<commit_before>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package images

import (
	"context"
	"encoding/json"
	"strings"
	"time"

	"github.com/containerd/containerd/content"
	"github.com/containerd/containerd/errdefs"
	"github.com/containerd/containerd/log"
	"github.com/containerd/containerd/platforms"
	digest "github.com/opencontainers/go-digest"
	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
)

// Image provides the model for how containerd views container images.
type Image struct {
	// Name of the image.
	//
	// To be pulled, it must be a reference compatible with resolvers.
	//
	// This field is required.
	Name string

	// Labels provide runtime decoration for the image record.
	//
	// There is no default behavior for how these labels are propagated. They
	// only decorate the static metadata object.
	//
	// This field is optional.
	Labels map[string]string

	// Target describes the root content for this image. Typically, this is
	// a manifest, index or manifest list.
	Target ocispec.Descriptor

	CreatedAt, UpdatedAt time.Time
}

// DeleteOptions provide options on image delete
type DeleteOptions struct {
	Synchronous bool
}

// DeleteOpt allows configuring a delete operation
type DeleteOpt func(context.Context, *DeleteOptions) error

// SynchronousDelete is used to indicate that an image deletion and removal of
// the image resources should occur synchronously before returning a result.
func SynchronousDelete() DeleteOpt {
	return func(ctx context.Context, o *DeleteOptions) error {
		o.Synchronous = true
		return nil
	}
}

// Store and interact with images
type Store interface {
	Get(ctx context.Context, name string) (Image, error)
	List(ctx context.Context, filters ...string) ([]Image, error)
	Create(ctx context.Context, image Image) (Image, error)

	// Update will replace the data in the store with the provided image. If
	// one or more fieldpaths are provided, only those fields will be updated.
	Update(ctx context.Context, image Image, fieldpaths ...string) (Image, error)

	Delete(ctx context.Context, name string, opts ...DeleteOpt) error
}

// TODO(stevvooe): Many of these functions make strong platform assumptions,
// which are untrue in a lot of cases. More refactoring must be done here to
// make this work in all cases.

// Config resolves the image configuration descriptor.
//
// The caller can then use the descriptor to resolve and process the
// configuration of the image.
func (image *Image) Config(ctx context.Context, provider content.Provider, platform string) (ocispec.Descriptor, error) {
	return Config(ctx, provider, image.Target, platform)
}

// RootFS returns the unpacked diffids that make up and images rootfs.
//
// These are used to verify that a set of layers unpacked to the expected
// values.
func (image *Image) RootFS(ctx context.Context, provider content.Provider, platform string) ([]digest.Digest, error) {
	desc, err := image.Config(ctx, provider, platform)
	if err != nil {
		return nil, err
	}
	return RootFS(ctx, provider, desc)
}

// Size returns the total size of an image's packed resources.
func (image *Image) Size(ctx context.Context, provider content.Provider, platform string) (int64, error) {
	var size int64
	return size, Walk(ctx, Handlers(HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
		if desc.Size < 0 {
			return nil, errors.Errorf("invalid size %v in %v (%v)", desc.Size, desc.Digest, desc.MediaType)
		}
		size += desc.Size
		return nil, nil
	}), FilterPlatforms(ChildrenHandler(provider), platform)), image.Target)
}

// Manifest resolves a manifest from the image for the given platform.
//
// TODO(stevvooe): This violates the current platform agnostic approach to this
// package by returning a specific manifest type. We'll need to refactor this
// to return a manifest descriptor or decide that we want to bring the API in
// this direction because this abstraction is not needed.`
func Manifest(ctx context.Context, provider content.Provider, image ocispec.Descriptor, platform string) (ocispec.Manifest, error) {
	var (
		matcher platforms.Matcher
		m       *ocispec.Manifest
	)
	if platform != "" {
		p, err := platforms.Parse(platform)
		if err != nil {
			return ocispec.Manifest{}, err
		}
		matcher = platforms.NewMatcher(p)
	}

	if err := Walk(ctx, HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
		switch desc.MediaType {
		case MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest:
			p, err := content.ReadBlob(ctx, provider, desc)
			if err != nil {
				return nil, err
			}

			var manifest ocispec.Manifest
			if err := json.Unmarshal(p, &manifest); err != nil {
				return nil, err
			}

			if platform != "" {
				if desc.Platform != nil && !matcher.Match(*desc.Platform) {
					return nil, nil
				}

				if desc.Platform == nil {
					p, err := content.ReadBlob(ctx, provider, manifest.Config)
					if err != nil {
						return nil, err
					}

					var image ocispec.Image
					if err := json.Unmarshal(p, &image); err != nil {
						return nil, err
					}

					if !matcher.Match(platforms.Normalize(ocispec.Platform{OS: image.OS, Architecture: image.Architecture})) {
						return nil, nil
					}

				}
			}

			m = &manifest

			return nil, nil
		case MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex:
			p, err := content.ReadBlob(ctx, provider, desc)
			if err != nil {
				return nil, err
			}

			var idx ocispec.Index
			if err := json.Unmarshal(p, &idx); err != nil {
				return nil, err
			}

			if platform == "" {
				return idx.Manifests, nil
			}

			var descs []ocispec.Descriptor
			for _, d := range idx.Manifests {
				if d.Platform == nil || matcher.Match(*d.Platform) {
					descs = append(descs, d)
				}
			}

			return descs, nil

		}
		return nil, errors.Wrapf(errdefs.ErrNotFound, "unexpected media type %v for %v", desc.MediaType, desc.Digest)
	}), image); err != nil {
		return ocispec.Manifest{}, err
	}

	if m == nil {
		return ocispec.Manifest{}, errors.Wrapf(errdefs.ErrNotFound, "manifest %v", image.Digest)
	}

	return *m, nil
}

// Config resolves the image configuration descriptor using a content provided
// to resolve child resources on the image.
//
// The caller can then use the descriptor to resolve and process the
// configuration of the image.
func Config(ctx context.Context, provider content.Provider, image ocispec.Descriptor, platform string) (ocispec.Descriptor, error) {
	manifest, err := Manifest(ctx, provider, image, platform)
	if err != nil {
		return ocispec.Descriptor{}, err
	}
	return manifest.Config, err
}

// Platforms returns one or more platforms supported by the image.
func Platforms(ctx context.Context, provider content.Provider, image ocispec.Descriptor) ([]ocispec.Platform, error) {
	var platformSpecs []ocispec.Platform
	return platformSpecs, Walk(ctx, Handlers(HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
		if desc.Platform != nil {
			platformSpecs = append(platformSpecs, *desc.Platform)
			return nil, ErrSkipDesc
		}

		switch desc.MediaType {
		case MediaTypeDockerSchema2Config, ocispec.MediaTypeImageConfig:
			p, err := content.ReadBlob(ctx, provider, desc)
			if err != nil {
				return nil, err
			}

			var image ocispec.Image
			if err := json.Unmarshal(p, &image); err != nil {
				return nil, err
			}

			platformSpecs = append(platformSpecs,
				platforms.Normalize(ocispec.Platform{OS: image.OS, Architecture: image.Architecture}))
		}
		return nil, nil
	}), ChildrenHandler(provider)), image)
}

// Check returns nil if the all components of an image are available in the
// provider for the specified platform.
//
// If available is true, the caller can assume that required represents the
// complete set of content required for the image.
//
// missing will have the components that are part of required but not avaiiable
// in the provider.
//
// If there is a problem resolving content, an error will be returned.
func Check(ctx context.Context, provider content.Provider, image ocispec.Descriptor, platform string) (available bool, required, present, missing []ocispec.Descriptor, err error) {
	mfst, err := Manifest(ctx, provider, image, platform)
	if err != nil {
		if errdefs.IsNotFound(err) {
			return false, []ocispec.Descriptor{image}, nil, []ocispec.Descriptor{image}, nil
		}

		return false, nil, nil, nil, errors.Wrapf(err, "failed to check image %v", image.Digest)
	}

	// TODO(stevvooe): It is possible that referenced conponents could have
	// children, but this is rare. For now, we ignore this and only verify
	// that manifest components are present.
	required = append([]ocispec.Descriptor{mfst.Config}, mfst.Layers...)

	for _, desc := range required {
		ra, err := provider.ReaderAt(ctx, desc)
		if err != nil {
			if errdefs.IsNotFound(err) {
				missing = append(missing, desc)
				continue
			} else {
				return false, nil, nil, nil, errors.Wrapf(err, "failed to check image %v", desc.Digest)
			}
		}
		ra.Close()
		present = append(present, desc)

	}

	return true, required, present, missing, nil
}

// Children returns the immediate children of content described by the descriptor.
func Children(ctx context.Context, provider content.Provider, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
	var descs []ocispec.Descriptor
	switch desc.MediaType {
	case MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest:
		p, err := content.ReadBlob(ctx, provider, desc)
		if err != nil {
			return nil, err
		}

		// TODO(stevvooe): We just assume oci manifest, for now. There may be
		// subtle differences from the docker version.
		var manifest ocispec.Manifest
		if err := json.Unmarshal(p, &manifest); err != nil {
			return nil, err
		}

		descs = append(descs, manifest.Config)
		descs = append(descs, manifest.Layers...)
	case MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex:
		p, err := content.ReadBlob(ctx, provider, desc)
		if err != nil {
			return nil, err
		}

		var index ocispec.Index
		if err := json.Unmarshal(p, &index); err != nil {
			return nil, err
		}

		descs = append(descs, index.Manifests...)
	case MediaTypeDockerSchema2Layer, MediaTypeDockerSchema2LayerGzip,
		MediaTypeDockerSchema2LayerForeign, MediaTypeDockerSchema2LayerForeignGzip,
		MediaTypeDockerSchema2Config, ocispec.MediaTypeImageConfig,
		ocispec.MediaTypeImageLayer, ocispec.MediaTypeImageLayerGzip,
		ocispec.MediaTypeImageLayerNonDistributable, ocispec.MediaTypeImageLayerNonDistributableGzip,
		MediaTypeContainerd1Checkpoint, MediaTypeContainerd1CheckpointConfig:
		// childless data types.
		return nil, nil
	default:
		log.G(ctx).Warnf("encountered unknown type %v; children may not be fetched", desc.MediaType)
	}

	return descs, nil
}

// RootFS returns the unpacked diffids that make up and images rootfs.
//
// These are used to verify that a set of layers unpacked to the expected
// values.
func RootFS(ctx context.Context, provider content.Provider, configDesc ocispec.Descriptor) ([]digest.Digest, error) {
	p, err := content.ReadBlob(ctx, provider, configDesc)
	if err != nil {
		return nil, err
	}

	var config ocispec.Image
	if err := json.Unmarshal(p, &config); err != nil {
		return nil, err
	}
	return config.RootFS.DiffIDs, nil
}

// IsCompressedDiff returns true if mediaType is a known compressed diff media type.
// It returns false if the media type is a diff, but not compressed. If the media type
// is not a known diff type, it returns errdefs.ErrNotImplemented
func IsCompressedDiff(ctx context.Context, mediaType string) (bool, error) {
	switch mediaType {
	case ocispec.MediaTypeImageLayer, MediaTypeDockerSchema2Layer:
	case ocispec.MediaTypeImageLayerGzip, MediaTypeDockerSchema2LayerGzip:
		return true, nil
	default:
		// Still apply all generic media types *.tar[.+]gzip and *.tar
		if strings.HasSuffix(mediaType, ".tar.gzip") || strings.HasSuffix(mediaType, ".tar+gzip") {
			return true, nil
		} else if !strings.HasSuffix(mediaType, ".tar") {
			return false, errdefs.ErrNotImplemented
		}
	}
	return false, nil
}
<commit_msg>images: provide better error for manifest list match error<commit_after>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package images

import (
	"context"
	"encoding/json"
	"strings"
	"time"

	"github.com/containerd/containerd/content"
	"github.com/containerd/containerd/errdefs"
	"github.com/containerd/containerd/log"
	"github.com/containerd/containerd/platforms"
	digest "github.com/opencontainers/go-digest"
	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
)

// Image provides the model for how containerd views container images.
type Image struct {
	// Name of the image.
	//
	// To be pulled, it must be a reference compatible with resolvers.
	//
	// This field is required.
	Name string

	// Labels provide runtime decoration for the image record.
	//
	// There is no default behavior for how these labels are propagated. They
	// only decorate the static metadata object.
	//
	// This field is optional.
	Labels map[string]string

	// Target describes the root content for this image. Typically, this is
	// a manifest, index or manifest list.
	Target ocispec.Descriptor

	CreatedAt, UpdatedAt time.Time
}

// DeleteOptions provide options on image delete
type DeleteOptions struct {
	Synchronous bool
}

// DeleteOpt allows configuring a delete operation
type DeleteOpt func(context.Context, *DeleteOptions) error

// SynchronousDelete is used to indicate that an image deletion and removal of
// the image resources should occur synchronously before returning a result.
func SynchronousDelete() DeleteOpt {
	return func(ctx context.Context, o *DeleteOptions) error {
		o.Synchronous = true
		return nil
	}
}

// Store and interact with images
type Store interface {
	Get(ctx context.Context, name string) (Image, error)
	List(ctx context.Context, filters ...string) ([]Image, error)
	Create(ctx context.Context, image Image) (Image, error)

	// Update will replace the data in the store with the provided image. If
	// one or more fieldpaths are provided, only those fields will be updated.
	Update(ctx context.Context, image Image, fieldpaths ...string) (Image, error)

	Delete(ctx context.Context, name string, opts ...DeleteOpt) error
}

// TODO(stevvooe): Many of these functions make strong platform assumptions,
// which are untrue in a lot of cases. More refactoring must be done here to
// make this work in all cases.

// Config resolves the image configuration descriptor.
//
// The caller can then use the descriptor to resolve and process the
// configuration of the image.
func (image *Image) Config(ctx context.Context, provider content.Provider, platform string) (ocispec.Descriptor, error) {
	return Config(ctx, provider, image.Target, platform)
}

// RootFS returns the unpacked diffids that make up and images rootfs.
//
// These are used to verify that a set of layers unpacked to the expected
// values.
func (image *Image) RootFS(ctx context.Context, provider content.Provider, platform string) ([]digest.Digest, error) {
	desc, err := image.Config(ctx, provider, platform)
	if err != nil {
		return nil, err
	}
	return RootFS(ctx, provider, desc)
}

// Size returns the total size of an image's packed resources.
func (image *Image) Size(ctx context.Context, provider content.Provider, platform string) (int64, error) {
	var size int64
	return size, Walk(ctx, Handlers(HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
		if desc.Size < 0 {
			return nil, errors.Errorf("invalid size %v in %v (%v)", desc.Size, desc.Digest, desc.MediaType)
		}
		size += desc.Size
		return nil, nil
	}), FilterPlatforms(ChildrenHandler(provider), platform)), image.Target)
}

// Manifest resolves a manifest from the image for the given platform.
//
// TODO(stevvooe): This violates the current platform agnostic approach to this
// package by returning a specific manifest type. We'll need to refactor this
// to return a manifest descriptor or decide that we want to bring the API in
// this direction because this abstraction is not needed.`
func Manifest(ctx context.Context, provider content.Provider, image ocispec.Descriptor, platform string) (ocispec.Manifest, error) {
	var (
		matcher  platforms.Matcher
		m        *ocispec.Manifest
		p        ocispec.Platform
		wasIndex bool
	)
	if platform != "" {
		var err error
		p, err = platforms.Parse(platform)
		if err != nil {
			return ocispec.Manifest{}, err
		}
		matcher = platforms.NewMatcher(p)
	}

	if err := Walk(ctx, HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
		switch desc.MediaType {
		case MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest:
			p, err := content.ReadBlob(ctx, provider, desc)
			if err != nil {
				return nil, err
			}

			var manifest ocispec.Manifest
			if err := json.Unmarshal(p, &manifest); err != nil {
				return nil, err
			}

			if platform != "" {
				if desc.Platform != nil && !matcher.Match(*desc.Platform) {
					return nil, nil
				}

				if desc.Platform == nil {
					p, err := content.ReadBlob(ctx, provider, manifest.Config)
					if err != nil {
						return nil, err
					}

					var image ocispec.Image
					if err := json.Unmarshal(p, &image); err != nil {
						return nil, err
					}

					if !matcher.Match(platforms.Normalize(ocispec.Platform{OS: image.OS, Architecture: image.Architecture})) {
						return nil, nil
					}

				}
			}

			m = &manifest

			return nil, nil
		case MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex:
			p, err := content.ReadBlob(ctx, provider, desc)
			if err != nil {
				return nil, err
			}

			var idx ocispec.Index
			if err := json.Unmarshal(p, &idx); err != nil {
				return nil, err
			}

			if platform == "" {
				return idx.Manifests, nil
			}

			var descs []ocispec.Descriptor
			for _, d := range idx.Manifests {
				if d.Platform == nil || matcher.Match(*d.Platform) {
					descs = append(descs, d)
				}
			}

			wasIndex = true

			return descs, nil

		}
		return nil, errors.Wrapf(errdefs.ErrNotFound, "unexpected media type %v for %v", desc.MediaType, desc.Digest)
	}), image); err != nil {
		return ocispec.Manifest{}, err
	}

	if m == nil {
		err := errors.Wrapf(errdefs.ErrNotFound, "manifest %v", image.Digest)
		if wasIndex {
			err = errors.Wrapf(errdefs.ErrNotFound, "no match for current platform %s in manifest %v", platforms.Format(p), image.Digest)
		}
		return ocispec.Manifest{}, err
	}

	return *m, nil
}

// Config resolves the image configuration descriptor using a content provided
// to resolve child resources on the image.
//
// The caller can then use the descriptor to resolve and process the
// configuration of the image.
func Config(ctx context.Context, provider content.Provider, image ocispec.Descriptor, platform string) (ocispec.Descriptor, error) {
	manifest, err := Manifest(ctx, provider, image, platform)
	if err != nil {
		return ocispec.Descriptor{}, err
	}
	return manifest.Config, err
}

// Platforms returns one or more platforms supported by the image.
func Platforms(ctx context.Context, provider content.Provider, image ocispec.Descriptor) ([]ocispec.Platform, error) {
	var platformSpecs []ocispec.Platform
	return platformSpecs, Walk(ctx, Handlers(HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
		if desc.Platform != nil {
			platformSpecs = append(platformSpecs, *desc.Platform)
			return nil, ErrSkipDesc
		}

		switch desc.MediaType {
		case MediaTypeDockerSchema2Config, ocispec.MediaTypeImageConfig:
			p, err := content.ReadBlob(ctx, provider, desc)
			if err != nil {
				return nil, err
			}

			var image ocispec.Image
			if err := json.Unmarshal(p, &image); err != nil {
				return nil, err
			}

			platformSpecs = append(platformSpecs,
				platforms.Normalize(ocispec.Platform{OS: image.OS, Architecture: image.Architecture}))
		}
		return nil, nil
	}), ChildrenHandler(provider)), image)
}

// Check returns nil if the all components of an image are available in the
// provider for the specified platform.
//
// If available is true, the caller can assume that required represents the
// complete set of content required for the image.
//
// missing will have the components that are part of required but not avaiiable
// in the provider.
//
// If there is a problem resolving content, an error will be returned.
func Check(ctx context.Context, provider content.Provider, image ocispec.Descriptor, platform string) (available bool, required, present, missing []ocispec.Descriptor, err error) {
	mfst, err := Manifest(ctx, provider, image, platform)
	if err != nil {
		if errdefs.IsNotFound(err) {
			return false, []ocispec.Descriptor{image}, nil, []ocispec.Descriptor{image}, nil
		}

		return false, nil, nil, nil, errors.Wrapf(err, "failed to check image %v", image.Digest)
	}

	// TODO(stevvooe): It is possible that referenced conponents could have
	// children, but this is rare. For now, we ignore this and only verify
	// that manifest components are present.
	required = append([]ocispec.Descriptor{mfst.Config}, mfst.Layers...)

	for _, desc := range required {
		ra, err := provider.ReaderAt(ctx, desc)
		if err != nil {
			if errdefs.IsNotFound(err) {
				missing = append(missing, desc)
				continue
			} else {
				return false, nil, nil, nil, errors.Wrapf(err, "failed to check image %v", desc.Digest)
			}
		}
		ra.Close()
		present = append(present, desc)

	}

	return true, required, present, missing, nil
}

// Children returns the immediate children of content described by the descriptor.
func Children(ctx context.Context, provider content.Provider, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
	var descs []ocispec.Descriptor
	switch desc.MediaType {
	case MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest:
		p, err := content.ReadBlob(ctx, provider, desc)
		if err != nil {
			return nil, err
		}

		// TODO(stevvooe): We just assume oci manifest, for now. There may be
		// subtle differences from the docker version.
		var manifest ocispec.Manifest
		if err := json.Unmarshal(p, &manifest); err != nil {
			return nil, err
		}

		descs = append(descs, manifest.Config)
		descs = append(descs, manifest.Layers...)
	case MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex:
		p, err := content.ReadBlob(ctx, provider, desc)
		if err != nil {
			return nil, err
		}

		var index ocispec.Index
		if err := json.Unmarshal(p, &index); err != nil {
			return nil, err
		}

		descs = append(descs, index.Manifests...)
	case MediaTypeDockerSchema2Layer, MediaTypeDockerSchema2LayerGzip,
		MediaTypeDockerSchema2LayerForeign, MediaTypeDockerSchema2LayerForeignGzip,
		MediaTypeDockerSchema2Config, ocispec.MediaTypeImageConfig,
		ocispec.MediaTypeImageLayer, ocispec.MediaTypeImageLayerGzip,
		ocispec.MediaTypeImageLayerNonDistributable, ocispec.MediaTypeImageLayerNonDistributableGzip,
		MediaTypeContainerd1Checkpoint, MediaTypeContainerd1CheckpointConfig:
		// childless data types.
		return nil, nil
	default:
		log.G(ctx).Warnf("encountered unknown type %v; children may not be fetched", desc.MediaType)
	}

	return descs, nil
}

// RootFS returns the unpacked diffids that make up and images rootfs.
//
// These are used to verify that a set of layers unpacked to the expected
// values.
func RootFS(ctx context.Context, provider content.Provider, configDesc ocispec.Descriptor) ([]digest.Digest, error) {
	p, err := content.ReadBlob(ctx, provider, configDesc)
	if err != nil {
		return nil, err
	}

	var config ocispec.Image
	if err := json.Unmarshal(p, &config); err != nil {
		return nil, err
	}
	return config.RootFS.DiffIDs, nil
}

// IsCompressedDiff returns true if mediaType is a known compressed diff media type.
// It returns false if the media type is a diff, but not compressed. If the media type
// is not a known diff type, it returns errdefs.ErrNotImplemented
func IsCompressedDiff(ctx context.Context, mediaType string) (bool, error) {
	switch mediaType {
	case ocispec.MediaTypeImageLayer, MediaTypeDockerSchema2Layer:
	case ocispec.MediaTypeImageLayerGzip, MediaTypeDockerSchema2LayerGzip:
		return true, nil
	default:
		// Still apply all generic media types *.tar[.+]gzip and *.tar
		if strings.HasSuffix(mediaType, ".tar.gzip") || strings.HasSuffix(mediaType, ".tar+gzip") {
			return true, nil
		} else if !strings.HasSuffix(mediaType, ".tar") {
			return false, errdefs.ErrNotImplemented
		}
	}
	return false, nil
}
<|endoftext|>


<commit_before>// The MIT License (MIT)
//
// Copyright (c) 2013-2017 Oryx(ossrs)
//
// Permission is hereby granted, free of charge, to any person obtaining a copy of
// this software and associated documentation files (the "Software"), to deal in
// the Software without restriction, including without limitation the rights to
// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
// the Software, and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
// FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
// IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package aac_test

import (
	"fmt"
	"github.com/ossrs/go-oryx-lib/aac"
)

func ExampleDecode() {
	var err error
	var adts aac.ADTS
	if adts, err = aac.NewADTS(); err != nil {
		fmt.Println(fmt.Sprintf("APP: Create ADTS failed, err is %+v", err))
		return
	}

	var data []byte // Read ADTS data from file or network.

	// Ignore the left, assume that the RAW only contains one AAC frame.
	var raw []byte
	if raw, _, err = adts.Decode(data); err != nil {
		fmt.Println(fmt.Sprintf("APP: ADTS decode failed, err is %+v", err))
		return
	}

	// Use the RAW data.
	_ = raw

	// Use the asc object, for example, used as RTMP audio sequence header.
	_ = adts.ASC()
}

func ExampleEncode() {
	var err error
	var adts aac.ADTS
	if adts, err = aac.NewADTS(); err != nil {
		fmt.Println(fmt.Sprintf("APP: Create ADTS failed, err is %+v", err))
		return
	}

	var raw []byte // Read RAW AAC from file or network.
	var data []byte
	if data, err = adts.Encode(raw); err != nil {
		fmt.Println(fmt.Sprintf("APP: ADTS encode failed, err is %+v", err))
		return
	}

	// Use the ADTS data.
	_ = data
}
<commit_msg>Refine example, bind to func<commit_after>// The MIT License (MIT)
//
// Copyright (c) 2013-2017 Oryx(ossrs)
//
// Permission is hereby granted, free of charge, to any person obtaining a copy of
// this software and associated documentation files (the "Software"), to deal in
// the Software without restriction, including without limitation the rights to
// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
// the Software, and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
// FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
// IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package aac_test

import (
	"fmt"
	"github.com/ossrs/go-oryx-lib/aac"
)

func Exampleadts_Decode() {
	var err error
	var adts aac.ADTS
	if adts, err = aac.NewADTS(); err != nil {
		fmt.Println(fmt.Sprintf("APP: Create ADTS failed, err is %+v", err))
		return
	}

	var data []byte // Read ADTS data from file or network.

	// Ignore the left, assume that the RAW only contains one AAC frame.
	var raw []byte
	if raw, _, err = adts.Decode(data); err != nil {
		fmt.Println(fmt.Sprintf("APP: ADTS decode failed, err is %+v", err))
		return
	}

	// Use the RAW data.
	_ = raw

	// Use the asc object, for example, used as RTMP audio sequence header.
	_ = adts.ASC()
}

func Exampleadts_Encode() {
	var err error
	var adts aac.ADTS
	if adts, err = aac.NewADTS(); err != nil {
		fmt.Println(fmt.Sprintf("APP: Create ADTS failed, err is %+v", err))
		return
	}

	var raw []byte // Read RAW AAC from file or network.
	var data []byte
	if data, err = adts.Encode(raw); err != nil {
		fmt.Println(fmt.Sprintf("APP: ADTS encode failed, err is %+v", err))
		return
	}

	// Use the ADTS data.
	_ = data
}
<|endoftext|>


<commit_before>package mod

import (
	"image"
	"image/color"

	"github.com/disintegration/gift"
)

// GiftTransform converts any set of gift.Filters into a Mod.
func GiftTransform(fs ...gift.Filter) Mod {
	return func(rgba image.Image) *image.RGBA {
		filter := gift.New(fs...)
		dst := image.NewRGBA(filter.Bounds(rgba.Bounds()))
		filter.Draw(dst, rgba)
		return dst
	}
}

// GiftFilter converts any set of gift.Filters into a Filter.
// if a filter is internally a transformation in gift, this will
// not work and GiftTransform should be used instead.
func GiftFilter(fs ...gift.Filter) Filter {
	return func(rgba *image.RGBA) {
		gift.New(fs...).Draw(rgba, rgba)
	}
}

// Brighten brightens an image between -100 and 100. 100 will be solid white,
// -100 will be solid black, for all colors not zero before filtering.
func Brighten(brightenBy float32) Filter {
	return GiftFilter(gift.Brightness(brightenBy))
}

// Saturate saturates the input between -100 and 500 percent.
func Saturate(saturateBy float32) Filter {
	return GiftFilter(gift.Saturation(saturateBy))
}

// ColorBalance takes in 3 numbers between -100 and 500 and applies it to the given image
func ColorBalance(r, g, b float32) Filter {
	return GiftFilter(gift.ColorBalance(r, g, b))
}

// Crop will return the given rectangle portion of transformed images. See gift.Crop
func Crop(rect image.Rectangle) Mod {
	return GiftTransform(gift.Crop(rect))
}

// CropToSize applies crop with an optional anchor. See gift.CropToSize
func CropToSize(width, height int, anchor gift.Anchor) Mod {
	return GiftTransform(gift.CropToSize(width, height, anchor))
}

// FlipX returns a new rgba which is flipped
// over the horizontal axis.
var FlipX = GiftTransform(gift.FlipHorizontal())

// FlipY returns a new rgba which is flipped
// over the vertical axis.
var FlipY = GiftTransform(gift.FlipVertical())

// todo: alias gift types to this package

// Resize will transform images to match the input dimensions. See gift.Resize.
func Resize(width, height int, resampling gift.Resampling) Mod {
	return GiftTransform(gift.Resize(width, height, resampling))
}

// ResizeToFill will resize to fit and then crop using the given anchor. See gift.ResizeToFill.
func ResizeToFill(width, height int, resampling gift.Resampling, anchor gift.Anchor) Mod {
	return GiftTransform(gift.ResizeToFill(width, height, resampling, anchor))
}

// ResizeToFit will resize while preserving aspect ratio. See gift.ResizeToFit.
func ResizeToFit(width, height int, resampling gift.Resampling) Mod {
	return GiftTransform(gift.ResizeToFit(width, height, resampling))
}

// Rotate returns a rotated rgba.
func Rotate(degrees float32) Mod {
	return RotateInterpolated(degrees, gift.CubicInterpolation)
}

// RotateInterpolated acts as Rotate, but accepts an interpolation argument.
// standard rotation does this with Cubic Interpolation.
func RotateInterpolated(degrees float32, interpolation gift.Interpolation) Mod {
	return RotateBackground(degrees, color.RGBA{0, 0, 0, 0}, interpolation)
}

// RotateBackground acts as RotateInterpolated, but allows for supplying a specific
// background color to the rotation.
func RotateBackground(degrees float32, bckgrnd color.Color, interpolation gift.Interpolation) Mod {
	return GiftTransform(gift.Rotate(degrees, bckgrnd, interpolation))
}

// Rotate180 performs a specialized rotation for 180 degrees.
var Rotate180 = GiftTransform(gift.Rotate180())

// Rotate270 performs a specialized rotation for 270 degrees.
var Rotate270 = GiftTransform(gift.Rotate270())

// Rotate90 performs a specialized rotation for 360 degrees.
var Rotate90 = GiftTransform(gift.Rotate90())

// Transpose flips horizontally and rotates 90 degrees counter clockwise.
var Transpose = GiftTransform(gift.Transpose())

// Transverse flips vertically and rotates 90 degrees counter clockwise.
var Transverse = GiftTransform(gift.Transverse())
<commit_msg>render/mod: alias resampling type from gift<commit_after>package mod

import (
	"image"
	"image/color"

	"github.com/disintegration/gift"
)

// GiftTransform converts any set of gift.Filters into a Mod.
func GiftTransform(fs ...gift.Filter) Mod {
	return func(rgba image.Image) *image.RGBA {
		filter := gift.New(fs...)
		dst := image.NewRGBA(filter.Bounds(rgba.Bounds()))
		filter.Draw(dst, rgba)
		return dst
	}
}

// GiftFilter converts any set of gift.Filters into a Filter.
// if a filter is internally a transformation in gift, this will
// not work and GiftTransform should be used instead.
func GiftFilter(fs ...gift.Filter) Filter {
	return func(rgba *image.RGBA) {
		gift.New(fs...).Draw(rgba, rgba)
	}
}

// Brighten brightens an image between -100 and 100. 100 will be solid white,
// -100 will be solid black, for all colors not zero before filtering.
func Brighten(brightenBy float32) Filter {
	return GiftFilter(gift.Brightness(brightenBy))
}

// Saturate saturates the input between -100 and 500 percent.
func Saturate(saturateBy float32) Filter {
	return GiftFilter(gift.Saturation(saturateBy))
}

// ColorBalance takes in 3 numbers between -100 and 500 and applies it to the given image
func ColorBalance(r, g, b float32) Filter {
	return GiftFilter(gift.ColorBalance(r, g, b))
}

// Crop will return the given rectangle portion of transformed images. See gift.Crop
func Crop(rect image.Rectangle) Mod {
	return GiftTransform(gift.Crop(rect))
}

// CropToSize applies crop with an optional anchor. See gift.CropToSize
func CropToSize(width, height int, anchor gift.Anchor) Mod {
	return GiftTransform(gift.CropToSize(width, height, anchor))
}

// FlipX returns a new rgba which is flipped
// over the horizontal axis.
var FlipX = GiftTransform(gift.FlipHorizontal())

// FlipY returns a new rgba which is flipped
// over the vertical axis.
var FlipY = GiftTransform(gift.FlipVertical())

type Resampling = gift.Resampling

// NearestNeighborResampling is a nearest neighbor resampling filter.
var NearestNeighborResampling Resampling = gift.NearestNeighborResampling

// BoxResampling is a box resampling filter (average of surrounding pixels).
var BoxResampling Resampling = gift.BoxResampling

// LinearResampling is a bilinear resampling filter.
var LinearResampling Resampling = gift.LinearResampling

// CubicResampling is a bicubic resampling filter (Catmull-Rom).
var CubicResampling Resampling = gift.CubicResampling

// LanczosResampling is a Lanczos resampling filter (3 lobes).
var LanczosResampling Resampling = gift.LanczosResampling

// Resize will transform images to match the input dimensions. See gift.Resize.
func Resize(width, height int, resampling Resampling) Mod {
	return GiftTransform(gift.Resize(width, height, resampling))
}

// ResizeToFill will resize to fit and then crop using the given anchor. See gift.ResizeToFill.
func ResizeToFill(width, height int, resampling Resampling, anchor gift.Anchor) Mod {
	return GiftTransform(gift.ResizeToFill(width, height, resampling, anchor))
}

// ResizeToFit will resize while preserving aspect ratio. See gift.ResizeToFit.
func ResizeToFit(width, height int, resampling Resampling) Mod {
	return GiftTransform(gift.ResizeToFit(width, height, resampling))
}

// Rotate returns a rotated rgba.
func Rotate(degrees float32) Mod {
	return RotateInterpolated(degrees, gift.CubicInterpolation)
}

// RotateInterpolated acts as Rotate, but accepts an interpolation argument.
// standard rotation does this with Cubic Interpolation.
func RotateInterpolated(degrees float32, interpolation gift.Interpolation) Mod {
	return RotateBackground(degrees, color.RGBA{0, 0, 0, 0}, interpolation)
}

// RotateBackground acts as RotateInterpolated, but allows for supplying a specific
// background color to the rotation.
func RotateBackground(degrees float32, bckgrnd color.Color, interpolation gift.Interpolation) Mod {
	return GiftTransform(gift.Rotate(degrees, bckgrnd, interpolation))
}

// Rotate180 performs a specialized rotation for 180 degrees.
var Rotate180 = GiftTransform(gift.Rotate180())

// Rotate270 performs a specialized rotation for 270 degrees.
var Rotate270 = GiftTransform(gift.Rotate270())

// Rotate90 performs a specialized rotation for 360 degrees.
var Rotate90 = GiftTransform(gift.Rotate90())

// Transpose flips horizontally and rotates 90 degrees counter clockwise.
var Transpose = GiftTransform(gift.Transpose())

// Transverse flips vertically and rotates 90 degrees counter clockwise.
var Transverse = GiftTransform(gift.Transverse())
<|endoftext|>


<commit_before>package repo

import (
	"bytes"
	"encoding/json"
	"fmt"
	"html/template"
	"io"
	"math"
	"math/rand"
	"sort"
	"time"

	"github.com/PuerkitoBio/goquery"
	"github.com/flimzy/go-pouchdb"
	"github.com/flimzy/log"
	"github.com/pkg/errors"
	"golang.org/x/net/html"

	"github.com/FlashbackSRS/flashback-model"
	"github.com/FlashbackSRS/flashback/repository/done"
	"github.com/FlashbackSRS/flashback/util"
	"github.com/FlashbackSRS/flashback/webclient/views/studyview"
)

const newPriority = 0.5

func init() {
	rand.Seed(int64(time.Now().UnixNano()))
}

// Card represents a generic card-like object.
type Card interface {
	DocID() string
	Buttons(face int) (studyview.ButtonMap, error)
	Body(face int) (body string, err error)
	Action(face *int, startTime time.Time, button studyview.Button) (done bool, err error)
}

// PouchCard provides a convenient interface to fb.Card and dependencies
type PouchCard struct {
	*fb.Card
	db       *DB
	note     *Note
	priority float32
}

type cardList []*PouchCard

func (c cardList) Len() int { return len(c) }
func (c cardList) Less(i, j int) bool {
	return c[i].priority > c[j].priority || c[i].Created.Before(c[j].Created)
}
func (c cardList) Swap(i, j int) { c[i], c[j] = c[j], c[i] }

type jsCard struct {
	ID string `json:"id"`
}

// MarshalJSON marshals a Card for the benefit of javascript context in HTML
// templates.
func (c *PouchCard) MarshalJSON() ([]byte, error) {
	card := &jsCard{
		ID: c.DocID(),
	}
	return json.Marshal(card)
}

// Save saves the card's current state to the database.
func (c *PouchCard) Save() error {
	log.Debugf("Attempting to save card %s\n", c.Identity())
	return c.db.Save(c.Card)
}

// Note returns the card's associated Note
func (c *PouchCard) Note() (*Note, error) {
	if err := c.fetchNote(); err != nil {
		return nil, errors.Wrap(err, "Error fetching note for Note()")
	}
	return c.note, nil
}

func (c *PouchCard) fetchNote() error {
	if c.note != nil {
		// Nothing to do
		return nil
	}
	log.Debugf("Fetching note %s", c.NoteID())
	db, err := c.db.User.NewDB(c.BundleID())
	if err != nil {
		return errors.Wrap(err, "fetchNote() can't connect to bundle DB")
	}
	n := &fb.Note{}
	if err := db.Get(c.NoteID(), n, pouchdb.Options{Attachments: true}); err != nil {
		return errors.Wrapf(err, "fetchNote() can't fetch %s", c.NoteID())
	}
	c.note = &Note{
		Note: n,
		db:   db,
	}
	return nil
}

// GetCard fetches the requested card
func (u *User) GetCard(id string) (*PouchCard, error) {
	db, err := u.DB()
	if err != nil {
		return nil, errors.Wrap(err, "connect to user DB")
	}
	return db.GetCard(id)
}

// GetCard fetches the requested card
func (db *DB) GetCard(id string) (*PouchCard, error) {
	card := &fb.Card{}
	if err := db.Get(id, card, pouchdb.Options{}); err != nil {
		return nil, errors.Wrap(err, "fetch card")
	}
	return &PouchCard{
		Card: card,
		db:   db,
	}, nil
}

type cardPriority struct {
	Card     *fb.Card
	Priority float32
}

type prioritizedCards []cardPriority

func (p prioritizedCards) Len() int { return len(p) }
func (p prioritizedCards) Less(i, j int) bool {
	return p[i].Priority > p[j].Priority || p[i].Card.Created.Before(p[j].Card.Created)
}
func (p prioritizedCards) Swap(i, j int) { p[i], p[j] = p[j], p[i] }

// CardPrio returns a number 0 or greater, as a priority to be used in
// determining card study order.
func CardPrio(due *fb.Due, interval *fb.Interval, now time.Time) float32 {
	if due == nil || interval == nil {
		return newPriority
	}
	// Remove the timezone
	_, offset := now.Zone()
	utc := now.UTC().Add(time.Duration(offset) * time.Second)

	return float32(math.Pow(1+float64(utc.Sub(time.Time(*due)))/float64(time.Duration(*interval)), 3))
}

// GetCards fetches up to limit cards from the db, in priority order.
func GetCards(db *DB, now time.Time, limit int) ([]*PouchCard, error) {
	var cards []*PouchCard
	query := map[string]interface{}{
		"selector": map[string]interface{}{
			"type":    "card",
			"due":     map[string]interface{}{"$gte": nil},
			"created": map[string]interface{}{"$gte": nil},
			// Don't review cards we already saw today, with an interval >= 1d;
			// they would make no progress.
			"$not": map[string]interface{}{
				"interval":   map[string]interface{}{"$gte": 1},
				"lastReview": map[string]interface{}{"$gte": fb.Today().String()},
			},
		},
		//		"fields": []string{"_id", "due", "interval", "model", "created"},
		"sort":  []string{"due", "created"},
		"limit": limit,
	}
	if err := db.Find(query, &cards); err != nil {
		return nil, errors.Wrap(err, "card list")
	}
	for _, card := range cards {
		card.db = db
		card.priority = CardPrio(card.Due, card.Interval, now)
	}
	sort.Sort(cardList(cards))
	return cards, nil
}

// UnmarshalJSON wraps fb.Card's Unmarshaler
func (c *PouchCard) UnmarshalJSON(data []byte) error {
	fbCard := &fb.Card{}
	err := json.Unmarshal(data, fbCard)
	c.Card = fbCard
	return err
}

// GetNextCard gets the next card to study
func (u *User) GetNextCard() (Card, error) {
	db, err := u.DB()
	if err != nil {
		return nil, errors.Wrap(err, "GetNextCard(): Error connecting to User DB")
	}

	cards, err := GetCards(db, time.Now(), 20)
	if err != nil {
		return nil, errors.Wrap(err, "get card list")
	}
	if len(cards) == 0 {
		return done.GetCard(), nil
	}

	var weights float32
	for _, c := range cards {
		weights += c.priority
	}

	r := rand.Float32() * weights
	log.Debugf("Random key / total: %f / %f (%d)\n", r, weights, len(cards))
	for i, c := range cards {
		r -= c.priority
		if r < 0 {
			log.Debugf("Selected card %d: %s\n", i, c.Identity())
			return db.GetCard(c.DocID())
		}
	}
	return nil, errors.New("failed to fetch card")
}

type cardContext struct {
	Card *PouchCard
	Note *Note
	// Model    *Model
	// Deck     *Deck
	BaseURI string
	Fields  map[string]template.HTML
}

const (
	// Question is a card's first face
	Question = iota
	// Answer is a card's second face
	Answer
)

var faces = map[int]string{
	Question: "question",
	Answer:   "answer",
}

// Buttons returns the button states for the given card/face.
func (c *PouchCard) Buttons(face int) (studyview.ButtonMap, error) {
	cont, err := c.getModelController()
	if err != nil {
		return nil, err
	}
	return cont.Buttons(face)
}

// Action handles the action on the card, such as a button press.
func (c *PouchCard) Action(face *int, startTime time.Time, button studyview.Button) (done bool, err error) {
	cont, err := c.getModelController()
	if err != nil {
		return false, err
	}
	return cont.Action(c, face, startTime, button)
}

// Model returns the model for the card
func (c *PouchCard) Model() (*Model, error) {
	note, err := c.Note()
	if err != nil {
		return nil, errors.Wrap(err, "retrieve Note")
	}
	model, err := note.Model()
	if err != nil {
		return nil, errors.Wrap(err, "retrieve Model")
	}
	return model, nil
}

// Body returns the requested card face
func (c *PouchCard) Body(face int) (body string, err error) {
	note, err := c.Note()
	if err != nil {
		return "", errors.Wrap(err, "Unable to retrieve Note")
	}
	model, err := c.Model()
	if err != nil {
		return "", errors.Wrap(err, "Unable to retrieve Model")
	}
	tmpl, err := model.GenerateTemplate()
	if err != nil {
		return "", errors.Wrap(err, "Error generating template")
	}
	ctx := cardContext{
		Card: c,
		Note: note,
		// Model:    model,
		BaseURI: util.BaseURI(),
		Fields:  make(map[string]template.HTML),
	}

	for i, f := range model.Fields {
		switch note.FieldValues[i].Type() {
		case fb.AnkiField, fb.TextField:
			text, e := note.FieldValues[i].Text()
			if e != nil {
				return "", errors.Wrap(e, "Unable to fetch text for field value")
			}
			ctx.Fields[f.Name] = template.HTML(text)
		}
	}

	htmlDoc := new(bytes.Buffer)
	if e := tmpl.Execute(htmlDoc, ctx); e != nil {
		return "", errors.Wrap(e, "Unable to execute template")
	}
	cont, err := c.getModelController()
	if err != nil {
		return "", errors.Wrap(err, "get model controller")
	}
	log.Debugf("original size = %d\n", htmlDoc.Len())
	newBody, err := prepareBody(face, c.TemplateID(), cont, htmlDoc)
	if err != nil {
		return "", errors.Wrap(err, "prepare body")
	}

	nbString := string(newBody)
	log.Debugf("new body size = %d\n", len(nbString))
	return nbString, nil
}

func prepareBody(face int, templateID uint32, cont ModelController, r io.Reader) ([]byte, error) {
	cardFace, ok := faces[face]
	if !ok {
		return nil, errors.Errorf("Unrecognized card face %d", face)
	}
	doc, err := goquery.NewDocumentFromReader(r)
	if err != nil {
		return nil, errors.Wrap(err, "goquery parse")
	}
	body := doc.Find("body")
	if body == nil {
		return nil, errors.New("no body in template output")
	}
	sel := fmt.Sprintf("div.%s[data-id='%d']", cardFace, templateID)
	container := body.Find(sel)
	if container.Length() == 0 {
		return nil, errors.Errorf("No div matching '%s' found in template output", sel)
	}

	containerHTML, err := container.Html()
	if err != nil {
		return nil, errors.Wrap(err, "error extracting div html")
	}

	body.Empty()
	body.AppendHtml(containerHTML)

	doc.Find("head").AppendHtml(fmt.Sprintf(`<script type="text/javascript">%s</script>`, string(cont.IframeScript())))

	newBody, err := goquery.OuterHtml(doc.Selection)
	if err != nil {
		return nil, errors.Wrap(err, "outer html failed")
	}
	return []byte(newBody), nil
}

func findBody(n *html.Node) *html.Node {
	if n.Type == html.ElementNode && n.Data == "body" {
		return n
	}
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		if body := findBody(c); body != nil {
			return body
		}
	}
	return nil
}

func findContainer(n *html.Node, targetID, targetClass string) *html.Node {
	if n == nil {
		return nil
	}
	if n.Type == html.ElementNode && n.Data == "div" {
		var class, id string
		for _, a := range n.Attr {
			switch a.Key {
			case "class":
				class = a.Val
			case "data-id":
				id = a.Val
			}
			if class != "" && id != "" {
				break
			}
		}
		if class == targetClass && id == targetID {
			return n
		}
	}
	return findContainer(n.NextSibling, targetID, targetClass)
}

// GetAttachment fetches an attachment from the note, failling back to the model
func (c *PouchCard) GetAttachment(filename string) (*Attachment, error) {
	n, err := c.Note()
	if err != nil {
		return nil, errors.Wrap(err, "Error fetching Note for GetAttachment()")
	}
	if file, ok := n.Attachments.GetFile(filename); ok {
		return &Attachment{file}, nil
	}

	m, err := n.Model()
	if err != nil {
		return nil, errors.Wrap(err, "Error fetching Model for GetAttachments()")
	}
	if file, ok := m.Files.GetFile(filename); ok {
		return &Attachment{file}, nil
	}
	return nil, errors.Errorf("File '%s' not found", filename)
}
<commit_msg>Skip sub-day scheduled cards that aren't due yet.<commit_after>package repo

import (
	"bytes"
	"encoding/json"
	"fmt"
	"html/template"
	"io"
	"math"
	"math/rand"
	"sort"
	"time"

	"github.com/PuerkitoBio/goquery"
	"github.com/flimzy/go-pouchdb"
	"github.com/flimzy/log"
	"github.com/pkg/errors"
	"golang.org/x/net/html"

	"github.com/FlashbackSRS/flashback-model"
	"github.com/FlashbackSRS/flashback/repository/done"
	"github.com/FlashbackSRS/flashback/util"
	"github.com/FlashbackSRS/flashback/webclient/views/studyview"
)

const newPriority = 0.5

func init() {
	rand.Seed(int64(time.Now().UnixNano()))
}

// Card represents a generic card-like object.
type Card interface {
	DocID() string
	Buttons(face int) (studyview.ButtonMap, error)
	Body(face int) (body string, err error)
	Action(face *int, startTime time.Time, button studyview.Button) (done bool, err error)
}

// PouchCard provides a convenient interface to fb.Card and dependencies
type PouchCard struct {
	*fb.Card
	db       *DB
	note     *Note
	priority float32
}

type cardList []*PouchCard

func (c cardList) Len() int { return len(c) }
func (c cardList) Less(i, j int) bool {
	return c[i].priority > c[j].priority || c[i].Created.Before(c[j].Created)
}
func (c cardList) Swap(i, j int) { c[i], c[j] = c[j], c[i] }

type jsCard struct {
	ID string `json:"id"`
}

// MarshalJSON marshals a Card for the benefit of javascript context in HTML
// templates.
func (c *PouchCard) MarshalJSON() ([]byte, error) {
	card := &jsCard{
		ID: c.DocID(),
	}
	return json.Marshal(card)
}

// Save saves the card's current state to the database.
func (c *PouchCard) Save() error {
	log.Debugf("Attempting to save card %s\n", c.Identity())
	return c.db.Save(c.Card)
}

// Note returns the card's associated Note
func (c *PouchCard) Note() (*Note, error) {
	if err := c.fetchNote(); err != nil {
		return nil, errors.Wrap(err, "Error fetching note for Note()")
	}
	return c.note, nil
}

func (c *PouchCard) fetchNote() error {
	if c.note != nil {
		// Nothing to do
		return nil
	}
	log.Debugf("Fetching note %s", c.NoteID())
	db, err := c.db.User.NewDB(c.BundleID())
	if err != nil {
		return errors.Wrap(err, "fetchNote() can't connect to bundle DB")
	}
	n := &fb.Note{}
	if err := db.Get(c.NoteID(), n, pouchdb.Options{Attachments: true}); err != nil {
		return errors.Wrapf(err, "fetchNote() can't fetch %s", c.NoteID())
	}
	c.note = &Note{
		Note: n,
		db:   db,
	}
	return nil
}

// GetCard fetches the requested card
func (u *User) GetCard(id string) (*PouchCard, error) {
	db, err := u.DB()
	if err != nil {
		return nil, errors.Wrap(err, "connect to user DB")
	}
	return db.GetCard(id)
}

// GetCard fetches the requested card
func (db *DB) GetCard(id string) (*PouchCard, error) {
	card := &fb.Card{}
	if err := db.Get(id, card, pouchdb.Options{}); err != nil {
		return nil, errors.Wrap(err, "fetch card")
	}
	return &PouchCard{
		Card: card,
		db:   db,
	}, nil
}

type cardPriority struct {
	Card     *fb.Card
	Priority float32
}

type prioritizedCards []cardPriority

func (p prioritizedCards) Len() int { return len(p) }
func (p prioritizedCards) Less(i, j int) bool {
	return p[i].Priority > p[j].Priority || p[i].Card.Created.Before(p[j].Card.Created)
}
func (p prioritizedCards) Swap(i, j int) { p[i], p[j] = p[j], p[i] }

// CardPrio returns a number 0 or greater, as a priority to be used in
// determining card study order.
func CardPrio(due *fb.Due, interval *fb.Interval, now time.Time) float32 {
	if due == nil || interval == nil {
		return newPriority
	}
	// Remove the timezone
	_, offset := now.Zone()
	utc := now.UTC().Add(time.Duration(offset) * time.Second)

	return float32(math.Pow(1+float64(utc.Sub(time.Time(*due)))/float64(time.Duration(*interval)), 3))
}

func init() {
	// pouchdb.Debug("*")
	pouchdb.DebugDisable()
}

// GetCards fetches up to limit cards from the db, in priority order.
func GetCards(db *DB, now time.Time, limit int) ([]*PouchCard, error) {
	var cards []*PouchCard
	query := map[string]interface{}{
		"selector": map[string]interface{}{
			"type":    "card",
			"due":     map[string]interface{}{"$gte": nil},
			"created": map[string]interface{}{"$gte": nil},
			"$not": map[string]interface{}{
				"$or": []interface{}{
					map[string]interface{}{
						// Don't review cards we already saw today, with an
						// interval >= 1d; they would make no progress.
						"interval":   map[string]interface{}{"$gte": 1},
						"lastReview": map[string]interface{}{"$gte": fb.Today().String()},
					},
					map[string]interface{}{
						// Ignore sub-day intervals that aren't due yet. We only allow
						// forward-fuzzing for intervals > 1day
						"interval": map[string]interface{}{"$lt": 0},
						"due":      map[string]interface{}{"$gt": fb.Now().String()},
					},
				},
			},
		},
		"sort":  []string{"due", "created"},
		"limit": limit,
	}
	if err := db.Find(query, &cards); err != nil {
		return nil, errors.Wrap(err, "card list")
	}
	for _, card := range cards {
		card.db = db
		card.priority = CardPrio(card.Due, card.Interval, now)
	}
	sort.Sort(cardList(cards))
	return cards, nil
}

// UnmarshalJSON wraps fb.Card's Unmarshaler
func (c *PouchCard) UnmarshalJSON(data []byte) error {
	fbCard := &fb.Card{}
	err := json.Unmarshal(data, fbCard)
	c.Card = fbCard
	return err
}

// cardBatchSize is the number of cards we fetch at once, using simple schedule
// prioritization. This number should be large enough that the intelligent
// scheduling has room to function, but small enough that performance isn't
// a big problem due to fetching and prioritizing many cards we don't actually
// use.
const cardBatchSize = 100

// GetNextCard gets the next card to study
func (u *User) GetNextCard() (Card, error) {
	db, err := u.DB()
	if err != nil {
		return nil, errors.Wrap(err, "GetNextCard(): Error connecting to User DB")
	}

	cards, err := GetCards(db, time.Now(), cardBatchSize)
	if err != nil {
		return nil, errors.Wrap(err, "get card list")
	}
	if len(cards) == 0 {
		return done.GetCard(), nil
	}

	var weights float32
	for _, c := range cards {
		weights += c.priority
	}

	r := rand.Float32() * weights
	log.Debugf("Random key / total: %f / %f (%d)\n", r, weights, len(cards))
	for i, c := range cards {
		r -= c.priority
		if r < 0 {
			log.Debugf("Selected card %d: %s\n", i, c.Identity())
			return c, nil
		}
	}
	return nil, errors.New("failed to fetch card")
}

type cardContext struct {
	Card *PouchCard
	Note *Note
	// Model    *Model
	// Deck     *Deck
	BaseURI string
	Fields  map[string]template.HTML
}

const (
	// Question is a card's first face
	Question = iota
	// Answer is a card's second face
	Answer
)

var faces = map[int]string{
	Question: "question",
	Answer:   "answer",
}

// Buttons returns the button states for the given card/face.
func (c *PouchCard) Buttons(face int) (studyview.ButtonMap, error) {
	cont, err := c.getModelController()
	if err != nil {
		return nil, err
	}
	return cont.Buttons(face)
}

// Action handles the action on the card, such as a button press.
func (c *PouchCard) Action(face *int, startTime time.Time, button studyview.Button) (done bool, err error) {
	cont, err := c.getModelController()
	if err != nil {
		return false, err
	}
	return cont.Action(c, face, startTime, button)
}

// Model returns the model for the card
func (c *PouchCard) Model() (*Model, error) {
	note, err := c.Note()
	if err != nil {
		return nil, errors.Wrap(err, "retrieve Note")
	}
	model, err := note.Model()
	if err != nil {
		return nil, errors.Wrap(err, "retrieve Model")
	}
	return model, nil
}

// Body returns the requested card face
func (c *PouchCard) Body(face int) (body string, err error) {
	note, err := c.Note()
	if err != nil {
		return "", errors.Wrap(err, "Unable to retrieve Note")
	}
	model, err := c.Model()
	if err != nil {
		return "", errors.Wrap(err, "Unable to retrieve Model")
	}
	tmpl, err := model.GenerateTemplate()
	if err != nil {
		return "", errors.Wrap(err, "Error generating template")
	}
	ctx := cardContext{
		Card: c,
		Note: note,
		// Model:    model,
		BaseURI: util.BaseURI(),
		Fields:  make(map[string]template.HTML),
	}

	for i, f := range model.Fields {
		switch note.FieldValues[i].Type() {
		case fb.AnkiField, fb.TextField:
			text, e := note.FieldValues[i].Text()
			if e != nil {
				return "", errors.Wrap(e, "Unable to fetch text for field value")
			}
			ctx.Fields[f.Name] = template.HTML(text)
		}
	}

	htmlDoc := new(bytes.Buffer)
	if e := tmpl.Execute(htmlDoc, ctx); e != nil {
		return "", errors.Wrap(e, "Unable to execute template")
	}
	cont, err := c.getModelController()
	if err != nil {
		return "", errors.Wrap(err, "get model controller")
	}
	log.Debugf("original size = %d\n", htmlDoc.Len())
	newBody, err := prepareBody(face, c.TemplateID(), cont, htmlDoc)
	if err != nil {
		return "", errors.Wrap(err, "prepare body")
	}

	nbString := string(newBody)
	log.Debugf("new body size = %d\n", len(nbString))
	return nbString, nil
}

func prepareBody(face int, templateID uint32, cont ModelController, r io.Reader) ([]byte, error) {
	cardFace, ok := faces[face]
	if !ok {
		return nil, errors.Errorf("Unrecognized card face %d", face)
	}
	doc, err := goquery.NewDocumentFromReader(r)
	if err != nil {
		return nil, errors.Wrap(err, "goquery parse")
	}
	body := doc.Find("body")
	if body == nil {
		return nil, errors.New("no body in template output")
	}
	sel := fmt.Sprintf("div.%s[data-id='%d']", cardFace, templateID)
	container := body.Find(sel)
	if container.Length() == 0 {
		return nil, errors.Errorf("No div matching '%s' found in template output", sel)
	}

	containerHTML, err := container.Html()
	if err != nil {
		return nil, errors.Wrap(err, "error extracting div html")
	}

	body.Empty()
	body.AppendHtml(containerHTML)

	doc.Find("head").AppendHtml(fmt.Sprintf(`<script type="text/javascript">%s</script>`, string(cont.IframeScript())))

	newBody, err := goquery.OuterHtml(doc.Selection)
	if err != nil {
		return nil, errors.Wrap(err, "outer html failed")
	}
	return []byte(newBody), nil
}

func findBody(n *html.Node) *html.Node {
	if n.Type == html.ElementNode && n.Data == "body" {
		return n
	}
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		if body := findBody(c); body != nil {
			return body
		}
	}
	return nil
}

func findContainer(n *html.Node, targetID, targetClass string) *html.Node {
	if n == nil {
		return nil
	}
	if n.Type == html.ElementNode && n.Data == "div" {
		var class, id string
		for _, a := range n.Attr {
			switch a.Key {
			case "class":
				class = a.Val
			case "data-id":
				id = a.Val
			}
			if class != "" && id != "" {
				break
			}
		}
		if class == targetClass && id == targetID {
			return n
		}
	}
	return findContainer(n.NextSibling, targetID, targetClass)
}

// GetAttachment fetches an attachment from the note, failling back to the model
func (c *PouchCard) GetAttachment(filename string) (*Attachment, error) {
	n, err := c.Note()
	if err != nil {
		return nil, errors.Wrap(err, "Error fetching Note for GetAttachment()")
	}
	if file, ok := n.Attachments.GetFile(filename); ok {
		return &Attachment{file}, nil
	}

	m, err := n.Model()
	if err != nil {
		return nil, errors.Wrap(err, "Error fetching Model for GetAttachments()")
	}
	if file, ok := m.Files.GetFile(filename); ok {
		return &Attachment{file}, nil
	}
	return nil, errors.Errorf("File '%s' not found", filename)
}
<|endoftext|>


<commit_before>package blobstore

import (
	"fmt"
	_ "gnd.la/blobstore/driver/file"
	_ "gnd.la/blobstore/driver/gridfs"
	_ "gnd.la/blobstore/driver/s3"
	"hash/adler32"
	"io/ioutil"
	"net"
	"os"
	"testing"
)

const (
	dataSize = 1 << 20 // 1MiB
)

func testPort(port int) bool {
	conn, err := net.Dial("tcp", fmt.Sprintf("localhost:%d", port))
	if err != nil {
		return false
	}
	conn.Close()
	return true
}

type Meta struct {
	Foo int
}

func fileData(t *testing.T, file string, size int64) []byte {
	f, err := os.Open(file)
	if err != nil {
		t.Fatal(err)
	}
	defer f.Close()
	b := make([]byte, size)
	if _, err := f.Read(b); err != nil {
		t.Fatal(err)
	}
	return b
}

func randData(t *testing.T, size int64) []byte {
	return fileData(t, "/dev/urandom", size)
}

func zeroData(t *testing.T, size int64) []byte {
	return fileData(t, "/dev/zero", size)
}

func testStore(t *testing.T, meta *Meta, config string) {
	t.Logf("Testing store with config %s", config)
	store, err := New(config)
	if err != nil {
		t.Fatal(err)
	}
	var ids []string
	var hashes []uint32
	for ii := 0; ii < 10; ii++ {
		var r []byte
		if ii%2 == 0 {
			r = zeroData(t, dataSize)
		} else {
			r = randData(t, dataSize)
		}
		id, err := store.Store(r, meta)
		if err != nil {
			t.Error(err)
			continue
		}
		t.Logf("Stored file with id %s", id)
		ids = append(ids, id)
		hashes = append(hashes, adler32.Checksum(r))
	}
	for ii, v := range ids {
		f, err := store.Open(v)
		if err != nil {
			t.Error(err)
			continue
		}
		t.Logf("Opened file %s", v)
		if f.Size() != dataSize {
			t.Errorf("Invalid data size for file %s. Want %v, got %v.", v, dataSize, f.Size())
		}
		if meta != nil {
			var m Meta
			if err := f.GetMeta(&m); err != nil {
				t.Errorf("error loading metadata from %v: %s", v, err)
			} else {
				if m.Foo != meta.Foo {
					t.Errorf("Invalid metadata value. Want %v, got %v.", meta.Foo, m.Foo)
				}
			}
		}
		if err := f.Verify(); err != nil {
			t.Errorf("error checking file %v: %s", v, err)
		}
		b, err := f.ReadAll()
		if err != nil {
			t.Error(err)
			continue
		}
		if len(b) != dataSize {
			t.Errorf("expecting %d bytes, got %d instead", dataSize, len(b))
			continue
		}
		if h := adler32.Checksum(b); h != hashes[ii] {
			t.Errorf("invalid hash %v for file %v, expected %v", h, v, hashes[ii])
		}
	}
}

func TestFileStore(t *testing.T) {
	dir, err := ioutil.TempDir("", "pool-test")
	if err != nil {
		t.Fatal(err)
	}
	//defer os.RemoveAll(dir)
	cfg := "file://" + dir
	testStore(t, nil, cfg)
}

func TestFileStoreMeta(t *testing.T) {
	dir, err := ioutil.TempDir("", "pool-test")
	if err != nil {
		t.Fatal(err)
	}
	//defer os.RemoveAll(dir)
	cfg := "file://" + dir
	testStore(t, &Meta{Foo: 5}, cfg)
}

func TestGridfs(t *testing.T) {
	if !testPort(27017) {
		t.Skip("mongodb is not running. mongodb memcache on localhost to run this test")
	}
	testStore(t, nil, "gridfs://localhost/blobstore_test")
}

func TestS3(t *testing.T) {
	b, err := ioutil.ReadFile("s3.txt")
	if err != nil || !strings.HasPrefix(string(b), "s3://") {
		abs, _ := filepath.Abs("s3.txt")
		t.Skipf("please, provide a file with a at %s s3 blobstore url to execute this test (e.g. \"s3://my-blobstore-test?access_key=akey&secret_key=some_secret\"", abs)
	}
	testStore(t, nil, strings.TrimSpace(string(b)))
}
<commit_msg>When testing a blobstore backend, test deletion too<commit_after>package blobstore

import (
	"fmt"
	_ "gnd.la/blobstore/driver/file"
	_ "gnd.la/blobstore/driver/gridfs"
	_ "gnd.la/blobstore/driver/s3"
	"hash/adler32"
	"io/ioutil"
	"net"
	"os"
	"path/filepath"
	"strings"
	"testing"
)

const (
	dataSize = 1 << 20 // 1MiB
)

func testPort(port int) bool {
	conn, err := net.Dial("tcp", fmt.Sprintf("localhost:%d", port))
	if err != nil {
		return false
	}
	conn.Close()
	return true
}

type Meta struct {
	Foo int
}

func fileData(t *testing.T, file string, size int64) []byte {
	f, err := os.Open(file)
	if err != nil {
		t.Fatal(err)
	}
	defer f.Close()
	b := make([]byte, size)
	if _, err := f.Read(b); err != nil {
		t.Fatal(err)
	}
	return b
}

func randData(t *testing.T, size int64) []byte {
	return fileData(t, "/dev/urandom", size)
}

func zeroData(t *testing.T, size int64) []byte {
	return fileData(t, "/dev/zero", size)
}

func testStore(t *testing.T, meta *Meta, config string) {
	t.Logf("Testing store with config %s", config)
	store, err := New(config)
	if err != nil {
		t.Fatal(err)
	}
	defer store.Close()
	var ids []string
	var hashes []uint32
	for ii := 0; ii < 10; ii++ {
		var r []byte
		if ii%2 == 0 {
			r = zeroData(t, dataSize)
		} else {
			r = randData(t, dataSize)
		}
		id, err := store.Store(r, meta)
		if err != nil {
			t.Error(err)
			continue
		}
		t.Logf("Stored file with id %s", id)
		ids = append(ids, id)
		hashes = append(hashes, adler32.Checksum(r))
	}
	for ii, v := range ids {
		f, err := store.Open(v)
		if err != nil {
			t.Error(err)
			continue
		}
		defer f.Close()
		t.Logf("Opened file %s", v)
		if f.Size() != dataSize {
			t.Errorf("Invalid data size for file %s. Want %v, got %v.", v, dataSize, f.Size())
		}
		if meta != nil {
			var m Meta
			if err := f.GetMeta(&m); err != nil {
				t.Errorf("error loading metadata from %v: %s", v, err)
			} else {
				if m.Foo != meta.Foo {
					t.Errorf("Invalid metadata value. Want %v, got %v.", meta.Foo, m.Foo)
				}
			}
		}
		if err := f.Verify(); err != nil {
			t.Errorf("error checking file %v: %s", v, err)
		}
		b, err := f.ReadAll()
		if err != nil {
			t.Error(err)
			continue
		}
		if len(b) != dataSize {
			t.Errorf("expecting %d bytes, got %d instead", dataSize, len(b))
			continue
		}
		if h := adler32.Checksum(b); h != hashes[ii] {
			t.Errorf("invalid hash %v for file %v, expected %v", h, v, hashes[ii])
		}
	}
	// Now remove all the files
	for _, v := range ids {
		if err := store.Delete(v); err != nil {
			t.Error(err)
		} else {
			t.Logf("deleted file %s", v)
		}
	}
	// Check that the files do not exist
	for _, v := range ids {
		if f, err := store.Open(v); err == nil || f != nil {
			t.Errorf("expecting nil file and non-nil err, got file %v and err %v instead", f, err)
			if f == nil {
				f.Close()
			}
		} else {
			t.Logf("file %s was deleted", v)
		}
	}
}

func TestFileStore(t *testing.T) {
	dir, err := ioutil.TempDir("", "pool-test")
	if err != nil {
		t.Fatal(err)
	}
	//defer os.RemoveAll(dir)
	cfg := "file://" + dir
	testStore(t, nil, cfg)
}

func TestFileStoreMeta(t *testing.T) {
	dir, err := ioutil.TempDir("", "pool-test")
	if err != nil {
		t.Fatal(err)
	}
	//defer os.RemoveAll(dir)
	cfg := "file://" + dir
	testStore(t, &Meta{Foo: 5}, cfg)
}

func TestGridfs(t *testing.T) {
	if !testPort(27017) {
		t.Skip("mongodb is not running. mongodb memcache on localhost to run this test")
	}
	testStore(t, nil, "gridfs://localhost/blobstore_test")
}

func TestS3(t *testing.T) {
	b, err := ioutil.ReadFile("s3.txt")
	if err != nil || !strings.HasPrefix(string(b), "s3://") {
		abs, _ := filepath.Abs("s3.txt")
		t.Skipf("please, provide a file with a at %s s3 blobstore url to execute this test (e.g. \"s3://my-blobstore-test?access_key=akey&secret_key=some_secret\"", abs)
	}
	testStore(t, nil, strings.TrimSpace(string(b)))
}
<|endoftext|>


<commit_before>package main

import (
	"time"

	"fmt"

	"github.com/JessonChan/yorm"
)
import _ "github.com/go-sql-driver/mysql"

/*
CREATE TABLE `program_language` (
  `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
  `name` varchar(32) DEFAULT NULL,
  `rank_month` date DEFAULT NULL,
  `position` int(11) DEFAULT NULL,
  `created` datetime DEFAULT NULL,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB  DEFAULT CHARSET=utf8;
*/

type ProgramLanguage struct {
	Id        int64
	Position  int
	Name      string
	RankMonth time.Time
	Created   time.Time
}

func main() {
	yorm.SetLoggerLevel(yorm.Debug)
	//设置自己的数据地址
	yorm.Register("root:@tcp(127.0.0.1:3306)/yorm_test?charset=utf8")

	//插入一条数据
	php := ProgramLanguage{Name: "PHP", Position: 7, RankMonth: time.Now(), Created: time.Now()}
	yorm.Insert(&php)

	//更新一条数据
	yorm.Update("update program_language set position=? where id=?", 8, php.Id)

	//删除一条
	yorm.Delete("delete from program_language where id=? ", php.Id)

	var ps []ProgramLanguage

	//读取所有的数据
	yorm.R(&ps)
	fmt.Println(ps)

	//读取所有小于10的数据
	yorm.R(&ps, "where id<10")
	fmt.Println(ps)

	//读取所有小于10的数据
	yorm.R(&ps, "where id<?", 10)
	fmt.Println(ps)

	//也可以
	yorm.Select(&ps, "where id<?", 10)
	fmt.Println(ps)

	//也可以
	yorm.Select(&ps, "where <10")
	fmt.Println(ps)

	var p ProgramLanguage = ProgramLanguage{Id: 1}

	//读取id为1的某条数据
	yorm.R(&p)
	fmt.Println(p)

	//读取id为1的某条数据
	yorm.SelectByPK(&p)
	fmt.Println(p)
}
<commit_msg>better format<commit_after>package main

import (
	"time"

	"fmt"

	"github.com/JessonChan/yorm"
)
import _ "github.com/go-sql-driver/mysql"

/*
CREATE TABLE `program_language` (
  `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
  `name` varchar(32) DEFAULT NULL,
  `rank_month` date DEFAULT NULL,
  `position` int(11) DEFAULT NULL,
  `created` datetime DEFAULT NULL,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB  DEFAULT CHARSET=utf8;
*/

type ProgramLanguage struct {
	Id        int64
	Position  int
	Name      string
	RankMonth time.Time
	Created   time.Time
}

func main() {
	yorm.SetLoggerLevel(yorm.Debug)
	// 设置自己的数据地址
	yorm.Register("root:@tcp(127.0.0.1:3306)/yorm_test?charset=utf8")

	// 插入一条数据
	php := ProgramLanguage{Name: "PHP", Position: 7, RankMonth: time.Now(), Created: time.Now()}
	yorm.Insert(&php)

	// 更新一条数据
	yorm.Update("update program_language set position=? where id=?", 8, php.Id)

	// 删除一条
	yorm.Delete("delete from program_language where id=? ", php.Id)

	var ps []ProgramLanguage

	// 读取所有的数据
	yorm.R(&ps)
	fmt.Println(ps)

	// 读取所有小于10的数据
	yorm.R(&ps, "where id<10")
	fmt.Println(ps)

	// 读取所有小于10的数据
	yorm.R(&ps, "where id<?", 10)
	fmt.Println(ps)

	// 也可以
	yorm.Select(&ps, "where id<?", 10)
	fmt.Println(ps)

	// 也可以
	yorm.Select(&ps, "where <10")
	fmt.Println(ps)

	var p ProgramLanguage = ProgramLanguage{Id: 1}

	// 读取id为1的某条数据
	yorm.R(&p)
	fmt.Println(p)

	// 读取id为1的某条数据
	yorm.SelectByPK(&p)
	fmt.Println(p)
}
<|endoftext|>


<commit_before>package main

import "fmt"

func main() {
    fmt.Println("Hello world!")
}
<commit_msg>Http reequest<commit_after>package main

import (
    "bufio"
    "fmt"
    "net/http"
)

func main() {
    resp, err := http.Get("http://localhost:19071/ApplicationStatus")
    if err != nil {
        panic(err)
    }
    defer resp.Body.Close()

    fmt.Println("Response status: ", resp.Status)
    scanner := bufio.NewScanner(resp.Body)
    for i := 0; scanner.Scan() && i < 5; i++ {
        fmt.Println(scanner.Text())
    }

    if err := scanner.Err(); err != nil {
        panic(err)
    }
}
<|endoftext|>


<commit_before>// Copyright 2014 Hari haran. All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"os"

	"github.com/gorilla/websocket"
	"golang.org/x/crypto/ssh/terminal"
)

func speculoHandler(w http.ResponseWriter, r *http.Request) {
	ws, err := websocket.Upgrade(w, r, nil, 1024, 1024)
	if _, ok := err.(websocket.HandshakeError); ok {
		http.Error(w, "Not a websocket handshake", 400)
		return
	} else if err != nil {
		log.Fatal(err)
		return
	}

	oldState, err := terminal.MakeRaw(0)
	if err != nil {
		panic(err)
	}
	defer terminal.Restore(0, oldState)
	term := terminal.NewTerminal(os.Stdin, "> ")

	term.AutoCompleteCallback = func(line string, pos int, key rune) (newLine string, newPos int, ok bool) {
		if key == '\x03' { // Ctrl-C keycode is \x03
			fmt.Println()
			if oldState != nil {
				terminal.Restore(0, oldState)
			}
			os.Exit(0)
		}
		return "", 0, false
	}

	go func() {
		newline := []byte("\n")
		for {
			_, r, err := ws.ReadMessage() // messageType is ignored
			if err != nil {
				log.Fatal(err)
			}
			term.Write(r)
			term.Write(newline)
		}
	}()

	for {
		text, err := term.ReadLine()
		if err != nil {
			// Ctrl-D is EOF. So, quit without printing stacktrace.
			if err == io.EOF {
				fmt.Println()
				if oldState != nil {
					terminal.Restore(0, oldState)
				}
				os.Exit(0)
			}
			panic(err)
		}
		inputcommand := []byte(text)
		if err := ws.WriteMessage(1, inputcommand); err != nil {
			log.Println(err)
		}
	}
}

func main() {
	path, err := os.Getwd()
	if err != nil {
		log.Fatal(err)
	}

	http.Handle("/", http.FileServer(http.Dir(path)))
	http.HandleFunc("/repl", speculoHandler)

	fmt.Println("Open http://localhost:8080 and type js commands in the terminal!")
	http.ListenAndServe(":8080", nil)
}
<commit_msg>handle exit cleanly and also when the browser window is closed<commit_after>// Copyright 2014 Hari haran. All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"os"

	"github.com/gorilla/websocket"
	"golang.org/x/crypto/ssh/terminal"
)

func speculoHandler(w http.ResponseWriter, r *http.Request) {
	ws, err := websocket.Upgrade(w, r, nil, 1024, 1024)
	if _, ok := err.(websocket.HandshakeError); ok {
		http.Error(w, "Not a websocket handshake", 400)
		return
	} else if err != nil {
		log.Println(err)
		return
	}

	oldState, err := terminal.MakeRaw(0)
	if err != nil {
		panic(err)
	}
	defer terminal.Restore(0, oldState)
	term := terminal.NewTerminal(os.Stdin, "> ")

	term.AutoCompleteCallback = func(line string, pos int, key rune) (newLine string, newPos int, ok bool) {
		if key == '\x03' { // Ctrl-C keycode is \x03
			cleanAndExit(oldState)
		}
		return "", 0, false
	}

	notify := w.(http.CloseNotifier).CloseNotify()

	go func() {
		<-notify
		cleanAndExit(oldState)
	}()

	go func() {
		newline := []byte("\n")
		for {
			_, r, err := ws.ReadMessage() // messageType is ignored
			if err != nil {
				log.Println(err)
				return
			}
			term.Write(r)
			term.Write(newline)
		}
	}()

	for {
		text, err := term.ReadLine()
		if err != nil {
			// Ctrl-D is EOF. So, quit without printing stacktrace.
			if err == io.EOF {
				cleanAndExit(oldState)
			}
			log.Println(err)
			return
		}
		inputcommand := []byte(text)
		if err := ws.WriteMessage(1, inputcommand); err != nil {
			log.Println(err)
			return
		}
	}
}

func cleanAndExit(oldState *terminal.State) {
	fmt.Println()
	if oldState != nil {
		terminal.Restore(0, oldState)
	}
	os.Exit(0)
}

func main() {
	log.SetFlags(0)
	log.SetPrefix("speculo: ")

	path, err := os.Getwd()
	if err != nil {
		log.Fatal(err)
	}

	http.Handle("/", http.FileServer(http.Dir(path)))
	http.HandleFunc("/repl", speculoHandler)

	fmt.Println("Open http://localhost:8080 and type js commands in the terminal!")
	http.ListenAndServe(":8080", nil)
}
<|endoftext|>


<commit_before>// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package spinner is a simple package to add a spinner / progress indicator to any terminal application.
package spinner

import (
	"errors"
	"fmt"
	"io"
	"strconv"
	"sync"
	"time"
	"unicode/utf8"

	"github.com/fatih/color"
)

// errInvalidColor is returned when attempting to set an invalid color
var errInvalidColor = errors.New("invalid color")

// validColors holds an array of the only colors allowed
var validColors = []string{"red", "green", "yellow", "blue", "magenta", "cyan", "white"}

// validColor will make sure the given color is actually allowed
func validColor(c string) bool {
	for _, i := range validColors {
		if c == i {
			return true
		}
	}
	return false
}

// Spinner struct to hold the provided options
type Spinner struct {
	Delay      time.Duration                 // Delay is the speed of the indicator
	chars      []string                      // chars holds the chosen character set
	Prefix     string                        // Prefix is the text preppended to the indicator
	Suffix     string                        // Suffix is the text appended to the indicator
	FinalMSG   string                        // string displayed after Stop() is called
	lastOutput string                        // last character(set) written
	color      func(a ...interface{}) string // default color is white
	lock       *sync.RWMutex                 // Lock useed for
	Writer     io.Writer                     // to make testing better, exported so users have access
	active     bool                          // active holds the state of the spinner
	stopChan   chan struct{}                 // stopChan is a channel used to stop the indicator
}

// New provides a pointer to an instance of Spinner with the supplied options
func New(cs []string, d time.Duration) *Spinner {
	return &Spinner{
		Delay:    d,
		chars:    cs,
		color:    color.New(color.FgWhite).SprintFunc(),
		lock:     &sync.RWMutex{},
		Writer:   color.Output,
		active:   false,
		stopChan: make(chan struct{}, 1),
	}
}

// Start will start the indicator
func (s *Spinner) Start() {
	if s.active {
		return
	}
	s.active = true

	go func() {
		for {
			for i := 0; i < len(s.chars); i++ {
				select {
				case <-s.stopChan:
					s.erase(s.lastOutput)
					fmt.Fprint(s.Writer, s.lastOutput)
					return
				default:
					fmt.Fprint(s.Writer, fmt.Sprintf("%s%s%s ", s.Prefix, s.color(s.chars[i]), s.Suffix))
					out := fmt.Sprintf("%s%s%s ", s.Prefix, s.chars[i], s.Suffix)
					s.lastOutput = out
					s.lock.RLock()
					time.Sleep(s.Delay)
					s.lock.RUnlock()
					s.erase(out)
				}
			}
		}
	}()
}

// Stop stops the indicator
func (s *Spinner) Stop() {
	s.lock.Lock()
	defer s.lock.Unlock()
	if s.active {
		s.stopChan <- struct{}{}
		s.active = false
		if s.FinalMSG != "" {
			fmt.Fprintf(s.Writer, s.FinalMSG)
		}
	}
}

// Restart will stop and start the indicator
func (s *Spinner) Restart() {
	s.Stop()
	s.Start()
}

// Reverse will reverse the order of the slice assigned to the indicator
func (s *Spinner) Reverse() {
	s.lock.Lock()
	defer s.lock.Unlock()
	for i, j := 0, len(s.chars)-1; i < j; i, j = i+1, j-1 {
		s.chars[i], s.chars[j] = s.chars[j], s.chars[i]
	}
}

// Color will set the struct field for the given color to be used
func (s *Spinner) Color(c string) error {
	if validColor(c) {
		switch c {
		case "red":
			s.color = color.New(color.FgRed).SprintFunc()
			s.Restart()
		case "yellow":
			s.color = color.New(color.FgYellow).SprintFunc()
			s.Restart()
		case "green":
			s.color = color.New(color.FgGreen).SprintFunc()
			s.Restart()
		case "magenta":
			s.color = color.New(color.FgMagenta).SprintFunc()
			s.Restart()
		case "blue":
			s.color = color.New(color.FgBlue).SprintFunc()
			s.Restart()
		case "cyan":
			s.color = color.New(color.FgCyan).SprintFunc()
			s.Restart()
		case "white":
			s.color = color.New(color.FgWhite).SprintFunc()
			s.Restart()
		default:
			return errInvalidColor
		}
	}
	return nil
}

// UpdateSpeed will set the indicator delay to the given value
func (s *Spinner) UpdateSpeed(d time.Duration) {
	s.lock.Lock()
	defer s.lock.Unlock()
	s.Delay = d
}

// UpdateCharSet will change the current character set to the given one
func (s *Spinner) UpdateCharSet(cs []string) {
	s.lock.Lock()
	defer s.lock.Unlock()
	s.chars = cs
}

// erase deletes written characters
func (s *Spinner) erase(a string) {
	n := utf8.RuneCountInString(a)
	s.lock.RLock()
	defer s.lock.RUnlock()
	for i := 0; i < n; i++ {
		fmt.Fprintf(s.Writer, "\b")
	}
}

// GenerateNumberSequence will generate a slice of integers at the
// provided length and convert them each to a string
func GenerateNumberSequence(length int) []string {
	var numSeq []string
	for i := 0; i < length; i++ {
		numSeq = append(numSeq, strconv.Itoa(i))
	}
	return numSeq
}
<commit_msg>formatting, small functioanl changes<commit_after>// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package spinner is a simple package to add a spinner / progress indicator to any terminal application.
package spinner

import (
	"errors"
	"fmt"
	"io"
	"strconv"
	"sync"
	"time"
	"unicode/utf8"

	"github.com/fatih/color"
)

// errInvalidColor is returned when attempting to set an invalid color
var errInvalidColor = errors.New("invalid color")

// validColors holds an array of the only colors allowed
var validColors = []string{"red", "green", "yellow", "blue", "magenta", "cyan", "white"}

// validColor will make sure the given color is actually allowed
func validColor(c string) bool {
	for _, i := range validColors {
		if c == i {
			return true
		}
	}
	return false
}

// Spinner struct to hold the provided options
type Spinner struct {
	Delay          time.Duration                 // Delay is the speed of the indicator
	chars          []string                      // chars holds the chosen character set
	Prefix         string                        // Prefix is the text preppended to the indicator
	Suffix         string                        // Suffix is the text appended to the indicator
	FinalMSG       string                        // string displayed after Stop() is called
	lastOutput     string                        // last character(set) written
	color          func(a ...interface{}) string // default color is white
	lock           *sync.RWMutex                 // Lock useed for
	Writer         io.Writer                     // to make testing better, exported so users have access
	active         bool                          // active holds the state of the spinner
	stopChan       chan struct{}                 // stopChan is a channel used to stop the indicator
	lastOutputChan chan string                   // allows main to safely get the last output from the spinner goroutine
}

// New provides a pointer to an instance of Spinner with the supplied options
func New(cs []string, d time.Duration) *Spinner {
	return &Spinner{
		Delay:          d,
		chars:          cs,
		color:          color.New(color.FgWhite).SprintFunc(),
		lock:           &sync.RWMutex{},
		Writer:         color.Output,
		active:         false,
		stopChan:       make(chan struct{}, 1),
		lastOutputChan: make(chan string, 1),
	}
}

// Start will start the indicator
func (s *Spinner) Start() {
	if s.active {
		return
	}
	s.active = true

	go func() {
		for {
			for i := 0; i < len(s.chars); i++ {
				select {
				case <-s.stopChan:
					s.erase(<-s.lastOutputChan)
					return
				default:
					fmt.Fprint(s.Writer, fmt.Sprintf("%s%s%s ", s.Prefix, s.color(s.chars[i]), s.Suffix))
					out := fmt.Sprintf("%s%s%s ", s.Prefix, s.chars[i], s.Suffix)
					s.lastOutput = out
					s.lock.RLock()
					time.Sleep(s.Delay)
					s.lock.RUnlock()
					s.erase(out)
				}
			}
		}
	}()
}

// Stop stops the indicator
func (s *Spinner) Stop() {
	s.lock.Lock()
	defer s.lock.Unlock()
	if s.active {
		s.active = false
		if s.FinalMSG != "" {
			fmt.Fprintf(s.Writer, s.FinalMSG)
		}
		s.stopChan <- struct{}{}
	}
}

// Restart will stop and start the indicator
func (s *Spinner) Restart() {
	s.Stop()
	s.Start()
}

// Reverse will reverse the order of the slice assigned to the indicator
func (s *Spinner) Reverse() {
	s.lock.Lock()
	defer s.lock.Unlock()
	for i, j := 0, len(s.chars)-1; i < j; i, j = i+1, j-1 {
		s.chars[i], s.chars[j] = s.chars[j], s.chars[i]
	}
}

// Color will set the struct field for the given color to be used
func (s *Spinner) Color(c string) error {
	if validColor(c) {
		switch c {
		case "red":
			s.color = color.New(color.FgRed).SprintFunc()
			s.Restart()
		case "yellow":
			s.color = color.New(color.FgYellow).SprintFunc()
			s.Restart()
		case "green":
			s.color = color.New(color.FgGreen).SprintFunc()
			s.Restart()
		case "magenta":
			s.color = color.New(color.FgMagenta).SprintFunc()
			s.Restart()
		case "blue":
			s.color = color.New(color.FgBlue).SprintFunc()
			s.Restart()
		case "cyan":
			s.color = color.New(color.FgCyan).SprintFunc()
			s.Restart()
		case "white":
			s.color = color.New(color.FgWhite).SprintFunc()
			s.Restart()
		default:
			return errInvalidColor
		}
	}
	return nil
}

// UpdateSpeed will set the indicator delay to the given value
func (s *Spinner) UpdateSpeed(d time.Duration) {
	s.lock.Lock()
	defer s.lock.Unlock()
	s.Delay = d
}

// UpdateCharSet will change the current character set to the given one
func (s *Spinner) UpdateCharSet(cs []string) {
	s.lock.Lock()
	defer s.lock.Unlock()
	s.chars = cs
}

// erase deletes written characters
func (s *Spinner) erase(a string) {
	n := utf8.RuneCountInString(a)
	s.lock.Lock()
	defer s.lock.Unlock()
	for i := 0; i < n; i++ {
		fmt.Fprintf(s.Writer, "\b")
	}
}

// GenerateNumberSequence will generate a slice of integers at the
// provided length and convert them each to a string
func GenerateNumberSequence(length int) []string {
	numSeq := make([]string, length)
	for i := 0; i < length; i++ {
		numSeq = append(numSeq, strconv.Itoa(i))
	}
	return numSeq
}
<|endoftext|>


<commit_before>package apptail

import (
	"fmt"
	"github.com/ActiveState/log"
	"github.com/ActiveState/tail"
	"github.com/ActiveState/zmqpubsub"
	"logyard"
	"logyard/clients/messagecommon"
	"os"
	"path/filepath"
	"strings"
	"time"
)

// Instance is the NATS message sent by dea_ng to notify of new instances.
type Instance struct {
	AppGUID  string
	AppName  string
	AppSpace string
	Type     string
	Index    int
	DockerId string `json:"docker_id"`
	RootPath string
	LogFiles map[string]string
}

func (instance *Instance) Identifier() string {
	return fmt.Sprintf("%v[%v:%v]", instance.AppName, instance.Index, instance.DockerId[:ID_LENGTH])
}

// Tail begins tailing the files for this instance.
func (instance *Instance) Tail() {
	log.Infof("Tailing %v logs for %v -- %+v",
		instance.Type, instance.Identifier(), instance)

	stopCh := make(chan bool)
	logfiles := instance.getLogFiles()

	log.Infof("Determined log files: %+v", logfiles)

	for name, filename := range logfiles {
		go instance.tailFile(name, filename, stopCh)
	}

	go func() {
		DockerListener.WaitForContainer(instance.DockerId)
		log.Infof("Container for %v exited", instance.Identifier())
		close(stopCh)
	}()
}

func (instance *Instance) tailFile(name, filename string, stopCh chan bool) {
	var err error

	pub := logyard.Broker.NewPublisherMust()
	defer pub.Stop()

	limit, err := instance.getReadLimit(pub, name, filename)
	if err != nil {
		log.Warn(err)
		return
	}

	t, err := tail.TailFile(filename, tail.Config{
		MaxLineSize: GetConfig().MaxRecordSize,
		MustExist:   true,
		Follow:      true,
		Location:    &tail.SeekInfo{-limit, os.SEEK_END},
		ReOpen:      false,
		Poll:        false,
		LimitRate:   GetConfig().RateLimit})
	if err != nil {
		log.Warnf("Cannot tail file (%s); %s", filename, err)
		return
	}

FORLOOP:
	for {
		select {
		case line, ok := <-t.Lines:
			if !ok {
				err = t.Wait()
				break FORLOOP
			}
			instance.publishLine(pub, name, line)
		case <-stopCh:
			err = t.Stop()
			break FORLOOP
		}
	}

	if err != nil {
		log.Warn(err)
	}

	log.Infof("Completed tailing %v log for %v", name, instance.Identifier())
}

func (instance *Instance) getLogFiles() map[string]string {
	var logfiles map[string]string

	rawMode := len(instance.LogFiles) > 0
	if rawMode {
		// If the logfiles list was explicitly passed, use it as is.
		logfiles = instance.LogFiles
	} else {
		// Use $STACKATO_LOG_FILES
		logfiles = make(map[string]string)
		if env, err := GetDockerAppEnv(instance.RootPath); err != nil {
			log.Errorf("Failed to read docker image env: %v", err)
		} else {
			if s, ok := env["STACKATO_LOG_FILES"]; ok {
				parts := strings.Split(s, ":")
				if len(parts) > 7 {
					log.Warnf("$STACKATO_LOG_FILES contains more than 7 parts; using only last 7 parts")
					parts = parts[len(parts)-7 : len(parts)]
				}
				for _, f := range parts {
					parts := strings.SplitN(f, "=", 2)
					logfiles[parts[0]] = parts[1]
				}
			} else {
				log.Errorf("Expected env $STACKATO_LOG_FILES not found in docker image")
			}
		}
	}

	pub := logyard.Broker.NewPublisherMust()
	defer pub.Stop()
	// XXX: this delay is unfortunately required, else the publish calls
	// (instance.notify) below for warnings will get ignored.
	time.Sleep(100 * time.Millisecond)

	// Expand paths, and securely ensure they fall within the app root.
	logfilesSecure := make(map[string]string)
	for name, path := range logfiles {
		// Treat relative paths as being relative to the app directory.
		if !filepath.IsAbs(path) {
			path = filepath.Join("/app/app/", path)
		}

		fullpath := filepath.Join(instance.RootPath, path)
		fullpath, err := filepath.Abs(fullpath)
		if err != nil {
			log.Warnf("Cannot find Abs of %v <join> %v: %v", instance.RootPath, path, err)
			instance.notify(pub, fmt.Sprintf("WARN -- Failed to find absolute path for %v", path))
			continue
		}
		fullpath, err = filepath.EvalSymlinks(fullpath)
		if err != nil {
			instance.notify(pub, fmt.Sprintf("WARN -- Ignoring missing/inaccessible path %v", path))
			continue
		}
		if !strings.HasPrefix(fullpath, instance.RootPath) {
			log.Warnf("Ignoring insecure log path %v (via %v) in instance %+v", fullpath, path, instance)
			// This user warning is exactly the same as above, lest we provide
			// a backdoor for a malicious user to list the directory tree on
			// the host.
			instance.notify(pub, fmt.Sprintf("WARN -- Ignoring missing/inaccessible path %v", path))
			continue
		}
		logfilesSecure[name] = fullpath
	}

	return logfilesSecure
}

func (instance *Instance) getReadLimit(
	pub *zmqpubsub.Publisher,
	logname string,
	filename string) (int64, error) {
	// convert MB to limit in bytes.
	filesizeLimit := GetConfig().FileSizeLimit * 1024 * 1024
	if !(filesizeLimit > 0) {
		panic("invalid value for `read_limit' in apptail config")
	}

	fi, err := os.Stat(filename)
	if err != nil {
		return -1, fmt.Errorf("Cannot stat file (%s); %s", filename, err)
	}
	size := fi.Size()
	limit := filesizeLimit
	if size > filesizeLimit {
		err := fmt.Errorf("Skipping much of a large log file (%s); size (%v bytes) > read_limit (%v bytes)",
			logname, size, filesizeLimit)
		// Publish special error message.
		instance.publishLine(pub, logname, &tail.Line{
			Text: err.Error(),
			Time: time.Now(),
			Err:  err})
	} else {
		limit = size
	}
	return limit, nil
}

// publishLine publishes a log line corresponding to this instance.
func (instance *Instance) publishLine(pub *zmqpubsub.Publisher, logname string, line *tail.Line) {
	instance.publishLineAs(pub, instance.Type, logname, line)
}

func (instance *Instance) notify(pub *zmqpubsub.Publisher, line string) {
	instance.publishLineAs(pub, "stackato.apptail", "", tail.NewLine(line))
}

func (instance *Instance) publishLineAs(pub *zmqpubsub.Publisher, source string, logname string, line *tail.Line) {
	if line == nil {
		panic("line is nil")
	}

	msg := &Message{
		LogFilename:   logname,
		Source:        source,
		InstanceIndex: instance.Index,
		AppGUID:       instance.AppGUID,
		AppName:       instance.AppName,
		AppSpace:      instance.AppSpace,
		MessageCommon: messagecommon.New(line.Text, line.Time, LocalNodeId()),
	}

	if line.Err != nil {
		// Mark this as a special error record, as it is
		// coming from tail, not the app.
		msg.Source = "stackato.apptail"
		msg.LogFilename = ""
		log.Warnf("[%s] %s", instance.AppName, line.Text)
	}

	err := msg.Publish(pub, false)
	if err != nil {
		Fatal("unable to publish: %v", err)
	}
}
<commit_msg>use the user provide path in warnings<commit_after>package apptail

import (
	"fmt"
	"github.com/ActiveState/log"
	"github.com/ActiveState/tail"
	"github.com/ActiveState/zmqpubsub"
	"logyard"
	"logyard/clients/messagecommon"
	"os"
	"path/filepath"
	"strings"
	"time"
)

// Instance is the NATS message sent by dea_ng to notify of new instances.
type Instance struct {
	AppGUID  string
	AppName  string
	AppSpace string
	Type     string
	Index    int
	DockerId string `json:"docker_id"`
	RootPath string
	LogFiles map[string]string
}

func (instance *Instance) Identifier() string {
	return fmt.Sprintf("%v[%v:%v]", instance.AppName, instance.Index, instance.DockerId[:ID_LENGTH])
}

// Tail begins tailing the files for this instance.
func (instance *Instance) Tail() {
	log.Infof("Tailing %v logs for %v -- %+v",
		instance.Type, instance.Identifier(), instance)

	stopCh := make(chan bool)
	logfiles := instance.getLogFiles()

	log.Infof("Determined log files: %+v", logfiles)

	for name, filename := range logfiles {
		go instance.tailFile(name, filename, stopCh)
	}

	go func() {
		DockerListener.WaitForContainer(instance.DockerId)
		log.Infof("Container for %v exited", instance.Identifier())
		close(stopCh)
	}()
}

func (instance *Instance) tailFile(name, filename string, stopCh chan bool) {
	var err error

	pub := logyard.Broker.NewPublisherMust()
	defer pub.Stop()

	limit, err := instance.getReadLimit(pub, name, filename)
	if err != nil {
		log.Warn(err)
		return
	}

	t, err := tail.TailFile(filename, tail.Config{
		MaxLineSize: GetConfig().MaxRecordSize,
		MustExist:   true,
		Follow:      true,
		Location:    &tail.SeekInfo{-limit, os.SEEK_END},
		ReOpen:      false,
		Poll:        false,
		LimitRate:   GetConfig().RateLimit})
	if err != nil {
		log.Warnf("Cannot tail file (%s); %s", filename, err)
		return
	}

FORLOOP:
	for {
		select {
		case line, ok := <-t.Lines:
			if !ok {
				err = t.Wait()
				break FORLOOP
			}
			instance.publishLine(pub, name, line)
		case <-stopCh:
			err = t.Stop()
			break FORLOOP
		}
	}

	if err != nil {
		log.Warn(err)
	}

	log.Infof("Completed tailing %v log for %v", name, instance.Identifier())
}

func (instance *Instance) getLogFiles() map[string]string {
	var logfiles map[string]string

	rawMode := len(instance.LogFiles) > 0
	if rawMode {
		// If the logfiles list was explicitly passed, use it as is.
		logfiles = instance.LogFiles
	} else {
		// Use $STACKATO_LOG_FILES
		logfiles = make(map[string]string)
		if env, err := GetDockerAppEnv(instance.RootPath); err != nil {
			log.Errorf("Failed to read docker image env: %v", err)
		} else {
			if s, ok := env["STACKATO_LOG_FILES"]; ok {
				parts := strings.Split(s, ":")
				if len(parts) > 7 {
					log.Warnf("$STACKATO_LOG_FILES contains more than 7 parts; using only last 7 parts")
					parts = parts[len(parts)-7 : len(parts)]
				}
				for _, f := range parts {
					parts := strings.SplitN(f, "=", 2)
					logfiles[parts[0]] = parts[1]
				}
			} else {
				log.Errorf("Expected env $STACKATO_LOG_FILES not found in docker image")
			}
		}
	}

	pub := logyard.Broker.NewPublisherMust()
	defer pub.Stop()
	// XXX: this delay is unfortunately required, else the publish calls
	// (instance.notify) below for warnings will get ignored.
	time.Sleep(100 * time.Millisecond)

	// Expand paths, and securely ensure they fall within the app root.
	logfilesSecure := make(map[string]string)
	for name, path := range logfiles {
		var fullpath string

		// Treat relative paths as being relative to the app directory.
		if !filepath.IsAbs(path) {
			fullpath = filepath.Join(instance.RootPath, "/app/app/", path)
		} else {
			fullpath = filepath.Join(instance.RootPath, path)
		}

		fullpath, err := filepath.Abs(fullpath)
		if err != nil {
			log.Warnf("Cannot find Abs of %v <join> %v: %v", instance.RootPath, path, err)
			instance.notify(pub, fmt.Sprintf("WARN -- Failed to find absolute path for %v", path))
			continue
		}
		fullpath, err = filepath.EvalSymlinks(fullpath)
		if err != nil {
			instance.notify(pub, fmt.Sprintf("WARN -- Ignoring missing/inaccessible path %v", path))
			continue
		}
		if !strings.HasPrefix(fullpath, instance.RootPath) {
			log.Warnf("Ignoring insecure log path %v (via %v) in instance %+v", fullpath, path, instance)
			// This user warning is exactly the same as above, lest we provide
			// a backdoor for a malicious user to list the directory tree on
			// the host.
			instance.notify(pub, fmt.Sprintf("WARN -- Ignoring missing/inaccessible path %v", path))
			continue
		}
		logfilesSecure[name] = fullpath
	}

	return logfilesSecure
}

func (instance *Instance) getReadLimit(
	pub *zmqpubsub.Publisher,
	logname string,
	filename string) (int64, error) {
	// convert MB to limit in bytes.
	filesizeLimit := GetConfig().FileSizeLimit * 1024 * 1024
	if !(filesizeLimit > 0) {
		panic("invalid value for `read_limit' in apptail config")
	}

	fi, err := os.Stat(filename)
	if err != nil {
		return -1, fmt.Errorf("Cannot stat file (%s); %s", filename, err)
	}
	size := fi.Size()
	limit := filesizeLimit
	if size > filesizeLimit {
		err := fmt.Errorf("Skipping much of a large log file (%s); size (%v bytes) > read_limit (%v bytes)",
			logname, size, filesizeLimit)
		// Publish special error message.
		instance.publishLine(pub, logname, &tail.Line{
			Text: err.Error(),
			Time: time.Now(),
			Err:  err})
	} else {
		limit = size
	}
	return limit, nil
}

// publishLine publishes a log line corresponding to this instance.
func (instance *Instance) publishLine(pub *zmqpubsub.Publisher, logname string, line *tail.Line) {
	instance.publishLineAs(pub, instance.Type, logname, line)
}

func (instance *Instance) notify(pub *zmqpubsub.Publisher, line string) {
	instance.publishLineAs(pub, "stackato.apptail", "", tail.NewLine(line))
}

func (instance *Instance) publishLineAs(pub *zmqpubsub.Publisher, source string, logname string, line *tail.Line) {
	if line == nil {
		panic("line is nil")
	}

	msg := &Message{
		LogFilename:   logname,
		Source:        source,
		InstanceIndex: instance.Index,
		AppGUID:       instance.AppGUID,
		AppName:       instance.AppName,
		AppSpace:      instance.AppSpace,
		MessageCommon: messagecommon.New(line.Text, line.Time, LocalNodeId()),
	}

	if line.Err != nil {
		// Mark this as a special error record, as it is
		// coming from tail, not the app.
		msg.Source = "stackato.apptail"
		msg.LogFilename = ""
		log.Warnf("[%s] %s", instance.AppName, line.Text)
	}

	err := msg.Publish(pub, false)
	if err != nil {
		Fatal("unable to publish: %v", err)
	}
}
<|endoftext|>