<commit_before>/*
Copyright 2017 Google, Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package util

import (
	"archive/tar"
	"encoding/json"
	"errors"
	"io/ioutil"
	"os"
	"strings"

	"github.com/GoogleCloudPlatform/container-diff/cmd/util/output"
	"github.com/containers/image/docker"
	"github.com/containers/image/manifest"
	"github.com/containers/image/pkg/compression"
	"github.com/containers/image/types"
	"github.com/sirupsen/logrus"
)

type Prepper interface {
	Name() string
	GetConfig() (ConfigSchema, error)
	GetFileSystem() (string, error)
	GetImage() (Image, error)
	GetSource() string
}

type ImageType int

const (
	ImageTypeTar ImageType = iota
	ImageTypeDaemon
	ImageTypeCloud
)

type Image struct {
	Source string
	FSPath string
	Config ConfigSchema
	Type   ImageType
}

func (i *Image) IsTar() bool {
	return i.Type == ImageTypeTar
}

func (i *Image) IsDaemon() bool {
	return i.Type == ImageTypeDaemon
}

func (i *Image) IsCloud() bool {
	return i.Type == ImageTypeCloud
}

func (i *Image) GetRemoteDigest() (string, error) {
	ref, err := docker.ParseReference("//" + i.Source)
	if err != nil {
		return "", err
	}
	return getDigestFromReference(ref, i.Source)
}

func (i *Image) GetName() string {
	return strings.Split(i.Source, ":")[0]
}

type ImageHistoryItem struct {
	CreatedBy string `json:"created_by"`
}

type ConfigObject struct {
	Env          []string            `json:"Env"`
	Entrypoint   []string            `json:"Entrypoint"`
	ExposedPorts map[string]struct{} `json:"ExposedPorts"`
	Cmd          []string            `json:"Cmd"`
	Volumes      map[string]struct{} `json:"Volumes"`
	Workdir      string              `json:"WorkingDir"`
	// Labels       map[string]string   `json:"Labels"`
}

type ConfigSchema struct {
	Config  ConfigObject       `json:"config"`
	History []ImageHistoryItem `json:"history"`
}

func getImage(p Prepper) (Image, error) {
	output.PrintToStdErr("Retrieving image %s from source %s\n", p.GetSource(), p.Name())
	imgPath, err := p.GetFileSystem()
	if err != nil {
		return Image{}, err
	}

	config, err := p.GetConfig()
	if err != nil {
		logrus.Error("Error retrieving History: ", err)
	}

	logrus.Infof("Finished prepping image %s", p.GetSource())
	return Image{
		Source: p.GetSource(),
		FSPath: imgPath,
		Config: config,
	}, nil
}

func getImageFromTar(tarPath string) (string, error) {
	logrus.Info("Extracting image tar to obtain image file system")
	tempPath, err := ioutil.TempDir("", ".container-diff")
	if err != nil {
		return "", err
	}
	return tempPath, unpackDockerSave(tarPath, tempPath)
}

func getFileSystemFromReference(ref types.ImageReference, imgSrc types.ImageSource, path string) error {
	img, err := ref.NewImage(nil)
	if err != nil {
		return err
	}
	defer img.Close()
	for _, b := range img.LayerInfos() {
		bi, _, err := imgSrc.GetBlob(b)
		if err != nil {
			return err
		}
		defer bi.Close()
		f, reader, err := compression.DetectCompression(bi)
		if err != nil {
			return err
		}
		// Decompress if necessary.
		if f != nil {
			reader, err = f(reader)
			if err != nil {
				return err
			}
		}
		tr := tar.NewReader(reader)
		if err := unpackTar(tr, path); err != nil {
			return err
		}
	}
	return nil
}

func getDigestFromReference(ref types.ImageReference, source string) (string, error) {
	img, err := ref.NewImage(nil)
	if err != nil {
		logrus.Errorf("Error referencing image %s from registry: %s", source, err)
		return "", errors.New("Could not obtain image digest")
	}
	defer img.Close()

	rawManifest, _, err := img.Manifest()
	if err != nil {
		logrus.Errorf("Error referencing image %s from registry: %s", source, err)
		return "", errors.New("Could not obtain image digest")
	}

	digest, err := manifest.Digest(rawManifest)
	if err != nil {
		logrus.Errorf("Error referencing image %s from registry: %s", source, err)
		return "", errors.New("Could not obtain image digest")
	}

	return digest.String(), nil
}

func getConfigFromReference(ref types.ImageReference, source string) (ConfigSchema, error) {
	img, err := ref.NewImage(nil)
	if err != nil {
		logrus.Errorf("Error referencing image %s from registry: %s", source, err)
		return ConfigSchema{}, errors.New("Could not obtain image config")
	}
	defer img.Close()

	configBlob, err := img.ConfigBlob()
	if err != nil {
		logrus.Errorf("Error obtaining config blob for image %s from registry: %s", source, err)
		return ConfigSchema{}, errors.New("Could not obtain image config")
	}

	var config ConfigSchema
	err = json.Unmarshal(configBlob, &config)
	if err != nil {
		logrus.Errorf("Error with config file struct for image %s: %s", source, err)
		return ConfigSchema{}, errors.New("Could not obtain image config")
	}
	return config, nil
}

func CleanupImage(image Image) {
	if image.FSPath != "" {
		logrus.Infof("Removing image filesystem directory %s from system", image.FSPath)
		if err := os.RemoveAll(image.FSPath); err != nil {
			logrus.Error(err.Error())
		}
	}
}
<commit_msg>restored the Labels change<commit_after>/*
Copyright 2017 Google, Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package util

import (
	"archive/tar"
	"encoding/json"
	"errors"
	"io/ioutil"
	"os"
	"strings"

	"github.com/GoogleCloudPlatform/container-diff/cmd/util/output"
	"github.com/containers/image/docker"
	"github.com/containers/image/manifest"
	"github.com/containers/image/pkg/compression"
	"github.com/containers/image/types"
	"github.com/sirupsen/logrus"
)

type Prepper interface {
	Name() string
	GetConfig() (ConfigSchema, error)
	GetFileSystem() (string, error)
	GetImage() (Image, error)
	GetSource() string
}

type ImageType int

const (
	ImageTypeTar ImageType = iota
	ImageTypeDaemon
	ImageTypeCloud
)

type Image struct {
	Source string
	FSPath string
	Config ConfigSchema
	Type   ImageType
}

func (i *Image) IsTar() bool {
	return i.Type == ImageTypeTar
}

func (i *Image) IsDaemon() bool {
	return i.Type == ImageTypeDaemon
}

func (i *Image) IsCloud() bool {
	return i.Type == ImageTypeCloud
}

func (i *Image) GetRemoteDigest() (string, error) {
	ref, err := docker.ParseReference("//" + i.Source)
	if err != nil {
		return "", err
	}
	return getDigestFromReference(ref, i.Source)
}

func (i *Image) GetName() string {
	return strings.Split(i.Source, ":")[0]
}

type ImageHistoryItem struct {
	CreatedBy string `json:"created_by"`
}

type ConfigObject struct {
	Env          []string            `json:"Env"`
	Entrypoint   []string            `json:"Entrypoint"`
	ExposedPorts map[string]struct{} `json:"ExposedPorts"`
	Cmd          []string            `json:"Cmd"`
	Volumes      map[string]struct{} `json:"Volumes"`
	Workdir      string              `json:"WorkingDir"`
	Labels       map[string]string   `json:"Labels"`
}

type ConfigSchema struct {
	Config  ConfigObject       `json:"config"`
	History []ImageHistoryItem `json:"history"`
}

func getImage(p Prepper) (Image, error) {
	output.PrintToStdErr("Retrieving image %s from source %s\n", p.GetSource(), p.Name())
	imgPath, err := p.GetFileSystem()
	if err != nil {
		return Image{}, err
	}

	config, err := p.GetConfig()
	if err != nil {
		logrus.Error("Error retrieving History: ", err)
	}

	logrus.Infof("Finished prepping image %s", p.GetSource())
	return Image{
		Source: p.GetSource(),
		FSPath: imgPath,
		Config: config,
	}, nil
}

func getImageFromTar(tarPath string) (string, error) {
	logrus.Info("Extracting image tar to obtain image file system")
	tempPath, err := ioutil.TempDir("", ".container-diff")
	if err != nil {
		return "", err
	}
	return tempPath, unpackDockerSave(tarPath, tempPath)
}

func getFileSystemFromReference(ref types.ImageReference, imgSrc types.ImageSource, path string) error {
	img, err := ref.NewImage(nil)
	if err != nil {
		return err
	}
	defer img.Close()
	for _, b := range img.LayerInfos() {
		bi, _, err := imgSrc.GetBlob(b)
		if err != nil {
			return err
		}
		defer bi.Close()
		f, reader, err := compression.DetectCompression(bi)
		if err != nil {
			return err
		}
		// Decompress if necessary.
		if f != nil {
			reader, err = f(reader)
			if err != nil {
				return err
			}
		}
		tr := tar.NewReader(reader)
		if err := unpackTar(tr, path); err != nil {
			return err
		}
	}
	return nil
}

func getDigestFromReference(ref types.ImageReference, source string) (string, error) {
	img, err := ref.NewImage(nil)
	if err != nil {
		logrus.Errorf("Error referencing image %s from registry: %s", source, err)
		return "", errors.New("Could not obtain image digest")
	}
	defer img.Close()

	rawManifest, _, err := img.Manifest()
	if err != nil {
		logrus.Errorf("Error referencing image %s from registry: %s", source, err)
		return "", errors.New("Could not obtain image digest")
	}

	digest, err := manifest.Digest(rawManifest)
	if err != nil {
		logrus.Errorf("Error referencing image %s from registry: %s", source, err)
		return "", errors.New("Could not obtain image digest")
	}

	return digest.String(), nil
}

func getConfigFromReference(ref types.ImageReference, source string) (ConfigSchema, error) {
	img, err := ref.NewImage(nil)
	if err != nil {
		logrus.Errorf("Error referencing image %s from registry: %s", source, err)
		return ConfigSchema{}, errors.New("Could not obtain image config")
	}
	defer img.Close()

	configBlob, err := img.ConfigBlob()
	if err != nil {
		logrus.Errorf("Error obtaining config blob for image %s from registry: %s", source, err)
		return ConfigSchema{}, errors.New("Could not obtain image config")
	}

	var config ConfigSchema
	err = json.Unmarshal(configBlob, &config)
	if err != nil {
		logrus.Errorf("Error with config file struct for image %s: %s", source, err)
		return ConfigSchema{}, errors.New("Could not obtain image config")
	}
	return config, nil
}

func CleanupImage(image Image) {
	if image.FSPath != "" {
		logrus.Infof("Removing image filesystem directory %s from system", image.FSPath)
		if err := os.RemoveAll(image.FSPath); err != nil {
			logrus.Error(err.Error())
		}
	}
}
<|endoftext|>


<commit_before>package util

import (
	"fmt"

	"github.com/SeerUK/tid/pkg/errhandling"
	"github.com/SeerUK/tid/pkg/state"
)

// WorkspaceFacade provides a simpler interface for common general workspace-related tasks.
type WorkspaceFacade struct {
	// backend is a lower-level backend storage interface.
	backend state.Backend
	// sysGateway is a SysGateway used for accessing system storage.
	sysGateway state.SysGateway
}

// NewWorkspaceFacade creates a new WorkspaceFacade instance.
func NewWorkspaceFacade(backend state.Backend, sysGateway state.SysGateway) *WorkspaceFacade {
	return &WorkspaceFacade{
		backend:    backend,
		sysGateway: sysGateway,
	}
}

// Create attempts to create a new workspace.
func (f *WorkspaceFacade) Create(name string) error {
	index, err := f.sysGateway.FindWorkspaceIndex()
	if err != nil {
		return err
	}

	for _, workspace := range index.Workspaces {
		if workspace == name {
			return fmt.Errorf("util: Workspace '%s' already exists", workspace)
		}
	}

	bucketName := fmt.Sprintf(
		state.BackendBucketWorkspaceFmt,
		name,
	)

	err = f.backend.CreateBucketIfNotExists(bucketName)
	if err != nil {
		return err
	}

	index.Workspaces = append(index.Workspaces, name)

	return f.sysGateway.PersistWorkspaceIndex(index)
}

// Delete attempts to delete a workspace.
func (f *WorkspaceFacade) Delete(name string) error {
	index, err := f.sysGateway.FindWorkspaceIndex()
	if err != nil {
		return err
	}

	exists := false

	// Remove the workspace from the index.
	for i, ws := range index.Workspaces {
		if ws == name {
			index.Workspaces = append(index.Workspaces[:i], index.Workspaces[i+1:]...)
			exists = true
			break
		}
	}

	if !exists {
		return fmt.Errorf("util: Workspace '%s' does not exist", name)
	}

	err = f.sysGateway.PersistWorkspaceIndex(index)
	if err != nil {
		return err
	}

	return f.backend.DeleteBucket(fmt.Sprintf(
		state.BackendBucketWorkspaceFmt,
		name,
	))
}

// Switch attempts to switch to another workspace.
func (f *WorkspaceFacade) Switch(name string) error {
	index, err1 := f.sysGateway.FindWorkspaceIndex()
	status, err2 := f.sysGateway.FindOrCreateStatus()

	errs := errhandling.NewErrorStack()
	errs.Add(err1)
	errs.Add(err2)

	if !errs.Empty() {
		return errs.Errors()
	}

	exists := false

	for _, workspace := range index.Workspaces {
		if workspace == name {
			exists = true
			break
		}
	}

	if !exists {
		return fmt.Errorf("util: Workspace '%s' does not exist", name)
	}

	status.Workspace = name

	return f.sysGateway.PersistStatus(status)
}
<commit_msg>Switching workspace also clears the status to avoid errors.<commit_after>package util

import (
	"fmt"

	"github.com/SeerUK/tid/pkg/errhandling"
	"github.com/SeerUK/tid/pkg/state"
)

// WorkspaceFacade provides a simpler interface for common general workspace-related tasks.
type WorkspaceFacade struct {
	// backend is a lower-level backend storage interface.
	backend state.Backend
	// sysGateway is a SysGateway used for accessing system storage.
	sysGateway state.SysGateway
}

// NewWorkspaceFacade creates a new WorkspaceFacade instance.
func NewWorkspaceFacade(backend state.Backend, sysGateway state.SysGateway) *WorkspaceFacade {
	return &WorkspaceFacade{
		backend:    backend,
		sysGateway: sysGateway,
	}
}

// Create attempts to create a new workspace.
func (f *WorkspaceFacade) Create(name string) error {
	index, err := f.sysGateway.FindWorkspaceIndex()
	if err != nil {
		return err
	}

	for _, workspace := range index.Workspaces {
		if workspace == name {
			return fmt.Errorf("util: Workspace '%s' already exists", workspace)
		}
	}

	bucketName := fmt.Sprintf(
		state.BackendBucketWorkspaceFmt,
		name,
	)

	err = f.backend.CreateBucketIfNotExists(bucketName)
	if err != nil {
		return err
	}

	index.Workspaces = append(index.Workspaces, name)

	return f.sysGateway.PersistWorkspaceIndex(index)
}

// Delete attempts to delete a workspace.
func (f *WorkspaceFacade) Delete(name string) error {
	index, err := f.sysGateway.FindWorkspaceIndex()
	if err != nil {
		return err
	}

	exists := false

	// Remove the workspace from the index.
	for i, ws := range index.Workspaces {
		if ws == name {
			index.Workspaces = append(index.Workspaces[:i], index.Workspaces[i+1:]...)
			exists = true
			break
		}
	}

	if !exists {
		return fmt.Errorf("util: Workspace '%s' does not exist", name)
	}

	err = f.sysGateway.PersistWorkspaceIndex(index)
	if err != nil {
		return err
	}

	return f.backend.DeleteBucket(fmt.Sprintf(
		state.BackendBucketWorkspaceFmt,
		name,
	))
}

// Switch attempts to switch to another workspace.
func (f *WorkspaceFacade) Switch(name string) error {
	index, err1 := f.sysGateway.FindWorkspaceIndex()
	status, err2 := f.sysGateway.FindOrCreateStatus()

	errs := errhandling.NewErrorStack()
	errs.Add(err1)
	errs.Add(err2)

	if !errs.Empty() {
		return errs.Errors()
	}

	status, err := f.sysGateway.FindOrCreateStatus()
	if err != nil {
		return err
	}

	status.StopAndClear()

	exists := false

	for _, workspace := range index.Workspaces {
		if workspace == name {
			exists = true
			break
		}
	}

	if !exists {
		return fmt.Errorf("util: Workspace '%s' does not exist", name)
	}

	status.Workspace = name

	return f.sysGateway.PersistStatus(status)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"image"
	"image/color"
	"math"
	"math/rand"
	"time"

	"github.com/ambientsound/wirelight/blinken/lib"
	colorful "github.com/lucasb-eyer/go-colorful"
)

func fill(canvas *image.RGBA, col color.Color) {
	b := canvas.Bounds()
	for x := b.Min.X; x < b.Max.X; x++ {
		for y := b.Min.Y; y < b.Max.Y; y++ {
			canvas.Set(x, y, col)
		}
	}
}

func northernLights(canvas *image.RGBA) {
	b := canvas.Bounds()
	old := make([]colorful.Color, b.Max.X*b.Max.Y)
	for {
		for angle := 0.0; angle < 360.0; angle++ {
			for x := b.Min.X; x < b.Max.X; x++ {
				for y := b.Min.Y; y < b.Max.Y; y++ {
					i := (y * b.Max.X) + x
					col := colorful.Hsl(angle+rand.Float64()*50.0, 1, rand.Float64()*0.6)
					step := col.BlendHcl(old[i], 0.92).Clamped()
					canvas.Set(x, y, step)
					old[i] = step
				}
			}
			time.Sleep(time.Millisecond * 100)
		}
	}
}

func black(canvas *image.RGBA) {
	for {
		fill(canvas, colorful.Hsv(0, 0, 0))
	}
}

func white(canvas *image.RGBA) {
	for {
		hue := rand.Float64() * 360.0
		for deg := 0.0; deg <= 180.0; deg += 1 {
			l := math.Sin(lib.Rad(deg))
			col := colorful.Hsv(hue, 1.0, l*0.5).Clamped()
			fill(canvas, col)
			time.Sleep(time.Microsecond * 1500)
		}
		time.Sleep(time.Millisecond * 185)
	}
}

func gradients(canvas *image.RGBA) {
	var h, c, l float64
	h = 0.0
	c = 0.8
	l = 0.5
	_, _ = c, l
	src := colorful.Hsv(h, 1, 1)
	dst := colorful.Hsv(h, 1, 1)

	for {
		src = dst
		h += 30
		if h >= 360 {
			h = 0
		}
		dst = colorful.Hsv(h, 1, 1)
		fmt.Printf("hue=%.2f, blend %#v %#v\n", h, src, dst)

		// interpolate between the two colors.
		for n := 0.0; n < 1.0; n += 0.01 {
			col := src.BlendHcl(dst, n).Clamped()
			fill(canvas, col)
			time.Sleep(time.Millisecond * 20)
		}
	}
}

func wheelHCL(canvas *image.RGBA) {
	var h float64
	for {
		h += 1
		if h > 360 {
			h = 0
		}
		col := colorful.Hcl(h, 0.2, 0).Clamped()
		fill(canvas, col)
		time.Sleep(time.Millisecond * 10)
	}
}

func wheelHSV(canvas *image.RGBA) {
	var h float64
	for {
		h += 1
		if h > 360 {
			h = 0
		}
		col := colorful.Hsv(h, 1, 1)
		fill(canvas, col)
		time.Sleep(time.Millisecond * 10)
	}
}
<commit_msg>blinken: two new effects, mostly for testing.<commit_after>package main

import (
	"fmt"
	"image"
	"image/color"
	"math"
	"math/rand"
	"time"

	"github.com/ambientsound/wirelight/blinken/lib"
	colorful "github.com/lucasb-eyer/go-colorful"
)

func fill(canvas *image.RGBA, col color.Color) {
	b := canvas.Bounds()
	for x := b.Min.X; x < b.Max.X; x++ {
		for y := b.Min.Y; y < b.Max.Y; y++ {
			canvas.Set(x, y, col)
		}
	}
}

func northernLights(canvas *image.RGBA) {
	b := canvas.Bounds()
	old := make([]colorful.Color, b.Max.X*b.Max.Y)
	for {
		for angle := 0.0; angle < 360.0; angle++ {
			for x := b.Min.X; x < b.Max.X; x++ {
				for y := b.Min.Y; y < b.Max.Y; y++ {
					i := (y * b.Max.X) + x
					col := colorful.Hsl(angle+rand.Float64()*50.0, 1, rand.Float64()*0.1)
					step := col.BlendHcl(old[i], 0.92).Clamped()
					canvas.Set(x, y, step)
					old[i] = step
				}
			}
			time.Sleep(time.Millisecond * 100)
		}
	}
}

func black(canvas *image.RGBA) {
	for {
		fill(canvas, colorful.Hsv(0, 0, 0))
	}
}

func white(canvas *image.RGBA) {
	for {
		hue := rand.Float64() * 360.0
		for deg := 0.0; deg <= 180.0; deg += 1 {
			l := math.Sin(lib.Rad(deg))
			col := colorful.Hsv(hue, 1.0, l*0.5).Clamped()
			fill(canvas, col)
			time.Sleep(time.Microsecond * 1500)
		}
		time.Sleep(time.Millisecond * 185)
	}
}

// directionTest draws up a gradient on each strip.
func directionTest(canvas *image.RGBA) {
	c := 1.0
	l := 0.05

	src := colorful.Hcl(0.0, c, l)
	dst := colorful.Hcl(160.0, c, l)
	b := canvas.Bounds()
	count := b.Max.X - b.Min.X
	step := float64(1.0) / float64(count)

	for {
		for y := b.Min.Y; y < b.Max.Y; y++ {
			n := 0.0
			for x := b.Min.X; x < b.Max.X; x++ {
				n += step
				col := src.BlendHcl(dst, n).Clamped()
				canvas.Set(x, y, col)
			}
		}
		time.Sleep(time.Millisecond * 1000)
	}
}

func gradients(canvas *image.RGBA) {
	var h, c, l float64
	h = 0.0
	c = 0.8
	l = 0.5
	_, _ = c, l
	src := colorful.Hsv(h, 1, 1)
	dst := colorful.Hsv(h, 1, 1)

	for {
		src = dst
		h += 30
		if h >= 360 {
			h = 0
		}
		dst = colorful.Hsv(h, 1, 1)
		fmt.Printf("hue=%.2f, blend %#v %#v\n", h, src, dst)

		// interpolate between the two colors.
		for n := 0.0; n < 1.0; n += 0.01 {
			col := src.BlendHcl(dst, n).Clamped()
			fill(canvas, col)
			time.Sleep(time.Millisecond * 20)
		}
	}
}

func staccatoWheel(canvas *image.RGBA) {
	var h float64
	for {
		h += 31
		if h > 360 {
			h -= 360
		}
		col := colorful.Hsv(h, 1, 0.25).Clamped()
		fill(canvas, col)
		time.Sleep(time.Millisecond * 250)
	}
}

func wheelHCL(canvas *image.RGBA) {
	var h float64
	for {
		h += 1
		if h > 360 {
			h = 0
		}
		col := colorful.Hcl(h, 0.2, 0).Clamped()
		fill(canvas, col)
		time.Sleep(time.Millisecond * 10)
	}
}

func wheelHSV(canvas *image.RGBA) {
	var h float64
	for {
		h += 1
		if h > 360 {
			h = 0
		}
		col := colorful.Hsv(h, 1, 1)
		fill(canvas, col)
		time.Sleep(time.Millisecond * 10)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package etcd

import (
	"k8s.io/kubernetes/federation/apis/federation"
	"k8s.io/kubernetes/federation/registry/cluster"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/rest"
	"k8s.io/kubernetes/pkg/registry/generic"
	"k8s.io/kubernetes/pkg/registry/generic/registry"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/storage"
)

type REST struct {
	*registry.Store
}

type StatusREST struct {
	store *registry.Store
}

func (r *StatusREST) New() runtime.Object {
	return &federation.Cluster{}
}

// Update alters the status subset of an object.
func (r *StatusREST) Update(ctx api.Context, name string, objInfo rest.UpdatedObjectInfo) (runtime.Object, bool, error) {
	return r.store.Update(ctx, name, objInfo)
}

// NewREST returns a RESTStorage object that will work against clusters.
func NewREST(opts generic.RESTOptions) (*REST, *StatusREST) {
	prefix := "/" + opts.ResourcePrefix

	newListFunc := func() runtime.Object { return &federation.ClusterList{} }
	storageInterface, _ := opts.Decorator(
		opts.StorageConfig,
		100,
		&federation.Cluster{},
		prefix,
		cluster.Strategy,
		newListFunc,
		storage.NoTriggerPublisher,
	)

	store := &registry.Store{
		NewFunc:     func() runtime.Object { return &federation.Cluster{} },
		NewListFunc: newListFunc,
		KeyRootFunc: func(ctx api.Context) string {
			return prefix
		},
		KeyFunc: func(ctx api.Context, name string) (string, error) {
			return registry.NoNamespaceKeyFunc(ctx, prefix, name)
		},
		ObjectNameFunc: func(obj runtime.Object) (string, error) {
			return obj.(*federation.Cluster).Name, nil
		},
		PredicateFunc:           cluster.MatchCluster,
		QualifiedResource:       federation.Resource("clusters"),
		DeleteCollectionWorkers: opts.DeleteCollectionWorkers,

		CreateStrategy: cluster.Strategy,
		UpdateStrategy: cluster.Strategy,
		DeleteStrategy: cluster.Strategy,

		ReturnDeletedObject: true,

		Storage: storageInterface,
	}

	statusStore := *store
	statusStore.UpdateStrategy = cluster.StatusStrategy

	return &REST{store}, &StatusREST{store: &statusStore}
}
<commit_msg>All REST that set DeleteCollectionWorkers should set EnableGC<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package etcd

import (
	"k8s.io/kubernetes/federation/apis/federation"
	"k8s.io/kubernetes/federation/registry/cluster"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/rest"
	"k8s.io/kubernetes/pkg/registry/generic"
	"k8s.io/kubernetes/pkg/registry/generic/registry"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/storage"
)

type REST struct {
	*registry.Store
}

type StatusREST struct {
	store *registry.Store
}

func (r *StatusREST) New() runtime.Object {
	return &federation.Cluster{}
}

// Update alters the status subset of an object.
func (r *StatusREST) Update(ctx api.Context, name string, objInfo rest.UpdatedObjectInfo) (runtime.Object, bool, error) {
	return r.store.Update(ctx, name, objInfo)
}

// NewREST returns a RESTStorage object that will work against clusters.
func NewREST(opts generic.RESTOptions) (*REST, *StatusREST) {
	prefix := "/" + opts.ResourcePrefix

	newListFunc := func() runtime.Object { return &federation.ClusterList{} }
	storageInterface, _ := opts.Decorator(
		opts.StorageConfig,
		100,
		&federation.Cluster{},
		prefix,
		cluster.Strategy,
		newListFunc,
		storage.NoTriggerPublisher,
	)

	store := &registry.Store{
		NewFunc:     func() runtime.Object { return &federation.Cluster{} },
		NewListFunc: newListFunc,
		KeyRootFunc: func(ctx api.Context) string {
			return prefix
		},
		KeyFunc: func(ctx api.Context, name string) (string, error) {
			return registry.NoNamespaceKeyFunc(ctx, prefix, name)
		},
		ObjectNameFunc: func(obj runtime.Object) (string, error) {
			return obj.(*federation.Cluster).Name, nil
		},
		PredicateFunc:           cluster.MatchCluster,
		QualifiedResource:       federation.Resource("clusters"),
		EnableGarbageCollection: opts.EnableGarbageCollection,
		DeleteCollectionWorkers: opts.DeleteCollectionWorkers,

		CreateStrategy: cluster.Strategy,
		UpdateStrategy: cluster.Strategy,
		DeleteStrategy: cluster.Strategy,

		ReturnDeletedObject: true,

		Storage: storageInterface,
	}

	statusStore := *store
	statusStore.UpdateStrategy = cluster.StatusStrategy

	return &REST{store}, &StatusREST{store: &statusStore}
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package blob

import (
	"encoding/hex"
	"fmt"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"testing"
)

func TestRegister(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func fromHex(h string) []byte {
	b, err := hex.DecodeString(h)
	if err != nil {
		panic(fmt.Sprintf("Invalid hex string: %s", h))
	}

	return b
}

type ScoreTest struct{}

func init() { RegisterTestSuite(&ScoreTest{}) }

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *ScoreTest) EmptySlice() {
	data := []byte{}
	golden := "da39a3ee5e6b4b0d3255bfef95601890afd80709"

	score := ComputeScore(data)
	AssertEq(20, len(score))

	ExpectEq(golden, score.Hex())
	ExpectThat(score, DeepEquals(fromHex(golden)))
}

func (t *ScoreTest) HashStartsWithZero() {
	data := []byte("hello_5")
	golden := "086766b9ba6a30e3792c05b00c5fb0e85a18a040"

	score := ComputeScore(data)
	AssertEq(20, len(score))

	ExpectEq(golden, score.Hex())
	ExpectThat(score, DeepEquals(fromHex(golden)))
}

func (t *ScoreTest) HexHashStartsWithNonZeroNumber() {
	data := []byte("hello_0")
	golden := "3966a6c98206d4cda8fd000656ed4f279a35726b"

	score := ComputeScore(data)
	AssertEq(20, len(score))

	ExpectEq(golden, score.Hex())
	ExpectThat(score, DeepEquals(fromHex(golden)))
}

func (t *ScoreTest) HexHashStartsWithLetter() {
	data := []byte("foo_barbazqux")
	golden := "ccf73cc0bfe964b652934764f847699e4005205e"

	score := ComputeScore(data)
	AssertEq(20, len(score))

	ExpectEq(golden, score.Hex())
	ExpectThat(score, DeepEquals(fromHex(golden)))
}

func (t *ScoreTest) DataContainsNonUtf8() {
	data := []byte{0x4a, 0x80, 0x81, 0x82, 0x4b}
	golden := "2feba26855d9f4e8b76d36c34dc385c8afe622c8"

	score := ComputeScore(data)
	AssertEq(20, len(score))

	ExpectEq(golden, score.Hex())
	ExpectThat(score, DeepEquals(fromHex(golden)))
}
<commit_msg>Fixed some test bugs.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package blob

import (
	"encoding/hex"
	"fmt"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"testing"
)

func TestRegister(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func fromHex(h string) Score {
	b, err := hex.DecodeString(h)
	if err != nil {
		panic(fmt.Sprintf("Invalid hex string: %s", h))
	}

	return Score(b)
}

type ScoreTest struct{}

func init() { RegisterTestSuite(&ScoreTest{}) }

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *ScoreTest) EmptySlice() {
	data := []byte{}
	golden := "da39a3ee5e6b4b0d3255bfef95601890afd80709"

	score := ComputeScore(data)
	AssertEq(20, len(score))

	ExpectEq(golden, score.Hex())
	ExpectThat(score, DeepEquals(fromHex(golden)))
}

func (t *ScoreTest) HashStartsWithZero() {
	data := []byte("hello_5")
	golden := "086766b9ba6a30e3792c05b00c5fb0e85a18a040"

	score := ComputeScore(data)
	AssertEq(20, len(score))

	ExpectEq(golden, score.Hex())
	ExpectThat(score, DeepEquals(fromHex(golden)))
}

func (t *ScoreTest) HexHashStartsWithNonZeroNumber() {
	data := []byte("hello_0")
	golden := "3966a6c98206d4cda8fd000656ed4f279a35726b"

	score := ComputeScore(data)
	AssertEq(20, len(score))

	ExpectEq(golden, score.Hex())
	ExpectThat(score, DeepEquals(fromHex(golden)))
}

func (t *ScoreTest) HexHashStartsWithLetter() {
	data := []byte("foo_barbazqux")
	golden := "ccf73cc0bfe964b652934764f847699e4005205e"

	score := ComputeScore(data)
	AssertEq(20, len(score))

	ExpectEq(golden, score.Hex())
	ExpectThat(score, DeepEquals(fromHex(golden)))
}

func (t *ScoreTest) DataContainsNonUtf8() {
	data := []byte{0x4a, 0x80, 0x81, 0x82, 0x4b}
	golden := "2feba26855d9f4e8b76d36c34dc385c8afe622c8"

	score := ComputeScore(data)
	AssertEq(20, len(score))

	ExpectEq(golden, score.Hex())
	ExpectThat(score, DeepEquals(fromHex(golden)))
}
<|endoftext|>


<commit_before>// +build !containers_image_ostree_stub

package ostree

import (
	"bytes"
	"compress/gzip"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"time"

	"github.com/containers/image/manifest"
	"github.com/containers/image/types"
	"github.com/containers/storage/pkg/archive"
	"github.com/opencontainers/go-digest"
	"github.com/ostreedev/ostree-go/pkg/otbuiltin"
	"github.com/pkg/errors"
	"github.com/vbatts/tar-split/tar/asm"
	"github.com/vbatts/tar-split/tar/storage"
)

// #cgo pkg-config: glib-2.0 gobject-2.0 ostree-1
// #include <glib.h>
// #include <glib-object.h>
// #include <gio/gio.h>
// #include <stdlib.h>
// #include <ostree.h>
// #include <gio/ginputstream.h>
import "C"

type blobToImport struct {
	Size     int64
	Digest   digest.Digest
	BlobPath string
}

type descriptor struct {
	Size   int64         `json:"size"`
	Digest digest.Digest `json:"digest"`
}

type fsLayersSchema1 struct {
	BlobSum digest.Digest `json:"blobSum"`
}

type manifestSchema struct {
	LayersDescriptors []descriptor      `json:"layers"`
	FSLayers          []fsLayersSchema1 `json:"fsLayers"`
}

type ostreeImageDestination struct {
	ref           ostreeReference
	manifest      string
	schema        manifestSchema
	tmpDirPath    string
	blobs         map[string]*blobToImport
	digest        digest.Digest
	signaturesLen int
	repo          *C.struct_OstreeRepo
}

// newImageDestination returns an ImageDestination for writing to an existing ostree.
func newImageDestination(ref ostreeReference, tmpDirPath string) (types.ImageDestination, error) {
	tmpDirPath = filepath.Join(tmpDirPath, ref.branchName)
	if err := ensureDirectoryExists(tmpDirPath); err != nil {
		return nil, err
	}
	return &ostreeImageDestination{ref, "", manifestSchema{}, tmpDirPath, map[string]*blobToImport{}, "", 0, nil}, nil
}

// Reference returns the reference used to set up this destination.  Note that this should directly correspond to user's intent,
// e.g. it should use the public hostname instead of the result of resolving CNAMEs or following redirects.
func (d *ostreeImageDestination) Reference() types.ImageReference {
	return d.ref
}

// Close removes resources associated with an initialized ImageDestination, if any.
func (d *ostreeImageDestination) Close() error {
	if d.repo != nil {
		C.g_object_unref(C.gpointer(d.repo))
	}
	return os.RemoveAll(d.tmpDirPath)
}

func (d *ostreeImageDestination) SupportedManifestMIMETypes() []string {
	return []string{
		manifest.DockerV2Schema2MediaType,
	}
}

// SupportsSignatures returns an error (to be displayed to the user) if the destination certainly can't store signatures.
// Note: It is still possible for PutSignatures to fail if SupportsSignatures returns nil.
func (d *ostreeImageDestination) SupportsSignatures() error {
	return nil
}

// ShouldCompressLayers returns true iff it is desirable to compress layer blobs written to this destination.
func (d *ostreeImageDestination) ShouldCompressLayers() bool {
	return false
}

// AcceptsForeignLayerURLs returns false iff foreign layers in manifest should be actually
// uploaded to the image destination, true otherwise.
func (d *ostreeImageDestination) AcceptsForeignLayerURLs() bool {
	return false
}

// MustMatchRuntimeOS returns true iff the destination can store only images targeted for the current runtime OS. False otherwise.
func (d *ostreeImageDestination) MustMatchRuntimeOS() bool {
	return true
}

func (d *ostreeImageDestination) PutBlob(stream io.Reader, inputInfo types.BlobInfo) (types.BlobInfo, error) {
	tmpDir, err := ioutil.TempDir(d.tmpDirPath, "blob")
	if err != nil {
		return types.BlobInfo{}, err
	}

	blobPath := filepath.Join(tmpDir, "content")
	blobFile, err := os.Create(blobPath)
	if err != nil {
		return types.BlobInfo{}, err
	}
	defer blobFile.Close()

	digester := digest.Canonical.Digester()
	tee := io.TeeReader(stream, digester.Hash())

	size, err := io.Copy(blobFile, tee)
	if err != nil {
		return types.BlobInfo{}, err
	}
	computedDigest := digester.Digest()
	if inputInfo.Size != -1 && size != inputInfo.Size {
		return types.BlobInfo{}, errors.Errorf("Size mismatch when copying %s, expected %d, got %d", computedDigest, inputInfo.Size, size)
	}
	if err := blobFile.Sync(); err != nil {
		return types.BlobInfo{}, err
	}

	hash := computedDigest.Hex()
	d.blobs[hash] = &blobToImport{Size: size, Digest: computedDigest, BlobPath: blobPath}
	return types.BlobInfo{Digest: computedDigest, Size: size}, nil
}

func fixFiles(dir string, usermode bool) error {
	entries, err := ioutil.ReadDir(dir)
	if err != nil {
		return err
	}

	for _, info := range entries {
		fullpath := filepath.Join(dir, info.Name())
		if info.Mode()&(os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 {
			if err := os.Remove(fullpath); err != nil {
				return err
			}
			continue
		}
		if info.IsDir() {
			if usermode {
				if err := os.Chmod(fullpath, info.Mode()|0700); err != nil {
					return err
				}
			}
			err = fixFiles(fullpath, usermode)
			if err != nil {
				return err
			}
		} else if usermode && (info.Mode().IsRegular()) {
			if err := os.Chmod(fullpath, info.Mode()|0600); err != nil {
				return err
			}
		}
	}

	return nil
}

func (d *ostreeImageDestination) ostreeCommit(repo *otbuiltin.Repo, branch string, root string, metadata []string) error {
	opts := otbuiltin.NewCommitOptions()
	opts.AddMetadataString = metadata
	opts.Timestamp = time.Now()
	// OCI layers have no parent OSTree commit
	opts.Parent = "0000000000000000000000000000000000000000000000000000000000000000"
	_, err := repo.Commit(root, branch, opts)
	return err
}

func generateTarSplitMetadata(output *bytes.Buffer, file string) error {
	mfz := gzip.NewWriter(output)
	defer mfz.Close()
	metaPacker := storage.NewJSONPacker(mfz)

	stream, err := os.OpenFile(file, os.O_RDONLY, 0)
	if err != nil {
		return err
	}
	defer stream.Close()

	gzReader, err := gzip.NewReader(stream)
	if err != nil {
		return err
	}
	defer gzReader.Close()

	its, err := asm.NewInputTarStream(gzReader, metaPacker, nil)
	if err != nil {
		return err
	}

	_, err = io.Copy(ioutil.Discard, its)
	if err != nil {
		return err
	}
	return nil
}

func (d *ostreeImageDestination) importBlob(repo *otbuiltin.Repo, blob *blobToImport) error {
	ostreeBranch := fmt.Sprintf("ociimage/%s", blob.Digest.Hex())
	destinationPath := filepath.Join(d.tmpDirPath, blob.Digest.Hex(), "root")
	if err := ensureDirectoryExists(destinationPath); err != nil {
		return err
	}
	defer func() {
		os.Remove(blob.BlobPath)
		os.RemoveAll(destinationPath)
	}()

	var tarSplitOutput bytes.Buffer
	if err := generateTarSplitMetadata(&tarSplitOutput, blob.BlobPath); err != nil {
		return err
	}

	if os.Getuid() == 0 {
		if err := archive.UntarPath(blob.BlobPath, destinationPath); err != nil {
			return err
		}
		if err := fixFiles(destinationPath, false); err != nil {
			return err
		}
	} else {
		os.MkdirAll(destinationPath, 0755)
		if err := exec.Command("tar", "-C", destinationPath, "--no-same-owner", "--no-same-permissions", "--delay-directory-restore", "-xf", blob.BlobPath).Run(); err != nil {
			return err
		}

		if err := fixFiles(destinationPath, true); err != nil {
			return err
		}
	}
	return d.ostreeCommit(repo, ostreeBranch, destinationPath, []string{fmt.Sprintf("docker.size=%d", blob.Size),
		fmt.Sprintf("tarsplit.output=%s", base64.StdEncoding.EncodeToString(tarSplitOutput.Bytes()))})

}

func (d *ostreeImageDestination) importConfig(repo *otbuiltin.Repo, blob *blobToImport) error {
	ostreeBranch := fmt.Sprintf("ociimage/%s", blob.Digest.Hex())
	destinationPath := filepath.Dir(blob.BlobPath)

	return d.ostreeCommit(repo, ostreeBranch, destinationPath, []string{fmt.Sprintf("docker.size=%d", blob.Size)})
}

func (d *ostreeImageDestination) HasBlob(info types.BlobInfo) (bool, int64, error) {

	if d.repo == nil {
		repo, err := openRepo(d.ref.repo)
		if err != nil {
			return false, 0, err
		}
		d.repo = repo
	}
	branch := fmt.Sprintf("ociimage/%s", info.Digest.Hex())

	found, data, err := readMetadata(d.repo, branch, "docker.size")
	if err != nil || !found {
		return found, -1, err
	}

	size, err := strconv.ParseInt(data, 10, 64)
	if err != nil {
		return false, -1, err
	}

	return true, size, nil
}

func (d *ostreeImageDestination) ReapplyBlob(info types.BlobInfo) (types.BlobInfo, error) {
	return info, nil
}

// PutManifest writes manifest to the destination.
// FIXME? This should also receive a MIME type if known, to differentiate between schema versions.
// If the destination is in principle available, refuses this manifest type (e.g. it does not recognize the schema),
// but may accept a different manifest type, the returned error must be an ManifestTypeRejectedError.
func (d *ostreeImageDestination) PutManifest(manifestBlob []byte) error {
	d.manifest = string(manifestBlob)

	if err := json.Unmarshal(manifestBlob, &d.schema); err != nil {
		return err
	}

	manifestPath := filepath.Join(d.tmpDirPath, d.ref.manifestPath())
	if err := ensureParentDirectoryExists(manifestPath); err != nil {
		return err
	}

	digest, err := manifest.Digest(manifestBlob)
	if err != nil {
		return err
	}
	d.digest = digest

	return ioutil.WriteFile(manifestPath, manifestBlob, 0644)
}

func (d *ostreeImageDestination) PutSignatures(signatures [][]byte) error {
	path := filepath.Join(d.tmpDirPath, d.ref.signaturePath(0))
	if err := ensureParentDirectoryExists(path); err != nil {
		return err
	}

	for i, sig := range signatures {
		signaturePath := filepath.Join(d.tmpDirPath, d.ref.signaturePath(i))
		if err := ioutil.WriteFile(signaturePath, sig, 0644); err != nil {
			return err
		}
	}
	d.signaturesLen = len(signatures)
	return nil
}

func (d *ostreeImageDestination) Commit() error {
	repo, err := otbuiltin.OpenRepo(d.ref.repo)
	if err != nil {
		return err
	}

	_, err = repo.PrepareTransaction()
	if err != nil {
		return err
	}

	checkLayer := func(hash string) error {
		blob := d.blobs[hash]
		// if the blob is not present in d.blobs then it is already stored in OSTree,
		// and we don't need to import it.
		if blob == nil {
			return nil
		}
		err := d.importBlob(repo, blob)
		if err != nil {
			return err
		}

		delete(d.blobs, hash)
		return nil
	}
	for _, layer := range d.schema.LayersDescriptors {
		hash := layer.Digest.Hex()
		if err = checkLayer(hash); err != nil {
			return err
		}
	}
	for _, layer := range d.schema.FSLayers {
		hash := layer.BlobSum.Hex()
		if err = checkLayer(hash); err != nil {
			return err
		}
	}

	// Import the other blobs that are not layers
	for _, blob := range d.blobs {
		err := d.importConfig(repo, blob)
		if err != nil {
			return err
		}
	}

	manifestPath := filepath.Join(d.tmpDirPath, "manifest")

	metadata := []string{fmt.Sprintf("docker.manifest=%s", string(d.manifest)),
		fmt.Sprintf("signatures=%d", d.signaturesLen),
		fmt.Sprintf("docker.digest=%s", string(d.digest))}
	err = d.ostreeCommit(repo, fmt.Sprintf("ociimage/%s", d.ref.branchName), manifestPath, metadata)

	_, err = repo.CommitTransaction()
	return err
}

func ensureDirectoryExists(path string) error {
	if _, err := os.Stat(path); err != nil && os.IsNotExist(err) {
		if err := os.MkdirAll(path, 0755); err != nil {
			return err
		}
	}
	return nil
}

func ensureParentDirectoryExists(path string) error {
	return ensureDirectoryExists(filepath.Dir(path))
}
<commit_msg>ostree: add selinux label for each file<commit_after>// +build !containers_image_ostree_stub

package ostree

import (
	"bytes"
	"compress/gzip"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"syscall"
	"time"
	"unsafe"

	"github.com/containers/image/manifest"
	"github.com/containers/image/types"
	"github.com/containers/storage/pkg/archive"
	"github.com/opencontainers/go-digest"
	selinux "github.com/opencontainers/selinux/go-selinux"
	"github.com/ostreedev/ostree-go/pkg/otbuiltin"
	"github.com/pkg/errors"
	"github.com/vbatts/tar-split/tar/asm"
	"github.com/vbatts/tar-split/tar/storage"
)

// #cgo pkg-config: glib-2.0 gobject-2.0 ostree-1 libselinux
// #include <glib.h>
// #include <glib-object.h>
// #include <gio/gio.h>
// #include <stdlib.h>
// #include <ostree.h>
// #include <gio/ginputstream.h>
// #include <selinux/selinux.h>
// #include <selinux/label.h>
import "C"

type blobToImport struct {
	Size     int64
	Digest   digest.Digest
	BlobPath string
}

type descriptor struct {
	Size   int64         `json:"size"`
	Digest digest.Digest `json:"digest"`
}

type fsLayersSchema1 struct {
	BlobSum digest.Digest `json:"blobSum"`
}

type manifestSchema struct {
	LayersDescriptors []descriptor      `json:"layers"`
	FSLayers          []fsLayersSchema1 `json:"fsLayers"`
}

type ostreeImageDestination struct {
	ref           ostreeReference
	manifest      string
	schema        manifestSchema
	tmpDirPath    string
	blobs         map[string]*blobToImport
	digest        digest.Digest
	signaturesLen int
	repo          *C.struct_OstreeRepo
}

// newImageDestination returns an ImageDestination for writing to an existing ostree.
func newImageDestination(ref ostreeReference, tmpDirPath string) (types.ImageDestination, error) {
	tmpDirPath = filepath.Join(tmpDirPath, ref.branchName)
	if err := ensureDirectoryExists(tmpDirPath); err != nil {
		return nil, err
	}
	return &ostreeImageDestination{ref, "", manifestSchema{}, tmpDirPath, map[string]*blobToImport{}, "", 0, nil}, nil
}

// Reference returns the reference used to set up this destination.  Note that this should directly correspond to user's intent,
// e.g. it should use the public hostname instead of the result of resolving CNAMEs or following redirects.
func (d *ostreeImageDestination) Reference() types.ImageReference {
	return d.ref
}

// Close removes resources associated with an initialized ImageDestination, if any.
func (d *ostreeImageDestination) Close() error {
	if d.repo != nil {
		C.g_object_unref(C.gpointer(d.repo))
	}
	return os.RemoveAll(d.tmpDirPath)
}

func (d *ostreeImageDestination) SupportedManifestMIMETypes() []string {
	return []string{
		manifest.DockerV2Schema2MediaType,
	}
}

// SupportsSignatures returns an error (to be displayed to the user) if the destination certainly can't store signatures.
// Note: It is still possible for PutSignatures to fail if SupportsSignatures returns nil.
func (d *ostreeImageDestination) SupportsSignatures() error {
	return nil
}

// ShouldCompressLayers returns true iff it is desirable to compress layer blobs written to this destination.
func (d *ostreeImageDestination) ShouldCompressLayers() bool {
	return false
}

// AcceptsForeignLayerURLs returns false iff foreign layers in manifest should be actually
// uploaded to the image destination, true otherwise.
func (d *ostreeImageDestination) AcceptsForeignLayerURLs() bool {
	return false
}

// MustMatchRuntimeOS returns true iff the destination can store only images targeted for the current runtime OS. False otherwise.
func (d *ostreeImageDestination) MustMatchRuntimeOS() bool {
	return true
}

func (d *ostreeImageDestination) PutBlob(stream io.Reader, inputInfo types.BlobInfo) (types.BlobInfo, error) {
	tmpDir, err := ioutil.TempDir(d.tmpDirPath, "blob")
	if err != nil {
		return types.BlobInfo{}, err
	}

	blobPath := filepath.Join(tmpDir, "content")
	blobFile, err := os.Create(blobPath)
	if err != nil {
		return types.BlobInfo{}, err
	}
	defer blobFile.Close()

	digester := digest.Canonical.Digester()
	tee := io.TeeReader(stream, digester.Hash())

	size, err := io.Copy(blobFile, tee)
	if err != nil {
		return types.BlobInfo{}, err
	}
	computedDigest := digester.Digest()
	if inputInfo.Size != -1 && size != inputInfo.Size {
		return types.BlobInfo{}, errors.Errorf("Size mismatch when copying %s, expected %d, got %d", computedDigest, inputInfo.Size, size)
	}
	if err := blobFile.Sync(); err != nil {
		return types.BlobInfo{}, err
	}

	hash := computedDigest.Hex()
	d.blobs[hash] = &blobToImport{Size: size, Digest: computedDigest, BlobPath: blobPath}
	return types.BlobInfo{Digest: computedDigest, Size: size}, nil
}

func fixFiles(selinuxHnd *C.struct_selabel_handle, root string, dir string, usermode bool) error {
	entries, err := ioutil.ReadDir(dir)
	if err != nil {
		return err
	}

	for _, info := range entries {
		fullpath := filepath.Join(dir, info.Name())
		if info.Mode()&(os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 {
			if err := os.Remove(fullpath); err != nil {
				return err
			}
			continue
		}

		if selinuxHnd != nil {
			relPath, err := filepath.Rel(root, fullpath)
			if err != nil {
				return err
			}
			relPath = fmt.Sprintf("/%s", relPath)

			relPathC := C.CString(relPath)
			defer C.free(unsafe.Pointer(relPathC))
			var context *C.char

			res, err := C.selabel_lookup_raw(selinuxHnd, &context, relPathC, C.int(info.Mode()&os.ModePerm))
			if int(res) < 0 && err != syscall.ENOENT {
				return errors.Wrapf(err, "cannot selabel_lookup_raw %s", relPath)
			}
			if int(res) == 0 {
				defer C.freecon(context)
				fullpathC := C.CString(fullpath)
				defer C.free(unsafe.Pointer(fullpathC))
				res, err = C.lsetfilecon_raw(fullpathC, context)
				if int(res) < 0 {
					return errors.Wrapf(err, "cannot setfilecon_raw %s", fullpath)
				}
			}
		}

		if info.IsDir() {
			if usermode {
				if err := os.Chmod(fullpath, info.Mode()|0700); err != nil {
					return err
				}
			}
			err = fixFiles(selinuxHnd, root, fullpath, usermode)
			if err != nil {
				return err
			}
		} else if usermode && (info.Mode().IsRegular()) {
			if err := os.Chmod(fullpath, info.Mode()|0600); err != nil {
				return err
			}
		}
	}

	return nil
}

func (d *ostreeImageDestination) ostreeCommit(repo *otbuiltin.Repo, branch string, root string, metadata []string) error {
	opts := otbuiltin.NewCommitOptions()
	opts.AddMetadataString = metadata
	opts.Timestamp = time.Now()
	// OCI layers have no parent OSTree commit
	opts.Parent = "0000000000000000000000000000000000000000000000000000000000000000"
	_, err := repo.Commit(root, branch, opts)
	return err
}

func generateTarSplitMetadata(output *bytes.Buffer, file string) error {
	mfz := gzip.NewWriter(output)
	defer mfz.Close()
	metaPacker := storage.NewJSONPacker(mfz)

	stream, err := os.OpenFile(file, os.O_RDONLY, 0)
	if err != nil {
		return err
	}
	defer stream.Close()

	gzReader, err := gzip.NewReader(stream)
	if err != nil {
		return err
	}
	defer gzReader.Close()

	its, err := asm.NewInputTarStream(gzReader, metaPacker, nil)
	if err != nil {
		return err
	}

	_, err = io.Copy(ioutil.Discard, its)
	if err != nil {
		return err
	}
	return nil
}

func (d *ostreeImageDestination) importBlob(selinuxHnd *C.struct_selabel_handle, repo *otbuiltin.Repo, blob *blobToImport) error {
	ostreeBranch := fmt.Sprintf("ociimage/%s", blob.Digest.Hex())
	destinationPath := filepath.Join(d.tmpDirPath, blob.Digest.Hex(), "root")
	if err := ensureDirectoryExists(destinationPath); err != nil {
		return err
	}
	defer func() {
		os.Remove(blob.BlobPath)
		os.RemoveAll(destinationPath)
	}()

	var tarSplitOutput bytes.Buffer
	if err := generateTarSplitMetadata(&tarSplitOutput, blob.BlobPath); err != nil {
		return err
	}

	if os.Getuid() == 0 {
		if err := archive.UntarPath(blob.BlobPath, destinationPath); err != nil {
			return err
		}
		if err := fixFiles(selinuxHnd, destinationPath, destinationPath, false); err != nil {
			return err
		}
	} else {
		os.MkdirAll(destinationPath, 0755)
		if err := exec.Command("tar", "-C", destinationPath, "--no-same-owner", "--no-same-permissions", "--delay-directory-restore", "-xf", blob.BlobPath).Run(); err != nil {
			return err
		}

		if err := fixFiles(selinuxHnd, destinationPath, destinationPath, true); err != nil {
			return err
		}
	}
	return d.ostreeCommit(repo, ostreeBranch, destinationPath, []string{fmt.Sprintf("docker.size=%d", blob.Size),
		fmt.Sprintf("tarsplit.output=%s", base64.StdEncoding.EncodeToString(tarSplitOutput.Bytes()))})

}

func (d *ostreeImageDestination) importConfig(repo *otbuiltin.Repo, blob *blobToImport) error {
	ostreeBranch := fmt.Sprintf("ociimage/%s", blob.Digest.Hex())
	destinationPath := filepath.Dir(blob.BlobPath)

	return d.ostreeCommit(repo, ostreeBranch, destinationPath, []string{fmt.Sprintf("docker.size=%d", blob.Size)})
}

func (d *ostreeImageDestination) HasBlob(info types.BlobInfo) (bool, int64, error) {

	if d.repo == nil {
		repo, err := openRepo(d.ref.repo)
		if err != nil {
			return false, 0, err
		}
		d.repo = repo
	}
	branch := fmt.Sprintf("ociimage/%s", info.Digest.Hex())

	found, data, err := readMetadata(d.repo, branch, "docker.size")
	if err != nil || !found {
		return found, -1, err
	}

	size, err := strconv.ParseInt(data, 10, 64)
	if err != nil {
		return false, -1, err
	}

	return true, size, nil
}

func (d *ostreeImageDestination) ReapplyBlob(info types.BlobInfo) (types.BlobInfo, error) {
	return info, nil
}

// PutManifest writes manifest to the destination.
// FIXME? This should also receive a MIME type if known, to differentiate between schema versions.
// If the destination is in principle available, refuses this manifest type (e.g. it does not recognize the schema),
// but may accept a different manifest type, the returned error must be an ManifestTypeRejectedError.
func (d *ostreeImageDestination) PutManifest(manifestBlob []byte) error {
	d.manifest = string(manifestBlob)

	if err := json.Unmarshal(manifestBlob, &d.schema); err != nil {
		return err
	}

	manifestPath := filepath.Join(d.tmpDirPath, d.ref.manifestPath())
	if err := ensureParentDirectoryExists(manifestPath); err != nil {
		return err
	}

	digest, err := manifest.Digest(manifestBlob)
	if err != nil {
		return err
	}
	d.digest = digest

	return ioutil.WriteFile(manifestPath, manifestBlob, 0644)
}

func (d *ostreeImageDestination) PutSignatures(signatures [][]byte) error {
	path := filepath.Join(d.tmpDirPath, d.ref.signaturePath(0))
	if err := ensureParentDirectoryExists(path); err != nil {
		return err
	}

	for i, sig := range signatures {
		signaturePath := filepath.Join(d.tmpDirPath, d.ref.signaturePath(i))
		if err := ioutil.WriteFile(signaturePath, sig, 0644); err != nil {
			return err
		}
	}
	d.signaturesLen = len(signatures)
	return nil
}

func (d *ostreeImageDestination) Commit() error {
	repo, err := otbuiltin.OpenRepo(d.ref.repo)
	if err != nil {
		return err
	}

	_, err = repo.PrepareTransaction()
	if err != nil {
		return err
	}

	var selinuxHnd *C.struct_selabel_handle

	if os.Getuid() == 0 && selinux.GetEnabled() {
		selinuxHnd, err := C.selabel_open(C.SELABEL_CTX_FILE, nil, 0)
		if selinuxHnd == nil {
			return errors.Wrapf(err, "cannot open the SELinux DB")
		}

		defer C.selabel_close(selinuxHnd)
	}

	checkLayer := func(hash string) error {
		blob := d.blobs[hash]
		// if the blob is not present in d.blobs then it is already stored in OSTree,
		// and we don't need to import it.
		if blob == nil {
			return nil
		}
		err := d.importBlob(selinuxHnd, repo, blob)
		if err != nil {
			return err
		}

		delete(d.blobs, hash)
		return nil
	}
	for _, layer := range d.schema.LayersDescriptors {
		hash := layer.Digest.Hex()
		if err = checkLayer(hash); err != nil {
			return err
		}
	}
	for _, layer := range d.schema.FSLayers {
		hash := layer.BlobSum.Hex()
		if err = checkLayer(hash); err != nil {
			return err
		}
	}

	// Import the other blobs that are not layers
	for _, blob := range d.blobs {
		err := d.importConfig(repo, blob)
		if err != nil {
			return err
		}
	}

	manifestPath := filepath.Join(d.tmpDirPath, "manifest")

	metadata := []string{fmt.Sprintf("docker.manifest=%s", string(d.manifest)),
		fmt.Sprintf("signatures=%d", d.signaturesLen),
		fmt.Sprintf("docker.digest=%s", string(d.digest))}
	err = d.ostreeCommit(repo, fmt.Sprintf("ociimage/%s", d.ref.branchName), manifestPath, metadata)

	_, err = repo.CommitTransaction()
	return err
}

func ensureDirectoryExists(path string) error {
	if _, err := os.Stat(path); err != nil && os.IsNotExist(err) {
		if err := os.MkdirAll(path, 0755); err != nil {
			return err
		}
	}
	return nil
}

func ensureParentDirectoryExists(path string) error {
	return ensureDirectoryExists(filepath.Dir(path))
}
<|endoftext|>


<commit_before>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package ddl

import (
	"time"

	. "github.com/pingcap/check"
	"github.com/pingcap/tidb/meta"
	"github.com/pingcap/tidb/model"
	"github.com/pingcap/tidb/util/errors2"
	"github.com/pingcap/tidb/util/mock"
)

func (s *testDDLSuite) TestSchema(c *C) {
	store := createTestStore(c, "test_schema")
	defer store.Close()

	lease := 100 * time.Millisecond

	d1 := newDDL(store, nil, nil, lease)
	defer d1.close()

	ctx := mock.NewContext()

	schema := model.NewCIStr("test")
	schemaID, err := d1.meta.GenGlobalID()
	c.Assert(err, IsNil)

	job := &model.Job{
		SchemaID: schemaID,
		Type:     model.ActionCreateSchema,
		Args:     []interface{}{schema},
	}

	err = d1.startJob(ctx, job)
	c.Assert(err, IsNil)

	d1.meta.RunInNewTxn(false, func(t *meta.TMeta) error {
		var dbInfo *model.DBInfo
		dbInfo, err = t.GetDatabase(schemaID)
		c.Assert(err, IsNil)
		c.Assert(dbInfo.Name, DeepEquals, model.NewCIStr("test"))
		c.Assert(dbInfo.State, Equals, model.StatePublic)

		var historyJob *model.Job
		historyJob, err = t.GetHistoryDDLJob(job.ID)
		c.Assert(err, IsNil)
		c.Assert(historyJob.State, Equals, model.JobDone)
		c.Assert(historyJob.SchemaState, Equals, model.StatePublic)
		return nil
	})

	job = &model.Job{
		SchemaID: schemaID,
		Type:     model.ActionDropSchema,
	}

	err = d1.startJob(ctx, job)
	c.Assert(err, IsNil)

	d1.meta.RunInNewTxn(false, func(t *meta.TMeta) error {
		dbInfo, err := t.GetDatabase(schemaID)
		c.Assert(err, IsNil)
		c.Assert(dbInfo, IsNil)

		var historyJob *model.Job
		historyJob, err = t.GetHistoryDDLJob(job.ID)
		c.Assert(err, IsNil)
		c.Assert(historyJob.State, Equals, model.JobDone)
		c.Assert(historyJob.SchemaState, Equals, model.StateNone)
		return nil
	})

	job = &model.Job{
		SchemaID: schemaID,
		Type:     model.ActionDropSchema,
	}

	err = d1.startJob(ctx, job)
	c.Assert(errors2.ErrorEqual(err, ErrNotExists), IsTrue)
}

func (s *testDDLSuite) TestSchemaWaitJob(c *C) {
	store := createTestStore(c, "test_schema_wait")
	defer store.Close()

	ctx := mock.NewContext()

	lease := 50 * time.Millisecond

	d1 := newDDL(store, nil, nil, lease)
	defer d1.close()

	testCheckOwner(c, d1, true)

	d2 := newDDL(store, nil, nil, lease)
	defer d2.close()

	// d2 must not be owner.
	testCheckOwner(c, d2, false)

	schema := model.NewCIStr("test")
	schemaID, err := d2.meta.GenGlobalID()
	c.Assert(err, IsNil)

	job := &model.Job{
		SchemaID: schemaID,
		Type:     model.ActionCreateSchema,
		Args:     []interface{}{schema},
	}

	err = d2.startJob(ctx, job)
	c.Assert(err, IsNil)

	d2.meta.RunInNewTxn(false, func(t *meta.TMeta) error {
		var dbInfo *model.DBInfo
		dbInfo, err = t.GetDatabase(schemaID)
		c.Assert(err, IsNil)
		c.Assert(dbInfo.Name, DeepEquals, model.NewCIStr("test"))
		c.Assert(dbInfo.State, Equals, model.StatePublic)
		return nil
	})

	// d2 must not be owner.
	testCheckOwner(c, d2, false)

	schemaID, err = d2.meta.GenGlobalID()
	c.Assert(err, IsNil)

	job = &model.Job{
		SchemaID: schemaID,
		Type:     model.ActionCreateSchema,
		Args:     []interface{}{schema},
	}

	err = d2.startJob(ctx, job)
	c.Assert(err, NotNil)

	d2.meta.RunInNewTxn(false, func(t *meta.TMeta) error {
		var historyJob *model.Job
		historyJob, err = t.GetHistoryDDLJob(job.ID)
		c.Assert(err, IsNil)
		c.Assert(historyJob.State, Equals, model.JobCancelled)
		return nil
	})

	// d2 must not be owner.
	testCheckOwner(c, d2, false)
}

func testRunInterruptedJob(c *C, d *ddl, job *model.Job) {
	ctx := mock.NewContext()
	done := make(chan error, 1)
	go func() {
		done <- d.startJob(ctx, job)
	}()

	ticker := time.NewTicker(d.lease * 1)
	defer ticker.Stop()

LOOP:
	for {
		select {
		case <-ticker.C:
			d.close()
			d.start()
			asyncNotify(d.jobCh)
		case err := <-done:
			c.Assert(err, IsNil)
			break LOOP
		}
	}
}

func (s *testDDLSuite) TestSchemaResume(c *C) {
	store := createTestStore(c, "test_schema_resume")
	defer store.Close()

	lease := 50 * time.Millisecond

	d1 := newDDL(store, nil, nil, lease)
	defer d1.close()

	testCheckOwner(c, d1, true)

	schema := model.NewCIStr("test")
	schemaID, err := d1.meta.GenGlobalID()
	c.Assert(err, IsNil)

	job := &model.Job{
		SchemaID: schemaID,
		Type:     model.ActionCreateSchema,
		Args:     []interface{}{schema},
	}

	testRunInterruptedJob(c, d1, job)

	d1.meta.RunInNewTxn(false, func(t *meta.TMeta) error {
		var dbInfo *model.DBInfo
		dbInfo, err = t.GetDatabase(schemaID)
		c.Assert(err, IsNil)
		c.Assert(dbInfo.Name, DeepEquals, model.NewCIStr("test"))
		c.Assert(dbInfo.State, Equals, model.StatePublic)
		return nil
	})

	job = &model.Job{
		SchemaID: schemaID,
		Type:     model.ActionDropSchema,
	}

	testRunInterruptedJob(c, d1, job)

	d1.meta.RunInNewTxn(false, func(t *meta.TMeta) error {
		var dbInfo *model.DBInfo
		dbInfo, err = t.GetDatabase(schemaID)
		c.Assert(err, IsNil)
		c.Assert(dbInfo, IsNil)
		return nil
	})
}
<commit_msg>ddl: fix make check<commit_after>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package ddl

import (
	"time"

	. "github.com/pingcap/check"
	"github.com/pingcap/tidb/meta"
	"github.com/pingcap/tidb/model"
	"github.com/pingcap/tidb/util/errors2"
	"github.com/pingcap/tidb/util/mock"
)

func (s *testDDLSuite) TestSchema(c *C) {
	store := createTestStore(c, "test_schema")
	defer store.Close()

	lease := 100 * time.Millisecond

	d1 := newDDL(store, nil, nil, lease)
	defer d1.close()

	ctx := mock.NewContext()

	schema := model.NewCIStr("test")
	schemaID, err := d1.meta.GenGlobalID()
	c.Assert(err, IsNil)

	job := &model.Job{
		SchemaID: schemaID,
		Type:     model.ActionCreateSchema,
		Args:     []interface{}{schema},
	}

	err = d1.startJob(ctx, job)
	c.Assert(err, IsNil)

	d1.meta.RunInNewTxn(false, func(t *meta.TMeta) error {
		var dbInfo *model.DBInfo
		dbInfo, err = t.GetDatabase(schemaID)
		c.Assert(err, IsNil)
		c.Assert(dbInfo.Name, DeepEquals, model.NewCIStr("test"))
		c.Assert(dbInfo.State, Equals, model.StatePublic)

		var historyJob *model.Job
		historyJob, err = t.GetHistoryDDLJob(job.ID)
		c.Assert(err, IsNil)
		c.Assert(historyJob.State, Equals, model.JobDone)
		c.Assert(historyJob.SchemaState, Equals, model.StatePublic)
		return nil
	})

	job = &model.Job{
		SchemaID: schemaID,
		Type:     model.ActionDropSchema,
	}

	err = d1.startJob(ctx, job)
	c.Assert(err, IsNil)

	d1.meta.RunInNewTxn(false, func(t *meta.TMeta) error {
		var dbInfo *model.DBInfo
		dbInfo, err = t.GetDatabase(schemaID)
		c.Assert(err, IsNil)
		c.Assert(dbInfo, IsNil)

		var historyJob *model.Job
		historyJob, err = t.GetHistoryDDLJob(job.ID)
		c.Assert(err, IsNil)
		c.Assert(historyJob.State, Equals, model.JobDone)
		c.Assert(historyJob.SchemaState, Equals, model.StateNone)
		return nil
	})

	job = &model.Job{
		SchemaID: schemaID,
		Type:     model.ActionDropSchema,
	}

	err = d1.startJob(ctx, job)
	c.Assert(errors2.ErrorEqual(err, ErrNotExists), IsTrue)
}

func (s *testDDLSuite) TestSchemaWaitJob(c *C) {
	store := createTestStore(c, "test_schema_wait")
	defer store.Close()

	ctx := mock.NewContext()

	lease := 50 * time.Millisecond

	d1 := newDDL(store, nil, nil, lease)
	defer d1.close()

	testCheckOwner(c, d1, true)

	d2 := newDDL(store, nil, nil, lease)
	defer d2.close()

	// d2 must not be owner.
	testCheckOwner(c, d2, false)

	schema := model.NewCIStr("test")
	schemaID, err := d2.meta.GenGlobalID()
	c.Assert(err, IsNil)

	job := &model.Job{
		SchemaID: schemaID,
		Type:     model.ActionCreateSchema,
		Args:     []interface{}{schema},
	}

	err = d2.startJob(ctx, job)
	c.Assert(err, IsNil)

	d2.meta.RunInNewTxn(false, func(t *meta.TMeta) error {
		var dbInfo *model.DBInfo
		dbInfo, err = t.GetDatabase(schemaID)
		c.Assert(err, IsNil)
		c.Assert(dbInfo.Name, DeepEquals, model.NewCIStr("test"))
		c.Assert(dbInfo.State, Equals, model.StatePublic)
		return nil
	})

	// d2 must not be owner.
	testCheckOwner(c, d2, false)

	schemaID, err = d2.meta.GenGlobalID()
	c.Assert(err, IsNil)

	job = &model.Job{
		SchemaID: schemaID,
		Type:     model.ActionCreateSchema,
		Args:     []interface{}{schema},
	}

	err = d2.startJob(ctx, job)
	c.Assert(err, NotNil)

	d2.meta.RunInNewTxn(false, func(t *meta.TMeta) error {
		var historyJob *model.Job
		historyJob, err = t.GetHistoryDDLJob(job.ID)
		c.Assert(err, IsNil)
		c.Assert(historyJob.State, Equals, model.JobCancelled)
		return nil
	})

	// d2 must not be owner.
	testCheckOwner(c, d2, false)
}

func testRunInterruptedJob(c *C, d *ddl, job *model.Job) {
	ctx := mock.NewContext()
	done := make(chan error, 1)
	go func() {
		done <- d.startJob(ctx, job)
	}()

	ticker := time.NewTicker(d.lease * 1)
	defer ticker.Stop()

LOOP:
	for {
		select {
		case <-ticker.C:
			d.close()
			d.start()
			asyncNotify(d.jobCh)
		case err := <-done:
			c.Assert(err, IsNil)
			break LOOP
		}
	}
}

func (s *testDDLSuite) TestSchemaResume(c *C) {
	store := createTestStore(c, "test_schema_resume")
	defer store.Close()

	lease := 50 * time.Millisecond

	d1 := newDDL(store, nil, nil, lease)
	defer d1.close()

	testCheckOwner(c, d1, true)

	schema := model.NewCIStr("test")
	schemaID, err := d1.meta.GenGlobalID()
	c.Assert(err, IsNil)

	job := &model.Job{
		SchemaID: schemaID,
		Type:     model.ActionCreateSchema,
		Args:     []interface{}{schema},
	}

	testRunInterruptedJob(c, d1, job)

	d1.meta.RunInNewTxn(false, func(t *meta.TMeta) error {
		var dbInfo *model.DBInfo
		dbInfo, err = t.GetDatabase(schemaID)
		c.Assert(err, IsNil)
		c.Assert(dbInfo.Name, DeepEquals, model.NewCIStr("test"))
		c.Assert(dbInfo.State, Equals, model.StatePublic)
		return nil
	})

	job = &model.Job{
		SchemaID: schemaID,
		Type:     model.ActionDropSchema,
	}

	testRunInterruptedJob(c, d1, job)

	d1.meta.RunInNewTxn(false, func(t *meta.TMeta) error {
		var dbInfo *model.DBInfo
		dbInfo, err = t.GetDatabase(schemaID)
		c.Assert(err, IsNil)
		c.Assert(dbInfo, IsNil)
		return nil
	})
}
<|endoftext|>


<commit_before>package main

const code = `{{ if .Services }}
package {{.PackageName}}

import (
	"errors"
	"fmt"
	"log"
	"math"
	"time"

	"github.com/AsynkronIT/protoactor-go/actor"
	"github.com/AsynkronIT/protoactor-go/cluster"
	"github.com/AsynkronIT/protoactor-go/remote"
	"github.com/gogo/protobuf/proto"
)

var _ = proto.Marshal
var _ = fmt.Errorf
var _ = math.Inf

var rootContext = actor.EmptyRootContext
{{ range $service := .Services}}
var x{{ $service.Name }}Factory func() {{ $service.Name }}

// {{ $service.Name }}Factory produces a {{ $service.Name }}
func {{ $service.Name }}Factory(factory func() {{ $service.Name }}) {
	x{{ $service.Name }}Factory = factory
}

// Get{{ $service.Name }}Grain instantiates a new {{ $service.Name }}Grain with given ID
func Get{{ $service.Name }}Grain(id string) *{{ $service.Name }}Grain {
	return &{{ $service.Name }}Grain{ID: id}
}

// {{ $service.Name }} interfaces the services available to the {{ $service.Name }}
type {{ $service.Name }} interface {
	Init(id string)
	Terminate()
	{{ range $method := $service.Methods}}
	{{ $method.Name }}(*{{ $method.Input.Name }}, cluster.GrainContext) (*{{ $method.Output.Name }}, error)
	{{ end }}
}

// {{ $service.Name }}Grain holds the base data for the {{ $service.Name }}Grain
type {{ $service.Name }}Grain struct {
	ID string
}
{{ range $method := $service.Methods}}
// {{ $method.Name }} requests the execution on to the cluster using default options
func (g *{{ $service.Name }}Grain) {{ $method.Name }}(r *{{ $method.Input.Name }}) (*{{ $method.Output.Name }}, error) {
	return g.{{ $method.Name }}WithOpts(r, cluster.DefaultGrainCallOptions())
}

// {{ $method.Name }}WithOpts requests the execution on to the cluster
func (g *{{ $service.Name }}Grain) {{ $method.Name }}WithOpts(r *{{ $method.Input.Name }}, opts *cluster.GrainCallOptions) (*{{ $method.Output.Name }}, error) {
	fun := func() (*{{ $method.Output.Name }}, error) {
			pid, statusCode := cluster.Get(g.ID, "{{ $service.Name }}")
			if statusCode != remote.ResponseStatusCodeOK {
				return nil, fmt.Errorf("get PID failed with StatusCode: %v", statusCode)
			}
			bytes, err := proto.Marshal(r)
			if err != nil {
				return nil, err
			}
			request := &cluster.GrainRequest{MethodIndex: {{ $method.Index }}, MessageData: bytes}
			response, err := rootContext.RequestFuture(pid, request, opts.Timeout).Result()
			if err != nil {
				return nil, err
			}
			switch msg := response.(type) {
			case *cluster.GrainResponse:
				result := &{{ $method.Output.Name }}{}
				err = proto.Unmarshal(msg.MessageData, result)
				if err != nil {
					return nil, err
				}
				return result, nil
			case *cluster.GrainErrorResponse:
				return nil, errors.New(msg.Err)
			default:
				return nil, errors.New("unknown response")
			}
		}

	var res *{{ $method.Output.Name }}
	var err error
	for i := 0; i < opts.RetryCount; i++ {
		res, err = fun()
		if err == nil || err.Error() != "future: timeout" {
			return res, err
		} else if opts.RetryAction != nil {
				opts.RetryAction(i)
		}
	}
	return nil, err
}

// {{ $method.Name }}Chan allows to use a channel to execute the method using default options
func (g *{{ $service.Name }}Grain) {{ $method.Name }}Chan(r *{{ $method.Input.Name }}) (<-chan *{{ $method.Output.Name }}, <-chan error) {
	return g.{{ $method.Name }}ChanWithOpts(r, cluster.DefaultGrainCallOptions())
}

// {{ $method.Name }}ChanWithOpts allows to use a channel to execute the method
func (g *{{ $service.Name }}Grain) {{ $method.Name }}ChanWithOpts(r *{{ $method.Input.Name }}, opts *cluster.GrainCallOptions) (<-chan *{{ $method.Output.Name }}, <-chan error) {
	c := make(chan *{{ $method.Output.Name }})
	e := make(chan error)
	go func() {
		res, err := g.{{ $method.Name }}WithOpts(r, opts)
		if err != nil {
			e <- err
		} else {
			c <- res
		}
		close(c)
		close(e)
	}()
	return c, e
}
{{ end }}

// {{ $service.Name }}Actor represents the actor structure
type {{ $service.Name }}Actor struct {
	inner {{ $service.Name }}
	Timeout *time.Duration
}

// Receive ensures the lifecycle of the actor for the received message
func (a *{{ $service.Name }}Actor) Receive(ctx actor.Context) {
	switch msg := ctx.Message().(type) {
	case *actor.Started:
		a.inner = x{{ $service.Name }}Factory()
		id := ctx.Self().Id
		a.inner.Init(id[7:]) // skip "remote$"
		if a.Timeout != nil {
			ctx.SetReceiveTimeout(*a.Timeout)
		}
	case *actor.ReceiveTimeout:
		a.inner.Terminate()
		ctx.Self().Poison()

	case actor.AutoReceiveMessage: // pass
	case actor.SystemMessage: // pass

	case *cluster.GrainRequest:
		switch msg.MethodIndex {
		{{ range $method := $service.Methods}}
		case {{ $method.Index }}:
			req := &{{ $method.Input.Name }}{}
			err := proto.Unmarshal(msg.MessageData, req)
			if err != nil {
				log.Fatalf("[GRAIN] proto.Unmarshal failed %v", err)
			}
			r0, err := a.inner.{{ $method.Name }}(req, ctx)
			if err == nil {
				bytes, errMarshal := proto.Marshal(r0)
				if errMarshal != nil {
					log.Fatalf("[GRAIN] proto.Marshal failed %v", errMarshal)
				}
				resp := &cluster.GrainResponse{MessageData: bytes}
				ctx.Respond(resp)
			} else {
				resp := &cluster.GrainErrorResponse{Err: err.Error()}
				ctx.Respond(resp)
			}
		{{ end }}
		}
	default:
		log.Printf("Unknown message %v", msg)
	}
}

{{ end }}

{{ end}}

`
<commit_msg>Adding PROCESSNAMEALREADYEXIST doesnt seem to be a failure<commit_after>package main

const code = `{{ if .Services }}
package {{.PackageName}}

import (
	"errors"
	"fmt"
	"log"
	"math"
	"time"

	"github.com/AsynkronIT/protoactor-go/actor"
	"github.com/AsynkronIT/protoactor-go/cluster"
	"github.com/AsynkronIT/protoactor-go/remote"
	"github.com/gogo/protobuf/proto"
)

var _ = proto.Marshal
var _ = fmt.Errorf
var _ = math.Inf

var rootContext = actor.EmptyRootContext
{{ range $service := .Services}}
var x{{ $service.Name }}Factory func() {{ $service.Name }}

// {{ $service.Name }}Factory produces a {{ $service.Name }}
func {{ $service.Name }}Factory(factory func() {{ $service.Name }}) {
	x{{ $service.Name }}Factory = factory
}

// Get{{ $service.Name }}Grain instantiates a new {{ $service.Name }}Grain with given ID
func Get{{ $service.Name }}Grain(id string) *{{ $service.Name }}Grain {
	return &{{ $service.Name }}Grain{ID: id}
}

// {{ $service.Name }} interfaces the services available to the {{ $service.Name }}
type {{ $service.Name }} interface {
	Init(id string)
	Terminate()
	{{ range $method := $service.Methods}}
	{{ $method.Name }}(*{{ $method.Input.Name }}, cluster.GrainContext) (*{{ $method.Output.Name }}, error)
	{{ end }}
}

// {{ $service.Name }}Grain holds the base data for the {{ $service.Name }}Grain
type {{ $service.Name }}Grain struct {
	ID string
}
{{ range $method := $service.Methods}}
// {{ $method.Name }} requests the execution on to the cluster using default options
func (g *{{ $service.Name }}Grain) {{ $method.Name }}(r *{{ $method.Input.Name }}) (*{{ $method.Output.Name }}, error) {
	return g.{{ $method.Name }}WithOpts(r, cluster.DefaultGrainCallOptions())
}

// {{ $method.Name }}WithOpts requests the execution on to the cluster
func (g *{{ $service.Name }}Grain) {{ $method.Name }}WithOpts(r *{{ $method.Input.Name }}, opts *cluster.GrainCallOptions) (*{{ $method.Output.Name }}, error) {
	fun := func() (*{{ $method.Output.Name }}, error) {
			pid, statusCode := cluster.Get(g.ID, "{{ $service.Name }}")
			if statusCode != remote.ResponseStatusCodeOK && statusCode != remote.ResponseStatusCodePROCESSNAMEALREADYEXIST {
				return nil, fmt.Errorf("get PID failed with StatusCode: %v", statusCode)
			}
			bytes, err := proto.Marshal(r)
			if err != nil {
				return nil, err
			}
			request := &cluster.GrainRequest{MethodIndex: {{ $method.Index }}, MessageData: bytes}
			response, err := rootContext.RequestFuture(pid, request, opts.Timeout).Result()
			if err != nil {
				return nil, err
			}
			switch msg := response.(type) {
			case *cluster.GrainResponse:
				result := &{{ $method.Output.Name }}{}
				err = proto.Unmarshal(msg.MessageData, result)
				if err != nil {
					return nil, err
				}
				return result, nil
			case *cluster.GrainErrorResponse:
				return nil, errors.New(msg.Err)
			default:
				return nil, errors.New("unknown response")
			}
		}

	var res *{{ $method.Output.Name }}
	var err error
	for i := 0; i < opts.RetryCount; i++ {
		res, err = fun()
		if err == nil || err.Error() != "future: timeout" {
			return res, err
		} else if opts.RetryAction != nil {
				opts.RetryAction(i)
		}
	}
	return nil, err
}

// {{ $method.Name }}Chan allows to use a channel to execute the method using default options
func (g *{{ $service.Name }}Grain) {{ $method.Name }}Chan(r *{{ $method.Input.Name }}) (<-chan *{{ $method.Output.Name }}, <-chan error) {
	return g.{{ $method.Name }}ChanWithOpts(r, cluster.DefaultGrainCallOptions())
}

// {{ $method.Name }}ChanWithOpts allows to use a channel to execute the method
func (g *{{ $service.Name }}Grain) {{ $method.Name }}ChanWithOpts(r *{{ $method.Input.Name }}, opts *cluster.GrainCallOptions) (<-chan *{{ $method.Output.Name }}, <-chan error) {
	c := make(chan *{{ $method.Output.Name }})
	e := make(chan error)
	go func() {
		res, err := g.{{ $method.Name }}WithOpts(r, opts)
		if err != nil {
			e <- err
		} else {
			c <- res
		}
		close(c)
		close(e)
	}()
	return c, e
}
{{ end }}

// {{ $service.Name }}Actor represents the actor structure
type {{ $service.Name }}Actor struct {
	inner {{ $service.Name }}
	Timeout *time.Duration
}

// Receive ensures the lifecycle of the actor for the received message
func (a *{{ $service.Name }}Actor) Receive(ctx actor.Context) {
	switch msg := ctx.Message().(type) {
	case *actor.Started:
		a.inner = x{{ $service.Name }}Factory()
		id := ctx.Self().Id
		a.inner.Init(id[7:]) // skip "remote$"
		if a.Timeout != nil {
			ctx.SetReceiveTimeout(*a.Timeout)
		}
	case *actor.ReceiveTimeout:
		a.inner.Terminate()
		ctx.Self().Poison()

	case actor.AutoReceiveMessage: // pass
	case actor.SystemMessage: // pass

	case *cluster.GrainRequest:
		switch msg.MethodIndex {
		{{ range $method := $service.Methods}}
		case {{ $method.Index }}:
			req := &{{ $method.Input.Name }}{}
			err := proto.Unmarshal(msg.MessageData, req)
			if err != nil {
				log.Fatalf("[GRAIN] proto.Unmarshal failed %v", err)
			}
			r0, err := a.inner.{{ $method.Name }}(req, ctx)
			if err == nil {
				bytes, errMarshal := proto.Marshal(r0)
				if errMarshal != nil {
					log.Fatalf("[GRAIN] proto.Marshal failed %v", errMarshal)
				}
				resp := &cluster.GrainResponse{MessageData: bytes}
				ctx.Respond(resp)
			} else {
				resp := &cluster.GrainErrorResponse{Err: err.Error()}
				ctx.Respond(resp)
			}
		{{ end }}
		}
	default:
		log.Printf("Unknown message %v", msg)
	}
}

{{ end }}

{{ end}}

`
<|endoftext|>


<commit_before>package mpb_test

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"regexp"
	"strings"
	"testing"
	"time"

	"github.com/vbauerster/mpb"
)

func TestPrependName(t *testing.T) {
	var buf bytes.Buffer
	p := mpb.New().SetOut(&buf)
	name := "TestBar"
	bar := p.AddBar(100).TrimLeftSpace().TrimRightSpace().
		PrependName(name, 0, 0)
	for i := 0; i < 100; i++ {
		bar.Incr(1)
	}

	p.Stop()

	want := name + "["
	barOut := buf.String()
	if !strings.Contains(barOut, want) {
		t.Errorf("%q not found in bar: %s\n", want, barOut)
	}
}

func TestPrependNameDindentRight(t *testing.T) {
	var buf bytes.Buffer
	p := mpb.New().SetOut(&buf)
	name := "TestBar"
	bar := p.AddBar(100).TrimLeftSpace().TrimRightSpace().
		PrependName(name, len(name)+1, mpb.DidentRight)
	for i := 0; i < 100; i++ {
		bar.Incr(1)
	}

	p.Stop()

	want := name + " ["
	barOut := buf.String()
	if !strings.Contains(barOut, want) {
		t.Errorf("%q not found in bar: %s\n", want, barOut)
	}
}

func TestPrependCounters(t *testing.T) {
	var buf bytes.Buffer
	p := mpb.New().SetOut(&buf)

	reader := strings.NewReader(content)

	total := int64(len(content))
	bar := p.AddBar(total).TrimLeftSpace().TrimRightSpace().
		PrependCounters("%3s / %3s", mpb.UnitBytes, 0, 0)
	preader := bar.ProxyReader(reader)

	_, err := io.Copy(ioutil.Discard, preader)
	if err != nil {
		t.Errorf("Error copying from reader: %+v\n", err)
	}

	p.Stop()

	barOut := buf.String()
	want := fmt.Sprintf("%[1]db / %[1]db[", total)
	if !strings.Contains(barOut, want) {
		t.Errorf("%q not found in bar: %s\n", want, barOut)
	}
}

func TestPrependCountersDindentRight(t *testing.T) {
	var buf bytes.Buffer
	p := mpb.New().SetOut(&buf)

	reader := strings.NewReader(content)

	total := int64(len(content))
	bar := p.AddBar(total).TrimLeftSpace().TrimRightSpace().
		PrependCounters("%3s / %3s", mpb.UnitBytes, 12, mpb.DidentRight)
	preader := bar.ProxyReader(reader)

	_, err := io.Copy(ioutil.Discard, preader)
	if err != nil {
		t.Errorf("Error copying from reader: %+v\n", err)
	}

	p.Stop()

	barOut := buf.String()
	want := fmt.Sprintf("%[1]db / %[1]db [", total)
	if !strings.Contains(barOut, want) {
		t.Errorf("%q not found in bar: %s\n", want, barOut)
	}
}

func TestAppendPercentage(t *testing.T) {
	var buf bytes.Buffer
	p := mpb.New().SetOut(&buf)

	bar := p.AddBar(100).TrimLeftSpace().TrimRightSpace().
		AppendPercentage(6, 0)

	for i := 0; i < 100; i++ {
		time.Sleep(10 * time.Millisecond)
		bar.Incr(1)
	}

	p.Stop()

	want := "] 100 %"
	barOut := buf.String()
	if !strings.Contains(barOut, want) {
		t.Errorf("%q not found in bar: %s\n", want, barOut)
	}
}

func TestAppendPercentageDindentRight(t *testing.T) {
	var buf bytes.Buffer
	p := mpb.New().SetOut(&buf)

	bar := p.AddBar(100).TrimLeftSpace().TrimRightSpace().
		AppendPercentage(6, mpb.DidentRight)

	for i := 0; i < 100; i++ {
		time.Sleep(10 * time.Millisecond)
		bar.Incr(1)
	}

	p.Stop()

	want := "]100 % "
	barOut := buf.String()
	if !strings.Contains(barOut, want) {
		t.Errorf("%q not found in bar: %s\n", want, barOut)
	}
}

func TestPrependPercentage(t *testing.T) {
	var buf bytes.Buffer
	p := mpb.New().SetOut(&buf)

	bar := p.AddBar(100).TrimLeftSpace().TrimRightSpace().
		PrependPercentage(6, 0)

	for i := 0; i < 100; i++ {
		time.Sleep(10 * time.Millisecond)
		bar.Incr(1)
	}

	p.Stop()

	want := " 100 %["
	barOut := buf.String()
	if !strings.Contains(barOut, want) {
		t.Errorf("%q not found in bar: %s\n", want, barOut)
	}
}

func TestPrependPercentageDindentRight(t *testing.T) {
	var buf bytes.Buffer
	p := mpb.New().SetOut(&buf)

	bar := p.AddBar(100).TrimLeftSpace().TrimRightSpace().
		PrependPercentage(6, mpb.DidentRight)

	for i := 0; i < 100; i++ {
		time.Sleep(10 * time.Millisecond)
		bar.Incr(1)
	}

	p.Stop()

	want := "100 % ["
	barOut := buf.String()
	if !strings.Contains(barOut, want) {
		t.Errorf("%q not found in bar: %s\n", want, barOut)
	}
}

func TestPrependElapsed(t *testing.T) {
	var buf bytes.Buffer
	p := mpb.New().SetOut(&buf)

	bar := p.AddBar(100).TrimLeftSpace().TrimRightSpace().
		PrependElapsed(0, 0)

	for i := 0; i < 100; i++ {
		time.Sleep(10 * time.Millisecond)
		bar.Incr(1)
	}

	p.Stop()

	want := "1s["
	barOut := buf.String()
	if !strings.Contains(barOut, want) {
		t.Errorf("%q not found in bar: %s\n", want, barOut)
	}
}

func TestPrependElapsedDindentRight(t *testing.T) {
	var buf bytes.Buffer
	p := mpb.New().SetOut(&buf)

	bar := p.AddBar(100).TrimLeftSpace().TrimRightSpace().
		PrependElapsed(3, mpb.DidentRight)

	for i := 0; i < 100; i++ {
		time.Sleep(10 * time.Millisecond)
		bar.Incr(1)
	}

	p.Stop()

	want := "1s ["
	barOut := buf.String()
	if !strings.Contains(barOut, want) {
		t.Errorf("%q not found in bar: %s\n", want, barOut)
	}
}

func TestAppendElapsed(t *testing.T) {
	var buf bytes.Buffer
	p := mpb.New().SetOut(&buf)

	bar := p.AddBar(100).TrimLeftSpace().TrimRightSpace().
		AppendElapsed(0, 0)

	for i := 0; i < 100; i++ {
		time.Sleep(10 * time.Millisecond)
		bar.Incr(1)
	}

	p.Stop()

	want := "]1s"
	barOut := buf.String()
	if !strings.Contains(barOut, want) {
		t.Errorf("%q not found in bar: %s\n", want, barOut)
	}
}

func TestAppendElapsedDindentRight(t *testing.T) {
	var buf bytes.Buffer
	p := mpb.New().SetOut(&buf)

	bar := p.AddBar(100).TrimLeftSpace().TrimRightSpace().
		AppendElapsed(3, mpb.DidentRight)

	for i := 0; i < 100; i++ {
		time.Sleep(10 * time.Millisecond)
		bar.Incr(1)
	}

	p.Stop()

	want := "]1s "
	barOut := buf.String()
	if !strings.Contains(barOut, want) {
		t.Errorf("%q not found in bar: %s\n", want, barOut)
	}
}

func TestPrependETA(t *testing.T) {
	var buf bytes.Buffer
	p := mpb.New().SetOut(&buf)

	bar := p.AddBar(100).TrimLeftSpace().TrimRightSpace().
		PrependETA(0, 0)

	for i := 0; i < 100; i++ {
		time.Sleep(10 * time.Millisecond)
		bar.Incr(1)
	}

	p.Stop()

	want := `0s?\[`
	barOut := buf.String()

	matched, err := regexp.MatchString(want, barOut)
	if err != nil {
		t.Logf("Regex %q err: %+v\n", want, err)
		t.FailNow()
	}

	if !matched {
		t.Errorf("%q not found in bar: %s\n", want, barOut)
	}
}

func TestPrependETADindentRight(t *testing.T) {
	var buf bytes.Buffer
	p := mpb.New().SetOut(&buf)

	bar := p.AddBar(100).TrimLeftSpace().TrimRightSpace().
		PrependETA(3, mpb.DidentRight)

	for i := 0; i < 100; i++ {
		time.Sleep(10 * time.Millisecond)
		bar.Incr(1)
	}

	p.Stop()

	want := `0s? \[`
	barOut := buf.String()

	matched, err := regexp.MatchString(want, barOut)
	if err != nil {
		t.Logf("Regex %q err: %+v\n", want, err)
		t.FailNow()
	}

	if !matched {
		t.Errorf("%q not found in bar: %s\n", want, barOut)
	}
}

func TestAppendETA(t *testing.T) {
	var buf bytes.Buffer
	p := mpb.New().SetOut(&buf)

	bar := p.AddBar(100).TrimLeftSpace().TrimRightSpace().
		AppendETA(0, 0)

	for i := 0; i < 100; i++ {
		time.Sleep(10 * time.Millisecond)
		bar.Incr(1)
	}

	p.Stop()

	want := `\]0s?`
	barOut := buf.String()

	matched, err := regexp.MatchString(want, barOut)
	if err != nil {
		t.Logf("Regex %q err: %+v\n", want, err)
		t.FailNow()
	}

	if !matched {
		t.Errorf("%q not found in bar: %s\n", want, barOut)
	}
}

func TestAppendETADindentRight(t *testing.T) {
	var buf bytes.Buffer
	p := mpb.New().SetOut(&buf)

	bar := p.AddBar(100).TrimLeftSpace().TrimRightSpace().
		AppendETA(3, mpb.DidentRight)

	for i := 0; i < 100; i++ {
		time.Sleep(10 * time.Millisecond)
		bar.Incr(1)
	}

	p.Stop()

	want := `\]0s? `
	barOut := buf.String()

	matched, err := regexp.MatchString(want, barOut)
	if err != nil {
		t.Logf("Regex %q err: %+v\n", want, err)
		t.FailNow()
	}

	if !matched {
		t.Errorf("%q not found in bar: %s\n", want, barOut)
	}
}
<commit_msg>One more test fix<commit_after>package mpb_test

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"regexp"
	"strings"
	"testing"
	"time"

	"github.com/vbauerster/mpb"
)

func TestPrependName(t *testing.T) {
	var buf bytes.Buffer
	p := mpb.New().SetOut(&buf)
	name := "TestBar"
	bar := p.AddBar(100).TrimLeftSpace().TrimRightSpace().
		PrependName(name, 0, 0)
	for i := 0; i < 100; i++ {
		bar.Incr(1)
	}

	p.Stop()

	want := name + "["
	barOut := buf.String()
	if !strings.Contains(barOut, want) {
		t.Errorf("%q not found in bar: %s\n", want, barOut)
	}
}

func TestPrependNameDindentRight(t *testing.T) {
	var buf bytes.Buffer
	p := mpb.New().SetOut(&buf)
	name := "TestBar"
	bar := p.AddBar(100).TrimLeftSpace().TrimRightSpace().
		PrependName(name, len(name)+1, mpb.DidentRight)
	for i := 0; i < 100; i++ {
		bar.Incr(1)
	}

	p.Stop()

	want := name + " ["
	barOut := buf.String()
	if !strings.Contains(barOut, want) {
		t.Errorf("%q not found in bar: %s\n", want, barOut)
	}
}

func TestPrependCounters(t *testing.T) {
	var buf bytes.Buffer
	p := mpb.New().SetOut(&buf)

	reader := strings.NewReader(content)

	total := int64(len(content))
	bar := p.AddBar(total).TrimLeftSpace().TrimRightSpace().
		PrependCounters("%3s / %3s", mpb.UnitBytes, 0, 0)
	preader := bar.ProxyReader(reader)

	_, err := io.Copy(ioutil.Discard, preader)
	if err != nil {
		t.Errorf("Error copying from reader: %+v\n", err)
	}

	p.Stop()

	barOut := buf.String()
	want := fmt.Sprintf("%[1]db / %[1]db[", total)
	if !strings.Contains(barOut, want) {
		t.Errorf("%q not found in bar: %s\n", want, barOut)
	}
}

func TestPrependCountersDindentRight(t *testing.T) {
	var buf bytes.Buffer
	p := mpb.New().SetOut(&buf)

	reader := strings.NewReader(content)

	total := int64(len(content))
	bar := p.AddBar(total).TrimLeftSpace().TrimRightSpace().
		PrependCounters("%3s / %3s", mpb.UnitBytes, 12, mpb.DidentRight)
	preader := bar.ProxyReader(reader)

	_, err := io.Copy(ioutil.Discard, preader)
	if err != nil {
		t.Errorf("Error copying from reader: %+v\n", err)
	}

	p.Stop()

	barOut := buf.String()
	want := fmt.Sprintf("%[1]db / %[1]db [", total)
	if !strings.Contains(barOut, want) {
		t.Errorf("%q not found in bar: %s\n", want, barOut)
	}
}

func TestAppendPercentage(t *testing.T) {
	var buf bytes.Buffer
	p := mpb.New().SetOut(&buf)

	bar := p.AddBar(100).TrimLeftSpace().TrimRightSpace().
		AppendPercentage(6, 0)

	for i := 0; i < 100; i++ {
		time.Sleep(10 * time.Millisecond)
		bar.Incr(1)
	}

	p.Stop()

	want := "] 100 %"
	barOut := buf.String()
	if !strings.Contains(barOut, want) {
		t.Errorf("%q not found in bar: %s\n", want, barOut)
	}
}

func TestAppendPercentageDindentRight(t *testing.T) {
	var buf bytes.Buffer
	p := mpb.New().SetOut(&buf)

	bar := p.AddBar(100).TrimLeftSpace().TrimRightSpace().
		AppendPercentage(6, mpb.DidentRight)

	for i := 0; i < 100; i++ {
		time.Sleep(10 * time.Millisecond)
		bar.Incr(1)
	}

	p.Stop()

	want := "]100 % "
	barOut := buf.String()
	if !strings.Contains(barOut, want) {
		t.Errorf("%q not found in bar: %s\n", want, barOut)
	}
}

func TestPrependPercentage(t *testing.T) {
	var buf bytes.Buffer
	p := mpb.New().SetOut(&buf)

	bar := p.AddBar(100).TrimLeftSpace().TrimRightSpace().
		PrependPercentage(6, 0)

	for i := 0; i < 100; i++ {
		time.Sleep(10 * time.Millisecond)
		bar.Incr(1)
	}

	p.Stop()

	want := " 100 %["
	barOut := buf.String()
	if !strings.Contains(barOut, want) {
		t.Errorf("%q not found in bar: %s\n", want, barOut)
	}
}

func TestPrependPercentageDindentRight(t *testing.T) {
	var buf bytes.Buffer
	p := mpb.New().SetOut(&buf)

	bar := p.AddBar(100).TrimLeftSpace().TrimRightSpace().
		PrependPercentage(6, mpb.DidentRight)

	for i := 0; i < 100; i++ {
		time.Sleep(10 * time.Millisecond)
		bar.Incr(1)
	}

	p.Stop()

	want := "100 % ["
	barOut := buf.String()
	if !strings.Contains(barOut, want) {
		t.Errorf("%q not found in bar: %s\n", want, barOut)
	}
}

func TestPrependElapsed(t *testing.T) {
	var buf bytes.Buffer
	p := mpb.New().SetOut(&buf)

	bar := p.AddBar(100).TrimLeftSpace().TrimRightSpace().
		PrependElapsed(0, 0)

	for i := 0; i < 100; i++ {
		time.Sleep(10 * time.Millisecond)
		bar.Incr(1)
	}

	p.Stop()

	want := "1s["
	barOut := buf.String()
	if !strings.Contains(barOut, want) {
		t.Errorf("%q not found in bar: %s\n", want, barOut)
	}
}

func TestPrependElapsedDindentRight(t *testing.T) {
	var buf bytes.Buffer
	p := mpb.New().SetOut(&buf)

	bar := p.AddBar(100).TrimLeftSpace().TrimRightSpace().
		PrependElapsed(3, mpb.DidentRight)

	for i := 0; i < 100; i++ {
		time.Sleep(10 * time.Millisecond)
		bar.Incr(1)
	}

	p.Stop()

	want := "1s ["
	barOut := buf.String()
	if !strings.Contains(barOut, want) {
		t.Errorf("%q not found in bar: %s\n", want, barOut)
	}
}

func TestAppendElapsed(t *testing.T) {
	var buf bytes.Buffer
	p := mpb.New().SetOut(&buf)

	bar := p.AddBar(100).TrimLeftSpace().TrimRightSpace().
		AppendElapsed(0, 0)

	for i := 0; i < 100; i++ {
		time.Sleep(10 * time.Millisecond)
		bar.Incr(1)
	}

	p.Stop()

	want := "]1s"
	barOut := buf.String()
	if !strings.Contains(barOut, want) {
		t.Errorf("%q not found in bar: %s\n", want, barOut)
	}
}

func TestAppendElapsedDindentRight(t *testing.T) {
	var buf bytes.Buffer
	p := mpb.New().SetOut(&buf)

	bar := p.AddBar(100).TrimLeftSpace().TrimRightSpace().
		AppendElapsed(3, mpb.DidentRight)

	for i := 0; i < 100; i++ {
		time.Sleep(10 * time.Millisecond)
		bar.Incr(1)
	}

	p.Stop()

	want := "]1s "
	barOut := buf.String()
	if !strings.Contains(barOut, want) {
		t.Errorf("%q not found in bar: %s\n", want, barOut)
	}
}

func TestPrependETA(t *testing.T) {
	var buf bytes.Buffer
	p := mpb.New().SetOut(&buf)

	bar := p.AddBar(100).TrimLeftSpace().TrimRightSpace().
		PrependETA(0, 0)

	for i := 0; i < 100; i++ {
		time.Sleep(10 * time.Millisecond)
		bar.Incr(1)
	}

	p.Stop()

	want := `0s?\[`
	barOut := buf.String()

	matched, err := regexp.MatchString(want, barOut)
	if err != nil {
		t.Logf("Regex %q err: %+v\n", want, err)
		t.FailNow()
	}

	if !matched {
		t.Errorf("%q not found in bar: %s\n", want, barOut)
	}
}

func TestPrependETADindentRight(t *testing.T) {
	var buf bytes.Buffer
	p := mpb.New().SetOut(&buf)

	bar := p.AddBar(100).TrimLeftSpace().TrimRightSpace().
		PrependETA(3, mpb.DidentRight)

	for i := 0; i < 100; i++ {
		time.Sleep(10 * time.Millisecond)
		bar.Incr(1)
	}

	p.Stop()

	want := `0s?\s+\[`
	barOut := buf.String()

	matched, err := regexp.MatchString(want, barOut)
	if err != nil {
		t.Logf("Regex %q err: %+v\n", want, err)
		t.FailNow()
	}

	if !matched {
		t.Errorf("%q not found in bar: %s\n", want, barOut)
	}
}

func TestAppendETA(t *testing.T) {
	var buf bytes.Buffer
	p := mpb.New().SetOut(&buf)

	bar := p.AddBar(100).TrimLeftSpace().TrimRightSpace().
		AppendETA(0, 0)

	for i := 0; i < 100; i++ {
		time.Sleep(10 * time.Millisecond)
		bar.Incr(1)
	}

	p.Stop()

	want := `\]0s?`
	barOut := buf.String()

	matched, err := regexp.MatchString(want, barOut)
	if err != nil {
		t.Logf("Regex %q err: %+v\n", want, err)
		t.FailNow()
	}

	if !matched {
		t.Errorf("%q not found in bar: %s\n", want, barOut)
	}
}

func TestAppendETADindentRight(t *testing.T) {
	var buf bytes.Buffer
	p := mpb.New().SetOut(&buf)

	bar := p.AddBar(100).TrimLeftSpace().TrimRightSpace().
		AppendETA(3, mpb.DidentRight)

	for i := 0; i < 100; i++ {
		time.Sleep(10 * time.Millisecond)
		bar.Incr(1)
	}

	p.Stop()

	want := `\]0s? `
	barOut := buf.String()

	matched, err := regexp.MatchString(want, barOut)
	if err != nil {
		t.Logf("Regex %q err: %+v\n", want, err)
		t.FailNow()
	}

	if !matched {
		t.Errorf("%q not found in bar: %s\n", want, barOut)
	}
}
<|endoftext|>


<commit_before>package build

import (
	"fmt"
	"neon/util"
	"os"
	"os/exec"
	"runtime"
	"strings"
)

type Step interface {
	Run() error
}

func NewStep(target *Target, step interface{}) (Step, error) {
	switch step := step.(type) {
	case string:
		return NewShellStep(target, step)
	case map[interface{}]interface{}:
		return NewTaskStep(target, step)
	default:
		return nil, fmt.Errorf("step must be string or map")
	}
}

type ShellStep struct {
	Target  *Target
	Command string
}

func NewShellStep(target *Target, shell string) (Step, error) {
	step := ShellStep{
		Target:  target,
		Command: shell,
	}
	return step, nil
}

func (step ShellStep) Run() error {
	cmd, err := step.Target.Build.Context.EvaluateString(step.Command)
	if err != nil {
		return fmt.Errorf("evaluating shell expression: %v", err)
	}
	var command *exec.Cmd
	if runtime.GOOS == "windows" {
		command = exec.Command("cmd.exe", "/C", cmd)
	} else {
		command = exec.Command("sh", "-c", cmd)
	}
	dir, err := os.Getwd()
	if err != nil {
		return fmt.Errorf("getting current working directory: %v", err)
	}
	command.Dir = dir
	command.Stdout = os.Stdout
	command.Stderr = os.Stderr
	command.Env, err = step.Target.Build.Context.EvaluateEnvironment()
	if err != nil {
		return fmt.Errorf("building environment: %v", err)
	}
	return command.Run()
}

type TaskStep struct {
	Target *Target
	Task   Task
}

func NewTaskStep(target *Target, m map[interface{}]interface{}) (Step, error) {
	object, err := util.NewObject(m)
	if err != nil {
		return nil, fmt.Errorf("a task must be a map with string keys")
	}
	fields := object.Fields()
	for name, descriptor := range TaskMap {
		for _, field := range fields {
			if name == field {
				task, err := descriptor.Constructor(target, object)
				if err != nil {
					return nil, fmt.Errorf("parsing task '%s': %v", name, err)
				}
				step := TaskStep{
					Target: target,
					Task:   task,
				}
				return step, nil
			}
		}
	}
	return nil, fmt.Errorf("unknown task '%s'", strings.Join(fields, "/"))
}

func (step TaskStep) Run() error {
	return step.Task()
}
<commit_msg>Code cleanin<commit_after>package build

import (
	"fmt"
	"neon/util"
	"os"
	"os/exec"
	"runtime"
	"strings"
)

// A step has a Run() method
type Step interface {
	Run() error
}

// Make a step inside given target
func NewStep(target *Target, step interface{}) (Step, error) {
	switch step := step.(type) {
	case string:
		return NewShellStep(target, step)
	case map[interface{}]interface{}:
		return NewTaskStep(target, step)
	default:
		return nil, fmt.Errorf("step must be string or map")
	}
}

// A shell step
type ShellStep struct {
	Target  *Target
	Command string
}

// Make a shell step
func NewShellStep(target *Target, shell string) (Step, error) {
	step := ShellStep{
		Target:  target,
		Command: shell,
	}
	return step, nil
}

// Run a shell step:
// - If running on windows, run shell with "cmd.exe"
// - Otherwise, run shell with "sh"
func (step ShellStep) Run() error {
	cmd, err := step.Target.Build.Context.EvaluateString(step.Command)
	if err != nil {
		return fmt.Errorf("evaluating shell expression: %v", err)
	}
	var command *exec.Cmd
	if runtime.GOOS == "windows" {
		command = exec.Command("cmd.exe", "/C", cmd)
	} else {
		command = exec.Command("sh", "-c", cmd)
	}
	dir, err := os.Getwd()
	if err != nil {
		return fmt.Errorf("getting current working directory: %v", err)
	}
	command.Dir = dir
	command.Stdout = os.Stdout
	command.Stderr = os.Stderr
	command.Env, err = step.Target.Build.Context.EvaluateEnvironment()
	if err != nil {
		return fmt.Errorf("building environment: %v", err)
	}
	return command.Run()
}

// Structure for a task step
type TaskStep struct {
	Target *Target
	Task   Task
}

// Make a task step
func NewTaskStep(target *Target, m map[interface{}]interface{}) (Step, error) {
	object, err := util.NewObject(m)
	if err != nil {
		return nil, fmt.Errorf("a task must be a map with string keys")
	}
	fields := object.Fields()
	for name, descriptor := range TaskMap {
		for _, field := range fields {
			if name == field {
				task, err := descriptor.Constructor(target, object)
				if err != nil {
					return nil, fmt.Errorf("parsing task '%s': %v", name, err)
				}
				step := TaskStep{
					Target: target,
					Task:   task,
				}
				return step, nil
			}
		}
	}
	return nil, fmt.Errorf("unknown task '%s'", strings.Join(fields, "/"))
}

// Run a task step, calling the function for the step
func (step TaskStep) Run() error {
	return step.Task()
}
<|endoftext|>