<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ratelimit

import (
	"errors"
	"io"

	"github.com/jacobsa/gcloud/gcs"
	"golang.org/x/net/context"
)

// Create a bucket that limits the rate at which it calls the wrapped bucket
// using opThrottle, and limits the bandwidth with which it reads from the
// wrapped bucket using egressThrottle.
func NewThrottledBucket(
	opThrottle Throttle,
	egressThrottle Throttle,
	wrapped gcs.Bucket) (b gcs.Bucket) {
	b = &throttledBucket{
		opThrottle:     opThrottle,
		egressThrottle: egressThrottle,
		wrapped:        wrapped,
	}

	return
}

////////////////////////////////////////////////////////////////////////
// throttledBucket
////////////////////////////////////////////////////////////////////////

type throttledBucket struct {
	opThrottle     Throttle
	egressThrottle Throttle
	wrapped        gcs.Bucket
}

func (b *throttledBucket) Name() string {
	return b.wrapped.Name()
}

func (b *throttledBucket) NewReader(
	ctx context.Context,
	req *gcs.ReadObjectRequest) (rc io.ReadCloser, err error) {
	// Wait for permission to call through.
	err = b.opThrottle.Wait(ctx, 1)
	if err != nil {
		return
	}

	// Call through.
	rc, err = b.wrapped.NewReader(ctx, req)
	if err != nil {
		return
	}

	// Wrap the result in a throttled layer.
	rc = &readerCloser{
		Reader: ThrottledReader(ctx, rc, b.egressThrottle),
		Closer: rc,
	}

	return
}

func (b *throttledBucket) CreateObject(
	ctx context.Context,
	req *gcs.CreateObjectRequest) (o *gcs.Object, err error) {
	err = errors.New("TODO")
	return
}

func (b *throttledBucket) CopyObject(
	ctx context.Context,
	req *gcs.CopyObjectRequest) (o *gcs.Object, err error) {
	err = errors.New("TODO")
	return
}

func (b *throttledBucket) StatObject(
	ctx context.Context,
	req *gcs.StatObjectRequest) (o *gcs.Object, err error) {
	err = errors.New("TODO")
	return
}

func (b *throttledBucket) ListObjects(
	ctx context.Context,
	req *gcs.ListObjectsRequest) (listing *gcs.Listing, err error) {
	err = errors.New("TODO")
	return
}

func (b *throttledBucket) UpdateObject(
	ctx context.Context,
	req *gcs.UpdateObjectRequest) (o *gcs.Object, err error) {
	err = errors.New("TODO")
	return
}

func (b *throttledBucket) DeleteObject(
	ctx context.Context,
	name string) (err error) {
	err = errors.New("TODO")
	return
}

////////////////////////////////////////////////////////////////////////
// readerCloser
////////////////////////////////////////////////////////////////////////

// An io.ReadCloser that forwards read requests to an io.Reader and close
// requests to an io.Closer.
type readerCloser struct {
	Reader io.Reader
	Closer io.Closer
}

func (rc *readerCloser) Read(p []byte) (n int, err error) {
	n, err = rc.Reader.Read(p)
	return
}

func (rc *readerCloser) Close() (err error) {
	err = rc.Closer.Close()
	return
}
<commit_msg>Other throttledBucket methods.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ratelimit

import (
	"io"

	"github.com/jacobsa/gcloud/gcs"
	"golang.org/x/net/context"
)

// Create a bucket that limits the rate at which it calls the wrapped bucket
// using opThrottle, and limits the bandwidth with which it reads from the
// wrapped bucket using egressThrottle.
func NewThrottledBucket(
	opThrottle Throttle,
	egressThrottle Throttle,
	wrapped gcs.Bucket) (b gcs.Bucket) {
	b = &throttledBucket{
		opThrottle:     opThrottle,
		egressThrottle: egressThrottle,
		wrapped:        wrapped,
	}

	return
}

////////////////////////////////////////////////////////////////////////
// throttledBucket
////////////////////////////////////////////////////////////////////////

type throttledBucket struct {
	opThrottle     Throttle
	egressThrottle Throttle
	wrapped        gcs.Bucket
}

func (b *throttledBucket) Name() string {
	return b.wrapped.Name()
}

func (b *throttledBucket) NewReader(
	ctx context.Context,
	req *gcs.ReadObjectRequest) (rc io.ReadCloser, err error) {
	// Wait for permission to call through.
	err = b.opThrottle.Wait(ctx, 1)
	if err != nil {
		return
	}

	// Call through.
	rc, err = b.wrapped.NewReader(ctx, req)
	if err != nil {
		return
	}

	// Wrap the result in a throttled layer.
	rc = &readerCloser{
		Reader: ThrottledReader(ctx, rc, b.egressThrottle),
		Closer: rc,
	}

	return
}

func (b *throttledBucket) CreateObject(
	ctx context.Context,
	req *gcs.CreateObjectRequest) (o *gcs.Object, err error) {
	// Wait for permission to call through.
	err = b.opThrottle.Wait(ctx, 1)
	if err != nil {
		return
	}

	// Call through.
	o, err = b.wrapped.CreateObject(ctx, req)

	return
}

func (b *throttledBucket) CopyObject(
	ctx context.Context,
	req *gcs.CopyObjectRequest) (o *gcs.Object, err error) {
	// Wait for permission to call through.
	err = b.opThrottle.Wait(ctx, 1)
	if err != nil {
		return
	}

	// Call through.
	o, err = b.wrapped.CopyObject(ctx, req)

	return
}

func (b *throttledBucket) StatObject(
	ctx context.Context,
	req *gcs.StatObjectRequest) (o *gcs.Object, err error) {
	// Wait for permission to call through.
	err = b.opThrottle.Wait(ctx, 1)
	if err != nil {
		return
	}

	// Call through.
	o, err = b.wrapped.StatObject(ctx, req)

	return
}

func (b *throttledBucket) ListObjects(
	ctx context.Context,
	req *gcs.ListObjectsRequest) (listing *gcs.Listing, err error) {
	// Wait for permission to call through.
	err = b.opThrottle.Wait(ctx, 1)
	if err != nil {
		return
	}

	// Call through.
	listing, err = b.wrapped.ListObjects(ctx, req)

	return
}

func (b *throttledBucket) UpdateObject(
	ctx context.Context,
	req *gcs.UpdateObjectRequest) (o *gcs.Object, err error) {
	// Wait for permission to call through.
	err = b.opThrottle.Wait(ctx, 1)
	if err != nil {
		return
	}

	// Call through.
	o, err = b.wrapped.UpdateObject(ctx, req)

	return
}

func (b *throttledBucket) DeleteObject(
	ctx context.Context,
	name string) (err error) {
	// Wait for permission to call through.
	err = b.opThrottle.Wait(ctx, 1)
	if err != nil {
		return
	}

	// Call through.
	err = b.wrapped.DeleteObject(ctx, name)

	return
}

////////////////////////////////////////////////////////////////////////
// readerCloser
////////////////////////////////////////////////////////////////////////

// An io.ReadCloser that forwards read requests to an io.Reader and close
// requests to an io.Closer.
type readerCloser struct {
	Reader io.Reader
	Closer io.Closer
}

func (rc *readerCloser) Read(p []byte) (n int, err error) {
	n, err = rc.Reader.Read(p)
	return
}

func (rc *readerCloser) Close() (err error) {
	err = rc.Closer.Close()
	return
}
<|endoftext|>


<commit_before>package release

import (
	"bytes"
	"errors"
	"fmt"
	mo_path2 "github.com/watermint/toolbox/domain/common/model/mo_path"
	"github.com/watermint/toolbox/domain/common/model/mo_string"
	"github.com/watermint/toolbox/domain/dropbox/api/dbx_conn_impl"
	"github.com/watermint/toolbox/domain/github/api/gh_conn"
	"github.com/watermint/toolbox/domain/github/api/gh_context"
	"github.com/watermint/toolbox/domain/github/api/gh_context_impl"
	"github.com/watermint/toolbox/domain/github/model/mo_release"
	"github.com/watermint/toolbox/domain/github/service/sv_reference"
	"github.com/watermint/toolbox/domain/github/service/sv_release"
	"github.com/watermint/toolbox/domain/github/service/sv_release_asset"
	"github.com/watermint/toolbox/infra/api/api_auth"
	"github.com/watermint/toolbox/infra/api/api_auth_impl"
	"github.com/watermint/toolbox/infra/app"
	"github.com/watermint/toolbox/infra/control/app_control"
	"github.com/watermint/toolbox/infra/recipe/rc_exec"
	"github.com/watermint/toolbox/infra/recipe/rc_recipe"
	"github.com/watermint/toolbox/infra/ui/app_lang"
	"github.com/watermint/toolbox/infra/ui/app_msg"
	"github.com/watermint/toolbox/infra/ui/app_ui"
	"github.com/watermint/toolbox/infra/util/ut_filehash"
	"github.com/watermint/toolbox/quality/infra/qt_endtoend"
	"github.com/watermint/toolbox/quality/infra/qt_errors"
	"github.com/watermint/toolbox/quality/infra/qt_file"
	"github.com/watermint/toolbox/quality/infra/qt_recipe"
	"github.com/watermint/toolbox/quality/infra/qt_runtime"
	"github.com/watermint/toolbox/recipe/dev/test"
	"go.uber.org/zap"
	"golang.org/x/text/language/display"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

var (
	ErrorBuildIsNotReadyForRelease = errors.New("the build does not satisfy release criteria")
	ErrorNoArtifactFound           = errors.New("no artifact found")
)

type Publish struct {
	TestResource              string
	Branch                    string
	SkipTests                 bool
	ArtifactPath              mo_path2.FileSystemPath
	ConnGithub                gh_conn.ConnGithubRepo
	HeadingReleaseTheme       app_msg.Message
	HeadingChanges            app_msg.Message
	ListSpecChange            app_msg.Message
	HeadingDocument           app_msg.Message
	ListReadme                app_msg.Message
	HeadingBinary             app_msg.Message
	BinaryTableHeaderFilename app_msg.Message
	BinaryTableHeaderSize     app_msg.Message
	BinaryTableHeaderMD5      app_msg.Message
	BinaryTableHeaderSHA256   app_msg.Message
	TagCommitMessage          app_msg.Message
	ReleaseName               app_msg.Message
}

type ArtifactSum struct {
	Filename string
	Size     int64
	MD5      string
	SHA256   string
}

func (z *Publish) Preset() {
	z.TestResource = defaultTestResource
	z.Branch = "master"
}

func (z *Publish) artifactAssets(c app_control.Control) (paths []string, sizes map[string]int64, err error) {
	l := c.Log()

	entries, err := ioutil.ReadDir(z.ArtifactPath.Path())
	if err != nil {
		return nil, nil, err
	}
	paths = make([]string, 0)
	sizes = make(map[string]int64)
	for _, e := range entries {
		if !strings.HasPrefix(e.Name(), "tbx-"+app.Version) || !strings.HasSuffix(e.Name(), ".zip") {
			l.Debug("Ignore non artifact file", zap.Any("file", e))
			continue
		}
		path := filepath.Join(z.ArtifactPath.Path(), e.Name())
		paths = append(paths, path)
		sizes[path] = e.Size()
	}
	return paths, sizes, nil
}

func (z *Publish) verifyArtifacts(c app_control.Control) (a []*ArtifactSum, err error) {
	l := c.Log()
	a = make([]*ArtifactSum, 0)

	assets, assetSize, err := z.artifactAssets(c)

	h := ut_filehash.NewHash(l)
	for _, p := range assets {
		sum := &ArtifactSum{
			Filename: filepath.Base(p),
			Size:     assetSize[p],
		}
		sum.MD5, err = h.MD5(p)
		if err != nil {
			l.Debug("Unable to calc MD5", zap.Error(err))
			return nil, err
		}
		sum.SHA256, err = h.SHA256(p)
		if err != nil {
			l.Debug("Unable to calc SHA256", zap.Error(err))
			return nil, err
		}
		a = append(a, sum)
	}
	if len(a) < 1 {
		return nil, ErrorNoArtifactFound
	}
	return a, nil
}

func (z *Publish) releaseNotes(c app_control.Control, sum []*ArtifactSum) (relNote string, err error) {
	l := c.Log()
	baseUrl := "https://github.com/watermint/toolbox/blob/" + app.Version

	var buf bytes.Buffer
	mui := app_ui.NewMarkdown(c.Messages(), &buf, true)
	mui.Header(z.HeadingReleaseTheme)
	mui.Break()

	mui.Header(z.HeadingChanges)

	for _, lang := range app_lang.SupportedLanguages {
		mui.Info(z.ListSpecChange.
			With("Link", baseUrl+"/doc/generated"+app_lang.PathSuffix(lang)+"/changes.md").
			With("Lang", display.Self.Name(lang)),
		)
	}

	mui.Break()
	mui.Header(z.HeadingDocument)

	for _, lang := range app_lang.SupportedLanguages {
		name := "README" + app_lang.PathSuffix(lang) + ".md"
		mui.Info(z.ListReadme.
			With("Name", name).
			With("Link", baseUrl+"/"+name).
			With("Lang", display.Self.Name(lang)),
		)
	}

	mui.Break()
	mui.Header(z.HeadingBinary)
	mit := mui.InfoTable("Binaries")
	mit.Header(z.BinaryTableHeaderFilename, z.BinaryTableHeaderSize, z.BinaryTableHeaderMD5, z.BinaryTableHeaderSHA256)

	for _, s := range sum {
		mit.RowRaw(s.Filename, fmt.Sprintf("%d", s.Size), s.MD5, s.SHA256)
	}
	mit.Flush()

	relNotesPath := filepath.Join(c.Workspace().Report(), "release_notes.md")
	err = ioutil.WriteFile(relNotesPath, buf.Bytes(), 0644)
	if err != nil {
		l.Debug("Unable to write release notes", zap.Error(err), zap.String("path", relNotesPath))
		return "", err
	}
	l.Info("Release note created", zap.String("path", relNotesPath))

	return buf.String(), nil
}

func (z *Publish) endToEndTest(c app_control.Control) error {
	l := c.Log()
	if c.Feature().IsTest() {
		l.Info("Skip tests")
		return nil
	}

	if c.Feature().IsProduction() {
		l.Info("Prepare resources")
		if !api_auth_impl.IsCacheAvailable(c, qt_endtoend.EndToEndPeer, []string{
			api_auth.DropboxTokenFull,
			api_auth.DropboxTokenBusinessAudit,
			api_auth.DropboxTokenBusinessManagement,
			api_auth.DropboxTokenBusinessFile,
			api_auth.DropboxTokenBusinessInfo,
		}) {
			return qt_errors.ErrorNotEnoughResource
		}
	}

	l.Info("Ensure end to end resource availability")
	if !dbx_conn_impl.IsEndToEndTokenAllAvailable(c) {
		l.Error("At least one of end to end resource is not available.")
		return errors.New("end to end resource is not available")
	}

	l.Info("Testing all end to end test")
	err := rc_exec.Exec(c, &test.Recipe{}, func(r rc_recipe.Recipe) {
		m := r.(*test.Recipe)
		m.All = true
		_, err := os.Lstat(z.TestResource)
		if err == nil {
			m.Resource = mo_string.NewOptional(z.TestResource)
		} else {
			l.Warn("Unable to read test resource", zap.String("path", z.TestResource), zap.Error(err))
		}
	})
	return err
}

func (z *Publish) ghCtx(c app_control.Control) gh_context.Context {
	if c.Feature().IsTest() {
		return gh_context_impl.NewMock()
	} else {
		return z.ConnGithub.Context()
	}
}

func (z *Publish) createTag(c app_control.Control) error {
	l := c.Log().With(
		zap.String("owner", app.RepositoryOwner),
		zap.String("repository", app.RepositoryName),
		zap.String("version", app.Version),
		zap.String("hash", app.Hash))
	svt := sv_reference.New(z.ghCtx(c), app.RepositoryOwner, app.RepositoryName)
	l.Debug("Create tag")
	tag, err := svt.Create(
		"refs/tags/"+app.Version,
		app.Hash,
	)
	if err != nil && err != qt_errors.ErrorMock {
		l.Debug("Unable to create tag", zap.Error(err))
		return err
	}
	if err == qt_errors.ErrorMock {
		return nil
	}
	l.Info("The tag created", zap.Any("tag", tag))
	return nil
}

func (z *Publish) createReleaseDraft(c app_control.Control, relNote string) (rel *mo_release.Release, err error) {
	l := c.Log().With(
		zap.String("owner", app.RepositoryOwner),
		zap.String("repository", app.RepositoryName),
		zap.String("version", app.Version),
		zap.String("hash", app.Hash))
	ui := c.UI()
	svr := sv_release.New(z.ghCtx(c), app.RepositoryOwner, app.RepositoryName)
	rel, err = svr.CreateDraft(
		app.Version,
		ui.Text(z.ReleaseName.With("Version", app.Version)),
		relNote,
		z.Branch,
	)
	if err != nil && err != qt_errors.ErrorMock {
		l.Debug("Unable to create release draft", zap.Error(err))
		return nil, err
	}
	if err == qt_errors.ErrorMock {
		return &mo_release.Release{}, nil
	}
	l.Info("Release created", zap.Any("rel", rel))
	return rel, nil
}

func (z *Publish) uploadAssets(c app_control.Control, rel *mo_release.Release) error {
	l := c.Log()
	assets, _, err := z.artifactAssets(c)
	if err != nil {
		return err
	}

	sva := sv_release_asset.New(z.ghCtx(c), app.RepositoryOwner, app.RepositoryName, rel.Id)
	for _, p := range assets {
		l.Info("Uploading asset", zap.String("path", p))
		a, err := sva.Upload(mo_path2.NewExistingFileSystemPath(p))
		if err != nil && err != qt_errors.ErrorMock {
			return err
		}
		if err == qt_errors.ErrorMock {
			continue
		}
		l.Info("Uploaded", zap.Any("asset", a))
	}
	return nil
}

func (z *Publish) Exec(c app_control.Control) error {
	l := c.Log()
	ready := true

	if app.IsProduction() {
		l.Info("Verify embedded resources")
		qt_runtime.Suite(c)
	} else {
		l.Info("Run as dev mode")
		ready = false
	}

	if z.SkipTests {
		ready = false
	} else {
		err := z.endToEndTest(c)
		if err != nil {
			return err
		}
	}

	sum, err := z.verifyArtifacts(c)
	if err != nil {
		return err
	}

	relNote, err := z.releaseNotes(c, sum)
	if err != nil {
		return err
	}

	if err := z.createTag(c); err != nil {
		return nil
	}

	rel, err := z.createReleaseDraft(c, relNote)
	if err != nil {
		return err
	}

	if err := z.uploadAssets(c, rel); err != nil {
		return err
	}

	if !ready {
		l.Warn("The build does not satisfy release criteria")
		return ErrorBuildIsNotReadyForRelease
	}
	l.Info("The build is ready to publish")
	return nil
}

func (z *Publish) Test(c app_control.Control) error {
	d, err := qt_file.MakeTestFolder("release-publish", false)
	if err != nil {
		return err
	}

	platforms := []string{"linux", "mac", "win"}
	for _, platform := range platforms {
		app.Version = "dev-test"
		err = ioutil.WriteFile(filepath.Join(d, "tbx-"+app.Version+"-"+platform+".zip"), []byte("Test artifact"), 0644)
		if err != nil {
			c.Log().Warn("Unable to create test artifact", zap.Error(err))
			return err
		}
	}
	defer os.RemoveAll(d)

	err = rc_exec.ExecMock(c, &Publish{}, func(r rc_recipe.Recipe) {
		m := r.(*Publish)
		m.ArtifactPath = mo_path2.NewFileSystemPath(d)
	})
	if err, _ = qt_recipe.RecipeError(c.Log(), err); err != ErrorBuildIsNotReadyForRelease && err != nil {
		return err
	}
	return nil
}
<commit_msg>#333 : always generate release notes in English<commit_after>package release

import (
	"bytes"
	"errors"
	"fmt"
	mo_path2 "github.com/watermint/toolbox/domain/common/model/mo_path"
	"github.com/watermint/toolbox/domain/common/model/mo_string"
	"github.com/watermint/toolbox/domain/dropbox/api/dbx_conn_impl"
	"github.com/watermint/toolbox/domain/github/api/gh_conn"
	"github.com/watermint/toolbox/domain/github/api/gh_context"
	"github.com/watermint/toolbox/domain/github/api/gh_context_impl"
	"github.com/watermint/toolbox/domain/github/model/mo_release"
	"github.com/watermint/toolbox/domain/github/service/sv_reference"
	"github.com/watermint/toolbox/domain/github/service/sv_release"
	"github.com/watermint/toolbox/domain/github/service/sv_release_asset"
	"github.com/watermint/toolbox/infra/api/api_auth"
	"github.com/watermint/toolbox/infra/api/api_auth_impl"
	"github.com/watermint/toolbox/infra/app"
	"github.com/watermint/toolbox/infra/control/app_control"
	"github.com/watermint/toolbox/infra/control/app_control_launcher"
	"github.com/watermint/toolbox/infra/recipe/rc_exec"
	"github.com/watermint/toolbox/infra/recipe/rc_recipe"
	"github.com/watermint/toolbox/infra/ui/app_lang"
	"github.com/watermint/toolbox/infra/ui/app_msg"
	"github.com/watermint/toolbox/infra/ui/app_ui"
	"github.com/watermint/toolbox/infra/util/ut_filehash"
	"github.com/watermint/toolbox/quality/infra/qt_endtoend"
	"github.com/watermint/toolbox/quality/infra/qt_errors"
	"github.com/watermint/toolbox/quality/infra/qt_file"
	"github.com/watermint/toolbox/quality/infra/qt_recipe"
	"github.com/watermint/toolbox/quality/infra/qt_runtime"
	"github.com/watermint/toolbox/recipe/dev/test"
	"go.uber.org/zap"
	"golang.org/x/text/language/display"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

var (
	ErrorBuildIsNotReadyForRelease = errors.New("the build does not satisfy release criteria")
	ErrorNoArtifactFound           = errors.New("no artifact found")
)

type Publish struct {
	TestResource              string
	Branch                    string
	SkipTests                 bool
	ArtifactPath              mo_path2.FileSystemPath
	ConnGithub                gh_conn.ConnGithubRepo
	HeadingReleaseTheme       app_msg.Message
	HeadingChanges            app_msg.Message
	ListSpecChange            app_msg.Message
	HeadingDocument           app_msg.Message
	ListReadme                app_msg.Message
	HeadingBinary             app_msg.Message
	BinaryTableHeaderFilename app_msg.Message
	BinaryTableHeaderSize     app_msg.Message
	BinaryTableHeaderMD5      app_msg.Message
	BinaryTableHeaderSHA256   app_msg.Message
	TagCommitMessage          app_msg.Message
	ReleaseName               app_msg.Message
}

type ArtifactSum struct {
	Filename string
	Size     int64
	MD5      string
	SHA256   string
}

func (z *Publish) Preset() {
	z.TestResource = defaultTestResource
	z.Branch = "master"
}

func (z *Publish) artifactAssets(c app_control.Control) (paths []string, sizes map[string]int64, err error) {
	l := c.Log()

	entries, err := ioutil.ReadDir(z.ArtifactPath.Path())
	if err != nil {
		return nil, nil, err
	}
	paths = make([]string, 0)
	sizes = make(map[string]int64)
	for _, e := range entries {
		if !strings.HasPrefix(e.Name(), "tbx-"+app.Version) || !strings.HasSuffix(e.Name(), ".zip") {
			l.Debug("Ignore non artifact file", zap.Any("file", e))
			continue
		}
		path := filepath.Join(z.ArtifactPath.Path(), e.Name())
		paths = append(paths, path)
		sizes[path] = e.Size()
	}
	return paths, sizes, nil
}

func (z *Publish) verifyArtifacts(c app_control.Control) (a []*ArtifactSum, err error) {
	l := c.Log()
	a = make([]*ArtifactSum, 0)

	assets, assetSize, err := z.artifactAssets(c)

	h := ut_filehash.NewHash(l)
	for _, p := range assets {
		sum := &ArtifactSum{
			Filename: filepath.Base(p),
			Size:     assetSize[p],
		}
		sum.MD5, err = h.MD5(p)
		if err != nil {
			l.Debug("Unable to calc MD5", zap.Error(err))
			return nil, err
		}
		sum.SHA256, err = h.SHA256(p)
		if err != nil {
			l.Debug("Unable to calc SHA256", zap.Error(err))
			return nil, err
		}
		a = append(a, sum)
	}
	if len(a) < 1 {
		return nil, ErrorNoArtifactFound
	}
	return a, nil
}

func (z *Publish) releaseNotes(c app_control.Control, sum []*ArtifactSum) (relNote string, err error) {
	if cl, ok := app_control_launcher.ControlWithLang("en", c); ok {
		c = cl
	}

	l := c.Log()
	baseUrl := "https://github.com/watermint/toolbox/blob/" + app.Version

	var buf bytes.Buffer
	mui := app_ui.NewMarkdown(c.Messages(), &buf, true)
	mui.Header(z.HeadingReleaseTheme)
	mui.Break()

	mui.Header(z.HeadingChanges)

	for _, lang := range app_lang.SupportedLanguages {
		mui.Info(z.ListSpecChange.
			With("Link", baseUrl+"/doc/generated"+app_lang.PathSuffix(lang)+"/changes.md").
			With("Lang", display.Self.Name(lang)),
		)
	}

	mui.Break()
	mui.Header(z.HeadingDocument)

	for _, lang := range app_lang.SupportedLanguages {
		name := "README" + app_lang.PathSuffix(lang) + ".md"
		mui.Info(z.ListReadme.
			With("Name", name).
			With("Link", baseUrl+"/"+name).
			With("Lang", display.Self.Name(lang)),
		)
	}

	mui.Break()
	mui.Header(z.HeadingBinary)
	mit := mui.InfoTable("Binaries")
	mit.Header(z.BinaryTableHeaderFilename, z.BinaryTableHeaderSize, z.BinaryTableHeaderMD5, z.BinaryTableHeaderSHA256)

	for _, s := range sum {
		mit.RowRaw(s.Filename, fmt.Sprintf("%d", s.Size), s.MD5, s.SHA256)
	}
	mit.Flush()

	relNotesPath := filepath.Join(c.Workspace().Report(), "release_notes.md")
	err = ioutil.WriteFile(relNotesPath, buf.Bytes(), 0644)
	if err != nil {
		l.Debug("Unable to write release notes", zap.Error(err), zap.String("path", relNotesPath))
		return "", err
	}
	l.Info("Release note created", zap.String("path", relNotesPath))

	return buf.String(), nil
}

func (z *Publish) endToEndTest(c app_control.Control) error {
	l := c.Log()
	if c.Feature().IsTest() {
		l.Info("Skip tests")
		return nil
	}

	if c.Feature().IsProduction() {
		l.Info("Prepare resources")
		if !api_auth_impl.IsCacheAvailable(c, qt_endtoend.EndToEndPeer, []string{
			api_auth.DropboxTokenFull,
			api_auth.DropboxTokenBusinessAudit,
			api_auth.DropboxTokenBusinessManagement,
			api_auth.DropboxTokenBusinessFile,
			api_auth.DropboxTokenBusinessInfo,
		}) {
			return qt_errors.ErrorNotEnoughResource
		}
	}

	l.Info("Ensure end to end resource availability")
	if !dbx_conn_impl.IsEndToEndTokenAllAvailable(c) {
		l.Error("At least one of end to end resource is not available.")
		return errors.New("end to end resource is not available")
	}

	l.Info("Testing all end to end test")
	err := rc_exec.Exec(c, &test.Recipe{}, func(r rc_recipe.Recipe) {
		m := r.(*test.Recipe)
		m.All = true
		_, err := os.Lstat(z.TestResource)
		if err == nil {
			m.Resource = mo_string.NewOptional(z.TestResource)
		} else {
			l.Warn("Unable to read test resource", zap.String("path", z.TestResource), zap.Error(err))
		}
	})
	return err
}

func (z *Publish) ghCtx(c app_control.Control) gh_context.Context {
	if c.Feature().IsTest() {
		return gh_context_impl.NewMock()
	} else {
		return z.ConnGithub.Context()
	}
}

func (z *Publish) createTag(c app_control.Control) error {
	l := c.Log().With(
		zap.String("owner", app.RepositoryOwner),
		zap.String("repository", app.RepositoryName),
		zap.String("version", app.Version),
		zap.String("hash", app.Hash))
	svt := sv_reference.New(z.ghCtx(c), app.RepositoryOwner, app.RepositoryName)
	l.Debug("Create tag")
	tag, err := svt.Create(
		"refs/tags/"+app.Version,
		app.Hash,
	)
	if err != nil && err != qt_errors.ErrorMock {
		l.Debug("Unable to create tag", zap.Error(err))
		return err
	}
	if err == qt_errors.ErrorMock {
		return nil
	}
	l.Info("The tag created", zap.Any("tag", tag))
	return nil
}

func (z *Publish) createReleaseDraft(c app_control.Control, relNote string) (rel *mo_release.Release, err error) {
	l := c.Log().With(
		zap.String("owner", app.RepositoryOwner),
		zap.String("repository", app.RepositoryName),
		zap.String("version", app.Version),
		zap.String("hash", app.Hash))
	ui := c.UI()
	svr := sv_release.New(z.ghCtx(c), app.RepositoryOwner, app.RepositoryName)
	rel, err = svr.CreateDraft(
		app.Version,
		ui.Text(z.ReleaseName.With("Version", app.Version)),
		relNote,
		z.Branch,
	)
	if err != nil && err != qt_errors.ErrorMock {
		l.Debug("Unable to create release draft", zap.Error(err))
		return nil, err
	}
	if err == qt_errors.ErrorMock {
		return &mo_release.Release{}, nil
	}
	l.Info("Release created", zap.Any("rel", rel))
	return rel, nil
}

func (z *Publish) uploadAssets(c app_control.Control, rel *mo_release.Release) error {
	l := c.Log()
	assets, _, err := z.artifactAssets(c)
	if err != nil {
		return err
	}

	sva := sv_release_asset.New(z.ghCtx(c), app.RepositoryOwner, app.RepositoryName, rel.Id)
	for _, p := range assets {
		l.Info("Uploading asset", zap.String("path", p))
		a, err := sva.Upload(mo_path2.NewExistingFileSystemPath(p))
		if err != nil && err != qt_errors.ErrorMock {
			return err
		}
		if err == qt_errors.ErrorMock {
			continue
		}
		l.Info("Uploaded", zap.Any("asset", a))
	}
	return nil
}

func (z *Publish) Exec(c app_control.Control) error {
	l := c.Log()
	ready := true

	if app.IsProduction() {
		l.Info("Verify embedded resources")
		qt_runtime.Suite(c)
	} else {
		l.Info("Run as dev mode")
		ready = false
	}

	if z.SkipTests {
		ready = false
	} else {
		err := z.endToEndTest(c)
		if err != nil {
			return err
		}
	}

	sum, err := z.verifyArtifacts(c)
	if err != nil {
		return err
	}

	relNote, err := z.releaseNotes(c, sum)
	if err != nil {
		return err
	}

	if err := z.createTag(c); err != nil {
		return nil
	}

	rel, err := z.createReleaseDraft(c, relNote)
	if err != nil {
		return err
	}

	if err := z.uploadAssets(c, rel); err != nil {
		return err
	}

	if !ready {
		l.Warn("The build does not satisfy release criteria")
		return ErrorBuildIsNotReadyForRelease
	}
	l.Info("The build is ready to publish")
	return nil
}

func (z *Publish) Test(c app_control.Control) error {
	d, err := qt_file.MakeTestFolder("release-publish", false)
	if err != nil {
		return err
	}

	platforms := []string{"linux", "mac", "win"}
	for _, platform := range platforms {
		app.Version = "dev-test"
		err = ioutil.WriteFile(filepath.Join(d, "tbx-"+app.Version+"-"+platform+".zip"), []byte("Test artifact"), 0644)
		if err != nil {
			c.Log().Warn("Unable to create test artifact", zap.Error(err))
			return err
		}
	}
	defer os.RemoveAll(d)

	err = rc_exec.ExecMock(c, &Publish{}, func(r rc_recipe.Recipe) {
		m := r.(*Publish)
		m.ArtifactPath = mo_path2.NewFileSystemPath(d)
	})
	if err, _ = qt_recipe.RecipeError(c.Log(), err); err != ErrorBuildIsNotReadyForRelease && err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package controller

import (
	"errors"
	"fmt"
	mongomodels "koding/db/models"
	"koding/db/mongodb/modelhelper"
	"socialapi/models"
	"strings"

	"github.com/VerbalExpressions/GoVerbalExpressions"
	"github.com/koding/logging"
	"labix.org/v2/mgo/bson"
)

var (
	ErrMigrated     = errors.New("already migrated")
	kodingChannelId int64
	tagRegex        = verbalexpressions.New().
			BeginCapture().
			Find("|#:JTag:").
			Anything().
			Then(":").
			Anything().
			Then("|").
			EndCapture().
			Regex()
)

type Controller struct {
	log logging.Logger
}

func New(log logging.Logger) (*Controller, error) {
	wc := &Controller{
		log: log,
	}

	return wc, nil
}

func (mwc *Controller) Start() error {
	if err := mwc.migrateAllGroups(); err != nil {
		return err
	}

	if err := mwc.migrateAllTags(); err != nil {
		return err
	}

	if err := mwc.migrateAllPosts(); err != nil {
		return err
	}

	return nil
}

func (mwc *Controller) migrateAllPosts() error {
	o := modelhelper.Options{
		Sort: "meta.createdAt",
	}
	s := modelhelper.Selector{
		"socialMessageId": modelhelper.Selector{"$exists": false},
	}
	kodingChannel, err := mwc.createGroupChannel("koding")
	if err != nil {
		return fmt.Errorf("Koding channel cannot be created: %s", err)
	}
	kodingChannelId = kodingChannel.Id

	errCount := 0
	successCount := 0

	handleError := func(su *mongomodels.StatusUpdate, err error) {
		mwc.log.Error("an error occured for %s: %s", su.Id.Hex(), err)
		errCount++
	}

	iter := modelhelper.GetStatusUpdateIter(s, o)
	defer iter.Close()

	var su mongomodels.StatusUpdate
	for iter.Next(&su) {
		channelId, err := mwc.fetchGroupChannelId(su.Group)
		if err != nil {
			return fmt.Errorf("Post migration is interrupted with %d errors: channel id cannot be fetched :%s", errCount, err)
		}

		// create channel message
		cm := mapStatusUpdateToChannelMessage(&su)
		cm.InitialChannelId = channelId
		if err := insertChannelMessage(cm, su.OriginId.Hex()); err != nil {
			handleError(&su, err)
			continue
		}

		if err := addChannelMessageToMessageList(cm); err != nil {
			handleError(&su, err)
			continue
		}

		// create reply messages
		if err := mwc.migrateComments(cm, &su, channelId); err != nil {
			handleError(&su, err)
			continue
		}

		if err := mwc.migrateLikes(cm, su.Id); err != nil {
			handleError(&su, err)
			continue
		}

		if err := mwc.migrateTags(cm, su.Id); err != nil {
			handleError(&su, err)
			continue
		}

		// update mongo status update channelMessageId field
		if err := completePostMigration(&su, cm); err != nil {
			handleError(&su, err)
			continue
		}
		successCount++
	}

	if err := iter.Err(); err != nil {
		return fmt.Errorf("Post migration is interrupted with %d errors: %s", errCount, err)
	}

	mwc.log.Notice("Post migration completed for %d status updates with %d errors", successCount, errCount)

	return nil
}

func insertChannelMessage(cm *models.ChannelMessage, accountOldId string) error {

	if err := prepareMessageAccount(cm, accountOldId); err != nil {
		return err
	}

	if err := cm.CreateRaw(); err != nil {
		return err
	}

	return nil
}

func addChannelMessageToMessageList(cm *models.ChannelMessage) error {
	cml := models.NewChannelMessageList()
	cml.ChannelId = cm.InitialChannelId
	cml.MessageId = cm.Id
	cml.AddedAt = cm.CreatedAt

	return cml.CreateRaw()
}

func (mwc *Controller) migrateComments(parentMessage *models.ChannelMessage, su *mongomodels.StatusUpdate, channelId int64) error {

	s := modelhelper.Selector{
		"sourceId":   su.Id,
		"targetName": "JComment",
	}
	rels, err := modelhelper.GetAllRelationships(s)
	if err != nil {
		if err == modelhelper.ErrNotFound {
			return nil
		}
		return fmt.Errorf("comment relationships cannot be fetched: %s", err)
	}

	for _, r := range rels {
		comment, err := modelhelper.GetCommentById(r.TargetId.Hex())
		if err != nil {
			return fmt.Errorf("comment cannot be fetched %s", err)
		}
		// comment is already migrated
		if comment.SocialMessageId != 0 {
			continue
		}

		reply := mapCommentToChannelMessage(comment)
		reply.InitialChannelId = channelId
		// insert as channel message
		if err := insertChannelMessage(reply, comment.OriginId.Hex()); err != nil {
			return fmt.Errorf("comment cannot be inserted %s", err)
		}

		// insert as message reply
		mr := models.NewMessageReply()
		mr.MessageId = parentMessage.Id
		mr.ReplyId = reply.Id
		mr.CreatedAt = reply.CreatedAt
		if err := mr.CreateRaw(); err != nil {
			return fmt.Errorf("comment cannot be inserted to message reply %s", err)
		}

		if err := mwc.migrateLikes(reply, comment.Id); err != nil {
			return fmt.Errorf("likes cannot be migrated %s", err)
		}

		if err := completeCommentMigration(comment, reply); err != nil {
			return fmt.Errorf("old comment cannot be flagged with new message id %s", err)
		}
	}

	return nil
}

func (mwc *Controller) migrateLikes(cm *models.ChannelMessage, oldId bson.ObjectId) error {
	s := modelhelper.Selector{
		"sourceId": oldId,
		"as":       "like",
	}
	rels, err := modelhelper.GetAllRelationships(s)
	if err != nil {
		return fmt.Errorf("likes cannot be fetched %s", err)
	}
	for _, r := range rels {
		a := models.NewAccount()
		a.OldId = r.TargetId.Hex()
		if err := a.FetchOrCreate(); err != nil {
			mwc.log.Error("interactor account could not found: %s", err)
			continue
		}
		i := models.NewInteraction()
		i.MessageId = cm.Id
		i.AccountId = a.Id
		i.TypeConstant = models.Interaction_TYPE_LIKE
		i.CreatedAt = r.TimeStamp
		if err := i.CreateRaw(); err != nil {
			mwc.log.Error("interaction could not created: %s", err)
		}
	}

	return nil
}

func prepareMessageAccount(cm *models.ChannelMessage, accountOldId string) error {
	a := models.NewAccount()
	a.OldId = accountOldId
	if err := a.FetchOrCreate(); err != nil {
		return fmt.Errorf("account could not found: %s", err)
	}

	cm.AccountId = a.Id

	return nil
}

func (mwc *Controller) fetchGroupChannelId(groupName string) (int64, error) {
	// koding group channel id is prefetched
	if groupName == "koding" {
		return kodingChannelId, nil
	}

	c := models.NewChannel()
	channelId, err := c.FetchChannelIdByNameAndGroupName(groupName, groupName)
	if err != nil {
		return 0, err
	}

	return channelId, nil
}

func mapStatusUpdateToChannelMessage(su *mongomodels.StatusUpdate) *models.ChannelMessage {
	cm := models.NewChannelMessage()
	cm.Slug = su.Slug
	prepareBody(cm, su.Body)
	cm.TypeConstant = models.ChannelMessage_TYPE_POST
	cm.CreatedAt = su.Meta.CreatedAt
	prepareMessageMetaDates(cm, &su.Meta)

	return cm
}

func mapCommentToChannelMessage(c *mongomodels.Comment) *models.ChannelMessage {
	cm := models.NewChannelMessage()
	cm.Body = c.Body
	cm.TypeConstant = models.ChannelMessage_TYPE_REPLY
	cm.CreatedAt = c.Meta.CreatedAt
	prepareMessageMetaDates(cm, &c.Meta)

	return cm
}

func prepareMessageMetaDates(cm *models.ChannelMessage, meta *mongomodels.Meta) {
	// this is added because status update->modified at field is before createdAt
	if cm.CreatedAt.After(meta.ModifiedAt) {
		cm.UpdatedAt = cm.CreatedAt
	} else {
		cm.UpdatedAt = meta.ModifiedAt
	}
}

func prepareBody(cm *models.ChannelMessage, body string) {
	res := tagRegex.FindAllStringSubmatch(body, -1)
	cm.Body = body
	if len(res) == 0 {
		return
	}

	for _, element := range res {
		tag := element[1][1 : len(element[1])-1]
		tag = strings.Split(tag, ":")[3]
		tag = "#" + tag
		cm.Body = verbalexpressions.New().Find(element[1]).Replace(cm.Body, tag)
	}

}

func completePostMigration(su *mongomodels.StatusUpdate, cm *models.ChannelMessage) error {
	su.SocialMessageId = cm.Id

	return modelhelper.UpdateStatusUpdate(su)
}

func completeCommentMigration(reply *mongomodels.Comment, cm *models.ChannelMessage) error {
	reply.SocialMessageId = cm.Id

	return modelhelper.UpdateComment(reply)
}
<commit_msg>Social: Migrate embedly links<commit_after>package controller

import (
	"encoding/json"
	"errors"
	"fmt"
	mongomodels "koding/db/models"
	"koding/db/mongodb/modelhelper"
	"reflect"
	"socialapi/models"
	"strings"

	"github.com/VerbalExpressions/GoVerbalExpressions"
	"github.com/koding/logging"
	"labix.org/v2/mgo/bson"
)

var (
	ErrMigrated     = errors.New("already migrated")
	kodingChannelId int64
	tagRegex        = verbalexpressions.New().
			BeginCapture().
			Find("|#:JTag:").
			Anything().
			Then(":").
			Anything().
			Then("|").
			EndCapture().
			Regex()
)

type Controller struct {
	log logging.Logger
}

func New(log logging.Logger) (*Controller, error) {
	wc := &Controller{
		log: log,
	}

	return wc, nil
}

func (mwc *Controller) Start() error {
	if err := mwc.migrateAllGroups(); err != nil {
		return err
	}

	if err := mwc.migrateAllTags(); err != nil {
		return err
	}

	if err := mwc.migrateAllPosts(); err != nil {
		return err
	}

	return nil
}

func (mwc *Controller) migrateAllPosts() error {
	o := modelhelper.Options{
		Sort: "meta.createdAt",
	}
	s := modelhelper.Selector{
		"socialMessageId": modelhelper.Selector{"$exists": false},
	}
	kodingChannel, err := mwc.createGroupChannel("koding")
	if err != nil {
		return fmt.Errorf("Koding channel cannot be created: %s", err)
	}
	kodingChannelId = kodingChannel.Id

	errCount := 0
	successCount := 0

	handleError := func(su *mongomodels.StatusUpdate, err error) {
		mwc.log.Error("an error occured for %s: %s", su.Id.Hex(), err)
		errCount++
	}

	iter := modelhelper.GetStatusUpdateIter(s, o)
	defer iter.Close()

	var su mongomodels.StatusUpdate
	for iter.Next(&su) {
		channelId, err := mwc.fetchGroupChannelId(su.Group)
		if err != nil {
			return fmt.Errorf("Post migration is interrupted with %d errors: channel id cannot be fetched :%s", errCount, err)
		}

		// create channel message
		cm, err := mapStatusUpdateToChannelMessage(&su)
		if err != nil {
			handleError(&su, err)
			continue
		}

		cm.InitialChannelId = channelId
		if err := insertChannelMessage(cm, su.OriginId.Hex()); err != nil {
			handleError(&su, err)
			continue
		}

		if err := addChannelMessageToMessageList(cm); err != nil {
			handleError(&su, err)
			continue
		}

		// create reply messages
		if err := mwc.migrateComments(cm, &su, channelId); err != nil {
			handleError(&su, err)
			continue
		}

		if err := mwc.migrateLikes(cm, su.Id); err != nil {
			handleError(&su, err)
			continue
		}

		if err := mwc.migrateTags(cm, su.Id); err != nil {
			handleError(&su, err)
			continue
		}

		// update mongo status update channelMessageId field
		if err := completePostMigration(&su, cm); err != nil {
			handleError(&su, err)
			continue
		}
		successCount++
	}

	if err := iter.Err(); err != nil {
		return fmt.Errorf("Post migration is interrupted with %d errors: %s", errCount, err)
	}

	mwc.log.Notice("Post migration completed for %d status updates with %d errors", successCount, errCount)

	return nil
}

func insertChannelMessage(cm *models.ChannelMessage, accountOldId string) error {

	if err := prepareMessageAccount(cm, accountOldId); err != nil {
		return err
	}

	if err := cm.CreateRaw(); err != nil {
		return err
	}

	return nil
}

func addChannelMessageToMessageList(cm *models.ChannelMessage) error {
	cml := models.NewChannelMessageList()
	cml.ChannelId = cm.InitialChannelId
	cml.MessageId = cm.Id
	cml.AddedAt = cm.CreatedAt

	return cml.CreateRaw()
}

func (mwc *Controller) migrateComments(parentMessage *models.ChannelMessage, su *mongomodels.StatusUpdate, channelId int64) error {

	s := modelhelper.Selector{
		"sourceId":   su.Id,
		"targetName": "JComment",
	}
	rels, err := modelhelper.GetAllRelationships(s)
	if err != nil {
		if err == modelhelper.ErrNotFound {
			return nil
		}
		return fmt.Errorf("comment relationships cannot be fetched: %s", err)
	}

	for _, r := range rels {
		comment, err := modelhelper.GetCommentById(r.TargetId.Hex())
		if err != nil {
			return fmt.Errorf("comment cannot be fetched %s", err)
		}
		// comment is already migrated
		if comment.SocialMessageId != 0 {
			continue
		}

		reply := mapCommentToChannelMessage(comment)
		reply.InitialChannelId = channelId
		// insert as channel message
		if err := insertChannelMessage(reply, comment.OriginId.Hex()); err != nil {
			return fmt.Errorf("comment cannot be inserted %s", err)
		}

		// insert as message reply
		mr := models.NewMessageReply()
		mr.MessageId = parentMessage.Id
		mr.ReplyId = reply.Id
		mr.CreatedAt = reply.CreatedAt
		if err := mr.CreateRaw(); err != nil {
			return fmt.Errorf("comment cannot be inserted to message reply %s", err)
		}

		if err := mwc.migrateLikes(reply, comment.Id); err != nil {
			return fmt.Errorf("likes cannot be migrated %s", err)
		}

		if err := completeCommentMigration(comment, reply); err != nil {
			return fmt.Errorf("old comment cannot be flagged with new message id %s", err)
		}
	}

	return nil
}

func (mwc *Controller) migrateLikes(cm *models.ChannelMessage, oldId bson.ObjectId) error {
	s := modelhelper.Selector{
		"sourceId": oldId,
		"as":       "like",
	}
	rels, err := modelhelper.GetAllRelationships(s)
	if err != nil {
		return fmt.Errorf("likes cannot be fetched %s", err)
	}
	for _, r := range rels {
		a := models.NewAccount()
		a.OldId = r.TargetId.Hex()
		if err := a.FetchOrCreate(); err != nil {
			mwc.log.Error("interactor account could not found: %s", err)
			continue
		}
		i := models.NewInteraction()
		i.MessageId = cm.Id
		i.AccountId = a.Id
		i.TypeConstant = models.Interaction_TYPE_LIKE
		i.CreatedAt = r.TimeStamp
		if err := i.CreateRaw(); err != nil {
			mwc.log.Error("interaction could not created: %s", err)
		}
	}

	return nil
}

func prepareMessageAccount(cm *models.ChannelMessage, accountOldId string) error {
	a := models.NewAccount()
	a.OldId = accountOldId
	if err := a.FetchOrCreate(); err != nil {
		return fmt.Errorf("account could not found: %s", err)
	}

	cm.AccountId = a.Id

	return nil
}

func (mwc *Controller) fetchGroupChannelId(groupName string) (int64, error) {
	// koding group channel id is prefetched
	if groupName == "koding" {
		return kodingChannelId, nil
	}

	c := models.NewChannel()
	channelId, err := c.FetchChannelIdByNameAndGroupName(groupName, groupName)
	if err != nil {
		return 0, err
	}

	return channelId, nil
}

func mapStatusUpdateToChannelMessage(su *mongomodels.StatusUpdate) (*models.ChannelMessage, error) {
	cm := models.NewChannelMessage()
	cm.Slug = su.Slug
	prepareBody(cm, su.Body)
	cm.TypeConstant = models.ChannelMessage_TYPE_POST
	cm.CreatedAt = su.Meta.CreatedAt
	payload, err := mapEmbeddedLink(su.Link)
	if err != nil {
		return nil, err
	}
	cm.Payload = payload

	prepareMessageMetaDates(cm, &su.Meta)

	return cm, nil
}

func mapEmbeddedLink(link map[string]interface{}) (map[string]*string, error) {
	resultMap := make(map[string]*string)
	for key, value := range link {
		// when value is a map, then marshal it
		if reflect.ValueOf(value).Kind() == reflect.Map {
			res, err := json.Marshal(value)
			if err != nil {
				return nil, err
			}

			s := string(res)
			resultMap[key] = &s
			continue
		}

		// for the other types convert value to string
		str := fmt.Sprintf("%v", value)
		resultMap[key] = &str
	}

	return resultMap, nil
}

func mapCommentToChannelMessage(c *mongomodels.Comment) *models.ChannelMessage {
	cm := models.NewChannelMessage()
	cm.Body = c.Body
	cm.TypeConstant = models.ChannelMessage_TYPE_REPLY
	cm.CreatedAt = c.Meta.CreatedAt
	prepareMessageMetaDates(cm, &c.Meta)

	return cm
}

func prepareMessageMetaDates(cm *models.ChannelMessage, meta *mongomodels.Meta) {
	// this is added because status update->modified at field is before createdAt
	if cm.CreatedAt.After(meta.ModifiedAt) {
		cm.UpdatedAt = cm.CreatedAt
	} else {
		cm.UpdatedAt = meta.ModifiedAt
	}
}

func prepareBody(cm *models.ChannelMessage, body string) {
	res := tagRegex.FindAllStringSubmatch(body, -1)
	cm.Body = body
	if len(res) == 0 {
		return
	}

	for _, element := range res {
		tag := element[1][1 : len(element[1])-1]
		tag = strings.Split(tag, ":")[3]
		tag = "#" + tag
		cm.Body = verbalexpressions.New().Find(element[1]).Replace(cm.Body, tag)
	}

}

func completePostMigration(su *mongomodels.StatusUpdate, cm *models.ChannelMessage) error {
	su.SocialMessageId = cm.Id

	return modelhelper.UpdateStatusUpdate(su)
}

func completeCommentMigration(reply *mongomodels.Comment, cm *models.ChannelMessage) error {
	reply.SocialMessageId = cm.Id

	return modelhelper.UpdateComment(reply)
}
<|endoftext|>


<commit_before>/*
Copyright 2021 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mccontroller

import (
	v1 "k8s.io/api/core/v1"
	storagev1 "k8s.io/api/storage/v1"
	"k8s.io/apimachinery/pkg/runtime"
)

type ObjCRF = func(objectType runtime.Object) runtime.Object

var TargetObjCR [](ObjCRF)
var TargetObjCRList [](ObjCRF)

func AddTargetObjCR(objCRF ObjCRF) {
	TargetObjCR = append(TargetObjCR, objCRF)
}

func AddTargetObjCRList(objCRF ObjCRF) {
	TargetObjCRList = append(TargetObjCRList, objCRF)
}

func getTargetObject(objectType runtime.Object) runtime.Object {
	for _, f := range TargetObjCR {
		ro := f(objectType)
		if ro != nil {
			return ro
		}
	}
	return nil
}

func getTargetObjectList(objectType runtime.Object) runtime.Object {
	for _, f := range TargetObjCRList {
		ro := f(objectType)
		if ro != nil {
			return ro
		}
	}
	return nil
}

func init() {
	AddTargetObjCR(getTargetObj)
	AddTargetObjCRList(getTargetObjList)
}

func getTargetObj(objectType runtime.Object) runtime.Object {
	switch objectType.(type) {
	case *v1.ConfigMap:
		return &v1.ConfigMap{}
	case *v1.Namespace:
		return &v1.Namespace{}
	case *v1.Node:
		return &v1.Node{}
	case *v1.Event:
		return &v1.Event{}
	case *v1.Pod:
		return &v1.Pod{}
	case *v1.Secret:
		return &v1.Secret{}
	case *v1.Service:
		return &v1.Service{}
	case *v1.ServiceAccount:
		return &v1.ServiceAccount{}
	case *storagev1.StorageClass:
		return &storagev1.StorageClass{}
	case *v1.PersistentVolumeClaim:
		return &v1.PersistentVolumeClaim{}
	case *v1.PersistentVolume:
		return &v1.PersistentVolume{}
	case *v1.Endpoints:
		return &v1.Endpoints{}
	default:
		return nil
	}
}

func getTargetObjList(objectType runtime.Object) runtime.Object {
	switch objectType.(type) {
	case *v1.ConfigMap:
		return &v1.ConfigMapList{}
	case *v1.Namespace:
		return &v1.NamespaceList{}
	case *v1.Node:
		return &v1.NodeList{}
	case *v1.Event:
		return &v1.EventList{}
	case *v1.Pod:
		return &v1.PodList{}
	case *v1.Secret:
		return &v1.SecretList{}
	case *v1.Service:
		return &v1.ServiceList{}
	case *v1.ServiceAccount:
		return &v1.ServiceAccountList{}
	case *storagev1.StorageClass:
		return &storagev1.StorageClassList{}
	case *v1.PersistentVolumeClaim:
		return &v1.PersistentVolumeClaimList{}
	case *v1.PersistentVolume:
		return &v1.PersistentVolumeList{}
	case *v1.Endpoints:
		return &v1.EndpointsList{}
	default:
		return nil
	}
}
<commit_msg>update to add 2 missing object<commit_after>/*
Copyright 2021 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mccontroller

import (
	v1 "k8s.io/api/core/v1"
	extensionsv1beta1 "k8s.io/api/extensions/v1beta1"
	schedulingv1 "k8s.io/api/scheduling/v1"
	storagev1 "k8s.io/api/storage/v1"
	"k8s.io/apimachinery/pkg/runtime"
)

type ObjCRF = func(objectType runtime.Object) runtime.Object

var TargetObjCR [](ObjCRF)
var TargetObjCRList [](ObjCRF)

func AddTargetObjCR(objCRF ObjCRF) {
	TargetObjCR = append(TargetObjCR, objCRF)
}

func AddTargetObjCRList(objCRF ObjCRF) {
	TargetObjCRList = append(TargetObjCRList, objCRF)
}

func getTargetObject(objectType runtime.Object) runtime.Object {
	for _, f := range TargetObjCR {
		ro := f(objectType)
		if ro != nil {
			return ro
		}
	}
	return nil
}

func getTargetObjectList(objectType runtime.Object) runtime.Object {
	for _, f := range TargetObjCRList {
		ro := f(objectType)
		if ro != nil {
			return ro
		}
	}
	return nil
}

func init() {
	AddTargetObjCR(getTargetObj)
	AddTargetObjCRList(getTargetObjList)
}

func getTargetObj(objectType runtime.Object) runtime.Object {
	switch objectType.(type) {
	case *v1.ConfigMap:
		return &v1.ConfigMap{}
	case *v1.Namespace:
		return &v1.Namespace{}
	case *v1.Node:
		return &v1.Node{}
	case *v1.Event:
		return &v1.Event{}
	case *v1.Pod:
		return &v1.Pod{}
	case *v1.Secret:
		return &v1.Secret{}
	case *v1.Service:
		return &v1.Service{}
	case *v1.ServiceAccount:
		return &v1.ServiceAccount{}
	case *storagev1.StorageClass:
		return &storagev1.StorageClass{}
	case *v1.PersistentVolumeClaim:
		return &v1.PersistentVolumeClaim{}
	case *v1.PersistentVolume:
		return &v1.PersistentVolume{}
	case *v1.Endpoints:
		return &v1.Endpoints{}
	case *schedulingv1.PriorityClass:
		return &schedulingv1.PriorityClass{}
	case *extensionsv1beta1.Ingress:
		return &extensionsv1beta1.Ingress{}
	default:
		return nil
	}
}

func getTargetObjList(objectType runtime.Object) runtime.Object {
	switch objectType.(type) {
	case *v1.ConfigMap:
		return &v1.ConfigMapList{}
	case *v1.Namespace:
		return &v1.NamespaceList{}
	case *v1.Node:
		return &v1.NodeList{}
	case *v1.Event:
		return &v1.EventList{}
	case *v1.Pod:
		return &v1.PodList{}
	case *v1.Secret:
		return &v1.SecretList{}
	case *v1.Service:
		return &v1.ServiceList{}
	case *v1.ServiceAccount:
		return &v1.ServiceAccountList{}
	case *storagev1.StorageClass:
		return &storagev1.StorageClassList{}
	case *v1.PersistentVolumeClaim:
		return &v1.PersistentVolumeClaimList{}
	case *v1.PersistentVolume:
		return &v1.PersistentVolumeList{}
	case *v1.Endpoints:
		return &v1.EndpointsList{}
	case *schedulingv1.PriorityClass:
		return &schedulingv1.PriorityClassList{}
	case *extensionsv1beta1.Ingress:
		return &extensionsv1beta1.IngressList{}
	default:
		return nil
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Ceph-CSI Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package persistentvolume

import (
	"context"
	"errors"
	"fmt"
	"strconv"

	ctrl "github.com/ceph/ceph-csi/internal/controller"
	"github.com/ceph/ceph-csi/internal/rbd"
	"github.com/ceph/ceph-csi/internal/util"
	"github.com/ceph/ceph-csi/internal/util/log"

	corev1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/types"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/controller"
	"sigs.k8s.io/controller-runtime/pkg/handler"
	"sigs.k8s.io/controller-runtime/pkg/manager"
	"sigs.k8s.io/controller-runtime/pkg/reconcile"
	"sigs.k8s.io/controller-runtime/pkg/source"
)

// ReconcilePersistentVolume reconciles a PersistentVolume object.
type ReconcilePersistentVolume struct {
	client client.Client
	config ctrl.Config
	Locks  *util.VolumeLocks
}

var (
	_ reconcile.Reconciler = &ReconcilePersistentVolume{}
	_ ctrl.Manager         = &ReconcilePersistentVolume{}
)

// Init will add the ReconcilePersistentVolume to the list.
func Init() {
	// add ReconcilePersistentVolume to the list
	ctrl.ControllerList = append(ctrl.ControllerList, ReconcilePersistentVolume{})
}

// Add adds the newPVReconciler.
func (r ReconcilePersistentVolume) Add(mgr manager.Manager, config ctrl.Config) error {
	return add(mgr, newPVReconciler(mgr, config))
}

// newReconciler returns a ReconcilePersistentVolume.
func newPVReconciler(mgr manager.Manager, config ctrl.Config) reconcile.Reconciler {
	r := &ReconcilePersistentVolume{
		client: mgr.GetClient(),
		config: config,
		Locks:  util.NewVolumeLocks(),
	}

	return r
}

func add(mgr manager.Manager, r reconcile.Reconciler) error {
	// Create a new controller
	c, err := controller.New(
		"persistentvolume-controller",
		mgr,
		controller.Options{MaxConcurrentReconciles: 1, Reconciler: r})
	if err != nil {
		return err
	}

	// Watch for changes to PersistentVolumes
	err = c.Watch(&source.Kind{Type: &corev1.PersistentVolume{}}, &handler.EnqueueRequestForObject{})
	if err != nil {
		return fmt.Errorf("failed to watch the changes: %w", err)
	}

	return nil
}

func (r *ReconcilePersistentVolume) getCredentials(
	ctx context.Context,
	name,
	namespace string) (*util.Credentials, error) {
	var cr *util.Credentials

	if name == "" || namespace == "" {
		errStr := "secret name or secret namespace is empty"
		log.ErrorLogMsg(errStr)

		return nil, errors.New(errStr)
	}
	secret := &corev1.Secret{}
	err := r.client.Get(ctx,
		types.NamespacedName{Name: name, Namespace: namespace},
		secret)
	if err != nil {
		return nil, fmt.Errorf("error getting secret %s in namespace %s: %w", name, namespace, err)
	}

	credentials := map[string]string{}
	for key, value := range secret.Data {
		credentials[key] = string(value)
	}

	cr, err = util.NewUserCredentials(credentials)
	if err != nil {
		log.ErrorLogMsg("failed to get user credentials %s", err)

		return nil, err
	}

	return cr, nil
}

func checkStaticVolume(pv *corev1.PersistentVolume) (bool, error) {
	static := false
	var err error

	staticVol := pv.Spec.CSI.VolumeAttributes["staticVolume"]
	if staticVol != "" {
		static, err = strconv.ParseBool(staticVol)
		if err != nil {
			return false, fmt.Errorf("failed to parse preProvisionedVolume: %w", err)
		}
	}

	return static, nil
}

// storeVolumeIDInPV stores the new volumeID in PV object.
func (r ReconcilePersistentVolume) storeVolumeIDInPV(
	ctx context.Context,
	pv *corev1.PersistentVolume,
	newVolumeID string) error {
	if v, ok := pv.Annotations[rbd.PVVolumeHandleAnnotationKey]; ok {
		if v == newVolumeID {
			return nil
		}
	}
	if pv.Annotations == nil {
		pv.Annotations = make(map[string]string)
	}
	if pv.Labels == nil {
		pv.Labels = make(map[string]string)
	}
	pv.Labels[rbd.PVReplicatedLabelKey] = rbd.PVReplicatedLabelValue
	pv.Annotations[rbd.PVVolumeHandleAnnotationKey] = newVolumeID

	return r.client.Update(ctx, pv)
}

// reconcilePV will extract the image details from the pv spec and regenerates
// the omap data.
func (r ReconcilePersistentVolume) reconcilePV(ctx context.Context, obj runtime.Object) error {
	pv, ok := obj.(*corev1.PersistentVolume)
	if !ok {
		return nil
	}
	if pv.Spec.CSI == nil || pv.Spec.CSI.Driver != r.config.DriverName {
		return nil
	}
	requestName := pv.Name
	volumeHandler := pv.Spec.CSI.VolumeHandle
	secretName := ""
	secretNamespace := ""
	// check static volume
	static, err := checkStaticVolume(pv)
	if err != nil {
		return err
	}
	// if the volume is static, dont generate OMAP data
	if static {
		return nil
	}
	if pv.Spec.CSI.ControllerExpandSecretRef != nil {
		secretName = pv.Spec.CSI.ControllerExpandSecretRef.Name
		secretNamespace = pv.Spec.CSI.ControllerExpandSecretRef.Namespace
	} else if pv.Spec.CSI.NodeStageSecretRef != nil {
		secretName = pv.Spec.CSI.NodeStageSecretRef.Name
		secretNamespace = pv.Spec.CSI.NodeStageSecretRef.Namespace
	}

	// Take lock to process only one volumeHandle at a time.
	if ok := r.Locks.TryAcquire(pv.Spec.CSI.VolumeHandle); !ok {
		return fmt.Errorf(util.VolumeOperationAlreadyExistsFmt, pv.Spec.CSI.VolumeHandle)
	}
	defer r.Locks.Release(pv.Spec.CSI.VolumeHandle)

	cr, err := r.getCredentials(ctx, secretName, secretNamespace)
	if err != nil {
		log.ErrorLogMsg("failed to get credentials from secret %s", err)

		return err
	}
	defer cr.DeleteCredentials()

	rbdVolID, err := rbd.RegenerateJournal(pv.Spec.CSI.VolumeAttributes, volumeHandler, requestName, cr)
	if err != nil {
		log.ErrorLogMsg("failed to regenerate journal %s", err)

		return err
	}
	if rbdVolID != volumeHandler {
		err = r.storeVolumeIDInPV(ctx, pv, rbdVolID)
		if err != nil {
			log.ErrorLogMsg("failed to store volumeID in PV %s", err)

			return err
		}
	}

	return nil
}

// Reconcile reconciles the PersistentVolume object and creates a new omap entries
// for the volume.
func (r *ReconcilePersistentVolume) Reconcile(ctx context.Context,
	request reconcile.Request) (reconcile.Result, error) {
	pv := &corev1.PersistentVolume{}
	err := r.client.Get(ctx, request.NamespacedName, pv)
	if err != nil {
		if apierrors.IsNotFound(err) {
			return reconcile.Result{}, nil
		}

		return reconcile.Result{}, err
	}
	// Check if the object is under deletion
	if !pv.GetDeletionTimestamp().IsZero() {
		return reconcile.Result{}, nil
	}

	err = r.reconcilePV(ctx, pv)
	if err != nil {
		return reconcile.Result{}, err
	}

	return reconcile.Result{}, nil
}
<commit_msg>cleanup: simplify checkStaticVolume function and remove unwanted vars<commit_after>/*
Copyright 2020 The Ceph-CSI Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package persistentvolume

import (
	"context"
	"errors"
	"fmt"

	ctrl "github.com/ceph/ceph-csi/internal/controller"
	"github.com/ceph/ceph-csi/internal/rbd"
	"github.com/ceph/ceph-csi/internal/util"
	"github.com/ceph/ceph-csi/internal/util/log"

	corev1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/types"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/controller"
	"sigs.k8s.io/controller-runtime/pkg/handler"
	"sigs.k8s.io/controller-runtime/pkg/manager"
	"sigs.k8s.io/controller-runtime/pkg/reconcile"
	"sigs.k8s.io/controller-runtime/pkg/source"
)

// ReconcilePersistentVolume reconciles a PersistentVolume object.
type ReconcilePersistentVolume struct {
	client client.Client
	config ctrl.Config
	Locks  *util.VolumeLocks
}

var (
	_ reconcile.Reconciler = &ReconcilePersistentVolume{}
	_ ctrl.Manager         = &ReconcilePersistentVolume{}
)

// Init will add the ReconcilePersistentVolume to the list.
func Init() {
	// add ReconcilePersistentVolume to the list
	ctrl.ControllerList = append(ctrl.ControllerList, ReconcilePersistentVolume{})
}

// Add adds the newPVReconciler.
func (r ReconcilePersistentVolume) Add(mgr manager.Manager, config ctrl.Config) error {
	return add(mgr, newPVReconciler(mgr, config))
}

// newReconciler returns a ReconcilePersistentVolume.
func newPVReconciler(mgr manager.Manager, config ctrl.Config) reconcile.Reconciler {
	r := &ReconcilePersistentVolume{
		client: mgr.GetClient(),
		config: config,
		Locks:  util.NewVolumeLocks(),
	}

	return r
}

func add(mgr manager.Manager, r reconcile.Reconciler) error {
	// Create a new controller
	c, err := controller.New(
		"persistentvolume-controller",
		mgr,
		controller.Options{MaxConcurrentReconciles: 1, Reconciler: r})
	if err != nil {
		return err
	}

	// Watch for changes to PersistentVolumes
	err = c.Watch(&source.Kind{Type: &corev1.PersistentVolume{}}, &handler.EnqueueRequestForObject{})
	if err != nil {
		return fmt.Errorf("failed to watch the changes: %w", err)
	}

	return nil
}

func (r *ReconcilePersistentVolume) getCredentials(
	ctx context.Context,
	name,
	namespace string) (*util.Credentials, error) {
	var cr *util.Credentials

	if name == "" || namespace == "" {
		errStr := "secret name or secret namespace is empty"
		log.ErrorLogMsg(errStr)

		return nil, errors.New(errStr)
	}
	secret := &corev1.Secret{}
	err := r.client.Get(ctx,
		types.NamespacedName{Name: name, Namespace: namespace},
		secret)
	if err != nil {
		return nil, fmt.Errorf("error getting secret %s in namespace %s: %w", name, namespace, err)
	}

	credentials := map[string]string{}
	for key, value := range secret.Data {
		credentials[key] = string(value)
	}

	cr, err = util.NewUserCredentials(credentials)
	if err != nil {
		log.ErrorLogMsg("failed to get user credentials %s", err)

		return nil, err
	}

	return cr, nil
}

func checkStaticVolume(pv *corev1.PersistentVolume) bool {
	return pv.Spec.CSI.VolumeAttributes["staticVolume"] == "true"
}

// storeVolumeIDInPV stores the new volumeID in PV object.
func (r ReconcilePersistentVolume) storeVolumeIDInPV(
	ctx context.Context,
	pv *corev1.PersistentVolume,
	newVolumeID string) error {
	if v, ok := pv.Annotations[rbd.PVVolumeHandleAnnotationKey]; ok {
		if v == newVolumeID {
			return nil
		}
	}
	if pv.Annotations == nil {
		pv.Annotations = make(map[string]string)
	}
	if pv.Labels == nil {
		pv.Labels = make(map[string]string)
	}
	pv.Labels[rbd.PVReplicatedLabelKey] = rbd.PVReplicatedLabelValue
	pv.Annotations[rbd.PVVolumeHandleAnnotationKey] = newVolumeID

	return r.client.Update(ctx, pv)
}

// reconcilePV will extract the image details from the pv spec and regenerates
// the omap data.
func (r ReconcilePersistentVolume) reconcilePV(ctx context.Context, obj runtime.Object) error {
	pv, ok := obj.(*corev1.PersistentVolume)
	if !ok {
		return nil
	}
	if pv.Spec.CSI == nil || pv.Spec.CSI.Driver != r.config.DriverName {
		return nil
	}
	requestName := pv.Name
	volumeHandler := pv.Spec.CSI.VolumeHandle
	secretName := ""
	secretNamespace := ""
	// check static volume
	static := checkStaticVolume(pv)
	// if the volume is static, dont generate OMAP data
	if static {
		return nil
	}
	if pv.Spec.CSI.ControllerExpandSecretRef != nil {
		secretName = pv.Spec.CSI.ControllerExpandSecretRef.Name
		secretNamespace = pv.Spec.CSI.ControllerExpandSecretRef.Namespace
	} else if pv.Spec.CSI.NodeStageSecretRef != nil {
		secretName = pv.Spec.CSI.NodeStageSecretRef.Name
		secretNamespace = pv.Spec.CSI.NodeStageSecretRef.Namespace
	}

	// Take lock to process only one volumeHandle at a time.
	if ok := r.Locks.TryAcquire(pv.Spec.CSI.VolumeHandle); !ok {
		return fmt.Errorf(util.VolumeOperationAlreadyExistsFmt, pv.Spec.CSI.VolumeHandle)
	}
	defer r.Locks.Release(pv.Spec.CSI.VolumeHandle)

	cr, err := r.getCredentials(ctx, secretName, secretNamespace)
	if err != nil {
		log.ErrorLogMsg("failed to get credentials from secret %s", err)

		return err
	}
	defer cr.DeleteCredentials()

	rbdVolID, err := rbd.RegenerateJournal(pv.Spec.CSI.VolumeAttributes, volumeHandler, requestName, cr)
	if err != nil {
		log.ErrorLogMsg("failed to regenerate journal %s", err)

		return err
	}
	if rbdVolID != volumeHandler {
		err = r.storeVolumeIDInPV(ctx, pv, rbdVolID)
		if err != nil {
			log.ErrorLogMsg("failed to store volumeID in PV %s", err)

			return err
		}
	}

	return nil
}

// Reconcile reconciles the PersistentVolume object and creates a new omap entries
// for the volume.
func (r *ReconcilePersistentVolume) Reconcile(ctx context.Context,
	request reconcile.Request) (reconcile.Result, error) {
	pv := &corev1.PersistentVolume{}
	err := r.client.Get(ctx, request.NamespacedName, pv)
	if err != nil {
		if apierrors.IsNotFound(err) {
			return reconcile.Result{}, nil
		}

		return reconcile.Result{}, err
	}
	// Check if the object is under deletion
	if !pv.GetDeletionTimestamp().IsZero() {
		return reconcile.Result{}, nil
	}

	err = r.reconcilePV(ctx, pv)
	if err != nil {
		return reconcile.Result{}, err
	}

	return reconcile.Result{}, nil
}
<|endoftext|>


<commit_before>package meters

import (
	"github.com/rackspace/gophercloud"
)

// ListOptsBuilder allows extensions to add additional parameters to the
// List request.
type ListOptsBuilder interface {
	ToMeterListQuery() (string, error)
}

// ListOpts allows the filtering and sorting of collections through
// the API. Filtering is achieved by passing in struct field values that map to
// the server attributes you want to see returned.
type ListOpts struct {
}

// ToServerListQuery formats a ListOpts into a query string.
func (opts ListOpts) ToMeterListQuery() (string, error) {
	q, err := gophercloud.BuildQueryString(opts)
	if err != nil {
		return "", err
	}
	return q.String(), nil
}

// List makes a request against the API to list meters accessible to you.
func List(client *gophercloud.ServiceClient, opts ListOptsBuilder) listResult {
	var res listResult
	url := listURL(client)

	if opts != nil {
		query, err := opts.ToMeterListQuery()
		if err != nil {
			res.Err = err
			return res
		}
		url += query
	}

	_, res.Err = client.Get(url, &res.Body, &gophercloud.RequestOpts{})
	return res
}

// StatisticsOptsBuilder allows extensions to add additional parameters to the
// List request.
type MeterStatisticsOptsBuilder interface {
	ToMeterStatisticsQuery() (string, error)
}

// StatisticsOpts allows the filtering and sorting of collections through
// the API. Filtering is achieved by passing in struct field values that map to
// the server attributes you want to see returned.
type MeterStatisticsOpts struct {
	QueryField string `q:"q.field"`
	QueryOp    string `q:"q.op"`
	QueryValue string `q:"q.value"`

	// Optional group by
	GroupBy string `q:"groupby"`

	// Optional number of seconds in a period
	Period int `q:"period"`
}

// ToStatisticsQuery formats a StatisticsOpts into a query string.
func (opts MeterStatisticsOpts) ToMeterStatisticsQuery() (string, error) {
	q, err := gophercloud.BuildQueryString(opts)
	if err != nil {
		return "", err
	}
	return q.String(), nil
}

// List makes a request against the API to list meters accessible to you.
func MeterStatistics(client *gophercloud.ServiceClient, n string, opts MeterStatisticsOptsBuilder) statisticsResult {
	var res statisticsResult
	url := statisticsURL(client, n)

	if opts != nil {
		query, err := opts.ToMeterStatisticsQuery()
		if err != nil {
			res.Err = err
			return res
		}
		url += query
	}

	_, res.Err = client.Get(url, &res.Body, &gophercloud.RequestOpts{})
	return res
}
<commit_msg>add(statistics): allow json queries<commit_after>package meters

import (
	"github.com/rackspace/gophercloud"
)

// ListOptsBuilder allows extensions to add additional parameters to the
// List request.
type ListOptsBuilder interface {
	ToMeterListQuery() (string, error)
}

// ListOpts allows the filtering and sorting of collections through
// the API. Filtering is achieved by passing in struct field values that map to
// the server attributes you want to see returned.
type ListOpts struct {
}

// ToMeterListQuery formats a ListOpts into a query string.
func (opts ListOpts) ToMeterListQuery() (string, error) {
	q, err := gophercloud.BuildQueryString(opts)
	if err != nil {
		return "", err
	}
	return q.String(), nil
}

// List makes a request against the API to list meters accessible to you.
func List(client *gophercloud.ServiceClient, opts ListOptsBuilder) listResult {
	var res listResult
	url := listURL(client)

	if opts != nil {
		query, err := opts.ToMeterListQuery()
		if err != nil {
			res.Err = err
			return res
		}
		url += query
	}

	_, res.Err = client.Get(url, &res.Body, &gophercloud.RequestOpts{})
	return res
}

//OptsKind describes the mode with which a given set of opts should be tranferred
type OptsKind string

var (
	//BodyContentOpts is a kind of option serialization. The MeterStatisticsOptsBuilder is expected
	//to emit JSON from ToMeterStatisticsQuery()
	BodyContentOpts = OptsKind("Body")
	//QueryOpts is a kind of option serialization. The MeterStatisticsOptsBuilder is expected
	//to emit uri encoded fields from ToMeterStatisticsQuery()
	QueryOpts = OptsKind("Query")
)

// MeterStatisticsOptsBuilder allows extensions to add additional parameters to the
// List request.
type MeterStatisticsOptsBuilder interface {
	Kind() OptsKind
	ToMeterStatisticsQuery() (string, error)
}

// MeterStatisticsOpts allows the filtering and sorting of collections through
// the API. Filtering is achieved by passing in struct field values that map to
// the server attributes you want to see returned.
type MeterStatisticsOpts struct {
	QueryField string `q:"q.field"`
	QueryOp    string `q:"q.op"`
	QueryValue string `q:"q.value"`

	// Optional group by
	GroupBy string `q:"groupby"`

	// Optional number of seconds in a period
	Period int `q:"period"`
}

// Kind returns QueryOpts by default for MeterStatisticsOpts
func (opts MeterStatisticsOpts) Kind() OptsKind {
	return QueryOpts
}

// ToMeterStatisticsQuery formats a StatisticsOpts into a query string.
func (opts MeterStatisticsOpts) ToMeterStatisticsQuery() (string, error) {
	q, err := gophercloud.BuildQueryString(opts)
	if err != nil {
		return "", err
	}
	return q.String(), nil
}

//MeterStatistics gathers statistics based on filters, groups, and period options
func MeterStatistics(client *gophercloud.ServiceClient, n string, optsBuilder MeterStatisticsOptsBuilder) statisticsResult {
	var (
		res  statisticsResult
		url  = statisticsURL(client, n)
		opts gophercloud.RequestOpts
		err  error
	)

	if optsBuilder != nil && optsBuilder.Kind() == QueryOpts {
		query, err := optsBuilder.ToMeterStatisticsQuery()
		if err != nil {
			res.Err = err
			return res
		}
		url += query
	} else if optsBuilder != nil && optsBuilder.Kind() == BodyContentOpts {
		opts.JSONBody, err = optsBuilder.ToMeterStatisticsQuery()
		if err != nil {
			res.Err = err
			return res
		}
	}

	_, res.Err = client.Get(url, &res.Body, &opts)
	return res
}
<|endoftext|>


<commit_before>package terraform

// The main version number that is being run at the moment.
const Version = "0.6.15"

// A pre-release marker for the version. If this is "" (empty string)
// then it means that it is a final release. Otherwise, this is a pre-release
// such as "dev" (in development), "beta", "rc1", etc.
const VersionPrerelease = ""
<commit_msg>release: clean up after v0.6.15<commit_after>package terraform

// The main version number that is being run at the moment.
const Version = "0.6.16"

// A pre-release marker for the version. If this is "" (empty string)
// then it means that it is a final release. Otherwise, this is a pre-release
// such as "dev" (in development), "beta", "rc1", etc.
const VersionPrerelease = "dev"
<|endoftext|>


<commit_before>package terraform

import (
	"fmt"

	"github.com/hashicorp/go-version"
)

// The main version number that is being run at the moment.
const Version = "0.8.0"

// A pre-release marker for the version. If this is "" (empty string)
// then it means that it is a final release. Otherwise, this is a pre-release
// such as "dev" (in development), "beta", "rc1", etc.
const VersionPrerelease = "dev"

// SemVersion is an instance of version.Version. This has the secondary
// benefit of verifying during tests and init time that our version is a
// proper semantic version, which should always be the case.
var SemVersion = version.Must(version.NewVersion(Version))

// VersionHeader is the header name used to send the current terraform version
// in http requests.
const VersionHeader = "Terraform-Version"

func VersionString() string {
	if VersionPrerelease != "" {
		return fmt.Sprintf("%s-%s", Version, VersionPrerelease)
	}
	return Version
}
<commit_msg>v0.8.0-rc3<commit_after>package terraform

import (
	"fmt"

	"github.com/hashicorp/go-version"
)

// The main version number that is being run at the moment.
const Version = "0.8.0"

// A pre-release marker for the version. If this is "" (empty string)
// then it means that it is a final release. Otherwise, this is a pre-release
// such as "dev" (in development), "beta", "rc1", etc.
const VersionPrerelease = "rc3"

// SemVersion is an instance of version.Version. This has the secondary
// benefit of verifying during tests and init time that our version is a
// proper semantic version, which should always be the case.
var SemVersion = version.Must(version.NewVersion(Version))

// VersionHeader is the header name used to send the current terraform version
// in http requests.
const VersionHeader = "Terraform-Version"

func VersionString() string {
	if VersionPrerelease != "" {
		return fmt.Sprintf("%s-%s", Version, VersionPrerelease)
	}
	return Version
}
<|endoftext|>


<commit_before>// $G $D/$F.go && $L $F.$A && ./$A.out

// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Tests verifying the semantics of the select statement
// for basic empty/non-empty cases.

package main

import "time"

const always = "function did not"
const never = "function did"

func unreachable() {
	panic("control flow shouldn't reach here")
}

// Calls f and verifies that f always/never panics depending on signal.
func testPanic(signal string, f func()) {
	defer func() {
		s := never
		if recover() != nil {
			s = always  // f panicked
		}
		if s != signal {
			panic(signal + " panic")
		}
	}()
	f()
}

// Calls f and empirically verifies that f always/never blocks depending on signal.
func testBlock(signal string, f func()) {
	c := make(chan string)
	go func() {
		f()
		c <- never // f didn't block
	}()
	go func() {
		time.Sleep(1e8) // 0.1s seems plenty long
		c <- always     // f blocked always
	}()
	if <-c != signal {
		panic(signal + " block")
	}
}

func main() {
	const async = 1 // asynchronous channels
	var nilch chan int

	// sending/receiving from a nil channel outside a select panics
	testPanic(always, func() {
		nilch <- 7
	})
	testPanic(always, func() {
		<-nilch
	})

	// sending/receiving from a nil channel inside a select never panics
	testPanic(never, func() {
		select {
		case nilch <- 7:
			unreachable()
		default:
		}
	})
	testPanic(never, func() {
		select {
		case <-nilch:
			unreachable()
		default:
		}
	})

	// sending to an async channel with free buffer space never blocks
	testBlock(never, func() {
		ch := make(chan int, async)
		ch <- 7
	})

	// receiving from a non-ready channel always blocks
	testBlock(always, func() {
		ch := make(chan int)
		<-ch
	})

	// TODO(gri) remove this if once 6g accepts empty selects
	enabled := false
	if enabled {
		// empty selects always block
		testBlock(always, func() {
			select {
			case <-make(chan int): // remove this once 6g accepts empty selects
			}
		})

		// selects with only nil channels always block
		testBlock(always, func() {
			select {
			case <-nilch:
				unreachable()
			}
		})
		testBlock(always, func() {
			select {
			case nilch <- 7:
				unreachable()
			}
		})
		testBlock(always, func() {
			select {
			case <-nilch:
				unreachable()
			case nilch <- 7:
				unreachable()
			}
		})
	}

	// selects with non-ready non-nil channels always block
	testBlock(always, func() {
		ch := make(chan int)
		select {
		case <-ch:
			unreachable()
		}
	})

	// selects with default cases don't block
	testBlock(never, func() {
		select {
		default:
		}
	})
	testBlock(never, func() {
		select {
		case <-nilch:
			unreachable()
		default:
		}
	})
	testBlock(never, func() {
		select {
		case nilch <- 7:
			unreachable()
		default:
		}
	})

	// selects with ready channels don't block
	testBlock(never, func() {
		ch := make(chan int, async)
		select {
		case ch <- 7:
		default:
			unreachable()
		}
	})
	testBlock(never, func() {
		ch := make(chan int, async)
		ch <- 7
		select {
		case <-ch:
		default:
			unreachable()
		}
	})
}
<commit_msg>channel tests: added a couple of tests with closed channels<commit_after>// $G $D/$F.go && $L $F.$A && ./$A.out

// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Tests verifying the semantics of the select statement
// for basic empty/non-empty cases.

package main

import "time"

const always = "function did not"
const never = "function did"

func unreachable() {
	panic("control flow shouldn't reach here")
}

// Calls f and verifies that f always/never panics depending on signal.
func testPanic(signal string, f func()) {
	defer func() {
		s := never
		if recover() != nil {
			s = always // f panicked
		}
		if s != signal {
			panic(signal + " panic")
		}
	}()
	f()
}

// Calls f and empirically verifies that f always/never blocks depending on signal.
func testBlock(signal string, f func()) {
	c := make(chan string)
	go func() {
		f()
		c <- never // f didn't block
	}()
	go func() {
		time.Sleep(1e8) // 0.1s seems plenty long
		c <- always     // f blocked always
	}()
	if <-c != signal {
		panic(signal + " block")
	}
}

func main() {
	const async = 1 // asynchronous channels
	var nilch chan int
	closedch := make(chan int)
	close(closedch)

	// sending/receiving from a nil channel outside a select panics
	testPanic(always, func() {
		nilch <- 7
	})
	testPanic(always, func() {
		<-nilch
	})

	// sending/receiving from a nil channel inside a select never panics
	testPanic(never, func() {
		select {
		case nilch <- 7:
			unreachable()
		default:
		}
	})
	testPanic(never, func() {
		select {
		case <-nilch:
			unreachable()
		default:
		}
	})

	// sending to an async channel with free buffer space never blocks
	testBlock(never, func() {
		ch := make(chan int, async)
		ch <- 7
	})

	// receiving (a small number of times) from a closed channel never blocks
	testBlock(never, func() {
		for i := 0; i < 10; i++ {
			if <-closedch != 0 {
				panic("expected zero value when reading from closed channel")
			}
		}
	})

	// sending (a small number of times) to a closed channel is not specified
	// but the current implementation doesn't block: test that different
	// implementations behave the same
	testBlock(never, func() {
		for i := 0; i < 10; i++ {
			closedch <- 7
		}
	})

	// receiving from a non-ready channel always blocks
	testBlock(always, func() {
		ch := make(chan int)
		<-ch
	})

	// TODO(gri) remove this if once 6g accepts empty selects
	enabled := false
	if enabled {
		// empty selects always block
		testBlock(always, func() {
			select {
			case <-make(chan int): // remove this once 6g accepts empty selects
			}
		})

		// selects with only nil channels always block
		testBlock(always, func() {
			select {
			case <-nilch:
				unreachable()
			}
		})
		testBlock(always, func() {
			select {
			case nilch <- 7:
				unreachable()
			}
		})
		testBlock(always, func() {
			select {
			case <-nilch:
				unreachable()
			case nilch <- 7:
				unreachable()
			}
		})
	}

	// selects with non-ready non-nil channels always block
	testBlock(always, func() {
		ch := make(chan int)
		select {
		case <-ch:
			unreachable()
		}
	})

	// selects with default cases don't block
	testBlock(never, func() {
		select {
		default:
		}
	})
	testBlock(never, func() {
		select {
		case <-nilch:
			unreachable()
		default:
		}
	})
	testBlock(never, func() {
		select {
		case nilch <- 7:
			unreachable()
		default:
		}
	})

	// selects with ready channels don't block
	testBlock(never, func() {
		ch := make(chan int, async)
		select {
		case ch <- 7:
		default:
			unreachable()
		}
	})
	testBlock(never, func() {
		ch := make(chan int, async)
		ch <- 7
		select {
		case <-ch:
		default:
			unreachable()
		}
	})

	// selects with closed channels don't block
	testBlock(never, func() {
		select {
		case <-closedch:
		}
	})
	testBlock(never, func() {
		select {
		case closedch <- 7:
		}
	})
}
<|endoftext|>


<commit_before>package main

import (
	"os/exec"
	"strings"
	"testing"
)

func TestCommitAfterContainerIsDone(t *testing.T) {
	runCmd := exec.Command(dockerBinary, "run", "-i", "-a", "stdin", "busybox", "echo", "foo")
	out, _, _, err := runCommandWithStdoutStderr(runCmd)
	if err != nil {
		t.Fatalf("failed to run container: %s, %v", out, err)
	}

	cleanedContainerID := stripTrailingCharacters(out)

	waitCmd := exec.Command(dockerBinary, "wait", cleanedContainerID)
	if _, _, err = runCommandWithOutput(waitCmd); err != nil {
		t.Fatalf("error thrown while waiting for container: %s, %v", out, err)
	}

	commitCmd := exec.Command(dockerBinary, "commit", cleanedContainerID)
	out, _, err = runCommandWithOutput(commitCmd)
	if err != nil {
		t.Fatalf("failed to commit container to image: %s, %v", out, err)
	}

	cleanedImageID := stripTrailingCharacters(out)

	inspectCmd := exec.Command(dockerBinary, "inspect", cleanedImageID)
	if out, _, err = runCommandWithOutput(inspectCmd); err != nil {
		t.Fatalf("failed to inspect image: %s, %v", out, err)
	}

	deleteContainer(cleanedContainerID)
	deleteImages(cleanedImageID)

	logDone("commit - echo foo and commit the image")
}

func TestCommitWithoutPause(t *testing.T) {
	runCmd := exec.Command(dockerBinary, "run", "-i", "-a", "stdin", "busybox", "echo", "foo")
	out, _, _, err := runCommandWithStdoutStderr(runCmd)
	if err != nil {
		t.Fatalf("failed to run container: %s, %v", out, err)
	}

	cleanedContainerID := stripTrailingCharacters(out)

	waitCmd := exec.Command(dockerBinary, "wait", cleanedContainerID)
	if _, _, err = runCommandWithOutput(waitCmd); err != nil {
		t.Fatalf("error thrown while waiting for container: %s, %v", out, err)
	}

	commitCmd := exec.Command(dockerBinary, "commit", "-p=false", cleanedContainerID)
	out, _, err = runCommandWithOutput(commitCmd)
	if err != nil {
		t.Fatalf("failed to commit container to image: %s, %v", out, err)
	}

	cleanedImageID := stripTrailingCharacters(out)

	inspectCmd := exec.Command(dockerBinary, "inspect", cleanedImageID)
	if out, _, err = runCommandWithOutput(inspectCmd); err != nil {
		t.Fatalf("failed to inspect image: %s, %v", out, err)
	}

	deleteContainer(cleanedContainerID)
	deleteImages(cleanedImageID)

	logDone("commit - echo foo and commit the image with --pause=false")
}

//test commit a paused container should not unpause it after commit
func TestCommitPausedContainer(t *testing.T) {
	defer deleteAllContainers()
	defer unpauseAllContainers()
	cmd := exec.Command(dockerBinary, "run", "-i", "-d", "busybox")
	out, _, _, err := runCommandWithStdoutStderr(cmd)
	if err != nil {
		t.Fatalf("failed to run container: %v, output: %q", err, out)
	}

	cleanedContainerID := stripTrailingCharacters(out)
	cmd = exec.Command(dockerBinary, "pause", cleanedContainerID)
	out, _, _, err = runCommandWithStdoutStderr(cmd)
	if err != nil {
		t.Fatalf("failed to pause container: %v, output: %q", err, out)
	}

	commitCmd := exec.Command(dockerBinary, "commit", cleanedContainerID)
	out, _, err = runCommandWithOutput(commitCmd)
	if err != nil {
		t.Fatalf("failed to commit container to image: %s, %v", out, err)
	}
	cleanedImageID := stripTrailingCharacters(out)
	defer deleteImages(cleanedImageID)

	cmd = exec.Command(dockerBinary, "inspect", "-f", "{{.State.Paused}}", cleanedContainerID)
	out, _, _, err = runCommandWithStdoutStderr(cmd)
	if err != nil {
		t.Fatalf("failed to inspect container: %v, output: %q", err, out)
	}

	if !strings.Contains(out, "true") {
		t.Fatalf("commit should not unpause a paused container")
	}

	logDone("commit - commit a paused container will not unpause it")
}

func TestCommitNewFile(t *testing.T) {
	defer deleteAllContainers()

	cmd := exec.Command(dockerBinary, "run", "--name", "commiter", "busybox", "/bin/sh", "-c", "echo koye > /foo")
	if _, err := runCommand(cmd); err != nil {
		t.Fatal(err)
	}

	cmd = exec.Command(dockerBinary, "commit", "commiter")
	imageID, _, err := runCommandWithOutput(cmd)
	if err != nil {
		t.Fatal(err)
	}
	imageID = strings.Trim(imageID, "\r\n")
	defer deleteImages(imageID)

	cmd = exec.Command(dockerBinary, "run", imageID, "cat", "/foo")

	out, _, err := runCommandWithOutput(cmd)
	if err != nil {
		t.Fatal(err, out)
	}
	if actual := strings.Trim(out, "\r\n"); actual != "koye" {
		t.Fatalf("expected output koye received %q", actual)
	}

	logDone("commit - commit file and read")
}

func TestCommitHardlink(t *testing.T) {
	defer deleteAllContainers()

	cmd := exec.Command(dockerBinary, "run", "-t", "--name", "hardlinks", "busybox", "sh", "-c", "touch file1 && ln file1 file2 && ls -di file1 file2")
	firstOuput, _, err := runCommandWithOutput(cmd)
	if err != nil {
		t.Fatal(err)
	}

	chunks := strings.Split(strings.TrimSpace(firstOuput), " ")
	inode := chunks[0]
	found := false
	for _, chunk := range chunks[1:] {
		if chunk == inode {
			found = true
			break
		}
	}
	if !found {
		t.Fatalf("Failed to create hardlink in a container. Expected to find %q in %q", inode, chunks[1:])
	}

	cmd = exec.Command(dockerBinary, "commit", "hardlinks", "hardlinks")
	imageID, _, err := runCommandWithOutput(cmd)
	if err != nil {
		t.Fatal(imageID, err)
	}
	imageID = strings.Trim(imageID, "\r\n")
	defer deleteImages(imageID)

	cmd = exec.Command(dockerBinary, "run", "-t", "hardlinks", "ls", "-di", "file1", "file2")
	secondOuput, _, err := runCommandWithOutput(cmd)
	if err != nil {
		t.Fatal(err)
	}

	chunks = strings.Split(strings.TrimSpace(secondOuput), " ")
	inode = chunks[0]
	found = false
	for _, chunk := range chunks[1:] {
		if chunk == inode {
			found = true
			break
		}
	}
	if !found {
		t.Fatalf("Failed to create hardlink in a container. Expected to find %q in %q", inode, chunks[1:])
	}

	logDone("commit - commit hardlinks")
}

func TestCommitTTY(t *testing.T) {
	defer deleteImages("ttytest")
	defer deleteAllContainers()

	cmd := exec.Command(dockerBinary, "run", "-t", "--name", "tty", "busybox", "/bin/ls")
	if _, err := runCommand(cmd); err != nil {
		t.Fatal(err)
	}

	cmd = exec.Command(dockerBinary, "commit", "tty", "ttytest")
	imageID, _, err := runCommandWithOutput(cmd)
	if err != nil {
		t.Fatal(err)
	}
	imageID = strings.Trim(imageID, "\r\n")

	cmd = exec.Command(dockerBinary, "run", "ttytest", "/bin/ls")
	if _, err := runCommand(cmd); err != nil {
		t.Fatal(err)
	}

	logDone("commit - commit tty")
}

func TestCommitWithHostBindMount(t *testing.T) {
	defer deleteAllContainers()

	cmd := exec.Command(dockerBinary, "run", "--name", "bind-commit", "-v", "/dev/null:/winning", "busybox", "true")
	if _, err := runCommand(cmd); err != nil {
		t.Fatal(err)
	}

	cmd = exec.Command(dockerBinary, "commit", "bind-commit", "bindtest")
	imageID, _, err := runCommandWithOutput(cmd)
	if err != nil {
		t.Fatal(imageID, err)
	}

	imageID = strings.Trim(imageID, "\r\n")
	defer deleteImages(imageID)

	cmd = exec.Command(dockerBinary, "run", "bindtest", "true")

	if _, err := runCommand(cmd); err != nil {
		t.Fatal(err)
	}

	logDone("commit - commit bind mounted file")
}

func TestCommitChange(t *testing.T) {
	defer deleteAllContainers()

	cmd := exec.Command(dockerBinary, "run", "--name", "test", "busybox", "true")
	if _, err := runCommand(cmd); err != nil {
		t.Fatal(err)
	}

	cmd = exec.Command(dockerBinary, "commit",
		"--change", "EXPOSE 8080",
		"--change", "ENV DEBUG true",
		"--change", "ENV test 1",
		"test", "test-commit")
	imageId, _, err := runCommandWithOutput(cmd)
	if err != nil {
		t.Fatal(imageId, err)
	}
	imageId = strings.Trim(imageId, "\r\n")
	defer deleteImages(imageId)

	expected := map[string]string{
		"Config.ExposedPorts": "map[8080/tcp:map[]]",
		"Config.Env":          "[DEBUG=true test=1]",
	}

	for conf, value := range expected {
		res, err := inspectField(imageId, conf)
		if err != nil {
			t.Errorf("failed to get value %s, error: %s", conf, err)
		}
		if res != value {
			t.Errorf("%s('%s'), expected %s", conf, res, value)
		}
	}

	logDone("commit - commit --change")
}
<commit_msg>integ-cli: fix TestCommitChange for pulled busybox<commit_after>package main

import (
	"os/exec"
	"strings"
	"testing"
)

func TestCommitAfterContainerIsDone(t *testing.T) {
	runCmd := exec.Command(dockerBinary, "run", "-i", "-a", "stdin", "busybox", "echo", "foo")
	out, _, _, err := runCommandWithStdoutStderr(runCmd)
	if err != nil {
		t.Fatalf("failed to run container: %s, %v", out, err)
	}

	cleanedContainerID := stripTrailingCharacters(out)

	waitCmd := exec.Command(dockerBinary, "wait", cleanedContainerID)
	if _, _, err = runCommandWithOutput(waitCmd); err != nil {
		t.Fatalf("error thrown while waiting for container: %s, %v", out, err)
	}

	commitCmd := exec.Command(dockerBinary, "commit", cleanedContainerID)
	out, _, err = runCommandWithOutput(commitCmd)
	if err != nil {
		t.Fatalf("failed to commit container to image: %s, %v", out, err)
	}

	cleanedImageID := stripTrailingCharacters(out)

	inspectCmd := exec.Command(dockerBinary, "inspect", cleanedImageID)
	if out, _, err = runCommandWithOutput(inspectCmd); err != nil {
		t.Fatalf("failed to inspect image: %s, %v", out, err)
	}

	deleteContainer(cleanedContainerID)
	deleteImages(cleanedImageID)

	logDone("commit - echo foo and commit the image")
}

func TestCommitWithoutPause(t *testing.T) {
	runCmd := exec.Command(dockerBinary, "run", "-i", "-a", "stdin", "busybox", "echo", "foo")
	out, _, _, err := runCommandWithStdoutStderr(runCmd)
	if err != nil {
		t.Fatalf("failed to run container: %s, %v", out, err)
	}

	cleanedContainerID := stripTrailingCharacters(out)

	waitCmd := exec.Command(dockerBinary, "wait", cleanedContainerID)
	if _, _, err = runCommandWithOutput(waitCmd); err != nil {
		t.Fatalf("error thrown while waiting for container: %s, %v", out, err)
	}

	commitCmd := exec.Command(dockerBinary, "commit", "-p=false", cleanedContainerID)
	out, _, err = runCommandWithOutput(commitCmd)
	if err != nil {
		t.Fatalf("failed to commit container to image: %s, %v", out, err)
	}

	cleanedImageID := stripTrailingCharacters(out)

	inspectCmd := exec.Command(dockerBinary, "inspect", cleanedImageID)
	if out, _, err = runCommandWithOutput(inspectCmd); err != nil {
		t.Fatalf("failed to inspect image: %s, %v", out, err)
	}

	deleteContainer(cleanedContainerID)
	deleteImages(cleanedImageID)

	logDone("commit - echo foo and commit the image with --pause=false")
}

//test commit a paused container should not unpause it after commit
func TestCommitPausedContainer(t *testing.T) {
	defer deleteAllContainers()
	defer unpauseAllContainers()
	cmd := exec.Command(dockerBinary, "run", "-i", "-d", "busybox")
	out, _, _, err := runCommandWithStdoutStderr(cmd)
	if err != nil {
		t.Fatalf("failed to run container: %v, output: %q", err, out)
	}

	cleanedContainerID := stripTrailingCharacters(out)
	cmd = exec.Command(dockerBinary, "pause", cleanedContainerID)
	out, _, _, err = runCommandWithStdoutStderr(cmd)
	if err != nil {
		t.Fatalf("failed to pause container: %v, output: %q", err, out)
	}

	commitCmd := exec.Command(dockerBinary, "commit", cleanedContainerID)
	out, _, err = runCommandWithOutput(commitCmd)
	if err != nil {
		t.Fatalf("failed to commit container to image: %s, %v", out, err)
	}
	cleanedImageID := stripTrailingCharacters(out)
	defer deleteImages(cleanedImageID)

	cmd = exec.Command(dockerBinary, "inspect", "-f", "{{.State.Paused}}", cleanedContainerID)
	out, _, _, err = runCommandWithStdoutStderr(cmd)
	if err != nil {
		t.Fatalf("failed to inspect container: %v, output: %q", err, out)
	}

	if !strings.Contains(out, "true") {
		t.Fatalf("commit should not unpause a paused container")
	}

	logDone("commit - commit a paused container will not unpause it")
}

func TestCommitNewFile(t *testing.T) {
	defer deleteAllContainers()

	cmd := exec.Command(dockerBinary, "run", "--name", "commiter", "busybox", "/bin/sh", "-c", "echo koye > /foo")
	if _, err := runCommand(cmd); err != nil {
		t.Fatal(err)
	}

	cmd = exec.Command(dockerBinary, "commit", "commiter")
	imageID, _, err := runCommandWithOutput(cmd)
	if err != nil {
		t.Fatal(err)
	}
	imageID = strings.Trim(imageID, "\r\n")
	defer deleteImages(imageID)

	cmd = exec.Command(dockerBinary, "run", imageID, "cat", "/foo")

	out, _, err := runCommandWithOutput(cmd)
	if err != nil {
		t.Fatal(err, out)
	}
	if actual := strings.Trim(out, "\r\n"); actual != "koye" {
		t.Fatalf("expected output koye received %q", actual)
	}

	logDone("commit - commit file and read")
}

func TestCommitHardlink(t *testing.T) {
	defer deleteAllContainers()

	cmd := exec.Command(dockerBinary, "run", "-t", "--name", "hardlinks", "busybox", "sh", "-c", "touch file1 && ln file1 file2 && ls -di file1 file2")
	firstOuput, _, err := runCommandWithOutput(cmd)
	if err != nil {
		t.Fatal(err)
	}

	chunks := strings.Split(strings.TrimSpace(firstOuput), " ")
	inode := chunks[0]
	found := false
	for _, chunk := range chunks[1:] {
		if chunk == inode {
			found = true
			break
		}
	}
	if !found {
		t.Fatalf("Failed to create hardlink in a container. Expected to find %q in %q", inode, chunks[1:])
	}

	cmd = exec.Command(dockerBinary, "commit", "hardlinks", "hardlinks")
	imageID, _, err := runCommandWithOutput(cmd)
	if err != nil {
		t.Fatal(imageID, err)
	}
	imageID = strings.Trim(imageID, "\r\n")
	defer deleteImages(imageID)

	cmd = exec.Command(dockerBinary, "run", "-t", "hardlinks", "ls", "-di", "file1", "file2")
	secondOuput, _, err := runCommandWithOutput(cmd)
	if err != nil {
		t.Fatal(err)
	}

	chunks = strings.Split(strings.TrimSpace(secondOuput), " ")
	inode = chunks[0]
	found = false
	for _, chunk := range chunks[1:] {
		if chunk == inode {
			found = true
			break
		}
	}
	if !found {
		t.Fatalf("Failed to create hardlink in a container. Expected to find %q in %q", inode, chunks[1:])
	}

	logDone("commit - commit hardlinks")
}

func TestCommitTTY(t *testing.T) {
	defer deleteImages("ttytest")
	defer deleteAllContainers()

	cmd := exec.Command(dockerBinary, "run", "-t", "--name", "tty", "busybox", "/bin/ls")
	if _, err := runCommand(cmd); err != nil {
		t.Fatal(err)
	}

	cmd = exec.Command(dockerBinary, "commit", "tty", "ttytest")
	imageID, _, err := runCommandWithOutput(cmd)
	if err != nil {
		t.Fatal(err)
	}
	imageID = strings.Trim(imageID, "\r\n")

	cmd = exec.Command(dockerBinary, "run", "ttytest", "/bin/ls")
	if _, err := runCommand(cmd); err != nil {
		t.Fatal(err)
	}

	logDone("commit - commit tty")
}

func TestCommitWithHostBindMount(t *testing.T) {
	defer deleteAllContainers()

	cmd := exec.Command(dockerBinary, "run", "--name", "bind-commit", "-v", "/dev/null:/winning", "busybox", "true")
	if _, err := runCommand(cmd); err != nil {
		t.Fatal(err)
	}

	cmd = exec.Command(dockerBinary, "commit", "bind-commit", "bindtest")
	imageID, _, err := runCommandWithOutput(cmd)
	if err != nil {
		t.Fatal(imageID, err)
	}

	imageID = strings.Trim(imageID, "\r\n")
	defer deleteImages(imageID)

	cmd = exec.Command(dockerBinary, "run", "bindtest", "true")

	if _, err := runCommand(cmd); err != nil {
		t.Fatal(err)
	}

	logDone("commit - commit bind mounted file")
}

func TestCommitChange(t *testing.T) {
	defer deleteAllContainers()

	cmd := exec.Command(dockerBinary, "run", "--name", "test", "busybox", "true")
	if _, err := runCommand(cmd); err != nil {
		t.Fatal(err)
	}

	cmd = exec.Command(dockerBinary, "commit",
		"--change", "EXPOSE 8080",
		"--change", "ENV DEBUG true",
		"--change", "ENV test 1",
		"--change", "ENV PATH /foo",
		"test", "test-commit")
	imageId, _, err := runCommandWithOutput(cmd)
	if err != nil {
		t.Fatal(imageId, err)
	}
	imageId = strings.Trim(imageId, "\r\n")
	defer deleteImages(imageId)

	expected := map[string]string{
		"Config.ExposedPorts": "map[8080/tcp:map[]]",
		"Config.Env":          "[DEBUG=true test=1 PATH=/foo]",
	}

	for conf, value := range expected {
		res, err := inspectField(imageId, conf)
		if err != nil {
			t.Errorf("failed to get value %s, error: %s", conf, err)
		}
		if res != value {
			t.Errorf("%s('%s'), expected %s", conf, res, value)
		}
	}

	logDone("commit - commit --change")
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
)

type Element struct {
	Name         string  `json:"name"`
	Symbol       string  `json:"symbol"`
	AtomicNumber int     `json:"atomic_number"`
	AtomicWeight float64 `json:"atomic_weight"`
	Category     string  `json:"category"`
	Group        int     `json:"group"`
	Period       int     `json:"period"`
}

func main() {
	data, err := ioutil.ReadFile("exhibit-c/chemistry.json")
	if err != nil {
		panic("Error reading phones.json")
	}

	e := Element{}
	err = json.Unmarshal(data, &e)
	if err != nil {
		panic(err)
	}

	fmt.Printf("Name:     %s\n", e.Name)
	fmt.Printf("Symbol:   %s\n", e.Symbol)
	fmt.Printf("Number:   %d\n", e.AtomicNumber)
	fmt.Printf("Weight:   %f\n", e.AtomicWeight)
	fmt.Printf("Category: %s\n", e.Category)
	fmt.Printf("Group:    %d\n", e.Group)
	fmt.Printf("Period:   %d\n", e.Period)
}
<commit_msg>Update app.go<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
)

type Element struct {
	Name         string  `json:"name"`
	Symbol       string  `json:"symbol"`
	AtomicNumber int     `json:"atomic_number"`
	AtomicWeight float64 `json:"atomic_weight"`
	Category     string  `json:"category"`
	Group        int     `json:"group"`
	Period       int     `json:"period"`
}

func main() {
	data, err := ioutil.ReadFile("exhibit-c/chemistry.json")
	if err != nil {
		panic("Error reading file")
	}

	e := Element{}
	err = json.Unmarshal(data, &e)
	if err != nil {
		panic(err)
	}

	fmt.Printf("Name:     %s\n", e.Name)
	fmt.Printf("Symbol:   %s\n", e.Symbol)
	fmt.Printf("Number:   %d\n", e.AtomicNumber)
	fmt.Printf("Weight:   %f\n", e.AtomicWeight)
	fmt.Printf("Category: %s\n", e.Category)
	fmt.Printf("Group:    %d\n", e.Group)
	fmt.Printf("Period:   %d\n", e.Period)
}
<|endoftext|>