<commit_before>package konnectors

import (
	"archive/tar"
	"bufio"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path"
	"runtime"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/cozy/cozy-stack/pkg/apps"
	"github.com/cozy/cozy-stack/pkg/config"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/jobs"
	"github.com/cozy/cozy-stack/pkg/realtime"
	"github.com/cozy/cozy-stack/pkg/stack"
	"github.com/cozy/cozy-stack/pkg/workers/mails"
	"github.com/spf13/afero"
)

func init() {
	jobs.AddWorker("konnector", &jobs.WorkerConfig{
		Concurrency:  runtime.NumCPU(),
		MaxExecCount: 2,
		MaxExecTime:  120 * time.Second,
		Timeout:      60 * time.Second,
		WorkerFunc:   Worker,
		WorkerCommit: commit,
	})
}

// Options contains the options to execute a konnector.
type Options struct {
	Konnector    string `json:"konnector"`
	Account      string `json:"account"`
	FolderToSave string `json:"folder_to_save"`
}

// result stores the result of a konnector execution.
type result struct {
	DocID     string    `json:"_id,omitempty"`
	DocRev    string    `json:"_rev,omitempty"`
	CreatedAt time.Time `json:"last_execution"`
	State     string    `json:"state"`
	Error     string    `json:"error"`
}

func (r *result) ID() string         { return r.DocID }
func (r *result) Rev() string        { return r.DocRev }
func (r *result) DocType() string    { return consts.KonnectorResults }
func (r *result) Clone() couchdb.Doc { return r }
func (r *result) SetID(id string)    { r.DocID = id }
func (r *result) SetRev(rev string)  { r.DocRev = rev }

// Worker is the worker that runs a konnector by executing an external process.
func Worker(ctx context.Context, m *jobs.Message) error {
	opts := &Options{}
	if err := m.Unmarshal(&opts); err != nil {
		return err
	}

	slug := opts.Konnector
	fields := struct {
		Account      string `json:"account"`
		FolderToSave string `json:"folder_to_save"`
	}{
		Account:      opts.Account,
		FolderToSave: opts.FolderToSave,
	}
	domain := ctx.Value(jobs.ContextDomainKey).(string)
	worker := ctx.Value(jobs.ContextWorkerKey).(string)
	jobID := fmt.Sprintf("%s/%s/%s", worker, slug, domain)

	inst, err := instance.Get(domain)
	if err != nil {
		return err
	}

	man, err := apps.GetKonnectorBySlug(inst, slug)
	if err != nil {
		return err
	}
	if man.State() != apps.Ready {
		return errors.New("Konnector is not ready")
	}

	token := inst.BuildKonnectorToken(man)

	osFS := afero.NewOsFs()
	workDir, err := afero.TempDir(osFS, "", "konnector-"+slug)
	if err != nil {
		return err
	}
	defer osFS.RemoveAll(workDir)
	workFS := afero.NewBasePathFs(osFS, workDir)

	fileServer := inst.KonnectorsFileServer()
	tarFile, err := fileServer.Open(slug, man.Version(), apps.KonnectorArchiveName)
	if err != nil {
		return err
	}

	tr := tar.NewReader(tarFile)
	for {
		var hdr *tar.Header
		hdr, err = tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}
		dirname := path.Dir(hdr.Name)
		if dirname != "." {
			if err = workFS.MkdirAll(dirname, 0755); err != nil {
				return nil
			}
		}
		var f afero.File
		f, err = workFS.OpenFile(hdr.Name, os.O_CREATE|os.O_WRONLY, os.FileMode(hdr.Mode))
		if err != nil {
			return err
		}
		_, err = io.Copy(f, tr)
		if err != nil {
			return err
		}
	}

	fieldsJSON, err := json.Marshal(fields)
	if err != nil {
		return err
	}

	konnCmd := config.GetConfig().Konnectors.Cmd
	cmd := exec.CommandContext(ctx, konnCmd, workDir) // #nosec
	cmd.Env = []string{
		"COZY_URL=" + inst.PageURL("/", nil),
		"COZY_CREDENTIALS=" + token,
		"COZY_FIELDS=" + string(fieldsJSON),
		"COZY_TYPE=" + man.Type,
		"COZY_JOB_ID=" + jobID,
	}

	cmdErr, err := cmd.StderrPipe()
	if err != nil {
		return err
	}
	cmdOut, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}

	scanErr := bufio.NewScanner(cmdErr)
	scanOut := bufio.NewScanner(cmdOut)
	scanOut.Buffer(nil, 256*1024)

	go doScanOut(jobID, scanOut, domain)
	go doScanErr(jobID, scanErr)

	if err = cmd.Start(); err != nil {
		return wrapErr(ctx, err)
	}
	if err = cmd.Wait(); err != nil {
		return wrapErr(ctx, err)
	}
	return nil
}

func doScanOut(jobID string, scanner *bufio.Scanner, domain string) {
	hub := realtime.GetHub()
	for scanner.Scan() {
		doc := couchdb.JSONDoc{Type: consts.JobEvents}
		err := json.Unmarshal(scanner.Bytes(), &doc.M)
		if err != nil {
			log.Warnf("[konnector] %s: Could not parse Stdout as JSON: %s", jobID, err)
			log.Warnf("[konnector] %s: Stdout: %s", jobID, scanner.Text())
			continue
		}
		hub.Publish(&realtime.Event{
			Type:   realtime.EventCreate,
			Doc:    doc,
			Domain: domain,
		})
	}
	if err := scanner.Err(); err != nil {
		log.Errorf("[konnector] %s: Error while reading stdout: %s", jobID, err)
	}
}

func doScanErr(jobID string, scanner *bufio.Scanner) {
	for scanner.Scan() {
		log.Errorf("[konnector] %s: Stderr: %s", jobID, scanner.Text())
	}
	if err := scanner.Err(); err != nil {
		log.Errorf("[konnector] %s: Error while reading stderr: %s", jobID, err)
	}
}

func commit(ctx context.Context, m *jobs.Message, errjob error) error {
	opts := &Options{}
	if err := m.Unmarshal(&opts); err != nil {
		return err
	}

	slug := opts.Konnector
	domain := ctx.Value(jobs.ContextDomainKey).(string)

	inst, err := instance.Get(domain)
	if err != nil {
		return err
	}

	lastResult := &result{}
	err = couchdb.GetDoc(inst, consts.KonnectorResults, slug, lastResult)
	if err != nil {
		if !couchdb.IsNotFoundError(err) {
			return err
		}
		lastResult = nil
	}

	var state, errstr string
	if errjob != nil {
		errstr = errjob.Error()
		state = jobs.Errored
	} else {
		state = jobs.Done
	}
	result := &result{
		DocID:     slug,
		CreatedAt: time.Now(),
		State:     state,
		Error:     errstr,
	}
	if lastResult == nil {
		err = couchdb.CreateNamedDocWithDB(inst, result)
	} else {
		result.SetRev(lastResult.Rev())
		err = couchdb.UpdateDoc(inst, result)
	}
	if err != nil {
		return err
	}

	// if it is the first try we do not take into account an error, we bail.
	if lastResult == nil {
		return nil
	}
	// if the job has not errored, or the last one was already errored, we bail.
	if state != jobs.Errored || lastResult.State == jobs.Errored {
		return nil
	}

	msg, err := jobs.NewMessage(jobs.JSONEncoding, &mails.Options{
		Mode:           mails.ModeNoReply,
		Subject:        inst.Translate("Konnector execution error"),
		TemplateName:   "konnector_error_" + inst.Locale,
		TemplateValues: map[string]string{"KonnectorName": slug},
	})
	if err != nil {
		return err
	}
	_, err = stack.GetBroker().PushJob(&jobs.JobRequest{
		Domain:     domain,
		WorkerType: "sendmail",
		Message:    msg,
	})
	return err
}

func wrapErr(ctx context.Context, err error) error {
	if ctx.Err() == context.DeadlineExceeded {
		return context.DeadlineExceeded
	}
	return err
}
<commit_msg>parse json message from konnector and fail job if an error occured<commit_after>package konnectors

import (
	"archive/tar"
	"bufio"
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path"
	"runtime"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/cozy/cozy-stack/pkg/apps"
	"github.com/cozy/cozy-stack/pkg/config"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/jobs"
	"github.com/cozy/cozy-stack/pkg/realtime"
	"github.com/cozy/cozy-stack/pkg/stack"
	"github.com/cozy/cozy-stack/pkg/workers/mails"
	"github.com/spf13/afero"
)

func init() {
	jobs.AddWorker("konnector", &jobs.WorkerConfig{
		Concurrency:  runtime.NumCPU(),
		MaxExecCount: 2,
		MaxExecTime:  120 * time.Second,
		Timeout:      60 * time.Second,
		WorkerFunc:   Worker,
		WorkerCommit: commit,
	})
}

// Options contains the options to execute a konnector.
type Options struct {
	Konnector    string `json:"konnector"`
	Account      string `json:"account"`
	FolderToSave string `json:"folder_to_save"`
}

// result stores the result of a konnector execution.
type result struct {
	DocID     string    `json:"_id,omitempty"`
	DocRev    string    `json:"_rev,omitempty"`
	CreatedAt time.Time `json:"last_execution"`
	State     string    `json:"state"`
	Error     string    `json:"error"`
}

func (r *result) ID() string         { return r.DocID }
func (r *result) Rev() string        { return r.DocRev }
func (r *result) DocType() string    { return consts.KonnectorResults }
func (r *result) Clone() couchdb.Doc { return r }
func (r *result) SetID(id string)    { r.DocID = id }
func (r *result) SetRev(rev string)  { r.DocRev = rev }

const konnectorMsgTypeError string = "error"
const konnectorMsgTypeDebug string = "debug"
const konnectorMsgTypeWarning string = "warning"
const konnectorMsgTypeProgress string = "progress"

type konnectorMsg struct {
	Type    string `json:"type"`
	Message string `json:"message"`
}

// Worker is the worker that runs a konnector by executing an external process.
func Worker(ctx context.Context, m *jobs.Message) error {
	opts := &Options{}
	if err := m.Unmarshal(&opts); err != nil {
		return err
	}

	slug := opts.Konnector
	fields := struct {
		Account      string `json:"account"`
		FolderToSave string `json:"folder_to_save"`
	}{
		Account:      opts.Account,
		FolderToSave: opts.FolderToSave,
	}
	domain := ctx.Value(jobs.ContextDomainKey).(string)
	worker := ctx.Value(jobs.ContextWorkerKey).(string)
	jobID := fmt.Sprintf("%s/%s/%s", worker, slug, domain)

	inst, err := instance.Get(domain)
	if err != nil {
		return err
	}

	man, err := apps.GetKonnectorBySlug(inst, slug)
	if err != nil {
		return err
	}
	if man.State() != apps.Ready {
		return errors.New("Konnector is not ready")
	}

	token := inst.BuildKonnectorToken(man)

	osFS := afero.NewOsFs()
	workDir, err := afero.TempDir(osFS, "", "konnector-"+slug)
	if err != nil {
		return err
	}
	defer osFS.RemoveAll(workDir)
	workFS := afero.NewBasePathFs(osFS, workDir)

	fileServer := inst.KonnectorsFileServer()
	tarFile, err := fileServer.Open(slug, man.Version(), apps.KonnectorArchiveName)
	if err != nil {
		return err
	}

	tr := tar.NewReader(tarFile)
	for {
		var hdr *tar.Header
		hdr, err = tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}
		dirname := path.Dir(hdr.Name)
		if dirname != "." {
			if err = workFS.MkdirAll(dirname, 0755); err != nil {
				return nil
			}
		}
		var f afero.File
		f, err = workFS.OpenFile(hdr.Name, os.O_CREATE|os.O_WRONLY, os.FileMode(hdr.Mode))
		if err != nil {
			return err
		}
		_, err = io.Copy(f, tr)
		if err != nil {
			return err
		}
	}

	fieldsJSON, err := json.Marshal(fields)
	if err != nil {
		return err
	}

	konnCmd := config.GetConfig().Konnectors.Cmd
	cmd := exec.CommandContext(ctx, konnCmd, workDir) // #nosec
	cmd.Env = []string{
		"COZY_URL=" + inst.PageURL("/", nil),
		"COZY_CREDENTIALS=" + token,
		"COZY_FIELDS=" + string(fieldsJSON),
		"COZY_TYPE=" + man.Type,
		"COZY_JOB_ID=" + jobID,
	}

	cmdErr, err := cmd.StderrPipe()
	if err != nil {
		return err
	}
	cmdOut, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}

	scanErr := bufio.NewScanner(cmdErr)
	scanOut := bufio.NewScanner(cmdOut)
	scanOut.Buffer(nil, 256*1024)

	var msgChan = make(chan konnectorMsg)
	var messages []konnectorMsg

	go doScanOut(jobID, scanOut, domain, msgChan)
	go doScanErr(jobID, scanErr)
	go func() {
		hub := realtime.GetHub()
		for msg := range msgChan {
			messages = append(messages, msg)
			hub.Publish(&realtime.Event{
				Type: realtime.EventCreate,
				Doc: couchdb.JSONDoc{Type: consts.JobEvents, M: map[string]interface{}{
					"type":    msg.Type,
					"message": msg.Message,
				}},
				Domain: domain,
			})
		}
	}()

	if err = cmd.Start(); err != nil {
		return wrapErr(ctx, err)
	}
	if err = cmd.Wait(); err != nil {
		return wrapErr(ctx, err)
	}
	for _, msg := range messages {
		if msg.Type == konnectorMsgTypeError {
			err = errors.New(msg.Message)
			return err
		}
	}
	close(msgChan)
	return nil
}

func doScanOut(jobID string, scanner *bufio.Scanner, domain string, msgs chan konnectorMsg) {
	for scanner.Scan() {
		linebb := scanner.Bytes()
		from := bytes.IndexByte(linebb, '{')
		to := bytes.LastIndexByte(linebb, '}')
		var msg konnectorMsg
		log.Infof("[konnector] %s: Stdout: %s", jobID, string(linebb[from:to+1]))
		if from > -1 && to > -1 {
			err := json.Unmarshal(linebb[from:to+1], &msg)
			if err == nil {
				msgs <- msg
				continue
			}
		}
		log.Warnf("[konnector] %s: Could not parse as JSON", jobID)
	}
	if err := scanner.Err(); err != nil {
		log.Errorf("[konnector] %s: Error while reading stdout: %s", jobID, err)
	}
}

func doScanErr(jobID string, scanner *bufio.Scanner) {
	for scanner.Scan() {
		log.Errorf("[konnector] %s: Stderr: %s", jobID, scanner.Text())
	}
	if err := scanner.Err(); err != nil {
		log.Errorf("[konnector] %s: Error while reading stderr: %s", jobID, err)
	}
}

func commit(ctx context.Context, m *jobs.Message, errjob error) error {
	opts := &Options{}
	if err := m.Unmarshal(&opts); err != nil {
		return err
	}

	slug := opts.Konnector
	domain := ctx.Value(jobs.ContextDomainKey).(string)

	inst, err := instance.Get(domain)
	if err != nil {
		return err
	}

	lastResult := &result{}
	err = couchdb.GetDoc(inst, consts.KonnectorResults, slug, lastResult)
	if err != nil {
		if !couchdb.IsNotFoundError(err) {
			return err
		}
		lastResult = nil
	}

	var state, errstr string
	if errjob != nil {
		errstr = errjob.Error()
		state = jobs.Errored
	} else {
		state = jobs.Done
	}
	result := &result{
		DocID:     slug,
		CreatedAt: time.Now(),
		State:     state,
		Error:     errstr,
	}
	if lastResult == nil {
		err = couchdb.CreateNamedDocWithDB(inst, result)
	} else {
		result.SetRev(lastResult.Rev())
		err = couchdb.UpdateDoc(inst, result)
	}
	if err != nil {
		return err
	}

	// if it is the first try we do not take into account an error, we bail.
	if lastResult == nil {
		return nil
	}
	// if the job has not errored, or the last one was already errored, we bail.
	if state != jobs.Errored || lastResult.State == jobs.Errored {
		return nil
	}

	msg, err := jobs.NewMessage(jobs.JSONEncoding, &mails.Options{
		Mode:           mails.ModeNoReply,
		Subject:        inst.Translate("Konnector execution error"),
		TemplateName:   "konnector_error_" + inst.Locale,
		TemplateValues: map[string]string{"KonnectorName": slug},
	})
	if err != nil {
		return err
	}
	_, err = stack.GetBroker().PushJob(&jobs.JobRequest{
		Domain:     domain,
		WorkerType: "sendmail",
		Message:    msg,
	})
	return err
}

func wrapErr(ctx context.Context, err error) error {
	if ctx.Err() == context.DeadlineExceeded {
		return context.DeadlineExceeded
	}
	return err
}
<|endoftext|>


<commit_before>// Copyright 2016 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"encoding/json"
	"fmt"
	"strconv"

	"github.com/google/uuid"
	"github.com/maruel/subcommands"

	"go.chromium.org/luci/auth"
	"go.chromium.org/luci/common/api/buildbucket/buildbucket/v1"
	"go.chromium.org/luci/common/cli"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
)

func cmdPutBatch(defaultAuthOpts auth.Options) *subcommands.Command {
	return &subcommands.Command{
		UsageLine: `put [flags] <JSON request>...`,
		ShortDesc: "schedule builds",
		LongDesc: "Schedule builds. \n" +
			"See https://godoc.org/go.chromium.org/luci/common/api/" +
			"buildbucket/buildbucket/v1#ApiPutBatchRequestMessage " +
			"for JSON request message schema.",
		CommandRun: func() subcommands.CommandRun {
			c := &putBatchRun{}
			c.SetDefaultFlags(defaultAuthOpts)
			return c
		},
	}
}

type putBatchRun struct {
	baseCommandRun
}

func parsePutRequest(args []string, prefixFunc func() (string, error)) (*buildbucket.ApiPutBatchRequestMessage, error) {
	if len(args) <= 0 {
		return nil, fmt.Errorf("missing parameter: <JSON Request>")
	}

	req := &buildbucket.ApiPutBatchRequestMessage{}
	opIDPrefix := ""
	for i, a := range args {
		// verify that args are valid here before sending the request.
		putReq := &buildbucket.ApiPutRequestMessage{}
		if err := json.Unmarshal([]byte(a), putReq); err != nil {
			return nil, errors.Annotate(err, "invalid build request #%d", i).Err()
		}
		if putReq.ClientOperationId == "" {
			if opIDPrefix == "" {
				var err error
				opIDPrefix, err = prefixFunc()
				if err != nil {
					return nil, err
				}
			}
			putReq.ClientOperationId = opIDPrefix + "-" + strconv.Itoa(i)
		}
		req.Builds = append(req.Builds, putReq)
	}

	return req, nil
}

func (r *putBatchRun) Run(a subcommands.Application, args []string, env subcommands.Env) int {
	ctx := cli.GetContext(a, r, env)

	req, err := parsePutRequest(args, func() (string, error) {
		id, err := uuid.NewRandom()
		return id.String(), err
	})
	if err != nil {
		return r.done(ctx, err)
	}

	client, err := r.createClient(ctx)
	if err != nil {
		return r.done(ctx, err)
	}

	resBytes, err := client.call(ctx, "PUT", "builds/batch", req)
	if err != nil {
		return r.done(ctx, err)
	}

	var res buildbucket.ApiPutBatchResponseMessage
	if err := json.Unmarshal(resBytes, &res); err != nil {
		return r.done(ctx, err)
	}

	hasErrors := false
	for i, r := range res.Results {
		if r.Error != nil {
			hasErrors = true
			logging.Errorf(ctx, "Failed to schedule build #%d (reason: %s): %s\n",
				i, r.Error.Reason, r.Error.Message)
		}
	}
	if hasErrors {
		return 1
	}

	fmt.Printf("%s\n", resBytes)
	return 0

}
<commit_msg>[buildbucket] Check response-level error<commit_after>// Copyright 2016 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"encoding/json"
	"fmt"
	"strconv"

	"github.com/google/uuid"
	"github.com/maruel/subcommands"

	"go.chromium.org/luci/auth"
	"go.chromium.org/luci/common/api/buildbucket/buildbucket/v1"
	"go.chromium.org/luci/common/cli"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
)

func cmdPutBatch(defaultAuthOpts auth.Options) *subcommands.Command {
	return &subcommands.Command{
		UsageLine: `put [flags] <JSON request>...`,
		ShortDesc: "schedule builds",
		LongDesc: "Schedule builds. \n" +
			"See https://godoc.org/go.chromium.org/luci/common/api/" +
			"buildbucket/buildbucket/v1#ApiPutBatchRequestMessage " +
			"for JSON request message schema.",
		CommandRun: func() subcommands.CommandRun {
			c := &putBatchRun{}
			c.SetDefaultFlags(defaultAuthOpts)
			return c
		},
	}
}

type putBatchRun struct {
	baseCommandRun
}

func parsePutRequest(args []string, prefixFunc func() (string, error)) (*buildbucket.ApiPutBatchRequestMessage, error) {
	if len(args) <= 0 {
		return nil, fmt.Errorf("missing parameter: <JSON Request>")
	}

	req := &buildbucket.ApiPutBatchRequestMessage{}
	opIDPrefix := ""
	for i, a := range args {
		// verify that args are valid here before sending the request.
		putReq := &buildbucket.ApiPutRequestMessage{}
		if err := json.Unmarshal([]byte(a), putReq); err != nil {
			return nil, errors.Annotate(err, "invalid build request #%d", i).Err()
		}
		if putReq.ClientOperationId == "" {
			if opIDPrefix == "" {
				var err error
				opIDPrefix, err = prefixFunc()
				if err != nil {
					return nil, err
				}
			}
			putReq.ClientOperationId = opIDPrefix + "-" + strconv.Itoa(i)
		}
		req.Builds = append(req.Builds, putReq)
	}

	return req, nil
}

func (r *putBatchRun) Run(a subcommands.Application, args []string, env subcommands.Env) int {
	ctx := cli.GetContext(a, r, env)

	req, err := parsePutRequest(args, func() (string, error) {
		id, err := uuid.NewRandom()
		return id.String(), err
	})
	if err != nil {
		return r.done(ctx, err)
	}

	client, err := r.createClient(ctx)
	if err != nil {
		return r.done(ctx, err)
	}

	resBytes, err := client.call(ctx, "PUT", "builds/batch", req)
	if err != nil {
		return r.done(ctx, err)
	}

	var res buildbucket.ApiPutBatchResponseMessage
	if err := json.Unmarshal(resBytes, &res); err != nil {
		return r.done(ctx, err)
	}

	hasErrors := false

	if res.Error != nil {
		hasErrors = true
		logging.Errorf(ctx, "Failed to schedule builds (reason: %s): %s",
			res.Error.Reason, res.Error.Message)
	}

	for i, r := range res.Results {
		if r.Error != nil {
			hasErrors = true
			logging.Errorf(ctx, "Failed to schedule build #%d (reason: %s): %s\n",
				i, r.Error.Reason, r.Error.Message)
		}
	}
	if hasErrors {
		return 1
	}

	fmt.Printf("%s\n", resBytes)
	return 0

}
<|endoftext|>


<commit_before>package libvirt

import (
	"fmt"
	"log"
	"strings"
	"time"
	"unicode/utf8"

	"github.com/mitchellh/multistep"
	"github.com/mitchellh/packer/packer"
	"gopkg.in/alexzorin/libvirt-go.v2"
)

const KeyLeftShift uint32 = 0xFFE1

type bootCommandTemplateData struct {
	HTTPIP   string
	HTTPPort uint
	Name     string
}

// This step "types" the boot command into the VM over VNC.
//
// Uses:
//   config *config
//   http_port int
//   ui     packer.Ui
//
// Produces:
//   <nothing>
type stepTypeBootCommand struct{}

func (s *stepTypeBootCommand) Run(state multistep.StateBag) multistep.StepAction {
	config := state.Get("config").(*Config)
	//	httpPort := state.Get("http_port").(uint)
	//	hostIp := state.Get("host_ip").(string)
	ui := state.Get("ui").(packer.Ui)

	var lvd libvirt.VirDomain
	lv, err := libvirt.NewVirConnection(config.LibvirtUrl)
	if err != nil {
		err := fmt.Errorf("Error connecting to libvirt: %s", err)
		state.Put("error", err)
		ui.Error(err.Error())
		return multistep.ActionHalt
	}
	defer lv.CloseConnection()
	if lvd, err = lv.LookupDomainByName(config.VMName); err != nil {
		err := fmt.Errorf("Error lookup domain: %s", err)
		state.Put("error", err)
		ui.Error(err.Error())
		return multistep.ActionHalt
	}
	defer lvd.Free()

	//	tplData := &bootCommandTemplateData{
	//		hostIp,
	//		httpPort,
	//		config.VMName,
	//	}

	ui.Say("Typing the boot command...")
	for _, command := range config.BootCommand {
		//		command, err := config.tpl.Process(command, tplData)
		//		if err != nil {
		//			err := fmt.Errorf("Error preparing boot command: %s", err)
		//			state.Put("error", err)
		//			ui.Error(err.Error())
		//			return multistep.ActionHalt
		//		}

		// Check for interrupts between typing things so we can cancel
		// since this isn't the fastest thing.
		if _, ok := state.GetOk(multistep.StateCancelled); ok {
			return multistep.ActionHalt
		}

		sendBootString(lvd, command)
	}

	return multistep.ActionContinue
}

func (*stepTypeBootCommand) Cleanup(multistep.StateBag) {}

func sendBootString(d libvirt.VirDomain, original string) {
	var err error
	var key uint

	for len(original) > 0 {
		time.Sleep(100 * time.Millisecond)
		if strings.HasPrefix(original, "<wait>") {
			log.Printf("Special code '<wait>' found, sleeping one second")
			time.Sleep(1 * time.Second)
			original = original[len("<wait>"):]
			continue
		}

		if strings.HasPrefix(original, "<wait5>") {
			log.Printf("Special code '<wait5>' found, sleeping 5 seconds")
			time.Sleep(5 * time.Second)
			original = original[len("<wait5>"):]
			continue
		}

		if strings.HasPrefix(original, "<wait10>") {
			log.Printf("Special code '<wait10>' found, sleeping 10 seconds")
			time.Sleep(10 * time.Second)
			original = original[len("<wait10>"):]
			continue
		}

		if strings.HasPrefix(original, "<esc>") {
			d.SendKey(libvirt.VIR_KEYCODE_SET_RFB, 100, []uint{ecodes["<esc>"]}, 0)
			original = original[len("<esc>"):]
			continue
		}

		if strings.HasPrefix(original, "<enter>") {
			d.SendKey(libvirt.VIR_KEYCODE_SET_RFB, 100, []uint{ecodes["<enter>"]}, 0)
			original = original[len("<enter>"):]
			continue
		}

		log.Printf("command %s", original)
		r, size := utf8.DecodeRuneInString(original)
		original = original[size:]
		key = ecodes[string(r)]
		log.Printf("find code for char %s %d", string(r), key)
		//VIR_KEYCODE_SET_LINUX, VIR_KEYCODE_SET_USB, VIR_KEYCODE_SET_RFB, VIR_KEYCODE_SET_WIN32, VIR_KEYCODE_SET_XT_KBD
		if err = d.SendKey(libvirt.VIR_KEYCODE_SET_RFB, 100, []uint{key}, 0); err != nil {
			log.Printf("Sending code %d failed: %s", key, err.Error())
		}
	}

}
<commit_msg>fix<commit_after>package libvirt

import (
	"fmt"
	"log"
	"strings"
	"time"
	"unicode/utf8"

	"github.com/mitchellh/multistep"
	"github.com/mitchellh/packer/packer"
	"gopkg.in/alexzorin/libvirt-go.v2"
)

const KeyLeftShift uint32 = 0xFFE1

type bootCommandTemplateData struct {
	HTTPIP   string
	HTTPPort uint
	Name     string
}

// This step "types" the boot command into the VM over VNC.
//
// Uses:
//   config *config
//   http_port int
//   ui     packer.Ui
//
// Produces:
//   <nothing>
type stepTypeBootCommand struct{}

func (s *stepTypeBootCommand) Run(state multistep.StateBag) multistep.StepAction {
	config := state.Get("config").(*Config)
	//	httpPort := state.Get("http_port").(uint)
	//	hostIp := state.Get("host_ip").(string)
	ui := state.Get("ui").(packer.Ui)

	var lvd libvirt.VirDomain
	lv, err := libvirt.NewVirConnection(config.LibvirtUrl)
	if err != nil {
		err := fmt.Errorf("Error connecting to libvirt: %s", err)
		state.Put("error", err)
		ui.Error(err.Error())
		return multistep.ActionHalt
	}
	defer lv.CloseConnection()
	if lvd, err = lv.LookupDomainByName(config.VMName); err != nil {
		err := fmt.Errorf("Error lookup domain: %s", err)
		state.Put("error", err)
		ui.Error(err.Error())
		return multistep.ActionHalt
	}
	defer lvd.Free()

	//	tplData := &bootCommandTemplateData{
	//		hostIp,
	//		httpPort,
	//		config.VMName,
	//	}

	ui.Say("Typing the boot command...")
	for _, command := range config.BootCommand {
		//		command, err := config.tpl.Process(command, tplData)
		//		if err != nil {
		//			err := fmt.Errorf("Error preparing boot command: %s", err)
		//			state.Put("error", err)
		//			ui.Error(err.Error())
		//			return multistep.ActionHalt
		//		}

		// Check for interrupts between typing things so we can cancel
		// since this isn't the fastest thing.
		if _, ok := state.GetOk(multistep.StateCancelled); ok {
			return multistep.ActionHalt
		}

		sendBootString(lvd, command)
	}

	return multistep.ActionContinue
}

func (*stepTypeBootCommand) Cleanup(multistep.StateBag) {}

func sendBootString(d libvirt.VirDomain, original string) {
	var err error
	var key uint

	for len(original) > 0 {
		time.Sleep(50 * time.Millisecond)
		if strings.HasPrefix(original, "<wait>") {
			log.Printf("Special code '<wait>' found, sleeping one second")
			time.Sleep(1 * time.Second)
			original = original[len("<wait>"):]
			continue
		}

		if strings.HasPrefix(original, "<wait5>") {
			log.Printf("Special code '<wait5>' found, sleeping 5 seconds")
			time.Sleep(5 * time.Second)
			original = original[len("<wait5>"):]
			continue
		}

		if strings.HasPrefix(original, "<wait10>") {
			log.Printf("Special code '<wait10>' found, sleeping 10 seconds")
			time.Sleep(10 * time.Second)
			original = original[len("<wait10>"):]
			continue
		}

		if strings.HasPrefix(original, "<esc>") {
			d.SendKey(libvirt.VIR_KEYCODE_SET_RFB, 50, []uint{ecodes["<esc>"]}, 0)
			original = original[len("<esc>"):]
			continue
		}

		if strings.HasPrefix(original, "<enter>") {
			d.SendKey(libvirt.VIR_KEYCODE_SET_RFB, 50, []uint{ecodes["<enter>"]}, 0)
			original = original[len("<enter>"):]
			continue
		}

		log.Printf("command %s", original)
		r, size := utf8.DecodeRuneInString(original)
		original = original[size:]
		key = ecodes[string(r)]
		log.Printf("find code for char %s %d", string(r), key)
		//VIR_KEYCODE_SET_LINUX, VIR_KEYCODE_SET_USB, VIR_KEYCODE_SET_RFB, VIR_KEYCODE_SET_WIN32, VIR_KEYCODE_SET_XT_KBD
		if err = d.SendKey(libvirt.VIR_KEYCODE_SET_RFB, 50, []uint{key}, 0); err != nil {
			log.Printf("Sending code %d failed: %s", key, err.Error())
		}
	}

}
<|endoftext|>


<commit_before>package repo_manager

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strings"
	"text/template"
	"time"

	"go.skia.org/infra/go/depot_tools"
	"go.skia.org/infra/go/exec"
	"go.skia.org/infra/go/gerrit"
	"go.skia.org/infra/go/issues"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/util"
)

const (
	GCLIENT = "gclient.py"

	TMPL_COMMIT_MESSAGE = `Roll {{.ChildPath}} {{.From}}..{{.To}} ({{.NumCommits}} commits)

{{.ChildRepo}}/+log/{{.From}}..{{.To}}

{{.LogStr}}
Created with:
  gclient setdep -r {{.ChildPath}}@{{.To}}

The AutoRoll server is located here: {{.ServerURL}}

Documentation for the AutoRoller is here:
https://skia.googlesource.com/buildbot/+/master/autoroll/README.md

If the roll is causing failures, please contact the current sheriff, who should
be CC'd on the roll, and stop the roller if necessary.

{{.Footer}}
`
	TMPL_CQ_INCLUDE_TRYBOTS = "CQ_INCLUDE_TRYBOTS=%s"
)

var (
	// Use this function to instantiate a RepoManager. This is able to be
	// overridden for testing.
	NewDEPSRepoManager func(context.Context, *DEPSRepoManagerConfig, string, *gerrit.Gerrit, string, string) (RepoManager, error) = newDEPSRepoManager

	commitMsgTmpl = template.Must(template.New("commitMsg").Parse(TMPL_COMMIT_MESSAGE))
)

// issueJson is the structure of "git cl issue --json"
type issueJson struct {
	Issue    int64  `json:"issue"`
	IssueUrl string `json:"issue_url"`
}

// depsRepoManager is a struct used by DEPs AutoRoller for managing checkouts.
type depsRepoManager struct {
	*depotToolsRepoManager
	includeLog bool
}

// DEPSRepoManagerConfig provides configuration for the DEPS RepoManager.
type DEPSRepoManagerConfig struct {
	DepotToolsRepoManagerConfig

	// If false, roll CLs do not include a git log.
	IncludeLog bool `json:"includeLog"`
}

// Validate the config.
func (c *DEPSRepoManagerConfig) Validate() error {
	return c.DepotToolsRepoManagerConfig.Validate()
}

// newDEPSRepoManager returns a RepoManager instance which operates in the given
// working directory and updates at the given frequency.
func newDEPSRepoManager(ctx context.Context, c *DEPSRepoManagerConfig, workdir string, g *gerrit.Gerrit, recipeCfgFile, serverURL string) (RepoManager, error) {
	if err := c.Validate(); err != nil {
		return nil, err
	}
	drm, err := newDepotToolsRepoManager(ctx, c.DepotToolsRepoManagerConfig, path.Join(workdir, "repo_manager"), recipeCfgFile, serverURL, g)
	if err != nil {
		return nil, err
	}
	dr := &depsRepoManager{
		depotToolsRepoManager: drm,
		includeLog:            c.IncludeLog,
	}

	// TODO(borenet): This update can be extremely expensive. Consider
	// moving it out of the startup critical path.
	return dr, dr.Update(ctx)
}

// Update syncs code in the relevant repositories.
func (dr *depsRepoManager) Update(ctx context.Context) error {
	// Sync the projects.
	dr.repoMtx.Lock()
	defer dr.repoMtx.Unlock()

	if err := dr.createAndSyncParent(ctx); err != nil {
		return fmt.Errorf("Could not create and sync parent repo: %s", err)
	}

	// Get the last roll revision.
	lastRollRev, err := dr.getLastRollRev(ctx)
	if err != nil {
		return err
	}

	// Get the next roll revision.
	nextRollRev, err := dr.strategy.GetNextRollRev(ctx, dr.childRepo, lastRollRev)
	if err != nil {
		return err
	}

	// Find the number of not-rolled child repo commits.
	notRolled, err := dr.getCommitsNotRolled(ctx, lastRollRev)
	if err != nil {
		return err
	}

	dr.infoMtx.Lock()
	defer dr.infoMtx.Unlock()
	dr.lastRollRev = lastRollRev
	dr.nextRollRev = nextRollRev
	dr.commitsNotRolled = notRolled
	return nil
}

// getLastRollRev returns the commit hash of the last-completed DEPS roll.
func (dr *depsRepoManager) getLastRollRev(ctx context.Context) (string, error) {
	output, err := exec.RunCwd(ctx, dr.parentDir, "python", dr.gclient, "getdep", "-r", dr.childPath)
	if err != nil {
		return "", err
	}
	commit := strings.TrimSpace(output)
	if len(commit) != 40 {
		return "", fmt.Errorf("Got invalid output for `gclient getdep`: %s", output)
	}
	return commit, nil
}

func getLocalPartOfEmailAddress(emailAddress string) string {
	return strings.SplitN(emailAddress, "@", 2)[0]
}

// Helper function for building the commit message.
func (dr *depsRepoManager) buildCommitMsg(ctx context.Context, from, to, cqExtraTrybots string, bugs []string) (string, error) {
	logStr, err := exec.RunCwd(ctx, dr.childDir, "git", "log", fmt.Sprintf("%s..%s", from, to), "--date=short", "--no-merges", "--format='%ad %ae %s'")
	if err != nil {
		return "", err
	}
	numCommits := len(strings.Split(logStr, "\n"))
	remoteUrl, err := exec.RunCwd(ctx, dr.childDir, "git", "remote", "get-url", "origin")
	if err != nil {
		return "", err
	}
	remoteUrl = strings.TrimSpace(remoteUrl)
	data := struct {
		ChildPath  string
		ChildRepo  string
		From       string
		To         string
		NumCommits int
		LogURL     string
		LogStr     string
		ServerURL  string
		Footer     string
	}{
		ChildPath:  dr.childPath,
		ChildRepo:  remoteUrl,
		From:       from[:7],
		To:         to[:7],
		NumCommits: numCommits,
		LogStr:     "",
		ServerURL:  dr.serverURL,
		Footer:     "",
	}
	if cqExtraTrybots != "" {
		data.Footer += fmt.Sprintf(TMPL_CQ_INCLUDE_TRYBOTS, cqExtraTrybots)
	}
	if len(bugs) > 0 {
		data.Footer += "\n\nBUG=" + strings.Join(bugs, ",")
	}
	if dr.includeLog {
		data.LogStr = fmt.Sprintf("\ngit log %s..%s --date=short --no-merges --format='%%ad %%ae %%s'", from[:7], to[:7])
		data.LogStr += logStr
	}
	var buf bytes.Buffer
	if err := commitMsgTmpl.Execute(&buf, data); err != nil {
		return "", err
	}
	commitMsg := buf.String()
	return commitMsg, nil
}

// CreateNewRoll creates and uploads a new DEPS roll to the given commit.
// Returns the issue number of the uploaded roll.
func (dr *depsRepoManager) CreateNewRoll(ctx context.Context, from, to string, emails []string, cqExtraTrybots string, dryRun bool) (int64, error) {
	dr.repoMtx.Lock()
	defer dr.repoMtx.Unlock()

	// Clean the checkout, get onto a fresh branch.
	if err := dr.cleanParent(ctx); err != nil {
		return 0, err
	}
	if _, err := exec.RunCwd(ctx, dr.parentDir, "git", "checkout", "-b", ROLL_BRANCH, "-t", fmt.Sprintf("origin/%s", dr.parentBranch), "-f"); err != nil {
		return 0, err
	}

	// Defer some more cleanup.
	defer func() {
		util.LogErr(dr.cleanParent(ctx))
	}()

	// Create the roll CL.
	cr := dr.childRepo
	commits, err := cr.RevList(ctx, fmt.Sprintf("%s..%s", from, to))
	if err != nil {
		return 0, fmt.Errorf("Failed to list revisions: %s", err)
	}

	if _, err := exec.RunCwd(ctx, dr.parentDir, "git", "config", "user.name", getLocalPartOfEmailAddress(dr.user)); err != nil {
		return 0, err
	}
	if _, err := exec.RunCwd(ctx, dr.parentDir, "git", "config", "user.email", dr.user); err != nil {
		return 0, err
	}

	// Find relevant bugs.
	bugs := []string{}
	monorailProject := issues.REPO_PROJECT_MAPPING[dr.parentRepo]
	if monorailProject == "" {
		sklog.Warningf("Found no entry in issues.REPO_PROJECT_MAPPING for %q", dr.parentRepo)
	} else {
		for _, c := range commits {
			d, err := cr.Details(ctx, c)
			if err != nil {
				return 0, fmt.Errorf("Failed to obtain commit details: %s", err)
			}
			b := util.BugsFromCommitMsg(d.Body)
			for _, bug := range b[monorailProject] {
				bugs = append(bugs, fmt.Sprintf("%s:%s", monorailProject, bug))
			}
		}
	}

	// Run "gclient setdep".
	args := []string{"setdep", "-r", fmt.Sprintf("%s@%s", dr.childPath, to)}
	sklog.Infof("Running command: gclient %s", strings.Join(args, " "))
	if _, err := exec.RunCommand(ctx, &exec.Command{
		Dir:  dr.parentDir,
		Env:  depot_tools.Env(dr.depotTools),
		Name: dr.gclient,
		Args: args,
	}); err != nil {
		return 0, err
	}

	// Build the commit message.
	commitMsg, err := dr.buildCommitMsg(ctx, from, to, cqExtraTrybots, bugs)
	if err != nil {
		return 0, err
	}

	// Run the pre-upload steps.
	for _, s := range dr.PreUploadSteps() {
		if err := s(ctx, dr.parentDir); err != nil {
			return 0, fmt.Errorf("Failed pre-upload step: %s", err)
		}
	}

	// Commit.
	if _, err := exec.RunCwd(ctx, dr.parentDir, "git", "commit", "-a", "-m", commitMsg); err != nil {
		return 0, err
	}

	// Upload the CL.
	uploadCmd := &exec.Command{
		Dir:     dr.parentDir,
		Env:     depot_tools.Env(dr.depotTools),
		Name:    "git",
		Args:    []string{"cl", "upload", "--bypass-hooks", "-f", "-v", "-v"},
		Timeout: 2 * time.Minute,
	}
	if dryRun {
		uploadCmd.Args = append(uploadCmd.Args, "--cq-dry-run")
	} else {
		uploadCmd.Args = append(uploadCmd.Args, "--use-commit-queue")
	}
	uploadCmd.Args = append(uploadCmd.Args, "--gerrit")
	tbr := "\nTBR="
	if emails != nil && len(emails) > 0 {
		emailStr := strings.Join(emails, ",")
		tbr += emailStr
		uploadCmd.Args = append(uploadCmd.Args, "--send-mail", "--cc", emailStr)
	}
	commitMsg += tbr
	uploadCmd.Args = append(uploadCmd.Args, "-m", commitMsg)

	// Upload the CL.
	sklog.Infof("Running command: git %s", strings.Join(uploadCmd.Args, " "))
	if _, err := exec.RunCommand(ctx, uploadCmd); err != nil {
		return 0, err
	}

	// Obtain the issue number.
	tmp, err := ioutil.TempDir("", "")
	if err != nil {
		return 0, err
	}
	defer util.RemoveAll(tmp)
	jsonFile := path.Join(tmp, "issue.json")
	if _, err := exec.RunCommand(ctx, &exec.Command{
		Dir:  dr.parentDir,
		Env:  depot_tools.Env(dr.depotTools),
		Name: "git",
		Args: []string{"cl", "issue", fmt.Sprintf("--json=%s", jsonFile)},
	}); err != nil {
		return 0, err
	}
	f, err := os.Open(jsonFile)
	if err != nil {
		return 0, err
	}
	var issue issueJson
	if err := json.NewDecoder(f).Decode(&issue); err != nil {
		return 0, err
	}
	return issue.Issue, nil
}
<commit_msg>[autoroll] Fix commit message for new setdep flow<commit_after>package repo_manager

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strings"
	"text/template"
	"time"

	"go.skia.org/infra/go/depot_tools"
	"go.skia.org/infra/go/exec"
	"go.skia.org/infra/go/gerrit"
	"go.skia.org/infra/go/issues"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/util"
)

const (
	GCLIENT = "gclient.py"

	TMPL_COMMIT_MESSAGE = `Roll {{.ChildPath}} {{.From}}..{{.To}} ({{.NumCommits}} commits)

{{.ChildRepo}}/+log/{{.From}}..{{.To}}

{{.LogStr}}
Created with:
  gclient setdep -r {{.ChildPath}}@{{.To}}

The AutoRoll server is located here: {{.ServerURL}}

Documentation for the AutoRoller is here:
https://skia.googlesource.com/buildbot/+/master/autoroll/README.md

If the roll is causing failures, please contact the current sheriff, who should
be CC'd on the roll, and stop the roller if necessary.

{{.Footer}}
`
	TMPL_CQ_INCLUDE_TRYBOTS = "CQ_INCLUDE_TRYBOTS=%s"
)

var (
	// Use this function to instantiate a RepoManager. This is able to be
	// overridden for testing.
	NewDEPSRepoManager func(context.Context, *DEPSRepoManagerConfig, string, *gerrit.Gerrit, string, string) (RepoManager, error) = newDEPSRepoManager

	commitMsgTmpl = template.Must(template.New("commitMsg").Parse(TMPL_COMMIT_MESSAGE))
)

// issueJson is the structure of "git cl issue --json"
type issueJson struct {
	Issue    int64  `json:"issue"`
	IssueUrl string `json:"issue_url"`
}

// depsRepoManager is a struct used by DEPs AutoRoller for managing checkouts.
type depsRepoManager struct {
	*depotToolsRepoManager
	includeLog bool
}

// DEPSRepoManagerConfig provides configuration for the DEPS RepoManager.
type DEPSRepoManagerConfig struct {
	DepotToolsRepoManagerConfig

	// If false, roll CLs do not include a git log.
	IncludeLog bool `json:"includeLog"`
}

// Validate the config.
func (c *DEPSRepoManagerConfig) Validate() error {
	return c.DepotToolsRepoManagerConfig.Validate()
}

// newDEPSRepoManager returns a RepoManager instance which operates in the given
// working directory and updates at the given frequency.
func newDEPSRepoManager(ctx context.Context, c *DEPSRepoManagerConfig, workdir string, g *gerrit.Gerrit, recipeCfgFile, serverURL string) (RepoManager, error) {
	if err := c.Validate(); err != nil {
		return nil, err
	}
	drm, err := newDepotToolsRepoManager(ctx, c.DepotToolsRepoManagerConfig, path.Join(workdir, "repo_manager"), recipeCfgFile, serverURL, g)
	if err != nil {
		return nil, err
	}
	dr := &depsRepoManager{
		depotToolsRepoManager: drm,
		includeLog:            c.IncludeLog,
	}

	// TODO(borenet): This update can be extremely expensive. Consider
	// moving it out of the startup critical path.
	return dr, dr.Update(ctx)
}

// Update syncs code in the relevant repositories.
func (dr *depsRepoManager) Update(ctx context.Context) error {
	// Sync the projects.
	dr.repoMtx.Lock()
	defer dr.repoMtx.Unlock()

	if err := dr.createAndSyncParent(ctx); err != nil {
		return fmt.Errorf("Could not create and sync parent repo: %s", err)
	}

	// Get the last roll revision.
	lastRollRev, err := dr.getLastRollRev(ctx)
	if err != nil {
		return err
	}

	// Get the next roll revision.
	nextRollRev, err := dr.strategy.GetNextRollRev(ctx, dr.childRepo, lastRollRev)
	if err != nil {
		return err
	}

	// Find the number of not-rolled child repo commits.
	notRolled, err := dr.getCommitsNotRolled(ctx, lastRollRev)
	if err != nil {
		return err
	}

	dr.infoMtx.Lock()
	defer dr.infoMtx.Unlock()
	dr.lastRollRev = lastRollRev
	dr.nextRollRev = nextRollRev
	dr.commitsNotRolled = notRolled
	return nil
}

// getLastRollRev returns the commit hash of the last-completed DEPS roll.
func (dr *depsRepoManager) getLastRollRev(ctx context.Context) (string, error) {
	output, err := exec.RunCwd(ctx, dr.parentDir, "python", dr.gclient, "getdep", "-r", dr.childPath)
	if err != nil {
		return "", err
	}
	commit := strings.TrimSpace(output)
	if len(commit) != 40 {
		return "", fmt.Errorf("Got invalid output for `gclient getdep`: %s", output)
	}
	return commit, nil
}

func getLocalPartOfEmailAddress(emailAddress string) string {
	return strings.SplitN(emailAddress, "@", 2)[0]
}

// Helper function for building the commit message.
func (dr *depsRepoManager) buildCommitMsg(ctx context.Context, from, to, cqExtraTrybots string, bugs []string) (string, error) {
	logStr, err := exec.RunCwd(ctx, dr.childDir, "git", "log", fmt.Sprintf("%s..%s", from, to), "--date=short", "--no-merges", "--format=%ad %ae %s")
	if err != nil {
		return "", err
	}
	logStr = strings.TrimSpace(logStr)
	numCommits := len(strings.Split(logStr, "\n"))
	remoteUrl, err := exec.RunCwd(ctx, dr.childDir, "git", "remote", "get-url", "origin")
	if err != nil {
		return "", err
	}
	remoteUrl = strings.TrimSpace(remoteUrl)
	data := struct {
		ChildPath  string
		ChildRepo  string
		From       string
		To         string
		NumCommits int
		LogURL     string
		LogStr     string
		ServerURL  string
		Footer     string
	}{
		ChildPath:  dr.childPath,
		ChildRepo:  remoteUrl,
		From:       from[:7],
		To:         to[:7],
		NumCommits: numCommits,
		LogStr:     "",
		ServerURL:  dr.serverURL,
		Footer:     "",
	}
	if cqExtraTrybots != "" {
		data.Footer += fmt.Sprintf(TMPL_CQ_INCLUDE_TRYBOTS, cqExtraTrybots)
	}
	if len(bugs) > 0 {
		data.Footer += "\n\nBUG=" + strings.Join(bugs, ",")
	}
	if dr.includeLog {
		data.LogStr = fmt.Sprintf("\ngit log %s..%s --date=short --no-merges --format='%%ad %%ae %%s'\n", from[:7], to[:7])
		data.LogStr += logStr + "\n"
	}
	var buf bytes.Buffer
	if err := commitMsgTmpl.Execute(&buf, data); err != nil {
		return "", err
	}
	commitMsg := buf.String()
	return commitMsg, nil
}

// CreateNewRoll creates and uploads a new DEPS roll to the given commit.
// Returns the issue number of the uploaded roll.
func (dr *depsRepoManager) CreateNewRoll(ctx context.Context, from, to string, emails []string, cqExtraTrybots string, dryRun bool) (int64, error) {
	dr.repoMtx.Lock()
	defer dr.repoMtx.Unlock()

	// Clean the checkout, get onto a fresh branch.
	if err := dr.cleanParent(ctx); err != nil {
		return 0, err
	}
	if _, err := exec.RunCwd(ctx, dr.parentDir, "git", "checkout", "-b", ROLL_BRANCH, "-t", fmt.Sprintf("origin/%s", dr.parentBranch), "-f"); err != nil {
		return 0, err
	}

	// Defer some more cleanup.
	defer func() {
		util.LogErr(dr.cleanParent(ctx))
	}()

	// Create the roll CL.
	cr := dr.childRepo
	commits, err := cr.RevList(ctx, fmt.Sprintf("%s..%s", from, to))
	if err != nil {
		return 0, fmt.Errorf("Failed to list revisions: %s", err)
	}

	if _, err := exec.RunCwd(ctx, dr.parentDir, "git", "config", "user.name", getLocalPartOfEmailAddress(dr.user)); err != nil {
		return 0, err
	}
	if _, err := exec.RunCwd(ctx, dr.parentDir, "git", "config", "user.email", dr.user); err != nil {
		return 0, err
	}

	// Find relevant bugs.
	bugs := []string{}
	monorailProject := issues.REPO_PROJECT_MAPPING[dr.parentRepo]
	if monorailProject == "" {
		sklog.Warningf("Found no entry in issues.REPO_PROJECT_MAPPING for %q", dr.parentRepo)
	} else {
		for _, c := range commits {
			d, err := cr.Details(ctx, c)
			if err != nil {
				return 0, fmt.Errorf("Failed to obtain commit details: %s", err)
			}
			b := util.BugsFromCommitMsg(d.Body)
			for _, bug := range b[monorailProject] {
				bugs = append(bugs, fmt.Sprintf("%s:%s", monorailProject, bug))
			}
		}
	}

	// Run "gclient setdep".
	args := []string{"setdep", "-r", fmt.Sprintf("%s@%s", dr.childPath, to)}
	sklog.Infof("Running command: gclient %s", strings.Join(args, " "))
	if _, err := exec.RunCommand(ctx, &exec.Command{
		Dir:  dr.parentDir,
		Env:  depot_tools.Env(dr.depotTools),
		Name: dr.gclient,
		Args: args,
	}); err != nil {
		return 0, err
	}

	// Build the commit message.
	commitMsg, err := dr.buildCommitMsg(ctx, from, to, cqExtraTrybots, bugs)
	if err != nil {
		return 0, err
	}

	// Run the pre-upload steps.
	for _, s := range dr.PreUploadSteps() {
		if err := s(ctx, dr.parentDir); err != nil {
			return 0, fmt.Errorf("Failed pre-upload step: %s", err)
		}
	}

	// Commit.
	if _, err := exec.RunCwd(ctx, dr.parentDir, "git", "commit", "-a", "-m", commitMsg); err != nil {
		return 0, err
	}

	// Upload the CL.
	uploadCmd := &exec.Command{
		Dir:     dr.parentDir,
		Env:     depot_tools.Env(dr.depotTools),
		Name:    "git",
		Args:    []string{"cl", "upload", "--bypass-hooks", "-f", "-v", "-v"},
		Timeout: 2 * time.Minute,
	}
	if dryRun {
		uploadCmd.Args = append(uploadCmd.Args, "--cq-dry-run")
	} else {
		uploadCmd.Args = append(uploadCmd.Args, "--use-commit-queue")
	}
	uploadCmd.Args = append(uploadCmd.Args, "--gerrit")
	tbr := "\nTBR="
	if emails != nil && len(emails) > 0 {
		emailStr := strings.Join(emails, ",")
		tbr += emailStr
		uploadCmd.Args = append(uploadCmd.Args, "--send-mail", "--cc", emailStr)
	}
	commitMsg += tbr
	uploadCmd.Args = append(uploadCmd.Args, "-m", commitMsg)

	// Upload the CL.
	sklog.Infof("Running command: git %s", strings.Join(uploadCmd.Args, " "))
	if _, err := exec.RunCommand(ctx, uploadCmd); err != nil {
		return 0, err
	}

	// Obtain the issue number.
	tmp, err := ioutil.TempDir("", "")
	if err != nil {
		return 0, err
	}
	defer util.RemoveAll(tmp)
	jsonFile := path.Join(tmp, "issue.json")
	if _, err := exec.RunCommand(ctx, &exec.Command{
		Dir:  dr.parentDir,
		Env:  depot_tools.Env(dr.depotTools),
		Name: "git",
		Args: []string{"cl", "issue", fmt.Sprintf("--json=%s", jsonFile)},
	}); err != nil {
		return 0, err
	}
	f, err := os.Open(jsonFile)
	if err != nil {
		return 0, err
	}
	var issue issueJson
	if err := json.NewDecoder(f).Decode(&issue); err != nil {
		return 0, err
	}
	return issue.Issue, nil
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsEbsEncryptionByDefault() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsEbsEncryptionByDefaultCreate,
		Read:   resourceAwsEbsEncryptionByDefaultRead,
		Update: resourceAwsEbsEncryptionByDefaultUpdate,
		Delete: resourceAwsEbsEncryptionByDefaultDelete,

		Schema: map[string]*schema.Schema{
			"enabled": {
				Type:     schema.TypeBool,
				Required: true,
			},
		},
	}
}

func resourceAwsEbsEncryptionByDefaultCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn

	enabled := d.Get("enabled").(bool)
	if err := setEbsEncryptionByDefault(conn, enabled); err != nil {
		return fmt.Errorf("error creating EBS encryption by default (%t): %s", enabled, err)
	}

	d.SetId(resource.UniqueId())

	return resourceAwsEbsEncryptionByDefaultRead(d, meta)
}

func resourceAwsEbsEncryptionByDefaultRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn

	resp, err := conn.GetEbsEncryptionByDefault(&ec2.GetEbsEncryptionByDefaultInput{})
	if err != nil {
		return fmt.Errorf("error reading EBS encryption by default: %s", err)
	}

	d.Set("enabled", aws.BoolValue(resp.EbsEncryptionByDefault))

	return nil
}

func resourceAwsEbsEncryptionByDefaultUpdate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn

	enabled := d.Get("enabled").(bool)
	if err := setEbsEncryptionByDefault(conn, enabled); err != nil {
		return fmt.Errorf("error updating EBS encryption by default (%t): %s", enabled, err)
	}

	return resourceAwsEbsEncryptionByDefaultRead(d, meta)
}

func resourceAwsEbsEncryptionByDefaultDelete(d *schema.ResourceData, meta interface{}) error {
	return nil
}

func setEbsEncryptionByDefault(conn *ec2.EC2, enabled bool) error {
	var err error

	if enabled {
		_, err = conn.EnableEbsEncryptionByDefault(&ec2.EnableEbsEncryptionByDefaultInput{})
	} else {
		_, err = conn.DisableEbsEncryptionByDefault(&ec2.DisableEbsEncryptionByDefaultInput{})
	}

	return err
}
<commit_msg>Update aws/resource_aws_ebs_encryption_by_default.go<commit_after>package aws

import (
	"fmt"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsEbsEncryptionByDefault() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsEbsEncryptionByDefaultCreate,
		Read:   resourceAwsEbsEncryptionByDefaultRead,
		Update: resourceAwsEbsEncryptionByDefaultUpdate,
		Delete: resourceAwsEbsEncryptionByDefaultDelete,

		Schema: map[string]*schema.Schema{
			"enabled": {
				Type:     schema.TypeBool,
				Optional: true,
				Default:  true,
			},
		},
	}
}

func resourceAwsEbsEncryptionByDefaultCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn

	enabled := d.Get("enabled").(bool)
	if err := setEbsEncryptionByDefault(conn, enabled); err != nil {
		return fmt.Errorf("error creating EBS encryption by default (%t): %s", enabled, err)
	}

	d.SetId(resource.UniqueId())

	return resourceAwsEbsEncryptionByDefaultRead(d, meta)
}

func resourceAwsEbsEncryptionByDefaultRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn

	resp, err := conn.GetEbsEncryptionByDefault(&ec2.GetEbsEncryptionByDefaultInput{})
	if err != nil {
		return fmt.Errorf("error reading EBS encryption by default: %s", err)
	}

	d.Set("enabled", aws.BoolValue(resp.EbsEncryptionByDefault))

	return nil
}

func resourceAwsEbsEncryptionByDefaultUpdate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn

	enabled := d.Get("enabled").(bool)
	if err := setEbsEncryptionByDefault(conn, enabled); err != nil {
		return fmt.Errorf("error updating EBS encryption by default (%t): %s", enabled, err)
	}

	return resourceAwsEbsEncryptionByDefaultRead(d, meta)
}

func resourceAwsEbsEncryptionByDefaultDelete(d *schema.ResourceData, meta interface{}) error {
	return nil
}

func setEbsEncryptionByDefault(conn *ec2.EC2, enabled bool) error {
	var err error

	if enabled {
		_, err = conn.EnableEbsEncryptionByDefault(&ec2.EnableEbsEncryptionByDefaultInput{})
	} else {
		_, err = conn.DisableEbsEncryptionByDefault(&ec2.DisableEbsEncryptionByDefaultInput{})
	}

	return err
}
<|endoftext|>


<commit_before>package cachestore

import (
	"appengine"
	"appengine/datastore"
	"appengine/memcache"
	"encoding/gob"
	"fmt"
	"github.com/oschmid/appenginetesting"
	"reflect"
	"testing"
)

var c = must(appenginetesting.NewContext(nil))

func must(c *appenginetesting.Context, err error) *appenginetesting.Context {
	if err != nil {
		panic(err)
	}
	return c
}

func init() {
	gob.Register(*new(Struct))
}

type Struct struct {
	I int
}

type PropertyLoadSaver struct {
	S string
}

func (p *PropertyLoadSaver) Load(c <-chan datastore.Property) error {
	if err := datastore.LoadStruct(p, c); err != nil {
		return err
	}
	p.S += ".load"
	return nil
}

func (p *PropertyLoadSaver) Save(c chan<- datastore.Property) error {
	defer close(c)
	c <- datastore.Property{
		Name:  "S",
		Value: p.S + ".save",
	}
	return nil
}

func TestWithStruct(t *testing.T) {
	src := Struct{I: 3}
	key := datastore.NewIncompleteKey(c, "Struct", nil)
	// Put
	key, err := Put(c, key, &src)
	if err != nil {
		t.Fatal(err)
	}
	// Get
	dst := *new(Struct)
	err = Get(c, key, &dst)
	if err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(src, dst) {
		t.Fatalf("expected=%#v actual=%#v", src, dst)
	}
	// Delete
	err = Delete(c, key)
	if err != nil {
		t.Fatal(err)
	}
	err = Get(c, key, &dst)
	if err != datastore.ErrNoSuchEntity {
		t.Fatal("expected=%#v actual=%#v", datastore.ErrNoSuchEntity, err)
	}
}

func TestWithStructArray(t *testing.T) {
	src := *new([]Struct)
	key := *new([]*datastore.Key)
	for i := 1; i < 11; i++ {
		src = append(src, Struct{I: i})
		key = append(key, datastore.NewIncompleteKey(c, "Struct", nil))
	}
	// PutMulti
	key, err := PutMulti(c, key, src)
	if err != nil {
		t.Fatal(err)
	}
	// GetMulti
	dst := make([]Struct, len(src))
	err = GetMulti(c, key, dst)
	if err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(src, dst) {
		t.Fatalf("expected=%#v actual=%#v", src, dst)
	}
	// DeleteMulti
	err = DeleteMulti(c, key)
	if err != nil {
		t.Fatal(err)
	}
	err = GetMulti(c, key, dst)
	if me, ok := err.(appengine.MultiError); ok {
		for _, e := range me {
			if e != datastore.ErrNoSuchEntity {
				t.Fatal(e)
			}
		}
	} else {
		t.Fatal(err)
	}
}

// TODO test []*S

// TODO test []I

func TestWithPropertyLoadSaver(t *testing.T) {
	src := PropertyLoadSaver{}
	key := datastore.NewIncompleteKey(c, "PropertyLoadSaver", nil)
	// Put
	key, err := Put(c, key, &src)
	if err != nil {
		t.Fatal(err)
	}
	// Get
	dst := *new(PropertyLoadSaver)
	err = Get(c, key, &dst)
	if err != nil {
		t.Fatal(err)
	}
	if dst.S != src.S+".save.load" {
		t.Fatalf("actual=%#v", dst.S)
	}
	// Delete
	err = Delete(c, key)
	if err != nil {
		t.Fatal(err)
	}
	err = Get(c, key, &dst)
	if err != datastore.ErrNoSuchEntity {
		t.Fatal("expected=%#v actual=%#v", datastore.ErrNoSuchEntity, err)
	}
}

func TestWithPropertyLoadSaverArray(t *testing.T) {
	src := *new([]PropertyLoadSaver)
	key := *new([]*datastore.Key)
	for i := 1; i < 11; i++ {
		src = append(src, PropertyLoadSaver{S: fmt.Sprint(i)})
		key = append(key, datastore.NewIncompleteKey(c, "PropertyLoadSaver", nil))
	}
	// PutMulti
	key, err := PutMulti(c, key, src)
	if err != nil {
		t.Fatal(err)
	}
	// GetMulti
	dst := make([]PropertyLoadSaver, len(src))
	err = GetMulti(c, key, dst)
	if err != nil {
		t.Fatal(err)
	}
	for i, d := range dst {
		if d.S != src[i].S+".save.load" {
			t.Fatalf("actual%#v", d.S)
		}
	}
	// DeleteMulti
	err = DeleteMulti(c, key)
	if err != nil {
		t.Fatal(err)
	}
	err = GetMulti(c, key, dst)
	if me, ok := err.(appengine.MultiError); ok {
		for _, e := range me {
			if e != datastore.ErrNoSuchEntity {
				t.Fatal(e)
			}
		}
	} else {
		t.Fatal(err)
	}
}

func TestGetFromMemcache(t *testing.T) {
	src := Struct{I: 3}
	key := datastore.NewIncompleteKey(c, "Struct", nil)
	// Put
	key, err := Put(c, key, &src)
	if err != nil {
		t.Fatal(err)
	}
	// remove from datastore
	err = datastore.Delete(c, key)
	if err != nil {
		t.Fatal(err)
	}
	// Get
	dst := *new(Struct)
	err = Get(c, key, &dst)
	if err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(src, dst) {
		t.Fatalf("expected=%#v actual=%#v", src, dst)
	}
	// Delete
	err = Delete(c, key)
	if err != nil {
		t.Fatal(err)
	}
	err = Get(c, key, &dst)
	if err != datastore.ErrNoSuchEntity {
		t.Fatal("expected=%#v actual=%#v", datastore.ErrNoSuchEntity, err)
	}
}

func TestGetFromDatastore(t *testing.T) {
	src := Struct{I: 3}
	key := datastore.NewIncompleteKey(c, "Struct", nil)
	// Put
	key, err := Put(c, key, &src)
	if err != nil {
		t.Fatal(err)
	}
	// remove from memcache
	err = memcache.Delete(c, key.Encode())
	if err != nil {
		t.Fatal(err)
	}
	// Get
	dst := *new(Struct)
	err = Get(c, key, &dst)
	if err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(src, dst) {
		t.Fatalf("expected=%#v actual=%#v", src, dst)
	}
	// Delete
	err = Delete(c, key)
	if err != nil {
		t.Fatal(err)
	}
	err = Get(c, key, &dst)
	if err != datastore.ErrNoSuchEntity {
		t.Fatal("expected=%#v actual=%#v", datastore.ErrNoSuchEntity, err)
	}
}
<commit_msg>use aetest<commit_after>package cachestore

import (
	"appengine"
	"appengine/aetest"
	"appengine/datastore"
	"appengine/memcache"
	"encoding/gob"
	"fmt"
	"reflect"
	"testing"
)

var c = must(aetest.NewContext(nil))

func must(c aetest.Context, err error) aetest.Context {
	if err != nil {
		panic(err)
	}
	return c
}

func init() {
	gob.Register(*new(Struct))
}

type Struct struct {
	I int
}

type PropertyLoadSaver struct {
	S string
}

func (p *PropertyLoadSaver) Load(c <-chan datastore.Property) error {
	if err := datastore.LoadStruct(p, c); err != nil {
		return err
	}
	p.S += ".load"
	return nil
}

func (p *PropertyLoadSaver) Save(c chan<- datastore.Property) error {
	defer close(c)
	c <- datastore.Property{
		Name:  "S",
		Value: p.S + ".save",
	}
	return nil
}

func TestWithStruct(t *testing.T) {
	src := Struct{I: 3}
	key := datastore.NewIncompleteKey(c, "Struct", nil)
	// Put
	key, err := Put(c, key, &src)
	if err != nil {
		t.Fatal(err)
	}
	// Get
	dst := *new(Struct)
	err = Get(c, key, &dst)
	if err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(src, dst) {
		t.Fatalf("expected=%#v actual=%#v", src, dst)
	}
	// Delete
	err = Delete(c, key)
	if err != nil {
		t.Fatal(err)
	}
	err = Get(c, key, &dst)
	if err != datastore.ErrNoSuchEntity {
		t.Fatal("expected=%#v actual=%#v", datastore.ErrNoSuchEntity, err)
	}
}

func TestWithStructArray(t *testing.T) {
	src := *new([]Struct)
	key := *new([]*datastore.Key)
	for i := 1; i < 11; i++ {
		src = append(src, Struct{I: i})
		key = append(key, datastore.NewIncompleteKey(c, "Struct", nil))
	}
	// PutMulti
	key, err := PutMulti(c, key, src)
	if err != nil {
		t.Fatal(err)
	}
	// GetMulti
	dst := make([]Struct, len(src))
	err = GetMulti(c, key, dst)
	if err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(src, dst) {
		t.Fatalf("expected=%#v actual=%#v", src, dst)
	}
	// DeleteMulti
	err = DeleteMulti(c, key)
	if err != nil {
		t.Fatal(err)
	}
	err = GetMulti(c, key, dst)
	if me, ok := err.(appengine.MultiError); ok {
		for _, e := range me {
			if e != datastore.ErrNoSuchEntity {
				t.Fatal(e)
			}
		}
	} else {
		t.Fatal(err)
	}
}

// TODO test []*S

// TODO test []I

func TestWithPropertyLoadSaver(t *testing.T) {
	src := PropertyLoadSaver{}
	key := datastore.NewIncompleteKey(c, "PropertyLoadSaver", nil)
	// Put
	key, err := Put(c, key, &src)
	if err != nil {
		t.Fatal(err)
	}
	// Get
	dst := *new(PropertyLoadSaver)
	err = Get(c, key, &dst)
	if err != nil {
		t.Fatal(err)
	}
	if dst.S != src.S+".save.load" {
		t.Fatalf("actual=%#v", dst.S)
	}
	// Delete
	err = Delete(c, key)
	if err != nil {
		t.Fatal(err)
	}
	err = Get(c, key, &dst)
	if err != datastore.ErrNoSuchEntity {
		t.Fatal("expected=%#v actual=%#v", datastore.ErrNoSuchEntity, err)
	}
}

func TestWithPropertyLoadSaverArray(t *testing.T) {
	src := *new([]PropertyLoadSaver)
	key := *new([]*datastore.Key)
	for i := 1; i < 11; i++ {
		src = append(src, PropertyLoadSaver{S: fmt.Sprint(i)})
		key = append(key, datastore.NewIncompleteKey(c, "PropertyLoadSaver", nil))
	}
	// PutMulti
	key, err := PutMulti(c, key, src)
	if err != nil {
		t.Fatal(err)
	}
	// GetMulti
	dst := make([]PropertyLoadSaver, len(src))
	err = GetMulti(c, key, dst)
	if err != nil {
		t.Fatal(err)
	}
	for i, d := range dst {
		if d.S != src[i].S+".save.load" {
			t.Fatalf("actual%#v", d.S)
		}
	}
	// DeleteMulti
	err = DeleteMulti(c, key)
	if err != nil {
		t.Fatal(err)
	}
	err = GetMulti(c, key, dst)
	if me, ok := err.(appengine.MultiError); ok {
		for _, e := range me {
			if e != datastore.ErrNoSuchEntity {
				t.Fatal(e)
			}
		}
	} else {
		t.Fatal(err)
	}
}

func TestGetFromMemcache(t *testing.T) {
	src := Struct{I: 3}
	key := datastore.NewIncompleteKey(c, "Struct", nil)
	// Put
	key, err := Put(c, key, &src)
	if err != nil {
		t.Fatal(err)
	}
	// remove from datastore
	err = datastore.Delete(c, key)
	if err != nil {
		t.Fatal(err)
	}
	// Get
	dst := *new(Struct)
	err = Get(c, key, &dst)
	if err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(src, dst) {
		t.Fatalf("expected=%#v actual=%#v", src, dst)
	}
	// Delete
	err = Delete(c, key)
	if err != nil {
		t.Fatal(err)
	}
	err = Get(c, key, &dst)
	if err != datastore.ErrNoSuchEntity {
		t.Fatal("expected=%#v actual=%#v", datastore.ErrNoSuchEntity, err)
	}
}

func TestGetFromDatastore(t *testing.T) {
	src := Struct{I: 3}
	key := datastore.NewIncompleteKey(c, "Struct", nil)
	// Put
	key, err := Put(c, key, &src)
	if err != nil {
		t.Fatal(err)
	}
	// remove from memcache
	err = memcache.Delete(c, key.Encode())
	if err != nil {
		t.Fatal(err)
	}
	// Get
	dst := *new(Struct)
	err = Get(c, key, &dst)
	if err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(src, dst) {
		t.Fatalf("expected=%#v actual=%#v", src, dst)
	}
	// Delete
	err = Delete(c, key)
	if err != nil {
		t.Fatal(err)
	}
	err = Get(c, key, &dst)
	if err != datastore.ErrNoSuchEntity {
		t.Fatal("expected=%#v actual=%#v", datastore.ErrNoSuchEntity, err)
	}
}
<|endoftext|>


<commit_before>package types

import (
	"fmt"
	"github.com/pkg/errors"
)

// A Place isn't something directly used by the Navitia.io api.
//
// However, it allows the library user to use idiomatic go when working with the library.
// If you want a countainer, see PlaceContainer
//
// Place is satisfied by:
// 	- StopArea
// 	- POI
// 	- Address
// 	- StopPoint
// 	- Admin
type Place interface {
	// PlaceID returns the ID associated with the Place
	PlaceID() ID

	// PlaceName returns the name of the Place
	PlaceName() string

	// PlaceType returns the name of the type of the Place
	PlaceType() string

	// String() for string printing
	String() string
}

// PlaceContainer is the ugly countainer sent by Navitia to make us all cry :(
//
// However, this can be useful. May be removed from the public API in gonavitia v1.
type PlaceContainer struct {
	ID           ID     `json:"id"`
	Name         string `json:"name"`
	Quality      uint   `json:"quality,omitempty"`
	EmbeddedType string `json:"embedded_type"`

	// Four possibilitiess
	StopArea             *StopArea             `json:"stop_area,omitempty"`
	POI                  *POI                  `json:"POI,omitempty"`
	Address              *Address              `json:"address,omitempty"`
	StopPoint            *StopPoint            `json:"stop_point,omitempty"`
	Admin *Admin `json:"administrative_region,omitempty"`
}

// ErrInvalidPlaceContainer is returned after a check on a PlaceContainer
type ErrInvalidPlaceContainer struct {
	// If the PlaceContainer has a zero ID.
	NoID bool

	// If the PlaceContainer has a zero EmbeddedType.
	NoEmbeddedType bool

	// If the PlaceContainer has an unknown EmbeddedType
	UnknownEmbeddedType bool

	// If the PlaceContainer has a known EmbeddedType but the corresponding concrete value is nil.
	NilConcretePlaceValue bool

	// TODO:
	// If the PlaceContainer has more than one non-nil concrete place values.
	// MultipleNonNilConcretePlaceValues bool
}

// Error satisfies the error interface
func (err ErrInvalidPlaceContainer) Error() string {
	// Count the number of anomalies
	var anomalies uint

	msg := "Error: Invalid non-empty PlaceContainer (%d anomalies):"

	if err.NoID {
		msg += "\n\tNo ID specified"
		anomalies++
	}
	if err.NoEmbeddedType {
		msg += "\n\tEmpty EmbeddedType"
		anomalies++
	}
	if err.UnknownEmbeddedType {
		msg += "\n\tUnknown EmbeddedType"
		anomalies++
	}
	if err.NilConcretePlaceValue {
		msg += "\n\tKnown EmbeddedType but nil corresponding concrete value"
		anomalies++
	}

	return fmt.Sprintf(msg, anomalies)
}

// IsEmpty returns whether or not pc is empty
func (pc PlaceContainer) IsEmpty() bool {
	empty := PlaceContainer{}
	return pc == empty
}

var placeTypes = []string{
	embeddedStopArea,
	embeddedPOI,
	embeddedAddress,
	embeddedStopPoint,
	embeddedAddress,
}

const (
	embeddedStopArea  string = "stop_area"
	embeddedPOI              = "poi"
	embeddedAddress          = "address"
	embeddedStopPoint        = "stop_point"
	embeddedAdmin            = "administrative_region"
)

// Check checks the validity of the PlaceContainer. Returns an ErrInvalidPlaceContainer.
//
// An empty PlaceContainer is valid. But those cases aren't:
// 	- If the PlaceContainer has a zero ID.
// 	- If the PlaceContainer has a zero EmbeddedType.
// 	- If the PlaceContainer has an unknown EmbeddedType.
// 	- If the PlaceContainer has a known EmbeddedType but the corresponding concrete value is nil.
func (pc PlaceContainer) Check() error {
	if pc.IsEmpty() {
		return nil
	}
	err := ErrInvalidPlaceContainer{}

	// Check for zero ID
	err.NoID = (pc.ID == "")

	// Check if known EmbeddedType but the corresponding concrete value is nil.
	// Also check for empty and unknown embedded type
	absent := &err.NilConcretePlaceValue
	switch pc.EmbeddedType {
	case embeddedStopArea:
		*absent = (pc.StopArea == nil)
	case embeddedPOI:
		*absent = (pc.POI == nil)
	case embeddedAddress:
		*absent = (pc.Address == nil)
	case embeddedStopPoint:
		*absent = (pc.StopPoint == nil)
	case embeddedAdmin:
		*absent = (pc.Admin == nil)
	default:
		// Check for an empty embedded type
		err.NoEmbeddedType = (pc.EmbeddedType == "")
		// In case its not empty yet doesn't match with a known type
		err.UnknownEmbeddedType = !err.NoEmbeddedType
	}

	emptyErr := ErrInvalidPlaceContainer{}
	if err != emptyErr {
		return err
	}

	return nil
}

// Place returns the Place countained in the PlaceContainer
// If PlaceContainer is empty, Place returns an error.
// Check() is run on the PlaceContainer.
func (pc PlaceContainer) Place() (Place, error) {
	// If PlaceContainer is empty, return an error
	if pc.IsEmpty() {
		return nil, errors.Errorf("this place countainer is empty, can't extract a Place from it")
	}

	// Check validity
	err := pc.Check()
	if err != nil {
		return nil, err
	}

	// Check for each type
	switch pc.EmbeddedType {
	case embeddedStopArea:
		return pc.StopArea, nil
	case embeddedPOI:
		return pc.POI, nil
	case embeddedAddress:
		return pc.Address, nil
	case embeddedStopPoint:
		return pc.StopPoint, nil
	case embeddedAdmin:
		return pc.Admin, nil
	default:
		return nil, errors.Errorf("no known embedded type indicated (we have \"%s\"), can't return a place !", pc.EmbeddedType) // THIS IS VERY SERIOUS AS WE ALREADY CHECKED THE STRUCTURE
	}
}

// A StopArea represents a stop area: a nameable zone, where there are some stop points.
type StopArea struct {
	ID   ID     `json:"id"`
	Name string `json:"name"`

	// Label of the stop area.
	// The name is directly taken from the data whereas the label is something computed by navitia for better traveler information.
	// If you don't know what to display, display the label
	Label string `json:"label"`

	// Coordinates of the stop area
	Coord Coordinates `json:"coord"`

	// Administrative regions of the stop area in which is placed the stop area
	Admins []Admin `json:"administrative_regions"`

	// Stop points countained in this stop area
	StopPoints []StopPoint `json:"stop_points"`
}

// PlaceID returns the ID associated with the StopArea
// Helps satisfy Place
func (sa StopArea) PlaceID() ID {
	return sa.ID
}

// PlaceName returns the name of the StopArea
// Helps satisfy Place
func (sa StopArea) PlaceName() string {
	return sa.Name
}

// PlaceType returns the type of place, in this case "stop_area"
// Helps satisfy Place
func (sa StopArea) PlaceType() string {
	return embeddedStopArea
}

// String pretty-prints the StopArea.
// Satisfies Stringer and helps satisfy Place
func (sa StopArea) String() string {
	var label string
	if sa.Label == "" {
		label = sa.Name
	} else {
		label = sa.Label
	}

	format := "%s (id: %s)"
	return fmt.Sprintf(format, label, sa.ID)
}

// A POI is a Point Of Interest. A loosely-defined place.
type POI struct {
	ID   ID     `json:"id"`
	Name string `json:"name"`

	// The name is directly taken from the data whereas the label is something computed by navitia for better traveler information.
	// If you don't know what to display, display the label
	Label string `json:"label"`

	// The type of the POI
	Type POIType `json:"poi_type"`
}

// PlaceID returns the ID associated with the POI.
// Helps satisfy Place
func (poi POI) PlaceID() ID {
	return poi.ID
}

// PlaceName returns the name of the POI.
// Helps satisfy Place
func (poi POI) PlaceName() string {
	return poi.Name
}

// PlaceType returns the type of place, in this case "poi".
// Helps satisfy Place
func (poi POI) PlaceType() string {
	return embeddedPOI
}

// String pretty-prints the POI.
// Satisfies Stringer and helps satisfy Place
func (poi POI) String() string {
	var label string
	if poi.Label == "" {
		label = poi.Name
	} else {
		label = poi.Label
	}

	format := "%s (id: %s)"
	return fmt.Sprintf(format, label, poi.ID)
}

// A POIType codes for the type of the point of interest
// TODO: A list of usual types ?
type POIType struct {
	ID   ID     `json:"id"`
	Name string `json:"name"`
}

// An Address codes for a real-world address: a point located in a street.
type Address struct {
	ID   ID     `json:"id"`
	Name string `json:"name"`

	// Label of the address
	// The name is directly taken from the data whereas the label is something computed by navitia for better traveler information.
	// If you don't know what to display, display the label
	Label string `json:"label"`

	// Coordinates of the address
	Coord Coordinates `json:"coord"`

	// House number of the address
	HouseNumber uint `json:"house_number"`

	// Administrative regions of the stop area in which is placed the stop area
	Admins []Admin `json:"administrative_regions"`
}

// PlaceID returns the ID associated with the Address
// Helps satisfy Place
func (add Address) PlaceID() ID {
	return add.ID
}

// PlaceName returns the name of the Address
// Helps satisfy Place
func (add Address) PlaceName() string {
	return add.Name
}

// PlaceType returns the type of place, in this case "address"
// Helps satisfy Place
func (add Address) PlaceType() string {
	return embeddedAddress
}

// String pretty-prints the Address.
// Satisfies Stringer and helps satisfy Place
func (add Address) String() string {
	var label string
	if add.Label == "" {
		label = add.Name
	} else {
		label = add.Label
	}

	format := "%s (id: %s)"
	return fmt.Sprintf(format, label, add.ID)
}

// A StopPoint codes for a stop point in a line: a location where vehicles can pickup or drop off passengers.
type StopPoint struct {
	ID ID `json:"id"`

	// Name of the stop point
	Name string `json:"name"`

	// Coordinates of the stop point
	Coord Coordinates `json:"coord"`

	// Administrative regions of the stop point
	Admins []Admin `json:"administrative_regions"`

	// List of equipments of the stop point
	Equipments []Equipment `json:"equipment"`

	// Stop Area countaining the stop point
	StopArea *StopArea `json:"stop_area"`
}

// PlaceID returns the ID associated with the Stop Point
// Helps satisfy Place
func (sp StopPoint) PlaceID() ID {
	return sp.ID
}

// PlaceName returns the name of the Stop Point
// Helps satisfy Place
func (sp StopPoint) PlaceName() string {
	return sp.Name
}

// PlaceType returns the type of place, in this case "stop_point"
// Helps satisfy Place
func (sp StopPoint) PlaceType() string {
	return embeddedStopPoint
}

// String pretty-prints the StopPoint.
// Satisfies Stringer and helps satisfy Place
func (sp StopPoint) String() string {
	format := "%s (id: %s)"
	return fmt.Sprintf(format, sp.Name, sp.ID)
}

// An Admin represents an administrative region: a region under the control/responsibility of a specific organisation.
// It can be a city, a district, a neightborhood, etc.
type Admin struct {
	ID   ID     `json:"id"`
	Name string `json:"name"`

	// Label of the address
	// The name is directly taken from the data whereas the label is something computed by navitia for better traveler information.
	// If you don't know what to display, display the label
	Label string `json:"label"`

	// Coordinates of the administrative region
	Coord Coordinates `json:"coord"`

	// Level of the administrative region
	Level int `json:"level"`

	// Zip code of the administrative region
	ZipCode string `json:"zip_code"`
}

// PlaceID returns the ID associated with the Admin
// Helps satisfy Place
func (ar Admin) PlaceID() ID {
	return ar.ID
}

// PlaceName returns the name of the Admin
// Helps satisfy Place
func (ar Admin) PlaceName() string {
	return ar.Name
}

// PlaceType returns the type of place, in this case "administrative_region"
// Helps satisfy Place
func (ar Admin) PlaceType() string {
	return embeddedAdmin
}

// String pretty-prints the Admin.
// Satisfies Stringer and helps satisfy Place
func (ar Admin) String() string {
	var label string
	if ar.Label == "" {
		label = ar.Name
	} else {
		label = ar.Label
	}

	format := "%s (id: %s)"
	return fmt.Sprintf(format, label, ar.ID)
}
<commit_msg>Fix indentation<commit_after>package types

import (
	"fmt"
	"github.com/pkg/errors"
)

// A Place isn't something directly used by the Navitia.io api.
//
// However, it allows the library user to use idiomatic go when working with the library.
// If you want a countainer, see PlaceContainer
//
// Place is satisfied by:
// 	- StopArea
// 	- POI
// 	- Address
// 	- StopPoint
// 	- Admin
type Place interface {
	// PlaceID returns the ID associated with the Place
	PlaceID() ID

	// PlaceName returns the name of the Place
	PlaceName() string

	// PlaceType returns the name of the type of the Place
	PlaceType() string

	// String() for string printing
	String() string
}

// PlaceContainer is the ugly countainer sent by Navitia to make us all cry :(
//
// However, this can be useful. May be removed from the public API in gonavitia v1.
type PlaceContainer struct {
	ID           ID     `json:"id"`
	Name         string `json:"name"`
	Quality      uint   `json:"quality,omitempty"`
	EmbeddedType string `json:"embedded_type"`

	// Four possibilitiess
	StopArea  *StopArea  `json:"stop_area,omitempty"`
	POI       *POI       `json:"POI,omitempty"`
	Address   *Address   `json:"address,omitempty"`
	StopPoint *StopPoint `json:"stop_point,omitempty"`
	Admin     *Admin     `json:"administrative_region,omitempty"`
}

// ErrInvalidPlaceContainer is returned after a check on a PlaceContainer
type ErrInvalidPlaceContainer struct {
	// If the PlaceContainer has a zero ID.
	NoID bool

	// If the PlaceContainer has a zero EmbeddedType.
	NoEmbeddedType bool

	// If the PlaceContainer has an unknown EmbeddedType
	UnknownEmbeddedType bool

	// If the PlaceContainer has a known EmbeddedType but the corresponding concrete value is nil.
	NilConcretePlaceValue bool

	// TODO:
	// If the PlaceContainer has more than one non-nil concrete place values.
	// MultipleNonNilConcretePlaceValues bool
}

// Error satisfies the error interface
func (err ErrInvalidPlaceContainer) Error() string {
	// Count the number of anomalies
	var anomalies uint

	msg := "Error: Invalid non-empty PlaceContainer (%d anomalies):"

	if err.NoID {
		msg += "\n\tNo ID specified"
		anomalies++
	}
	if err.NoEmbeddedType {
		msg += "\n\tEmpty EmbeddedType"
		anomalies++
	}
	if err.UnknownEmbeddedType {
		msg += "\n\tUnknown EmbeddedType"
		anomalies++
	}
	if err.NilConcretePlaceValue {
		msg += "\n\tKnown EmbeddedType but nil corresponding concrete value"
		anomalies++
	}

	return fmt.Sprintf(msg, anomalies)
}

// IsEmpty returns whether or not pc is empty
func (pc PlaceContainer) IsEmpty() bool {
	empty := PlaceContainer{}
	return pc == empty
}

var placeTypes = []string{
	embeddedStopArea,
	embeddedPOI,
	embeddedAddress,
	embeddedStopPoint,
	embeddedAddress,
}

const (
	embeddedStopArea  string = "stop_area"
	embeddedPOI              = "poi"
	embeddedAddress          = "address"
	embeddedStopPoint        = "stop_point"
	embeddedAdmin            = "administrative_region"
)

// Check checks the validity of the PlaceContainer. Returns an ErrInvalidPlaceContainer.
//
// An empty PlaceContainer is valid. But those cases aren't:
// 	- If the PlaceContainer has a zero ID.
// 	- If the PlaceContainer has a zero EmbeddedType.
// 	- If the PlaceContainer has an unknown EmbeddedType.
// 	- If the PlaceContainer has a known EmbeddedType but the corresponding concrete value is nil.
func (pc PlaceContainer) Check() error {
	if pc.IsEmpty() {
		return nil
	}
	err := ErrInvalidPlaceContainer{}

	// Check for zero ID
	err.NoID = (pc.ID == "")

	// Check if known EmbeddedType but the corresponding concrete value is nil.
	// Also check for empty and unknown embedded type
	absent := &err.NilConcretePlaceValue
	switch pc.EmbeddedType {
	case embeddedStopArea:
		*absent = (pc.StopArea == nil)
	case embeddedPOI:
		*absent = (pc.POI == nil)
	case embeddedAddress:
		*absent = (pc.Address == nil)
	case embeddedStopPoint:
		*absent = (pc.StopPoint == nil)
	case embeddedAdmin:
		*absent = (pc.Admin == nil)
	default:
		// Check for an empty embedded type
		err.NoEmbeddedType = (pc.EmbeddedType == "")
		// In case its not empty yet doesn't match with a known type
		err.UnknownEmbeddedType = !err.NoEmbeddedType
	}

	emptyErr := ErrInvalidPlaceContainer{}
	if err != emptyErr {
		return err
	}

	return nil
}

// Place returns the Place countained in the PlaceContainer
// If PlaceContainer is empty, Place returns an error.
// Check() is run on the PlaceContainer.
func (pc PlaceContainer) Place() (Place, error) {
	// If PlaceContainer is empty, return an error
	if pc.IsEmpty() {
		return nil, errors.Errorf("this place countainer is empty, can't extract a Place from it")
	}

	// Check validity
	err := pc.Check()
	if err != nil {
		return nil, err
	}

	// Check for each type
	switch pc.EmbeddedType {
	case embeddedStopArea:
		return pc.StopArea, nil
	case embeddedPOI:
		return pc.POI, nil
	case embeddedAddress:
		return pc.Address, nil
	case embeddedStopPoint:
		return pc.StopPoint, nil
	case embeddedAdmin:
		return pc.Admin, nil
	default:
		return nil, errors.Errorf("no known embedded type indicated (we have \"%s\"), can't return a place !", pc.EmbeddedType) // THIS IS VERY SERIOUS AS WE ALREADY CHECKED THE STRUCTURE
	}
}

// A StopArea represents a stop area: a nameable zone, where there are some stop points.
type StopArea struct {
	ID   ID     `json:"id"`
	Name string `json:"name"`

	// Label of the stop area.
	// The name is directly taken from the data whereas the label is something computed by navitia for better traveler information.
	// If you don't know what to display, display the label
	Label string `json:"label"`

	// Coordinates of the stop area
	Coord Coordinates `json:"coord"`

	// Administrative regions of the stop area in which is placed the stop area
	Admins []Admin `json:"administrative_regions"`

	// Stop points countained in this stop area
	StopPoints []StopPoint `json:"stop_points"`
}

// PlaceID returns the ID associated with the StopArea
// Helps satisfy Place
func (sa StopArea) PlaceID() ID {
	return sa.ID
}

// PlaceName returns the name of the StopArea
// Helps satisfy Place
func (sa StopArea) PlaceName() string {
	return sa.Name
}

// PlaceType returns the type of place, in this case "stop_area"
// Helps satisfy Place
func (sa StopArea) PlaceType() string {
	return embeddedStopArea
}

// String pretty-prints the StopArea.
// Satisfies Stringer and helps satisfy Place
func (sa StopArea) String() string {
	var label string
	if sa.Label == "" {
		label = sa.Name
	} else {
		label = sa.Label
	}

	format := "%s (id: %s)"
	return fmt.Sprintf(format, label, sa.ID)
}

// A POI is a Point Of Interest. A loosely-defined place.
type POI struct {
	ID   ID     `json:"id"`
	Name string `json:"name"`

	// The name is directly taken from the data whereas the label is something computed by navitia for better traveler information.
	// If you don't know what to display, display the label
	Label string `json:"label"`

	// The type of the POI
	Type POIType `json:"poi_type"`
}

// PlaceID returns the ID associated with the POI.
// Helps satisfy Place
func (poi POI) PlaceID() ID {
	return poi.ID
}

// PlaceName returns the name of the POI.
// Helps satisfy Place
func (poi POI) PlaceName() string {
	return poi.Name
}

// PlaceType returns the type of place, in this case "poi".
// Helps satisfy Place
func (poi POI) PlaceType() string {
	return embeddedPOI
}

// String pretty-prints the POI.
// Satisfies Stringer and helps satisfy Place
func (poi POI) String() string {
	var label string
	if poi.Label == "" {
		label = poi.Name
	} else {
		label = poi.Label
	}

	format := "%s (id: %s)"
	return fmt.Sprintf(format, label, poi.ID)
}

// A POIType codes for the type of the point of interest
// TODO: A list of usual types ?
type POIType struct {
	ID   ID     `json:"id"`
	Name string `json:"name"`
}

// An Address codes for a real-world address: a point located in a street.
type Address struct {
	ID   ID     `json:"id"`
	Name string `json:"name"`

	// Label of the address
	// The name is directly taken from the data whereas the label is something computed by navitia for better traveler information.
	// If you don't know what to display, display the label
	Label string `json:"label"`

	// Coordinates of the address
	Coord Coordinates `json:"coord"`

	// House number of the address
	HouseNumber uint `json:"house_number"`

	// Administrative regions of the stop area in which is placed the stop area
	Admins []Admin `json:"administrative_regions"`
}

// PlaceID returns the ID associated with the Address
// Helps satisfy Place
func (add Address) PlaceID() ID {
	return add.ID
}

// PlaceName returns the name of the Address
// Helps satisfy Place
func (add Address) PlaceName() string {
	return add.Name
}

// PlaceType returns the type of place, in this case "address"
// Helps satisfy Place
func (add Address) PlaceType() string {
	return embeddedAddress
}

// String pretty-prints the Address.
// Satisfies Stringer and helps satisfy Place
func (add Address) String() string {
	var label string
	if add.Label == "" {
		label = add.Name
	} else {
		label = add.Label
	}

	format := "%s (id: %s)"
	return fmt.Sprintf(format, label, add.ID)
}

// A StopPoint codes for a stop point in a line: a location where vehicles can pickup or drop off passengers.
type StopPoint struct {
	ID ID `json:"id"`

	// Name of the stop point
	Name string `json:"name"`

	// Coordinates of the stop point
	Coord Coordinates `json:"coord"`

	// Administrative regions of the stop point
	Admins []Admin `json:"administrative_regions"`

	// List of equipments of the stop point
	Equipments []Equipment `json:"equipment"`

	// Stop Area countaining the stop point
	StopArea *StopArea `json:"stop_area"`
}

// PlaceID returns the ID associated with the Stop Point
// Helps satisfy Place
func (sp StopPoint) PlaceID() ID {
	return sp.ID
}

// PlaceName returns the name of the Stop Point
// Helps satisfy Place
func (sp StopPoint) PlaceName() string {
	return sp.Name
}

// PlaceType returns the type of place, in this case "stop_point"
// Helps satisfy Place
func (sp StopPoint) PlaceType() string {
	return embeddedStopPoint
}

// String pretty-prints the StopPoint.
// Satisfies Stringer and helps satisfy Place
func (sp StopPoint) String() string {
	format := "%s (id: %s)"
	return fmt.Sprintf(format, sp.Name, sp.ID)
}

// An Admin represents an administrative region: a region under the control/responsibility of a specific organisation.
// It can be a city, a district, a neightborhood, etc.
type Admin struct {
	ID   ID     `json:"id"`
	Name string `json:"name"`

	// Label of the address
	// The name is directly taken from the data whereas the label is something computed by navitia for better traveler information.
	// If you don't know what to display, display the label
	Label string `json:"label"`

	// Coordinates of the administrative region
	Coord Coordinates `json:"coord"`

	// Level of the administrative region
	Level int `json:"level"`

	// Zip code of the administrative region
	ZipCode string `json:"zip_code"`
}

// PlaceID returns the ID associated with the Admin
// Helps satisfy Place
func (ar Admin) PlaceID() ID {
	return ar.ID
}

// PlaceName returns the name of the Admin
// Helps satisfy Place
func (ar Admin) PlaceName() string {
	return ar.Name
}

// PlaceType returns the type of place, in this case "administrative_region"
// Helps satisfy Place
func (ar Admin) PlaceType() string {
	return embeddedAdmin
}

// String pretty-prints the Admin.
// Satisfies Stringer and helps satisfy Place
func (ar Admin) String() string {
	var label string
	if ar.Label == "" {
		label = ar.Name
	} else {
		label = ar.Label
	}

	format := "%s (id: %s)"
	return fmt.Sprintf(format, label, ar.ID)
}
<|endoftext|>


<commit_before>package airstrike

import (
	"errors"
	"runtime"
	"strings"

	"github.com/Sirupsen/logrus"
	"github.com/dysolution/airstrike/ordnance"
	"github.com/dysolution/sleepwalker"
)

var log = logrus.New()

// A Plane has an arsenal of deployable weapons. It represents a list of
// tasks that, perfored serially, compose a workflow.
//
// Many planes can deploy their arsenal at the same time, but each
// weapon in a plane's arsenal must be deployed one at a time.
//
// For example, a common workflow would be:
//   1. GET index - list all items in a collection
//   2. GET show - get the metadata for an item
//   3. POST create - create and/or associate an item related to the first
//
type Plane struct {
	Name    string `json:"name"`
	Client  sleepwalker.RESTClient
	Arsenal ordnance.Arsenal `json:"arsenal"`
}

// NewPlane ensures that the creation of each Plane is logged.
func NewPlane(name string, client sleepwalker.RESTClient) Plane {
	myPC, _, _, _ := runtime.Caller(0)
	desc := runtime.FuncForPC(myPC).Name()
	desc = strings.SplitAfter(desc, "github.com/dysolution/")[1]
	log.WithFields(logrus.Fields{
		"name":   name,
		"client": client,
	}).Debug(desc)
	return Plane{Name: name, Client: client}
}

// Arm loads the given arsenal into the Plane and logs error conditions.
func (p *Plane) Arm(weapons ordnance.Arsenal) {
	desc := "airstrike.(*Plane).Arm"
	if len(weapons) == 0 {
		log.WithFields(logrus.Fields{
			"plane":   p.Name,
			"weapons": weapons,
			"error":   errors.New("no weapons provided"),
		}).Error(desc)
	} else {
		log.WithFields(logrus.Fields{
			"plane":   p.Name,
			"weapons": weapons,
		}).Debug(desc)
		p.Arsenal = weapons
	}
}

// Launch tells a Plane to sequentially fires all of its weapons and report
// the results.
func (p Plane) Launch() ([]sleepwalker.Result, error) {
	var results []sleepwalker.Result

	myPC, _, _, _ := runtime.Caller(0)
	desc := runtime.FuncForPC(myPC).Name()
	desc = strings.SplitAfter(desc, "github.com/dysolution/")[1]

	log.WithFields(logrus.Fields{
		"plane": p,
	}).Info(desc)

	for _, weapon := range p.Arsenal {

		if weapon == nil {
			log.WithFields(logrus.Fields{
				"error":  "nil weapon",
				"plane":  p,
				"weapon": weapon,
			}).Error(desc)
			continue
		}

		log.WithFields(logrus.Fields{
			"client": p.Client,
			"msg":    "firing weapon",
			"plane":  p,
			"weapon": weapon,
		}).Debug(desc)

		if p.Client == nil {
			log.WithFields(logrus.Fields{
				"plane": p,
				"error": "nil client",
			}).Warn(desc)
			continue
		}

		result, _ := weapon.Fire(p.Client) // Fire does its own error logging
		results = append(results, result)
	}
	log.Debugf(desc+" is returning %v results", len(results))
	return results, nil
}
<commit_msg>fix: hide Client from marshaled output<commit_after>package airstrike

import (
	"errors"
	"runtime"
	"strings"

	"github.com/Sirupsen/logrus"
	"github.com/dysolution/airstrike/ordnance"
	"github.com/dysolution/sleepwalker"
)

var log = logrus.New()

// A Plane has an arsenal of deployable weapons. It represents a list of
// tasks that, perfored serially, compose a workflow.
//
// Many planes can deploy their arsenal at the same time, but each
// weapon in a plane's arsenal must be deployed one at a time.
//
// For example, a common workflow would be:
//   1. GET index - list all items in a collection
//   2. GET show - get the metadata for an item
//   3. POST create - create and/or associate an item related to the first
//
type Plane struct {
	Name    string                 `json:"name"`
	Client  sleepwalker.RESTClient `json:"-"`
	Arsenal ordnance.Arsenal       `json:"arsenal"`
}

// NewPlane ensures that the creation of each Plane is logged.
func NewPlane(name string, client sleepwalker.RESTClient) Plane {
	myPC, _, _, _ := runtime.Caller(0)
	desc := runtime.FuncForPC(myPC).Name()
	desc = strings.SplitAfter(desc, "github.com/dysolution/")[1]
	log.WithFields(logrus.Fields{
		"name":   name,
		"client": client,
	}).Debug(desc)
	return Plane{Name: name, Client: client}
}

// Arm loads the given arsenal into the Plane and logs error conditions.
func (p *Plane) Arm(weapons ordnance.Arsenal) {
	desc := "airstrike.(*Plane).Arm"
	if len(weapons) == 0 {
		log.WithFields(logrus.Fields{
			"plane":   p.Name,
			"weapons": weapons,
			"error":   errors.New("no weapons provided"),
		}).Error(desc)
	} else {
		log.WithFields(logrus.Fields{
			"plane":   p.Name,
			"weapons": weapons,
		}).Debug(desc)
		p.Arsenal = weapons
	}
}

// Launch tells a Plane to sequentially fires all of its weapons and report
// the results.
func (p Plane) Launch() ([]sleepwalker.Result, error) {
	var results []sleepwalker.Result

	myPC, _, _, _ := runtime.Caller(0)
	desc := runtime.FuncForPC(myPC).Name()
	desc = strings.SplitAfter(desc, "github.com/dysolution/")[1]

	log.WithFields(logrus.Fields{
		"plane": p,
	}).Info(desc)

	for _, weapon := range p.Arsenal {

		if weapon == nil {
			log.WithFields(logrus.Fields{
				"error":  "nil weapon",
				"plane":  p,
				"weapon": weapon,
			}).Error(desc)
			continue
		}

		log.WithFields(logrus.Fields{
			"client": p.Client,
			"msg":    "firing weapon",
			"plane":  p,
			"weapon": weapon,
		}).Debug(desc)

		if p.Client == nil {
			log.WithFields(logrus.Fields{
				"plane": p,
				"error": "nil client",
			}).Warn(desc)
			continue
		}

		result, _ := weapon.Fire(p.Client) // Fire does its own error logging
		results = append(results, result)
	}
	log.Debugf(desc+" is returning %v results", len(results))
	return results, nil
}
<|endoftext|>