<commit_before>// Copyright 2019 The GCR Cleaner Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcrcleaner

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"sort"
	"strings"
	"time"
)

const (
	contentTypeHeader = "Content-Type"
	contentTypeJSON   = "application/json"
)

// Server is a cleaning server.
type Server struct {
	cleaner *Cleaner
	logger  *Logger
}

// NewServer creates a new server for handler functions.
func NewServer(cleaner *Cleaner) (*Server, error) {
	if cleaner == nil {
		return nil, fmt.Errorf("missing cleaner")
	}

	return &Server{
		cleaner: cleaner,
		logger:  cleaner.logger,
	}, nil
}

// PubSubHandler is an http handler that invokes the cleaner from a pubsub
// request. Unlike an HTTP request, the pubsub endpoint always returns a success
// unless the pubsub message is malformed.
func (s *Server) PubSubHandler(cache Cache) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		var m pubsubMessage
		if err := json.NewDecoder(r.Body).Decode(&m); err != nil {
			err = fmt.Errorf("failed to decode pubsub message: %w", err)
			s.handleError(w, err, 400)
			return
		}

		// PubSub is "at least once" delivery. The cleaner is idempotent, but
		// let's try to prevent unnecessary work by not processing messages we've
		// already received.
		msgID := m.Subscription + "/" + m.Message.ID
		if exists := cache.Insert(msgID); exists {
			s.logger.Info("already processed message", "id", msgID)
			w.WriteHeader(204)
			return
		}

		if len(m.Message.Data) == 0 {
			err := fmt.Errorf("missing data in pubsub payload")
			s.handleError(w, err, 400)
			return
		}

		// Start a goroutine to delete the images
		body := ioutil.NopCloser(bytes.NewReader(m.Message.Data))
		go func() {
			// Intentionally don't use the request context, since it terminates but
			// the background job should still be processing.
			ctx := context.Background()
			if _, _, err := s.clean(ctx, body); err != nil {
				s.logger.Error("failed to clean", "error", err)
			}
		}()

		w.WriteHeader(204)
	}
}

// HTTPHandler is an http handler that invokes the cleaner with the given
// parameters.
func (s *Server) HTTPHandler() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		ctx := r.Context()

		deleted, status, err := s.clean(ctx, r.Body)
		if err != nil {
			s.handleError(w, err, status)
			return
		}

		refs := make([]string, 0, 16)
		for _, v := range deleted {
			refs = append(refs, v...)
		}
		sort.Strings(refs)

		b, err := json.Marshal(&cleanResp{
			Count:      len(deleted),
			Refs:       refs,
			RefsByRepo: deleted,
		})
		if err != nil {
			err = fmt.Errorf("failed to marshal JSON errors: %w", err)
			s.handleError(w, err, 500)
			return
		}

		w.WriteHeader(200)
		w.Header().Set(contentTypeHeader, contentTypeJSON)
		fmt.Fprint(w, string(b))
	}
}

// clean reads the given body as JSON and starts a cleaner instance.
func (s *Server) clean(ctx context.Context, r io.ReadCloser) (map[string][]string, int, error) {
	var p Payload
	if err := json.NewDecoder(r).Decode(&p); err != nil {
		return nil, 500, fmt.Errorf("failed to decode payload as JSON: %w", err)
	}

	// Convert duration to a negative value, since we're about to "add" it to the
	// since time.
	sub := time.Duration(p.Grace)
	if p.Grace > 0 {
		sub = sub * -1
	}

	since := time.Now().UTC().Add(sub)
	tagFilter, err := BuildTagFilter(p.TagFilterFirst, p.TagFilterAny, p.TagFilterAll)
	if err != nil {
		return nil, http.StatusBadRequest, fmt.Errorf("failed to build tag filter: %w", err)
	}

	// Gather all the repositories.
	repos := make([]string, 0, len(p.Repos))
	for _, v := range p.Repos {
		if t := strings.TrimSpace(v); t != "" {
			repos = append(repos, t)
		}
	}
	if t := strings.TrimSpace(p.Repo); t != "" {
		s.logger.Warn("specifying \"repo\" is deprecated - switch to \"repos\"")
		repos = append(repos, t)
	}
	if p.Recursive {
		s.logger.Debug("gathering child repositories recursively")

		allRepos, err := s.cleaner.ListChildRepositories(ctx, repos)
		if err != nil {
			return nil, http.StatusBadRequest, fmt.Errorf("failed to list child repositories for %q: %w", p.Repo, err)
		}
		s.logger.Debug("recursively listed child repositories",
			"in", repos,
			"out", allRepos)

		// This is safe because ListChildRepositories is guaranteed to include at
		// least the list repos givenh to it.
		repos = allRepos
	}

	s.logger.Info("deleting refs",
		"since", since,
		"repos", repos)

	// Do the deletion.
	deleted := make(map[string][]string, len(repos))
	for _, repo := range repos {
		s.logger.Info("deleting refs for repo", "repo", repo)

		childrenDeleted, err := s.cleaner.Clean(ctx, repo, since, p.Keep, tagFilter, p.DryRun)
		if err != nil {
			return nil, http.StatusBadRequest, fmt.Errorf("failed to clean repo %q: %w", repo, err)
		}

		if len(childrenDeleted) > 0 {
			s.logger.Info("deleted refs", "repo", repo, "refs", childrenDeleted)
			deleted[repo] = append(deleted[repo], childrenDeleted...)
		}
	}

	s.logger.Info("deleted refs", "refs", deleted)

	return deleted, http.StatusOK, nil
}

// handleError returns a JSON-formatted error message
func (s *Server) handleError(w http.ResponseWriter, err error, status int) {
	s.logger.Error(err.Error(), "error", err)

	b, err := json.Marshal(&errorResp{Error: err.Error()})
	if err != nil {
		err = fmt.Errorf("failed to marshal JSON errors: %w", err)
		http.Error(w, err.Error(), 500)
		return
	}

	w.WriteHeader(status)
	w.Header().Set(contentTypeHeader, contentTypeJSON)
	fmt.Fprint(w, string(b))
}

// Payload is the expected incoming payload format.
type Payload struct {
	// Repo is the name of the repo to clean.
	//
	// Deprecated: Use Repos instead.
	Repo string `json:"repo"`

	// Repos is the list of repositories to clean.
	Repos sortedStringSlice `json:"repos"`

	// Grace is a time.Duration value indicating how much grade period should be
	// given to new, untagged layers. The default is no grace.
	Grace duration `json:"grace"`

	// Keep is the minimum number of images to keep.
	Keep int `json:"keep"`

	// TagFilterAny is the tags pattern to be allowed removing. If given, any
	// image with at least one tag that matches this given regular expression will
	// be deleted. The image will be deleted even if it has other tags that do not
	// match the given regular expression.
	TagFilterAny string `json:"tag_filter_any"`

	// TagFilterAll is the tags pattern to be allowed removing. If given, any
	// image where all tags match this given regular expression will be deleted.
	// The image will not be delete if it has other tags that do not match the
	// given regular expression.
	TagFilterAll string `json:"tag_filter_all"`

	// DryRun instructs the server to not perform actual cleaning. The response
	// will include repositories that would have been deleted.
	DryRun bool `json:"dry_run"`

	// Recursive enables cleaning all child repositories.
	Recursive bool `json:"recursive"`

	// TagFilterFirst is the tags pattern to be allowed removing. If specified, any
	// images where the first tag matches the given regular expression will be
	// deleted.
	//
	// Deprecated: Use tag_filter_all or tag_filter_any instead.
	TagFilterFirst string `json:"tag_filter"`

	// AllowTagged is a Boolean value determine if tagged images are allowed to be
	// deleted.
	//
	// Deprecated: Use tag_filter_all or tag_filter_any instead. Setting either of
	// these values enables deleting tagged images.
	AllowTagged bool `json:"allow_tagged"`
}

type pubsubMessage struct {
	Message struct {
		Data []byte `json:"data"`
		ID   string `json:"message_id"`
	} `json:"message"`
	Subscription string `json:"subscription"`
}

type cleanResp struct {
	Count      int                 `json:"count"`
	Refs       []string            `json:"refs"`
	RefsByRepo map[string][]string `json:"refs_by_repo"`
}

type errorResp struct {
	Error string `json:"error"`
}

type sortedStringSlice []string

func (s sortedStringSlice) MarshalJSON() ([]byte, error) {
	return json.Marshal([]string(s))
}

func (s *sortedStringSlice) UnmarshalJSON(b []byte) error {
	var v any
	if err := json.Unmarshal(b, &v); err != nil {
		return err
	}

	m := make(map[string]struct{}, 4)

	switch val := v.(type) {
	case string:
		if t := strings.TrimSpace(val); t != "" {
			m[t] = struct{}{}
		}
	case any:
		for i, v := range val {
			s, ok := v.(string)
			if !ok {
				return fmt.Errorf("list must contain only strings (got %T at index %d)", v, i)
			}
			if t := strings.TrimSpace(s); t != "" {
				m[t] = struct{}{}
			}
		}
	case []string:
		for _, v := range val {
			if t := strings.TrimSpace(v); t != "" {
				m[t] = struct{}{}
			}
		}
	default:
		return fmt.Errorf("invalid list type %T", val)
	}

	list := make([]string, 0, len(m))
	for v := range m {
		list = append(list, v)
	}
	sort.Strings(list)
	*s = list

	return nil
}

type duration time.Duration

func (d duration) MarshalJSON() ([]byte, error) {
	return json.Marshal(time.Duration(d).String())
}

func (d *duration) UnmarshalJSON(b []byte) error {
	var v any
	if err := json.Unmarshal(b, &v); err != nil {
		return err
	}

	switch val := v.(type) {
	case float64:
		*d = duration(time.Duration(val))
		return nil
	case string:
		s, err := time.ParseDuration(val)
		if err != nil {
			return err
		}
		*d = duration(s)
		return nil
	default:
		return fmt.Errorf("invalid duration type %T", val)
	}
}
<commit_msg>Fix iteration<commit_after>// Copyright 2019 The GCR Cleaner Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcrcleaner

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"sort"
	"strings"
	"time"
)

const (
	contentTypeHeader = "Content-Type"
	contentTypeJSON   = "application/json"
)

// Server is a cleaning server.
type Server struct {
	cleaner *Cleaner
	logger  *Logger
}

// NewServer creates a new server for handler functions.
func NewServer(cleaner *Cleaner) (*Server, error) {
	if cleaner == nil {
		return nil, fmt.Errorf("missing cleaner")
	}

	return &Server{
		cleaner: cleaner,
		logger:  cleaner.logger,
	}, nil
}

// PubSubHandler is an http handler that invokes the cleaner from a pubsub
// request. Unlike an HTTP request, the pubsub endpoint always returns a success
// unless the pubsub message is malformed.
func (s *Server) PubSubHandler(cache Cache) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		var m pubsubMessage
		if err := json.NewDecoder(r.Body).Decode(&m); err != nil {
			err = fmt.Errorf("failed to decode pubsub message: %w", err)
			s.handleError(w, err, 400)
			return
		}

		// PubSub is "at least once" delivery. The cleaner is idempotent, but
		// let's try to prevent unnecessary work by not processing messages we've
		// already received.
		msgID := m.Subscription + "/" + m.Message.ID
		if exists := cache.Insert(msgID); exists {
			s.logger.Info("already processed message", "id", msgID)
			w.WriteHeader(204)
			return
		}

		if len(m.Message.Data) == 0 {
			err := fmt.Errorf("missing data in pubsub payload")
			s.handleError(w, err, 400)
			return
		}

		// Start a goroutine to delete the images
		body := ioutil.NopCloser(bytes.NewReader(m.Message.Data))
		go func() {
			// Intentionally don't use the request context, since it terminates but
			// the background job should still be processing.
			ctx := context.Background()
			if _, _, err := s.clean(ctx, body); err != nil {
				s.logger.Error("failed to clean", "error", err)
			}
		}()

		w.WriteHeader(204)
	}
}

// HTTPHandler is an http handler that invokes the cleaner with the given
// parameters.
func (s *Server) HTTPHandler() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		ctx := r.Context()

		deleted, status, err := s.clean(ctx, r.Body)
		if err != nil {
			s.handleError(w, err, status)
			return
		}

		refs := make([]string, 0, 16)
		for _, v := range deleted {
			refs = append(refs, v...)
		}
		sort.Strings(refs)

		b, err := json.Marshal(&cleanResp{
			Count:      len(deleted),
			Refs:       refs,
			RefsByRepo: deleted,
		})
		if err != nil {
			err = fmt.Errorf("failed to marshal JSON errors: %w", err)
			s.handleError(w, err, 500)
			return
		}

		w.WriteHeader(200)
		w.Header().Set(contentTypeHeader, contentTypeJSON)
		fmt.Fprint(w, string(b))
	}
}

// clean reads the given body as JSON and starts a cleaner instance.
func (s *Server) clean(ctx context.Context, r io.ReadCloser) (map[string][]string, int, error) {
	var p Payload
	if err := json.NewDecoder(r).Decode(&p); err != nil {
		return nil, 500, fmt.Errorf("failed to decode payload as JSON: %w", err)
	}

	// Convert duration to a negative value, since we're about to "add" it to the
	// since time.
	sub := time.Duration(p.Grace)
	if p.Grace > 0 {
		sub = sub * -1
	}

	since := time.Now().UTC().Add(sub)
	tagFilter, err := BuildTagFilter(p.TagFilterFirst, p.TagFilterAny, p.TagFilterAll)
	if err != nil {
		return nil, http.StatusBadRequest, fmt.Errorf("failed to build tag filter: %w", err)
	}

	// Gather all the repositories.
	repos := make([]string, 0, len(p.Repos))
	for _, v := range p.Repos {
		if t := strings.TrimSpace(v); t != "" {
			repos = append(repos, t)
		}
	}
	if t := strings.TrimSpace(p.Repo); t != "" {
		s.logger.Warn("specifying \"repo\" is deprecated - switch to \"repos\"")
		repos = append(repos, t)
	}
	if p.Recursive {
		s.logger.Debug("gathering child repositories recursively")

		allRepos, err := s.cleaner.ListChildRepositories(ctx, repos)
		if err != nil {
			return nil, http.StatusBadRequest, fmt.Errorf("failed to list child repositories for %q: %w", p.Repo, err)
		}
		s.logger.Debug("recursively listed child repositories",
			"in", repos,
			"out", allRepos)

		// This is safe because ListChildRepositories is guaranteed to include at
		// least the list repos givenh to it.
		repos = allRepos
	}

	s.logger.Info("deleting refs",
		"since", since,
		"repos", repos)

	// Do the deletion.
	deleted := make(map[string][]string, len(repos))
	for _, repo := range repos {
		s.logger.Info("deleting refs for repo", "repo", repo)

		childrenDeleted, err := s.cleaner.Clean(ctx, repo, since, p.Keep, tagFilter, p.DryRun)
		if err != nil {
			return nil, http.StatusBadRequest, fmt.Errorf("failed to clean repo %q: %w", repo, err)
		}

		if len(childrenDeleted) > 0 {
			s.logger.Info("deleted refs", "repo", repo, "refs", childrenDeleted)
			deleted[repo] = append(deleted[repo], childrenDeleted...)
		}
	}

	s.logger.Info("deleted refs", "refs", deleted)

	return deleted, http.StatusOK, nil
}

// handleError returns a JSON-formatted error message
func (s *Server) handleError(w http.ResponseWriter, err error, status int) {
	s.logger.Error(err.Error(), "error", err)

	b, err := json.Marshal(&errorResp{Error: err.Error()})
	if err != nil {
		err = fmt.Errorf("failed to marshal JSON errors: %w", err)
		http.Error(w, err.Error(), 500)
		return
	}

	w.WriteHeader(status)
	w.Header().Set(contentTypeHeader, contentTypeJSON)
	fmt.Fprint(w, string(b))
}

// Payload is the expected incoming payload format.
type Payload struct {
	// Repo is the name of the repo to clean.
	//
	// Deprecated: Use Repos instead.
	Repo string `json:"repo"`

	// Repos is the list of repositories to clean.
	Repos sortedStringSlice `json:"repos"`

	// Grace is a time.Duration value indicating how much grade period should be
	// given to new, untagged layers. The default is no grace.
	Grace duration `json:"grace"`

	// Keep is the minimum number of images to keep.
	Keep int `json:"keep"`

	// TagFilterAny is the tags pattern to be allowed removing. If given, any
	// image with at least one tag that matches this given regular expression will
	// be deleted. The image will be deleted even if it has other tags that do not
	// match the given regular expression.
	TagFilterAny string `json:"tag_filter_any"`

	// TagFilterAll is the tags pattern to be allowed removing. If given, any
	// image where all tags match this given regular expression will be deleted.
	// The image will not be delete if it has other tags that do not match the
	// given regular expression.
	TagFilterAll string `json:"tag_filter_all"`

	// DryRun instructs the server to not perform actual cleaning. The response
	// will include repositories that would have been deleted.
	DryRun bool `json:"dry_run"`

	// Recursive enables cleaning all child repositories.
	Recursive bool `json:"recursive"`

	// TagFilterFirst is the tags pattern to be allowed removing. If specified, any
	// images where the first tag matches the given regular expression will be
	// deleted.
	//
	// Deprecated: Use tag_filter_all or tag_filter_any instead.
	TagFilterFirst string `json:"tag_filter"`

	// AllowTagged is a Boolean value determine if tagged images are allowed to be
	// deleted.
	//
	// Deprecated: Use tag_filter_all or tag_filter_any instead. Setting either of
	// these values enables deleting tagged images.
	AllowTagged bool `json:"allow_tagged"`
}

type pubsubMessage struct {
	Message struct {
		Data []byte `json:"data"`
		ID   string `json:"message_id"`
	} `json:"message"`
	Subscription string `json:"subscription"`
}

type cleanResp struct {
	Count      int                 `json:"count"`
	Refs       []string            `json:"refs"`
	RefsByRepo map[string][]string `json:"refs_by_repo"`
}

type errorResp struct {
	Error string `json:"error"`
}

type sortedStringSlice []string

func (s sortedStringSlice) MarshalJSON() ([]byte, error) {
	return json.Marshal([]string(s))
}

func (s *sortedStringSlice) UnmarshalJSON(b []byte) error {
	var v any
	if err := json.Unmarshal(b, &v); err != nil {
		return err
	}

	m := make(map[string]struct{}, 4)

	switch val := v.(type) {
	case string:
		if t := strings.TrimSpace(val); t != "" {
			m[t] = struct{}{}
		}
	case []any:
		for i, v := range val {
			s, ok := v.(string)
			if !ok {
				return fmt.Errorf("list must contain only strings (got %T at index %d)", v, i)
			}
			if t := strings.TrimSpace(s); t != "" {
				m[t] = struct{}{}
			}
		}
	case []string:
		for _, v := range val {
			if t := strings.TrimSpace(v); t != "" {
				m[t] = struct{}{}
			}
		}
	default:
		return fmt.Errorf("invalid list type %T", val)
	}

	list := make([]string, 0, len(m))
	for v := range m {
		list = append(list, v)
	}
	sort.Strings(list)
	*s = list

	return nil
}

type duration time.Duration

func (d duration) MarshalJSON() ([]byte, error) {
	return json.Marshal(time.Duration(d).String())
}

func (d *duration) UnmarshalJSON(b []byte) error {
	var v any
	if err := json.Unmarshal(b, &v); err != nil {
		return err
	}

	switch val := v.(type) {
	case float64:
		*d = duration(time.Duration(val))
		return nil
	case string:
		s, err := time.ParseDuration(val)
		if err != nil {
			return err
		}
		*d = duration(s)
		return nil
	default:
		return fmt.Errorf("invalid duration type %T", val)
	}
}
<|endoftext|>


<commit_before>package integration

import (
	"archive/tar"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"reflect"
	"strings"
	"syscall"
	"time"

	"github.com/docker/docker/pkg/stringutils"
)

// GetExitCode returns the ExitStatus of the specified error if its type is
// exec.ExitError, returns 0 and an error otherwise.
func GetExitCode(err error) (int, error) {
	exitCode := 0
	if exiterr, ok := err.(*exec.ExitError); ok {
		if procExit, ok := exiterr.Sys().(syscall.WaitStatus); ok {
			return procExit.ExitStatus(), nil
		}
	}
	return exitCode, fmt.Errorf("failed to get exit code")
}

// ProcessExitCode process the specified error and returns the exit status code
// if the error was of type exec.ExitError, returns nothing otherwise.
func ProcessExitCode(err error) (exitCode int) {
	if err != nil {
		var exiterr error
		if exitCode, exiterr = GetExitCode(err); exiterr != nil {
			// TODO: Fix this so we check the error's text.
			// we've failed to retrieve exit code, so we set it to 127
			exitCode = 127
		}
	}
	return
}

// IsKilled process the specified error and returns whether the process was killed or not.
func IsKilled(err error) bool {
	if exitErr, ok := err.(*exec.ExitError); ok {
		status, ok := exitErr.Sys().(syscall.WaitStatus)
		if !ok {
			return false
		}
		// status.ExitStatus() is required on Windows because it does not
		// implement Signal() nor Signaled(). Just check it had a bad exit
		// status could mean it was killed (and in tests we do kill)
		return (status.Signaled() && status.Signal() == os.Kill) || status.ExitStatus() != 0
	}
	return false
}

// RunCommandWithOutput runs the specified command and returns the combined output (stdout/stderr)
// with the exitCode different from 0 and the error if something bad happened
func RunCommandWithOutput(cmd *exec.Cmd) (output string, exitCode int, err error) {
	exitCode = 0
	out, err := cmd.CombinedOutput()
	exitCode = ProcessExitCode(err)
	output = string(out)
	return
}

// RunCommandWithStdoutStderr runs the specified command and returns stdout and stderr separately
// with the exitCode different from 0 and the error if something bad happened
func RunCommandWithStdoutStderr(cmd *exec.Cmd) (stdout string, stderr string, exitCode int, err error) {
	var (
		stderrBuffer, stdoutBuffer bytes.Buffer
	)
	exitCode = 0
	cmd.Stderr = &stderrBuffer
	cmd.Stdout = &stdoutBuffer
	err = cmd.Run()
	exitCode = ProcessExitCode(err)

	stdout = stdoutBuffer.String()
	stderr = stderrBuffer.String()
	return
}

// RunCommandWithOutputForDuration runs the specified command "timeboxed" by the specified duration.
// If the process is still running when the timebox is finished, the process will be killed and .
// It will returns the output with the exitCode different from 0 and the error if something bad happened
// and a boolean whether it has been killed or not.
func RunCommandWithOutputForDuration(cmd *exec.Cmd, duration time.Duration) (output string, exitCode int, timedOut bool, err error) {
	var outputBuffer bytes.Buffer
	if cmd.Stdout != nil {
		err = errors.New("cmd.Stdout already set")
		return
	}
	cmd.Stdout = &outputBuffer

	if cmd.Stderr != nil {
		err = errors.New("cmd.Stderr already set")
		return
	}
	cmd.Stderr = &outputBuffer

	done := make(chan error)

	// Start the command in the main thread..
	err = cmd.Start()
	if err != nil {
		err = fmt.Errorf("Fail to start command %v : %v", cmd, err)
	}

	go func() {
		// And wait for it to exit in the goroutine :)
		exitErr := cmd.Wait()
		exitCode = ProcessExitCode(exitErr)
		done <- exitErr
	}()

	select {
	case <-time.After(duration):
		killErr := cmd.Process.Kill()
		if killErr != nil {
			fmt.Printf("failed to kill (pid=%d): %v\n", cmd.Process.Pid, killErr)
		}
		timedOut = true
		break
	case err = <-done:
		break
	}
	output = outputBuffer.String()
	return
}

var errCmdTimeout = fmt.Errorf("command timed out")

// RunCommandWithOutputAndTimeout runs the specified command "timeboxed" by the specified duration.
// It returns the output with the exitCode different from 0 and the error if something bad happened or
// if the process timed out (and has been killed).
func RunCommandWithOutputAndTimeout(cmd *exec.Cmd, timeout time.Duration) (output string, exitCode int, err error) {
	var timedOut bool
	output, exitCode, timedOut, err = RunCommandWithOutputForDuration(cmd, timeout)
	if timedOut {
		err = errCmdTimeout
	}
	return
}

// RunCommand runs the specified command and returns the exitCode different from 0
// and the error if something bad happened.
func RunCommand(cmd *exec.Cmd) (exitCode int, err error) {
	exitCode = 0
	err = cmd.Run()
	exitCode = ProcessExitCode(err)
	return
}

// RunCommandPipelineWithOutput runs the array of commands with the output
// of each pipelined with the following (like cmd1 | cmd2 | cmd3 would do).
// It returns the final output, the exitCode different from 0 and the error
// if something bad happened.
func RunCommandPipelineWithOutput(cmds ...*exec.Cmd) (output string, exitCode int, err error) {
	if len(cmds) < 2 {
		return "", 0, errors.New("pipeline does not have multiple cmds")
	}

	// connect stdin of each cmd to stdout pipe of previous cmd
	for i, cmd := range cmds {
		if i > 0 {
			prevCmd := cmds[i-1]
			cmd.Stdin, err = prevCmd.StdoutPipe()

			if err != nil {
				return "", 0, fmt.Errorf("cannot set stdout pipe for %s: %v", cmd.Path, err)
			}
		}
	}

	// start all cmds except the last
	for _, cmd := range cmds[:len(cmds)-1] {
		if err = cmd.Start(); err != nil {
			return "", 0, fmt.Errorf("starting %s failed with error: %v", cmd.Path, err)
		}
	}

	defer func() {
		// wait all cmds except the last to release their resources
		for _, cmd := range cmds[:len(cmds)-1] {
			cmd.Wait()
		}
	}()

	// wait on last cmd
	return RunCommandWithOutput(cmds[len(cmds)-1])
}

// UnmarshalJSON deserialize a JSON in the given interface.
func UnmarshalJSON(data []byte, result interface{}) error {
	if err := json.Unmarshal(data, result); err != nil {
		return err
	}

	return nil
}

// ConvertSliceOfStringsToMap converts a slices of string in a map
// with the strings as key and an empty string as values.
func ConvertSliceOfStringsToMap(input []string) map[string]struct{} {
	output := make(map[string]struct{})
	for _, v := range input {
		output[v] = struct{}{}
	}
	return output
}

// CompareDirectoryEntries compares two sets of FileInfo (usually taken from a directory)
// and returns an error if different.
func CompareDirectoryEntries(e1 []os.FileInfo, e2 []os.FileInfo) error {
	var (
		e1Entries = make(map[string]struct{})
		e2Entries = make(map[string]struct{})
	)
	for _, e := range e1 {
		e1Entries[e.Name()] = struct{}{}
	}
	for _, e := range e2 {
		e2Entries[e.Name()] = struct{}{}
	}
	if !reflect.DeepEqual(e1Entries, e2Entries) {
		return fmt.Errorf("entries differ")
	}
	return nil
}

// ListTar lists the entries of a tar.
func ListTar(f io.Reader) ([]string, error) {
	tr := tar.NewReader(f)
	var entries []string

	for {
		th, err := tr.Next()
		if err == io.EOF {
			// end of tar archive
			return entries, nil
		}
		if err != nil {
			return entries, err
		}
		entries = append(entries, th.Name)
	}
}

// RandomTmpDirPath provides a temporary path with rand string appended.
// does not create or checks if it exists.
func RandomTmpDirPath(s string, platform string) string {
	tmp := "/tmp"
	if platform == "windows" {
		tmp = os.Getenv("TEMP")
	}
	path := filepath.Join(tmp, fmt.Sprintf("%s.%s", s, stringutils.GenerateRandomAlphaOnlyString(10)))
	if platform == "windows" {
		return filepath.FromSlash(path) // Using \
	}
	return filepath.ToSlash(path) // Using /
}

// ConsumeWithSpeed reads chunkSize bytes from reader after every interval.
// Returns total read bytes.
func ConsumeWithSpeed(reader io.Reader, chunkSize int, interval time.Duration, stop chan bool) (n int, err error) {
	buffer := make([]byte, chunkSize)
	for {
		select {
		case <-stop:
			return
		default:
			var readBytes int
			readBytes, err = reader.Read(buffer)
			n += readBytes
			if err != nil {
				if err == io.EOF {
					err = nil
				}
				return
			}
			time.Sleep(interval)
		}
	}
}

// ParseCgroupPaths arses 'procCgroupData', which is output of '/proc/<pid>/cgroup', and returns
// a map which cgroup name as key and path as value.
func ParseCgroupPaths(procCgroupData string) map[string]string {
	cgroupPaths := map[string]string{}
	for _, line := range strings.Split(procCgroupData, "\n") {
		parts := strings.Split(line, ":")
		if len(parts) != 3 {
			continue
		}
		cgroupPaths[parts[1]] = parts[2]
	}
	return cgroupPaths
}

// ChannelBuffer holds a chan of byte array that can be populate in a goroutine.
type ChannelBuffer struct {
	C chan []byte
}

// Write implements Writer.
func (c *ChannelBuffer) Write(b []byte) (int, error) {
	c.C <- b
	return len(b), nil
}

// Close closes the go channel.
func (c *ChannelBuffer) Close() error {
	close(c.C)
	return nil
}

// ReadTimeout reads the content of the channel in the specified byte array with
// the specified duration as timeout.
func (c *ChannelBuffer) ReadTimeout(p []byte, n time.Duration) (int, error) {
	select {
	case b := <-c.C:
		return copy(p[0:], b), nil
	case <-time.After(n):
		return -1, fmt.Errorf("timeout reading from channel")
	}
}

// RunAtDifferentDate runs the specifed function with the given time.
// It changes the date of the system, which can led to weird behaviors.
func RunAtDifferentDate(date time.Time, block func()) {
	// Layout for date. MMDDhhmmYYYY
	const timeLayout = "010203042006"
	// Ensure we bring time back to now
	now := time.Now().Format(timeLayout)
	dateReset := exec.Command("date", now)
	defer RunCommand(dateReset)

	dateChange := exec.Command("date", date.Format(timeLayout))
	RunCommand(dateChange)
	block()
	return
}
<commit_msg>Fix race in RunCommandWithOutputForDuration<commit_after>package integration

import (
	"archive/tar"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"reflect"
	"strings"
	"syscall"
	"time"

	"github.com/docker/docker/pkg/stringutils"
)

// GetExitCode returns the ExitStatus of the specified error if its type is
// exec.ExitError, returns 0 and an error otherwise.
func GetExitCode(err error) (int, error) {
	exitCode := 0
	if exiterr, ok := err.(*exec.ExitError); ok {
		if procExit, ok := exiterr.Sys().(syscall.WaitStatus); ok {
			return procExit.ExitStatus(), nil
		}
	}
	return exitCode, fmt.Errorf("failed to get exit code")
}

// ProcessExitCode process the specified error and returns the exit status code
// if the error was of type exec.ExitError, returns nothing otherwise.
func ProcessExitCode(err error) (exitCode int) {
	if err != nil {
		var exiterr error
		if exitCode, exiterr = GetExitCode(err); exiterr != nil {
			// TODO: Fix this so we check the error's text.
			// we've failed to retrieve exit code, so we set it to 127
			exitCode = 127
		}
	}
	return
}

// IsKilled process the specified error and returns whether the process was killed or not.
func IsKilled(err error) bool {
	if exitErr, ok := err.(*exec.ExitError); ok {
		status, ok := exitErr.Sys().(syscall.WaitStatus)
		if !ok {
			return false
		}
		// status.ExitStatus() is required on Windows because it does not
		// implement Signal() nor Signaled(). Just check it had a bad exit
		// status could mean it was killed (and in tests we do kill)
		return (status.Signaled() && status.Signal() == os.Kill) || status.ExitStatus() != 0
	}
	return false
}

// RunCommandWithOutput runs the specified command and returns the combined output (stdout/stderr)
// with the exitCode different from 0 and the error if something bad happened
func RunCommandWithOutput(cmd *exec.Cmd) (output string, exitCode int, err error) {
	exitCode = 0
	out, err := cmd.CombinedOutput()
	exitCode = ProcessExitCode(err)
	output = string(out)
	return
}

// RunCommandWithStdoutStderr runs the specified command and returns stdout and stderr separately
// with the exitCode different from 0 and the error if something bad happened
func RunCommandWithStdoutStderr(cmd *exec.Cmd) (stdout string, stderr string, exitCode int, err error) {
	var (
		stderrBuffer, stdoutBuffer bytes.Buffer
	)
	exitCode = 0
	cmd.Stderr = &stderrBuffer
	cmd.Stdout = &stdoutBuffer
	err = cmd.Run()
	exitCode = ProcessExitCode(err)

	stdout = stdoutBuffer.String()
	stderr = stderrBuffer.String()
	return
}

// RunCommandWithOutputForDuration runs the specified command "timeboxed" by the specified duration.
// If the process is still running when the timebox is finished, the process will be killed and .
// It will returns the output with the exitCode different from 0 and the error if something bad happened
// and a boolean whether it has been killed or not.
func RunCommandWithOutputForDuration(cmd *exec.Cmd, duration time.Duration) (output string, exitCode int, timedOut bool, err error) {
	var outputBuffer bytes.Buffer
	if cmd.Stdout != nil {
		err = errors.New("cmd.Stdout already set")
		return
	}
	cmd.Stdout = &outputBuffer

	if cmd.Stderr != nil {
		err = errors.New("cmd.Stderr already set")
		return
	}
	cmd.Stderr = &outputBuffer

	// Start the command in the main thread..
	err = cmd.Start()
	if err != nil {
		err = fmt.Errorf("Fail to start command %v : %v", cmd, err)
	}

	type exitInfo struct {
		exitErr  error
		exitCode int
	}

	done := make(chan exitInfo, 1)

	go func() {
		// And wait for it to exit in the goroutine :)
		info := exitInfo{}
		info.exitErr = cmd.Wait()
		info.exitCode = ProcessExitCode(info.exitErr)
		done <- info
	}()

	select {
	case <-time.After(duration):
		killErr := cmd.Process.Kill()
		if killErr != nil {
			fmt.Printf("failed to kill (pid=%d): %v\n", cmd.Process.Pid, killErr)
		}
		timedOut = true
	case info := <-done:
		err = info.exitErr
		exitCode = info.exitCode
	}
	output = outputBuffer.String()
	return
}

var errCmdTimeout = fmt.Errorf("command timed out")

// RunCommandWithOutputAndTimeout runs the specified command "timeboxed" by the specified duration.
// It returns the output with the exitCode different from 0 and the error if something bad happened or
// if the process timed out (and has been killed).
func RunCommandWithOutputAndTimeout(cmd *exec.Cmd, timeout time.Duration) (output string, exitCode int, err error) {
	var timedOut bool
	output, exitCode, timedOut, err = RunCommandWithOutputForDuration(cmd, timeout)
	if timedOut {
		err = errCmdTimeout
	}
	return
}

// RunCommand runs the specified command and returns the exitCode different from 0
// and the error if something bad happened.
func RunCommand(cmd *exec.Cmd) (exitCode int, err error) {
	exitCode = 0
	err = cmd.Run()
	exitCode = ProcessExitCode(err)
	return
}

// RunCommandPipelineWithOutput runs the array of commands with the output
// of each pipelined with the following (like cmd1 | cmd2 | cmd3 would do).
// It returns the final output, the exitCode different from 0 and the error
// if something bad happened.
func RunCommandPipelineWithOutput(cmds ...*exec.Cmd) (output string, exitCode int, err error) {
	if len(cmds) < 2 {
		return "", 0, errors.New("pipeline does not have multiple cmds")
	}

	// connect stdin of each cmd to stdout pipe of previous cmd
	for i, cmd := range cmds {
		if i > 0 {
			prevCmd := cmds[i-1]
			cmd.Stdin, err = prevCmd.StdoutPipe()

			if err != nil {
				return "", 0, fmt.Errorf("cannot set stdout pipe for %s: %v", cmd.Path, err)
			}
		}
	}

	// start all cmds except the last
	for _, cmd := range cmds[:len(cmds)-1] {
		if err = cmd.Start(); err != nil {
			return "", 0, fmt.Errorf("starting %s failed with error: %v", cmd.Path, err)
		}
	}

	defer func() {
		// wait all cmds except the last to release their resources
		for _, cmd := range cmds[:len(cmds)-1] {
			cmd.Wait()
		}
	}()

	// wait on last cmd
	return RunCommandWithOutput(cmds[len(cmds)-1])
}

// UnmarshalJSON deserialize a JSON in the given interface.
func UnmarshalJSON(data []byte, result interface{}) error {
	if err := json.Unmarshal(data, result); err != nil {
		return err
	}

	return nil
}

// ConvertSliceOfStringsToMap converts a slices of string in a map
// with the strings as key and an empty string as values.
func ConvertSliceOfStringsToMap(input []string) map[string]struct{} {
	output := make(map[string]struct{})
	for _, v := range input {
		output[v] = struct{}{}
	}
	return output
}

// CompareDirectoryEntries compares two sets of FileInfo (usually taken from a directory)
// and returns an error if different.
func CompareDirectoryEntries(e1 []os.FileInfo, e2 []os.FileInfo) error {
	var (
		e1Entries = make(map[string]struct{})
		e2Entries = make(map[string]struct{})
	)
	for _, e := range e1 {
		e1Entries[e.Name()] = struct{}{}
	}
	for _, e := range e2 {
		e2Entries[e.Name()] = struct{}{}
	}
	if !reflect.DeepEqual(e1Entries, e2Entries) {
		return fmt.Errorf("entries differ")
	}
	return nil
}

// ListTar lists the entries of a tar.
func ListTar(f io.Reader) ([]string, error) {
	tr := tar.NewReader(f)
	var entries []string

	for {
		th, err := tr.Next()
		if err == io.EOF {
			// end of tar archive
			return entries, nil
		}
		if err != nil {
			return entries, err
		}
		entries = append(entries, th.Name)
	}
}

// RandomTmpDirPath provides a temporary path with rand string appended.
// does not create or checks if it exists.
func RandomTmpDirPath(s string, platform string) string {
	tmp := "/tmp"
	if platform == "windows" {
		tmp = os.Getenv("TEMP")
	}
	path := filepath.Join(tmp, fmt.Sprintf("%s.%s", s, stringutils.GenerateRandomAlphaOnlyString(10)))
	if platform == "windows" {
		return filepath.FromSlash(path) // Using \
	}
	return filepath.ToSlash(path) // Using /
}

// ConsumeWithSpeed reads chunkSize bytes from reader after every interval.
// Returns total read bytes.
func ConsumeWithSpeed(reader io.Reader, chunkSize int, interval time.Duration, stop chan bool) (n int, err error) {
	buffer := make([]byte, chunkSize)
	for {
		select {
		case <-stop:
			return
		default:
			var readBytes int
			readBytes, err = reader.Read(buffer)
			n += readBytes
			if err != nil {
				if err == io.EOF {
					err = nil
				}
				return
			}
			time.Sleep(interval)
		}
	}
}

// ParseCgroupPaths arses 'procCgroupData', which is output of '/proc/<pid>/cgroup', and returns
// a map which cgroup name as key and path as value.
func ParseCgroupPaths(procCgroupData string) map[string]string {
	cgroupPaths := map[string]string{}
	for _, line := range strings.Split(procCgroupData, "\n") {
		parts := strings.Split(line, ":")
		if len(parts) != 3 {
			continue
		}
		cgroupPaths[parts[1]] = parts[2]
	}
	return cgroupPaths
}

// ChannelBuffer holds a chan of byte array that can be populate in a goroutine.
type ChannelBuffer struct {
	C chan []byte
}

// Write implements Writer.
func (c *ChannelBuffer) Write(b []byte) (int, error) {
	c.C <- b
	return len(b), nil
}

// Close closes the go channel.
func (c *ChannelBuffer) Close() error {
	close(c.C)
	return nil
}

// ReadTimeout reads the content of the channel in the specified byte array with
// the specified duration as timeout.
func (c *ChannelBuffer) ReadTimeout(p []byte, n time.Duration) (int, error) {
	select {
	case b := <-c.C:
		return copy(p[0:], b), nil
	case <-time.After(n):
		return -1, fmt.Errorf("timeout reading from channel")
	}
}

// RunAtDifferentDate runs the specifed function with the given time.
// It changes the date of the system, which can led to weird behaviors.
func RunAtDifferentDate(date time.Time, block func()) {
	// Layout for date. MMDDhhmmYYYY
	const timeLayout = "010203042006"
	// Ensure we bring time back to now
	now := time.Now().Format(timeLayout)
	dateReset := exec.Command("date", now)
	defer RunCommand(dateReset)

	dateChange := exec.Command("date", date.Format(timeLayout))
	RunCommand(dateChange)
	block()
	return
}
<|endoftext|>


<commit_before>// Copyright 2014 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package keystore implements the ACI keystore.
package keystore

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/coreos/rocket/Godeps/_workspace/src/github.com/appc/spec/schema/types"
	"github.com/coreos/rocket/Godeps/_workspace/src/golang.org/x/crypto/openpgp"
)

// A Config structure is used to configure a Keystore.
type Config struct {
	RootPath         string
	PrefixPath       string
	SystemRootPath   string
	SystemPrefixPath string
}

// A Keystore represents a repository of trusted public keys which can be
// used to verify PGP signatures.
type Keystore struct {
	*Config
}

// New returns a new Keystore based on config.
func New(config *Config) *Keystore {
	if config == nil {
		config = defaultConfig
	}
	return &Keystore{config}
}

var defaultConfig = &Config{
	RootPath:         "/etc/rkt/trustedkeys/root.d",
	PrefixPath:       "/etc/rkt/trustedkeys/prefix.d",
	SystemRootPath:   "/usr/lib/rkt/trustedkeys/root.d",
	SystemPrefixPath: "/usr/lib/rkt/trustedkeys/prefix.d",
}

// CheckSignature is a convenience method for creating a Keystore with a default
// configuration and invoking CheckSignature.
func CheckSignature(prefix string, signed, signature io.Reader) (*openpgp.Entity, error) {
	ks := New(defaultConfig)
	return checkSignature(ks, prefix, signed, signature)
}

// CheckSignature takes a signed file and a detached signature and returns the signer
// if the signature is signed by a trusted signer.
// If the signer is unknown or not trusted, opengpg.ErrUnknownIssuer is returned.
func (ks *Keystore) CheckSignature(prefix string, signed, signature io.Reader) (*openpgp.Entity, error) {
	return checkSignature(ks, prefix, signed, signature)
}

func checkSignature(ks *Keystore, prefix string, signed, signature io.Reader) (*openpgp.Entity, error) {
	acname, err := types.NewACName(prefix)
	if err != nil {
		return nil, err
	}
	keyring, err := ks.loadKeyring(acname.String())
	if err != nil {
		return nil, fmt.Errorf("keystore: error loading keyring %v", err)
	}
	return openpgp.CheckArmoredDetachedSignature(keyring, signed, signature)
}

// DeleteTrustedKeyPrefix deletes the prefix trusted key identified by fingerprint.
func (ks *Keystore) DeleteTrustedKeyPrefix(prefix, fingerprint string) error {
	acname, err := types.NewACName(prefix)
	if err != nil {
		return err
	}
	return os.Remove(path.Join(ks.PrefixPath, acname.String(), fingerprint))
}

// MaskTrustedKeySystemPrefix masks the system prefix trusted key identified by fingerprint.
func (ks *Keystore) MaskTrustedKeySystemPrefix(prefix, fingerprint string) (string, error) {
	acname, err := types.NewACName(prefix)
	if err != nil {
		return "", err
	}
	dst := path.Join(ks.PrefixPath, acname.String(), fingerprint)
	return dst, ioutil.WriteFile(dst, []byte(""), 0644)
}

// DeleteTrustedKeyRoot deletes the root trusted key identified by fingerprint.
func (ks *Keystore) DeleteTrustedKeyRoot(fingerprint string) error {
	return os.Remove(path.Join(ks.RootPath, fingerprint))
}

// MaskTrustedKeySystemRoot masks the system root trusted key identified by fingerprint.
func (ks *Keystore) MaskTrustedKeySystemRoot(fingerprint string) (string, error) {
	dst := path.Join(ks.RootPath, fingerprint)
	return dst, ioutil.WriteFile(dst, []byte(""), 0644)
}

// StoreTrustedKeyPrefix stores the contents of public key r as a prefix trusted key.
func (ks *Keystore) StoreTrustedKeyPrefix(prefix string, r io.Reader) (string, error) {
	acname, err := types.NewACName(prefix)
	if err != nil {
		return "", err
	}
	return storeTrustedKey(path.Join(ks.PrefixPath, acname.String()), r)
}

// StoreTrustedKeyRoot stores the contents of public key r as a root trusted key.
func (ks *Keystore) StoreTrustedKeyRoot(r io.Reader) (string, error) {
	return storeTrustedKey(ks.RootPath, r)
}

func storeTrustedKey(dir string, r io.Reader) (string, error) {
	pubkeyBytes, err := ioutil.ReadAll(r)
	if err != nil {
		return "", err
	}
	if err := os.MkdirAll(dir, 0700); err != nil {
		return "", err
	}
	entityList, err := openpgp.ReadArmoredKeyRing(bytes.NewReader(pubkeyBytes))
	if err != nil {
		return "", err
	}
	pubKey := entityList[0].PrimaryKey
	trustedKeyPath := path.Join(dir, fingerprintToFilename(pubKey.Fingerprint))
	if err := ioutil.WriteFile(trustedKeyPath, pubkeyBytes, 0644); err != nil {
		return "", err
	}
	return trustedKeyPath, nil
}

func entityFromFile(path string) (*openpgp.Entity, error) {
	trustedKey, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer trustedKey.Close()
	entityList, err := openpgp.ReadArmoredKeyRing(trustedKey)
	if err != nil {
		return nil, err
	}
	if len(entityList) < 1 {
		return nil, errors.New("missing opengpg entity")
	}
	fingerprint := fingerprintToFilename(entityList[0].PrimaryKey.Fingerprint)
	if fingerprint != filepath.Base(trustedKey.Name()) {
		return nil, fmt.Errorf("fingerprint mismatch")
	}
	return entityList[0], nil
}

func (ks *Keystore) loadKeyring(prefix string) (openpgp.KeyRing, error) {
	acname, err := types.NewACName(prefix)
	if err != nil {
		return nil, err
	}
	var keyring openpgp.EntityList
	trustedKeys := make(map[string]*openpgp.Entity)

	prefixRoot := strings.Split(acname.String(), "/")[0]
	paths := []struct {
		root     string
		fullPath string
	}{
		{ks.SystemRootPath, ks.SystemRootPath},
		{ks.RootPath, ks.RootPath},
		{path.Join(ks.SystemPrefixPath, prefixRoot), path.Join(ks.SystemPrefixPath, acname.String())},
		{path.Join(ks.PrefixPath, prefixRoot), path.Join(ks.PrefixPath, acname.String())},
	}
	for _, p := range paths {
		err := filepath.Walk(p.root, func(path string, info os.FileInfo, err error) error {
			if err != nil && !os.IsNotExist(err) {
				return err
			}
			if info == nil {
				return nil
			}
			if info.IsDir() {
				switch {
				case strings.HasPrefix(p.fullPath, path):
					return nil
				default:
					return filepath.SkipDir
				}
			}
			// Remove trust for default keys.
			if info.Size() == 0 {
				delete(trustedKeys, info.Name())
				return nil
			}
			entity, err := entityFromFile(path)
			if err != nil {
				return err
			}
			trustedKeys[fingerprintToFilename(entity.PrimaryKey.Fingerprint)] = entity
			return nil
		})
		if err != nil {
			return nil, err
		}
	}

	for _, v := range trustedKeys {
		keyring = append(keyring, v)
	}
	return keyring, nil
}

func fingerprintToFilename(fp [20]byte) string {
	return fmt.Sprintf("%x", fp)
}

// NewTestKeystore creates a new KeyStore backed by a temp directory.
// NewTestKeystore returns a KeyStore, the path to the temp directory, and
// an error if any.
func NewTestKeystore() (*Keystore, string, error) {
	dir, err := ioutil.TempDir("", "keystore-test")
	if err != nil {
		return nil, "", err
	}
	c := &Config{
		RootPath:         path.Join(dir, "/etc/rkt/trustedkeys/root.d"),
		SystemRootPath:   path.Join(dir, "/usr/lib/rkt/trustedkeys/root.d"),
		PrefixPath:       path.Join(dir, "/etc/rkt/trustedkeys/prefix.d"),
		SystemPrefixPath: path.Join(dir, "/usr/lib/rkt/trustedkeys/prefix.d"),
	}
	for _, path := range []string{c.RootPath, c.SystemRootPath, c.PrefixPath, c.SystemPrefixPath} {
		if err := os.MkdirAll(path, 0755); err != nil {
			return nil, "", err
		}
	}
	return New(c), dir, nil
}
<commit_msg>pkg/keystore: error for no sigs in *.sig<commit_after>// Copyright 2014 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package keystore implements the ACI keystore.
package keystore

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/coreos/rocket/Godeps/_workspace/src/github.com/appc/spec/schema/types"
	"github.com/coreos/rocket/Godeps/_workspace/src/golang.org/x/crypto/openpgp"
)

// A Config structure is used to configure a Keystore.
type Config struct {
	RootPath         string
	PrefixPath       string
	SystemRootPath   string
	SystemPrefixPath string
}

// A Keystore represents a repository of trusted public keys which can be
// used to verify PGP signatures.
type Keystore struct {
	*Config
}

// New returns a new Keystore based on config.
func New(config *Config) *Keystore {
	if config == nil {
		config = defaultConfig
	}
	return &Keystore{config}
}

var defaultConfig = &Config{
	RootPath:         "/etc/rkt/trustedkeys/root.d",
	PrefixPath:       "/etc/rkt/trustedkeys/prefix.d",
	SystemRootPath:   "/usr/lib/rkt/trustedkeys/root.d",
	SystemPrefixPath: "/usr/lib/rkt/trustedkeys/prefix.d",
}

// CheckSignature is a convenience method for creating a Keystore with a default
// configuration and invoking CheckSignature.
func CheckSignature(prefix string, signed, signature io.Reader) (*openpgp.Entity, error) {
	ks := New(defaultConfig)
	return checkSignature(ks, prefix, signed, signature)
}

// CheckSignature takes a signed file and a detached signature and returns the signer
// if the signature is signed by a trusted signer.
// If the signer is unknown or not trusted, opengpg.ErrUnknownIssuer is returned.
func (ks *Keystore) CheckSignature(prefix string, signed, signature io.Reader) (*openpgp.Entity, error) {
	return checkSignature(ks, prefix, signed, signature)
}

func checkSignature(ks *Keystore, prefix string, signed, signature io.Reader) (*openpgp.Entity, error) {
	acname, err := types.NewACName(prefix)
	if err != nil {
		return nil, err
	}
	keyring, err := ks.loadKeyring(acname.String())
	if err != nil {
		return nil, fmt.Errorf("keystore: error loading keyring %v", err)
	}
	entities, err := openpgp.CheckArmoredDetachedSignature(keyring, signed, signature)
	if err == io.EOF {
		// otherwise, the client failure is just "EOF", which is not helpful
		return nil, fmt.Errorf("keystore: no signatures found")
	}
	return entities, err
}

// DeleteTrustedKeyPrefix deletes the prefix trusted key identified by fingerprint.
func (ks *Keystore) DeleteTrustedKeyPrefix(prefix, fingerprint string) error {
	acname, err := types.NewACName(prefix)
	if err != nil {
		return err
	}
	return os.Remove(path.Join(ks.PrefixPath, acname.String(), fingerprint))
}

// MaskTrustedKeySystemPrefix masks the system prefix trusted key identified by fingerprint.
func (ks *Keystore) MaskTrustedKeySystemPrefix(prefix, fingerprint string) (string, error) {
	acname, err := types.NewACName(prefix)
	if err != nil {
		return "", err
	}
	dst := path.Join(ks.PrefixPath, acname.String(), fingerprint)
	return dst, ioutil.WriteFile(dst, []byte(""), 0644)
}

// DeleteTrustedKeyRoot deletes the root trusted key identified by fingerprint.
func (ks *Keystore) DeleteTrustedKeyRoot(fingerprint string) error {
	return os.Remove(path.Join(ks.RootPath, fingerprint))
}

// MaskTrustedKeySystemRoot masks the system root trusted key identified by fingerprint.
func (ks *Keystore) MaskTrustedKeySystemRoot(fingerprint string) (string, error) {
	dst := path.Join(ks.RootPath, fingerprint)
	return dst, ioutil.WriteFile(dst, []byte(""), 0644)
}

// StoreTrustedKeyPrefix stores the contents of public key r as a prefix trusted key.
func (ks *Keystore) StoreTrustedKeyPrefix(prefix string, r io.Reader) (string, error) {
	acname, err := types.NewACName(prefix)
	if err != nil {
		return "", err
	}
	return storeTrustedKey(path.Join(ks.PrefixPath, acname.String()), r)
}

// StoreTrustedKeyRoot stores the contents of public key r as a root trusted key.
func (ks *Keystore) StoreTrustedKeyRoot(r io.Reader) (string, error) {
	return storeTrustedKey(ks.RootPath, r)
}

func storeTrustedKey(dir string, r io.Reader) (string, error) {
	pubkeyBytes, err := ioutil.ReadAll(r)
	if err != nil {
		return "", err
	}
	if err := os.MkdirAll(dir, 0700); err != nil {
		return "", err
	}
	entityList, err := openpgp.ReadArmoredKeyRing(bytes.NewReader(pubkeyBytes))
	if err != nil {
		return "", err
	}
	pubKey := entityList[0].PrimaryKey
	trustedKeyPath := path.Join(dir, fingerprintToFilename(pubKey.Fingerprint))
	if err := ioutil.WriteFile(trustedKeyPath, pubkeyBytes, 0644); err != nil {
		return "", err
	}
	return trustedKeyPath, nil
}

func entityFromFile(path string) (*openpgp.Entity, error) {
	trustedKey, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer trustedKey.Close()
	entityList, err := openpgp.ReadArmoredKeyRing(trustedKey)
	if err != nil {
		return nil, err
	}
	if len(entityList) < 1 {
		return nil, errors.New("missing opengpg entity")
	}
	fingerprint := fingerprintToFilename(entityList[0].PrimaryKey.Fingerprint)
	if fingerprint != filepath.Base(trustedKey.Name()) {
		return nil, fmt.Errorf("fingerprint mismatch")
	}
	return entityList[0], nil
}

func (ks *Keystore) loadKeyring(prefix string) (openpgp.KeyRing, error) {
	acname, err := types.NewACName(prefix)
	if err != nil {
		return nil, err
	}
	var keyring openpgp.EntityList
	trustedKeys := make(map[string]*openpgp.Entity)

	prefixRoot := strings.Split(acname.String(), "/")[0]
	paths := []struct {
		root     string
		fullPath string
	}{
		{ks.SystemRootPath, ks.SystemRootPath},
		{ks.RootPath, ks.RootPath},
		{path.Join(ks.SystemPrefixPath, prefixRoot), path.Join(ks.SystemPrefixPath, acname.String())},
		{path.Join(ks.PrefixPath, prefixRoot), path.Join(ks.PrefixPath, acname.String())},
	}
	for _, p := range paths {
		err := filepath.Walk(p.root, func(path string, info os.FileInfo, err error) error {
			if err != nil && !os.IsNotExist(err) {
				return err
			}
			if info == nil {
				return nil
			}
			if info.IsDir() {
				switch {
				case strings.HasPrefix(p.fullPath, path):
					return nil
				default:
					return filepath.SkipDir
				}
			}
			// Remove trust for default keys.
			if info.Size() == 0 {
				delete(trustedKeys, info.Name())
				return nil
			}
			entity, err := entityFromFile(path)
			if err != nil {
				return err
			}
			trustedKeys[fingerprintToFilename(entity.PrimaryKey.Fingerprint)] = entity
			return nil
		})
		if err != nil {
			return nil, err
		}
	}

	for _, v := range trustedKeys {
		keyring = append(keyring, v)
	}
	return keyring, nil
}

func fingerprintToFilename(fp [20]byte) string {
	return fmt.Sprintf("%x", fp)
}

// NewTestKeystore creates a new KeyStore backed by a temp directory.
// NewTestKeystore returns a KeyStore, the path to the temp directory, and
// an error if any.
func NewTestKeystore() (*Keystore, string, error) {
	dir, err := ioutil.TempDir("", "keystore-test")
	if err != nil {
		return nil, "", err
	}
	c := &Config{
		RootPath:         path.Join(dir, "/etc/rkt/trustedkeys/root.d"),
		SystemRootPath:   path.Join(dir, "/usr/lib/rkt/trustedkeys/root.d"),
		PrefixPath:       path.Join(dir, "/etc/rkt/trustedkeys/prefix.d"),
		SystemPrefixPath: path.Join(dir, "/usr/lib/rkt/trustedkeys/prefix.d"),
	}
	for _, path := range []string{c.RootPath, c.SystemRootPath, c.PrefixPath, c.SystemPrefixPath} {
		if err := os.MkdirAll(path, 0755); err != nil {
			return nil, "", err
		}
	}
	return New(c), dir, nil
}
<|endoftext|>


<commit_before>// SPDX-License-Identifier: Apache-2.0
// Copyright Authors of Cilium

package logging

import (
	"fmt"
	"strings"

	"github.com/sirupsen/logrus"
	"go.uber.org/fx"
	"go.uber.org/fx/fxevent"
	"golang.org/x/exp/slices"
)

type FxLogger struct {
	logrus.FieldLogger

	sups  []*fxevent.Supplied
	ctors []*fxevent.Provided
}

func FxLoggerOption(log logrus.FieldLogger) fx.Option {
	return fx.WithLogger(func() fxevent.Logger { return NewFxLogger(log) })
}

func NewFxLogger(log logrus.FieldLogger) *FxLogger {
	return &FxLogger{FieldLogger: log}
}

func (log *FxLogger) PrintObjects() {
	slices.SortFunc(log.sups, func(a, b *fxevent.Supplied) bool {
		return a.ModuleName < b.ModuleName || (a.ModuleName == b.ModuleName && a.TypeName < b.TypeName)
	})

	fmt.Print("Supplied objects:\n\n")
	for _, sup := range log.sups {
		if sup.ModuleName != "" {
			fmt.Printf("  🎁 %s from %s\n", sup.TypeName, sup.ModuleName)
		} else {
			fmt.Printf("  🎁 %s\n", sup.TypeName)
		}
		fmt.Println()
	}

	slices.SortFunc(log.ctors, func(a, b *fxevent.Provided) bool {
		return a.ModuleName < b.ModuleName || (a.ModuleName == b.ModuleName && a.ConstructorName < b.ConstructorName)
	})
	fmt.Print("Constructors:\n\n")
	for _, ctor := range log.ctors {
		if ctor.ModuleName != "" {
			fmt.Printf("  🛠  %s (%s):\n", ctor.ModuleName, ctor.ConstructorName)
		} else {
			fmt.Printf("  🛠  %s:\n", ctor.ConstructorName)
		}
		for _, rtype := range ctor.OutputTypeNames {
			fmt.Printf("    • %s\n", rtype)
		}
		if ctor.Err != nil {
			fmt.Printf("  ❌%s error: %s\n", ctor.ConstructorName, strings.Replace(ctor.Err.Error(), ":", ":\n\t", -1))
		}
		fmt.Println()
	}
}

func (log *FxLogger) LogEvent(event fxevent.Event) {
	switch e := event.(type) {
	case *fxevent.OnStartExecuting:
		log.WithField("callee", e.FunctionName).
			WithField("caller", e.CallerName).
			Debug("OnStart hook executing")

	case *fxevent.OnStartExecuted:
		if e.Err != nil {
			log.WithField("callee", e.FunctionName).
				WithField("caller", e.CallerName).
				WithError(e.Err).
				Debug("OnStart hook failed")
		} else {
			log.WithField("callee", e.FunctionName).
				WithField("caller", e.CallerName).
				WithField("runtime", e.Runtime.String()).
				Debug("OnStart hook executed")
		}

	case *fxevent.OnStopExecuting:
		log.WithField("callee", e.FunctionName).
			WithField("caller", e.CallerName).
			Debug("OnStop hook executing")

	case *fxevent.OnStopExecuted:
		if e.Err != nil {
			log.WithField("callee", e.FunctionName).
				WithField("caller", e.CallerName).
				WithError(e.Err).
				Error("OnStop hook failed")
		} else {
			log.WithField("callee", e.FunctionName).
				WithField("caller", e.CallerName).
				WithField("runtime", e.Runtime.String()).
				Debug("OnStop hook executed")
		}

	case *fxevent.Supplied:
		l := log.WithField("type", e.TypeName)
		if len(e.ModuleName) != 0 {
			l = l.WithField("module", e.ModuleName)
		}
		if e.Err != nil {
			l = l.WithError(e.Err)
		}
		l.Debug("Supplied")
		log.sups = append(log.sups, e)

	case *fxevent.Provided:
		l := log.WithField("constructor", e.ConstructorName)
		if len(e.ModuleName) != 0 {
			l = l.WithField("module", e.ModuleName)
		}

		for _, rtype := range e.OutputTypeNames {
			l.WithField("type", rtype).Debug("Provided")
		}
		if e.Err != nil {
			l.WithError(e.Err).
				Error("Error encountered while applying options")
		}
		log.ctors = append(log.ctors, e)

	case *fxevent.Decorated:
		l := log.WithField("decorator", e.DecoratorName)
		if len(e.ModuleName) != 0 {
			l = l.WithField("module", e.ModuleName)
		}
		for _, rtype := range e.OutputTypeNames {
			l.WithField("type", rtype).Debug("decorated")
		}
		if e.Err != nil {
			l.WithError(e.Err).
				Error("Error encountered while applying options")
		}

	case *fxevent.Invoking:
		l := log.WithField("function", e.FunctionName)
		if len(e.ModuleName) != 0 {
			l = l.WithField("module", e.ModuleName)
		}
		l.Debug("Invoking")

	case *fxevent.Invoked:
		if e.Err != nil {
			l := log.WithError(e.Err)
			if len(e.ModuleName) != 0 {
				l = l.WithField("module", e.ModuleName)
			}
			l.WithField("stack", e.Trace).
				WithField("function", e.FunctionName).
				Debug("Invoke failed")
		}

	case *fxevent.Stopping:
		log.WithField("signal", strings.ToUpper(e.Signal.String())).
			Info("Stopping")

	case *fxevent.Stopped:
		if e.Err != nil {
			log.WithError(e.Err).Error("Stop failed")
		} else {
			log.Info("Stopped")
		}

	case *fxevent.RollingBack:
		log.WithError(e.StartErr).Error("Start failed, rolling back")

	case *fxevent.RolledBack:
		if e.Err != nil {
			log.WithError(e.Err).Error("Rollback failed")
		}

	case *fxevent.Started:
		if e.Err != nil {
			log.WithError(e.Err).Error("Start failed")
		} else {
			log.Info("Started")
		}

	case *fxevent.LoggerInitialized:
		if e.Err != nil {
			log.WithError(e.Err).Error("Custom logger initialization failed")
		} else {
			log.WithField("function", e.ConstructorName).
				Debug("Initialized custom fxevent.Logger")
		}
	}
}
<commit_msg>logging: Collapse constructors by module in PrintObjects()<commit_after>// SPDX-License-Identifier: Apache-2.0
// Copyright Authors of Cilium

package logging

import (
	"fmt"
	"strings"

	"github.com/sirupsen/logrus"
	"go.uber.org/fx"
	"go.uber.org/fx/fxevent"
	"golang.org/x/exp/slices"
)

type FxLogger struct {
	logrus.FieldLogger

	sups  []*fxevent.Supplied
	ctors []*fxevent.Provided
}

func FxLoggerOption(log logrus.FieldLogger) fx.Option {
	return fx.WithLogger(func() fxevent.Logger { return NewFxLogger(log) })
}

func NewFxLogger(log logrus.FieldLogger) *FxLogger {
	return &FxLogger{FieldLogger: log}
}

func (log *FxLogger) PrintObjects() {
	slices.SortFunc(log.sups, func(a, b *fxevent.Supplied) bool {
		return a.ModuleName < b.ModuleName || (a.ModuleName == b.ModuleName && a.TypeName < b.TypeName)
	})

	fmt.Print("Supplied objects:\n\n")
	for _, sup := range log.sups {
		if sup.ModuleName != "" {
			fmt.Printf("  🎁 %s from %s\n", sup.TypeName, sup.ModuleName)
		} else {
			fmt.Printf("  🎁 %s\n", sup.TypeName)
		}
		fmt.Println()
	}

	slices.SortFunc(log.ctors, func(a, b *fxevent.Provided) bool {
		return a.ModuleName < b.ModuleName || (a.ModuleName == b.ModuleName && a.ConstructorName < b.ConstructorName)
	})
	// Collapse constructors by ModuleName
	ctorsByModule := make(map[string][]*fxevent.Provided)
	for _, ctor := range log.ctors {
		ctorsByModule[ctor.ModuleName] = append(ctorsByModule[ctor.ModuleName], ctor)
	}

	fmt.Print("Constructors:\n\n")
	for _, ctor := range ctorsByModule[""] {
		fmt.Printf("  🛠  %s:\n", ctor.ConstructorName)
		for _, rtype := range ctor.OutputTypeNames {
			fmt.Printf("    • %s\n", rtype)
		}
		if ctor.Err != nil {
			fmt.Printf("  ❌%s error: %s\n", ctor.ConstructorName, strings.Replace(ctor.Err.Error(), ":", ":\n\t", -1))
		}
		fmt.Println()
	}
	for modName, ctors := range ctorsByModule {
		if modName == "" {
			continue
		}

		fmt.Printf("  🛠  %s:\n", modName)
		for _, ctor := range ctors {
			for _, rtype := range ctor.OutputTypeNames {
				fmt.Printf("    • %s\n", rtype)
			}
			if ctor.Err != nil {
				fmt.Printf("  ❌%s error: %s\n", ctor.ConstructorName, strings.Replace(ctor.Err.Error(), ":", ":\n\t", -1))
			}
		}
		fmt.Println()
	}
}

func (log *FxLogger) LogEvent(event fxevent.Event) {
	switch e := event.(type) {
	case *fxevent.OnStartExecuting:
		log.WithField("callee", e.FunctionName).
			WithField("caller", e.CallerName).
			Debug("OnStart hook executing")

	case *fxevent.OnStartExecuted:
		if e.Err != nil {
			log.WithField("callee", e.FunctionName).
				WithField("caller", e.CallerName).
				WithError(e.Err).
				Debug("OnStart hook failed")
		} else {
			log.WithField("callee", e.FunctionName).
				WithField("caller", e.CallerName).
				WithField("runtime", e.Runtime.String()).
				Debug("OnStart hook executed")
		}

	case *fxevent.OnStopExecuting:
		log.WithField("callee", e.FunctionName).
			WithField("caller", e.CallerName).
			Debug("OnStop hook executing")

	case *fxevent.OnStopExecuted:
		if e.Err != nil {
			log.WithField("callee", e.FunctionName).
				WithField("caller", e.CallerName).
				WithError(e.Err).
				Error("OnStop hook failed")
		} else {
			log.WithField("callee", e.FunctionName).
				WithField("caller", e.CallerName).
				WithField("runtime", e.Runtime.String()).
				Debug("OnStop hook executed")
		}

	case *fxevent.Supplied:
		l := log.WithField("type", e.TypeName)
		if len(e.ModuleName) != 0 {
			l = l.WithField("module", e.ModuleName)
		}
		if e.Err != nil {
			l = l.WithError(e.Err)
		}
		l.Debug("Supplied")
		log.sups = append(log.sups, e)

	case *fxevent.Provided:
		l := log.WithField("constructor", e.ConstructorName)
		if len(e.ModuleName) != 0 {
			l = l.WithField("module", e.ModuleName)
		}

		for _, rtype := range e.OutputTypeNames {
			l.WithField("type", rtype).Debug("Provided")
		}
		if e.Err != nil {
			l.WithError(e.Err).
				Error("Error encountered while applying options")
		}
		log.ctors = append(log.ctors, e)

	case *fxevent.Decorated:
		l := log.WithField("decorator", e.DecoratorName)
		if len(e.ModuleName) != 0 {
			l = l.WithField("module", e.ModuleName)
		}
		for _, rtype := range e.OutputTypeNames {
			l.WithField("type", rtype).Debug("decorated")
		}
		if e.Err != nil {
			l.WithError(e.Err).
				Error("Error encountered while applying options")
		}

	case *fxevent.Invoking:
		l := log.WithField("function", e.FunctionName)
		if len(e.ModuleName) != 0 {
			l = l.WithField("module", e.ModuleName)
		}
		l.Debug("Invoking")

	case *fxevent.Invoked:
		if e.Err != nil {
			l := log.WithError(e.Err)
			if len(e.ModuleName) != 0 {
				l = l.WithField("module", e.ModuleName)
			}
			l.WithField("stack", e.Trace).
				WithField("function", e.FunctionName).
				Debug("Invoke failed")
		}

	case *fxevent.Stopping:
		log.WithField("signal", strings.ToUpper(e.Signal.String())).
			Info("Stopping")

	case *fxevent.Stopped:
		if e.Err != nil {
			log.WithError(e.Err).Error("Stop failed")
		} else {
			log.Info("Stopped")
		}

	case *fxevent.RollingBack:
		log.WithError(e.StartErr).Error("Start failed, rolling back")

	case *fxevent.RolledBack:
		if e.Err != nil {
			log.WithError(e.Err).Error("Rollback failed")
		}

	case *fxevent.Started:
		if e.Err != nil {
			log.WithError(e.Err).Error("Start failed")
		} else {
			log.Info("Started")
		}

	case *fxevent.LoggerInitialized:
		if e.Err != nil {
			log.WithError(e.Err).Error("Custom logger initialization failed")
		} else {
			log.WithField("function", e.ConstructorName).
				Debug("Initialized custom fxevent.Logger")
		}
	}
}
<|endoftext|>


<commit_before>package paxos

import (
    "fmt"
    "strings"
    "strconv"

    "borg/assert"
    "testing"
)

const (
    iSender = iota
    iCmd
    iRnd
    iNumParts
)

func accept(quorum int, ins, outs chan string) {
    var rnd, vrnd uint64
    var vval string

    ch, sent := make(chan int), 0
    for in := range ins {
        parts := strings.Split(in, ":", 3)
        if len(parts) != iNumParts {
            continue
        }
        i, _ := strconv.Btoui64(parts[iRnd], 10)
        // If parts[iRnd] is invalid, i is 0 and the message will be ignored
        switch {
            case i <= rnd:
            case i > rnd:
                rnd = i

                sent++
                msg := fmt.Sprintf("ACCEPT:%d:%d:%s", i, vrnd, vval)
                go func(msg string) { outs <- msg ; ch <- 1 }(msg)
        }
    }

    for x := 0; x < sent; x++ {
        <-ch
    }

    close(outs)
}

func TestAcceptsInvite(t *testing.T) {
    ins := make(chan string)
    outs := make(chan string)

    exp := "ACCEPT:1:0:"

    go accept(2, ins, outs)
    // Send a message with no senderId
    ins <- "1:INVITE:1"
    close(ins)

    got := ""
    for x := range outs {
        got += x
    }

    // outs was closed; therefore all messages have been processed
    assert.Equal(t, exp, got, "")
}

func TestIgnoresStaleInvites(t *testing.T) {
    ins := make(chan string)
    outs := make(chan string)

    exp := "ACCEPT:2:0:"

    go accept(2, ins, outs)
    // Send a message with no senderId
    ins <- "1:INVITE:2"
    ins <- "1:INVITE:1"
    close(ins)

    got := ""
    for x := range outs {
        got += x
    }

    // outs was closed; therefore all messages have been processed
    assert.Equal(t, exp, got, "")
}

func TestIgnoresMalformedMessageWithTooFewSeparators(t *testing.T) {
    totest := []string{
        "x",
        "x:x",
        "x:x:x:x",
    }
    for _, msg := range(totest) {
        ins := make(chan string)
        outs := make(chan string)

        exp := ""

        go accept(2, ins, outs)
        // Send a message with no senderId
        ins <- msg
        close(ins)

        got := ""
        for x := range outs {
            got += x
        }

        // outs was closed; therefore all messages have been processed
        assert.Equal(t, exp, got, "")
    }
}

func TestIgnoresMalformedMessageWithInvalidRound(t *testing.T) {
    ins := make(chan string)
    outs := make(chan string)

    exp := ""

    go accept(2, ins, outs)
    // Send a message with no senderId
    ins <- "1:INVITE:x"
    close(ins)

    got := ""
    for x := range outs {
        got += x
    }

    // outs was closed; therefore all messages have been processed
    assert.Equal(t, exp, got, "")
}

<commit_msg>ignore unknown commands<commit_after>package paxos

import (
    "fmt"
    "strings"
    "strconv"

    "borg/assert"
    "testing"
)

const (
    iSender = iota
    iCmd
    iRnd
    iNumParts
)

func accept(quorum int, ins, outs chan string) {
    var rnd, vrnd uint64
    var vval string

    ch, sent := make(chan int), 0
    for in := range ins {
        parts := strings.Split(in, ":", 3)
        if len(parts) != iNumParts {
            continue
        }
        switch parts[iCmd] {
        case "INVITE":
            i, _ := strconv.Btoui64(parts[iRnd], 10)
            // If parts[iRnd] is invalid, i is 0 and the message will be ignored
            switch {
                case i <= rnd:
                case i > rnd:
                    rnd = i

                    sent++
                    msg := fmt.Sprintf("ACCEPT:%d:%d:%s", i, vrnd, vval)
                    go func(msg string) { outs <- msg ; ch <- 1 }(msg)
            }
        }
    }

    for x := 0; x < sent; x++ {
        <-ch
    }

    close(outs)
}

func TestAcceptsInvite(t *testing.T) {
    ins := make(chan string)
    outs := make(chan string)

    exp := "ACCEPT:1:0:"

    go accept(2, ins, outs)
    // Send a message with no senderId
    ins <- "1:INVITE:1"
    close(ins)

    got := ""
    for x := range outs {
        got += x
    }

    // outs was closed; therefore all messages have been processed
    assert.Equal(t, exp, got, "")
}

func TestIgnoresStaleInvites(t *testing.T) {
    ins := make(chan string)
    outs := make(chan string)

    exp := "ACCEPT:2:0:"

    go accept(2, ins, outs)
    // Send a message with no senderId
    ins <- "1:INVITE:2"
    ins <- "1:INVITE:1"
    close(ins)

    got := ""
    for x := range outs {
        got += x
    }

    // outs was closed; therefore all messages have been processed
    assert.Equal(t, exp, got, "")
}

func TestIgnoresMalformedMessageWithTooFewSeparators(t *testing.T) {
    totest := []string{
        "x",
        "x:x",
        "x:x:x:x",
    }
    for _, msg := range(totest) {
        ins := make(chan string)
        outs := make(chan string)

        exp := ""

        go accept(2, ins, outs)
        // Send a message with no senderId
        ins <- msg
        close(ins)

        got := ""
        for x := range outs {
            got += x
        }

        // outs was closed; therefore all messages have been processed
        assert.Equal(t, exp, got, "")
    }
}

func TestIgnoresMalformedMessageWithInvalidRound(t *testing.T) {
    ins := make(chan string)
    outs := make(chan string)

    exp := ""

    go accept(2, ins, outs)
    // Send a message with no senderId
    ins <- "1:INVITE:x"
    close(ins)

    got := ""
    for x := range outs {
        got += x
    }

    // outs was closed; therefore all messages have been processed
    assert.Equal(t, exp, got, "")
}

func TestIgnoresUnknownCommands(t *testing.T) {
    ins := make(chan string)
    outs := make(chan string)

    exp := ""

    go accept(2, ins, outs)
    // Send a message with no senderId
    ins <- "1:x:1"
    close(ins)

    got := ""
    for x := range outs {
        got += x
    }

    // outs was closed; therefore all messages have been processed
    assert.Equal(t, exp, got, "")
}

<|endoftext|>


<commit_before>package store // import "a4.io/blobstash/pkg/stash/store"

import (
	"context"

	"a4.io/blobsfile"
	"a4.io/blobstash/pkg/blob"
	"a4.io/blobstash/pkg/vkv"
)

type KvStore interface {
	Put(ctx context.Context, key, ref string, data []byte, version int)
	Get(ctx context.Context, key string, version int) (*vkv.KeyValue, error)
	Versions(ctx context.Context, key string, start, limit int) (*vkv.KeyValueVersions, int, error)
	Keys(ctx context.Context, start, end string, limit int) ([]*vkv.KeyValue, string, error)
	ReverseKeys(start, end string, limit int) ([]*vkv.KeyValue, string, error)
}

type BlobStore interface {
	Put(ctx context.Context, blob *blob.Blob) error
	Get(ctx context.Context, hash string) ([]byte, error)
	Stat(ctx context.Context, hash string) (bool, error)
	Enumerate(ctx context.Context, start, end string, limit int) ([]*blob.SizedBlobRef, error)
	Scan(ctx context.Context) error
}

type BlobStoreProxy struct {
	BlobStore
	ReadSrc BlobStore
}

func (p *BlobStoreProxy) Get(ctx context.Context, hash string) ([]byte, error) {
	data, err := p.BlobStore.Get(ctx, hash)
	switch err {
	case nil:
	case blobsfile.ErrBlobNotFound:
		return p.readSrc.Get(ctx, hash)
	default:
		return nil, err
	}
}

func (p *BlobStoreProxy) Stat(ctx context.Context, hash string) (bool, error) {
	exists, err := p.BlobStore.Stat(ctx, hash)
	if err != nil {
		return false, err
	}
	if !exists {
		return p.readSrc.Stat(ctx, hash)
	}
	return exists, nil
}

func (p *BlobStoreProxy) Put(ctx context.Context, blob *blob.Blob) error {
	existsSrc, err := p.readSrc.Stat(ctx, blob.Hash)
	if err != nil {
		return err
	}
	if existsSrc {
		return nil
	}
	return p.BlobStore.Put(ctx, blob)
}
<commit_msg>stash/store: bugfix<commit_after>package store // import "a4.io/blobstash/pkg/stash/store"

import (
	"context"

	"a4.io/blobsfile"
	"a4.io/blobstash/pkg/blob"
	"a4.io/blobstash/pkg/vkv"
)

type KvStore interface {
	Put(ctx context.Context, key, ref string, data []byte, version int)
	Get(ctx context.Context, key string, version int) (*vkv.KeyValue, error)
	Versions(ctx context.Context, key string, start, limit int) (*vkv.KeyValueVersions, int, error)
	Keys(ctx context.Context, start, end string, limit int) ([]*vkv.KeyValue, string, error)
	ReverseKeys(start, end string, limit int) ([]*vkv.KeyValue, string, error)
}

type BlobStore interface {
	Put(ctx context.Context, blob *blob.Blob) error
	Get(ctx context.Context, hash string) ([]byte, error)
	Stat(ctx context.Context, hash string) (bool, error)
	Enumerate(ctx context.Context, start, end string, limit int) ([]*blob.SizedBlobRef, error)
	Scan(ctx context.Context) error
}

type BlobStoreProxy struct {
	BlobStore
	ReadSrc BlobStore
}

func (p *BlobStoreProxy) Get(ctx context.Context, hash string) ([]byte, error) {
	data, err := p.BlobStore.Get(ctx, hash)
	switch err {
	case nil:
	case blobsfile.ErrBlobNotFound:
		return p.ReadSrc.Get(ctx, hash)
	default:
		return nil, err
	}
	return data, nil
}

func (p *BlobStoreProxy) Stat(ctx context.Context, hash string) (bool, error) {
	exists, err := p.BlobStore.Stat(ctx, hash)
	if err != nil {
		return false, err
	}
	if !exists {
		return p.ReadSrc.Stat(ctx, hash)
	}
	return exists, nil
}

func (p *BlobStoreProxy) Put(ctx context.Context, blob *blob.Blob) error {
	existsSrc, err := p.ReadSrc.Stat(ctx, blob.Hash)
	if err != nil {
		return err
	}
	if existsSrc {
		return nil
	}
	return p.BlobStore.Put(ctx, blob)
}
<|endoftext|>


<commit_before>package template

import (
	"fmt"
	"regexp"
	"strings"

	"k8s.io/kubernetes/pkg/api/meta"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/util/fielderrors"

	"github.com/openshift/origin/pkg/template/api"
	. "github.com/openshift/origin/pkg/template/generator"
	"github.com/openshift/origin/pkg/util"
	"github.com/openshift/origin/pkg/util/stringreplace"
)

var parameterExp = regexp.MustCompile(`\$\{([a-zA-Z0-9\_]+)\}`)

// Processor process the Template into the List with substituted parameters
type Processor struct {
	Generators map[string]Generator
}

// NewProcessor creates new Processor and initializes its set of generators.
func NewProcessor(generators map[string]Generator) *Processor {
	return &Processor{Generators: generators}
}

// Process transforms Template object into List object. It generates
// Parameter values using the defined set of generators first, and then it
// substitutes all Parameter expression occurrences with their corresponding
// values (currently in the containers' Environment variables only).
func (p *Processor) Process(template *api.Template) fielderrors.ValidationErrorList {
	templateErrors := fielderrors.ValidationErrorList{}

	if err := p.GenerateParameterValues(template); err != nil {
		return append(templateErrors.Prefix("Template"), fielderrors.NewFieldInvalid("parameters", err, "failure to generate parameter value"))
	}

	for i, item := range template.Objects {
		if obj, ok := item.(*runtime.Unknown); ok {
			// TODO: use runtime.DecodeList when it returns ValidationErrorList
			obj, err := runtime.UnstructuredJSONScheme.Decode(obj.RawJSON)
			if err != nil {
				util.ReportError(&templateErrors, i, *fielderrors.NewFieldInvalid("objects", err, "unable to handle object"))
				continue
			}
			item = obj
		}

		newItem, err := p.SubstituteParameters(template.Parameters, item)
		if err != nil {
			util.ReportError(&templateErrors, i, *fielderrors.NewFieldInvalid("parameters", template.Parameters, err.Error()))
		}
		stripNamespace(newItem)
		if err := util.AddObjectLabels(newItem, template.ObjectLabels); err != nil {
			util.ReportError(&templateErrors, i, *fielderrors.NewFieldInvalid("labels", err, "label could not be applied"))
		}
		template.Objects[i] = newItem
	}

	return templateErrors
}

func stripNamespace(obj runtime.Object) {
	// Remove namespace from the item
	if itemMeta, err := meta.Accessor(obj); err == nil {
		itemMeta.SetNamespace("")
		return
	}
	// TODO: allow meta.Accessor to handle runtime.Unstructured
	if unstruct, ok := obj.(*runtime.Unstructured); ok && unstruct.Object != nil {
		if obj, ok := unstruct.Object["metadata"]; ok {
			if m, ok := obj.(map[string]interface{}); ok {
				if _, ok := m["namespace"]; ok {
					m["namespace"] = ""
				}
			}
			return
		}
		if _, ok := unstruct.Object["namespace"]; ok {
			unstruct.Object["namespace"] = ""
			return
		}
	}
}

// AddParameter adds new custom parameter to the Template. It overrides
// the existing parameter, if already defined.
func AddParameter(t *api.Template, param api.Parameter) {
	if existing := GetParameterByName(t, param.Name); existing != nil {
		*existing = param
	} else {
		t.Parameters = append(t.Parameters, param)
	}
}

// GetParameterByName searches for a Parameter in the Template
// based on its name.
func GetParameterByName(t *api.Template, name string) *api.Parameter {
	for i, param := range t.Parameters {
		if param.Name == name {
			return &(t.Parameters[i])
		}
	}
	return nil
}

// SubstituteParameters loops over all values defined in structured
// and unstructured types that are children of item.
//
// Example of Parameter expression:
//   - ${PARAMETER_NAME}
//
func (p *Processor) SubstituteParameters(params []api.Parameter, item runtime.Object) (runtime.Object, error) {
	// Make searching for given parameter name/value more effective
	paramMap := make(map[string]string, len(params))
	for _, param := range params {
		paramMap[param.Name] = param.Value
	}

	stringreplace.VisitObjectStrings(item, func(in string) string {
		for _, match := range parameterExp.FindAllStringSubmatch(in, -1) {
			if len(match) > 1 {
				if paramValue, found := paramMap[match[1]]; found {
					in = strings.Replace(in, match[0], paramValue, 1)
				}
			}
		}
		return in
	})

	return item, nil
}

// GenerateParameterValues generates Value for each Parameter of the given
// Template that has Generate field specified where Value is not already
// supplied.
//
// Examples:
//
// from             | value
// -----------------------------
// "test[0-9]{1}x"  | "test7x"
// "[0-1]{8}"       | "01001100"
// "0x[A-F0-9]{4}"  | "0xB3AF"
// "[a-zA-Z0-9]{8}" | "hW4yQU5i"
func (p *Processor) GenerateParameterValues(t *api.Template) error {
	for i := range t.Parameters {
		param := &t.Parameters[i]
		if len(param.Value) > 0 {
			continue
		}
		if param.Generate != "" {
			generator, ok := p.Generators[param.Generate]
			if !ok {
				return fmt.Errorf("template.parameters[%v]: Unable to find the '%v' generator", i, param.Generate)
			}
			if generator == nil {
				return fmt.Errorf("template.parameters[%v]: Invalid '%v' generator", i, param.Generate)
			}
			value, err := generator.GenerateValue(param.From)
			if err != nil {
				return err
			}
			param.Value, ok = value.(string)
			if !ok {
				return fmt.Errorf("template.parameters[%v]: Unable to convert the generated value '%#v' to string", i, value)
			}
		}
		if len(param.Value) == 0 && param.Required {
			err := fielderrors.NewFieldRequired(fmt.Sprintf("parameters[%d].value", i))
			err.Detail = fmt.Sprintf("parameter %s is required and must be specified", param.Name)
			return err
		}
	}
	return nil
}
<commit_msg>document why namespaces are stripped<commit_after>package template

import (
	"fmt"
	"regexp"
	"strings"

	"k8s.io/kubernetes/pkg/api/meta"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/util/fielderrors"

	"github.com/openshift/origin/pkg/template/api"
	. "github.com/openshift/origin/pkg/template/generator"
	"github.com/openshift/origin/pkg/util"
	"github.com/openshift/origin/pkg/util/stringreplace"
)

var parameterExp = regexp.MustCompile(`\$\{([a-zA-Z0-9\_]+)\}`)

// Processor process the Template into the List with substituted parameters
type Processor struct {
	Generators map[string]Generator
}

// NewProcessor creates new Processor and initializes its set of generators.
func NewProcessor(generators map[string]Generator) *Processor {
	return &Processor{Generators: generators}
}

// Process transforms Template object into List object. It generates
// Parameter values using the defined set of generators first, and then it
// substitutes all Parameter expression occurrences with their corresponding
// values (currently in the containers' Environment variables only).
func (p *Processor) Process(template *api.Template) fielderrors.ValidationErrorList {
	templateErrors := fielderrors.ValidationErrorList{}

	if err := p.GenerateParameterValues(template); err != nil {
		return append(templateErrors.Prefix("Template"), fielderrors.NewFieldInvalid("parameters", err, "failure to generate parameter value"))
	}

	for i, item := range template.Objects {
		if obj, ok := item.(*runtime.Unknown); ok {
			// TODO: use runtime.DecodeList when it returns ValidationErrorList
			obj, err := runtime.UnstructuredJSONScheme.Decode(obj.RawJSON)
			if err != nil {
				util.ReportError(&templateErrors, i, *fielderrors.NewFieldInvalid("objects", err, "unable to handle object"))
				continue
			}
			item = obj
		}

		newItem, err := p.SubstituteParameters(template.Parameters, item)
		if err != nil {
			util.ReportError(&templateErrors, i, *fielderrors.NewFieldInvalid("parameters", template.Parameters, err.Error()))
		}
		// If an object definition's metadata includes a namespace field, the field will be stripped out of
		// the definition during template instantiation.  This is necessary because all objects created during
		// instantiation are placed into the target namespace, so it would be invalid for the object to declare
		//a different namespace.
		stripNamespace(newItem)
		if err := util.AddObjectLabels(newItem, template.ObjectLabels); err != nil {
			util.ReportError(&templateErrors, i, *fielderrors.NewFieldInvalid("labels", err, "label could not be applied"))
		}
		template.Objects[i] = newItem
	}

	return templateErrors
}

func stripNamespace(obj runtime.Object) {
	// Remove namespace from the item
	if itemMeta, err := meta.Accessor(obj); err == nil {
		itemMeta.SetNamespace("")
		return
	}
	// TODO: allow meta.Accessor to handle runtime.Unstructured
	if unstruct, ok := obj.(*runtime.Unstructured); ok && unstruct.Object != nil {
		if obj, ok := unstruct.Object["metadata"]; ok {
			if m, ok := obj.(map[string]interface{}); ok {
				if _, ok := m["namespace"]; ok {
					m["namespace"] = ""
				}
			}
			return
		}
		if _, ok := unstruct.Object["namespace"]; ok {
			unstruct.Object["namespace"] = ""
			return
		}
	}
}

// AddParameter adds new custom parameter to the Template. It overrides
// the existing parameter, if already defined.
func AddParameter(t *api.Template, param api.Parameter) {
	if existing := GetParameterByName(t, param.Name); existing != nil {
		*existing = param
	} else {
		t.Parameters = append(t.Parameters, param)
	}
}

// GetParameterByName searches for a Parameter in the Template
// based on its name.
func GetParameterByName(t *api.Template, name string) *api.Parameter {
	for i, param := range t.Parameters {
		if param.Name == name {
			return &(t.Parameters[i])
		}
	}
	return nil
}

// SubstituteParameters loops over all values defined in structured
// and unstructured types that are children of item.
//
// Example of Parameter expression:
//   - ${PARAMETER_NAME}
//
func (p *Processor) SubstituteParameters(params []api.Parameter, item runtime.Object) (runtime.Object, error) {
	// Make searching for given parameter name/value more effective
	paramMap := make(map[string]string, len(params))
	for _, param := range params {
		paramMap[param.Name] = param.Value
	}

	stringreplace.VisitObjectStrings(item, func(in string) string {
		for _, match := range parameterExp.FindAllStringSubmatch(in, -1) {
			if len(match) > 1 {
				if paramValue, found := paramMap[match[1]]; found {
					in = strings.Replace(in, match[0], paramValue, 1)
				}
			}
		}
		return in
	})

	return item, nil
}

// GenerateParameterValues generates Value for each Parameter of the given
// Template that has Generate field specified where Value is not already
// supplied.
//
// Examples:
//
// from             | value
// -----------------------------
// "test[0-9]{1}x"  | "test7x"
// "[0-1]{8}"       | "01001100"
// "0x[A-F0-9]{4}"  | "0xB3AF"
// "[a-zA-Z0-9]{8}" | "hW4yQU5i"
func (p *Processor) GenerateParameterValues(t *api.Template) error {
	for i := range t.Parameters {
		param := &t.Parameters[i]
		if len(param.Value) > 0 {
			continue
		}
		if param.Generate != "" {
			generator, ok := p.Generators[param.Generate]
			if !ok {
				return fmt.Errorf("template.parameters[%v]: Unable to find the '%v' generator", i, param.Generate)
			}
			if generator == nil {
				return fmt.Errorf("template.parameters[%v]: Invalid '%v' generator", i, param.Generate)
			}
			value, err := generator.GenerateValue(param.From)
			if err != nil {
				return err
			}
			param.Value, ok = value.(string)
			if !ok {
				return fmt.Errorf("template.parameters[%v]: Unable to convert the generated value '%#v' to string", i, value)
			}
		}
		if len(param.Value) == 0 && param.Required {
			err := fielderrors.NewFieldRequired(fmt.Sprintf("parameters[%d].value", i))
			err.Detail = fmt.Sprintf("parameter %s is required and must be specified", param.Name)
			return err
		}
	}
	return nil
}
<|endoftext|>