<commit_before>package main

import (
	"context"
	"crypto/md5"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"html/template"
	"io"
	"net/http"
	"path/filepath"
	"runtime"
	"strings"

	"cloud.google.com/go/storage"
	"github.com/gorilla/mux"
	"go.skia.org/infra/go/allowed"
	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/httputils"
	"go.skia.org/infra/go/login"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/util"
	"google.golang.org/api/option"
)

const (
	// BUCKET is the Cloud Storage bucket we store files in.
	BUCKET          = "skottie-renderer"
	BUCKET_INTERNAL = "skottie-renderer-internal"
)

// flags
var (
	local        = flag.Bool("local", false, "Running locally if true. As opposed to in production.")
	lockedDown   = flag.Bool("locked_down", false, "Restricted to only @google.com accounts.")
	port         = flag.String("port", ":8000", "HTTP service address (e.g., ':8000')")
	promPort     = flag.String("prom_port", ":20000", "Metrics service address (e.g., ':10110')")
	resourcesDir = flag.String("resources_dir", "", "The directory to find templates, JS, and CSS files. If blank the current directory will be used.")
	skottieTool  = flag.String("skottie_tool", "", "[deprecated/unused]Absolute path to the skottie_tool executable.")
	versionFile  = flag.String("version_file", "[deprecated/unused]/etc/skia-prod/VERSION", "The full path of the Skia VERSION file.")
)

var (
	invalidRequestErr = errors.New("")
)

// Server is the state of the server.
type Server struct {
	bucket    *storage.BucketHandle
	templates *template.Template
}

func New() (*Server, error) {
	if *resourcesDir == "" {
		_, filename, _, _ := runtime.Caller(0)
		*resourcesDir = filepath.Join(filepath.Dir(filename), "../../dist")
	}

	ts, err := auth.NewDefaultTokenSource(*local, storage.ScopeFullControl)
	if err != nil {
		return nil, fmt.Errorf("Failed to get token source: %s", err)
	}
	client := httputils.DefaultClientConfig().WithTokenSource(ts).With2xxOnly().Client()
	storageClient, err := storage.NewClient(context.Background(), option.WithHTTPClient(client))
	if err != nil {
		return nil, fmt.Errorf("Problem creating storage client: %s", err)
	}

	if *lockedDown {
		allow := allowed.NewAllowedFromList([]string{"google.com"})
		login.InitWithAllow(*port, *local, nil, nil, allow)
	}

	bucket := BUCKET
	if *lockedDown {
		bucket = BUCKET_INTERNAL
	}

	srv := &Server{
		bucket: storageClient.Bucket(bucket),
	}
	srv.loadTemplates()
	return srv, nil
}

func (srv *Server) loadTemplates() {
	srv.templates = template.Must(template.New("").Delims("{%", "%}").ParseFiles(
		filepath.Join(*resourcesDir, "index.html"),
		filepath.Join(*resourcesDir, "embed.html"),
	))
}

func (srv *Server) mainHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/html")
	if *local {
		srv.loadTemplates()
	}
	if err := srv.templates.ExecuteTemplate(w, "index.html", nil); err != nil {
		sklog.Errorf("Failed to expand template: %s", err)
	}
}

func (srv *Server) embedHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/html")
	if *local {
		srv.loadTemplates()
	}
	if err := srv.templates.ExecuteTemplate(w, "embed.html", nil); err != nil {
		sklog.Errorf("Failed to expand template: %s", err)
	}
}

func (srv *Server) jsonHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json")
	w.Header().Set("Access-Control-Allow-Origin", "*")
	hash := mux.Vars(r)["hash"]
	path := strings.Join([]string{hash, "lottie.json"}, "/")
	reader, err := srv.bucket.Object(path).NewReader(r.Context())
	if err != nil {
		httputils.ReportError(w, r, err, "Can't load file from GCS")
		return
	}
	if _, err = io.Copy(w, reader); err != nil {
		httputils.ReportError(w, r, err, "Failed to write JSON file.")
		return
	}
}

type UploadRequest struct {
	Lottie   interface{} `json:"lottie"`
	Width    int         `json:"width"`
	Height   int         `json:"height"`
	FPS      float32     `json:"fps"`
	Filename string      `json:"filename"`
}

type UploadResponse struct {
	Hash string `json:"hash"`
}

func (req *UploadRequest) validate(w http.ResponseWriter) error {
	if req.FPS < 1 || req.FPS > 120 {
		http.Error(w, "FPS must be between 1 and 120.", http.StatusBadRequest)
		return invalidRequestErr
	}
	if req.Width < 1 || req.Width > 2048 {
		http.Error(w, "Width must be between 1 and 2048.", http.StatusBadRequest)
		return invalidRequestErr
	}
	if req.Height < 1 || req.Height > 2048 {
		http.Error(w, "Height must be between 1 and 2048.", http.StatusBadRequest)
		return invalidRequestErr
	}
	return nil
}

func (srv *Server) uploadHandler(w http.ResponseWriter, r *http.Request) {
	ctx := r.Context()
	// Extract json file.
	defer util.Close(r.Body)
	var req UploadRequest
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
		httputils.ReportError(w, r, err, "Error decoding JSON.")
		return
	}
	if err := req.validate(w); err != nil {
		return
	}

	b, err := json.Marshal(req.Lottie)
	if err != nil {
		httputils.ReportError(w, r, err, "Can't re-encode lottie file.")
		return
	}

	// Calculate md5 of file.
	// TODO(jcgregorio) include options in md5 calculation once they're added to the UI.
	h := md5.New()
	b, err = json.Marshal(req)
	if err != nil {
		httputils.ReportError(w, r, err, "Can't re-encode request.")
		return
	}
	if _, err = h.Write(b); err != nil {
		httputils.ReportError(w, r, err, "Failed calculating hash.")
		return
	}
	hash := fmt.Sprintf("%x", h.Sum(nil))

	// Write JSON file.
	path := strings.Join([]string{hash, "lottie.json"}, "/")
	obj := srv.bucket.Object(path)
	wr := obj.NewWriter(ctx)
	wr.ObjectAttrs.ContentEncoding = "application/json"
	if _, err := wr.Write(b); err != nil {
		httputils.ReportError(w, r, err, "Failed writing JSON to GCS.")
		return
	}
	if err := wr.Close(); err != nil {
		httputils.ReportError(w, r, err, "Failed writing JSON to GCS on close.")
		return
	}
	if !*lockedDown {
		if err := obj.ACL().Set(ctx, storage.AllUsers, storage.RoleReader); err != nil {
			sklog.Errorf("Failed to make JSON public: %s", err)
		}
	}

	resp := UploadResponse{
		Hash: hash,
	}
	w.Header().Set("Content-Type", "application/json")
	if err := json.NewEncoder(w).Encode(resp); err != nil {
		sklog.Errorf("Failed to write response: %s", err)
	}
}

func main() {
	common.InitWithMust(
		"skottie",
		common.PrometheusOpt(promPort),
		common.MetricsLoggingOpt(),
	)

	if *lockedDown && *local {
		sklog.Fatalf("Can't be run as both --locked_down and --local.")
	}

	srv, err := New()
	if err != nil {
		sklog.Fatalf("Failed to start: %s", err)
	}

	r := mux.NewRouter()
	r.HandleFunc("/{hash:[0-9A-Za-z]*}", srv.mainHandler)
	r.HandleFunc("/e/{hash:[0-9A-Za-z]*}", srv.embedHandler)

	r.HandleFunc("/_/j/{hash:[0-9A-Za-z]+}", srv.jsonHandler)
	r.HandleFunc("/_/upload", srv.uploadHandler)

	r.PathPrefix("/static/").Handler(http.StripPrefix("/static/", http.HandlerFunc(httputils.CorsHandler(httputils.MakeResourceHandler(*resourcesDir)))))

	// TODO(jcgregorio) Implement CSRF.
	h := httputils.LoggingGzipRequestResponse(r)
	if !*local {
		if *lockedDown {
			h = login.RestrictViewer(h)
			h = login.ForceAuth(h, login.DEFAULT_REDIRECT_URL)
		}
		h = httputils.HealthzAndHTTPS(h)
	}

	http.Handle("/", h)
	sklog.Infoln("Ready to serve.")
	sklog.Fatal(http.ListenAndServe(*port, nil))
}
<commit_msg>Set mime-type for wasm files.<commit_after>package main

import (
	"context"
	"crypto/md5"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"html/template"
	"io"
	"mime"
	"net/http"
	"path/filepath"
	"runtime"
	"strings"

	"cloud.google.com/go/storage"
	"github.com/gorilla/mux"
	"go.skia.org/infra/go/allowed"
	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/httputils"
	"go.skia.org/infra/go/login"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/util"
	"google.golang.org/api/option"
)

const (
	// BUCKET is the Cloud Storage bucket we store files in.
	BUCKET          = "skottie-renderer"
	BUCKET_INTERNAL = "skottie-renderer-internal"
)

// flags
var (
	local        = flag.Bool("local", false, "Running locally if true. As opposed to in production.")
	lockedDown   = flag.Bool("locked_down", false, "Restricted to only @google.com accounts.")
	port         = flag.String("port", ":8000", "HTTP service address (e.g., ':8000')")
	promPort     = flag.String("prom_port", ":20000", "Metrics service address (e.g., ':10110')")
	resourcesDir = flag.String("resources_dir", "", "The directory to find templates, JS, and CSS files. If blank the current directory will be used.")
	skottieTool  = flag.String("skottie_tool", "", "[deprecated/unused]Absolute path to the skottie_tool executable.")
	versionFile  = flag.String("version_file", "[deprecated/unused]/etc/skia-prod/VERSION", "The full path of the Skia VERSION file.")
)

var (
	invalidRequestErr = errors.New("")
)

// Server is the state of the server.
type Server struct {
	bucket    *storage.BucketHandle
	templates *template.Template
}

func New() (*Server, error) {
	if *resourcesDir == "" {
		_, filename, _, _ := runtime.Caller(0)
		*resourcesDir = filepath.Join(filepath.Dir(filename), "../../dist")
	}

	// Need to set the mime-type for wasm files so streaming compile works.
	if err := mime.AddExtensionType(".wasm", "application/wasm"); err != nil {
		sklog.Fatal(err)
	}

	ts, err := auth.NewDefaultTokenSource(*local, storage.ScopeFullControl)
	if err != nil {
		return nil, fmt.Errorf("Failed to get token source: %s", err)
	}
	client := httputils.DefaultClientConfig().WithTokenSource(ts).With2xxOnly().Client()
	storageClient, err := storage.NewClient(context.Background(), option.WithHTTPClient(client))
	if err != nil {
		return nil, fmt.Errorf("Problem creating storage client: %s", err)
	}

	if *lockedDown {
		allow := allowed.NewAllowedFromList([]string{"google.com"})
		login.InitWithAllow(*port, *local, nil, nil, allow)
	}

	bucket := BUCKET
	if *lockedDown {
		bucket = BUCKET_INTERNAL
	}

	srv := &Server{
		bucket: storageClient.Bucket(bucket),
	}
	srv.loadTemplates()
	return srv, nil
}

func (srv *Server) loadTemplates() {
	srv.templates = template.Must(template.New("").Delims("{%", "%}").ParseFiles(
		filepath.Join(*resourcesDir, "index.html"),
		filepath.Join(*resourcesDir, "embed.html"),
	))
}

func (srv *Server) mainHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/html")
	if *local {
		srv.loadTemplates()
	}
	if err := srv.templates.ExecuteTemplate(w, "index.html", nil); err != nil {
		sklog.Errorf("Failed to expand template: %s", err)
	}
}

func (srv *Server) embedHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/html")
	if *local {
		srv.loadTemplates()
	}
	if err := srv.templates.ExecuteTemplate(w, "embed.html", nil); err != nil {
		sklog.Errorf("Failed to expand template: %s", err)
	}
}

func (srv *Server) jsonHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json")
	w.Header().Set("Access-Control-Allow-Origin", "*")
	hash := mux.Vars(r)["hash"]
	path := strings.Join([]string{hash, "lottie.json"}, "/")
	reader, err := srv.bucket.Object(path).NewReader(r.Context())
	if err != nil {
		httputils.ReportError(w, r, err, "Can't load file from GCS")
		return
	}
	if _, err = io.Copy(w, reader); err != nil {
		httputils.ReportError(w, r, err, "Failed to write JSON file.")
		return
	}
}

type UploadRequest struct {
	Lottie   interface{} `json:"lottie"`
	Width    int         `json:"width"`
	Height   int         `json:"height"`
	FPS      float32     `json:"fps"`
	Filename string      `json:"filename"`
}

type UploadResponse struct {
	Hash string `json:"hash"`
}

func (req *UploadRequest) validate(w http.ResponseWriter) error {
	if req.FPS < 1 || req.FPS > 120 {
		http.Error(w, "FPS must be between 1 and 120.", http.StatusBadRequest)
		return invalidRequestErr
	}
	if req.Width < 1 || req.Width > 2048 {
		http.Error(w, "Width must be between 1 and 2048.", http.StatusBadRequest)
		return invalidRequestErr
	}
	if req.Height < 1 || req.Height > 2048 {
		http.Error(w, "Height must be between 1 and 2048.", http.StatusBadRequest)
		return invalidRequestErr
	}
	return nil
}

func (srv *Server) uploadHandler(w http.ResponseWriter, r *http.Request) {
	ctx := r.Context()
	// Extract json file.
	defer util.Close(r.Body)
	var req UploadRequest
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
		httputils.ReportError(w, r, err, "Error decoding JSON.")
		return
	}
	if err := req.validate(w); err != nil {
		return
	}

	b, err := json.Marshal(req.Lottie)
	if err != nil {
		httputils.ReportError(w, r, err, "Can't re-encode lottie file.")
		return
	}

	// Calculate md5 of file.
	// TODO(jcgregorio) include options in md5 calculation once they're added to the UI.
	h := md5.New()
	b, err = json.Marshal(req)
	if err != nil {
		httputils.ReportError(w, r, err, "Can't re-encode request.")
		return
	}
	if _, err = h.Write(b); err != nil {
		httputils.ReportError(w, r, err, "Failed calculating hash.")
		return
	}
	hash := fmt.Sprintf("%x", h.Sum(nil))

	// Write JSON file.
	path := strings.Join([]string{hash, "lottie.json"}, "/")
	obj := srv.bucket.Object(path)
	wr := obj.NewWriter(ctx)
	wr.ObjectAttrs.ContentEncoding = "application/json"
	if _, err := wr.Write(b); err != nil {
		httputils.ReportError(w, r, err, "Failed writing JSON to GCS.")
		return
	}
	if err := wr.Close(); err != nil {
		httputils.ReportError(w, r, err, "Failed writing JSON to GCS on close.")
		return
	}
	if !*lockedDown {
		if err := obj.ACL().Set(ctx, storage.AllUsers, storage.RoleReader); err != nil {
			sklog.Errorf("Failed to make JSON public: %s", err)
		}
	}

	resp := UploadResponse{
		Hash: hash,
	}
	w.Header().Set("Content-Type", "application/json")
	if err := json.NewEncoder(w).Encode(resp); err != nil {
		sklog.Errorf("Failed to write response: %s", err)
	}
}

func main() {
	common.InitWithMust(
		"skottie",
		common.PrometheusOpt(promPort),
		common.MetricsLoggingOpt(),
	)

	if *lockedDown && *local {
		sklog.Fatalf("Can't be run as both --locked_down and --local.")
	}

	srv, err := New()
	if err != nil {
		sklog.Fatalf("Failed to start: %s", err)
	}

	r := mux.NewRouter()
	r.HandleFunc("/{hash:[0-9A-Za-z]*}", srv.mainHandler)
	r.HandleFunc("/e/{hash:[0-9A-Za-z]*}", srv.embedHandler)

	r.HandleFunc("/_/j/{hash:[0-9A-Za-z]+}", srv.jsonHandler)
	r.HandleFunc("/_/upload", srv.uploadHandler)

	r.PathPrefix("/static/").Handler(http.StripPrefix("/static/", http.HandlerFunc(httputils.CorsHandler(httputils.MakeResourceHandler(*resourcesDir)))))

	// TODO(jcgregorio) Implement CSRF.
	h := httputils.LoggingGzipRequestResponse(r)
	if !*local {
		if *lockedDown {
			h = login.RestrictViewer(h)
			h = login.ForceAuth(h, login.DEFAULT_REDIRECT_URL)
		}
		h = httputils.HealthzAndHTTPS(h)
	}

	http.Handle("/", h)
	sklog.Infoln("Ready to serve.")
	sklog.Fatal(http.ListenAndServe(*port, nil))
}
<|endoftext|>


<commit_before>package otto

import (
	"fmt"
)

type _clone struct {
	runtime      *_runtime
	_object      map[*_object]*_object
	_objectStash map[*_objectStash]*_objectStash
	_dclStash    map[*_dclStash]*_dclStash
	_fnStash     map[*_fnStash]*_fnStash
}

func (in *_runtime) clone() *_runtime {

	in.lck.Lock()
	defer in.lck.Unlock()

	out := &_runtime{}
	clone := _clone{
		runtime:      out,
		_object:      make(map[*_object]*_object),
		_objectStash: make(map[*_objectStash]*_objectStash),
		_dclStash:    make(map[*_dclStash]*_dclStash),
		_fnStash:     make(map[*_fnStash]*_fnStash),
	}

	globalObject := clone.object(in.globalObject)
	out.globalStash = out.newObjectStash(globalObject, nil)
	out.globalObject = globalObject
	out.global = _global{
		clone.object(in.global.Object),
		clone.object(in.global.Function),
		clone.object(in.global.Array),
		clone.object(in.global.String),
		clone.object(in.global.Boolean),
		clone.object(in.global.Number),
		clone.object(in.global.Math),
		clone.object(in.global.Date),
		clone.object(in.global.RegExp),
		clone.object(in.global.Error),
		clone.object(in.global.EvalError),
		clone.object(in.global.TypeError),
		clone.object(in.global.RangeError),
		clone.object(in.global.ReferenceError),
		clone.object(in.global.SyntaxError),
		clone.object(in.global.URIError),
		clone.object(in.global.JSON),

		clone.object(in.global.ObjectPrototype),
		clone.object(in.global.FunctionPrototype),
		clone.object(in.global.ArrayPrototype),
		clone.object(in.global.StringPrototype),
		clone.object(in.global.BooleanPrototype),
		clone.object(in.global.NumberPrototype),
		clone.object(in.global.DatePrototype),
		clone.object(in.global.RegExpPrototype),
		clone.object(in.global.ErrorPrototype),
		clone.object(in.global.EvalErrorPrototype),
		clone.object(in.global.TypeErrorPrototype),
		clone.object(in.global.RangeErrorPrototype),
		clone.object(in.global.ReferenceErrorPrototype),
		clone.object(in.global.SyntaxErrorPrototype),
		clone.object(in.global.URIErrorPrototype),
	}

	out.eval = out.globalObject.property["eval"].value.(Value).value.(*_object)
	out.globalObject.prototype = out.global.ObjectPrototype

	// Not sure if this is necessary, but give some help to the GC
	clone.runtime = nil
	clone._object = nil
	clone._objectStash = nil
	clone._dclStash = nil
	clone._fnStash = nil

	return out
}

func (clone *_clone) object(in *_object) *_object {
	if out, exists := clone._object[in]; exists {
		return out
	}
	out := &_object{}
	clone._object[in] = out
	return in.objectClass.clone(in, out, clone)
}

func (clone *_clone) dclStash(in *_dclStash) (*_dclStash, bool) {
	if out, exists := clone._dclStash[in]; exists {
		return out, true
	}
	out := &_dclStash{}
	clone._dclStash[in] = out
	return out, false
}

func (clone *_clone) objectStash(in *_objectStash) (*_objectStash, bool) {
	if out, exists := clone._objectStash[in]; exists {
		return out, true
	}
	out := &_objectStash{}
	clone._objectStash[in] = out
	return out, false
}

func (clone *_clone) fnStash(in *_fnStash) (*_fnStash, bool) {
	if out, exists := clone._fnStash[in]; exists {
		return out, true
	}
	out := &_fnStash{}
	clone._fnStash[in] = out
	return out, false
}

func (clone *_clone) value(in Value) Value {
	out := in
	switch value := in.value.(type) {
	case *_object:
		out.value = clone.object(value)
	}
	return out
}

func (clone *_clone) valueArray(in []Value) []Value {
	out := make([]Value, len(in))
	for index, value := range in {
		out[index] = clone.value(value)
	}
	return out
}

func (clone *_clone) stash(in _stash) _stash {
	if in == nil {
		return nil
	}
	return in.clone(clone)
}

func (clone *_clone) property(in _property) _property {
	out := in

	switch v := in.value.(type) {
	case _propertyGetSet:
		out.value = _propertyGetSet{v[0], v[1]}
	case Value:
		out.value = clone.value(v)
	default:
		panic(fmt.Errorf("in.value.(Value) != true"))
	}
	return out
}

func (clone *_clone) dclProperty(in _dclProperty) _dclProperty {
	out := in
	out.value = clone.value(in.value)
	return out
}
<commit_msg>fix cloning getter/setter props<commit_after>package otto

import (
	"fmt"
)

type _clone struct {
	runtime      *_runtime
	_object      map[*_object]*_object
	_objectStash map[*_objectStash]*_objectStash
	_dclStash    map[*_dclStash]*_dclStash
	_fnStash     map[*_fnStash]*_fnStash
}

func (in *_runtime) clone() *_runtime {

	in.lck.Lock()
	defer in.lck.Unlock()

	out := &_runtime{}
	clone := _clone{
		runtime:      out,
		_object:      make(map[*_object]*_object),
		_objectStash: make(map[*_objectStash]*_objectStash),
		_dclStash:    make(map[*_dclStash]*_dclStash),
		_fnStash:     make(map[*_fnStash]*_fnStash),
	}

	globalObject := clone.object(in.globalObject)
	out.globalStash = out.newObjectStash(globalObject, nil)
	out.globalObject = globalObject
	out.global = _global{
		clone.object(in.global.Object),
		clone.object(in.global.Function),
		clone.object(in.global.Array),
		clone.object(in.global.String),
		clone.object(in.global.Boolean),
		clone.object(in.global.Number),
		clone.object(in.global.Math),
		clone.object(in.global.Date),
		clone.object(in.global.RegExp),
		clone.object(in.global.Error),
		clone.object(in.global.EvalError),
		clone.object(in.global.TypeError),
		clone.object(in.global.RangeError),
		clone.object(in.global.ReferenceError),
		clone.object(in.global.SyntaxError),
		clone.object(in.global.URIError),
		clone.object(in.global.JSON),

		clone.object(in.global.ObjectPrototype),
		clone.object(in.global.FunctionPrototype),
		clone.object(in.global.ArrayPrototype),
		clone.object(in.global.StringPrototype),
		clone.object(in.global.BooleanPrototype),
		clone.object(in.global.NumberPrototype),
		clone.object(in.global.DatePrototype),
		clone.object(in.global.RegExpPrototype),
		clone.object(in.global.ErrorPrototype),
		clone.object(in.global.EvalErrorPrototype),
		clone.object(in.global.TypeErrorPrototype),
		clone.object(in.global.RangeErrorPrototype),
		clone.object(in.global.ReferenceErrorPrototype),
		clone.object(in.global.SyntaxErrorPrototype),
		clone.object(in.global.URIErrorPrototype),
	}

	out.eval = out.globalObject.property["eval"].value.(Value).value.(*_object)
	out.globalObject.prototype = out.global.ObjectPrototype

	// Not sure if this is necessary, but give some help to the GC
	clone.runtime = nil
	clone._object = nil
	clone._objectStash = nil
	clone._dclStash = nil
	clone._fnStash = nil

	return out
}

func (clone *_clone) object(in *_object) *_object {
	if out, exists := clone._object[in]; exists {
		return out
	}
	out := &_object{}
	clone._object[in] = out
	return in.objectClass.clone(in, out, clone)
}

func (clone *_clone) dclStash(in *_dclStash) (*_dclStash, bool) {
	if out, exists := clone._dclStash[in]; exists {
		return out, true
	}
	out := &_dclStash{}
	clone._dclStash[in] = out
	return out, false
}

func (clone *_clone) objectStash(in *_objectStash) (*_objectStash, bool) {
	if out, exists := clone._objectStash[in]; exists {
		return out, true
	}
	out := &_objectStash{}
	clone._objectStash[in] = out
	return out, false
}

func (clone *_clone) fnStash(in *_fnStash) (*_fnStash, bool) {
	if out, exists := clone._fnStash[in]; exists {
		return out, true
	}
	out := &_fnStash{}
	clone._fnStash[in] = out
	return out, false
}

func (clone *_clone) value(in Value) Value {
	out := in
	switch value := in.value.(type) {
	case *_object:
		out.value = clone.object(value)
	}
	return out
}

func (clone *_clone) valueArray(in []Value) []Value {
	out := make([]Value, len(in))
	for index, value := range in {
		out[index] = clone.value(value)
	}
	return out
}

func (clone *_clone) stash(in _stash) _stash {
	if in == nil {
		return nil
	}
	return in.clone(clone)
}

func (clone *_clone) property(in _property) _property {
	out := in

	switch value := in.value.(type) {
	case Value:
		out.value = clone.value(value)
	case _propertyGetSet:
		p := _propertyGetSet{}
		if value[0] != nil {
			p[0] = clone.object(value[0])
		}
		if value[1] != nil {
			p[1] = clone.object(value[1])
		}
		out.value = p
	default:
		panic(fmt.Errorf("in.value.(Value) != true; in.value is %T", in.value))
	}

	return out
}

func (clone *_clone) dclProperty(in _dclProperty) _dclProperty {
	out := in
	out.value = clone.value(in.value)
	return out
}
<|endoftext|>


<commit_before>package clone

import (
	"crypto/md5"
	"encoding/hex"
	"errors"
	"github.com/jeffail/tunny"
	csv "github.com/whosonfirst/go-whosonfirst-csv"
	log "github.com/whosonfirst/go-whosonfirst-log"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"sync/atomic"
)

type WOFClone struct {
	Source    string
	Dest      string
	Count     int64
	Success   int64
	Error     int64
	Skipped   int64
	Scheduled int64
	Completed int64
	Logger    *log.WOFLogger
	client    *http.Client
	pool      *tunny.WorkPool
}

func NewWOFClone(source string, dest string, procs int, logger *log.WOFLogger) *WOFClone {

	// cd := &sync.Cond{L: &sync.Mutex{}}

	cl := &http.Client{}

	runtime.GOMAXPROCS(procs)

	pool, _ := tunny.CreatePoolGeneric(procs).Open()

	c := WOFClone{
		Count:   0,
		Success: 0,
		Error:   0,
		Skipped: 0,
		Source:  source,
		Dest:    dest,
		Logger:  logger,
		client:  cl,
		pool:    pool,
	}

	return &c
}

func (c *WOFClone) CloneMetaFile(file string) error {

	abs_path, _ := filepath.Abs(file)
	// c.Logger.Debug("Parse meta file %s", abs_path)

	reader, read_err := csv.NewDictReader(abs_path)

	if read_err != nil {
		c.Logger.Error("Failed to read %s, because %v", abs_path, read_err)
		return read_err
	}

	wg := new(sync.WaitGroup)

	for {
		row, err := reader.Read()

		if err == io.EOF {
			break
		}

		if err != nil {
			return err
		}

		rel_path, ok := row["path"]

		if !ok {
			continue
		}

		// this does not account for counts - need to think about who calls what, when
		// probably needs to moved in to ClonePath... (20151105/thisisaaronland)

		ensure_changes := true
		skip_existing := false

		remote := c.Source + rel_path
		local := path.Join(c.Dest, rel_path)

		if !os.IsNotExist(err) {

			if skip_existing {
				c.Logger.Debug("%s already exists and we are skipping things that exist", local)
				continue
			}

			change, _ := c.HasChanged(local, remote)

			if !change {
				c.Logger.Info("no changes to %s", local)
				continue
			}
		}

		wg.Add(1)
		atomic.AddInt64(&c.Scheduled, 1)

		go func() {

			defer wg.Done()

			_, err = c.pool.SendWork(func() {

				cl_err := c.ClonePath(rel_path, ensure_changes)

				if cl_err != nil {
					atomic.AddInt64(&c.Error, 1)
				} else {
					atomic.AddInt64(&c.Success, 1)
				}

				atomic.AddInt64(&c.Completed, 1)
				c.Status()
			})

		}()
	}

	wg.Wait()
	return nil
}

func (c *WOFClone) ClonePath(rel_path string, ensure_changes bool) error {

	atomic.AddInt64(&c.Count, 1)

	remote := c.Source + rel_path
	local := path.Join(c.Dest, rel_path)

	_, err := os.Stat(local)

	if !os.IsNotExist(err) && ensure_changes {

		change, _ := c.HasChanged(local, remote)

		if !change {

			c.Logger.Debug("%s has not changed so skipping", local)
			atomic.AddInt64(&c.Skipped, 1)
			return nil
		}

	}

	process_err := c.Process(remote, local)

	if process_err != nil {
		atomic.AddInt64(&c.Error, 1)
		return process_err
	}

	return nil
}

// don't return true if there's a problem - move that logic up above

func (c *WOFClone) HasChanged(local string, remote string) (bool, error) {

	change := true

	body, err := ioutil.ReadFile(local)

	if err != nil {
		c.Logger.Error("Failed to read %s, becase %v", local, err)
		return change, err
	}

	hash := md5.Sum(body)
	local_hash := hex.EncodeToString(hash[:])

	return c.HasHashChanged(local_hash, remote)
}

func (c *WOFClone) HasHashChanged(local_hash string, remote string) (bool, error) {

	change := true

	rsp, err := c.Fetch("HEAD", remote)

	if err != nil {
		return change, err
	}

	defer rsp.Body.Close()

	etag := rsp.Header.Get("Etag")
	remote_hash := strings.Replace(etag, "\"", "", -1)

	if local_hash == remote_hash {
		change = false
	}

	return change, nil
}

func (c *WOFClone) Process(remote string, local string) error {

	c.Logger.Debug("fetch %s and store in %s", remote, local)

	local_root := path.Dir(local)

	_, err := os.Stat(local_root)

	if os.IsNotExist(err) {
		c.Logger.Info("create %s", local_root)
		os.MkdirAll(local_root, 0755)
	}

	rsp, fetch_err := c.Fetch("GET", remote)

	if fetch_err != nil {
		return fetch_err
	}

	defer rsp.Body.Close()

	contents, read_err := ioutil.ReadAll(rsp.Body)

	if read_err != nil {
		c.Logger.Error("failed to read body for %s, because %v", remote, read_err)
		return read_err
	}

	go func() error {

		write_err := ioutil.WriteFile(local, contents, 0644)

		if write_err != nil {
			c.Logger.Error("Failed to write %s, because %v", local, write_err)

			atomic.AddInt64(&c.Success, -1)
			atomic.AddInt64(&c.Error, 1)

			return write_err
		}

		c.Logger.Debug("Wrote %s to disk", local)
		return nil
	}()

	return nil
}

func (c *WOFClone) Fetch(method string, url string) (*http.Response, error) {

	c.Logger.Debug("%s %s", method, url)

	req, _ := http.NewRequest(method, url, nil)
	req.Close = true

	rsp, err := c.client.Do(req)

	if err != nil {
		c.Logger.Error("Failed to %s %s, because %v", method, url, err)
		// golog.Fatal(err)
		return nil, err
	}

	// See also: https://github.com/whosonfirst/go-whosonfirst-clone/issues/6

	expected := 200

	if rsp.StatusCode != expected {
		c.Logger.Error("Failed to %s %s, because we expected %d from S3 and got '%s' instead", method, url, expected, rsp.Status)
		return nil, errors.New(rsp.Status)
	}

	return rsp, err
}

func (c *WOFClone) Status() {
	c.Logger.Info("scheduled: %d completed: %d success: %d error: %d goroutines: %d", c.Scheduled, c.Completed, c.Success, c.Error, runtime.NumGoroutine())
}
<commit_msg>return nil because pendatic<commit_after>package clone

import (
	"crypto/md5"
	"encoding/hex"
	"errors"
	"github.com/jeffail/tunny"
	csv "github.com/whosonfirst/go-whosonfirst-csv"
	log "github.com/whosonfirst/go-whosonfirst-log"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"sync/atomic"
)

type WOFClone struct {
	Source    string
	Dest      string
	Count     int64
	Success   int64
	Error     int64
	Skipped   int64
	Scheduled int64
	Completed int64
	Logger    *log.WOFLogger
	client    *http.Client
	pool      *tunny.WorkPool
}

func NewWOFClone(source string, dest string, procs int, logger *log.WOFLogger) *WOFClone {

	// cd := &sync.Cond{L: &sync.Mutex{}}

	cl := &http.Client{}

	runtime.GOMAXPROCS(procs)

	pool, _ := tunny.CreatePoolGeneric(procs).Open()

	c := WOFClone{
		Count:   0,
		Success: 0,
		Error:   0,
		Skipped: 0,
		Source:  source,
		Dest:    dest,
		Logger:  logger,
		client:  cl,
		pool:    pool,
	}

	return &c
}

func (c *WOFClone) CloneMetaFile(file string) error {

	abs_path, _ := filepath.Abs(file)
	// c.Logger.Debug("Parse meta file %s", abs_path)

	reader, read_err := csv.NewDictReader(abs_path)

	if read_err != nil {
		c.Logger.Error("Failed to read %s, because %v", abs_path, read_err)
		return read_err
	}

	wg := new(sync.WaitGroup)

	for {
		row, err := reader.Read()

		if err == io.EOF {
			break
		}

		if err != nil {
			return err
		}

		rel_path, ok := row["path"]

		if !ok {
			continue
		}

		// this does not account for counts - need to think about who calls what, when
		// probably needs to moved in to ClonePath... (20151105/thisisaaronland)

		ensure_changes := true
		skip_existing := false

		remote := c.Source + rel_path
		local := path.Join(c.Dest, rel_path)

		if !os.IsNotExist(err) {

			if skip_existing {
				c.Logger.Debug("%s already exists and we are skipping things that exist", local)
				continue
			}

			change, _ := c.HasChanged(local, remote)

			if !change {
				c.Logger.Info("no changes to %s", local)
				continue
			}
		}

		wg.Add(1)
		atomic.AddInt64(&c.Scheduled, 1)

		go func() {

			defer wg.Done()

			_, err = c.pool.SendWork(func() {

				cl_err := c.ClonePath(rel_path, ensure_changes)

				if cl_err != nil {
					atomic.AddInt64(&c.Error, 1)
				} else {
					atomic.AddInt64(&c.Success, 1)
				}

				atomic.AddInt64(&c.Completed, 1)
				c.Status()
			})

		}()
	}

	wg.Wait()
	return nil
}

func (c *WOFClone) ClonePath(rel_path string, ensure_changes bool) error {

	atomic.AddInt64(&c.Count, 1)

	remote := c.Source + rel_path
	local := path.Join(c.Dest, rel_path)

	_, err := os.Stat(local)

	if !os.IsNotExist(err) && ensure_changes {

		change, _ := c.HasChanged(local, remote)

		if !change {

			c.Logger.Debug("%s has not changed so skipping", local)
			atomic.AddInt64(&c.Skipped, 1)
			return nil
		}

	}

	process_err := c.Process(remote, local)

	if process_err != nil {
		atomic.AddInt64(&c.Error, 1)
		return process_err
	}

	return nil
}

// don't return true if there's a problem - move that logic up above

func (c *WOFClone) HasChanged(local string, remote string) (bool, error) {

	change := true

	body, err := ioutil.ReadFile(local)

	if err != nil {
		c.Logger.Error("Failed to read %s, becase %v", local, err)
		return change, err
	}

	hash := md5.Sum(body)
	local_hash := hex.EncodeToString(hash[:])

	return c.HasHashChanged(local_hash, remote)
}

func (c *WOFClone) HasHashChanged(local_hash string, remote string) (bool, error) {

	change := true

	rsp, err := c.Fetch("HEAD", remote)

	if err != nil {
		return change, err
	}

	defer rsp.Body.Close()

	etag := rsp.Header.Get("Etag")
	remote_hash := strings.Replace(etag, "\"", "", -1)

	if local_hash == remote_hash {
		change = false
	}

	return change, nil
}

func (c *WOFClone) Process(remote string, local string) error {

	c.Logger.Debug("fetch %s and store in %s", remote, local)

	local_root := path.Dir(local)

	_, err := os.Stat(local_root)

	if os.IsNotExist(err) {
		c.Logger.Info("create %s", local_root)
		os.MkdirAll(local_root, 0755)
	}

	rsp, fetch_err := c.Fetch("GET", remote)

	if fetch_err != nil {
		return fetch_err
	}

	defer rsp.Body.Close()

	contents, read_err := ioutil.ReadAll(rsp.Body)

	if read_err != nil {
		c.Logger.Error("failed to read body for %s, because %v", remote, read_err)
		return read_err
	}

	go func() error {

		write_err := ioutil.WriteFile(local, contents, 0644)

		if write_err != nil {
			c.Logger.Error("Failed to write %s, because %v", local, write_err)

			atomic.AddInt64(&c.Success, -1)
			atomic.AddInt64(&c.Error, 1)

			return write_err
		}

		c.Logger.Debug("Wrote %s to disk", local)
		return nil
	}()

	return nil
}

func (c *WOFClone) Fetch(method string, url string) (*http.Response, error) {

	c.Logger.Debug("%s %s", method, url)

	req, _ := http.NewRequest(method, url, nil)
	req.Close = true

	rsp, err := c.client.Do(req)

	if err != nil {
		c.Logger.Error("Failed to %s %s, because %v", method, url, err)
		// golog.Fatal(err)
		return nil, err
	}

	// See also: https://github.com/whosonfirst/go-whosonfirst-clone/issues/6

	expected := 200

	if rsp.StatusCode != expected {
		c.Logger.Error("Failed to %s %s, because we expected %d from S3 and got '%s' instead", method, url, expected, rsp.Status)
		return nil, errors.New(rsp.Status)
	}

	return rsp, nil
}

func (c *WOFClone) Status() {
	c.Logger.Info("scheduled: %d completed: %d success: %d error: %d goroutines: %d", c.Scheduled, c.Completed, c.Success, c.Error, runtime.NumGoroutine())
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"

	"github.com/codegangsta/cli"
	"github.com/docker/docker/daemon/graphdriver"
	"github.com/docker/docker/graph"
)

func initDriver(c *cli.Context) graphdriver.Driver {
	graphdriver.DefaultDriver = c.GlobalString("driver")
	homedir := c.GlobalString("home")
	drv, err := graphdriver.New(homedir, []string{})
	if err != nil {
		fmt.Printf("Failed to instantiate graphdriver: %s\n", err)
		os.Exit(1)
	}
	fmt.Printf("[DEBUG] Using driver %s.\n%g\nHome directory: %s\n", drv.String(), drv.Status(), homedir)
	return drv
}

func initGraph(c *cli.Context) *graph.Graph {
	drv := initDriver(c)
	homedir := c.GlobalString("home") + "/graph/"
	g, err := graph.NewGraph(homedir, drv)
	if err != nil {
		fmt.Printf("Failed to instantiate graph: %s\n", err)
		os.Exit(1)
	}
	return g
}

var commands []cli.Command

func main() {
	graphc := cli.NewApp()
	graphc.Name = "graphc"
	graphc.Usage = "manage graphc storage"
	graphc.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "home",
			Value:  "/var/lib/docker/",
			Usage:  "home directory for graphdriver storage operations",
			EnvVar: "GRAPHDRIVER_HOME",
		},
		cli.StringFlag{
			Name:   "driver, s",
			Value:  "",
			Usage:  "storage backend to use",
			EnvVar: "GRAPHDRIVER_BACKEND",
		},
		cli.StringFlag{
			Name:  "context, c",
			Value: "",
			Usage: "optional mountlabel (SELinux context)",
		},
	}
	graphc.EnableBashCompletion = true
	graphc.Commands = commands

	graphc.Run(os.Args)
}
<commit_msg>Add --storage-opt, mimicking docker's CLI flag<commit_after>package main

import (
	"fmt"
	"os"

	"github.com/codegangsta/cli"
	"github.com/docker/docker/daemon/graphdriver"
	"github.com/docker/docker/graph"
)

func initDriver(c *cli.Context) graphdriver.Driver {
	graphdriver.DefaultDriver = c.GlobalString("driver")
	homedir := c.GlobalString("home")
	drv, err := graphdriver.New(homedir, c.GlobalStringSlice("storage-opt"))
	if err != nil {
		fmt.Printf("Failed to instantiate graphdriver: %s\n", err)
		os.Exit(1)
	}
	fmt.Printf("[DEBUG] Using driver %s.\n%g\nHome directory: %s\n", drv.String(), drv.Status(), homedir)
	return drv
}

func initGraph(c *cli.Context) *graph.Graph {
	drv := initDriver(c)
	homedir := c.GlobalString("home") + "/graph/"
	g, err := graph.NewGraph(homedir, drv)
	if err != nil {
		fmt.Printf("Failed to instantiate graph: %s\n", err)
		os.Exit(1)
	}
	return g
}

var commands []cli.Command

func main() {
	graphc := cli.NewApp()
	graphc.Name = "graphc"
	graphc.Usage = "manage graphc storage"
	graphc.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "home",
			Value:  "/var/lib/docker/",
			Usage:  "home directory for graphdriver storage operations",
			EnvVar: "GRAPHDRIVER_HOME",
		},
		cli.StringFlag{
			Name:   "driver, s",
			Value:  "",
			Usage:  "storage driver to use",
			EnvVar: "GRAPHDRIVER_BACKEND",
		},
		cli.StringSliceFlag{
			Name:   "storage-opt",
			Value:  &cli.StringSlice{},
			Usage:  "set storage driver options",
			EnvVar: "GRAPHDRIVER_OPTIONS",
		},
		cli.StringFlag{
			Name:  "context, c",
			Value: "",
			Usage: "optional mountlabel (SELinux context)",
		},
	}
	graphc.EnableBashCompletion = true
	graphc.Commands = commands

	graphc.Run(os.Args)
}
<|endoftext|>


<commit_before>package dockerclient

import (
	"bytes"
	. "github.com/smartystreets/goconvey/convey"
	"testing"
)

func TestParseBuildResponse(t *testing.T) {
	Convey("Parse build response", t, func() {
		Convey("Parse old response", func() {
			r := bytes.NewBufferString(outputPre0_7)
			dh := &DockerHost{}
			streams := BuildResponse{}
			res, e := dh.handleBuildImageOld(r, func(s *Stream) {
				streams = append(streams, s)
			})
			So(e, ShouldBeNil)
			So(len(streams), ShouldEqual, 9)
			So(len(res), ShouldEqual, 9)
			So(streams[0].Stream, ShouldEqual, "Step 1 : FROM ubuntu\n")
			So(streams.ImageId(), ShouldEqual, "b30eb4fbfc51")
		})
		Convey("Parse json response", func() {
			r := bytes.NewBufferString(newResponse)
			dh := &DockerHost{}
			streams := BuildResponse{}
			res, e := dh.handleBuildImageJson(r, func(s *Stream) {
				streams = append(streams, s)
			})
			So(e, ShouldBeNil)
			So(len(streams), ShouldEqual, 9)
			So(len(res), ShouldEqual, 9)
			So(streams[0].Stream, ShouldEqual, "Step 1 : FROM ubuntu\n")
			So(streams.ImageId(), ShouldEqual, "0f101a4836f6")
		})
	})
}

const outputPre0_7 = `Step 1 : FROM ubuntu
---> 8dbd9e392a96
Step 2 : RUN apt-get update
---> Using cache
---> f7ada547a49b
Step 3 : RUN apt-get upgrade -y
---> Using cache
---> b30eb4fbfc51
Successfully built b30eb4fbfc51
`

const newResponse = `{"stream":"Step 1 : FROM ubuntu\n"}
{"stream":" ---\u003e 8dbd9e392a96\n"}
{"stream":"Step 2 : RUN apt-get update\n"}
{"stream":" ---\u003e Using cache\n"}
{"stream":" ---\u003e 30d9e1cb9bb8\n"}
{"stream":"Step 3 : RUN apt-get upgrade -y\n"}
{"stream":" ---\u003e Using cache\n"}
{"stream":" ---\u003e 0f101a4836f6\n"}
{"stream":"Successfully built 0f101a4836f6\n"}
`
<commit_msg>fixup tests<commit_after>package dockerclient

import (
	"bytes"
	. "github.com/smartystreets/goconvey/convey"
	"testing"
)

func TestParseBuildResponse(t *testing.T) {
	Convey("Parse build response", t, func() {
		Convey("Parse old response", func() {
			r := bytes.NewBufferString(outputPre0_7)
			dh := &DockerHost{}
			streams := BuildResponse{}
			res, e := dh.handleBuildImagePlain(r, func(s *Stream) {
				streams = append(streams, s)
			})
			So(e, ShouldBeNil)
			So(len(streams), ShouldEqual, 9)
			So(len(res), ShouldEqual, 9)
			So(streams[0].Stream, ShouldEqual, "Step 1 : FROM ubuntu\n")
			So(streams.ImageId(), ShouldEqual, "b30eb4fbfc51")
		})
		Convey("Parse json response", func() {
			r := bytes.NewBufferString(newResponse)
			dh := &DockerHost{}
			streams := BuildResponse{}
			res, e := dh.handleBuildImageJson(r, func(s *Stream) {
				streams = append(streams, s)
			})
			So(e, ShouldBeNil)
			So(len(streams), ShouldEqual, 9)
			So(len(res), ShouldEqual, 9)
			So(streams[0].Stream, ShouldEqual, "Step 1 : FROM ubuntu\n")
			So(streams.ImageId(), ShouldEqual, "0f101a4836f6")
		})
	})
}

const outputPre0_7 = `Step 1 : FROM ubuntu
---> 8dbd9e392a96
Step 2 : RUN apt-get update
---> Using cache
---> f7ada547a49b
Step 3 : RUN apt-get upgrade -y
---> Using cache
---> b30eb4fbfc51
Successfully built b30eb4fbfc51
`

const newResponse = `{"stream":"Step 1 : FROM ubuntu\n"}
{"stream":" ---\u003e 8dbd9e392a96\n"}
{"stream":"Step 2 : RUN apt-get update\n"}
{"stream":" ---\u003e Using cache\n"}
{"stream":" ---\u003e 30d9e1cb9bb8\n"}
{"stream":"Step 3 : RUN apt-get upgrade -y\n"}
{"stream":" ---\u003e Using cache\n"}
{"stream":" ---\u003e 0f101a4836f6\n"}
{"stream":"Successfully built 0f101a4836f6\n"}
`
<|endoftext|>


<commit_before>// Copyright 2018 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcp

import (
	"os"

	"istio.io/test-infra/toolbox/util"
)

// SetKubeConfig saves kube config from a given cluster to the given location
func SetKubeConfig(project, zone, cluster, kubeconfig string) error {
	if err := os.Setenv("KUBECONFIG", kubeconfig); err != nil {
		return err
	}
	_, err := util.Shell("gcloud", "container", "clusters", "get-credentials", cluster,
		"--project", project, "--zone", zone)
	return err
}
<commit_msg>Fixes format of gcloud command<commit_after>// Copyright 2018 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcp

import (
	"os"

	"istio.io/test-infra/toolbox/util"
)

// SetKubeConfig saves kube config from a given cluster to the given location
func SetKubeConfig(project, zone, cluster, kubeconfig string) error {
	if err := os.Setenv("KUBECONFIG", kubeconfig); err != nil {
		return err
	}
	_, err := util.Shell(
		"gcloud container clusters get-credentials %s --project=%s --zone=%s",
		cluster, project, zone)
	return err
}
<|endoftext|>


<commit_before>/*
   Hockeypuck - OpenPGP key server
   Copyright (C) 2012-2014  Casey Marshall

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published by
   the Free Software Foundation, version 3.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

package openpgp

import (
	"bytes"
	"crypto/md5"
	"crypto/sha1"
	"encoding/hex"
	"fmt"
	"strings"
	"time"

	"golang.org/x/crypto/openpgp/packet"
	"gopkg.in/errgo.v1"
)

type PublicKey struct {
	Packet

	RFingerprint string
	RKeyID       string
	RShortID     string

	// Creation stores the timestamp when the public key was created.
	Creation time.Time

	// Expiration stores the timestamp when the public key expires.
	Expiration time.Time

	// Algorithm stores the algorithm type of the public key.
	Algorithm int

	// BitLen stores the bit length of the public key.
	BitLen int

	Signatures []*Signature
	Others     []*Packet
}

func algoCode(algo int) string {
	switch algo {
	case 1, 2, 3:
		return "rsa"
	case 16:
		return "elg"
	case 17:
		return "dsa"
	case 18:
		return "ecdh"
	case 19:
		return "ecdsa"
	case 20:
		return "elg"
	case 22:
		return "eddsa"
	default:
		return fmt.Sprintf("unk%d?", algo)
	}
}

func (pk *PublicKey) QualifiedFingerprint() string {
	return fmt.Sprintf("%s%d/%s", algoCode(pk.Algorithm), pk.BitLen, Reverse(pk.RFingerprint))
}

func (pk *PublicKey) ShortID() string {
	return Reverse(pk.RShortID)
}

func (pk *PublicKey) KeyID() string {
	return Reverse(pk.RKeyID)
}

func (pk *PublicKey) Fingerprint() string {
	return Reverse(pk.RFingerprint)
}

// appendSignature implements signable.
func (pk *PublicKey) appendSignature(sig *Signature) {
	pk.Signatures = append(pk.Signatures, sig)
}

func (pkp *PublicKey) publicKeyPacket() (*packet.PublicKey, error) {
	op, err := pkp.opaquePacket()
	if err != nil {
		return nil, errgo.Mask(err)
	}
	p, err := op.Parse()
	if err != nil {
		return nil, errgo.Mask(err)
	}
	pk, ok := p.(*packet.PublicKey)
	if !ok {
		return nil, errgo.Newf("expected public key packet, got %T", p)
	}
	return pk, nil
}

func (pkp *PublicKey) publicKeyV3Packet() (*packet.PublicKeyV3, error) {
	op, err := pkp.opaquePacket()
	if err != nil {
		return nil, errgo.Mask(err)
	}
	p, err := op.Parse()
	if err != nil {
		return nil, errgo.Mask(err)
	}
	pk, ok := p.(*packet.PublicKeyV3)
	if !ok {
		return nil, errgo.Newf("expected public key V3 packet, got %T", p)
	}
	return pk, nil
}

func (pkp *PublicKey) parse(op *packet.OpaquePacket, subkey bool) error {
	p, err := op.Parse()
	if err != nil {
		return errgo.Mask(err)
	}

	switch pk := p.(type) {
	case *packet.PublicKey:
		if pk.IsSubkey != subkey {
			return ErrInvalidPacketType
		}
		return pkp.setPublicKey(pk)
	case *packet.PublicKeyV3:
		if pk.IsSubkey != subkey {
			return ErrInvalidPacketType
		}
		return pkp.setPublicKeyV3(pk)
	default:
	}

	return errgo.Mask(ErrInvalidPacketType)
}

func (pkp *PublicKey) setUnsupported(op *packet.OpaquePacket) error {
	// Calculate opaque fingerprint on unsupported public key packet
	h := sha1.New()
	h.Write([]byte{0x99, byte(len(op.Contents) >> 8), byte(len(op.Contents))})
	h.Write(op.Contents)
	fpr := hex.EncodeToString(h.Sum(nil))
	pkp.RFingerprint = Reverse(fpr)
	pkp.UUID = pkp.RFingerprint
	return pkp.setV4IDs(pkp.UUID)
}

func (pkp *PublicKey) setPublicKey(pk *packet.PublicKey) error {
	buf := bytes.NewBuffer(nil)
	err := pk.Serialize(buf)
	if err != nil {
		return errgo.Mask(err)
	}
	fingerprint := hex.EncodeToString(pk.Fingerprint[:])
	bitLen, err := pk.BitLength()
	if err != nil {
		return errgo.Mask(err)
	}
	pkp.RFingerprint = Reverse(fingerprint)
	pkp.UUID = pkp.RFingerprint
	err = pkp.setV4IDs(pkp.UUID)
	if err != nil {
		return err
	}
	pkp.Creation = pk.CreationTime
	pkp.Algorithm = int(pk.PubKeyAlgo)
	pkp.BitLen = int(bitLen)
	pkp.Parsed = true
	return nil
}

func (pkp *PublicKey) setV4IDs(rfp string) error {
	if len(rfp) < 8 {
		return errgo.Newf("invalid fingerprint %q", rfp)
	}
	pkp.RShortID = rfp[:8]
	if len(rfp) < 16 {
		return errgo.Newf("invalid fingerprint %q", rfp)
	}
	pkp.RKeyID = rfp[:16]
	return nil
}

func (pkp *PublicKey) setPublicKeyV3(pk *packet.PublicKeyV3) error {
	var buf bytes.Buffer
	err := pk.Serialize(&buf)
	if err != nil {
		return errgo.Mask(err)
	}
	fingerprint := hex.EncodeToString(pk.Fingerprint[:])
	bitLen, err := pk.BitLength()
	if err != nil {
		return errgo.Mask(err)
	}
	pkp.RFingerprint = Reverse(fingerprint)
	pkp.UUID = pkp.RFingerprint
	pkp.RShortID = Reverse(fmt.Sprintf("%08x", uint32(pk.KeyId)))
	pkp.RKeyID = Reverse(fmt.Sprintf("%016x", pk.KeyId))
	pkp.Creation = pk.CreationTime
	if pk.DaysToExpire > 0 {
		pkp.Expiration = pkp.Creation.Add(time.Duration(pk.DaysToExpire) * time.Hour * 24)
	}
	pkp.Algorithm = int(pk.PubKeyAlgo)
	pkp.BitLen = int(bitLen)
	pkp.Parsed = true
	return nil
}

type PrimaryKey struct {
	PublicKey

	MD5    string
	SHA256 string

	SubKeys        []*SubKey
	UserIDs        []*UserID
	UserAttributes []*UserAttribute
}

// contents implements the packetNode interface for top-level public keys.
func (pubkey *PrimaryKey) contents() []packetNode {
	result := []packetNode{pubkey}
	for _, sig := range pubkey.Signatures {
		result = append(result, sig.contents()...)
	}
	for _, uid := range pubkey.UserIDs {
		result = append(result, uid.contents()...)
	}
	for _, uat := range pubkey.UserAttributes {
		result = append(result, uat.contents()...)
	}
	for _, subkey := range pubkey.SubKeys {
		result = append(result, subkey.contents()...)
	}
	for _, other := range pubkey.Others {
		result = append(result, other.contents()...)
	}
	return result
}

func (*PrimaryKey) removeDuplicate(parent packetNode, dup packetNode) error {
	return errgo.New("cannot remove a duplicate primary pubkey")
}

func ParsePrimaryKey(op *packet.OpaquePacket) (*PrimaryKey, error) {
	var buf bytes.Buffer
	var err error

	if err = op.Serialize(&buf); err != nil {
		return nil, errgo.Mask(err)
	}
	pubkey := &PrimaryKey{
		PublicKey: PublicKey{
			Packet: Packet{
				Tag:    op.Tag,
				Packet: buf.Bytes(),
			},
		},
	}

	// Attempt to parse the opaque packet into a public key type.
	parseErr := pubkey.parse(op, false)
	if parseErr != nil {
		err = pubkey.setUnsupported(op)
		if err != nil {
			return nil, errgo.Mask(err)
		}
	} else {
		pubkey.Parsed = true
	}

	return pubkey, nil
}

func (pubkey *PrimaryKey) setPublicKey(pk *packet.PublicKey) error {
	if pk.IsSubkey {
		return errgo.NoteMask(ErrInvalidPacketType, "expected primary public key packet, got sub-key")
	}
	return pubkey.PublicKey.setPublicKey(pk)
}

func (pubkey *PrimaryKey) setPublicKeyV3(pk *packet.PublicKeyV3) error {
	if pk.IsSubkey {
		return errgo.NoteMask(ErrInvalidPacketType, "expected primary public key packet, got sub-key")
	}
	return pubkey.PublicKey.setPublicKeyV3(pk)
}

func (pubkey *PrimaryKey) SelfSigs() *SelfSigs {
	result := &SelfSigs{target: pubkey}
	for _, sig := range pubkey.Signatures {
		// Skip non-self-certifications.
		if !strings.HasPrefix(pubkey.UUID, sig.RIssuerKeyID) {
			continue
		}
		checkSig := &CheckSig{
			PrimaryKey: pubkey,
			Signature:  sig,
			Error:      pubkey.verifyPublicKeySelfSig(&pubkey.PublicKey, sig),
		}
		if checkSig.Error != nil {
			result.Errors = append(result.Errors, checkSig)
			continue
		}
		switch sig.SigType {
		case 0x20: // packet.SigTypeKeyRevocation
			result.Revocations = append(result.Revocations, checkSig)
		}
	}
	result.resolve()
	return result
}

func (pubkey *PrimaryKey) updateMD5() error {
	digest, err := SksDigest(pubkey, md5.New())
	if err != nil {
		return err
	}
	pubkey.MD5 = digest
	return nil
}
<commit_msg>Expose AlgorithmName.<commit_after>/*
   Hockeypuck - OpenPGP key server
   Copyright (C) 2012-2014  Casey Marshall

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published by
   the Free Software Foundation, version 3.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

package openpgp

import (
	"bytes"
	"crypto/md5"
	"crypto/sha1"
	"encoding/hex"
	"fmt"
	"strings"
	"time"

	"golang.org/x/crypto/openpgp/packet"
	"gopkg.in/errgo.v1"
)

type PublicKey struct {
	Packet

	RFingerprint string
	RKeyID       string
	RShortID     string

	// Creation stores the timestamp when the public key was created.
	Creation time.Time

	// Expiration stores the timestamp when the public key expires.
	Expiration time.Time

	// Algorithm stores the algorithm type of the public key.
	Algorithm int

	// BitLen stores the bit length of the public key.
	BitLen int

	Signatures []*Signature
	Others     []*Packet
}

func AlgorithmName(code int) string {
	switch code {
	case 1, 2, 3:
		return "rsa"
	case 16:
		return "elg"
	case 17:
		return "dsa"
	case 18:
		return "ecdh"
	case 19:
		return "ecdsa"
	case 20:
		return "elg"
	case 22:
		return "eddsa"
	default:
		return fmt.Sprintf("unk(#%d)", code)
	}
}

func (pk *PublicKey) QualifiedFingerprint() string {
	return fmt.Sprintf("%s%d/%s", AlgorithmName(pk.Algorithm), pk.BitLen, Reverse(pk.RFingerprint))
}

func (pk *PublicKey) ShortID() string {
	return Reverse(pk.RShortID)
}

func (pk *PublicKey) KeyID() string {
	return Reverse(pk.RKeyID)
}

func (pk *PublicKey) Fingerprint() string {
	return Reverse(pk.RFingerprint)
}

// appendSignature implements signable.
func (pk *PublicKey) appendSignature(sig *Signature) {
	pk.Signatures = append(pk.Signatures, sig)
}

func (pkp *PublicKey) publicKeyPacket() (*packet.PublicKey, error) {
	op, err := pkp.opaquePacket()
	if err != nil {
		return nil, errgo.Mask(err)
	}
	p, err := op.Parse()
	if err != nil {
		return nil, errgo.Mask(err)
	}
	pk, ok := p.(*packet.PublicKey)
	if !ok {
		return nil, errgo.Newf("expected public key packet, got %T", p)
	}
	return pk, nil
}

func (pkp *PublicKey) publicKeyV3Packet() (*packet.PublicKeyV3, error) {
	op, err := pkp.opaquePacket()
	if err != nil {
		return nil, errgo.Mask(err)
	}
	p, err := op.Parse()
	if err != nil {
		return nil, errgo.Mask(err)
	}
	pk, ok := p.(*packet.PublicKeyV3)
	if !ok {
		return nil, errgo.Newf("expected public key V3 packet, got %T", p)
	}
	return pk, nil
}

func (pkp *PublicKey) parse(op *packet.OpaquePacket, subkey bool) error {
	p, err := op.Parse()
	if err != nil {
		return errgo.Mask(err)
	}

	switch pk := p.(type) {
	case *packet.PublicKey:
		if pk.IsSubkey != subkey {
			return ErrInvalidPacketType
		}
		return pkp.setPublicKey(pk)
	case *packet.PublicKeyV3:
		if pk.IsSubkey != subkey {
			return ErrInvalidPacketType
		}
		return pkp.setPublicKeyV3(pk)
	default:
	}

	return errgo.Mask(ErrInvalidPacketType)
}

func (pkp *PublicKey) setUnsupported(op *packet.OpaquePacket) error {
	// Calculate opaque fingerprint on unsupported public key packet
	h := sha1.New()
	h.Write([]byte{0x99, byte(len(op.Contents) >> 8), byte(len(op.Contents))})
	h.Write(op.Contents)
	fpr := hex.EncodeToString(h.Sum(nil))
	pkp.RFingerprint = Reverse(fpr)
	pkp.UUID = pkp.RFingerprint
	return pkp.setV4IDs(pkp.UUID)
}

func (pkp *PublicKey) setPublicKey(pk *packet.PublicKey) error {
	buf := bytes.NewBuffer(nil)
	err := pk.Serialize(buf)
	if err != nil {
		return errgo.Mask(err)
	}
	fingerprint := hex.EncodeToString(pk.Fingerprint[:])
	bitLen, err := pk.BitLength()
	if err != nil {
		return errgo.Mask(err)
	}
	pkp.RFingerprint = Reverse(fingerprint)
	pkp.UUID = pkp.RFingerprint
	err = pkp.setV4IDs(pkp.UUID)
	if err != nil {
		return err
	}
	pkp.Creation = pk.CreationTime
	pkp.Algorithm = int(pk.PubKeyAlgo)
	pkp.BitLen = int(bitLen)
	pkp.Parsed = true
	return nil
}

func (pkp *PublicKey) setV4IDs(rfp string) error {
	if len(rfp) < 8 {
		return errgo.Newf("invalid fingerprint %q", rfp)
	}
	pkp.RShortID = rfp[:8]
	if len(rfp) < 16 {
		return errgo.Newf("invalid fingerprint %q", rfp)
	}
	pkp.RKeyID = rfp[:16]
	return nil
}

func (pkp *PublicKey) setPublicKeyV3(pk *packet.PublicKeyV3) error {
	var buf bytes.Buffer
	err := pk.Serialize(&buf)
	if err != nil {
		return errgo.Mask(err)
	}
	fingerprint := hex.EncodeToString(pk.Fingerprint[:])
	bitLen, err := pk.BitLength()
	if err != nil {
		return errgo.Mask(err)
	}
	pkp.RFingerprint = Reverse(fingerprint)
	pkp.UUID = pkp.RFingerprint
	pkp.RShortID = Reverse(fmt.Sprintf("%08x", uint32(pk.KeyId)))
	pkp.RKeyID = Reverse(fmt.Sprintf("%016x", pk.KeyId))
	pkp.Creation = pk.CreationTime
	if pk.DaysToExpire > 0 {
		pkp.Expiration = pkp.Creation.Add(time.Duration(pk.DaysToExpire) * time.Hour * 24)
	}
	pkp.Algorithm = int(pk.PubKeyAlgo)
	pkp.BitLen = int(bitLen)
	pkp.Parsed = true
	return nil
}

type PrimaryKey struct {
	PublicKey

	MD5    string
	SHA256 string

	SubKeys        []*SubKey
	UserIDs        []*UserID
	UserAttributes []*UserAttribute
}

// contents implements the packetNode interface for top-level public keys.
func (pubkey *PrimaryKey) contents() []packetNode {
	result := []packetNode{pubkey}
	for _, sig := range pubkey.Signatures {
		result = append(result, sig.contents()...)
	}
	for _, uid := range pubkey.UserIDs {
		result = append(result, uid.contents()...)
	}
	for _, uat := range pubkey.UserAttributes {
		result = append(result, uat.contents()...)
	}
	for _, subkey := range pubkey.SubKeys {
		result = append(result, subkey.contents()...)
	}
	for _, other := range pubkey.Others {
		result = append(result, other.contents()...)
	}
	return result
}

func (*PrimaryKey) removeDuplicate(parent packetNode, dup packetNode) error {
	return errgo.New("cannot remove a duplicate primary pubkey")
}

func ParsePrimaryKey(op *packet.OpaquePacket) (*PrimaryKey, error) {
	var buf bytes.Buffer
	var err error

	if err = op.Serialize(&buf); err != nil {
		return nil, errgo.Mask(err)
	}
	pubkey := &PrimaryKey{
		PublicKey: PublicKey{
			Packet: Packet{
				Tag:    op.Tag,
				Packet: buf.Bytes(),
			},
		},
	}

	// Attempt to parse the opaque packet into a public key type.
	parseErr := pubkey.parse(op, false)
	if parseErr != nil {
		err = pubkey.setUnsupported(op)
		if err != nil {
			return nil, errgo.Mask(err)
		}
	} else {
		pubkey.Parsed = true
	}

	return pubkey, nil
}

func (pubkey *PrimaryKey) setPublicKey(pk *packet.PublicKey) error {
	if pk.IsSubkey {
		return errgo.NoteMask(ErrInvalidPacketType, "expected primary public key packet, got sub-key")
	}
	return pubkey.PublicKey.setPublicKey(pk)
}

func (pubkey *PrimaryKey) setPublicKeyV3(pk *packet.PublicKeyV3) error {
	if pk.IsSubkey {
		return errgo.NoteMask(ErrInvalidPacketType, "expected primary public key packet, got sub-key")
	}
	return pubkey.PublicKey.setPublicKeyV3(pk)
}

func (pubkey *PrimaryKey) SelfSigs() *SelfSigs {
	result := &SelfSigs{target: pubkey}
	for _, sig := range pubkey.Signatures {
		// Skip non-self-certifications.
		if !strings.HasPrefix(pubkey.UUID, sig.RIssuerKeyID) {
			continue
		}
		checkSig := &CheckSig{
			PrimaryKey: pubkey,
			Signature:  sig,
			Error:      pubkey.verifyPublicKeySelfSig(&pubkey.PublicKey, sig),
		}
		if checkSig.Error != nil {
			result.Errors = append(result.Errors, checkSig)
			continue
		}
		switch sig.SigType {
		case 0x20: // packet.SigTypeKeyRevocation
			result.Revocations = append(result.Revocations, checkSig)
		}
	}
	result.resolve()
	return result
}

func (pubkey *PrimaryKey) updateMD5() error {
	digest, err := SksDigest(pubkey, md5.New())
	if err != nil {
		return err
	}
	pubkey.MD5 = digest
	return nil
}
<|endoftext|>


<commit_before>// PubSub implements a simple Redis backed publish-subscribe client.
package pubsub

import "github.com/garyburd/redigo/redis"

// PubSub is a Redis pub/sub client. It uses a connection pool for
// communicating with Redis.
type Conn struct {
	pool *redis.Pool
}

// Dial connects to the Redis server with the given network and address.
func Dial(network string, address string, idle int, active int) Conn {
	pool := &redis.Pool{
		MaxIdle:   idle,
		MaxActive: active,
		Dial: func() (redis.Conn, error) {
			c, err := redis.Dial(network, address)
			if err != nil {
				panic(err.Error())
			}
			return c, err
		},
	}
	return Conn{pool: pool}
}

// Publish a message with the given topic.
func (c Conn) Publish(topic string, msg []byte) {
	con := c.pool.Get()
	defer con.Close()
	con.Do("PUBLISH", topic, msg)
}

// Publish a message with the given topic.
func (c Conn) PublishX(topic string, msg []byte) {
	con := c.pool.Get()
	defer con.Close()
	con.Do("PUBLISH", topic, msg)
}

// Subscribe to topic and get messages received for this topic sent to the
// given channel.
func (c Conn) Subscribe(topic string, chn chan<- []byte) {
	con := redis.PubSubConn{Conn: c.pool.Get()}
	con.Subscribe(topic)

	go func(con redis.PubSubConn) {
		for {
			switch v := con.Receive().(type) {
			case redis.Message:
				chn <- v.Data
			case redis.Subscription:
			case error:
				break
			}
		}
		con.Unsubscribe(topic)
	}(con)
}

// Closes the connection to Redis. This closes the connection pool.
func (c Conn) Close() {
	if c.pool == nil {
		return
	}
	c.pool.Close()
}
<commit_msg>removed stale code<commit_after>// PubSub implements a simple Redis backed publish-subscribe client.
package pubsub

import "github.com/garyburd/redigo/redis"

// PubSub is a Redis pub/sub client. It uses a connection pool for
// communicating with Redis.
type Conn struct {
	pool *redis.Pool
}

// Dial connects to the Redis server with the given network and address.
func Dial(network string, address string, idle int, active int) Conn {
	pool := &redis.Pool{
		MaxIdle:   idle,
		MaxActive: active,
		Dial: func() (redis.Conn, error) {
			c, err := redis.Dial(network, address)
			if err != nil {
				panic(err.Error())
			}
			return c, err
		},
	}
	return Conn{pool: pool}
}

// Publish a message with the given topic.
func (c Conn) Publish(topic string, msg []byte) {
	con := c.pool.Get()
	defer con.Close()
	con.Do("PUBLISH", topic, msg)
}

// Subscribe to topic and get messages received for this topic sent to the
// given channel.
func (c Conn) Subscribe(topic string, chn chan<- []byte) {
	con := redis.PubSubConn{Conn: c.pool.Get()}
	con.Subscribe(topic)

	go func(con redis.PubSubConn) {
		for {
			switch v := con.Receive().(type) {
			case redis.Message:
				chn <- v.Data
			case redis.Subscription:
			case error:
				break
			}
		}
		con.Unsubscribe(topic)
	}(con)
}

// Closes the connection to Redis. This closes the connection pool.
func (c Conn) Close() {
	if c.pool == nil {
		return
	}
	c.pool.Close()
}
<|endoftext|>


<commit_before>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package cloud contains Google Cloud Platform APIs related types
// and common functions.
package cloud

import (
	"net/http"

	"google.golang.org/cloud/internal"

	"code.google.com/p/go.net/context"
	pubsub "code.google.com/p/google-api-go-client/pubsub/v1beta1"
	storage "code.google.com/p/google-api-go-client/storage/v1"
)

// NewContext returns a new context that uses the provided http.Client.
// Provided http.Client is responsible to authorize and authenticate
// the requests made to the Google Cloud APIs.
// It mutates the client's original Transport to append the cloud
// package's user-agent to the outgoing requests.
// You can obtain the project ID from the Google Developers Console,
// https://console.developers.google.com.
func NewContext(projID string, c *http.Client) context.Context {
	return WithContext(context.Background(), projID, c)
}

// WithContext returns a new context in a similar way NewContext does,
// but initiates the new context with the specified parent.
func WithContext(parent context.Context, projID string, c *http.Client) context.Context {
	c.Transport = &internal.UATransport{Base: c.Transport}
	vals := make(map[string]interface{})
	vals["project_id"] = projID
	vals["http_client"] = c
	vals["pubsub_service"], _ = pubsub.New(c)
	vals["storage_service"], _ = storage.New(c)
	return context.WithValue(parent, internal.Key(0), vals)
}
<commit_msg>Adding a TODO to initiate the service objects lazily.<commit_after>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package cloud contains Google Cloud Platform APIs related types
// and common functions.
package cloud

import (
	"net/http"

	"google.golang.org/cloud/internal"

	"code.google.com/p/go.net/context"
	pubsub "code.google.com/p/google-api-go-client/pubsub/v1beta1"
	storage "code.google.com/p/google-api-go-client/storage/v1"
)

// NewContext returns a new context that uses the provided http.Client.
// Provided http.Client is responsible to authorize and authenticate
// the requests made to the Google Cloud APIs.
// It mutates the client's original Transport to append the cloud
// package's user-agent to the outgoing requests.
// You can obtain the project ID from the Google Developers Console,
// https://console.developers.google.com.
func NewContext(projID string, c *http.Client) context.Context {
	return WithContext(context.Background(), projID, c)
}

// WithContext returns a new context in a similar way NewContext does,
// but initiates the new context with the specified parent.
func WithContext(parent context.Context, projID string, c *http.Client) context.Context {
	c.Transport = &internal.UATransport{Base: c.Transport}
	vals := make(map[string]interface{})
	vals["project_id"] = projID
	vals["http_client"] = c
	// TODO(jbd): Lazily initiate the service objects.
	vals["pubsub_service"], _ = pubsub.New(c)
	vals["storage_service"], _ = storage.New(c)
	return context.WithValue(parent, internal.Key(0), vals)
}
<|endoftext|>


<commit_before>// +build darwin

// Package cocoa provides functions and types that are shared between go-vu
// drivers for Apple platforms.
package cocoa

// #cgo CFLAGS: -x objective-c -fobjc-arc -Wno-unused-parameter
// #cgo LDFLAGS: -framework AppKit
import "C"
<commit_msg>[skip ci] update doc<commit_after>// +build darwin

// Package cocoa provides functions and types that are shared between go-vu
// drivers for Apple platforms.
//
// The intent of thie package is not to provide a full interface to the Cocoa
// framework in go but rather to share reusable code and expose convenient
// abstractions in the context of building OSX and iOS drivers for the go-vu
// project.
package cocoa

// #cgo CFLAGS: -x objective-c -fobjc-arc -Wno-unused-parameter
// #cgo LDFLAGS: -framework AppKit
import "C"
<|endoftext|>


<commit_before>// gitinfo enables querying info from a Git repository.
package gitinfo

import (
	"fmt"
	"os/exec"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"skia.googlesource.com/buildbot.git/perf/go/config"
	"skia.googlesource.com/buildbot.git/perf/go/types"
	"skia.googlesource.com/buildbot.git/perf/go/util"

	"github.com/golang/glog"
)

// commitLineRe matches one line of commit log and captures hash, author and
// subject groups.
var commitLineRe = regexp.MustCompile(`([0-9a-f]{40}),([^,\n]+),(.+)$`)

// GitInfo allows querying a Git repo.
type GitInfo struct {
	dir        string
	hashes     []string
	timestamps map[string]time.Time // Key is the hash.
	mutex      sync.Mutex
}

// NewGitInfo creates a new GitInfo for the Git repository found in directory
// dir. If pull is true then a git pull is done on the repo before querying it
// for history.
func NewGitInfo(dir string, pull bool) (*GitInfo, error) {
	g := &GitInfo{
		dir:    dir,
		hashes: []string{},
	}
	return g, g.Update(pull)
}

// Update refreshes the history that GitInfo stores for the repo. If pull is
// true then git pull is performed before refreshing.
func (g *GitInfo) Update(pull bool) error {
	g.mutex.Lock()
	defer g.mutex.Unlock()
	glog.Info("Beginning Update.")
	if pull {
		cmd := exec.Command("git", "pull")
		cmd.Dir = g.dir
		b, err := cmd.Output()
		if err != nil {
			return fmt.Errorf("Failed to sync to HEAD: %s - %s", err, string(b))
		}
	}

	hashes, timestamps, err := readCommitsFromGit(g.dir)
	if err != nil {
		return err
	}
	g.hashes = hashes
	g.timestamps = timestamps
	return nil
}

// Details returns the author, subject and timestamp for the given commit.
func (g *GitInfo) Details(hash string) (string, string, time.Time, error) {
	cmd := exec.Command("git", "log", "-n", "1", "--format=format:%an%x20(%ae)%n%s", hash)
	cmd.Dir = g.dir
	b, err := cmd.Output()
	if err != nil {
		return "", "", time.Time{}, fmt.Errorf("Failed to execute Git: %s", err)
	}
	lines := strings.SplitN(string(b), "\n", 2)
	if len(lines) == 2 {
		return lines[0], lines[1], g.timestamps[hash], nil
	} else {
		return lines[0], "", time.Time{}, nil
	}
}

// From returns all commits from 'start' to HEAD.
func (g *GitInfo) From(start time.Time) []string {
	ret := []string{}
	for _, h := range g.hashes {
		if g.timestamps[h].After(start) {
			ret = append(ret, h)
		}
	}
	return ret
}

// Log returns a --name-only short log for every commit in (begin, end].
//
// If end is "" then it returns just the short log for the single commit at
// begin.
//
// Example response:
//
//    commit b7988a21fdf23cc4ace6145a06ea824aa85db099
//    Author: Joe Gregorio <jcgregorio@google.com>
//    Date:   Tue Aug 5 16:19:48 2014 -0400
//
//        A description of the commit.
//
//    perf/go/skiaperf/perf.go
//    perf/go/types/types.go
//    perf/res/js/logic.js
//
func (g *GitInfo) Log(begin, end string) (string, error) {
	command := []string{"log", "--name-only"}
	hashrange := begin
	if end != "" {
		hashrange += ".." + end
		command = append(command, hashrange)
	} else {
		command = append(command, "-n", "1", hashrange)
	}
	cmd := exec.Command("git", command...)
	cmd.Dir = g.dir
	b, err := cmd.Output()
	if err != nil {
		return "", err
	}
	return string(b), nil
}

// ShortCommit stores the hash, author, and subject of a git commit.
type ShortCommit struct {
	Hash    string
	Author  string
	Subject string
}

// ShortCommits stores a slice of ShortCommit struct.
type ShortCommits struct {
	Commits []*ShortCommit
}

// ShortList returns a slice of ShortCommit for every commit in (begin, end].
func (g *GitInfo) ShortList(begin, end string) (*ShortCommits, error) {
	command := []string{"log", "--pretty='%H,%an,%s", begin + ".." + end}
	cmd := exec.Command("git", command...)
	cmd.Dir = g.dir
	b, err := cmd.Output()
	if err != nil {
		return nil, err
	}
	ret := &ShortCommits{
		Commits: []*ShortCommit{},
	}
	for _, line := range strings.Split(string(b), "\n") {
		match := commitLineRe.FindStringSubmatch(line)
		if match == nil {
			// This could happen if the subject has new line, in which case we truncate it and ignore the remainder.
			continue
		}
		commit := &ShortCommit{
			Hash:    match[1],
			Author:  match[2],
			Subject: match[3],
		}
		ret.Commits = append(ret.Commits, commit)
	}

	return ret, nil
}

// gitHash represents information on a single Git commit.
type gitHash struct {
	hash      string
	timeStamp time.Time
}

type gitHashSlice []*gitHash

func (p gitHashSlice) Len() int           { return len(p) }
func (p gitHashSlice) Less(i, j int) bool { return p[i].timeStamp.Before(p[j].timeStamp) }
func (p gitHashSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

// readCommitsFromGit reads the commit history from a Git repository.
func readCommitsFromGit(dir string) ([]string, map[string]time.Time, error) {
	cmd := exec.Command("git", "log", "--format=format:%H%x20%ci")
	cmd.Dir = dir
	b, err := cmd.Output()
	if err != nil {
		return nil, nil, fmt.Errorf("Failed to execute git log: %s - %s", err, string(b))
	}
	lines := strings.Split(string(b), "\n")
	gitHashes := make([]*gitHash, 0, len(lines))
	timestamps := map[string]time.Time{}
	for _, line := range lines {
		parts := strings.SplitN(line, " ", 2)
		if len(parts) == 2 {
			t, err := time.Parse("2006-01-02 15:04:05 -0700", parts[1])
			if err != nil {
				return nil, nil, fmt.Errorf("Failed parsing Git log timestamp: %s", err)
			}
			hash := parts[0]
			gitHashes = append(gitHashes, &gitHash{hash: hash, timeStamp: t})
			timestamps[hash] = t
		}
	}
	sort.Sort(gitHashSlice(gitHashes))
	hashes := make([]string, len(gitHashes), len(gitHashes))
	for i, h := range gitHashes {
		hashes[i] = h.hash
	}
	return hashes, timestamps, nil
}

// SkpCommits returns the indices for all the commits that contain SKP updates.
func (g *GitInfo) SkpCommits(tile *types.Tile) ([]int, error) {
	// Executes a git log command that looks like:
	//
	//   git log --format=format:%H  32956400b4d8f33394e2cdef9b66e8369ba2a0f3..e7416bfc9858bde8fc6eb5f3bfc942bc3350953a SKP_VERSION
	//
	// The output should be a \n separated list of hashes that match.
	first, last := tile.CommitRange()
	cmd := exec.Command("git", "log", "--format=format:%H", first+".."+last, "SKP_VERSION")
	cmd.Dir = g.dir
	b, err := cmd.Output()
	if err != nil {
		glog.Error(string(b))
		return nil, err
	}
	hashes := strings.Split(string(b), "\n")

	ret := []int{}
	for i, c := range tile.Commits {
		if c.CommitTime != 0 && util.In(c.Hash, hashes) {
			ret = append(ret, i)
		}
	}
	return ret, nil
}

// LastSkpCommit returns the time of the last change to the SKP_VERSION file.
func (g *GitInfo) LastSkpCommit() (time.Time, error) {
	// Executes a git log command that looks like:
	//
	// git log --format=format:%ct -n 1 SKP_VERSION
	//
	// The output should be a single unix timestamp.
	cmd := exec.Command("git", "log", "--format=format:%ct", "-n", "1", "SKP_VERSION")
	cmd.Dir = g.dir
	b, err := cmd.Output()
	if err != nil {
		glog.Error("Failed to read git log: ", err)
		return time.Time{}, err
	}
	ts, err := strconv.ParseInt(string(b), 10, 64)
	if err != nil {
		glog.Error("Failed to parse timestamp: ", string(b), err)
		return time.Time{}, err
	}
	return time.Unix(ts, 0), nil
}

// TileAddressFromHash takes a commit hash and time, then returns the Level 0
// tile number that contains the hash, and its position in the tile commit array.
// This assumes that tiles are built for commits since after the given time.
func (g *GitInfo) TileAddressFromHash(hash string, start time.Time) (num, offset int, err error) {
	i := 0
	for _, h := range g.hashes {
		if g.timestamps[h].Before(start) {
			continue
		}
		if h == hash {
			return i / config.TILE_SIZE, i % config.TILE_SIZE, nil
		}
		i++
	}
	return -1, -1, fmt.Errorf("Cannot find hash %s.\n", hash)
}
<commit_msg>Make gitinfo threadsafe.<commit_after>// gitinfo enables querying info from a Git repository.
package gitinfo

import (
	"fmt"
	"os/exec"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"skia.googlesource.com/buildbot.git/perf/go/config"
	"skia.googlesource.com/buildbot.git/perf/go/types"
	"skia.googlesource.com/buildbot.git/perf/go/util"

	"github.com/golang/glog"
)

// commitLineRe matches one line of commit log and captures hash, author and
// subject groups.
var commitLineRe = regexp.MustCompile(`([0-9a-f]{40}),([^,\n]+),(.+)$`)

// GitInfo allows querying a Git repo.
type GitInfo struct {
	dir        string
	hashes     []string
	timestamps map[string]time.Time // Key is the hash.

	// Any access to hashes or timestamps must be protected.
	mutex sync.Mutex
}

// NewGitInfo creates a new GitInfo for the Git repository found in directory
// dir. If pull is true then a git pull is done on the repo before querying it
// for history.
func NewGitInfo(dir string, pull bool) (*GitInfo, error) {
	g := &GitInfo{
		dir:    dir,
		hashes: []string{},
	}
	return g, g.Update(pull)
}

// Update refreshes the history that GitInfo stores for the repo. If pull is
// true then git pull is performed before refreshing.
func (g *GitInfo) Update(pull bool) error {
	g.mutex.Lock()
	defer g.mutex.Unlock()
	glog.Info("Beginning Update.")
	if pull {
		cmd := exec.Command("git", "pull")
		cmd.Dir = g.dir
		b, err := cmd.Output()
		if err != nil {
			return fmt.Errorf("Failed to sync to HEAD: %s - %s", err, string(b))
		}
	}

	hashes, timestamps, err := readCommitsFromGit(g.dir)
	if err != nil {
		return err
	}
	g.hashes = hashes
	g.timestamps = timestamps
	return nil
}

// Details returns the author, subject and timestamp for the given commit.
func (g GitInfo) Details(hash string) (string, string, time.Time, error) {
	g.mutex.Lock()
	defer g.mutex.Unlock()
	cmd := exec.Command("git", "log", "-n", "1", "--format=format:%an%x20(%ae)%n%s", hash)
	cmd.Dir = g.dir
	b, err := cmd.Output()
	if err != nil {
		return "", "", time.Time{}, fmt.Errorf("Failed to execute Git: %s", err)
	}
	lines := strings.SplitN(string(b), "\n", 2)
	if len(lines) == 2 {
		return lines[0], lines[1], g.timestamps[hash], nil
	} else {
		return lines[0], "", time.Time{}, nil
	}
}

// From returns all commits from 'start' to HEAD.
func (g GitInfo) From(start time.Time) []string {
	g.mutex.Lock()
	defer g.mutex.Unlock()
	ret := []string{}
	for _, h := range g.hashes {
		if g.timestamps[h].After(start) {
			ret = append(ret, h)
		}
	}
	return ret
}

// Log returns a --name-only short log for every commit in (begin, end].
//
// If end is "" then it returns just the short log for the single commit at
// begin.
//
// Example response:
//
//    commit b7988a21fdf23cc4ace6145a06ea824aa85db099
//    Author: Joe Gregorio <jcgregorio@google.com>
//    Date:   Tue Aug 5 16:19:48 2014 -0400
//
//        A description of the commit.
//
//    perf/go/skiaperf/perf.go
//    perf/go/types/types.go
//    perf/res/js/logic.js
//
func (g GitInfo) Log(begin, end string) (string, error) {
	command := []string{"log", "--name-only"}
	hashrange := begin
	if end != "" {
		hashrange += ".." + end
		command = append(command, hashrange)
	} else {
		command = append(command, "-n", "1", hashrange)
	}
	cmd := exec.Command("git", command...)
	cmd.Dir = g.dir
	b, err := cmd.Output()
	if err != nil {
		return "", err
	}
	return string(b), nil
}

// ShortCommit stores the hash, author, and subject of a git commit.
type ShortCommit struct {
	Hash    string
	Author  string
	Subject string
}

// ShortCommits stores a slice of ShortCommit struct.
type ShortCommits struct {
	Commits []*ShortCommit
}

// ShortList returns a slice of ShortCommit for every commit in (begin, end].
func (g *GitInfo) ShortList(begin, end string) (*ShortCommits, error) {
	command := []string{"log", "--pretty='%H,%an,%s", begin + ".." + end}
	cmd := exec.Command("git", command...)
	cmd.Dir = g.dir
	b, err := cmd.Output()
	if err != nil {
		return nil, err
	}
	ret := &ShortCommits{
		Commits: []*ShortCommit{},
	}
	for _, line := range strings.Split(string(b), "\n") {
		match := commitLineRe.FindStringSubmatch(line)
		if match == nil {
			// This could happen if the subject has new line, in which case we truncate it and ignore the remainder.
			continue
		}
		commit := &ShortCommit{
			Hash:    match[1],
			Author:  match[2],
			Subject: match[3],
		}
		ret.Commits = append(ret.Commits, commit)
	}

	return ret, nil
}

// gitHash represents information on a single Git commit.
type gitHash struct {
	hash      string
	timeStamp time.Time
}

type gitHashSlice []*gitHash

func (p gitHashSlice) Len() int           { return len(p) }
func (p gitHashSlice) Less(i, j int) bool { return p[i].timeStamp.Before(p[j].timeStamp) }
func (p gitHashSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

// readCommitsFromGit reads the commit history from a Git repository.
func readCommitsFromGit(dir string) ([]string, map[string]time.Time, error) {
	cmd := exec.Command("git", "log", "--format=format:%H%x20%ci")
	cmd.Dir = dir
	b, err := cmd.Output()
	if err != nil {
		return nil, nil, fmt.Errorf("Failed to execute git log: %s - %s", err, string(b))
	}
	lines := strings.Split(string(b), "\n")
	gitHashes := make([]*gitHash, 0, len(lines))
	timestamps := map[string]time.Time{}
	for _, line := range lines {
		parts := strings.SplitN(line, " ", 2)
		if len(parts) == 2 {
			t, err := time.Parse("2006-01-02 15:04:05 -0700", parts[1])
			if err != nil {
				return nil, nil, fmt.Errorf("Failed parsing Git log timestamp: %s", err)
			}
			hash := parts[0]
			gitHashes = append(gitHashes, &gitHash{hash: hash, timeStamp: t})
			timestamps[hash] = t
		}
	}
	sort.Sort(gitHashSlice(gitHashes))
	hashes := make([]string, len(gitHashes), len(gitHashes))
	for i, h := range gitHashes {
		hashes[i] = h.hash
	}
	return hashes, timestamps, nil
}

// SkpCommits returns the indices for all the commits that contain SKP updates.
func (g GitInfo) SkpCommits(tile *types.Tile) ([]int, error) {
	// Executes a git log command that looks like:
	//
	//   git log --format=format:%H  32956400b4d8f33394e2cdef9b66e8369ba2a0f3..e7416bfc9858bde8fc6eb5f3bfc942bc3350953a SKP_VERSION
	//
	// The output should be a \n separated list of hashes that match.
	first, last := tile.CommitRange()
	cmd := exec.Command("git", "log", "--format=format:%H", first+".."+last, "SKP_VERSION")
	cmd.Dir = g.dir
	b, err := cmd.Output()
	if err != nil {
		glog.Error(string(b))
		return nil, err
	}
	hashes := strings.Split(string(b), "\n")

	ret := []int{}
	for i, c := range tile.Commits {
		if c.CommitTime != 0 && util.In(c.Hash, hashes) {
			ret = append(ret, i)
		}
	}
	return ret, nil
}

// LastSkpCommit returns the time of the last change to the SKP_VERSION file.
func (g GitInfo) LastSkpCommit() (time.Time, error) {
	// Executes a git log command that looks like:
	//
	// git log --format=format:%ct -n 1 SKP_VERSION
	//
	// The output should be a single unix timestamp.
	cmd := exec.Command("git", "log", "--format=format:%ct", "-n", "1", "SKP_VERSION")
	cmd.Dir = g.dir
	b, err := cmd.Output()
	if err != nil {
		glog.Error("Failed to read git log: ", err)
		return time.Time{}, err
	}
	ts, err := strconv.ParseInt(string(b), 10, 64)
	if err != nil {
		glog.Error("Failed to parse timestamp: ", string(b), err)
		return time.Time{}, err
	}
	return time.Unix(ts, 0), nil
}

// TileAddressFromHash takes a commit hash and time, then returns the Level 0
// tile number that contains the hash, and its position in the tile commit array.
// This assumes that tiles are built for commits since after the given time.
func (g *GitInfo) TileAddressFromHash(hash string, start time.Time) (num, offset int, err error) {
	g.mutex.Lock()
	defer g.mutex.Unlock()
	i := 0
	for _, h := range g.hashes {
		if g.timestamps[h].Before(start) {
			continue
		}
		if h == hash {
			return i / config.TILE_SIZE, i % config.TILE_SIZE, nil
		}
		i++
	}
	return -1, -1, fmt.Errorf("Cannot find hash %s.\n", hash)
}
<|endoftext|>


<commit_before>// perfserver is the single executable that contains the sub-commands that make
// up a running Perf system, including the web ui, the ingestion process, and
// the regression detection process.
//
// This cli is built using Cobra (https://github.com/spf13/cobra/) and the cobra
// cli should be used to add new sub-commands.
package main

import (
	"context"
	"fmt"
	"os"

	"github.com/jcgregorio/logger"
	cli "github.com/urfave/cli/v2"
	"go.skia.org/infra/go/metrics2"
	"go.skia.org/infra/go/skerr"
	"go.skia.org/infra/go/sklog/glog_and_cloud"
	"go.skia.org/infra/go/urfavecli"
	"go.skia.org/infra/perf/go/config"
	"go.skia.org/infra/perf/go/frontend"
	"go.skia.org/infra/perf/go/ingest/process"
)

func main() {
	var clusterFlags config.FrontendFlags
	var frontendFlags config.FrontendFlags
	var ingestFlags config.IngestFlags

	cli.MarkdownDocTemplate = urfavecli.MarkdownDocTemplate

	cliApp := &cli.App{
		Name:  "perfserver",
		Usage: "Command line tool that runs the various components of Perf.",
		Before: func(c *cli.Context) error {
			// Log to stdout.
			glog_and_cloud.SetLogger(
				glog_and_cloud.NewSLogCloudLogger(logger.NewFromOptions(&logger.Options{
					SyncWriter: os.Stdout,
				})),
			)

			return nil
		},
		Commands: []*cli.Command{
			{
				Name:        "frontend",
				Usage:       "The main web UI.",
				Description: "Runs the process that serves the web UI for Perf.",
				Flags:       (&frontendFlags).AsCliFlags(false),
				Action: func(c *cli.Context) error {
					urfavecli.LogFlags(c)
					f, err := frontend.New(&frontendFlags)
					if err != nil {
						return err
					}
					f.Serve()
					return nil
				},
			},
			{
				Name:        "ingest",
				Usage:       "Run the ingestion process.",
				Description: "Continuously imports files as they arrive from the configured ingestion sources and populates the TraceStore with that data.",
				Flags:       (&ingestFlags).AsCliFlags(),
				Action: func(c *cli.Context) error {
					urfavecli.LogFlags(c)
					instanceConfig, err := config.InstanceConfigFromFile(ingestFlags.ConfigFilename)
					if err != nil {
						return err
					}
					if ingestFlags.ConnectionString != "" {
						instanceConfig.DataStoreConfig.ConnectionString = ingestFlags.ConnectionString
					}

					metrics2.InitPrometheus(ingestFlags.PromPort)

					return process.Start(context.Background(), ingestFlags.Local, ingestFlags.NumParallelIngesters, instanceConfig)
				},
			},
			{
				Name:        "cluster",
				Usage:       "Run the regression detection process.",
				Description: "Continuously runs over all the configured alerts and looks for regressions as new data arrives.",
				Flags:       (&clusterFlags).AsCliFlags(true),
				Action: func(c *cli.Context) error {
					urfavecli.LogFlags(c)
					f, err := frontend.New(&clusterFlags)
					if err != nil {
						return err
					}
					f.Serve()
					return nil
				},
			},
			{
				Name:  "markdown",
				Usage: "Generates markdown help for perfserver.",
				Action: func(c *cli.Context) error {
					body, err := c.App.ToMarkdown()
					if err != nil {
						return skerr.Wrap(err)
					}
					fmt.Println(body)
					return nil
				},
			},
		},
	}

	err := cliApp.Run(os.Args)
	if err != nil {
		fmt.Printf("\nError: %s\n", err.Error())
		os.Exit(2)
	}
}
<commit_msg>[perf] Remove cobra comment.<commit_after>// perfserver is the single executable that contains the sub-commands that make
// up a running Perf system, including the web ui, the ingestion process, and
// the regression detection process.
package main

import (
	"context"
	"fmt"
	"os"

	"github.com/jcgregorio/logger"
	cli "github.com/urfave/cli/v2"
	"go.skia.org/infra/go/metrics2"
	"go.skia.org/infra/go/skerr"
	"go.skia.org/infra/go/sklog/glog_and_cloud"
	"go.skia.org/infra/go/urfavecli"
	"go.skia.org/infra/perf/go/config"
	"go.skia.org/infra/perf/go/frontend"
	"go.skia.org/infra/perf/go/ingest/process"
)

func main() {
	var clusterFlags config.FrontendFlags
	var frontendFlags config.FrontendFlags
	var ingestFlags config.IngestFlags

	cli.MarkdownDocTemplate = urfavecli.MarkdownDocTemplate

	cliApp := &cli.App{
		Name:  "perfserver",
		Usage: "Command line tool that runs the various components of Perf.",
		Before: func(c *cli.Context) error {
			// Log to stdout.
			glog_and_cloud.SetLogger(
				glog_and_cloud.NewSLogCloudLogger(logger.NewFromOptions(&logger.Options{
					SyncWriter: os.Stdout,
				})),
			)

			return nil
		},
		Commands: []*cli.Command{
			{
				Name:        "frontend",
				Usage:       "The main web UI.",
				Description: "Runs the process that serves the web UI for Perf.",
				Flags:       (&frontendFlags).AsCliFlags(false),
				Action: func(c *cli.Context) error {
					urfavecli.LogFlags(c)
					f, err := frontend.New(&frontendFlags)
					if err != nil {
						return err
					}
					f.Serve()
					return nil
				},
			},
			{
				Name:        "ingest",
				Usage:       "Run the ingestion process.",
				Description: "Continuously imports files as they arrive from the configured ingestion sources and populates the TraceStore with that data.",
				Flags:       (&ingestFlags).AsCliFlags(),
				Action: func(c *cli.Context) error {
					urfavecli.LogFlags(c)
					instanceConfig, err := config.InstanceConfigFromFile(ingestFlags.ConfigFilename)
					if err != nil {
						return err
					}
					if ingestFlags.ConnectionString != "" {
						instanceConfig.DataStoreConfig.ConnectionString = ingestFlags.ConnectionString
					}

					metrics2.InitPrometheus(ingestFlags.PromPort)

					return process.Start(context.Background(), ingestFlags.Local, ingestFlags.NumParallelIngesters, instanceConfig)
				},
			},
			{
				Name:        "cluster",
				Usage:       "Run the regression detection process.",
				Description: "Continuously runs over all the configured alerts and looks for regressions as new data arrives.",
				Flags:       (&clusterFlags).AsCliFlags(true),
				Action: func(c *cli.Context) error {
					urfavecli.LogFlags(c)
					f, err := frontend.New(&clusterFlags)
					if err != nil {
						return err
					}
					f.Serve()
					return nil
				},
			},
			{
				Name:  "markdown",
				Usage: "Generates markdown help for perfserver.",
				Action: func(c *cli.Context) error {
					body, err := c.App.ToMarkdown()
					if err != nil {
						return skerr.Wrap(err)
					}
					fmt.Println(body)
					return nil
				},
			},
		},
	}

	err := cliApp.Run(os.Args)
	if err != nil {
		fmt.Printf("\nError: %s\n", err.Error())
		os.Exit(2)
	}
}
<|endoftext|>


<commit_before>package circular

// Implement a circular buffer of bytes supporting both overflow-checked writes
// and unconditional, possibly overwriting, writes.
//
//   type Buffer
//   func NewBuffer(size int) *Buffer
//   func (*Buffer) ReadByte() (byte, error)
//   func (*Buffer) WriteByte(c byte) error
//   func (*Buffer) Overwrite(c byte)
//   func (*Buffer) Reset() // put buffer in an empty state
//
// We chose the above API so that Buffer implements io.ByteReader
// and io.ByteWriter and can be used (size permitting) as a drop in
// replacement for anything using that interface.

import (
	"io"
	"testing"
)

const targetTestVersion = 4

func TestTestVersion(t *testing.T) {
	if testVersion != targetTestVersion {
		t.Errorf("Found testVersion = %v, want %v.", testVersion, targetTestVersion)
	}
}

// Here is one way you can have a test case verify that the expected
// interfaces are implemented.

var _ io.ByteReader = new(Buffer)
var _ io.ByteWriter = new(Buffer)

// testBuffer and methods support the tests, providing log and fail messages.

type testBuffer struct {
	*testing.T
	b *Buffer
}

func nb(size int, t *testing.T) testBuffer {
	t.Logf("NewBuffer(%d)", size)
	return testBuffer{t, NewBuffer(size)}
}

func (tb testBuffer) read(want byte) {
	switch c, err := tb.b.ReadByte(); {
	case err != nil:
		var _ error = err
		tb.Fatalf("ReadByte() failed unexpectedly: %v", err)
	case c != want:
		tb.Fatalf("ReadByte() = %c, want %c.", c, want)
	}
	tb.Logf("ReadByte %c", want)
}

func (tb testBuffer) readFail() {
	c, err := tb.b.ReadByte()
	if err == nil {
		tb.Fatalf("ReadByte() = %c, expected a failure", c)
	}
	var _ error = err
	tb.Log("ReadByte() fails as expected")
}

func (tb testBuffer) write(c byte) {
	if err := tb.b.WriteByte(c); err != nil {
		var _ error = err
		tb.Fatalf("WriteByte(%c) failed unexpectedly: %v", c, err)
	}
	tb.Logf("WriteByte(%c)", c)
}

func (tb testBuffer) writeFail(c byte) {
	err := tb.b.WriteByte(c)
	if err == nil {
		tb.Fatalf("WriteByte(%c) succeeded, expected a failure", c)
	}
	var _ error = err
	tb.Logf("WriteByte(%c) fails as expected", c)
}

func (tb testBuffer) reset() {
	tb.b.Reset()
	tb.Log("Reset()")
}

func (tb testBuffer) overwrite(c byte) {
	tb.b.Overwrite(c)
	tb.Logf("Overwrite(%c)", c)
}

// tests.  separate functions so log will have descriptive test name.

func TestReadEmptyBuffer(t *testing.T) {
	tb := nb(1, t)
	tb.readFail()
}

func TestWriteAndReadOneItem(t *testing.T) {
	tb := nb(1, t)
	tb.write('1')
	tb.read('1')
	tb.readFail()
}

func TestWriteAndReadMultipleItems(t *testing.T) {
	tb := nb(2, t)
	tb.write('1')
	tb.write('2')
	tb.read('1')
	tb.read('2')
	tb.readFail()
}

func TestReset(t *testing.T) {
	tb := nb(3, t)
	tb.write('1')
	tb.write('2')
	tb.write('3')
	tb.reset()
	tb.write('1')
	tb.write('3')
	tb.read('1')
	tb.write('4')
	tb.read('3')
}

func TestAlternateWriteAndRead(t *testing.T) {
	tb := nb(2, t)
	tb.write('1')
	tb.read('1')
	tb.write('2')
	tb.read('2')
}

func TestReadOldestItem(t *testing.T) {
	tb := nb(3, t)
	tb.write('1')
	tb.write('2')
	tb.read('1')
	tb.write('3')
	tb.read('2')
	tb.read('3')
}

func TestWriteFullBuffer(t *testing.T) {
	tb := nb(2, t)
	tb.write('1')
	tb.write('2')
	tb.writeFail('A')
}

func TestOverwriteFull(t *testing.T) {
	tb := nb(2, t)
	tb.write('1')
	tb.write('2')
	tb.overwrite('A')
	tb.read('2')
	tb.read('A')
	tb.readFail()
}

func TestOverwriteNonFull(t *testing.T) {
	tb := nb(2, t)
	tb.write('1')
	tb.overwrite('2')
	tb.read('1')
	tb.read('2')
	tb.readFail()
}

func TestAlternateReadAndOverwrite(t *testing.T) {
	tb := nb(5, t)
	tb.write('1')
	tb.write('2')
	tb.write('3')
	tb.read('1')
	tb.read('2')
	tb.write('4')
	tb.read('3')
	tb.write('5')
	tb.write('6')
	tb.write('7')
	tb.write('8')
	tb.overwrite('A')
	tb.overwrite('B')
	tb.read('6')
	tb.read('7')
	tb.read('8')
	tb.read('A')
	tb.read('B')
	tb.readFail()
}

func BenchmarkOverwrite(b *testing.B) {
	c := NewBuffer(100)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		c.Overwrite(0)
	}
	b.SetBytes(int64(b.N))
}

func BenchmarkWriteRead(b *testing.B) {
	c := NewBuffer(100)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		c.WriteByte(0)
		c.ReadByte()
	}
	b.SetBytes(int64(b.N))
}
<commit_msg>circular-buffer: testversiontest moved to start of test functions, errorf -> fatalf, see #470<commit_after>package circular

// Implement a circular buffer of bytes supporting both overflow-checked writes
// and unconditional, possibly overwriting, writes.
//
//   type Buffer
//   func NewBuffer(size int) *Buffer
//   func (*Buffer) ReadByte() (byte, error)
//   func (*Buffer) WriteByte(c byte) error
//   func (*Buffer) Overwrite(c byte)
//   func (*Buffer) Reset() // put buffer in an empty state
//
// We chose the above API so that Buffer implements io.ByteReader
// and io.ByteWriter and can be used (size permitting) as a drop in
// replacement for anything using that interface.

import (
	"io"
	"testing"
)

const targetTestVersion = 4

// Here is one way you can have a test case verify that the expected
// interfaces are implemented.

var _ io.ByteReader = new(Buffer)
var _ io.ByteWriter = new(Buffer)

// testBuffer and methods support the tests, providing log and fail messages.

type testBuffer struct {
	*testing.T
	b *Buffer
}

func nb(size int, t *testing.T) testBuffer {
	t.Logf("NewBuffer(%d)", size)
	return testBuffer{t, NewBuffer(size)}
}

func (tb testBuffer) read(want byte) {
	switch c, err := tb.b.ReadByte(); {
	case err != nil:
		var _ error = err
		tb.Fatalf("ReadByte() failed unexpectedly: %v", err)
	case c != want:
		tb.Fatalf("ReadByte() = %c, want %c.", c, want)
	}
	tb.Logf("ReadByte %c", want)
}

func (tb testBuffer) readFail() {
	c, err := tb.b.ReadByte()
	if err == nil {
		tb.Fatalf("ReadByte() = %c, expected a failure", c)
	}
	var _ error = err
	tb.Log("ReadByte() fails as expected")
}

func (tb testBuffer) write(c byte) {
	if err := tb.b.WriteByte(c); err != nil {
		var _ error = err
		tb.Fatalf("WriteByte(%c) failed unexpectedly: %v", c, err)
	}
	tb.Logf("WriteByte(%c)", c)
}

func (tb testBuffer) writeFail(c byte) {
	err := tb.b.WriteByte(c)
	if err == nil {
		tb.Fatalf("WriteByte(%c) succeeded, expected a failure", c)
	}
	var _ error = err
	tb.Logf("WriteByte(%c) fails as expected", c)
}

func (tb testBuffer) reset() {
	tb.b.Reset()
	tb.Log("Reset()")
}

func (tb testBuffer) overwrite(c byte) {
	tb.b.Overwrite(c)
	tb.Logf("Overwrite(%c)", c)
}

// tests.  separate functions so log will have descriptive test name.

func TestTestVersion(t *testing.T) {
	if testVersion != targetTestVersion {
		t.Fatalf("Found testVersion = %v, want %v.", testVersion, targetTestVersion)
	}
}

func TestReadEmptyBuffer(t *testing.T) {
	tb := nb(1, t)
	tb.readFail()
}

func TestWriteAndReadOneItem(t *testing.T) {
	tb := nb(1, t)
	tb.write('1')
	tb.read('1')
	tb.readFail()
}

func TestWriteAndReadMultipleItems(t *testing.T) {
	tb := nb(2, t)
	tb.write('1')
	tb.write('2')
	tb.read('1')
	tb.read('2')
	tb.readFail()
}

func TestReset(t *testing.T) {
	tb := nb(3, t)
	tb.write('1')
	tb.write('2')
	tb.write('3')
	tb.reset()
	tb.write('1')
	tb.write('3')
	tb.read('1')
	tb.write('4')
	tb.read('3')
}

func TestAlternateWriteAndRead(t *testing.T) {
	tb := nb(2, t)
	tb.write('1')
	tb.read('1')
	tb.write('2')
	tb.read('2')
}

func TestReadOldestItem(t *testing.T) {
	tb := nb(3, t)
	tb.write('1')
	tb.write('2')
	tb.read('1')
	tb.write('3')
	tb.read('2')
	tb.read('3')
}

func TestWriteFullBuffer(t *testing.T) {
	tb := nb(2, t)
	tb.write('1')
	tb.write('2')
	tb.writeFail('A')
}

func TestOverwriteFull(t *testing.T) {
	tb := nb(2, t)
	tb.write('1')
	tb.write('2')
	tb.overwrite('A')
	tb.read('2')
	tb.read('A')
	tb.readFail()
}

func TestOverwriteNonFull(t *testing.T) {
	tb := nb(2, t)
	tb.write('1')
	tb.overwrite('2')
	tb.read('1')
	tb.read('2')
	tb.readFail()
}

func TestAlternateReadAndOverwrite(t *testing.T) {
	tb := nb(5, t)
	tb.write('1')
	tb.write('2')
	tb.write('3')
	tb.read('1')
	tb.read('2')
	tb.write('4')
	tb.read('3')
	tb.write('5')
	tb.write('6')
	tb.write('7')
	tb.write('8')
	tb.overwrite('A')
	tb.overwrite('B')
	tb.read('6')
	tb.read('7')
	tb.read('8')
	tb.read('A')
	tb.read('B')
	tb.readFail()
}

func BenchmarkOverwrite(b *testing.B) {
	c := NewBuffer(100)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		c.Overwrite(0)
	}
	b.SetBytes(int64(b.N))
}

func BenchmarkWriteRead(b *testing.B) {
	c := NewBuffer(100)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		c.WriteByte(0)
		c.ReadByte()
	}
	b.SetBytes(int64(b.N))
}
<|endoftext|>