<commit_before>package main

import (
	"flag"
	"log"
	"net/http"
	"os"

	"github.com/julienschmidt/httprouter"
)

var (
	apiServerBaseURL       = flag.String("api-server-base-url", "https://kubernetes.default", "Kubernetes API server base URL")
	apiServerUser          = flag.String("api-server-username", "admin", "Kubernetes API server username to use if no service acccount API token is present.")
	apiServerPassword      = flag.String("api-server-password", "admin123", "Kubernetes API server password to use if no service acccount API token is present.")
	awsKey                 = flag.String("aws-access-key", "", "Default decap AWS access key.  /etc/secrets/aws-key in the cluster overrides this.")
	awsSecret              = flag.String("aws-secret-key", "", "Default decap AWS access secret.  /etc/secrets/aws-secret in the cluster overrides this.")
	awsRegion              = flag.String("aws-region", "us-west-1", "Default decap AWS region.  /etc/secrets/aws-region in the cluster overrides this.")
	githubClientID         = flag.String("github-client-id", "", "Default Github ClientID for quering Github repos.  /etc/secrets/github-client-id in the cluster overrides this.")
	githubClientSecret     = flag.String("github-client-secret", "", "Default Github Client Secret for quering Github repos.  /etc/secrets/github-client-secret in the cluster overrides this.")
	buildScriptsRepo       = flag.String("build-scripts-repo", "https://github.com/ae6rt/decap-build-scripts.git", "Git repo where userland build scripts are held.")
	buildScriptsRepoBranch = flag.String("build-scripts-repo-branch", "master", "Branch or revision to use on git repo where userland build scripts are held.")
	noWebsocket            = flag.Bool("no-websocket", false, "Do not start websocket client that watches pods.")
	versionFlag            = flag.Bool("version", false, "Print version info and exit.")

	Log *log.Logger = log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile)

	buildVersion string
	buildCommit  string
	buildDate    string
	buildGoSDK   string
)

func init() {
	flag.Parse()
	Log.Printf("Version: %s, Commit: %s, Date: %s, Go SDK: %s\n", buildVersion, buildCommit, buildDate, buildGoSDK)
	if *versionFlag {
		os.Exit(0)
	}

	*awsKey = kubeSecret("/etc/secrets/aws-key", *awsKey)
	*awsSecret = kubeSecret("/etc/secrets/aws-secret", *awsSecret)
	*awsRegion = kubeSecret("/etc/secrets/aws-region", *awsRegion)
	*githubClientID = kubeSecret("/etc/secrets/github-client-id", *githubClientID)
	*githubClientSecret = kubeSecret("/etc/secrets/github-client-secret", *githubClientSecret)
}

func writeCorsHeaders(w http.ResponseWriter) {
	w.Header().Set("Access-Control-Allow-Origin", "http://localhost:9000")
	w.Header().Set("Access-Control-Allow-Headers", "DECAP-APP-NAME, DECAP-API-TOKEN, Cache-Control, Pragma, Origin, Authorization, Content-Type, X-Requested-With")
	w.Header().Set("Access-Control-Allow-Methods", "GET, PUT, POST, DELETE, OPTIONS")
}

func handleOptions(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	writeCorsHeaders(w)
}

func main() {
	locker := NewDefaultLock([]string{"http://localhost:2379"})
	k8s := NewBuilder(*apiServerBaseURL, *apiServerUser, *apiServerPassword, *awsKey, *awsSecret, *awsRegion, locker, *buildScriptsRepo, *buildScriptsRepoBranch)
	awsStorageService := NewAWSStorageService(*awsKey, *awsSecret, *awsRegion)
	scmManagers := map[string]SCMClient{
		"github": NewGithubClient("https://api.github.com", *githubClientID, *githubClientSecret),
	}

	router := httprouter.New()
	router.ServeFiles("/decap/*filepath", http.Dir("./static"))
	router.GET("/api/v1/version", VersionHandler)
	router.GET("/api/v1/projects", ProjectsHandler)
	router.GET("/api/v1/projects/:team/:project/refs", ProjectRefsHandler(scmManagers))
	router.GET("/api/v1/builds/:team/:project", BuildsHandler(awsStorageService))
	router.DELETE("/api/v1/builds/:id", StopBuildHandler(k8s))
	router.POST("/api/v1/builds/:team/:project", ExecuteBuildHandler(k8s))
	router.GET("/api/v1/teams", TeamsHandler)
	router.GET("/api/v1/logs/:id", LogHandler(awsStorageService))
	router.GET("/api/v1/artifacts/:id", ArtifactsHandler(awsStorageService))
	router.POST("/api/v1/shutdown/:state", ShutdownHandler)
	router.POST("/hooks/:repomanager", HooksHandler(*buildScriptsRepo, *buildScriptsRepoBranch, k8s))
	router.Handle("OPTIONS", "/api/v1/*filepath", handleOptions)

	var err error
	atoms, err = assembleAtoms(*buildScriptsRepo, *buildScriptsRepoBranch)
	if err != nil {
		Log.Printf("Cannot clone build scripts repository: %v\n", err)
	}
	for _, v := range atoms {
		Log.Printf("Project: %+v\n", v)
	}

	if !*noWebsocket {
		go k8s.Websock()
	}

	corsWrapper := func(handler http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			writeCorsHeaders(w)
			handler.ServeHTTP(w, r)
		})
	}

	Log.Println("decap ready on port 9090...")
	http.ListenAndServe(":9090", corsWrapper(router))
}
<commit_msg>wire in deferred runner<commit_after>package main

import (
	"flag"
	"log"
	"net/http"
	"os"

	"github.com/julienschmidt/httprouter"
)

var (
	apiServerBaseURL       = flag.String("api-server-base-url", "https://kubernetes.default", "Kubernetes API server base URL")
	apiServerUser          = flag.String("api-server-username", "admin", "Kubernetes API server username to use if no service acccount API token is present.")
	apiServerPassword      = flag.String("api-server-password", "admin123", "Kubernetes API server password to use if no service acccount API token is present.")
	awsKey                 = flag.String("aws-access-key", "", "Default decap AWS access key.  /etc/secrets/aws-key in the cluster overrides this.")
	awsSecret              = flag.String("aws-secret-key", "", "Default decap AWS access secret.  /etc/secrets/aws-secret in the cluster overrides this.")
	awsRegion              = flag.String("aws-region", "us-west-1", "Default decap AWS region.  /etc/secrets/aws-region in the cluster overrides this.")
	githubClientID         = flag.String("github-client-id", "", "Default Github ClientID for quering Github repos.  /etc/secrets/github-client-id in the cluster overrides this.")
	githubClientSecret     = flag.String("github-client-secret", "", "Default Github Client Secret for quering Github repos.  /etc/secrets/github-client-secret in the cluster overrides this.")
	buildScriptsRepo       = flag.String("build-scripts-repo", "https://github.com/ae6rt/decap-build-scripts.git", "Git repo where userland build scripts are held.")
	buildScriptsRepoBranch = flag.String("build-scripts-repo-branch", "master", "Branch or revision to use on git repo where userland build scripts are held.")
	noWebsocket            = flag.Bool("no-websocket", false, "Do not start websocket client that watches pods.")
	versionFlag            = flag.Bool("version", false, "Print version info and exit.")

	Log *log.Logger = log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile)

	buildVersion string
	buildCommit  string
	buildDate    string
	buildGoSDK   string
)

func init() {
	flag.Parse()
	Log.Printf("Version: %s, Commit: %s, Date: %s, Go SDK: %s\n", buildVersion, buildCommit, buildDate, buildGoSDK)
	if *versionFlag {
		os.Exit(0)
	}

	*awsKey = kubeSecret("/etc/secrets/aws-key", *awsKey)
	*awsSecret = kubeSecret("/etc/secrets/aws-secret", *awsSecret)
	*awsRegion = kubeSecret("/etc/secrets/aws-region", *awsRegion)
	*githubClientID = kubeSecret("/etc/secrets/github-client-id", *githubClientID)
	*githubClientSecret = kubeSecret("/etc/secrets/github-client-secret", *githubClientSecret)
}

func writeCorsHeaders(w http.ResponseWriter) {
	w.Header().Set("Access-Control-Allow-Origin", "http://localhost:9000")
	w.Header().Set("Access-Control-Allow-Headers", "DECAP-APP-NAME, DECAP-API-TOKEN, Cache-Control, Pragma, Origin, Authorization, Content-Type, X-Requested-With")
	w.Header().Set("Access-Control-Allow-Methods", "GET, PUT, POST, DELETE, OPTIONS")
}

func handleOptions(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	writeCorsHeaders(w)
}

func main() {
	locker := NewDefaultLock([]string{"http://localhost:2379"})
	buildLauncher := NewBuilder(*apiServerBaseURL, *apiServerUser, *apiServerPassword, *awsKey, *awsSecret, *awsRegion, locker, *buildScriptsRepo, *buildScriptsRepoBranch)
	awsStorageService := NewAWSStorageService(*awsKey, *awsSecret, *awsRegion)
	scmManagers := map[string]SCMClient{
		"github": NewGithubClient("https://api.github.com", *githubClientID, *githubClientSecret),
	}

	router := httprouter.New()
	router.ServeFiles("/decap/*filepath", http.Dir("./static"))
	router.GET("/api/v1/version", VersionHandler)
	router.GET("/api/v1/projects", ProjectsHandler)
	router.GET("/api/v1/projects/:team/:project/refs", ProjectRefsHandler(scmManagers))
	router.GET("/api/v1/builds/:team/:project", BuildsHandler(awsStorageService))
	router.DELETE("/api/v1/builds/:id", StopBuildHandler(buildLauncher))
	router.POST("/api/v1/builds/:team/:project", ExecuteBuildHandler(buildLauncher))
	router.GET("/api/v1/teams", TeamsHandler)
	router.GET("/api/v1/logs/:id", LogHandler(awsStorageService))
	router.GET("/api/v1/artifacts/:id", ArtifactsHandler(awsStorageService))
	router.POST("/api/v1/shutdown/:state", ShutdownHandler)
	router.POST("/hooks/:repomanager", HooksHandler(*buildScriptsRepo, *buildScriptsRepoBranch, buildLauncher))
	router.Handle("OPTIONS", "/api/v1/*filepath", handleOptions)

	var err error
	atoms, err = assembleAtoms(*buildScriptsRepo, *buildScriptsRepoBranch)
	if err != nil {
		Log.Printf("Cannot clone build scripts repository: %v\n", err)
	}
	for _, v := range atoms {
		Log.Printf("Project: %+v\n", v)
	}

	if !*noWebsocket {
		go buildLauncher.Websock()
	}

	go buildLauncher.LaunchDeferred()

	corsWrapper := func(handler http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			writeCorsHeaders(w)
			handler.ServeHTTP(w, r)
		})
	}

	Log.Println("decap ready on port 9090...")
	http.ListenAndServe(":9090", corsWrapper(router))
}
<|endoftext|>


<commit_before>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package pkix

import (
	"encoding/asn1"
	"encoding/json"
	"fmt"
	"strconv"
	"strings"
)

// AuxOID behaves similar to asn1.ObjectIdentifier, except encodes to JSON as a
// string in dot notation. It is a type synonym for []int, and can be converted
// to an asn1.ObjectIdentifier by going through []int and back.
type AuxOID []int

// AsSlice returns a slice over the inner-representation
func (aux *AuxOID) AsSlice() []int {
	return *aux
}

// CopyAsSlice returns a copy of the inter-representation as a slice
func (aux *AuxOID) CopyAsSlice() []int {
	out := make([]int, len(*aux))
	copy(out, *aux)
	return out
}

// Equal tests (deep) equality of two AuxOIDs
func (aux *AuxOID) Equal(other *AuxOID) bool {
	var a []int = *aux
	var b []int = *other
	if len(a) != len(b) {
		return false
	}
	for idx := range a {
		if a[idx] != b[idx] {
			return false
		}
	}
	return true
}

// MarshalJSON implements the json.Marshaler interface
func (aux *AuxOID) MarshalJSON() ([]byte, error) {
	var oid asn1.ObjectIdentifier
	oid = []int(*aux)
	return json.Marshal(oid.String())
}

// UnmarshalJSON implements the json.Unmarshaler interface
func (aux *AuxOID) UnmarshalJSON(b []byte) error {
	var s string
	if err := json.Unmarshal(b, &s); err != nil {
		return err
	}
	parts := strings.Split(s, ".")
	if len(parts) < 1 {
		return fmt.Errorf("Invalid OID string %s", s)
	}
	slice := make([]int, len(parts))
	for idx := range parts {
		n, err := strconv.Atoi(parts[idx])
		if err != nil || n < 0 {
			return fmt.Errorf("Invalid OID integer %s", parts[idx])
		}
		slice[idx] = n
	}
	*aux = slice
	return nil
}
<commit_msg>Fix copyright year<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package pkix

import (
	"encoding/asn1"
	"encoding/json"
	"fmt"
	"strconv"
	"strings"
)

// AuxOID behaves similar to asn1.ObjectIdentifier, except encodes to JSON as a
// string in dot notation. It is a type synonym for []int, and can be converted
// to an asn1.ObjectIdentifier by going through []int and back.
type AuxOID []int

// AsSlice returns a slice over the inner-representation
func (aux *AuxOID) AsSlice() []int {
	return *aux
}

// CopyAsSlice returns a copy of the inter-representation as a slice
func (aux *AuxOID) CopyAsSlice() []int {
	out := make([]int, len(*aux))
	copy(out, *aux)
	return out
}

// Equal tests (deep) equality of two AuxOIDs
func (aux *AuxOID) Equal(other *AuxOID) bool {
	var a []int = *aux
	var b []int = *other
	if len(a) != len(b) {
		return false
	}
	for idx := range a {
		if a[idx] != b[idx] {
			return false
		}
	}
	return true
}

// MarshalJSON implements the json.Marshaler interface
func (aux *AuxOID) MarshalJSON() ([]byte, error) {
	var oid asn1.ObjectIdentifier
	oid = []int(*aux)
	return json.Marshal(oid.String())
}

// UnmarshalJSON implements the json.Unmarshaler interface
func (aux *AuxOID) UnmarshalJSON(b []byte) error {
	var s string
	if err := json.Unmarshal(b, &s); err != nil {
		return err
	}
	parts := strings.Split(s, ".")
	if len(parts) < 1 {
		return fmt.Errorf("Invalid OID string %s", s)
	}
	slice := make([]int, len(parts))
	for idx := range parts {
		n, err := strconv.Atoi(parts[idx])
		if err != nil || n < 0 {
			return fmt.Errorf("Invalid OID integer %s", parts[idx])
		}
		slice[idx] = n
	}
	*aux = slice
	return nil
}
<|endoftext|>


<commit_before>package channeldb

import (
	"bytes"

	"github.com/boltdb/bolt"
	"github.com/roasbeef/btcd/wire"
)

// deliveryScriptBugMigration is a database migration that patches an incorrect
// version of the database due to a typo in the key when fetching, putting,
// deleting the delivery scripts for a channel. As of database version 1, the
// querying logic expects delivery scripts to be in the proper place within the
// node's channel schema. This migration fixes the issue in the older version
// of the database so channels can properly be read from the database.
func deliveryScriptBugMigration(tx *bolt.Tx) error {
	// Get the bucket dedicated to storing the metadata for open channels.
	openChanBucket := tx.Bucket(openChannelBucket)
	if openChanBucket == nil {
		return nil
	}

	// Next, fetch the bucket dedicated to storing metadata related to all
	// nodes. All keys within this bucket are the serialized public keys of
	// all our direct counterparties.
	nodeMetaBucket := tx.Bucket(nodeInfoBucket)
	if nodeMetaBucket == nil {
		return nil
	}

	log.Infof("Migration database from legacy channel delivery script " +
		"schema")

	// Finally for each node public key in the bucket, fetch all the
	// channels related to this particular node.
	return nodeMetaBucket.ForEach(func(k, v []byte) error {
		// Within the meta node meta bucket, each key is the node's
		// serialized public key. Knowing this key allows us to fetch
		// the node's open channel bucket.
		nodeChanBucket := openChanBucket.Bucket(k)
		if nodeChanBucket == nil {
			return nil
		}

		// Once we have the open channel bucket for this particular
		// node, we then access another sub-bucket which is an index
		// that stores the channel points (chanID's) for all active
		// channels with the node.
		nodeChanIDBucket := nodeChanBucket.Bucket(chanIDBucket[:])
		if nodeChanIDBucket == nil {
			return nil
		}
		err := nodeChanIDBucket.ForEach(func(k, v []byte) error {
			if k == nil {
				return nil
			}

			// The old delivery script key within a node's channel
			// bucket for each channel was just the prefix key
			// itself. So we'll check if this key stores any data,
			// if not, then we don't need to migrate this channel.
			oldDeliverykey := deliveryScriptsKey
			deliveryScripts := nodeChanBucket.Get(oldDeliverykey)
			if deliveryScripts == nil {
				return nil
			}

			// Decode the stored outpoint so we can log our
			// progress.
			outBytes := bytes.NewReader(k)
			chanID := &wire.OutPoint{}
			if err := readOutpoint(outBytes, chanID); err != nil {
				return err
			}

			log.Debugf("Migration delivery scripts of "+
				"ChannelPoint(%v)", chanID)

			// Next we manually construct the _proper_ key which
			// uses the key prefix in conjunction with the chanID
			// to create the final key.
			deliveryKey := make([]byte, len(deliveryScriptsKey)+len(k))
			copy(deliveryKey[:3], deliveryScriptsKey)
			copy(deliveryKey[3:], k)

			// To complete the migration for this channel, we now
			// store the delivery scripts in their proper place.
			return nodeChanBucket.Put(deliveryKey, deliveryScripts)
		})
		if err != nil {
			return err
		}

		// Before we conclude, we'll also delete value of the incorrect
		// delivery storage.
		return nodeChanBucket.Delete(deliveryScriptsKey)
	})
}
<commit_msg>channeldb: remove prior migrations as their no longer needed<commit_after>package channeldb
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"net/url"
	"os"
	"sync"
	"sync/atomic"
	"time"
)

func main() {
	context, _ := prepareContext()
	parallelity := 200 // set well below the typical ulimit of 1024
	// to avoid "socket: too many open files".
	// Also fits AWS API limits, avoid "503 SlowDown: Please reduce your request rate."

	names := make(chan string)

	context.wg.Add(parallelity)
	for gr := 1; gr <= parallelity; gr++ {
		go cpworker(&context, names)
	}

	input := bufio.NewScanner(os.Stdin)
	for input.Scan() {
		names <- input.Text()
	}
	close(names)

	context.wg.Wait()
	fmt.Printf("\nDone.\n")
}

type CopyContext struct {
	s3svc      *s3.S3
	bucketname string
	newvalue   string

	copiedObjects   int64
	copiedBytes     int64
	expectedObjects int64
	start           time.Time
	statusLineMutex sync.Mutex
	lastStatusShown float64

	wg sync.WaitGroup
}

func prepareContext() (CopyContext, error) {
	// Session with the new library
	sess, err := session.NewSession(&aws.Config{
		Region: aws.String("eu-central-1")},
	)
	if err != nil {
		panic(fmt.Sprintf("Can not create AWS SDK session %s", err))
	}

	if len(os.Args) != 3 {
		panic("Please provide bucket name and desired Cache-Control setting")
	}
	return CopyContext{
		s3svc:           s3.New(sess),
		bucketname:      os.Args[1],
		expectedObjects: 3867874,
		newvalue:        os.Args[2],
		start:           time.Now(),
	}, nil
}

func cpworker(context *CopyContext, names <-chan string) {
	for {
		name, more := <-names
		if more {
			// fmt.Printf("Starting copy %v\n", name)
			if err := cp(context, name); err != nil {
				os.Stderr.WriteString(fmt.Sprintf("==> Failed processing '%s': %v\n", name, err))
				filename := "error_keys.txt"
				os.Stderr.WriteString(fmt.Sprintf("Adding name to '%s' for later processing or reference", filename))
				f, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0666)
				if err != nil {
					os.Stderr.WriteString(fmt.Sprintf("Could not write to 'TODO' file '%s'. Error: %v\n", filename, err))
					panic(err)
				}
				fmt.Fprintln(f, name)
				f.Close()
			}
		} else {
			// fmt.Printf("\nNo more data in names channel.\n")
			context.wg.Done()
			return
		}
	}
}

func cp(context *CopyContext, name string) error {
	//fmt.Println(context.bucketname)
	//fmt.Println(url.PathEscape(name))
	headin := s3.HeadObjectInput{
		Bucket: aws.String(context.bucketname),
		Key:    aws.String(name),
		//		Key:    aws.String(url.PathEscape(name)),
	}
	headresp, err := context.s3svc.HeadObject(&headin)
	if err != nil {
		return fmt.Errorf("aws sdk Head failed: %v", err)
	}

	contenttype := headresp.ContentType
	oldcachecontrol := headresp.CacheControl

	var status string
	// . - skip
	// X - type was not set, set to image/png
	// j - was image/jpeg; adjusted CacheControl
	// g - was image/png; adjusted CacheControl
	// P - pdf file; adjusted CacheControl
	// Y - other file type; adjusted CacheControl
	if oldcachecontrol != nil && *oldcachecontrol == context.newvalue {
		status = "."
	} else {
		if contenttype == nil {
			status = "X"
			contenttype = aws.String("image/png")
		} else {
			// contenttype = contenttypes[0] // theoretically, there can be multiple HTTP headers with the same key
			// but lets assume, there is at most one
			if *contenttype == "image/png" {
				status = "g"
			} else if *contenttype == "image/jpeg" {
				status = "j"
			} else if *contenttype == "application/pdf" {
				status = "P"
			} else {
				status = "Y"
			}
		}

		src := fmt.Sprintf("%s/%s", context.bucketname, url.PathEscape(name))
		inp := s3.CopyObjectInput{
			Bucket:            aws.String(context.bucketname),
			CopySource:        &src,
			Key:               &name,
			CacheControl:      &context.newvalue,
			ContentType:       contenttype,
			MetadataDirective: aws.String("REPLACE"),
		}
		_, err := context.s3svc.CopyObject(&inp)
		if err != nil {
			return fmt.Errorf("Failed changing (inplace-copying) object: %v", err)
		}
	}
	fmt.Print(status)

	atomic.AddInt64(&context.copiedObjects, 1)
	sec := time.Since(context.start).Seconds()
	o_s := float64(context.copiedObjects) / sec
	expectedDuration := time.Duration(int(float64(context.expectedObjects-context.copiedObjects)/o_s)) * time.Second
	days := int(expectedDuration.Hours() / 24)
	andHours := expectedDuration.Hours() - float64(days)*24
	eta := fmt.Sprintf("%dd %.1fh", days, andHours)
	if days == 0 {
		hours := int(expectedDuration.Minutes() / 60)
		andMinutes := expectedDuration.Minutes() - float64(hours)*60
		eta = fmt.Sprintf("%dh %.1fm", hours, andMinutes)
	}
	if context.expectedObjects < context.copiedObjects {
		eta = "-"
	}

	const everySeconds = 10
	show := false
	context.statusLineMutex.Lock()
	if context.lastStatusShown < sec-everySeconds {
		show = true
		context.lastStatusShown = sec
	}
	context.statusLineMutex.Unlock()

	if show {
		fmt.Printf("\n%-30s Totals: %d/%d objects. Avg: %.2f obj/s. ETA: %v    \n",
			name, context.copiedObjects, context.expectedObjects, o_s, eta,
		)
	}
	return nil
}
<commit_msg>Add godoc, try command vs. package docs<commit_after>/* put-cache-control copies objects in S3 bucket in-place, modifying the meta data, e.g. Cache-Control headers */
package main

import (
	"bufio"
	"fmt"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"net/url"
	"os"
	"sync"
	"sync/atomic"
	"time"
)

func main() {
	context, _ := prepareContext()
	parallelity := 200 // set well below the typical ulimit of 1024
	// to avoid "socket: too many open files".
	// Also fits AWS API limits, avoid "503 SlowDown: Please reduce your request rate."

	names := make(chan string)

	context.wg.Add(parallelity)
	for gr := 1; gr <= parallelity; gr++ {
		go cpworker(&context, names)
	}

	input := bufio.NewScanner(os.Stdin)
	for input.Scan() {
		names <- input.Text()
	}
	close(names)

	context.wg.Wait()
	fmt.Printf("\nDone.\n")
}

/* Hello world */
func CheckPublicCommentTmp() {
}

/* CopyContext defines context for running concurrent copy operations and remembers the progress */
type CopyContext struct {
	s3svc      *s3.S3
	bucketname string
	newvalue   string

	copiedObjects   int64
	copiedBytes     int64
	expectedObjects int64
	start           time.Time
	statusLineMutex sync.Mutex
	lastStatusShown float64

	wg sync.WaitGroup
}

func prepareContext() (CopyContext, error) {
	// Session with the new library
	sess, err := session.NewSession(&aws.Config{
		Region: aws.String("eu-central-1")},
	)
	if err != nil {
		panic(fmt.Sprintf("Can not create AWS SDK session %s", err))
	}

	if len(os.Args) != 3 {
		panic("Please provide bucket name and desired Cache-Control setting")
	}
	return CopyContext{
		s3svc:           s3.New(sess),
		bucketname:      os.Args[1],
		expectedObjects: 3867874,
		newvalue:        os.Args[2],
		start:           time.Now(),
	}, nil
}

func cpworker(context *CopyContext, names <-chan string) {
	for {
		name, more := <-names
		if more {
			// fmt.Printf("Starting copy %v\n", name)
			if err := cp(context, name); err != nil {
				os.Stderr.WriteString(fmt.Sprintf("==> Failed processing '%s': %v\n", name, err))
				filename := "error_keys.txt"
				os.Stderr.WriteString(fmt.Sprintf("Adding name to '%s' for later processing or reference", filename))
				f, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0666)
				if err != nil {
					os.Stderr.WriteString(fmt.Sprintf("Could not write to 'TODO' file '%s'. Error: %v\n", filename, err))
					panic(err)
				}
				fmt.Fprintln(f, name)
				f.Close()
			}
		} else {
			// fmt.Printf("\nNo more data in names channel.\n")
			context.wg.Done()
			return
		}
	}
}

func cp(context *CopyContext, name string) error {
	//fmt.Println(context.bucketname)
	//fmt.Println(url.PathEscape(name))
	headin := s3.HeadObjectInput{
		Bucket: aws.String(context.bucketname),
		Key:    aws.String(name),
		//		Key:    aws.String(url.PathEscape(name)),
	}
	headresp, err := context.s3svc.HeadObject(&headin)
	if err != nil {
		return fmt.Errorf("aws sdk Head failed: %v", err)
	}

	contenttype := headresp.ContentType
	oldcachecontrol := headresp.CacheControl

	var status string
	// . - skip
	// X - type was not set, set to image/png
	// j - was image/jpeg; adjusted CacheControl
	// g - was image/png; adjusted CacheControl
	// P - pdf file; adjusted CacheControl
	// Y - other file type; adjusted CacheControl
	if oldcachecontrol != nil && *oldcachecontrol == context.newvalue {
		status = "."
	} else {
		if contenttype == nil {
			status = "X"
			contenttype = aws.String("image/png")
		} else {
			// contenttype = contenttypes[0] // theoretically, there can be multiple HTTP headers with the same key
			// but lets assume, there is at most one
			if *contenttype == "image/png" {
				status = "g"
			} else if *contenttype == "image/jpeg" {
				status = "j"
			} else if *contenttype == "application/pdf" {
				status = "P"
			} else {
				status = "Y"
			}
		}

		src := fmt.Sprintf("%s/%s", context.bucketname, url.PathEscape(name))
		inp := s3.CopyObjectInput{
			Bucket:            aws.String(context.bucketname),
			CopySource:        &src,
			Key:               &name,
			CacheControl:      &context.newvalue,
			ContentType:       contenttype,
			MetadataDirective: aws.String("REPLACE"),
		}
		_, err := context.s3svc.CopyObject(&inp)
		if err != nil {
			return fmt.Errorf("Failed changing (inplace-copying) object: %v", err)
		}
	}
	fmt.Print(status)

	atomic.AddInt64(&context.copiedObjects, 1)
	sec := time.Since(context.start).Seconds()
	o_s := float64(context.copiedObjects) / sec
	expectedDuration := time.Duration(int(float64(context.expectedObjects-context.copiedObjects)/o_s)) * time.Second
	days := int(expectedDuration.Hours() / 24)
	andHours := expectedDuration.Hours() - float64(days)*24
	eta := fmt.Sprintf("%dd %.1fh", days, andHours)
	if days == 0 {
		hours := int(expectedDuration.Minutes() / 60)
		andMinutes := expectedDuration.Minutes() - float64(hours)*60
		eta = fmt.Sprintf("%dh %.1fm", hours, andMinutes)
	}
	if context.expectedObjects < context.copiedObjects {
		eta = "-"
	}

	const everySeconds = 10
	show := false
	context.statusLineMutex.Lock()
	if context.lastStatusShown < sec-everySeconds {
		show = true
		context.lastStatusShown = sec
	}
	context.statusLineMutex.Unlock()

	if show {
		fmt.Printf("\n%-30s Totals: %d/%d objects. Avg: %.2f obj/s. ETA: %v    \n",
			name, context.copiedObjects, context.expectedObjects, o_s, eta,
		)
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package isolate

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"

	"go.chromium.org/luci/client/archiver/pipeline"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/flag/stringmapflag"
	"go.chromium.org/luci/common/isolated"
	"go.chromium.org/luci/common/isolatedclient"
)

// IsolatedGenJSONVersion is used in the batcharchive json format.
//
// TODO(tandrii): Migrate to batch_archive.go.
const IsolatedGenJSONVersion = 1

// ValidVariable is the regexp of valid isolate variable name.
const ValidVariable = "[A-Za-z_][A-Za-z_0-9]*"

var validVariableMatcher = regexp.MustCompile(ValidVariable)
var variableSubstitutionMatcher = regexp.MustCompile("<\\(" + ValidVariable + "\\)")

// IsValidVariable returns true if the variable is a valid symbol name.
func IsValidVariable(variable string) bool {
	return validVariableMatcher.MatchString(variable)
}

// ArchiveOptions for archiving trees.
type ArchiveOptions struct {
	Isolate             string              `json:"isolate"`
	Isolated            string              `json:"isolated"`
	IgnoredPathFilterRe string              `json:"ignored_path_filter_re"`
	PathVariables       stringmapflag.Value `json:"path_variables"`
	ConfigVariables     stringmapflag.Value `json:"config_variables"`
	AllowMissingFileDir bool                `json:"allow_missing_file_dir"`
}

// Init initializes with non-nil values.
func (a *ArchiveOptions) Init() {
	a.PathVariables = map[string]string{}
	if runtime.GOOS == "windows" {
		a.PathVariables["EXECUTABLE_SUFFIX"] = ".exe"
	} else {
		a.PathVariables["EXECUTABLE_SUFFIX"] = ""
	}
	a.ConfigVariables = map[string]string{}
}

func genExtensionsRegex(exts ...string) string {
	if len(exts) == 0 {
		return ""
	}
	res := make([]string, len(exts))
	for i, e := range exts {
		res[i] = `(\.` + e + `)`
	}
	return "((" + strings.Join(res, "|") + ")$)"
}

func genDirectoriesRegex(dirs ...string) string {
	if len(dirs) == 0 {
		return ""
	}
	res := make([]string, len(dirs))
	for i, d := range dirs {
		res[i] = "(" + d + ")"
	}
	// #Backslashes: https://stackoverflow.com/a/4025505/12003165
	return `((^|[\\/])(` + strings.Join(res, "|") + `)([\\/]|$))`
}

// PostProcess post-processes the flags to fix any compatibility issue.
func (a *ArchiveOptions) PostProcess(cwd string) {
	if a.IgnoredPathFilterRe == "" {
		// Set default ignored paths regexp
		// .swp are vim files
		a.IgnoredPathFilterRe = genExtensionsRegex("pyc", "swp") + "|" + genDirectoriesRegex(`\.git`, `\.hg`, `\.svn`)
	}
	if !filepath.IsAbs(a.Isolate) {
		a.Isolate = filepath.Join(cwd, a.Isolate)
	}
	a.Isolate = filepath.Clean(a.Isolate)

	if !filepath.IsAbs(a.Isolated) {
		a.Isolated = filepath.Join(cwd, a.Isolated)
	}
	a.Isolated = filepath.Clean(a.Isolated)

	for k, v := range a.PathVariables {
		// This is due to a Windows + GYP specific issue, where double-quoted paths
		// would get mangled in a way that cannot be resolved unless a space is
		// injected.
		a.PathVariables[k] = strings.TrimSpace(v)
	}
}

// ReplaceVariables replaces any occurrences of '<(FOO)' in 'str' with the
// corresponding variable from 'opts'.
//
// If any substitution refers to a variable that is missing, the returned error will
// refer to the first such variable. In the case of errors, the returned string will
// still contain a valid result for any non-missing substitutions.
func ReplaceVariables(str string, opts *ArchiveOptions) (string, error) {
	var err error
	subst := variableSubstitutionMatcher.ReplaceAllStringFunc(str,
		func(match string) string {
			varName := match[2 : len(match)-1]
			if v, ok := opts.PathVariables[varName]; ok {
				return v
			}
			if v, ok := opts.ConfigVariables[varName]; ok {
				return v
			}
			if err == nil {
				err = errors.New("no value for variable '" + varName + "'")
			}
			return match
		})
	return subst, err
}

// Archive processes a .isolate, generates a .isolated and archive it.
// Returns a *PendingItem to the .isolated.
func Archive(arch *pipeline.Archiver, opts *ArchiveOptions) *pipeline.PendingItem {
	displayName := filepath.Base(opts.Isolated)
	f, err := archive(arch, opts, displayName)
	if err != nil {
		i := &pipeline.PendingItem{DisplayName: displayName}
		i.SetErr(err)
		return i
	}
	return f
}

func processDependencies(deps []string, isolateDir string, opts *ArchiveOptions) ([]string, string, error) {
	// Expand variables in the deps, and convert each path to an absolute form.
	for i := range deps {
		dep, err := ReplaceVariables(deps[i], opts)
		if err != nil {
			return nil, "", err
		}
		deps[i] = filepath.Join(isolateDir, dep)
	}

	// Find the root directory of all the files (the root might be above isolateDir).
	rootDir := isolateDir
	resultDeps := make([]string, 0, len(deps))
	for _, dep := range deps {
		// Check if the dep is outside isolateDir.
		info, err := os.Stat(dep)
		if err != nil {
			if !opts.AllowMissingFileDir {
				return nil, "", err
			}
			log.Printf("Ignore missing dep: %s, err: %v", dep, err)
			continue
		}
		base := filepath.Dir(dep)
		if info.IsDir() {
			base = dep
			// Downstream expects the dependency of a directory to always end
			// with '/', but filepath.Join() removes that, so we add it back.
			dep += osPathSeparator
		}
		resultDeps = append(resultDeps, dep)
		for {
			rel, err := filepath.Rel(rootDir, base)
			if err != nil {
				return nil, "", err
			}
			if !strings.HasPrefix(rel, "..") {
				break
			}
			newRootDir := filepath.Dir(rootDir)
			if newRootDir == rootDir {
				return nil, "", errors.New("failed to find root dir")
			}
			rootDir = newRootDir
		}
	}
	if rootDir != isolateDir {
		log.Printf("Root: %s", rootDir)
	}
	return resultDeps, rootDir, nil
}

// ProcessIsolate parses an isolate file, returning the list of dependencies
// (both files and directories), the root directory and the initial Isolated struct.
func ProcessIsolate(opts *ArchiveOptions) ([]string, string, *isolated.Isolated, error) {
	content, err := ioutil.ReadFile(opts.Isolate)
	if err != nil {
		return nil, "", nil, errors.Annotate(err, "failed to read file: %s", opts.Isolate).Err()
	}
	deps, isolateDir, err := LoadIsolateForConfig(filepath.Dir(opts.Isolate), content, opts.ConfigVariables)
	if err != nil {
		return nil, "", nil, err
	}

	deps, rootDir, err := processDependencies(deps, isolateDir, opts)
	if err != nil {
		return nil, "", nil, err
	}
	// Prepare the .isolated struct.
	isol := &isolated.Isolated{
		Algo:    "sha-1",
		Files:   map[string]isolated.File{},
		Version: isolated.IsolatedFormatVersion,
	}
	return deps, rootDir, isol, nil
}

// ProcessIsolateForCAS works similarly to ProcessIsolate. However, it is
// simpler in that it returns a list of dependency *relative* paths and the
// root directory, which are the necessary input to upload to RBE-CAS.
func ProcessIsolateForCAS(opts *ArchiveOptions) ([]string, string, error) {
	content, err := ioutil.ReadFile(opts.Isolate)
	if err != nil {
		return nil, "", err
	}
	deps, isolateDir, err := LoadIsolateForConfig(filepath.Dir(opts.Isolate), content, opts.ConfigVariables)
	if err != nil {
		return nil, "", err
	}

	deps, rootDir, err := processDependencies(deps, isolateDir, opts)
	if err != nil {
		return nil, "", err
	}
	relDeps := make([]string, len(deps))
	for i, dep := range deps {
		rel, err := filepath.Rel(rootDir, dep)
		if err != nil {
			return nil, "", err
		}
		if strings.HasSuffix(dep, osPathSeparator) && !strings.HasSuffix(rel, osPathSeparator) {
			// Make it consistent with the isolated format such that directory paths must end with osPathSeparator.
			rel += osPathSeparator
		}
		relDeps[i] = rel
	}
	return relDeps, rootDir, err
}

func archive(arch *pipeline.Archiver, opts *ArchiveOptions, displayName string) (*pipeline.PendingItem, error) {
	deps, rootDir, i, err := ProcessIsolate(opts)
	if err != nil {
		return nil, err
	}
	// Handle each dependency, either a file or a directory.
	var fileItems []*pipeline.PendingItem
	for _, dep := range deps {
		relPath, err := filepath.Rel(rootDir, dep)
		if err != nil {
			return nil, err
		}
		// Grab the stats right away; this can be used for both checking whether
		// it's a directory and checking whether it's a link.
		info, err := os.Lstat(dep)
		if err != nil {
			return nil, err
		}
		if mode := info.Mode(); mode.IsDir() {
			if relPath, err = filepath.Rel(rootDir, dep); err != nil {
				return nil, err
			}
			dirFItems, dirSymItems, err := pipeline.PushDirectory(arch, dep, relPath)
			if err != nil {
				return nil, err
			}
			for pending, item := range dirFItems {
				i.Files[item.RelPath] = isolated.BasicFile("", int(item.Info.Mode()), item.Info.Size())
				fileItems = append(fileItems, pending)
			}
			for relPath, dstPath := range dirSymItems {
				i.Files[relPath] = isolated.SymLink(dstPath)
			}
		} else {
			if mode&os.ModeSymlink == os.ModeSymlink {
				l, err := os.Readlink(dep)
				if err != nil {
					// Kill the process: there's no reason to continue if a file is
					// unavailable.
					log.Fatalf("Unable to stat %q: %v", dep, err)
				}
				i.Files[relPath] = isolated.SymLink(l)
			} else {
				i.Files[relPath] = isolated.BasicFile("", int(mode.Perm()), info.Size())
				fileItems = append(fileItems, arch.PushFile(relPath, dep, -info.Size()))
			}
		}
	}

	for _, item := range fileItems {
		item.WaitForHashed()
		if err = item.Error(); err != nil {
			return nil, err
		}
		f := i.Files[item.DisplayName]
		f.Digest = item.Digest()
		i.Files[item.DisplayName] = f
	}

	raw := &bytes.Buffer{}
	if err = json.NewEncoder(raw).Encode(i); err != nil {
		return nil, err
	}

	if err := ioutil.WriteFile(opts.Isolated, raw.Bytes(), 0644); err != nil {
		return nil, err
	}
	return arch.Push(displayName, isolatedclient.NewBytesSource(raw.Bytes()), 0), nil
}
<commit_msg>isolate: wrap some error<commit_after>// Copyright 2015 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package isolate

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"

	"go.chromium.org/luci/client/archiver/pipeline"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/flag/stringmapflag"
	"go.chromium.org/luci/common/isolated"
	"go.chromium.org/luci/common/isolatedclient"
)

// IsolatedGenJSONVersion is used in the batcharchive json format.
//
// TODO(tandrii): Migrate to batch_archive.go.
const IsolatedGenJSONVersion = 1

// ValidVariable is the regexp of valid isolate variable name.
const ValidVariable = "[A-Za-z_][A-Za-z_0-9]*"

var validVariableMatcher = regexp.MustCompile(ValidVariable)
var variableSubstitutionMatcher = regexp.MustCompile("<\\(" + ValidVariable + "\\)")

// IsValidVariable returns true if the variable is a valid symbol name.
func IsValidVariable(variable string) bool {
	return validVariableMatcher.MatchString(variable)
}

// ArchiveOptions for archiving trees.
type ArchiveOptions struct {
	Isolate             string              `json:"isolate"`
	Isolated            string              `json:"isolated"`
	IgnoredPathFilterRe string              `json:"ignored_path_filter_re"`
	PathVariables       stringmapflag.Value `json:"path_variables"`
	ConfigVariables     stringmapflag.Value `json:"config_variables"`
	AllowMissingFileDir bool                `json:"allow_missing_file_dir"`
}

// Init initializes with non-nil values.
func (a *ArchiveOptions) Init() {
	a.PathVariables = map[string]string{}
	if runtime.GOOS == "windows" {
		a.PathVariables["EXECUTABLE_SUFFIX"] = ".exe"
	} else {
		a.PathVariables["EXECUTABLE_SUFFIX"] = ""
	}
	a.ConfigVariables = map[string]string{}
}

func genExtensionsRegex(exts ...string) string {
	if len(exts) == 0 {
		return ""
	}
	res := make([]string, len(exts))
	for i, e := range exts {
		res[i] = `(\.` + e + `)`
	}
	return "((" + strings.Join(res, "|") + ")$)"
}

func genDirectoriesRegex(dirs ...string) string {
	if len(dirs) == 0 {
		return ""
	}
	res := make([]string, len(dirs))
	for i, d := range dirs {
		res[i] = "(" + d + ")"
	}
	// #Backslashes: https://stackoverflow.com/a/4025505/12003165
	return `((^|[\\/])(` + strings.Join(res, "|") + `)([\\/]|$))`
}

// PostProcess post-processes the flags to fix any compatibility issue.
func (a *ArchiveOptions) PostProcess(cwd string) {
	if a.IgnoredPathFilterRe == "" {
		// Set default ignored paths regexp
		// .swp are vim files
		a.IgnoredPathFilterRe = genExtensionsRegex("pyc", "swp") + "|" + genDirectoriesRegex(`\.git`, `\.hg`, `\.svn`)
	}
	if !filepath.IsAbs(a.Isolate) {
		a.Isolate = filepath.Join(cwd, a.Isolate)
	}
	a.Isolate = filepath.Clean(a.Isolate)

	if !filepath.IsAbs(a.Isolated) {
		a.Isolated = filepath.Join(cwd, a.Isolated)
	}
	a.Isolated = filepath.Clean(a.Isolated)

	for k, v := range a.PathVariables {
		// This is due to a Windows + GYP specific issue, where double-quoted paths
		// would get mangled in a way that cannot be resolved unless a space is
		// injected.
		a.PathVariables[k] = strings.TrimSpace(v)
	}
}

// ReplaceVariables replaces any occurrences of '<(FOO)' in 'str' with the
// corresponding variable from 'opts'.
//
// If any substitution refers to a variable that is missing, the returned error will
// refer to the first such variable. In the case of errors, the returned string will
// still contain a valid result for any non-missing substitutions.
func ReplaceVariables(str string, opts *ArchiveOptions) (string, error) {
	var err error
	subst := variableSubstitutionMatcher.ReplaceAllStringFunc(str,
		func(match string) string {
			varName := match[2 : len(match)-1]
			if v, ok := opts.PathVariables[varName]; ok {
				return v
			}
			if v, ok := opts.ConfigVariables[varName]; ok {
				return v
			}
			if err == nil {
				err = errors.New("no value for variable '" + varName + "'")
			}
			return match
		})
	return subst, err
}

// Archive processes a .isolate, generates a .isolated and archive it.
// Returns a *PendingItem to the .isolated.
func Archive(arch *pipeline.Archiver, opts *ArchiveOptions) *pipeline.PendingItem {
	displayName := filepath.Base(opts.Isolated)
	f, err := archive(arch, opts, displayName)
	if err != nil {
		i := &pipeline.PendingItem{DisplayName: displayName}
		i.SetErr(err)
		return i
	}
	return f
}

func processDependencies(deps []string, isolateDir string, opts *ArchiveOptions) ([]string, string, error) {
	// Expand variables in the deps, and convert each path to an absolute form.
	for i := range deps {
		dep, err := ReplaceVariables(deps[i], opts)
		if err != nil {
			return nil, "", err
		}
		deps[i] = filepath.Join(isolateDir, dep)
	}

	// Find the root directory of all the files (the root might be above isolateDir).
	rootDir := isolateDir
	resultDeps := make([]string, 0, len(deps))
	for _, dep := range deps {
		// Check if the dep is outside isolateDir.
		info, err := os.Stat(dep)
		if err != nil {
			if !opts.AllowMissingFileDir {
				return nil, "", errors.Annotate(err, "faled to call Stat for %s", dep).Err()
			}
			log.Printf("Ignore missing dep: %s, err: %v", dep, err)
			continue
		}
		base := filepath.Dir(dep)
		if info.IsDir() {
			base = dep
			// Downstream expects the dependency of a directory to always end
			// with '/', but filepath.Join() removes that, so we add it back.
			dep += osPathSeparator
		}
		resultDeps = append(resultDeps, dep)
		for {
			rel, err := filepath.Rel(rootDir, base)
			if err != nil {
				return nil, "", errors.Annotate(err, "failed to call filepath.Rel(%s, %s)", rootDir, base).Err()
			}
			if !strings.HasPrefix(rel, "..") {
				break
			}
			newRootDir := filepath.Dir(rootDir)
			if newRootDir == rootDir {
				return nil, "", errors.New("failed to find root dir")
			}
			rootDir = newRootDir
		}
	}
	if rootDir != isolateDir {
		log.Printf("Root: %s", rootDir)
	}
	return resultDeps, rootDir, nil
}

// ProcessIsolate parses an isolate file, returning the list of dependencies
// (both files and directories), the root directory and the initial Isolated struct.
func ProcessIsolate(opts *ArchiveOptions) ([]string, string, *isolated.Isolated, error) {
	content, err := ioutil.ReadFile(opts.Isolate)
	if err != nil {
		return nil, "", nil, errors.Annotate(err, "failed to read file: %s", opts.Isolate).Err()
	}
	deps, isolateDir, err := LoadIsolateForConfig(filepath.Dir(opts.Isolate), content, opts.ConfigVariables)
	if err != nil {
		return nil, "", nil, err
	}

	deps, rootDir, err := processDependencies(deps, isolateDir, opts)
	if err != nil {
		return nil, "", nil, err
	}
	// Prepare the .isolated struct.
	isol := &isolated.Isolated{
		Algo:    "sha-1",
		Files:   map[string]isolated.File{},
		Version: isolated.IsolatedFormatVersion,
	}
	return deps, rootDir, isol, nil
}

// ProcessIsolateForCAS works similarly to ProcessIsolate. However, it is
// simpler in that it returns a list of dependency *relative* paths and the
// root directory, which are the necessary input to upload to RBE-CAS.
func ProcessIsolateForCAS(opts *ArchiveOptions) ([]string, string, error) {
	content, err := ioutil.ReadFile(opts.Isolate)
	if err != nil {
		return nil, "", errors.Annotate(err, "failed to call ReadFile: %s", opts.Isolate).Err()
	}
	deps, isolateDir, err := LoadIsolateForConfig(filepath.Dir(opts.Isolate), content, opts.ConfigVariables)
	if err != nil {
		return nil, "", err
	}

	deps, rootDir, err := processDependencies(deps, isolateDir, opts)
	if err != nil {
		return nil, "", err
	}
	relDeps := make([]string, len(deps))
	for i, dep := range deps {
		rel, err := filepath.Rel(rootDir, dep)
		if err != nil {
			return nil, "", errors.Annotate(err, "failed to call filepath.Rel(%s, %s)", rootDir, dep).Err()
		}
		if strings.HasSuffix(dep, osPathSeparator) && !strings.HasSuffix(rel, osPathSeparator) {
			// Make it consistent with the isolated format such that directory paths must end with osPathSeparator.
			rel += osPathSeparator
		}
		relDeps[i] = rel
	}
	return relDeps, rootDir, nil
}

func archive(arch *pipeline.Archiver, opts *ArchiveOptions, displayName string) (*pipeline.PendingItem, error) {
	deps, rootDir, i, err := ProcessIsolate(opts)
	if err != nil {
		return nil, err
	}
	// Handle each dependency, either a file or a directory.
	var fileItems []*pipeline.PendingItem
	for _, dep := range deps {
		relPath, err := filepath.Rel(rootDir, dep)
		if err != nil {
			return nil, err
		}
		// Grab the stats right away; this can be used for both checking whether
		// it's a directory and checking whether it's a link.
		info, err := os.Lstat(dep)
		if err != nil {
			return nil, err
		}
		if mode := info.Mode(); mode.IsDir() {
			if relPath, err = filepath.Rel(rootDir, dep); err != nil {
				return nil, err
			}
			dirFItems, dirSymItems, err := pipeline.PushDirectory(arch, dep, relPath)
			if err != nil {
				return nil, err
			}
			for pending, item := range dirFItems {
				i.Files[item.RelPath] = isolated.BasicFile("", int(item.Info.Mode()), item.Info.Size())
				fileItems = append(fileItems, pending)
			}
			for relPath, dstPath := range dirSymItems {
				i.Files[relPath] = isolated.SymLink(dstPath)
			}
		} else {
			if mode&os.ModeSymlink == os.ModeSymlink {
				l, err := os.Readlink(dep)
				if err != nil {
					// Kill the process: there's no reason to continue if a file is
					// unavailable.
					log.Fatalf("Unable to stat %q: %v", dep, err)
				}
				i.Files[relPath] = isolated.SymLink(l)
			} else {
				i.Files[relPath] = isolated.BasicFile("", int(mode.Perm()), info.Size())
				fileItems = append(fileItems, arch.PushFile(relPath, dep, -info.Size()))
			}
		}
	}

	for _, item := range fileItems {
		item.WaitForHashed()
		if err = item.Error(); err != nil {
			return nil, err
		}
		f := i.Files[item.DisplayName]
		f.Digest = item.Digest()
		i.Files[item.DisplayName] = f
	}

	raw := &bytes.Buffer{}
	if err = json.NewEncoder(raw).Encode(i); err != nil {
		return nil, err
	}

	if err := ioutil.WriteFile(opts.Isolated, raw.Bytes(), 0644); err != nil {
		return nil, err
	}
	return arch.Push(displayName, isolatedclient.NewBytesSource(raw.Bytes()), 0), nil
}
<|endoftext|>


<commit_before>package checker

import (
	"bytes"
	"errors"
	"io"
	"net/url"
	"strings"

	"golang.org/x/net/html"

	"github.com/asciimoo/privacyscore/result"
)

type HTMLChecker struct{}

func (c *HTMLChecker) Check(r *result.Result) {
	if !strings.Contains(strings.ToLower(r.ContentType), "html") {
		r.AddError(errors.New("No HTML content found"))
		return
	}
	scriptTagFound := false
	forbidsReferrer := false
	hasExternalLink := false
	hasHTTPLink := false
	t := html.NewTokenizer(bytes.NewReader(r.ResponseBody))
	for {
		tagToken := t.Next()
		if tagToken == html.ErrorToken {
			if t.Err() == io.EOF {
				break
			} else {
				r.AddError(errors.New("Invalid HTML content"))
				return
			}
		}
		if tagToken != html.StartTagToken && tagToken != html.SelfClosingTagToken {
			continue
		}
		tagName, _ := t.TagName()
		switch string(tagName) {
		case "script":
			if !scriptTagFound {
				r.AddPenalty("Uses javascript", 5)
				scriptTagFound = true
			}
			src, found := getAttr(t, "src")
			if found {
				addExternalPenalty(r, src)
			}
		case "link":
			attrs := getAttrs(t)
			if _, found := attrs["href"]; found {
				addExternalPenalty(r, attrs["href"])
				break
			}
		case "img":
			src, found := getAttr(t, "src")
			if found {
				addExternalPenalty(r, src)
			}
		case "meta":
			attrs := getAttrs(t)
			if _, found := attrs["name"]; !found || attrs["name"] != "referrer" {
				break
			}
			if _, found := attrs["content"]; !found {
				break
			}
			switch strings.ToLower(attrs["content"]) {
			case "never", "none", "origin", "no-referrer":
				forbidsReferrer = true
			}
		case "a":
			src, found := getAttr(t, "href")
			if !found {
				break
			}
			u, _ := url.Parse(src)
			if (u.Scheme == "" && r.URL.Scheme != "https") || u.Scheme == "http" {
				hasHTTPLink = true
			}
			if forbidsReferrer || hasExternalLink {
				break
			}
			if r.IsNewForeignHost(u) {
				hasExternalLink = true
			}
		}
	}
	if hasExternalLink && !forbidsReferrer {
		r.AddPenalty("Has link to foreign host without HTTP referrer restrictions", 10)
	}
	if hasHTTPLink {
		r.AddPenalty("Has link to unencrypted service", 2)
	}
}

func addExternalPenalty(r *result.Result, src string) {
	u, _ := url.Parse(src)
	if r.IsNewForeignHost(u) {
		r.AddPenalty("Loads external resource from "+result.CropSubdomains(u.Host), 10)
	}
}

func getAttr(t *html.Tokenizer, name string) (string, bool) {
	for {
		attrName, attrValue, more := t.TagAttr()
		if string(attrName) == name {
			return string(attrValue), true
		}
		if !more {
			break
		}
	}
	return "", false
}

func getAttrs(t *html.Tokenizer) map[string]string {
	attrs := make(map[string]string)
	for {
		attrName, attrValue, more := t.TagAttr()
		attrs[string(attrName)] = string(attrValue)
		if !more {
			break
		}
	}
	return attrs
}
<commit_msg>[fix] stylesheet detection<commit_after>package checker

import (
	"bytes"
	"errors"
	"io"
	"net/url"
	"strings"

	"golang.org/x/net/html"

	"github.com/asciimoo/privacyscore/result"
)

type HTMLChecker struct{}

func (c *HTMLChecker) Check(r *result.Result) {
	if !strings.Contains(strings.ToLower(r.ContentType), "html") {
		r.AddError(errors.New("No HTML content found"))
		return
	}
	scriptTagFound := false
	forbidsReferrer := false
	hasExternalLink := false
	hasHTTPLink := false
	t := html.NewTokenizer(bytes.NewReader(r.ResponseBody))
	for {
		tagToken := t.Next()
		if tagToken == html.ErrorToken {
			if t.Err() == io.EOF {
				break
			} else {
				r.AddError(errors.New("Invalid HTML content"))
				return
			}
		}
		if tagToken != html.StartTagToken && tagToken != html.SelfClosingTagToken {
			continue
		}
		tagName, _ := t.TagName()
		switch string(tagName) {
		case "script":
			if !scriptTagFound {
				r.AddPenalty("Uses javascript", 5)
				scriptTagFound = true
			}
			src, found := getAttr(t, "src")
			if found {
				addExternalPenalty(r, src)
			}
		case "link":
			attrs := getAttrs(t)
			if rel, found := attrs["rel"]; !found || rel != "stylesheet" {
				break
			}
			if _, found := attrs["href"]; found {
				addExternalPenalty(r, attrs["href"])
			}
		case "img":
			src, found := getAttr(t, "src")
			if found {
				addExternalPenalty(r, src)
			}
		case "meta":
			attrs := getAttrs(t)
			if _, found := attrs["name"]; !found || attrs["name"] != "referrer" {
				break
			}
			if _, found := attrs["content"]; !found {
				break
			}
			switch strings.ToLower(attrs["content"]) {
			case "never", "none", "origin", "no-referrer":
				forbidsReferrer = true
			}
		case "a":
			src, found := getAttr(t, "href")
			if !found {
				break
			}
			u, _ := url.Parse(src)
			if (u.Scheme == "" && r.URL.Scheme != "https") || u.Scheme == "http" {
				hasHTTPLink = true
			}
			if forbidsReferrer || hasExternalLink {
				break
			}
			if r.IsNewForeignHost(u) {
				hasExternalLink = true
			}
		}
	}
	if hasExternalLink && !forbidsReferrer {
		r.AddPenalty("Has link to foreign host without HTTP referrer restrictions", 10)
	}
	if hasHTTPLink {
		r.AddPenalty("Has link to unencrypted service", 2)
	}
}

func addExternalPenalty(r *result.Result, src string) {
	u, _ := url.Parse(src)
	if r.IsNewForeignHost(u) {
		r.AddPenalty("Loads external resource from "+result.CropSubdomains(u.Host), 10)
	}
}

func getAttr(t *html.Tokenizer, name string) (string, bool) {
	for {
		attrName, attrValue, more := t.TagAttr()
		if string(attrName) == name {
			return string(attrValue), true
		}
		if !more {
			break
		}
	}
	return "", false
}

func getAttrs(t *html.Tokenizer) map[string]string {
	attrs := make(map[string]string)
	for {
		attrName, attrValue, more := t.TagAttr()
		attrs[string(attrName)] = string(attrValue)
		if !more {
			break
		}
	}
	return attrs
}
<|endoftext|>


<commit_before>package parse // import "github.com/tdewolff/parse"

import (
	"errors"
	"io"
)

// minBuf and maxBuf are the initial and maximal internal buffer size.
var minBuf = 1024
var maxBuf = 4096

// ErrBufferExceeded is returned when the internal buffer exceeds 4096 bytes, a string or comment must thus be smaller than 4kB!
var ErrBufferExceeded = errors.New("max buffer exceeded")

// ShiftBuffer is an interface for tokenisation readers, which enables peeking and shifting the buffer.
type ShiftBuffer interface {
	Read(int) byte
	Move(int)
	MoveTo(int)
	Len() int
	Bytes() []byte
	Shift() []byte
	Err() error
}

type defaultShiftBuffer struct {
	buf []byte
	pos int
	n   int
}

// Move advances the 0 position of read.
func (z *defaultShiftBuffer) Move(n int) {
	z.n += n
}

// MoveTo sets the 0 position of read.
func (z *defaultShiftBuffer) MoveTo(n int) {
	z.n = n
}

// Len returns the 0 position of read.
func (z defaultShiftBuffer) Len() int {
	return z.n
}

// Bytes returns the bytes of the current selection.
func (z defaultShiftBuffer) Bytes() []byte {
	return z.buf[z.pos : z.pos+z.n]
}

// Shift returns the bytes of the current selection and advances the position.
func (z *defaultShiftBuffer) Shift() []byte {
	b := z.buf[z.pos : z.pos+z.n]
	z.pos += z.n
	z.n = 0
	return b
}

////////////////////////////////////////////////////////////////

// ShiftBufferReader is a buffered reader for tokenisation, taking an io.Reader.
type ShiftBufferReader struct {
	defaultShiftBuffer
	r       io.Reader
	readErr error
}

// NewShiftBufferReader returns a new ShiftBufferReader.
func NewShiftBufferReader(r io.Reader) *ShiftBufferReader {
	return &ShiftBufferReader{
		defaultShiftBuffer{
			buf: make([]byte, 0, minBuf),
		},
		r,
		nil,
	}
}

// Read returns the ith byte and possible does a reallocation
func (z *ShiftBufferReader) Read(i int) byte {
	if z.pos+z.n+i >= len(z.buf) {
		if z.readErr != nil {
			return 0
		}

		// reallocate a new buffer (possibly larger)
		c := cap(z.buf)
		d := z.n + i
		var buf1 []byte
		if 2*d > c {
			if 2*c > maxBuf {
				z.readErr = ErrBufferExceeded
				return 0
			}
			buf1 = make([]byte, d, 2*c)
		} else {
			buf1 = z.buf[:d]
		}
		copy(buf1, z.buf[z.pos:z.pos+d])

		// Read in to fill the buffer till capacity
		var n int
		n, z.readErr = z.r.Read(buf1[d:cap(buf1)])
		z.pos, z.buf = 0, buf1[:d+n]
		if n == 0 {
			return 0
		}
	}
	return z.buf[z.pos+z.n+i]
}

// Err returns the error.
func (z ShiftBufferReader) Err() error {
	if z.readErr == io.EOF {
		if z.pos+z.n >= len(z.buf) {
			return io.EOF
		}
	} else if z.readErr != nil {
		return z.readErr
	}
	return nil
}

////////////////////////////////////////////////////////////////

// ShiftBufferBytes is a buffered reader for tokenisation, taking a []byte.
type ShiftBufferBytes struct {
	defaultShiftBuffer
}

// NewShiftBufferBytes returns a new ShiftBufferBytes.
func NewShiftBufferBytes(b []byte) *ShiftBufferBytes {
	return &ShiftBufferBytes{
		defaultShiftBuffer{
			buf: b,
		},
	}
}

// Read returns the ith byte and possible does a reallocation
func (z *ShiftBufferBytes) Read(i int) byte {
	if z.pos+z.n+i >= len(z.buf) {
		return 0
	}
	return z.buf[z.pos+z.n+i]
}

// Err returns the error.
func (z ShiftBufferBytes) Err() error {
	if z.pos+z.n >= len(z.buf) {
		return io.EOF
	}
	return nil
}
<commit_msg>Buffer size now const<commit_after>package parse // import "github.com/tdewolff/parse"

import (
	"errors"
	"io"
)

// minBuf and maxBuf are the initial and maximal internal buffer size.
const minBuf = 1024
const maxBuf = 4096

// ErrBufferExceeded is returned when the internal buffer exceeds 4096 bytes, a string or comment must thus be smaller than 4kB!
var ErrBufferExceeded = errors.New("max buffer exceeded")

// ShiftBuffer is an interface for tokenisation readers, which enables peeking and shifting the buffer.
type ShiftBuffer interface {
	Read(int) byte
	Move(int)
	MoveTo(int)
	Len() int
	Bytes() []byte
	Shift() []byte
	Err() error
}

type defaultShiftBuffer struct {
	buf []byte
	pos int
	n   int
}

// Move advances the 0 position of read.
func (z *defaultShiftBuffer) Move(n int) {
	z.n += n
}

// MoveTo sets the 0 position of read.
func (z *defaultShiftBuffer) MoveTo(n int) {
	z.n = n
}

// Len returns the 0 position of read.
func (z defaultShiftBuffer) Len() int {
	return z.n
}

// Bytes returns the bytes of the current selection.
func (z defaultShiftBuffer) Bytes() []byte {
	return z.buf[z.pos : z.pos+z.n]
}

// Shift returns the bytes of the current selection and advances the position.
func (z *defaultShiftBuffer) Shift() []byte {
	b := z.buf[z.pos : z.pos+z.n]
	z.pos += z.n
	z.n = 0
	return b
}

////////////////////////////////////////////////////////////////

// ShiftBufferReader is a buffered reader for tokenisation, taking an io.Reader.
type ShiftBufferReader struct {
	defaultShiftBuffer
	r       io.Reader
	readErr error
}

// NewShiftBufferReader returns a new ShiftBufferReader.
func NewShiftBufferReader(r io.Reader) *ShiftBufferReader {
	return &ShiftBufferReader{
		defaultShiftBuffer{
			buf: make([]byte, 0, minBuf),
		},
		r,
		nil,
	}
}

// Read returns the ith byte and possible does a reallocation
func (z *ShiftBufferReader) Read(i int) byte {
	if z.pos+z.n+i >= len(z.buf) {
		if z.readErr != nil {
			return 0
		}

		// reallocate a new buffer (possibly larger)
		c := cap(z.buf)
		d := z.n + i
		var buf1 []byte
		if 2*d > c {
			if 2*c > maxBuf {
				z.readErr = ErrBufferExceeded
				return 0
			}
			buf1 = make([]byte, d, 2*c)
		} else {
			buf1 = z.buf[:d]
		}
		copy(buf1, z.buf[z.pos:z.pos+d])

		// Read in to fill the buffer till capacity
		var n int
		n, z.readErr = z.r.Read(buf1[d:cap(buf1)])
		z.pos, z.buf = 0, buf1[:d+n]
		if n == 0 {
			return 0
		}
	}
	return z.buf[z.pos+z.n+i]
}

// Err returns the error.
func (z ShiftBufferReader) Err() error {
	if z.readErr == io.EOF {
		if z.pos+z.n >= len(z.buf) {
			return io.EOF
		}
	} else if z.readErr != nil {
		return z.readErr
	}
	return nil
}

////////////////////////////////////////////////////////////////

// ShiftBufferBytes is a buffered reader for tokenisation, taking a []byte.
type ShiftBufferBytes struct {
	defaultShiftBuffer
}

// NewShiftBufferBytes returns a new ShiftBufferBytes.
func NewShiftBufferBytes(b []byte) *ShiftBufferBytes {
	return &ShiftBufferBytes{
		defaultShiftBuffer{
			buf: b,
		},
	}
}

// Read returns the ith byte and possible does a reallocation
func (z *ShiftBufferBytes) Read(i int) byte {
	if z.pos+z.n+i >= len(z.buf) {
		return 0
	}
	return z.buf[z.pos+z.n+i]
}

// Err returns the error.
func (z ShiftBufferBytes) Err() error {
	if z.pos+z.n >= len(z.buf) {
		return io.EOF
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 The go-hep Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package rio

import (
	"bufio"
	"bytes"
	"io"

	riobin "github.com/gonuts/binary"
)

// Reader is a rio read-only stream
type Reader struct {
	r io.Reader

	rrr     *bytes.Buffer
	options Options
	version Version

	recs map[string]*Record // map of all connected records
}

// NewReader returns a new read-only rio stream
func NewReader(r io.Reader) (*Reader, error) {
	// a rio stream starts with rio magic
	hdr := [4]byte{}
	_, err := r.Read(hdr[:])
	if err != nil {
		return nil, errorf("rio: error reading magic-header: %v", err)
	}
	if hdr != rioMagic {
		return nil, errorf("rio: not a rio-stream. magic-header=%q. want=%q",
			string(hdr[:]),
			string(rioMagic[:]),
		)
	}

	buf := new(bytes.Buffer)
	r = io.TeeReader(r, buf)
	rr := bufio.NewReader(r) //Size(r, 4)
	return &Reader{
		r:       rr,
		options: 0,
		version: rioHdrVersion,
		recs:    make(map[string]*Record),
		rrr:     buf,
	}, nil
}

// Record adds a Record to the list of records to read or
// returns the Record with that name.
func (r *Reader) Record(name string) *Record {
	rec, ok := r.recs[name]
	if !ok {
		rec = newRecord(name, r.options)
		rec.r = r
		rec.unpack = true
		r.recs[name] = rec
	}
	return rec
}

// Records returns the list of connected Records
func (r *Reader) Records() []*Record {
	recs := make([]*Record, 0, len(r.recs))
	for _, rec := range r.recs {
		recs = append(recs, rec)
	}
	return recs
}

// Close finishes reading the rio read-only stream.
// It does not (and can not) close the underlying reader.
func (r *Reader) Close() error {
	var err error
	return err
}

// decoder manages the decoding of data values from rioRecords
type decoder struct {
	r io.Reader
}

func (dec *decoder) Decode(v interface{}) error {
	switch v := v.(type) {
	case RioDecoder:
		return v.RioDecode(dec.r)
	}

	d := riobin.NewDecoder(dec.r)
	d.Order = Endian
	return d.Decode(v)
}
<commit_msg>reader: cleanup<commit_after>// Copyright 2015 The go-hep Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package rio

import (
	"bufio"
	"io"

	riobin "github.com/gonuts/binary"
)

// Reader is a rio read-only stream
type Reader struct {
	r io.Reader

	options Options
	version Version

	recs map[string]*Record // map of all connected records
}

// NewReader returns a new read-only rio stream
func NewReader(r io.Reader) (*Reader, error) {
	// a rio stream starts with rio magic
	hdr := [4]byte{}
	_, err := r.Read(hdr[:])
	if err != nil {
		return nil, errorf("rio: error reading magic-header: %v", err)
	}
	if hdr != rioMagic {
		return nil, errorf("rio: not a rio-stream. magic-header=%q. want=%q",
			string(hdr[:]),
			string(rioMagic[:]),
		)
	}

	rr := bufio.NewReader(r)
	return &Reader{
		r:       rr,
		options: 0,
		version: rioHdrVersion,
		recs:    make(map[string]*Record),
	}, nil
}

// Record adds a Record to the list of records to read or
// returns the Record with that name.
func (r *Reader) Record(name string) *Record {
	rec, ok := r.recs[name]
	if !ok {
		rec = newRecord(name, r.options)
		rec.r = r
		rec.unpack = true
		r.recs[name] = rec
	}
	return rec
}

// Records returns the list of connected Records
func (r *Reader) Records() []*Record {
	recs := make([]*Record, 0, len(r.recs))
	for _, rec := range r.recs {
		recs = append(recs, rec)
	}
	return recs
}

// Close finishes reading the rio read-only stream.
// It does not (and can not) close the underlying reader.
func (r *Reader) Close() error {
	var err error
	return err
}

// decoder manages the decoding of data values from rioRecords
type decoder struct {
	r io.Reader
}

func (dec *decoder) Decode(v interface{}) error {
	switch v := v.(type) {
	case RioDecoder:
		return v.RioDecode(dec.r)
	}

	d := riobin.NewDecoder(dec.r)
	d.Order = Endian
	return d.Decode(v)
}
<|endoftext|>


<commit_before>// Package migration provides the primitives for migration in LXD.
//
// See https://github.com/lxc/lxd/blob/master/specs/migration.md for a complete
// description.

package main

import (
	"crypto/x509"
	"encoding/pem"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"strings"
	"sync"

	"github.com/golang/protobuf/proto"
	"github.com/gorilla/websocket"

	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/migration"
	"github.com/lxc/lxd/lxd/operations"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/idmap"
	"github.com/lxc/lxd/shared/logger"
)

type migrationFields struct {
	controlSecret string
	controlConn   *websocket.Conn
	controlLock   sync.Mutex

	criuSecret string
	criuConn   *websocket.Conn

	fsSecret string
	fsConn   *websocket.Conn

	// container specific fields
	live         bool
	instanceOnly bool
	instance     instance.Instance

	// storage specific fields
	volumeOnly bool
}

func (c *migrationFields) send(m proto.Message) error {
	/* gorilla websocket doesn't allow concurrent writes, and
	 * panic()s if it sees them (which is reasonable). If e.g. we
	 * happen to fail, get scheduled, start our write, then get
	 * unscheduled before the write is bit to a new thread which is
	 * receiving an error from the other side (due to our previous
	 * close), we can engage in these concurrent writes, which
	 * casuses the whole daemon to panic.
	 *
	 * Instead, let's lock sends to the controlConn so that we only ever
	 * write one message at the time.
	 */
	c.controlLock.Lock()
	defer c.controlLock.Unlock()

	err := migration.ProtoSend(c.controlConn, m)
	if err != nil {
		return err
	}

	return nil
}

func (c *migrationFields) recv(m proto.Message) error {
	return migration.ProtoRecv(c.controlConn, m)
}

func (c *migrationFields) disconnect() {
	closeMsg := websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")

	c.controlLock.Lock()
	if c.controlConn != nil {
		c.controlConn.WriteMessage(websocket.CloseMessage, closeMsg)
		c.controlConn.Close()
		c.controlConn = nil /* don't close twice */
	}
	c.controlLock.Unlock()

	/* Below we just Close(), which doesn't actually write to the
	 * websocket, it just closes the underlying connection. If e.g. there
	 * is still a filesystem transfer going on, but the other side has run
	 * out of disk space, writing an actual CloseMessage here will cause
	 * gorilla websocket to panic. Instead, we just force close this
	 * connection, since we report the error over the control channel
	 * anyway.
	 */
	if c.fsConn != nil {
		c.fsConn.Close()
	}

	if c.criuConn != nil {
		c.criuConn.Close()
	}
}

func (c *migrationFields) sendControl(err error) {
	c.controlLock.Lock()
	defer c.controlLock.Unlock()

	if c.controlConn != nil {
		migration.ProtoSendControl(c.controlConn, err)
	}

	if err != nil {
		c.disconnect()
	}
}

func (c *migrationFields) controlChannel() <-chan migration.MigrationControl {
	ch := make(chan migration.MigrationControl)
	go func() {
		msg := migration.MigrationControl{}
		err := c.recv(&msg)
		if err != nil {
			logger.Debugf("Got error reading migration control socket %s", err)
			close(ch)
			return
		}
		ch <- msg
	}()

	return ch
}

type migrationSourceWs struct {
	migrationFields

	allConnected chan bool
}

func (s *migrationSourceWs) Metadata() interface{} {
	secrets := shared.Jmap{
		"control": s.controlSecret,
		"fs":      s.fsSecret,
	}

	if s.criuSecret != "" {
		secrets["criu"] = s.criuSecret
	}

	return secrets
}

func (s *migrationSourceWs) Connect(op *operations.Operation, r *http.Request, w http.ResponseWriter) error {
	secret := r.FormValue("secret")
	if secret == "" {
		return fmt.Errorf("missing secret")
	}

	var conn **websocket.Conn

	switch secret {
	case s.controlSecret:
		conn = &s.controlConn
	case s.criuSecret:
		conn = &s.criuConn
	case s.fsSecret:
		conn = &s.fsConn
	default:
		// If we didn't find the right secret, the user provided a bad
		// one, which 403, not 404, since this operation actually
		// exists.
		return os.ErrPermission
	}

	c, err := shared.WebsocketUpgrader.Upgrade(w, r, nil)
	if err != nil {
		return err
	}

	*conn = c

	if s.controlConn != nil && (!s.live || s.criuConn != nil) && s.fsConn != nil {
		s.allConnected <- true
	}

	return nil
}

func (s *migrationSourceWs) ConnectTarget(certificate string, operation string, websockets map[string]string) error {
	var err error
	var cert *x509.Certificate

	if certificate != "" {
		certBlock, _ := pem.Decode([]byte(certificate))
		if certBlock == nil {
			return fmt.Errorf("Invalid certificate")
		}

		cert, err = x509.ParseCertificate(certBlock.Bytes)
		if err != nil {
			return err
		}
	}

	config, err := shared.GetTLSConfig("", "", "", cert)
	if err != nil {
		return err
	}

	dialer := websocket.Dialer{
		TLSClientConfig: config,
		NetDial:         shared.RFC3493Dialer,
	}

	for name, secret := range websockets {
		var conn **websocket.Conn

		switch name {
		case "control":
			conn = &s.controlConn
		case "fs":
			conn = &s.fsConn
		case "criu":
			conn = &s.criuConn
		default:
			return fmt.Errorf("Unknown secret provided: %s", name)
		}

		query := url.Values{"secret": []string{secret}}

		// The URL is a https URL to the operation, mangle to be a wss URL to the secret
		wsUrl := fmt.Sprintf("wss://%s/websocket?%s", strings.TrimPrefix(operation, "https://"), query.Encode())

		wsConn, _, err := dialer.Dial(wsUrl, http.Header{})
		if err != nil {
			return err
		}

		*conn = wsConn
	}

	s.allConnected <- true

	return nil
}

type migrationSink struct {
	// We are pulling the entity from src in pull mode.
	src migrationFields
	// The entity is pushed from src to dest in push mode. Note that
	// websocket connections are not set in push mode. Only the secret
	// fields are used since the client will connect to the sockets.
	dest migrationFields

	url          string
	dialer       websocket.Dialer
	allConnected chan bool
	push         bool
	refresh      bool
}

type MigrationSinkArgs struct {
	// General migration fields
	Dialer  websocket.Dialer
	Push    bool
	Secrets map[string]string
	Url     string

	// Instance specific fields
	Instance     instance.Instance
	InstanceOnly bool
	Idmap        *idmap.IdmapSet
	Live         bool
	Refresh      bool
	Snapshots    []*migration.Snapshot

	// Storage specific fields
	VolumeOnly bool
	VolumeSize int64

	// Transport specific fields
	RsyncFeatures []string
}

func (c *migrationSink) connectWithSecret(secret string) (*websocket.Conn, error) {
	query := url.Values{"secret": []string{secret}}

	// The URL is a https URL to the operation, mangle to be a wss URL to the secret
	wsUrl := fmt.Sprintf("wss://%s/websocket?%s", strings.TrimPrefix(c.url, "https://"), query.Encode())

	conn, _, err := c.dialer.Dial(wsUrl, http.Header{})
	if err != nil {
		return nil, err
	}

	return conn, err
}

func (s *migrationSink) Metadata() interface{} {
	secrets := shared.Jmap{
		"control": s.dest.controlSecret,
		"fs":      s.dest.fsSecret,
	}

	if s.dest.criuSecret != "" {
		secrets["criu"] = s.dest.criuSecret
	}

	return secrets
}

func (s *migrationSink) Connect(op *operations.Operation, r *http.Request, w http.ResponseWriter) error {
	secret := r.FormValue("secret")
	if secret == "" {
		return fmt.Errorf("missing secret")
	}

	var conn **websocket.Conn

	switch secret {
	case s.dest.controlSecret:
		conn = &s.dest.controlConn
	case s.dest.criuSecret:
		conn = &s.dest.criuConn
	case s.dest.fsSecret:
		conn = &s.dest.fsConn
	default:
		/* If we didn't find the right secret, the user provided a bad one,
		 * which 403, not 404, since this operation actually exists */
		return os.ErrPermission
	}

	c, err := shared.WebsocketUpgrader.Upgrade(w, r, nil)
	if err != nil {
		return err
	}

	*conn = c

	if s.dest.controlConn != nil && (!s.dest.live || s.dest.criuConn != nil) && s.dest.fsConn != nil {
		s.allConnected <- true
	}

	return nil
}
<commit_msg>lxd/migrate: Fix deadlock in sendControl<commit_after>// Package migration provides the primitives for migration in LXD.
//
// See https://github.com/lxc/lxd/blob/master/specs/migration.md for a complete
// description.

package main

import (
	"crypto/x509"
	"encoding/pem"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"strings"
	"sync"

	"github.com/golang/protobuf/proto"
	"github.com/gorilla/websocket"

	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/migration"
	"github.com/lxc/lxd/lxd/operations"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/idmap"
	"github.com/lxc/lxd/shared/logger"
)

type migrationFields struct {
	controlSecret string
	controlConn   *websocket.Conn
	controlLock   sync.Mutex

	criuSecret string
	criuConn   *websocket.Conn

	fsSecret string
	fsConn   *websocket.Conn

	// container specific fields
	live         bool
	instanceOnly bool
	instance     instance.Instance

	// storage specific fields
	volumeOnly bool
}

func (c *migrationFields) send(m proto.Message) error {
	/* gorilla websocket doesn't allow concurrent writes, and
	 * panic()s if it sees them (which is reasonable). If e.g. we
	 * happen to fail, get scheduled, start our write, then get
	 * unscheduled before the write is bit to a new thread which is
	 * receiving an error from the other side (due to our previous
	 * close), we can engage in these concurrent writes, which
	 * casuses the whole daemon to panic.
	 *
	 * Instead, let's lock sends to the controlConn so that we only ever
	 * write one message at the time.
	 */
	c.controlLock.Lock()
	defer c.controlLock.Unlock()

	err := migration.ProtoSend(c.controlConn, m)
	if err != nil {
		return err
	}

	return nil
}

func (c *migrationFields) recv(m proto.Message) error {
	return migration.ProtoRecv(c.controlConn, m)
}

func (c *migrationFields) disconnect() {
	closeMsg := websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")

	c.controlLock.Lock()
	if c.controlConn != nil {
		c.controlConn.WriteMessage(websocket.CloseMessage, closeMsg)
		c.controlConn.Close()
		c.controlConn = nil /* don't close twice */
	}
	c.controlLock.Unlock()

	/* Below we just Close(), which doesn't actually write to the
	 * websocket, it just closes the underlying connection. If e.g. there
	 * is still a filesystem transfer going on, but the other side has run
	 * out of disk space, writing an actual CloseMessage here will cause
	 * gorilla websocket to panic. Instead, we just force close this
	 * connection, since we report the error over the control channel
	 * anyway.
	 */
	if c.fsConn != nil {
		c.fsConn.Close()
	}

	if c.criuConn != nil {
		c.criuConn.Close()
	}
}

func (c *migrationFields) sendControl(err error) {
	c.controlLock.Lock()
	if c.controlConn != nil {
		migration.ProtoSendControl(c.controlConn, err)
	}
	c.controlLock.Unlock()

	if err != nil {
		c.disconnect()
	}
}

func (c *migrationFields) controlChannel() <-chan migration.MigrationControl {
	ch := make(chan migration.MigrationControl)
	go func() {
		msg := migration.MigrationControl{}
		err := c.recv(&msg)
		if err != nil {
			logger.Debugf("Got error reading migration control socket %s", err)
			close(ch)
			return
		}
		ch <- msg
	}()

	return ch
}

type migrationSourceWs struct {
	migrationFields

	allConnected chan bool
}

func (s *migrationSourceWs) Metadata() interface{} {
	secrets := shared.Jmap{
		"control": s.controlSecret,
		"fs":      s.fsSecret,
	}

	if s.criuSecret != "" {
		secrets["criu"] = s.criuSecret
	}

	return secrets
}

func (s *migrationSourceWs) Connect(op *operations.Operation, r *http.Request, w http.ResponseWriter) error {
	secret := r.FormValue("secret")
	if secret == "" {
		return fmt.Errorf("missing secret")
	}

	var conn **websocket.Conn

	switch secret {
	case s.controlSecret:
		conn = &s.controlConn
	case s.criuSecret:
		conn = &s.criuConn
	case s.fsSecret:
		conn = &s.fsConn
	default:
		// If we didn't find the right secret, the user provided a bad
		// one, which 403, not 404, since this operation actually
		// exists.
		return os.ErrPermission
	}

	c, err := shared.WebsocketUpgrader.Upgrade(w, r, nil)
	if err != nil {
		return err
	}

	*conn = c

	if s.controlConn != nil && (!s.live || s.criuConn != nil) && s.fsConn != nil {
		s.allConnected <- true
	}

	return nil
}

func (s *migrationSourceWs) ConnectTarget(certificate string, operation string, websockets map[string]string) error {
	var err error
	var cert *x509.Certificate

	if certificate != "" {
		certBlock, _ := pem.Decode([]byte(certificate))
		if certBlock == nil {
			return fmt.Errorf("Invalid certificate")
		}

		cert, err = x509.ParseCertificate(certBlock.Bytes)
		if err != nil {
			return err
		}
	}

	config, err := shared.GetTLSConfig("", "", "", cert)
	if err != nil {
		return err
	}

	dialer := websocket.Dialer{
		TLSClientConfig: config,
		NetDial:         shared.RFC3493Dialer,
	}

	for name, secret := range websockets {
		var conn **websocket.Conn

		switch name {
		case "control":
			conn = &s.controlConn
		case "fs":
			conn = &s.fsConn
		case "criu":
			conn = &s.criuConn
		default:
			return fmt.Errorf("Unknown secret provided: %s", name)
		}

		query := url.Values{"secret": []string{secret}}

		// The URL is a https URL to the operation, mangle to be a wss URL to the secret
		wsUrl := fmt.Sprintf("wss://%s/websocket?%s", strings.TrimPrefix(operation, "https://"), query.Encode())

		wsConn, _, err := dialer.Dial(wsUrl, http.Header{})
		if err != nil {
			return err
		}

		*conn = wsConn
	}

	s.allConnected <- true

	return nil
}

type migrationSink struct {
	// We are pulling the entity from src in pull mode.
	src migrationFields
	// The entity is pushed from src to dest in push mode. Note that
	// websocket connections are not set in push mode. Only the secret
	// fields are used since the client will connect to the sockets.
	dest migrationFields

	url          string
	dialer       websocket.Dialer
	allConnected chan bool
	push         bool
	refresh      bool
}

type MigrationSinkArgs struct {
	// General migration fields
	Dialer  websocket.Dialer
	Push    bool
	Secrets map[string]string
	Url     string

	// Instance specific fields
	Instance     instance.Instance
	InstanceOnly bool
	Idmap        *idmap.IdmapSet
	Live         bool
	Refresh      bool
	Snapshots    []*migration.Snapshot

	// Storage specific fields
	VolumeOnly bool
	VolumeSize int64

	// Transport specific fields
	RsyncFeatures []string
}

func (c *migrationSink) connectWithSecret(secret string) (*websocket.Conn, error) {
	query := url.Values{"secret": []string{secret}}

	// The URL is a https URL to the operation, mangle to be a wss URL to the secret
	wsUrl := fmt.Sprintf("wss://%s/websocket?%s", strings.TrimPrefix(c.url, "https://"), query.Encode())

	conn, _, err := c.dialer.Dial(wsUrl, http.Header{})
	if err != nil {
		return nil, err
	}

	return conn, err
}

func (s *migrationSink) Metadata() interface{} {
	secrets := shared.Jmap{
		"control": s.dest.controlSecret,
		"fs":      s.dest.fsSecret,
	}

	if s.dest.criuSecret != "" {
		secrets["criu"] = s.dest.criuSecret
	}

	return secrets
}

func (s *migrationSink) Connect(op *operations.Operation, r *http.Request, w http.ResponseWriter) error {
	secret := r.FormValue("secret")
	if secret == "" {
		return fmt.Errorf("missing secret")
	}

	var conn **websocket.Conn

	switch secret {
	case s.dest.controlSecret:
		conn = &s.dest.controlConn
	case s.dest.criuSecret:
		conn = &s.dest.criuConn
	case s.dest.fsSecret:
		conn = &s.dest.fsConn
	default:
		/* If we didn't find the right secret, the user provided a bad one,
		 * which 403, not 404, since this operation actually exists */
		return os.ErrPermission
	}

	c, err := shared.WebsocketUpgrader.Upgrade(w, r, nil)
	if err != nil {
		return err
	}

	*conn = c

	if s.dest.controlConn != nil && (!s.dest.live || s.dest.criuConn != nil) && s.dest.fsConn != nil {
		s.allConnected <- true
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"fmt"
	"time"
	"github.com/urfave/cli"
	"github.com/ebfe/go.pcsclite/scard"
)

type Reader struct {
	ctx *scard.Context
	c *cli.Context
	name string
	card *scard.Card
}

func NewReader(c *cli.Context) *Reader {
	ctx, err := scard.EstablishContext()
	if err != nil {
		fmt.Fprintf(os.Stderr, "error: %s\n", err)
		return nil
	}

	readers, err := ctx.ListReaders()
	if err != nil || len(readers) == 0 {
		fmt.Fprintf(os.Stderr, "エラー: リーダーが見つかりません。\n")
		return nil
	}
	if len(readers) >= 2 {
		fmt.Fprintf(os.Stderr,
			"警告: 複数のリーダーが見つかりました。最初のものを使います。\n")
	}

	reader := new(Reader)
	reader.ctx = ctx
	reader.c = c
	reader.name = readers[0]
	reader.card = nil
	return reader
}

func (self *Reader) Finalize() {
	self.ctx.Release()
}

func (self *Reader) CheckCard() {
	card := self.WaitForCard()
	aid := "D3 92 f0 00 26 01 00 00 00 01"
	apdu := "00 A4 04 0C" + " 0A " + aid
	sw1, sw2, _ := self.Tx(apdu)
	if sw1 == 0x90 && sw2 == 0x00 {
		return
	}

	fmt.Fprintf(os.Stderr, "これは個人番号カードではありません。\n")
	os.Exit(1)
}

func (self *Reader) GetCard() *scard.Card {
	card, _ := self.ctx.Connect(
		self.name, scard.SHARE_EXCLUSIVE, scard.PROTOCOL_ANY)
	self.card = card
	return card
}

func (self *Reader) WaitForCard() *scard.Card {
	rs := make([]scard.ReaderState, 1)
	rs[0].Reader = self.name
	rs[0].CurrentState = scard.STATE_UNAWARE
	for {
		fmt.Fprintf(os.Stderr, "wait for card:\n")
		err := self.ctx.GetStatusChange(rs, scard.INFINITE)
		if err != nil {
			fmt.Fprintf(os.Stderr, "エラー: %s\n", err)
			return nil
		}
		if rs[0].EventState&scard.STATE_PRESENT != 0 {
			card, _ := self.ctx.Connect(
				self.name, scard.SHARE_EXCLUSIVE, scard.PROTOCOL_ANY)
			self.card = card
			return card
		}
		time.Sleep(1 * time.Second)
	}
	panic("unreachable")
}

func (self *Reader) Tx(apdu string) (uint8, uint8, []byte) {
	card := reader.card
	//fmt.Printf("%v\n", c.GetInt("verbose"))
	fmt.Printf(">> %v\n", apdu)
	cmd := ToBytes(apdu)
	res, err := card.Transmit(cmd)
	if err != nil {
		fmt.Printf("err: %s\n", err)
		return 0, 0, nil
	}

	for i := 0; i < len(res); i++ {
		if i % 0x10 == 0 {
			fmt.Print("<<")
		}
		fmt.Printf(" %02X", res[i])
		if i % 0x10 == 0x0f {
			fmt.Println()
		}
	}
	fmt.Println()

	l := len(res)
	if l == 2 {
		return res[0], res[1], nil
	}else if l > 2 {
		return res[l-2], res[l-1], res[:l-2]
	}
	return 0, 0, nil
}

<commit_msg>add verbose option<commit_after>package main

import (
	"os"
	"fmt"
	"time"
	"github.com/urfave/cli"
	"github.com/ebfe/go.pcsclite/scard"
)

type Reader struct {
	ctx *scard.Context
	c *cli.Context
	name string
	card *scard.Card
}

func NewReader(c *cli.Context) *Reader {
	ctx, err := scard.EstablishContext()
	if err != nil {
		fmt.Fprintf(os.Stderr, "error: %s\n", err)
		return nil
	}

	readers, err := ctx.ListReaders()
	if err != nil || len(readers) == 0 {
		fmt.Fprintf(os.Stderr, "エラー: リーダーが見つかりません。\n")
		return nil
	}
	if len(readers) >= 2 {
		fmt.Fprintf(os.Stderr,
			"警告: 複数のリーダーが見つかりました。最初のものを使います。\n")
	}

	reader := new(Reader)
	reader.ctx = ctx
	reader.c = c
	reader.name = readers[0]
	reader.card = nil
	return reader
}

func (self *Reader) Finalize() {
	self.ctx.Release()
}

func (self *Reader) CheckCard() {
	self.WaitForCard()
	aid := "D3 92 f0 00 26 01 00 00 00 01"
	apdu := "00 A4 04 0C" + " 0A " + aid
	sw1, sw2, _ := self.Tx(apdu)
	if sw1 == 0x90 && sw2 == 0x00 {
		return
	}

	fmt.Fprintf(os.Stderr, "これは個人番号カードではありません。\n")
	os.Exit(1)
}

func (self *Reader) GetCard() *scard.Card {
	card, _ := self.ctx.Connect(
		self.name, scard.SHARE_EXCLUSIVE, scard.PROTOCOL_ANY)
	self.card = card
	return card
}

func (self *Reader) WaitForCard() *scard.Card {
	rs := make([]scard.ReaderState, 1)
	rs[0].Reader = self.name
	rs[0].CurrentState = scard.STATE_UNAWARE
	for {
		fmt.Fprintf(os.Stderr, "wait for card:\n")
		err := self.ctx.GetStatusChange(rs, scard.INFINITE)
		if err != nil {
			fmt.Fprintf(os.Stderr, "エラー: %s\n", err)
			return nil
		}
		if rs[0].EventState&scard.STATE_PRESENT != 0 {
			card, _ := self.ctx.Connect(
				self.name, scard.SHARE_EXCLUSIVE, scard.PROTOCOL_ANY)
			self.card = card
			return card
		}
		time.Sleep(1 * time.Second)
	}
	panic("unreachable")
}

func (self *Reader) Tx(apdu string) (uint8, uint8, []byte) {
	card := self.card
	if self.c.Bool("verbose") {
		fmt.Printf(">> %v\n", apdu)
	}
	cmd := ToBytes(apdu)
	res, err := card.Transmit(cmd)
	if err != nil {
		fmt.Printf("err: %s\n", err)
		return 0, 0, nil
	}

	if self.c.Bool("verbose") {
		for i := 0; i < len(res); i++ {
			if i % 0x10 == 0 {
				fmt.Print("<<")
			}
			fmt.Printf(" %02X", res[i])
			if i % 0x10 == 0x0f {
				fmt.Println()
			}
		}
		fmt.Println()
	}

	l := len(res)
	if l == 2 {
		return res[0], res[1], nil
	}else if l > 2 {
		return res[l-2], res[l-1], res[:l-2]
	}
	return 0, 0, nil
}

<|endoftext|>