<commit_before>package main

import (
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
)

type BaseMetadata struct {
	TypeName    string
	TypeURL     string
	TypeVersion string
	Name        string
	RepoUUID    string
	Compression string
	Checksum    string
	Persistence string
	Versioned   bool
}

type Metadata struct {
	Base     BaseMetadata
	Extended interface{}
}

func getLabelMetadata(dstURL string) *LabelMetadata {
	infoUrl := dstURL + "/info"
	resp, err := http.Get(infoUrl)
	if err != nil {
		fmt.Printf("Error on getting metadata: %s\n", err.Error())
		os.Exit(1)
	}
	if resp.StatusCode != http.StatusOK {
		fmt.Printf("Bad status on getting metadata: %d\n", resp.StatusCode)
		os.Exit(1)
	}
	metadata, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Printf("Could not read metadata: %s\n", err.Error())
		os.Exit(1)
	}
	m := new(LabelMetadata)
	if err := json.Unmarshal(metadata, m); err != nil {
		fmt.Printf("Error parsing metadata: %s\n", err.Error())
		os.Exit(1)
	}
	return m
}

type LabelMetadata struct {
	Base     BaseMetadata
	Extended struct {
		BlockSize [3]int
		MinIndex  [3]int
		MaxIndex  [3]int
	}
}

func sendLabels(src *LabelMetadata, name, srcURL string) {
	minIndex := src.Extended.MinIndex
	maxIndex := src.Extended.MaxIndex
	blockSize := src.Extended.BlockSize
	if blockSize[0] != blockSize[1] {
		fmt.Printf("Can't handle non-cubic block sizes as in %q data instance: %v\n", name, blockSize)
		os.Exit(1)
	}
	fmt.Printf("MinIndex: %v\n", minIndex)
	fmt.Printf("MaxIndex: %v\n", maxIndex)

	// Compute the output file name
	var ext string
	switch *compression {
	case "none":
		ext = "dat"
	case "lz4":
		ext = "lz4"
	case "gzip":
		ext = "gz"
	default:
		fmt.Printf("unknown compression type %q", *compression)
		os.Exit(1)
	}

	// Iterate through each XY plane of one block deep data.
	// Send it to destination, synchronously.
	nx := maxIndex[0] - minIndex[0] + 1
	ny := maxIndex[1] - minIndex[1] + 1

	// # voxels in x, and xy
	vx := nx * blockSize[0]
	vy := ny * blockSize[1]
	vz := blockSize[2]

	strips := 1
	// if vx*vy*vz*8 > 2000000000 {
	// 	strips = (vx*vy*vz*8)/2000000000 + 1
	// }
	byPerStrip := ny / strips
	fmt.Printf("Total uncompressed bytes per file: %d MB\n", vx*vy*vz*8/1000000)

	ox := minIndex[0] * blockSize[0]
	for z := minIndex[2]; z <= maxIndex[2]; z++ {
		oz := z * blockSize[2]
		if oz+blockSize[2] < *minz || oz > *maxz {
			continue
		}
		by0 := minIndex[1]
		for n := 0; n < strips; n++ {
			if by0 > maxIndex[1] {
				break
			}
			by1 := by0 + byPerStrip - 1
			if by1 > maxIndex[1] {
				by1 = maxIndex[1]
			}

			oy := by0 * blockSize[1]
			vy := (by1 - by0 + 1) * blockSize[1]
			getURL := fmt.Sprintf("%s/raw/0_1_2/%d_%d_%d/%d_%d_%d", srcURL, vx, vy, vz, ox, oy, oz)
			if *roi != "" {
				getURL += fmt.Sprintf("?roi=%s", *roi)
			}
			var resp http.Response
			if *dryrun {
				fmt.Printf("-- would GET %s\n", getURL)
			} else {
				resp, err := http.Get(getURL)
				if err != nil {
					fmt.Printf("Receive error: %s\n", err.Error())
					os.Exit(1)
				}
				if resp.StatusCode != http.StatusOK {
					fmt.Printf("Bad status on receiving data: %d\n", resp.StatusCode)
					os.Exit(1)
				}
			}
			if *url != "" {
				postURL := fmt.Sprintf("%s/raw/0_1_2/%d_%d_%d/%d_%d_%d", *url, vx, vy, vz, ox, oy, oz)
				if *dryrun {
					fmt.Printf("-- would POST %s\n", postURL)
				} else {
					resp2, err := http.Post(postURL, "application/octet-stream", resp.Body)
					if err != nil {
						fmt.Printf("Transmit error: %s\n", err.Error())
						os.Exit(1)
					}
					if resp2.StatusCode != http.StatusOK {
						fmt.Printf("Bad status on sending data: %d\n", resp2.StatusCode)
						os.Exit(1)
					}
				}
			}
			if *outdir != "" {
				base := fmt.Sprintf("%s-%dx%dx%d+%d+%d+%d.%s", name, vx, vy, vz, ox, oy, oz, ext)
				filename := filepath.Join(*outdir, base)
				if err := writeFile(resp.Body, filename); err != nil {
					fmt.Printf("Error writing file: %s\n", filename)
					os.Exit(1)
				}
			}

			by0 += byPerStrip
		}
	}
}

func writeFile(data io.ReadCloser, filename string) error {
	fmt.Printf("Writing data to %s\n", filename)
	if *dryrun {
		return nil
	}
	defer data.Close()

	// Setup file for write
	f, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer f.Close()

	var out io.Writer
	switch *compression {
	case "none":
		out = f
	case "gzip":
		out = gzip.NewWriter(f)
	default:
		return fmt.Errorf("Can't compress to type %q, only %q\n", *compression, "gzip")
	}

	io.Copy(out, data)
	return nil
}

func transferData(host, uuid, name string) {
	// If we are transferring to another DVID host, verify the destination.
	var dstMetadata *LabelMetadata
	if *url != "" {
		dstMetadata = getLabelMetadata(*url)
		dsttype := dstMetadata.Base.TypeName

		switch dsttype {
		case "labelblk":
			fmt.Printf("-- verified that POST URL (%v) is labelblk.\n", *url)
		default:
			fmt.Printf("POST URL (%v) must be labelblk but is %s data type\n", *url, dsttype)
			os.Exit(1)
		}
	}

	// Get the source metadata
	srcURL := fmt.Sprintf("%s/api/node/%s/%s", host, uuid, name)
	srcMetadata := getLabelMetadata(srcURL)
	if srcMetadata.Base.TypeName != "labelblk" {
		fmt.Printf("Source data instance %q is not labelblk type.  Aborting.\n", name)
		os.Exit(1)
	}

	// If we are writing files to output directory, make sure it's created.
	// Make sure output directory exists if it's specified.
	if *outdir != "" {
		if fileinfo, err := os.Stat(*outdir); os.IsNotExist(err) {
			fmt.Printf("Creating output directory: %s\n", *outdir)
			err := os.MkdirAll(*outdir, 0744)
			if err != nil {
				fmt.Printf("Can't make output directory: %s\n", err.Error())
				os.Exit(1)
			}
		} else if !fileinfo.IsDir() {
			fmt.Printf("Supplied output path (%s) is not a directory.", *outdir)
			os.Exit(1)
		}
	}

	sendLabels(srcMetadata, name, srcURL)
}
<commit_msg>enable striping for transfer command<commit_after>package main

import (
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
)

type BaseMetadata struct {
	TypeName    string
	TypeURL     string
	TypeVersion string
	Name        string
	RepoUUID    string
	Compression string
	Checksum    string
	Persistence string
	Versioned   bool
}

type Metadata struct {
	Base     BaseMetadata
	Extended interface{}
}

func getLabelMetadata(dstURL string) *LabelMetadata {
	infoUrl := dstURL + "/info"
	resp, err := http.Get(infoUrl)
	if err != nil {
		fmt.Printf("Error on getting metadata: %s\n", err.Error())
		os.Exit(1)
	}
	if resp.StatusCode != http.StatusOK {
		fmt.Printf("Bad status on getting metadata: %d\n", resp.StatusCode)
		os.Exit(1)
	}
	metadata, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Printf("Could not read metadata: %s\n", err.Error())
		os.Exit(1)
	}
	m := new(LabelMetadata)
	if err := json.Unmarshal(metadata, m); err != nil {
		fmt.Printf("Error parsing metadata: %s\n", err.Error())
		os.Exit(1)
	}
	return m
}

type LabelMetadata struct {
	Base     BaseMetadata
	Extended struct {
		BlockSize [3]int
		MinIndex  [3]int
		MaxIndex  [3]int
	}
}

func sendLabels(src *LabelMetadata, name, srcURL string) {
	minIndex := src.Extended.MinIndex
	maxIndex := src.Extended.MaxIndex
	blockSize := src.Extended.BlockSize
	if blockSize[0] != blockSize[1] {
		fmt.Printf("Can't handle non-cubic block sizes as in %q data instance: %v\n", name, blockSize)
		os.Exit(1)
	}
	fmt.Printf("MinIndex: %v\n", minIndex)
	fmt.Printf("MaxIndex: %v\n", maxIndex)

	// Compute the output file name
	var ext string
	switch *compression {
	case "none":
		ext = "dat"
	case "lz4":
		ext = "lz4"
	case "gzip":
		ext = "gz"
	default:
		fmt.Printf("unknown compression type %q", *compression)
		os.Exit(1)
	}

	// Iterate through each XY plane of one block deep data.
	// Send it to destination, synchronously.
	nx := maxIndex[0] - minIndex[0] + 1
	ny := maxIndex[1] - minIndex[1] + 1

	// # voxels in x, and xy
	vx := nx * blockSize[0]
	vy := ny * blockSize[1]
	vz := blockSize[2]

	strips := 1
	if vx*vy*vz*8 > 2000000000 {
		strips = (vx*vy*vz*8)/2000000000 + 1
	}
	byPerStrip := ny / strips
	fmt.Printf("Number of strips per layer: %d\n", strips)
	fmt.Printf("Total uncompressed bytes per file: %d MB\n", vx*vy*vz*8/(strips*1000000))

	ox := minIndex[0] * blockSize[0]
	for z := minIndex[2]; z <= maxIndex[2]; z++ {
		oz := z * blockSize[2]
		if oz+blockSize[2] < *minz || oz > *maxz {
			continue
		}
		by0 := minIndex[1]
		for n := 0; n < strips; n++ {
			if by0 > maxIndex[1] {
				break
			}
			by1 := by0 + byPerStrip - 1
			if by1 > maxIndex[1] {
				by1 = maxIndex[1]
			}

			oy := by0 * blockSize[1]
			vy := (by1 - by0 + 1) * blockSize[1]
			getURL := fmt.Sprintf("%s/raw/0_1_2/%d_%d_%d/%d_%d_%d", srcURL, vx, vy, vz, ox, oy, oz)
			if *roi != "" {
				getURL += fmt.Sprintf("?roi=%s", *roi)
			}
			var resp http.Response
			if *dryrun {
				fmt.Printf("-- would GET %s\n", getURL)
			} else {
				resp, err := http.Get(getURL)
				if err != nil {
					fmt.Printf("Receive error: %s\n", err.Error())
					os.Exit(1)
				}
				if resp.StatusCode != http.StatusOK {
					fmt.Printf("Bad status on receiving data: %d\n", resp.StatusCode)
					os.Exit(1)
				}
			}
			if *url != "" {
				postURL := fmt.Sprintf("%s/raw/0_1_2/%d_%d_%d/%d_%d_%d", *url, vx, vy, vz, ox, oy, oz)
				if *dryrun {
					fmt.Printf("-- would POST %s\n", postURL)
				} else {
					resp2, err := http.Post(postURL, "application/octet-stream", resp.Body)
					if err != nil {
						fmt.Printf("Transmit error: %s\n", err.Error())
						os.Exit(1)
					}
					if resp2.StatusCode != http.StatusOK {
						fmt.Printf("Bad status on sending data: %d\n", resp2.StatusCode)
						os.Exit(1)
					}
				}
			}
			if *outdir != "" {
				base := fmt.Sprintf("%s-%dx%dx%d+%d+%d+%d.%s", name, vx, vy, vz, ox, oy, oz, ext)
				filename := filepath.Join(*outdir, base)
				if err := writeFile(resp.Body, filename); err != nil {
					fmt.Printf("Error writing file: %s\n", filename)
					os.Exit(1)
				}
			}

			by0 += byPerStrip
		}
	}
}

func writeFile(data io.ReadCloser, filename string) error {
	fmt.Printf("Writing data to %s\n", filename)
	if *dryrun {
		return nil
	}
	defer data.Close()

	// Setup file for write
	f, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer f.Close()

	var out io.Writer
	switch *compression {
	case "none":
		out = f
	case "gzip":
		out = gzip.NewWriter(f)
	default:
		return fmt.Errorf("Can't compress to type %q, only %q\n", *compression, "gzip")
	}

	io.Copy(out, data)
	return nil
}

func transferData(host, uuid, name string) {
	// If we are transferring to another DVID host, verify the destination.
	var dstMetadata *LabelMetadata
	if *url != "" {
		dstMetadata = getLabelMetadata(*url)
		dsttype := dstMetadata.Base.TypeName

		switch dsttype {
		case "labelblk":
			fmt.Printf("-- verified that POST URL (%v) is labelblk.\n", *url)
		default:
			fmt.Printf("POST URL (%v) must be labelblk but is %s data type\n", *url, dsttype)
			os.Exit(1)
		}
	}

	// Get the source metadata
	srcURL := fmt.Sprintf("%s/api/node/%s/%s", host, uuid, name)
	srcMetadata := getLabelMetadata(srcURL)
	if srcMetadata.Base.TypeName != "labelblk" {
		fmt.Printf("Source data instance %q is not labelblk type.  Aborting.\n", name)
		os.Exit(1)
	}

	// If we are writing files to output directory, make sure it's created.
	// Make sure output directory exists if it's specified.
	if *outdir != "" {
		if fileinfo, err := os.Stat(*outdir); os.IsNotExist(err) {
			fmt.Printf("Creating output directory: %s\n", *outdir)
			err := os.MkdirAll(*outdir, 0744)
			if err != nil {
				fmt.Printf("Can't make output directory: %s\n", err.Error())
				os.Exit(1)
			}
		} else if !fileinfo.IsDir() {
			fmt.Printf("Supplied output path (%s) is not a directory.", *outdir)
			os.Exit(1)
		}
	}

	sendLabels(srcMetadata, name, srcURL)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"sort"
	"sync"

	"github.com/midbel/transmit"
)

var ErrDone = errors.New("done")

type Config struct {
	Listen  bool             `json:"-"`
	Verbose bool             `json:"-"`
	Address string           `json:"gateway"`
	Proxy   string           `json:"proxy"`
	Routes  []transmit.Route `json:"routes"`
}

func main() {
	config := new(Config)
	flag.BoolVar(&config.Listen, "l", config.Listen, "listen")
	flag.BoolVar(&config.Verbose, "v", config.Verbose, "verbose")
	flag.Parse()

	f, err := os.Open(flag.Arg(0))
	if err != nil {
		log.Fatalln(err)
	}
	f.Close()
	if err := json.NewDecoder(f).Decode(&config); err != nil {
		log.Fatalln(err)
	}

	switch {
	case config.Listen:
		if len(config.Routes) == 0 {
			log.Fatalln("no routes configured! abort...")
		}
		sort.Slice(config.Routes, func(i, j int) bool {
			return config.Routes[i].Addr < config.Routes[j].Addr
		})
		err = distribute(config.Address, config.Proxy, config.Routes)
	default:
		err = forward(config.Address, config.Routes)
	}
	if err != nil {
		log.Fatalln(err)
	}
}

func distribute(a, p string, rs []transmit.Route) error {
	r, err := transmit.NewRouter(a, rs)
	if err != nil {
		return err
	}
	log.Printf("start listening on %s", a)
	defer r.Close()

	var wg sync.WaitGroup
	for {
		f, s, err := r.Accept()
		if err != nil {
			log.Printf("connection rejected: %s", err)
			continue
		}
		wg.Add(1)
		go func(r, w net.Conn) {
			x, err := proxy(p, w.RemoteAddr().String(), rs)
			if err == nil {
				log.Printf("proxy packets from %s to %s", r.RemoteAddr(), x.RemoteAddr())
			} else {
				log.Println(err)
			}

			log.Printf("start transmitting from %s to %s", r.RemoteAddr(), w.RemoteAddr())
			if err := relay(r, w, x); err != nil && err != ErrDone {
				log.Println("unexpected error while transmitting packets:", err)
			}
			wg.Done()
			log.Printf("done transmitting from %s to %s", r.RemoteAddr(), w.RemoteAddr())
		}(f, s)
	}
	wg.Wait()

	return nil
}

func proxy(p, a string, rs []transmit.Route) (net.Conn, error) {
	ix := sort.Search(len(rs), func(i int) bool {
		return rs[i].Addr >= a
	})
	if ix < len(rs) && rs[ix].Addr == a {
		return transmit.Forward(p, rs[ix].Id)
	}
	return nil, fmt.Errorf("no suitable route found for %s", a)
}

func forward(a string, rs []transmit.Route) error {
	var wg sync.WaitGroup
	for _, r := range rs {
		f, err := transmit.Forward(a, r.Id)
		if err != nil {
			return err
		}
		s, err := transmit.Subscribe(r.Addr, r.Eth)
		if err != nil {
			return err
		}
		wg.Add(1)
		go func(r, w net.Conn) {
			log.Printf("start transmitting from %s to %s", r.LocalAddr(), w.RemoteAddr())
			if err := relay(r, w, nil); err != nil && err != ErrDone {
				log.Println("unexpected error while transmitting packets:", err)
			}
			wg.Done()
			log.Printf("done transmitting from %s to %s", r.LocalAddr(), w.RemoteAddr())
		}(s, f)
	}
	wg.Wait()
	return nil
}

type nopCloser struct {
	io.Writer
}

func (n *nopCloser) Close() error { return nil }

func relay(r io.ReadCloser, w, x io.WriteCloser) error {
	defer func() {
		r.Close()
		w.Close()
		if x != nil {
			x.Close()
		}
	}()
	if x != nil {
		w = &nopCloser{io.MultiWriter(x, w)}
	}
	for {
		_, err := io.Copy(w, r)
		switch err {
		case nil:
		case transmit.ErrCorrupted:
			log.Println(err)
		case io.EOF:
			return ErrDone
		default:
			if e, ok := err.(net.Error); !ok || !(e.Temporary() || e.Timeout()) {
				return err
			}
		}
	}
	return nil
}
<commit_msg>close file after reading it<commit_after>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"sort"
	"sync"

	"github.com/midbel/transmit"
)

var ErrDone = errors.New("done")

type Config struct {
	Listen  bool             `json:"-"`
	Verbose bool             `json:"-"`
	Address string           `json:"gateway"`
	Proxy   string           `json:"proxy"`
	Routes  []transmit.Route `json:"routes"`
}

func main() {
	config := new(Config)
	flag.BoolVar(&config.Listen, "l", config.Listen, "listen")
	flag.BoolVar(&config.Verbose, "v", config.Verbose, "verbose")
	flag.Parse()

	f, err := os.Open(flag.Arg(0))
	if err != nil {
		log.Fatalln(err)
	}
	if err := json.NewDecoder(f).Decode(&config); err != nil {
		log.Fatalln(err)
	}
	f.Close()

	switch {
	case config.Listen:
		if len(config.Routes) == 0 {
			log.Fatalln("no routes configured! abort...")
		}
		sort.Slice(config.Routes, func(i, j int) bool {
			return config.Routes[i].Addr < config.Routes[j].Addr
		})
		err = distribute(config.Address, config.Proxy, config.Routes)
	default:
		err = forward(config.Address, config.Routes)
	}
	if err != nil {
		log.Fatalln(err)
	}
}

func distribute(a, p string, rs []transmit.Route) error {
	r, err := transmit.NewRouter(a, rs)
	if err != nil {
		return err
	}
	log.Printf("start listening on %s", a)
	defer r.Close()

	var wg sync.WaitGroup
	for {
		f, s, err := r.Accept()
		if err != nil {
			log.Printf("connection rejected: %s", err)
			continue
		}
		wg.Add(1)
		go func(r, w net.Conn) {
			x, err := proxy(p, w.RemoteAddr().String(), rs)
			if err == nil {
				log.Printf("proxy packets from %s to %s", r.RemoteAddr(), x.RemoteAddr())
			} else {
				log.Println(err)
			}

			log.Printf("start transmitting from %s to %s", r.RemoteAddr(), w.RemoteAddr())
			if err := relay(r, w, x); err != nil && err != ErrDone {
				log.Println("unexpected error while transmitting packets:", err)
			}
			wg.Done()
			log.Printf("done transmitting from %s to %s", r.RemoteAddr(), w.RemoteAddr())
		}(f, s)
	}
	wg.Wait()

	return nil
}

func proxy(p, a string, rs []transmit.Route) (net.Conn, error) {
	ix := sort.Search(len(rs), func(i int) bool {
		return rs[i].Addr >= a
	})
	if ix < len(rs) && rs[ix].Addr == a {
		return transmit.Forward(p, rs[ix].Id)
	}
	return nil, fmt.Errorf("no suitable route found for %s", a)
}

func forward(a string, rs []transmit.Route) error {
	var wg sync.WaitGroup
	for _, r := range rs {
		f, err := transmit.Forward(a, r.Id)
		if err != nil {
			return err
		}
		s, err := transmit.Subscribe(r.Addr, r.Eth)
		if err != nil {
			return err
		}
		wg.Add(1)
		go func(r, w net.Conn) {
			log.Printf("start transmitting from %s to %s", r.LocalAddr(), w.RemoteAddr())
			if err := relay(r, w, nil); err != nil && err != ErrDone {
				log.Println("unexpected error while transmitting packets:", err)
			}
			wg.Done()
			log.Printf("done transmitting from %s to %s", r.LocalAddr(), w.RemoteAddr())
		}(s, f)
	}
	wg.Wait()
	return nil
}

type nopCloser struct {
	io.Writer
}

func (n *nopCloser) Close() error { return nil }

func relay(r io.ReadCloser, w, x io.WriteCloser) error {
	defer func() {
		r.Close()
		w.Close()
		if x != nil {
			x.Close()
		}
	}()
	if x != nil {
		w = &nopCloser{io.MultiWriter(x, w)}
	}
	for {
		_, err := io.Copy(w, r)
		switch err {
		case nil:
		case transmit.ErrCorrupted:
			log.Println(err)
		case io.EOF:
			return ErrDone
		default:
			if e, ok := err.(net.Error); !ok || !(e.Temporary() || e.Timeout()) {
				return err
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package cli

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/tus/tusd/pkg/azurestore"
	"github.com/tus/tusd/pkg/filelocker"
	"github.com/tus/tusd/pkg/filestore"
	"github.com/tus/tusd/pkg/gcsstore"
	"github.com/tus/tusd/pkg/handler"
	"github.com/tus/tusd/pkg/memorylocker"
	"github.com/tus/tusd/pkg/s3store"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
)

var Composer *handler.StoreComposer

func CreateComposer() {
	// Attempt to use S3 as a backend if the -s3-bucket option has been supplied.
	// If not, we default to storing them locally on disk.
	Composer = handler.NewStoreComposer()
	if Flags.S3Bucket != "" {
		s3Config := aws.NewConfig()

		if Flags.S3TransferAcceleration {
			s3Config = s3Config.WithS3UseAccelerate(true)
		}

		if Flags.S3DisableContentHashes {
			// Prevent the S3 service client from automatically
			// adding the Content-MD5 header to S3 Object Put and Upload API calls.
			s3Config = s3Config.WithS3DisableContentMD5Validation(true)
		}

		if Flags.S3DisableSSL {
			// Disable HTTPS and only use HTTP (helpful for debugging requests).
			s3Config = s3Config.WithDisableSSL(true)
		}

		if Flags.S3Endpoint == "" {

			if Flags.S3TransferAcceleration {
				stdout.Printf("Using 's3://%s' as S3 bucket for storage with AWS S3 Transfer Acceleration enabled.\n", Flags.S3Bucket)
			} else {
				stdout.Printf("Using 's3://%s' as S3 bucket for storage.\n", Flags.S3Bucket)
			}

		} else {
			stdout.Printf("Using '%s/%s' as S3 endpoint and bucket for storage.\n", Flags.S3Endpoint, Flags.S3Bucket)

			s3Config = s3Config.WithEndpoint(Flags.S3Endpoint).WithS3ForcePathStyle(true)
		}

		// Derive credentials from default credential chain (env, shared, ec2 instance role)
		// as per https://github.com/aws/aws-sdk-go#configuring-credentials
		store := s3store.New(Flags.S3Bucket, s3.New(session.Must(session.NewSession()), s3Config))
		store.ObjectPrefix = Flags.S3ObjectPrefix
		store.PreferredPartSize = Flags.S3PartSize
		store.DisableContentHashes = Flags.S3DisableContentHashes
		store.UseIn(Composer)

		locker := memorylocker.New()
		locker.UseIn(Composer)
	} else if Flags.GCSBucket != "" {
		if Flags.GCSObjectPrefix != "" && strings.Contains(Flags.GCSObjectPrefix, "_") {
			stderr.Fatalf("gcs-object-prefix value (%s) can't contain underscore. "+
				"Please remove underscore from the value", Flags.GCSObjectPrefix)
		}

		// Derivce credentials from service account file path passed in
		// GCS_SERVICE_ACCOUNT_FILE environment variable.
		gcsSAF := os.Getenv("GCS_SERVICE_ACCOUNT_FILE")
		if gcsSAF == "" {
			stderr.Fatalf("No service account file provided for Google Cloud Storage using the GCS_SERVICE_ACCOUNT_FILE environment variable.\n")
		}

		service, err := gcsstore.NewGCSService(gcsSAF)
		if err != nil {
			stderr.Fatalf("Unable to create Google Cloud Storage service: %s\n", err)
		}

		stdout.Printf("Using 'gcs://%s' as GCS bucket for storage.\n", Flags.GCSBucket)

		store := gcsstore.New(Flags.GCSBucket, service)
		store.ObjectPrefix = Flags.GCSObjectPrefix
		store.UseIn(Composer)

		locker := memorylocker.New()
		locker.UseIn(Composer)
	} else if Flags.AzStorage != "" {

		accountName := os.Getenv("AZURE_STORAGE_ACCOUNT")
		if accountName == "" {
			stderr.Fatalf("No service account name for Azure BlockBlob Storage using the AZURE_STORAGE_ACCOUNT environment variable.\n")
		}

		accountKey := os.Getenv("AZURE_STORAGE_KEY")
		if accountKey == "" {
			stderr.Fatalf("No service account key for Azure BlockBlob Storage using the AZURE_STORAGE_KEY environment variable.\n")
		}

		azureEndpoint := Flags.AzEndpoint
		// Enables support for using Azurite as a storage emulator without messing with proxies and stuff
		// e.g. http://127.0.0.1:10000/devstoreaccount1
		if azureEndpoint == "" {
			azureEndpoint = fmt.Sprintf("https://%s.blob.core.windows.net", accountName)
			stdout.Printf("Custom Azure Endpoint not specified in flag variable azure-endpoint.\n"+
				"Using endpoint %s\n", azureEndpoint)
		} else {
			stdout.Printf("Using Azure endpoint %s\n", azureEndpoint)
		}

		azConfig := &azurestore.AzConfig{
			AccountName:         accountName,
			AccountKey:          accountKey,
			ContainerName:       Flags.AzStorage,
			ContainerAccessType: Flags.AzContainerAccessType,
			BlobAccessTier:      Flags.AzBlobAccessTier,
			Endpoint:            azureEndpoint,
		}

		azService, err := azurestore.NewAzureService(azConfig)
		if err != nil {
			stderr.Fatalf(err.Error())
		}

		store := azurestore.New(azService)
		store.ObjectPrefix = Flags.AzObjectPrefix
		store.Container = Flags.AzStorage

		store.UseIn(Composer)

	} else {
		dir, err := filepath.Abs(Flags.UploadDir)
		if err != nil {
			stderr.Fatalf("Unable to make absolute path: %s", err)
		}

		stdout.Printf("Using '%s' as directory storage.\n", dir)
		if err := os.MkdirAll(dir, os.FileMode(0774)); err != nil {
			stderr.Fatalf("Unable to ensure directory exists: %s", err)
		}

		store := filestore.New(dir)
		store.UseIn(Composer)

		locker := filelocker.New(dir)
		locker.UseIn(Composer)
	}

	stdout.Printf("Using %.2fMB as maximum size.\n", float64(Flags.MaxSize)/1024/1024)
}
<commit_msg>cli: Add memorylocker when azurestore is used<commit_after>package cli

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/tus/tusd/pkg/azurestore"
	"github.com/tus/tusd/pkg/filelocker"
	"github.com/tus/tusd/pkg/filestore"
	"github.com/tus/tusd/pkg/gcsstore"
	"github.com/tus/tusd/pkg/handler"
	"github.com/tus/tusd/pkg/memorylocker"
	"github.com/tus/tusd/pkg/s3store"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
)

var Composer *handler.StoreComposer

func CreateComposer() {
	// Attempt to use S3 as a backend if the -s3-bucket option has been supplied.
	// If not, we default to storing them locally on disk.
	Composer = handler.NewStoreComposer()
	if Flags.S3Bucket != "" {
		s3Config := aws.NewConfig()

		if Flags.S3TransferAcceleration {
			s3Config = s3Config.WithS3UseAccelerate(true)
		}

		if Flags.S3DisableContentHashes {
			// Prevent the S3 service client from automatically
			// adding the Content-MD5 header to S3 Object Put and Upload API calls.
			s3Config = s3Config.WithS3DisableContentMD5Validation(true)
		}

		if Flags.S3DisableSSL {
			// Disable HTTPS and only use HTTP (helpful for debugging requests).
			s3Config = s3Config.WithDisableSSL(true)
		}

		if Flags.S3Endpoint == "" {

			if Flags.S3TransferAcceleration {
				stdout.Printf("Using 's3://%s' as S3 bucket for storage with AWS S3 Transfer Acceleration enabled.\n", Flags.S3Bucket)
			} else {
				stdout.Printf("Using 's3://%s' as S3 bucket for storage.\n", Flags.S3Bucket)
			}

		} else {
			stdout.Printf("Using '%s/%s' as S3 endpoint and bucket for storage.\n", Flags.S3Endpoint, Flags.S3Bucket)

			s3Config = s3Config.WithEndpoint(Flags.S3Endpoint).WithS3ForcePathStyle(true)
		}

		// Derive credentials from default credential chain (env, shared, ec2 instance role)
		// as per https://github.com/aws/aws-sdk-go#configuring-credentials
		store := s3store.New(Flags.S3Bucket, s3.New(session.Must(session.NewSession()), s3Config))
		store.ObjectPrefix = Flags.S3ObjectPrefix
		store.PreferredPartSize = Flags.S3PartSize
		store.DisableContentHashes = Flags.S3DisableContentHashes
		store.UseIn(Composer)

		locker := memorylocker.New()
		locker.UseIn(Composer)
	} else if Flags.GCSBucket != "" {
		if Flags.GCSObjectPrefix != "" && strings.Contains(Flags.GCSObjectPrefix, "_") {
			stderr.Fatalf("gcs-object-prefix value (%s) can't contain underscore. "+
				"Please remove underscore from the value", Flags.GCSObjectPrefix)
		}

		// Derivce credentials from service account file path passed in
		// GCS_SERVICE_ACCOUNT_FILE environment variable.
		gcsSAF := os.Getenv("GCS_SERVICE_ACCOUNT_FILE")
		if gcsSAF == "" {
			stderr.Fatalf("No service account file provided for Google Cloud Storage using the GCS_SERVICE_ACCOUNT_FILE environment variable.\n")
		}

		service, err := gcsstore.NewGCSService(gcsSAF)
		if err != nil {
			stderr.Fatalf("Unable to create Google Cloud Storage service: %s\n", err)
		}

		stdout.Printf("Using 'gcs://%s' as GCS bucket for storage.\n", Flags.GCSBucket)

		store := gcsstore.New(Flags.GCSBucket, service)
		store.ObjectPrefix = Flags.GCSObjectPrefix
		store.UseIn(Composer)

		locker := memorylocker.New()
		locker.UseIn(Composer)
	} else if Flags.AzStorage != "" {

		accountName := os.Getenv("AZURE_STORAGE_ACCOUNT")
		if accountName == "" {
			stderr.Fatalf("No service account name for Azure BlockBlob Storage using the AZURE_STORAGE_ACCOUNT environment variable.\n")
		}

		accountKey := os.Getenv("AZURE_STORAGE_KEY")
		if accountKey == "" {
			stderr.Fatalf("No service account key for Azure BlockBlob Storage using the AZURE_STORAGE_KEY environment variable.\n")
		}

		azureEndpoint := Flags.AzEndpoint
		// Enables support for using Azurite as a storage emulator without messing with proxies and stuff
		// e.g. http://127.0.0.1:10000/devstoreaccount1
		if azureEndpoint == "" {
			azureEndpoint = fmt.Sprintf("https://%s.blob.core.windows.net", accountName)
			stdout.Printf("Custom Azure Endpoint not specified in flag variable azure-endpoint.\n"+
				"Using endpoint %s\n", azureEndpoint)
		} else {
			stdout.Printf("Using Azure endpoint %s\n", azureEndpoint)
		}

		azConfig := &azurestore.AzConfig{
			AccountName:         accountName,
			AccountKey:          accountKey,
			ContainerName:       Flags.AzStorage,
			ContainerAccessType: Flags.AzContainerAccessType,
			BlobAccessTier:      Flags.AzBlobAccessTier,
			Endpoint:            azureEndpoint,
		}

		azService, err := azurestore.NewAzureService(azConfig)
		if err != nil {
			stderr.Fatalf(err.Error())
		}

		store := azurestore.New(azService)
		store.ObjectPrefix = Flags.AzObjectPrefix
		store.Container = Flags.AzStorage
		store.UseIn(Composer)

		locker := memorylocker.New()
		locker.UseIn(Composer)
	} else {
		dir, err := filepath.Abs(Flags.UploadDir)
		if err != nil {
			stderr.Fatalf("Unable to make absolute path: %s", err)
		}

		stdout.Printf("Using '%s' as directory storage.\n", dir)
		if err := os.MkdirAll(dir, os.FileMode(0774)); err != nil {
			stderr.Fatalf("Unable to ensure directory exists: %s", err)
		}

		store := filestore.New(dir)
		store.UseIn(Composer)

		locker := filelocker.New(dir)
		locker.UseIn(Composer)
	}

	stdout.Printf("Using %.2fMB as maximum size.\n", float64(Flags.MaxSize)/1024/1024)
}
<|endoftext|>


<commit_before>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// TODO(adg): support other kinds?

// Command upspinserver is a combined DirServer and StoreServer for use on
// stand-alone machines. It provides only the production implementations of the
// dir and store servers (dir/server and store/gcp).
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"sync"

	"upspin.io/errors"

	"upspin.io/client"
	"upspin.io/cloud/https"
	"upspin.io/config"
	dirServer "upspin.io/dir/server"
	"upspin.io/factotum"
	"upspin.io/flags"
	"upspin.io/log"
	"upspin.io/rpc/dirserver"
	"upspin.io/rpc/storeserver"
	"upspin.io/serverutil/perm"
	storeServer "upspin.io/store/server"
	"upspin.io/upspin"

	// Load useful packers
	_ "upspin.io/pack/ee"
	_ "upspin.io/pack/eeintegrity"
	_ "upspin.io/pack/plain"

	// Load required transports
	_ "upspin.io/transports"
)

var (
	cfgPath   = flag.String("serverconfig", filepath.Join(os.Getenv("HOME"), "upspin/server"), "server configuration `directory`")
	letsPath  = flag.String("letscache", filepath.Join(os.Getenv("HOME"), "upspin/letsencrypt"), "Let's Encrypt cache `directory`")
	enableWeb = flag.Bool("web", false, "enable Upspin web interface")
	ready     = make(chan struct{})
)

func main() {
	flags.Parse("https", "log")

	server, cfg, err := initServer()
	if err == noConfig {
		log.Print("Configuration file not found. Running in setup mode.")
		http.Handle("/", &setupHandler{})
	} else if err != nil {
		log.Fatal(err)
	} else {
		http.Handle("/", newWeb(cfg))
	}

	// Set up HTTPS server.
	opt := &https.Options{
		LetsEncryptCache: *letsPath,
	}
	if server != nil {
		host, _, err := net.SplitHostPort(string(server.Addr))
		if err != nil {
			log.Printf("Error parsing addr from config %q: %v", server.Addr, err)
			log.Printf("Warning: Let's Encrypt certificates will be fetched for any host.")
		} else {
			opt.LetsEncryptHosts = []string{host}
		}
	}
	https.ListenAndServe(ready, "upspinserver", flags.HTTPSAddr, opt)
}

var noConfig = errors.Str("no configuration")

func initServer() (*ServerConfig, upspin.Config, error) {
	serverConfig, err := readServerConfig()
	if os.IsNotExist(err) {
		return nil, nil, noConfig
	} else if err != nil {
		return nil, nil, err
	}

	os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", filepath.Join(*cfgPath, "serviceaccount.json"))

	cfg := config.New()
	cfg = config.SetUserName(cfg, serverConfig.User)

	f, err := factotum.NewFromDir(*cfgPath)
	if err != nil {
		return nil, nil, err
	}
	cfg = config.SetFactotum(cfg, f)

	ep := upspin.Endpoint{
		Transport: upspin.Remote,
		NetAddr:   serverConfig.Addr,
	}
	cfg = config.SetDirEndpoint(cfg, ep)
	cfg = config.SetStoreEndpoint(cfg, ep)

	storeCfg := config.SetPacking(cfg, upspin.EEIntegrityPack)
	dirCfg := config.SetPacking(cfg, upspin.EEPack)

	// Set up StoreServer.
	store, err := storeServer.New("gcpBucketName="+serverConfig.Bucket, "defaultACL=publicRead")
	if err != nil {
		return nil, nil, err
	}
	store, err = perm.WrapStore(storeCfg, ready, store)
	if err != nil {
		return nil, nil, fmt.Errorf("error wrapping store: %s", err)
	}

	// Set up DirServer.
	logDir := filepath.Join(*cfgPath, "dirserver-logs")
	if err := os.MkdirAll(logDir, 0700); err != nil {
		return nil, nil, err
	}
	dir, err := dirServer.New(dirCfg, "userCacheSize=1000", "logDir="+logDir)
	if err != nil {
		return nil, nil, err
	}
	dir, err = perm.WrapDir(dirCfg, ready, serverConfig.User, dir)
	if err != nil {
		return nil, nil, fmt.Errorf("Can't wrap DirServer monitoring %s: %s", flags.StoreServerUser, err)
	}

	// Set up RPC server.
	httpStore := storeserver.New(storeCfg, store, serverConfig.Addr)
	httpDir := dirserver.New(dirCfg, dir, serverConfig.Addr)
	http.Handle("/api/Store/", httpStore)
	http.Handle("/api/Dir/", httpDir)

	log.Println("Store and Directory servers initialized.")

	go func() {
		if err := setupWriters(storeCfg); err != nil {
			log.Printf("Error creating Writers file: %v", err)
		}
	}()
	return serverConfig, cfg, nil
}

type setupHandler struct {
	mu   sync.Mutex
	done bool
	web  http.Handler
}

func (h *setupHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	h.mu.Lock()
	if h.done {
		web := h.web
		h.mu.Unlock()
		if web != nil {
			web.ServeHTTP(w, r)
		} else {
			http.NotFound(w, r)
		}
		return
	}
	defer h.mu.Unlock()

	switch r.URL.Path {
	case "/":
		fmt.Fprint(w, "Unconfigured Upspin Server")
		return
	default:
		http.NotFound(w, r)
		return
	case "/setupserver":
		if r.Method != "POST" {
			http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
			return
		}
		// The rest of this function is the setupserver handler.
	}

	files := map[string][]byte{}
	if err := json.NewDecoder(r.Body).Decode(&files); err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	if err := os.MkdirAll(*cfgPath, 0700); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	for _, name := range configureServerFiles {
		body, ok := files[name]
		if !ok {
			http.Error(w, fmt.Sprintf("missing config file %q", name), http.StatusBadRequest)
			return
		}
		err := ioutil.WriteFile(filepath.Join(*cfgPath, name), body, 0600)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			os.RemoveAll(*cfgPath)
			return
		}
	}
	_, cfg, err := initServer()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	fmt.Fprintf(w, "OK")

	h.done = true
	h.web = newWeb(cfg)
}

func setupWriters(cfg upspin.Config) error {
	writers, err := ioutil.ReadFile(filepath.Join(*cfgPath, "Writers"))
	if err != nil {
		return err
	}

	c := client.New(cfg)

	dir := upspin.PathName(cfg.UserName())
	if err := existsOK(c.MakeDirectory(dir)); err != nil {
		return err
	}

	dir += "/Group"
	if err := existsOK(c.MakeDirectory(dir)); err != nil {
		return err
	}

	file := dir + "/Writers"
	_, err = c.Put(file, writers)
	return err
}

// existsOK returns err if it is not an errors.Exist error,
// in which case it returns nil.
func existsOK(_ *upspin.DirEntry, err error) error {
	if errors.Match(errors.E(errors.Exist), err) {
		return nil
	}
	return err
}

func readServerConfig() (*ServerConfig, error) {
	cfgFile := filepath.Join(*cfgPath, serverConfigFile)
	b, err := ioutil.ReadFile(cfgFile)
	if err != nil {
		return nil, err
	}
	cfg := &ServerConfig{}
	if err := json.Unmarshal(b, cfg); err != nil {
		return nil, err
	}
	return cfg, nil
}

// Keep the following declarations in sync with cmd/upspin/setupserver.go.
// TODO(adg): move these to their own package if/when there are more users.

type ServerConfig struct {
	Addr   upspin.NetAddr
	User   upspin.UserName
	Bucket string
}

const serverConfigFile = "serverconfig.json"

var configureServerFiles = []string{
	"Writers",
	"public.upspinkey",
	"secret.upspinkey",
	"serverconfig.json",
	"serviceaccount.json",
}
<commit_msg>cmd/upspinserver: only create Writers file on 'upspin setupserver'<commit_after>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// TODO(adg): support other kinds?

// Command upspinserver is a combined DirServer and StoreServer for use on
// stand-alone machines. It provides only the production implementations of the
// dir and store servers (dir/server and store/gcp).
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"sync"

	"upspin.io/errors"

	"upspin.io/client"
	"upspin.io/cloud/https"
	"upspin.io/config"
	dirServer "upspin.io/dir/server"
	"upspin.io/factotum"
	"upspin.io/flags"
	"upspin.io/log"
	"upspin.io/rpc/dirserver"
	"upspin.io/rpc/storeserver"
	"upspin.io/serverutil/perm"
	storeServer "upspin.io/store/server"
	"upspin.io/upspin"

	// Load useful packers
	_ "upspin.io/pack/ee"
	_ "upspin.io/pack/eeintegrity"
	_ "upspin.io/pack/plain"

	// Load required transports
	_ "upspin.io/transports"
)

var (
	cfgPath   = flag.String("serverconfig", filepath.Join(os.Getenv("HOME"), "upspin/server"), "server configuration `directory`")
	letsPath  = flag.String("letscache", filepath.Join(os.Getenv("HOME"), "upspin/letsencrypt"), "Let's Encrypt cache `directory`")
	enableWeb = flag.Bool("web", false, "enable Upspin web interface")
	ready     = make(chan struct{})
)

func main() {
	flags.Parse("https", "log")

	server, cfg, err := initServer(startup)
	if err == noConfig {
		log.Print("Configuration file not found. Running in setup mode.")
		http.Handle("/", &setupHandler{})
	} else if err != nil {
		log.Fatal(err)
	} else {
		http.Handle("/", newWeb(cfg))
	}

	// Set up HTTPS server.
	opt := &https.Options{
		LetsEncryptCache: *letsPath,
	}
	if server != nil {
		host, _, err := net.SplitHostPort(string(server.Addr))
		if err != nil {
			log.Printf("Error parsing addr from config %q: %v", server.Addr, err)
			log.Printf("Warning: Let's Encrypt certificates will be fetched for any host.")
		} else {
			opt.LetsEncryptHosts = []string{host}
		}
	}
	https.ListenAndServe(ready, "upspinserver", flags.HTTPSAddr, opt)
}

var noConfig = errors.Str("no configuration")

type initMode int

const (
	startup initMode = iota
	setupServer
)

func initServer(mode initMode) (*ServerConfig, upspin.Config, error) {
	serverConfig, err := readServerConfig()
	if os.IsNotExist(err) {
		return nil, nil, noConfig
	} else if err != nil {
		return nil, nil, err
	}

	os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", filepath.Join(*cfgPath, "serviceaccount.json"))

	cfg := config.New()
	cfg = config.SetUserName(cfg, serverConfig.User)

	f, err := factotum.NewFromDir(*cfgPath)
	if err != nil {
		return nil, nil, err
	}
	cfg = config.SetFactotum(cfg, f)

	ep := upspin.Endpoint{
		Transport: upspin.Remote,
		NetAddr:   serverConfig.Addr,
	}
	cfg = config.SetDirEndpoint(cfg, ep)
	cfg = config.SetStoreEndpoint(cfg, ep)

	storeCfg := config.SetPacking(cfg, upspin.EEIntegrityPack)
	dirCfg := config.SetPacking(cfg, upspin.EEPack)

	// Set up StoreServer.
	store, err := storeServer.New("gcpBucketName="+serverConfig.Bucket, "defaultACL=publicRead")
	if err != nil {
		return nil, nil, err
	}
	store, err = perm.WrapStore(storeCfg, ready, store)
	if err != nil {
		return nil, nil, fmt.Errorf("error wrapping store: %s", err)
	}

	// Set up DirServer.
	logDir := filepath.Join(*cfgPath, "dirserver-logs")
	if err := os.MkdirAll(logDir, 0700); err != nil {
		return nil, nil, err
	}
	dir, err := dirServer.New(dirCfg, "userCacheSize=1000", "logDir="+logDir)
	if err != nil {
		return nil, nil, err
	}
	dir, err = perm.WrapDir(dirCfg, ready, serverConfig.User, dir)
	if err != nil {
		return nil, nil, fmt.Errorf("Can't wrap DirServer monitoring %s: %s", flags.StoreServerUser, err)
	}

	// Set up RPC server.
	httpStore := storeserver.New(storeCfg, store, serverConfig.Addr)
	httpDir := dirserver.New(dirCfg, dir, serverConfig.Addr)
	http.Handle("/api/Store/", httpStore)
	http.Handle("/api/Dir/", httpDir)

	log.Println("Store and Directory servers initialized.")

	if mode == setupServer {
		// Create Writers file if this was triggered by 'upspin setupserver'.
		go func() {
			if err := setupWriters(storeCfg); err != nil {
				log.Printf("Error creating Writers file: %v", err)
			}
		}()
	}
	return serverConfig, cfg, nil
}

type setupHandler struct {
	mu   sync.Mutex
	done bool
	web  http.Handler
}

func (h *setupHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	h.mu.Lock()
	if h.done {
		web := h.web
		h.mu.Unlock()
		if web != nil {
			web.ServeHTTP(w, r)
		} else {
			http.NotFound(w, r)
		}
		return
	}
	defer h.mu.Unlock()

	switch r.URL.Path {
	case "/":
		fmt.Fprint(w, "Unconfigured Upspin Server")
		return
	default:
		http.NotFound(w, r)
		return
	case "/setupserver":
		if r.Method != "POST" {
			http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
			return
		}
		// The rest of this function is the setupserver handler.
	}

	files := map[string][]byte{}
	if err := json.NewDecoder(r.Body).Decode(&files); err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	if err := os.MkdirAll(*cfgPath, 0700); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	for _, name := range configureServerFiles {
		body, ok := files[name]
		if !ok {
			http.Error(w, fmt.Sprintf("missing config file %q", name), http.StatusBadRequest)
			return
		}
		err := ioutil.WriteFile(filepath.Join(*cfgPath, name), body, 0600)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			os.RemoveAll(*cfgPath)
			return
		}
	}
	_, cfg, err := initServer(setupServer)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	fmt.Fprintf(w, "OK")

	h.done = true
	h.web = newWeb(cfg)
}

func setupWriters(cfg upspin.Config) error {
	writers, err := ioutil.ReadFile(filepath.Join(*cfgPath, "Writers"))
	if err != nil {
		return err
	}

	c := client.New(cfg)

	dir := upspin.PathName(cfg.UserName())
	if err := existsOK(c.MakeDirectory(dir)); err != nil {
		return err
	}

	dir += "/Group"
	if err := existsOK(c.MakeDirectory(dir)); err != nil {
		return err
	}

	file := dir + "/Writers"
	_, err = c.Put(file, writers)
	return err
}

// existsOK returns err if it is not an errors.Exist error,
// in which case it returns nil.
func existsOK(_ *upspin.DirEntry, err error) error {
	if errors.Match(errors.E(errors.Exist), err) {
		return nil
	}
	return err
}

func readServerConfig() (*ServerConfig, error) {
	cfgFile := filepath.Join(*cfgPath, serverConfigFile)
	b, err := ioutil.ReadFile(cfgFile)
	if err != nil {
		return nil, err
	}
	cfg := &ServerConfig{}
	if err := json.Unmarshal(b, cfg); err != nil {
		return nil, err
	}
	return cfg, nil
}

// Keep the following declarations in sync with cmd/upspin/setupserver.go.
// TODO(adg): move these to their own package if/when there are more users.

type ServerConfig struct {
	Addr   upspin.NetAddr
	User   upspin.UserName
	Bucket string
}

const serverConfigFile = "serverconfig.json"

var configureServerFiles = []string{
	"Writers",
	"public.upspinkey",
	"secret.upspinkey",
	"serverconfig.json",
	"serviceaccount.json",
}
<|endoftext|>


<commit_before>package stathatrecipient

import (
	"bytes"
	"fmt"
	"strings"
	"text/template"

	"github.com/gentlemanautomaton/stathat"
	"github.com/scjalliance/power"
)

// DefaultFormat is the default StatHat statistic naming format.
const DefaultFormat = "{{.Source.Host}} {{.Stat.Name}}"

// Recipient is a StatHat recipient of power management values. It contains the
// ezkey and naming template.
type Recipient struct {
	ezkey string
	t     *template.Template // StatHat stat naming template parsed by text/template
}

// New returns a new StatHat recipient for the given ezkey. The default naming
// template is used.
func New(ezkey string) *Recipient {
	return &Recipient{
		ezkey: ezkey,
		t:     template.Must(template.New("stathat").Parse(DefaultFormat)),
	}
}

// NewWithNameTemplate returns a new StatHat recipient for the given ezkey and
// stat name template.
func NewWithNameTemplate(ezkey, nameTemplate string) (*Recipient, error) {
	t, err := template.New("stathat").Parse(nameTemplate)
	if err != nil {
		return nil, err
	}
	return &Recipient{
		ezkey: ezkey,
		t:     t,
	}, nil
}

// Send will send the report to the configured destination.
func (r *Recipient) Send(v power.Value) {
	if v.Err != nil {
		// Don't report stats that weren't collected successfully
		return
	}

	name := r.StatName(v)

	fmt.Printf("Sending data for \"%s\" to StatHat...", name)
	reporter := stathat.New().EZKey(r.ezkey)
	err := reporter.PostEZ(name, stathat.KindValue, v.Value, &v.Time)
	if err != nil {
		// TODO: Try again later?
		fmt.Printf("failed: %v\n", err)
	} else {
		fmt.Printf("done\n")
	}
}

// StatName returns the formatted name of the statistic in StatHat.
func (r *Recipient) StatName(v power.Value) string {
	var buf bytes.Buffer
	r.t.Execute(&buf, v)
	return buf.String()
}

// Parse will parse the given address, which is a string containing the StatHat
// ezkey.
func Parse(address string) (power.Recipient, error) {
	if elements := strings.SplitN(address, "~", 2); len(elements) == 2 {
		ezkey, tmpl := elements[0], elements[1]
		return NewWithNameTemplate(ezkey, tmpl)
	}
	return New(address), nil
}
<commit_msg>Improved logging for stathatrecipient<commit_after>package stathatrecipient

import (
	"bytes"
	"fmt"
	"strconv"
	"strings"
	"text/template"

	"github.com/gentlemanautomaton/stathat"
	"github.com/scjalliance/power"
)

// DefaultFormat is the default StatHat statistic naming format.
const DefaultFormat = "{{.Source.Host}} {{.Stat.Name}}"

// Recipient is a StatHat recipient of power management values. It contains the
// ezkey and naming template.
type Recipient struct {
	ezkey string
	t     *template.Template // StatHat stat naming template parsed by text/template
}

// New returns a new StatHat recipient for the given ezkey. The default naming
// template is used.
func New(ezkey string) *Recipient {
	return &Recipient{
		ezkey: ezkey,
		t:     template.Must(template.New("stathat").Parse(DefaultFormat)),
	}
}

// NewWithNameTemplate returns a new StatHat recipient for the given ezkey and
// stat name template.
func NewWithNameTemplate(ezkey, nameTemplate string) (*Recipient, error) {
	t, err := template.New("stathat").Parse(nameTemplate)
	if err != nil {
		return nil, err
	}
	return &Recipient{
		ezkey: ezkey,
		t:     t,
	}, nil
}

// Send will send the report to the configured destination.
func (r *Recipient) Send(v power.Value) {
	if v.Err != nil {
		// Don't report stats that weren't collected successfully
		return
	}

	name := r.StatName(v)

	fmt.Printf("Sending \"%s\" value %s to StatHat...", name, strconv.FormatFloat(v.Value, 'f', -1, 64))
	reporter := stathat.New().EZKey(r.ezkey)
	err := reporter.PostEZ(name, stathat.KindValue, v.Value, &v.Time)
	if err != nil {
		// TODO: Try again later?
		fmt.Printf("failed: %v\n", err)
	} else {
		fmt.Printf("done\n")
	}
}

// StatName returns the formatted name of the statistic in StatHat.
func (r *Recipient) StatName(v power.Value) string {
	var buf bytes.Buffer
	r.t.Execute(&buf, v)
	return buf.String()
}

// Parse will parse the given address, which is a string containing the StatHat
// ezkey.
func Parse(address string) (power.Recipient, error) {
	if elements := strings.SplitN(address, "~", 2); len(elements) == 2 {
		ezkey, tmpl := elements[0], elements[1]
		return NewWithNameTemplate(ezkey, tmpl)
	}
	return New(address), nil
}
<|endoftext|>


<commit_before>package agent

import (
	"github.com/marouenj/consul/consul/structs"
)

// ServiceDefinition is used to JSON decode the Service definitions
type ServiceDefinition struct {
	ID      string
	Name    string
	Tags    []string
	Address string
	Port    int
	Check   CheckType
	Checks  CheckTypes
}

func (s *ServiceDefinition) NodeService() *structs.NodeService {
	ns := &structs.NodeService{
		ID:      s.ID,
		Service: s.Name,
		Tags:    s.Tags,
		Address: s.Address,
		Port:    s.Port,
	}
	if ns.ID == "" && ns.Service != "" {
		ns.ID = ns.Service
	}
	return ns
}

func (s *ServiceDefinition) CheckTypes() (checks CheckTypes) {
	s.Checks = append(s.Checks, &s.Check)
	for _, check := range s.Checks {
		if check.Valid() {
			checks = append(checks, check)
		}
	}
	return
}

// ArchetypeDefinition is used to JSON decode the Archetype definitions
type ArchetypeDefinition struct {
	ID        string // identifies this definition
	PoolName  string // identifies an archetype. The convention is that definitions with the same PoolName relate to the same pool, 'redis_pool' for example
	Tags      []string
	Address   string
	Port      int
	Check     CheckType  // potentially a template, to be rendered by consul-template
	Checks    CheckTypes // ditto
	Templates []string   // templates for the config files of the service this archetype abstracts, to be rendered by consul-template
	Command   string     // launcher of the service this archetype abstracts, to be executed by consul-template
}

func (a *ArchetypeDefinition) NodeArchetype() *structs.NodeArchetype {
	na := &structs.NodeArchetype{
		ID:        a.ID,
		Archetype: a.PoolName,
		Tags:      a.Tags,
		Address:   a.Address,
		Port:      a.Port,
	}
	if na.ID == "" && na.Archetype != "" {
		na.ID = na.Archetype
	}
	return na
}

// ChecKDefinition is used to JSON decode the Check definitions
type CheckDefinition struct {
	ID        string
	Name      string
	Notes     string
	ServiceID string
	CheckType `mapstructure:",squash"`
}

func (c *CheckDefinition) HealthCheck(node string) *structs.HealthCheck {
	health := &structs.HealthCheck{
		Node:      node,
		CheckID:   c.ID,
		Name:      c.Name,
		Status:    structs.HealthCritical,
		Notes:     c.Notes,
		ServiceID: c.ServiceID,
	}
	if health.CheckID == "" && health.Name != "" {
		health.CheckID = health.Name
	}
	return health
}
<commit_msg>modify archetype definition's structure for representing a consul template<commit_after>package agent

import (
	"github.com/marouenj/consul-template/core"
	"github.com/marouenj/consul/consul/structs"
)

// ServiceDefinition is used to JSON decode the Service definitions
type ServiceDefinition struct {
	ID      string
	Name    string
	Tags    []string
	Address string
	Port    int
	Check   CheckType
	Checks  CheckTypes
}

func (s *ServiceDefinition) NodeService() *structs.NodeService {
	ns := &structs.NodeService{
		ID:      s.ID,
		Service: s.Name,
		Tags:    s.Tags,
		Address: s.Address,
		Port:    s.Port,
	}
	if ns.ID == "" && ns.Service != "" {
		ns.ID = ns.Service
	}
	return ns
}

func (s *ServiceDefinition) CheckTypes() (checks CheckTypes) {
	s.Checks = append(s.Checks, &s.Check)
	for _, check := range s.Checks {
		if check.Valid() {
			checks = append(checks, check)
		}
	}
	return
}

// ArchetypeDefinition is used to JSON decode the Archetype definitions
type ArchetypeDefinition struct {
	ID       string // identifies this definition
	PoolName string // identifies an archetype. The convention is that definitions with the same PoolName relate to the same pool, 'redis_pool' for example
	Tags     []string
	Address  string
	Port     int
	Check    CheckType           // potentially a template, to be rendered by consul-template
	Checks   CheckTypes          // ditto
	Template core.ConfigTemplate // template for the config file of the service this archetype abstracts, to be rendered by consul-template
}

func (a *ArchetypeDefinition) NodeArchetype() *structs.NodeArchetype {
	na := &structs.NodeArchetype{
		ID:        a.ID,
		Archetype: a.PoolName,
		Tags:      a.Tags,
		Address:   a.Address,
		Port:      a.Port,
	}
	if na.ID == "" && na.Archetype != "" {
		na.ID = na.Archetype
	}
	return na
}

// ChecKDefinition is used to JSON decode the Check definitions
type CheckDefinition struct {
	ID        string
	Name      string
	Notes     string
	ServiceID string
	CheckType `mapstructure:",squash"`
}

func (c *CheckDefinition) HealthCheck(node string) *structs.HealthCheck {
	health := &structs.HealthCheck{
		Node:      node,
		CheckID:   c.ID,
		Name:      c.Name,
		Status:    structs.HealthCritical,
		Notes:     c.Notes,
		ServiceID: c.ServiceID,
	}
	if health.CheckID == "" && health.Name != "" {
		health.CheckID = health.Name
	}
	return health
}
<|endoftext|>


<commit_before>package build

import (
	"bytes"
	"flag"
	"fmt"
	cmdcommon "github.com/mitchellh/packer/common/command"
	"github.com/mitchellh/packer/packer"
	"log"
	"os"
	"os/signal"
	"strings"
	"sync"
)

type Command byte

func (Command) Help() string {
	return strings.TrimSpace(helpText)
}

func (c Command) Run(env packer.Environment, args []string) int {
	var cfgDebug bool
	var cfgForce bool
	buildOptions := new(cmdcommon.BuildOptions)

	cmdFlags := flag.NewFlagSet("build", flag.ContinueOnError)
	cmdFlags.Usage = func() { env.Ui().Say(c.Help()) }
	cmdFlags.BoolVar(&cfgDebug, "debug", false, "debug mode for builds")
	cmdFlags.BoolVar(&cfgForce, "force", false, "force a build if artifacts exist")
	cmdcommon.BuildOptionFlags(cmdFlags, buildOptions)
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	args = cmdFlags.Args()
	if len(args) != 1 {
		cmdFlags.Usage()
		return 1
	}

	if err := buildOptions.Validate(); err != nil {
		env.Ui().Error(err.Error())
		env.Ui().Error("")
		env.Ui().Error(c.Help())
		return 1
	}

	userVars, err := buildOptions.AllUserVars()
	if err != nil {
		env.Ui().Error(fmt.Sprintf("Error compiling user variables: %s", err))
		env.Ui().Error("")
		env.Ui().Error(c.Help())
		return 1
	}

	// Read the file into a byte array so that we can parse the template
	log.Printf("Reading template: %s", args[0])
	tpl, err := packer.ParseTemplateFile(args[0])
	if err != nil {
		env.Ui().Error(fmt.Sprintf("Failed to parse template: %s", err))
		return 1
	}

	// The component finder for our builds
	components := &packer.ComponentFinder{
		Builder:       env.Builder,
		Hook:          env.Hook,
		PostProcessor: env.PostProcessor,
		Provisioner:   env.Provisioner,
	}

	// Go through each builder and compile the builds that we care about
	builds, err := buildOptions.Builds(tpl, components)
	if err != nil {
		env.Ui().Error(err.Error())
		return 1
	}

	if cfgDebug {
		env.Ui().Say("Debug mode enabled. Builds will not be parallelized.")
	}

	// Compile all the UIs for the builds
	colors := [5]packer.UiColor{
		packer.UiColorGreen,
		packer.UiColorCyan,
		packer.UiColorMagenta,
		packer.UiColorYellow,
		packer.UiColorBlue,
	}

	buildUis := make(map[string]packer.Ui)
	for i, b := range builds {
		ui := &packer.ColoredUi{
			Color: colors[i%len(colors)],
			Ui:    env.Ui(),
		}

		buildUis[b.Name()] = ui
		ui.Say(fmt.Sprintf("%s output will be in this color.", b.Name()))
	}

	// Add a newline between the color output and the actual output
	env.Ui().Say("")

	log.Printf("Build debug mode: %v", cfgDebug)
	log.Printf("Force build: %v", cfgForce)

	// Set the debug and force mode and prepare all the builds
	for _, b := range builds {
		log.Printf("Preparing build: %s", b.Name())
		b.SetDebug(cfgDebug)
		b.SetForce(cfgForce)
		err := b.Prepare(userVars)
		if err != nil {
			env.Ui().Error(err.Error())
			return 1
		}
	}

	// Run all the builds in parallel and wait for them to complete
	var interruptWg, wg sync.WaitGroup
	interrupted := false
	artifacts := make(map[string][]packer.Artifact)
	errors := make(map[string]error)
	for _, b := range builds {
		// Increment the waitgroup so we wait for this item to finish properly
		wg.Add(1)

		// Handle interrupts for this build
		sigCh := make(chan os.Signal, 1)
		signal.Notify(sigCh, os.Interrupt)
		defer signal.Stop(sigCh)
		go func(b packer.Build) {
			<-sigCh
			interruptWg.Add(1)
			defer interruptWg.Done()
			interrupted = true

			log.Printf("Stopping build: %s", b.Name())
			b.Cancel()
			log.Printf("Build cancelled: %s", b.Name())
		}(b)

		// Run the build in a goroutine
		go func(b packer.Build) {
			defer wg.Done()

			name := b.Name()
			log.Printf("Starting build run: %s", name)
			ui := buildUis[name]
			runArtifacts, err := b.Run(ui, env.Cache())

			if err != nil {
				ui.Error(fmt.Sprintf("Build '%s' errored: %s", name, err))
				errors[name] = err
			} else {
				ui.Say(fmt.Sprintf("Build '%s' finished.", name))
				artifacts[name] = runArtifacts
			}
		}(b)

		if cfgDebug {
			log.Printf("Debug enabled, so waiting for build to finish: %s", b.Name())
			wg.Wait()
		}

		if interrupted {
			log.Println("Interrupted, not going to start any more builds.")
			break
		}
	}

	// Wait for both the builds to complete and the interrupt handler,
	// if it is interrupted.
	log.Printf("Waiting on builds to complete...")
	wg.Wait()

	log.Printf("Builds completed. Waiting on interrupt barrier...")
	interruptWg.Wait()

	if interrupted {
		env.Ui().Say("Cleanly cancelled builds after being interrupted.")
		return 1
	}

	if len(errors) > 0 {
		env.Ui().Error("\n==> Some builds didn't complete successfully and had errors:")
		for name, err := range errors {
			env.Ui().Error(fmt.Sprintf("--> %s: %s", name, err))
		}
	}

	if len(artifacts) > 0 {
		env.Ui().Say("\n==> Builds finished. The artifacts of successful builds are:")
		for name, buildArtifacts := range artifacts {
			for _, artifact := range buildArtifacts {
				var message bytes.Buffer
				fmt.Fprintf(&message, "--> %s: ", name)

				if artifact != nil {
					fmt.Fprintf(&message, artifact.String())
				} else {
					fmt.Fprint(&message, "<nothing>")
				}

				env.Ui().Say(message.String())
			}
		}
	} else {
		env.Ui().Say("\n==> Builds finished but no artifacts were created.")
	}

	if len(errors) > 0 {
		// If any errors occurred, exit with a non-zero exit status
		return 1
	}

	return 0
}

func (Command) Synopsis() string {
	return "build image(s) from template"
}
<commit_msg>command/build: machine-readable artifacts<commit_after>package build

import (
	"bytes"
	"flag"
	"fmt"
	cmdcommon "github.com/mitchellh/packer/common/command"
	"github.com/mitchellh/packer/packer"
	"log"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"sync"
)

type Command byte

func (Command) Help() string {
	return strings.TrimSpace(helpText)
}

func (c Command) Run(env packer.Environment, args []string) int {
	var cfgDebug bool
	var cfgForce bool
	buildOptions := new(cmdcommon.BuildOptions)

	cmdFlags := flag.NewFlagSet("build", flag.ContinueOnError)
	cmdFlags.Usage = func() { env.Ui().Say(c.Help()) }
	cmdFlags.BoolVar(&cfgDebug, "debug", false, "debug mode for builds")
	cmdFlags.BoolVar(&cfgForce, "force", false, "force a build if artifacts exist")
	cmdcommon.BuildOptionFlags(cmdFlags, buildOptions)
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	args = cmdFlags.Args()
	if len(args) != 1 {
		cmdFlags.Usage()
		return 1
	}

	if err := buildOptions.Validate(); err != nil {
		env.Ui().Error(err.Error())
		env.Ui().Error("")
		env.Ui().Error(c.Help())
		return 1
	}

	userVars, err := buildOptions.AllUserVars()
	if err != nil {
		env.Ui().Error(fmt.Sprintf("Error compiling user variables: %s", err))
		env.Ui().Error("")
		env.Ui().Error(c.Help())
		return 1
	}

	// Read the file into a byte array so that we can parse the template
	log.Printf("Reading template: %s", args[0])
	tpl, err := packer.ParseTemplateFile(args[0])
	if err != nil {
		env.Ui().Error(fmt.Sprintf("Failed to parse template: %s", err))
		return 1
	}

	// The component finder for our builds
	components := &packer.ComponentFinder{
		Builder:       env.Builder,
		Hook:          env.Hook,
		PostProcessor: env.PostProcessor,
		Provisioner:   env.Provisioner,
	}

	// Go through each builder and compile the builds that we care about
	builds, err := buildOptions.Builds(tpl, components)
	if err != nil {
		env.Ui().Error(err.Error())
		return 1
	}

	if cfgDebug {
		env.Ui().Say("Debug mode enabled. Builds will not be parallelized.")
	}

	// Compile all the UIs for the builds
	colors := [5]packer.UiColor{
		packer.UiColorGreen,
		packer.UiColorCyan,
		packer.UiColorMagenta,
		packer.UiColorYellow,
		packer.UiColorBlue,
	}

	buildUis := make(map[string]packer.Ui)
	for i, b := range builds {
		ui := &packer.ColoredUi{
			Color: colors[i%len(colors)],
			Ui:    env.Ui(),
		}

		buildUis[b.Name()] = ui
		ui.Say(fmt.Sprintf("%s output will be in this color.", b.Name()))
	}

	// Add a newline between the color output and the actual output
	env.Ui().Say("")

	log.Printf("Build debug mode: %v", cfgDebug)
	log.Printf("Force build: %v", cfgForce)

	// Set the debug and force mode and prepare all the builds
	for _, b := range builds {
		log.Printf("Preparing build: %s", b.Name())
		b.SetDebug(cfgDebug)
		b.SetForce(cfgForce)
		err := b.Prepare(userVars)
		if err != nil {
			env.Ui().Error(err.Error())
			return 1
		}
	}

	// Run all the builds in parallel and wait for them to complete
	var interruptWg, wg sync.WaitGroup
	interrupted := false
	artifacts := make(map[string][]packer.Artifact)
	errors := make(map[string]error)
	for _, b := range builds {
		// Increment the waitgroup so we wait for this item to finish properly
		wg.Add(1)

		// Handle interrupts for this build
		sigCh := make(chan os.Signal, 1)
		signal.Notify(sigCh, os.Interrupt)
		defer signal.Stop(sigCh)
		go func(b packer.Build) {
			<-sigCh
			interruptWg.Add(1)
			defer interruptWg.Done()
			interrupted = true

			log.Printf("Stopping build: %s", b.Name())
			b.Cancel()
			log.Printf("Build cancelled: %s", b.Name())
		}(b)

		// Run the build in a goroutine
		go func(b packer.Build) {
			defer wg.Done()

			name := b.Name()
			log.Printf("Starting build run: %s", name)
			ui := buildUis[name]
			runArtifacts, err := b.Run(ui, env.Cache())

			if err != nil {
				ui.Error(fmt.Sprintf("Build '%s' errored: %s", name, err))
				errors[name] = err
			} else {
				ui.Say(fmt.Sprintf("Build '%s' finished.", name))
				artifacts[name] = runArtifacts
			}
		}(b)

		if cfgDebug {
			log.Printf("Debug enabled, so waiting for build to finish: %s", b.Name())
			wg.Wait()
		}

		if interrupted {
			log.Println("Interrupted, not going to start any more builds.")
			break
		}
	}

	// Wait for both the builds to complete and the interrupt handler,
	// if it is interrupted.
	log.Printf("Waiting on builds to complete...")
	wg.Wait()

	log.Printf("Builds completed. Waiting on interrupt barrier...")
	interruptWg.Wait()

	if interrupted {
		env.Ui().Say("Cleanly cancelled builds after being interrupted.")
		return 1
	}

	if len(errors) > 0 {
		env.Ui().Error("\n==> Some builds didn't complete successfully and had errors:")
		for name, err := range errors {
			env.Ui().Error(fmt.Sprintf("--> %s: %s", name, err))
		}
	}

	if len(artifacts) > 0 {
		env.Ui().Say("\n==> Builds finished. The artifacts of successful builds are:")
		for name, buildArtifacts := range artifacts {
			// Create a UI for the machine readable stuff to be targetted
			ui := &packer.TargettedUi{
				Target: name,
				Ui:     env.Ui(),
			}

			// Machine-readable helpful
			ui.Machine("artifact-count", strconv.FormatInt(int64(len(buildArtifacts)), 10))

			for i, artifact := range buildArtifacts {
				var message bytes.Buffer
				fmt.Fprintf(&message, "--> %s: ", name)

				if artifact != nil {
					fmt.Fprintf(&message, artifact.String())
				} else {
					fmt.Fprint(&message, "<nothing>")
				}

				iStr := strconv.FormatInt(int64(i), 10)
				if artifact != nil {
					ui.Machine("artifact", iStr, "builder-id", artifact.BuilderId())
					ui.Machine("artifact", iStr, "id", artifact.Id())
					ui.Machine("artifact", iStr, "string", message.String())

					files := artifact.Files()
					ui.Machine("artifact",
						iStr,
						"files-count", strconv.FormatInt(int64(len(files)), 10))
					for fi, file := range files {
						fiStr := strconv.FormatInt(int64(fi), 10)
						ui.Machine("artifact", iStr, "file", fiStr, file)
					}
				} else {
					ui.Machine("artifact", iStr, "nil")
				}

				env.Ui().Say(message.String())
			}
		}
	} else {
		env.Ui().Say("\n==> Builds finished but no artifacts were created.")
	}

	if len(errors) > 0 {
		// If any errors occurred, exit with a non-zero exit status
		return 1
	}

	return 0
}

func (Command) Synopsis() string {
	return "build image(s) from template"
}
<|endoftext|>


<commit_before>package command

import (
	"crypto/rand"
	"encoding/base64"
	"fmt"
	"os"
	"strings"

	"github.com/hashicorp/go-uuid"
	"github.com/hashicorp/vault/api"
	"github.com/hashicorp/vault/helper/password"
	"github.com/hashicorp/vault/helper/pgpkeys"
	"github.com/hashicorp/vault/helper/xor"
	"github.com/hashicorp/vault/meta"
)

// GenerateRootCommand is a Command that generates a new root token.
type GenerateRootCommand struct {
	meta.Meta

	// Key can be used to pre-seed the key. If it is set, it will not
	// be asked with the `password` helper.
	Key string

	// The nonce for the rekey request to send along
	Nonce string
}

func (c *GenerateRootCommand) Run(args []string) int {
	var init, cancel, status, genotp bool
	var nonce, decode, otp, pgpKey string
	var pgpKeyArr pgpkeys.PubKeyFilesFlag
	flags := c.Meta.FlagSet("generate-root", meta.FlagSetDefault)
	flags.BoolVar(&init, "init", false, "")
	flags.BoolVar(&cancel, "cancel", false, "")
	flags.BoolVar(&status, "status", false, "")
	flags.BoolVar(&genotp, "genotp", false, "")
	flags.StringVar(&decode, "decode", "", "")
	flags.StringVar(&otp, "otp", "", "")
	flags.StringVar(&nonce, "nonce", "", "")
	flags.Var(&pgpKeyArr, "pgp-key", "")
	flags.Usage = func() { c.Ui.Error(c.Help()) }
	if err := flags.Parse(args); err != nil {
		return 1
	}

	if genotp {
		buf := make([]byte, 16)
		readLen, err := rand.Read(buf)
		if err != nil {
			c.Ui.Error(fmt.Sprintf("Error reading random bytes: %s", err))
			return 1
		}
		if readLen != 16 {
			c.Ui.Error(fmt.Sprintf("Read %d bytes when we should have read 16", readLen))
			return 1
		}
		c.Ui.Output(fmt.Sprintf("OTP: %s", base64.StdEncoding.EncodeToString(buf)))
		return 0
	}

	if len(decode) > 0 {
		if len(otp) == 0 {
			c.Ui.Error("Both the value to decode and the OTP must be passed in")
			return 1
		}
		return c.decode(decode, otp)
	}

	client, err := c.Client()
	if err != nil {
		c.Ui.Error(fmt.Sprintf(
			"Error initializing client: %s", err))
		return 2
	}

	// Check if the root generation is started
	rootGenerationStatus, err := client.Sys().GenerateRootStatus()
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error reading root generation status: %s", err))
		return 1
	}

	// If we are initing, or if we are not started but are not running a
	// special function, check otp and pgpkey
	checkOtpPgp := false
	switch {
	case init:
		checkOtpPgp = true
	case cancel:
	case status:
	case genotp:
	case len(decode) != 0:
	case rootGenerationStatus.Started:
	default:
		checkOtpPgp = true
	}
	if checkOtpPgp {
		switch {
		case len(otp) == 0 && (pgpKeyArr == nil || len(pgpKeyArr) == 0):
			c.Ui.Error(c.Help())
			return 1
		case len(otp) != 0 && pgpKeyArr != nil && len(pgpKeyArr) != 0:
			c.Ui.Error(c.Help())
			return 1
		case len(otp) != 0:
			err := c.verifyOTP(otp)
			if err != nil {
				c.Ui.Error(fmt.Sprintf("Error verifying the provided OTP: %s", err))
				return 1
			}
		case pgpKeyArr != nil:
			if len(pgpKeyArr) != 1 {
				c.Ui.Error("Could not parse PGP key")
				return 1
			}
			if len(pgpKeyArr[0]) == 0 {
				c.Ui.Error("Got an empty PGP key")
				return 1
			}
			pgpKey = pgpKeyArr[0]
		default:
			panic("unreachable case")
		}
	}

	if nonce != "" {
		c.Nonce = nonce
	}

	// Check if we are running doing any restricted variants
	switch {
	case init:
		return c.initGenerateRoot(client, otp, pgpKey)
	case cancel:
		return c.cancelGenerateRoot(client)
	case status:
		return c.rootGenerationStatus(client)
	}

	// Start the root generation process if not started
	if !rootGenerationStatus.Started {
		rootGenerationStatus, err = client.Sys().GenerateRootInit(otp, pgpKey)
		if err != nil {
			c.Ui.Error(fmt.Sprintf("Error initializing root generation: %s", err))
			return 1
		}
		c.Nonce = rootGenerationStatus.Nonce
	}

	serverNonce := rootGenerationStatus.Nonce

	// Get the unseal key
	args = flags.Args()
	key := c.Key
	if len(args) > 0 {
		key = args[0]
	}
	if key == "" {
		c.Nonce = serverNonce
		fmt.Printf("Root generation operation nonce: %s\n", serverNonce)
		fmt.Printf("Key (will be hidden): ")
		key, err = password.Read(os.Stdin)
		fmt.Printf("\n")
		if err != nil {
			c.Ui.Error(fmt.Sprintf(
				"Error attempting to ask for password. The raw error message\n"+
					"is shown below, but the most common reason for this error is\n"+
					"that you attempted to pipe a value into unseal or you're\n"+
					"executing `vault generate-root` from outside of a terminal.\n\n"+
					"You should use `vault generate-root` from a terminal for maximum\n"+
					"security. If this isn't an option, the unseal key can be passed\n"+
					"in using the first parameter.\n\n"+
					"Raw error: %s", err))
			return 1
		}
	}

	// Provide the key, this may potentially complete the update
	statusResp, err := client.Sys().GenerateRootUpdate(strings.TrimSpace(key), c.Nonce)
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error attempting generate-root update: %s", err))
		return 1
	}

	c.dumpStatus(statusResp)

	return 0
}

func (c *GenerateRootCommand) verifyOTP(otp string) error {
	if len(otp) == 0 {
		return fmt.Errorf("No OTP passed in")
	}
	otpBytes, err := base64.StdEncoding.DecodeString(otp)
	if err != nil {
		return fmt.Errorf("Error decoding base64 OTP value: %s", err)
	}
	if otpBytes == nil || len(otpBytes) != 16 {
		return fmt.Errorf("Decoded OTP value is invalid or wrong length")
	}

	return nil
}

func (c *GenerateRootCommand) decode(encodedVal, otp string) int {
	tokenBytes, err := xor.XORBase64(encodedVal, otp)
	if err != nil {
		c.Ui.Error(err.Error())
		return 1
	}

	token, err := uuid.FormatUUID(tokenBytes)
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error formatting base64 token value: %v", err))
		return 1
	}

	c.Ui.Output(fmt.Sprintf("Root token: %s", token))

	return 0
}

// initGenerateRoot is used to start the generation process
func (c *GenerateRootCommand) initGenerateRoot(client *api.Client, otp string, pgpKey string) int {
	// Start the rekey
	status, err := client.Sys().GenerateRootInit(otp, pgpKey)
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error initializing root generation: %s", err))
		return 1
	}

	c.dumpStatus(status)

	return 0
}

// cancelGenerateRoot is used to abort the generation process
func (c *GenerateRootCommand) cancelGenerateRoot(client *api.Client) int {
	err := client.Sys().GenerateRootCancel()
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Failed to cancel root generation: %s", err))
		return 1
	}
	c.Ui.Output("Root generation canceled.")
	return 0
}

// rootGenerationStatus is used just to fetch and dump the status
func (c *GenerateRootCommand) rootGenerationStatus(client *api.Client) int {
	// Check the status
	status, err := client.Sys().GenerateRootStatus()
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error reading root generation status: %s", err))
		return 1
	}

	c.dumpStatus(status)

	return 0
}

// dumpStatus dumps the status to output
func (c *GenerateRootCommand) dumpStatus(status *api.GenerateRootStatusResponse) {
	// Dump the status
	statString := fmt.Sprintf(
		"Nonce: %s\n"+
			"Started: %v\n"+
			"Rekey Progress: %d\n"+
			"Required Keys: %d\n"+
			"Complete: %t",
		status.Nonce,
		status.Started,
		status.Progress,
		status.Required,
		status.Complete,
	)
	if len(status.PGPFingerprint) > 0 {
		statString = fmt.Sprintf("%s\nPGP Fingerprint: %s", statString, status.PGPFingerprint)
	}
	if len(status.EncodedRootToken) > 0 {
		statString = fmt.Sprintf("%s\n\nEncoded root token: %s", statString, status.EncodedRootToken)
	}
	c.Ui.Output(statString)
}

func (c *GenerateRootCommand) Synopsis() string {
	return "Generates a new root token"
}

func (c *GenerateRootCommand) Help() string {
	helpText := `
Usage: vault generate-root [options] [key]

  'generate-root' is used to create a new root token.

  Root generation can only be done when the Vault is already unsealed. The
  operation is done online, but requires that a threshold of the current unseal
  keys be provided.

  One (and only one) of the following must be provided at attempt
  initialization time:

  1) A 16-byte, base64-encoded One Time Password (OTP) provided in the '-otp'
  flag; the token is XOR'd with this value before it is returned once the final
  unseal key has been provided. The '-decode' operation can be used with this
  value and the OTP to output the final token value. The '-genotp' flag can be
  used to generate a suitable value.

  or

  2) A file containing a PGP key (binary or base64-encoded) or a Keybase.io
  username in the format of "keybase:<username>" in the '-pgp-key' flag. The
  final token value will be encrypted with this public key and base64-encoded.

General Options:
` + meta.GeneralOptionsUsage() + `
Rekey Options:

  -init                   Initialize the root generation attempt. This can only
                          be done if no generation is already initiated.

  -cancel                 Reset the root generation process by throwing away
                          prior unseal keys and the configuration.

  -status                 Prints the status of the current attempt. This can be
                          used to see the status without attempting to provide
                          an unseal key.

  -decode=abcd            Decodes and outputs the generated root token. The OTP
                          used at '-init' time must be provided in the '-otp'
                          parameter.

  -genotp                 Returns a high-quality OTP suitable for passing into
                          the '-init' method.

  -otp=abcd               The base64-encoded 16-byte OTP for use with the
                          '-init' or '-decode' methods.

  -pgp-key                A file on disk containing a binary- or base64-format
                          public PGP key, or a Keybase username specified as
                          "keybase:<username>". The output root token will be
                          encrypted and base64-encoded, in order, with the given
                          public key.

  -nonce=abcd             The nonce provided at initialization time. This same
                          nonce value must be provided with each unseal key. If
                          the unseal key is not being passed in via the command
                          line the nonce parameter is not required, and will
                          instead be displayed with the key prompt.
`
	return strings.TrimSpace(helpText)
}
<commit_msg>Fix generate-root help and progress output.<commit_after>package command

import (
	"crypto/rand"
	"encoding/base64"
	"fmt"
	"os"
	"strings"

	"github.com/hashicorp/go-uuid"
	"github.com/hashicorp/vault/api"
	"github.com/hashicorp/vault/helper/password"
	"github.com/hashicorp/vault/helper/pgpkeys"
	"github.com/hashicorp/vault/helper/xor"
	"github.com/hashicorp/vault/meta"
)

// GenerateRootCommand is a Command that generates a new root token.
type GenerateRootCommand struct {
	meta.Meta

	// Key can be used to pre-seed the key. If it is set, it will not
	// be asked with the `password` helper.
	Key string

	// The nonce for the rekey request to send along
	Nonce string
}

func (c *GenerateRootCommand) Run(args []string) int {
	var init, cancel, status, genotp bool
	var nonce, decode, otp, pgpKey string
	var pgpKeyArr pgpkeys.PubKeyFilesFlag
	flags := c.Meta.FlagSet("generate-root", meta.FlagSetDefault)
	flags.BoolVar(&init, "init", false, "")
	flags.BoolVar(&cancel, "cancel", false, "")
	flags.BoolVar(&status, "status", false, "")
	flags.BoolVar(&genotp, "genotp", false, "")
	flags.StringVar(&decode, "decode", "", "")
	flags.StringVar(&otp, "otp", "", "")
	flags.StringVar(&nonce, "nonce", "", "")
	flags.Var(&pgpKeyArr, "pgp-key", "")
	flags.Usage = func() { c.Ui.Error(c.Help()) }
	if err := flags.Parse(args); err != nil {
		return 1
	}

	if genotp {
		buf := make([]byte, 16)
		readLen, err := rand.Read(buf)
		if err != nil {
			c.Ui.Error(fmt.Sprintf("Error reading random bytes: %s", err))
			return 1
		}
		if readLen != 16 {
			c.Ui.Error(fmt.Sprintf("Read %d bytes when we should have read 16", readLen))
			return 1
		}
		c.Ui.Output(fmt.Sprintf("OTP: %s", base64.StdEncoding.EncodeToString(buf)))
		return 0
	}

	if len(decode) > 0 {
		if len(otp) == 0 {
			c.Ui.Error("Both the value to decode and the OTP must be passed in")
			return 1
		}
		return c.decode(decode, otp)
	}

	client, err := c.Client()
	if err != nil {
		c.Ui.Error(fmt.Sprintf(
			"Error initializing client: %s", err))
		return 2
	}

	// Check if the root generation is started
	rootGenerationStatus, err := client.Sys().GenerateRootStatus()
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error reading root generation status: %s", err))
		return 1
	}

	// If we are initing, or if we are not started but are not running a
	// special function, check otp and pgpkey
	checkOtpPgp := false
	switch {
	case init:
		checkOtpPgp = true
	case cancel:
	case status:
	case genotp:
	case len(decode) != 0:
	case rootGenerationStatus.Started:
	default:
		checkOtpPgp = true
	}
	if checkOtpPgp {
		switch {
		case len(otp) == 0 && (pgpKeyArr == nil || len(pgpKeyArr) == 0):
			c.Ui.Error(c.Help())
			return 1
		case len(otp) != 0 && pgpKeyArr != nil && len(pgpKeyArr) != 0:
			c.Ui.Error(c.Help())
			return 1
		case len(otp) != 0:
			err := c.verifyOTP(otp)
			if err != nil {
				c.Ui.Error(fmt.Sprintf("Error verifying the provided OTP: %s", err))
				return 1
			}
		case pgpKeyArr != nil:
			if len(pgpKeyArr) != 1 {
				c.Ui.Error("Could not parse PGP key")
				return 1
			}
			if len(pgpKeyArr[0]) == 0 {
				c.Ui.Error("Got an empty PGP key")
				return 1
			}
			pgpKey = pgpKeyArr[0]
		default:
			panic("unreachable case")
		}
	}

	if nonce != "" {
		c.Nonce = nonce
	}

	// Check if we are running doing any restricted variants
	switch {
	case init:
		return c.initGenerateRoot(client, otp, pgpKey)
	case cancel:
		return c.cancelGenerateRoot(client)
	case status:
		return c.rootGenerationStatus(client)
	}

	// Start the root generation process if not started
	if !rootGenerationStatus.Started {
		rootGenerationStatus, err = client.Sys().GenerateRootInit(otp, pgpKey)
		if err != nil {
			c.Ui.Error(fmt.Sprintf("Error initializing root generation: %s", err))
			return 1
		}
		c.Nonce = rootGenerationStatus.Nonce
	}

	serverNonce := rootGenerationStatus.Nonce

	// Get the unseal key
	args = flags.Args()
	key := c.Key
	if len(args) > 0 {
		key = args[0]
	}
	if key == "" {
		c.Nonce = serverNonce
		fmt.Printf("Root generation operation nonce: %s\n", serverNonce)
		fmt.Printf("Key (will be hidden): ")
		key, err = password.Read(os.Stdin)
		fmt.Printf("\n")
		if err != nil {
			c.Ui.Error(fmt.Sprintf(
				"Error attempting to ask for password. The raw error message\n"+
					"is shown below, but the most common reason for this error is\n"+
					"that you attempted to pipe a value into unseal or you're\n"+
					"executing `vault generate-root` from outside of a terminal.\n\n"+
					"You should use `vault generate-root` from a terminal for maximum\n"+
					"security. If this isn't an option, the unseal key can be passed\n"+
					"in using the first parameter.\n\n"+
					"Raw error: %s", err))
			return 1
		}
	}

	// Provide the key, this may potentially complete the update
	statusResp, err := client.Sys().GenerateRootUpdate(strings.TrimSpace(key), c.Nonce)
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error attempting generate-root update: %s", err))
		return 1
	}

	c.dumpStatus(statusResp)

	return 0
}

func (c *GenerateRootCommand) verifyOTP(otp string) error {
	if len(otp) == 0 {
		return fmt.Errorf("No OTP passed in")
	}
	otpBytes, err := base64.StdEncoding.DecodeString(otp)
	if err != nil {
		return fmt.Errorf("Error decoding base64 OTP value: %s", err)
	}
	if otpBytes == nil || len(otpBytes) != 16 {
		return fmt.Errorf("Decoded OTP value is invalid or wrong length")
	}

	return nil
}

func (c *GenerateRootCommand) decode(encodedVal, otp string) int {
	tokenBytes, err := xor.XORBase64(encodedVal, otp)
	if err != nil {
		c.Ui.Error(err.Error())
		return 1
	}

	token, err := uuid.FormatUUID(tokenBytes)
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error formatting base64 token value: %v", err))
		return 1
	}

	c.Ui.Output(fmt.Sprintf("Root token: %s", token))

	return 0
}

// initGenerateRoot is used to start the generation process
func (c *GenerateRootCommand) initGenerateRoot(client *api.Client, otp string, pgpKey string) int {
	// Start the rekey
	status, err := client.Sys().GenerateRootInit(otp, pgpKey)
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error initializing root generation: %s", err))
		return 1
	}

	c.dumpStatus(status)

	return 0
}

// cancelGenerateRoot is used to abort the generation process
func (c *GenerateRootCommand) cancelGenerateRoot(client *api.Client) int {
	err := client.Sys().GenerateRootCancel()
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Failed to cancel root generation: %s", err))
		return 1
	}
	c.Ui.Output("Root generation canceled.")
	return 0
}

// rootGenerationStatus is used just to fetch and dump the status
func (c *GenerateRootCommand) rootGenerationStatus(client *api.Client) int {
	// Check the status
	status, err := client.Sys().GenerateRootStatus()
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error reading root generation status: %s", err))
		return 1
	}

	c.dumpStatus(status)

	return 0
}

// dumpStatus dumps the status to output
func (c *GenerateRootCommand) dumpStatus(status *api.GenerateRootStatusResponse) {
	// Dump the status
	statString := fmt.Sprintf(
		"Nonce: %s\n"+
			"Started: %v\n"+
			"Generate Root Progress: %d\n"+
			"Required Keys: %d\n"+
			"Complete: %t",
		status.Nonce,
		status.Started,
		status.Progress,
		status.Required,
		status.Complete,
	)
	if len(status.PGPFingerprint) > 0 {
		statString = fmt.Sprintf("%s\nPGP Fingerprint: %s", statString, status.PGPFingerprint)
	}
	if len(status.EncodedRootToken) > 0 {
		statString = fmt.Sprintf("%s\n\nEncoded root token: %s", statString, status.EncodedRootToken)
	}
	c.Ui.Output(statString)
}

func (c *GenerateRootCommand) Synopsis() string {
	return "Generates a new root token"
}

func (c *GenerateRootCommand) Help() string {
	helpText := `
Usage: vault generate-root [options] [key]

  'generate-root' is used to create a new root token.

  Root generation can only be done when the Vault is already unsealed. The
  operation is done online, but requires that a threshold of the current unseal
  keys be provided.

  One (and only one) of the following must be provided at attempt
  initialization time:

  1) A 16-byte, base64-encoded One Time Password (OTP) provided in the '-otp'
  flag; the token is XOR'd with this value before it is returned once the final
  unseal key has been provided. The '-decode' operation can be used with this
  value and the OTP to output the final token value. The '-genotp' flag can be
  used to generate a suitable value.

  or

  2) A file containing a PGP key (binary or base64-encoded) or a Keybase.io
  username in the format of "keybase:<username>" in the '-pgp-key' flag. The
  final token value will be encrypted with this public key and base64-encoded.

General Options:
` + meta.GeneralOptionsUsage() + `
Generate Root Options:

  -init                   Initialize the root generation attempt. This can only
                          be done if no generation is already initiated.

  -cancel                 Reset the root generation process by throwing away
                          prior unseal keys and the configuration.

  -status                 Prints the status of the current attempt. This can be
                          used to see the status without attempting to provide
                          an unseal key.

  -decode=abcd            Decodes and outputs the generated root token. The OTP
                          used at '-init' time must be provided in the '-otp'
                          parameter.

  -genotp                 Returns a high-quality OTP suitable for passing into
                          the '-init' method.

  -otp=abcd               The base64-encoded 16-byte OTP for use with the
                          '-init' or '-decode' methods.

  -pgp-key                A file on disk containing a binary- or base64-format
                          public PGP key, or a Keybase username specified as
                          "keybase:<username>". The output root token will be
                          encrypted and base64-encoded, in order, with the given
                          public key.

  -nonce=abcd             The nonce provided at initialization time. This same
                          nonce value must be provided with each unseal key. If
                          the unseal key is not being passed in via the command
                          line the nonce parameter is not required, and will
                          instead be displayed with the key prompt.
`
	return strings.TrimSpace(helpText)
}
<|endoftext|>