<commit_before>package tmo

import (
	"image"
	"image/color"
	"math"
	"sync"

	colorful "github.com/lucasb-eyer/go-colorful"
	"github.com/mdouchement/hdr"
	"github.com/mdouchement/hdr/hdrcolor"
)

const (
	reinhardGamma = 1.8
)

// A Reinhard05 is a TMO implementation based on Erik Reinhard's 2005 white paper.
//
// Reference:
// Dynamic Range Reduction Inspired by Photoreceptor Physiology.
// E. Reinhard and K. Devlin.
// In IEEE Transactions on Visualization and Computer Graphics, 2005.
type Reinhard05 struct {
	HDRImage   hdr.Image
	Brightness float64
	Chromatic  float64
	Light      float64
	lumOnce    sync.Once
	cav        []float64
	lav        float64
	minLum     float64
	maxLum     float64
	worldLum   float64
	k          float64
	m          float64
	f          float64
}

// NewDefaultReinhard05 instanciates a new Reinhard05 TMO with default parameters.
func NewDefaultReinhard05(m hdr.Image) *Reinhard05 {
	return NewReinhard05(m, -5, 0.89, 0.89)
}

// NewReinhard05 instanciates a new Reinhard05 TMO.
func NewReinhard05(m hdr.Image, brightness, chromatic, light float64) *Reinhard05 {
	return &Reinhard05{
		HDRImage: m,
		// Brightness is included in [-20, 20] with 0.1 increment step.
		Brightness: brightness,
		// Chromatic is included in [0, 1] with 0.01 increment step.
		Chromatic: chromatic,
		// Light is included in [0, 1] with 0.01 increment step.
		Light:  light,
		cav:    make([]float64, 3),
		minLum: math.Inf(1),
		maxLum: math.Inf(-1),
	}
}

// Perform runs the TMO mapping.
func (t *Reinhard05) Perform() image.Image {
	img := image.NewRGBA64(t.HDRImage.Bounds())

	t.lumOnce.Do(t.luminance) // First pass

	// FIXME
	// Extra memory consumption (x2)
	// A temporary image avoids original image modifications and let user applies another TMO on the image.
	//   - It is quite speed to re-read the original file from filesystem.
	// We can avoid this tmp image by re-calculates the sampling but it costs an extra CPU consumption.
	// We can reduce memory consuption by streaming the data to a swap file but iowait could happen.
	tmp := hdr.NewRGB(t.HDRImage.Bounds())

	minSample, maxSample := t.tonemap(tmp) // Second pass

	t.normalize(img, tmp, minSample, maxSample) // Third pass

	return img
}

func (t *Reinhard05) luminance() {
	reinhardCh := make(chan *Reinhard05)

	completed := parallelR(t.HDRImage.Bounds(), func(x1, y1, x2, y2 int) {
		tt := NewDefaultReinhard05(nil)

		for y := y1; y < y2; y++ {
			for x := x1; x < x2; x++ {
				pixel := t.HDRImage.HDRAt(x, y)
				r, g, b, _ := pixel.HDRRGBA()

				_, lum, _ := colorful.Color{R: r, G: g, B: b}.Xyz() // Get luminance (Y) from the CIE XYZ-space.
				tt.minLum = math.Min(tt.minLum, lum)
				tt.maxLum = math.Max(tt.maxLum, lum)
				tt.worldLum += math.Log((2.3e-5) + lum)

				tt.cav[0] += r
				tt.cav[1] += g
				tt.cav[2] += b
				tt.lav += lum
			}
		}

		reinhardCh <- tt
	})

	for {
		select {
		case <-completed:
			goto NEXT
		case tt := <-reinhardCh:
			t.minLum = math.Min(t.minLum, tt.minLum)
			t.maxLum = math.Max(t.maxLum, tt.maxLum)
			t.worldLum += tt.worldLum

			t.cav[0] += tt.cav[0]
			t.cav[1] += tt.cav[1]
			t.cav[2] += tt.cav[2]
			t.lav += tt.lav
		}
	}
NEXT:

	size := float64(t.HDRImage.Size())
	t.worldLum /= size
	t.cav[0] /= size
	t.cav[1] /= size
	t.cav[2] /= size
	t.lav /= size

	t.minLum = math.Log(t.minLum)
	t.maxLum = math.Log(t.maxLum)

	// Image key
	t.k = (t.maxLum - t.worldLum) / (t.maxLum - t.minLum)
	// Image contrast based on key value
	t.m = (0.3 + (0.7 * math.Pow(t.k, 1.4)))
	// Image brightness
	t.f = math.Exp(-t.Brightness)
}

func (t *Reinhard05) tonemap(tmp *hdr.RGB) (minSample, maxSample float64) {
	minSample = 1.0
	maxSample = 0.0
	minCh := make(chan float64)
	maxCh := make(chan float64)

	completed := parallelR(t.HDRImage.Bounds(), func(x1, y1, x2, y2 int) {
		min := 1.0
		max := 0.0

		for y := y1; y < y2; y++ {
			for x := x1; x < x2; x++ {
				pixel := t.HDRImage.HDRAt(x, y)
				r, g, b, _ := pixel.HDRRGBA()

				_, lum, _ := colorful.Color{R: r, G: g, B: b}.Xyz() // Get luminance (Y) from the CIE XYZ-space.

				var sample float64
				p := hdrcolor.RGB{}

				if lum != 0.0 {
					sample = t.sampling(r, lum, 0)
					min = math.Min(min, sample)
					max = math.Max(max, sample)
					p.R = sample

					sample = t.sampling(g, lum, 1)
					min = math.Min(min, sample)
					max = math.Max(max, sample)
					p.G = sample

					sample = t.sampling(b, lum, 2)
					min = math.Min(min, sample)
					max = math.Max(max, sample)
					p.B = sample

					tmp.SetRGB(x, y, p)
				}
			}
		}

		minCh <- min
		maxCh <- max
	})

	for {
		select {
		case <-completed:
			return
		case sample := <-minCh:
			minSample = math.Min(minSample, sample)
		case sample := <-maxCh:
			maxSample = math.Max(maxSample, sample)
		}
	}
}

// sampling one channel
func (t *Reinhard05) sampling(sample, lum float64, c int) float64 {
	if sample != 0.0 {
		// Local light adaptation
		il := t.Chromatic*sample + (1-t.Chromatic)*lum
		// Global light adaptation
		ig := t.Chromatic*t.cav[c] + (1-t.Chromatic)*t.lav
		// Interpolated light adaptation
		ia := t.Light*il + (1-t.Light)*ig
		// Photoreceptor equation
		sample /= sample + math.Pow(t.f*ia, t.m)
	}

	return sample
}

func (t *Reinhard05) normalize(img *image.RGBA64, tmp *hdr.RGB, minSample, maxSample float64) {
	completed := parallelR(tmp.Bounds(), func(x1, y1, x2, y2 int) {
		for y := y1; y < y2; y++ {
			for x := x1; x < x2; x++ {
				pixel := tmp.HDRAt(x, y)
				r, g, b, _ := pixel.HDRRGBA()

				img.SetRGBA64(x, y, color.RGBA64{
					R: t.nrmz(r, minSample, maxSample),
					G: t.nrmz(g, minSample, maxSample),
					B: t.nrmz(b, minSample, maxSample),
					A: RangeMax,
				})
			}
		}
	})

	<-completed
}

// normalize one channel
func (t *Reinhard05) nrmz(channel, minSample, maxSample float64) uint16 {
	// Normalize intensities
	channel = (channel - minSample) / (maxSample - minSample)

	// Gamma correction
	if channel > RangeMin {
		channel = math.Pow(channel, 1/reinhardGamma)
	}

	// Inverse pixel mapping
	channel = LinearInversePixelMapping(channel, LumPixFloor, LumSize)

	// Clamp to solid black and solid white
	channel = Clamp(channel)

	return uint16(channel)
}
<commit_msg>Improved Reinhard05<commit_after>package tmo

import (
	"image"
	"image/color"
	"math"
	"sync"

	colorful "github.com/lucasb-eyer/go-colorful"
	"github.com/mdouchement/hdr"
	"github.com/mdouchement/hdr/filter"
)

const (
	reinhardGamma = 1.8
)

// A Reinhard05 is a TMO implementation based on Erik Reinhard's 2005 white paper.
//
// Reference:
// Dynamic Range Reduction Inspired by Photoreceptor Physiology.
// E. Reinhard and K. Devlin.
// In IEEE Transactions on Visualization and Computer Graphics, 2005.
type Reinhard05 struct {
	HDRImage   hdr.Image
	Brightness float64
	Chromatic  float64
	Light      float64
	lumOnce    sync.Once
	cav        []float64
	lav        float64
	minLum     float64
	maxLum     float64
	worldLum   float64
	k          float64
	m          float64
	f          float64
}

// NewDefaultReinhard05 instanciates a new Reinhard05 TMO with default parameters.
func NewDefaultReinhard05(m hdr.Image) *Reinhard05 {
	return NewReinhard05(m, -5, 0.89, 0.89)
}

// NewReinhard05 instanciates a new Reinhard05 TMO.
func NewReinhard05(m hdr.Image, brightness, chromatic, light float64) *Reinhard05 {
	return &Reinhard05{
		HDRImage: m,
		// Brightness is included in [-20, 20] with 0.1 increment step.
		Brightness: brightness,
		// Chromatic is included in [0, 1] with 0.01 increment step.
		Chromatic: chromatic,
		// Light is included in [0, 1] with 0.01 increment step.
		Light:  light,
		cav:    make([]float64, 3),
		minLum: math.Inf(1),
		maxLum: math.Inf(-1),
	}
}

// Perform runs the TMO mapping.
func (t *Reinhard05) Perform() image.Image {
	img := image.NewRGBA64(t.HDRImage.Bounds())

	t.lumOnce.Do(t.luminance) // First pass

	minSample, maxSample := t.tonemap() // Second pass

	t.normalize(img, minSample, maxSample) // Third pass

	return img
}

func (t *Reinhard05) luminance() {
	reinhardCh := make(chan *Reinhard05)
	qsImg := filter.NewQuickSampling(t.HDRImage, 0.6)

	completed := parallelR(qsImg.Bounds(), func(x1, y1, x2, y2 int) {
		tt := NewDefaultReinhard05(nil)

		for y := y1; y < y2; y++ {
			for x := x1; x < x2; x++ {
				pixel := qsImg.HDRAt(x, y)
				r, g, b, _ := pixel.HDRRGBA()

				_, lum, _ := colorful.Color{R: r, G: g, B: b}.Xyz() // Get luminance (Y) from the CIE XYZ-space.
				tt.minLum = math.Min(tt.minLum, lum)
				tt.maxLum = math.Max(tt.maxLum, lum)
				tt.worldLum += math.Log((2.3e-5) + lum)

				tt.cav[0] += r
				tt.cav[1] += g
				tt.cav[2] += b
				tt.lav += lum
			}
		}

		reinhardCh <- tt
	})

	for {
		select {
		case <-completed:
			goto NEXT
		case tt := <-reinhardCh:
			t.minLum = math.Min(t.minLum, tt.minLum)
			t.maxLum = math.Max(t.maxLum, tt.maxLum)
			t.worldLum += tt.worldLum

			t.cav[0] += tt.cav[0]
			t.cav[1] += tt.cav[1]
			t.cav[2] += tt.cav[2]
			t.lav += tt.lav
		}
	}
NEXT:

	size := float64(qsImg.Size())
	t.worldLum /= size
	t.cav[0] /= size
	t.cav[1] /= size
	t.cav[2] /= size
	t.lav /= size

	t.minLum = math.Log(t.minLum)
	t.maxLum = math.Log(t.maxLum)

	// Image key
	t.k = (t.maxLum - t.worldLum) / (t.maxLum - t.minLum)
	// Image contrast based on key value
	t.m = (0.3 + (0.7 * math.Pow(t.k, 1.4)))
	// Image brightness
	t.f = math.Exp(-t.Brightness)
}

func (t *Reinhard05) tonemap() (minSample, maxSample float64) {
	minSample = 1.0
	maxSample = 0.0
	minCh := make(chan float64)
	maxCh := make(chan float64)

	qsImg := filter.NewQuickSampling(t.HDRImage, 0.6)

	completed := parallelR(qsImg.Bounds(), func(x1, y1, x2, y2 int) {
		min := 1.0
		max := 0.0

		for y := y1; y < y2; y++ {
			for x := x1; x < x2; x++ {
				pixel := qsImg.HDRAt(x, y)
				r, g, b, _ := pixel.HDRRGBA()

				_, lum, _ := colorful.Color{R: r, G: g, B: b}.Xyz() // Get luminance (Y) from the CIE XYZ-space.

				var sample float64

				if lum != 0.0 {
					sample = t.sampling(r, lum, 0)
					min = math.Min(min, sample)
					max = math.Max(max, sample)

					sample = t.sampling(g, lum, 1)
					min = math.Min(min, sample)
					max = math.Max(max, sample)

					sample = t.sampling(b, lum, 2)
					min = math.Min(min, sample)
					max = math.Max(max, sample)
				}
			}
		}

		minCh <- min
		maxCh <- max
	})

	for {
		select {
		case <-completed:
			return
		case sample := <-minCh:
			minSample = math.Min(minSample, sample)
		case sample := <-maxCh:
			maxSample = math.Max(maxSample, sample)
		}
	}
}

// sampling one channel
func (t *Reinhard05) sampling(sample, lum float64, c int) float64 {
	if sample != 0.0 {
		// Local light adaptation
		il := t.Chromatic*sample + (1-t.Chromatic)*lum
		// Global light adaptation
		ig := t.Chromatic*t.cav[c] + (1-t.Chromatic)*t.lav
		// Interpolated light adaptation
		ia := t.Light*il + (1-t.Light)*ig
		// Photoreceptor equation
		sample /= sample + math.Pow(t.f*ia, t.m)
	}

	return sample
}

func (t *Reinhard05) normalize(img *image.RGBA64, minSample, maxSample float64) {
	completed := parallelR(t.HDRImage.Bounds(), func(x1, y1, x2, y2 int) {
		for y := y1; y < y2; y++ {
			for x := x1; x < x2; x++ {
				pixel := t.HDRImage.HDRAt(x, y)
				r, g, b, _ := pixel.HDRRGBA()

				_, lum, _ := colorful.Color{R: r, G: g, B: b}.Xyz() // Get luminance (Y) from the CIE XYZ-space.

				img.SetRGBA64(x, y, color.RGBA64{
					R: t.nrmz(t.sampling(r, lum, 0), minSample, maxSample),
					G: t.nrmz(t.sampling(g, lum, 1), minSample, maxSample),
					B: t.nrmz(t.sampling(b, lum, 2), minSample, maxSample),
					A: RangeMax,
				})
			}
		}
	})

	<-completed
}

// normalize one channel
func (t *Reinhard05) nrmz(channel, minSample, maxSample float64) uint16 {
	// Normalize intensities
	channel = (channel - minSample) / (maxSample - minSample)

	// Gamma correction
	if channel > RangeMin {
		channel = math.Pow(channel, 1/reinhardGamma)
	}

	// Inverse pixel mapping
	channel = LinearInversePixelMapping(channel, LumPixFloor, LumSize)

	// Clamp to solid black and solid white
	channel = Clamp(channel)

	return uint16(channel)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"log"
	"os/exec"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
)

func main() {
	path, err := exec.LookPath("mongodump")
	if err != nil {
		log.Fatal("mongodump could not be found")
	}
	fmt.Printf("mongodump is available at %s\n", path)

	dumpCmd := exec.Command(
		"mongodump",
		"--host",
		"10.10.1.103",
		"--port",
		"27017",
		"--archive"
	)
	body, err := dumpCmd.StdoutPipe()
	if err != nil {
		// handle error
	}

	if err := dumpCmd.Start(); err != nil {
		// handle error
	}

	// Wrap the pipe to hide the seek methods from the uploader
	var bodyWrap = struct {
		io.Reader
	}{body}

	uploader := s3manager.NewUploader(
		session.New(&aws.Config{Region: aws.String("us-east-1")})
	)
	result, err := uploader.Upload(&s3manager.UploadInput{
		Body:   bodyWrap,
		Bucket: aws.String("net-openwhere-mongodb-snapshots-dev"),
		Key:    aws.String("myKey2"),
	})
	if err != nil {
		log.Fatalln("Failed to upload", err)
	}

	if err := dumpCmd.Wait(); err != nil {
		// handle error
	}

	log.Println("Successfully uploaded to", result.Location)
}
<commit_msg>change formatting<commit_after>package main

import (
	"fmt"
	"io"
	"log"
	"os/exec"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
)

func main() {
	path, err := exec.LookPath("mongodump")
	if err != nil {
		log.Fatal("mongodump could not be found")
	}
	fmt.Printf("mongodump is available at %s\n", path)

	dumpCmd := exec.Command("mongodump", "--host", "10.10.1.103", "--port", "27017", "--archive")
	body, err := dumpCmd.StdoutPipe()
	if err != nil {
		// handle error
	}

	if err := dumpCmd.Start(); err != nil {
		// handle error
	}

	// Wrap the pipe to hide the seek methods from the uploader
	var bodyWrap = struct {
		io.Reader
	}{body}

	uploader := s3manager.NewUploader(session.New(&aws.Config{Region: aws.String("us-east-1")}))
	result, err := uploader.Upload(&s3manager.UploadInput{
		Body:   bodyWrap,
		Bucket: aws.String("net-openwhere-mongodb-snapshots-dev"),
		Key:    aws.String("myKey2"),
	})
	if err != nil {
		log.Fatalln("Failed to upload", err)
	}

	if err := dumpCmd.Wait(); err != nil {
		// handle error
	}

	log.Println("Successfully uploaded to", result.Location)
}
<|endoftext|>


<commit_before>package main

import (
	"compress/gzip"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/dustin/go-hashset"
	"github.com/dustin/gomemcached"

	"encoding/hex"
	"github.com/couchbaselabs/cbfs/config"
	"sync"
)

const backupKey = "/@backup"

type backupItem struct {
	Fn   string                `json:"filename"`
	Oid  string                `json:"oid"`
	When time.Time             `json:"when"`
	Conf cbfsconfig.CBFSConfig `json:"conf"`
}

type backups struct {
	Latest  backupItem   `json:"latest"`
	Backups []backupItem `json:"backups"`
}

func logDuration(m string, startTime time.Time) {
	log.Printf("Completed %v in %v", m, time.Since(startTime))
}

func streamFileMeta(w io.Writer,
	fch chan *namedFile,
	ech chan error) error {

	enc := json.NewEncoder(w)
	for {
		select {
		case f, ok := <-fch:
			if !ok {
				return nil
			}
			err := enc.Encode(map[string]interface{}{
				"path": f.name,
				"meta": f.meta,
			})
			if err != nil {
				return err
			}
		case e, ok := <-ech:
			if ok {
				return e
			}
			ech = nil
		}
	}
}

func backupTo(w io.Writer) (err error) {
	fch := make(chan *namedFile)
	ech := make(chan error)
	qch := make(chan bool)

	defer close(qch)

	defer logDuration("backup", time.Now())

	go pathGenerator("", fch, ech, qch)

	gz := gzip.NewWriter(w)
	defer func() {
		e := gz.Close()
		if err != nil {
			err = e
		}
	}()

	return streamFileMeta(gz, fch, ech)
}

func recordBackupObject() error {
	b := backups{}
	err := couchbase.Get(backupKey, &b)
	if err != nil {
		return err
	}

	removeDeadBackups(&b)

	f, err := os.Create(filepath.Join(*root, ".backup.json"))
	if err != nil {
		return err
	}
	defer f.Close()
	e := json.NewEncoder(f)
	return e.Encode(&b)
}

func recordRemoteBackupObjects() {
	rn, err := findRemoteNodes()
	if err != nil {
		log.Printf("Error getting remote nodes for recording backup: %v",
			err)
		return
	}
	for _, n := range rn {
		u := fmt.Sprintf("http://%s%s",
			n.Address(), markBackupPrefix)
		c := n.Client()
		res, err := c.Post(u, "application/octet-stream", nil)
		if err != nil {
			log.Printf("Error posting to %v: %v", u, err)
			continue
		}
		res.Body.Close()
		if res.StatusCode != 204 {
			log.Printf("HTTP Error posting to %v: %v", u, res.Status)
		}
	}
}

func removeDeadBackups(b *backups) {
	// Keep only backups we're pretty sure still exist.
	obn := b.Backups
	b.Backups = nil
	for _, bi := range obn {
		fm := fileMeta{}
		err := couchbase.Get(shortName(bi.Fn), &fm)
		if gomemcached.IsNotFound(err) {
			log.Printf("Dropping previous (deleted) backup: %v",
				bi.Fn)
		} else {
			b.Backups = append(b.Backups, bi)
		}
	}

}

func storeBackupObject(fn, h string) error {
	b := backups{}
	err := couchbase.Get(backupKey, &b)
	if err != nil && !gomemcached.IsNotFound(err) {
		log.Printf("Weird: %v", err)
		// return err
	}

	removeDeadBackups(&b)

	ob := backupItem{fn, h, time.Now().UTC(), *globalConfig}

	b.Latest = ob
	b.Backups = append(b.Backups, ob)

	return couchbase.Set(backupKey, 0, &b)
}

func backupToCBFS(fn string) error {
	f, err := NewHashRecord(*root, "")
	if err != nil {
		return err
	}
	defer f.Close()

	pr, pw := io.Pipe()

	go func() { pw.CloseWithError(backupTo(pw)) }()

	h, length, err := f.Process(pr)
	if err != nil {
		return err
	}

	err = recordBlobOwnership(h, length, true)
	if err != nil {
		return err
	}

	fm := fileMeta{
		OID:      h,
		Length:   length,
		Modified: time.Now().UTC(),
	}

	err = storeMeta(fn, 0, fm, 1, nil)
	if err != nil {
		return err
	}

	err = storeBackupObject(fn, h)
	if err != nil {
		return err
	}

	err = recordBackupObject()
	if err != nil {
		log.Printf("Failed to record backup OID: %v", err)
	}

	go recordRemoteBackupObjects()

	log.Printf("Replicating backup %v.", h)
	go increaseReplicaCount(h, length, globalConfig.MinReplicas-1)

	return nil
}

func doMarkBackup(w http.ResponseWriter, req *http.Request) {
	if req.FormValue("all") == "true" {
		go recordRemoteBackupObjects()
	}
	err := recordBackupObject()
	if err != nil {
		http.Error(w, fmt.Sprintf("Error marking backup; %v", err), 500)
		return
	}
	w.WriteHeader(204)
}

func doBackupDocs(w http.ResponseWriter, req *http.Request) {
	fn := req.FormValue("fn")
	if fn == "" {
		http.Error(w, "Missing fn parameter", 400)
		return
	}

	if bg, _ := strconv.ParseBool(req.FormValue("bg")); bg {
		go func() {
			err := backupToCBFS(fn)
			if err != nil {
				log.Printf("Error performing bg backup: %v", err)
			}
		}()
		w.WriteHeader(202)
		return
	}

	err := backupToCBFS(fn)
	if err != nil {
		http.Error(w, fmt.Sprintf("Error performing backup: %v", err), 500)
		return
	}

	w.WriteHeader(201)
}

func doGetBackupInfo(w http.ResponseWriter, req *http.Request) {
	b := backups{}
	err := couchbase.Get(backupKey, &b)
	if err != nil {
		code := 500
		if gomemcached.IsNotFound(err) {
			code = 404
		}
		http.Error(w, err.Error(), code)
		return
	}

	removeDeadBackups(&b)

	sendJson(w, req, &b)
}

var errExists = errors.New("item exists")

func maybeStoreMeta(k string, fm fileMeta, exp int, force bool) error {
	if force {
		return couchbase.Set(k, exp, fm)
	}
	added, err := couchbase.Add(k, exp, fm)
	if err == nil && !added {
		err = errExists
	}
	return err
}

func doRestoreDocument(w http.ResponseWriter, req *http.Request, fn string) {
	d := json.NewDecoder(req.Body)
	fm := fileMeta{}
	err := d.Decode(&fm)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	for len(fn) > 0 && fn[0] == '/' {
		fn = fn[1:]
	}

	if fn == "" {
		http.Error(w, "No filename", 400)
		return
	}

	if strings.Contains(fn, "//") {
		http.Error(w,
			fmt.Sprintf("Too many slashes in the path name: %v", fn), 400)
		return
	}

	_, err = referenceBlob(fm.OID)
	if err != nil {
		log.Printf("Missing blob %v while restoring %v - restoring anyway",
			fm.OID, fn)
	}

	exp := getExpiration(req.Header)
	if exp == 0 {
		exp = getExpiration(fm.Headers)
		if exp > 0 && exp < 60*60*24*30 {
			exp = int(fm.Modified.Add(time.Second * time.Duration(exp)).Unix())
		}
	}

	if exp < 0 {
		log.Printf("Attempt to restore expired file: %v", fn)
		w.WriteHeader(201)
		return
	}

	force := false
	err = maybeStoreMeta(fn, fm, exp, force)
	switch err {
	case errExists:
		http.Error(w, err.Error(), 409)
		return
	case nil:
	default:
		log.Printf("Error storing file meta of %v -> %v: %v",
			fn, fm.OID, err)
		http.Error(w,
			fmt.Sprintf("Error recording file meta: %v", err), 500)
		return
	}

	log.Printf("Restored %v -> %v (exp=%v)", fn, fm.OID, exp)

	w.WriteHeader(201)
}

func loadBackupHashes(oid string) (*hashset.Hashset, int, error) {
	rv := &hashset.Hashset{}

	r := blobReader(oid)
	defer r.Close()
	gz, err := gzip.NewReader(r)
	if err != nil {
		return nil, 0, err
	}
	defer gz.Close()

	d := json.NewDecoder(gz)

	hashlen := getHash().Size()

	visited := 0
	for {
		ob := struct {
			Meta struct {
				OID   string
				Older []struct {
					OID string
				}
			}
		}{}

		err := d.Decode(&ob)
		switch err {
		case nil:
			oid, err := hex.DecodeString(ob.Meta.OID)
			if err != nil {
				return nil, visited, err
			}
			if len(oid) != hashlen {
				log.Printf("Invalid hash from %#v", ob)
				continue
			}
			rv.Add(oid)
			visited++
			for _, obs := range ob.Meta.Older {
				oid, err = hex.DecodeString(obs.OID)
				if err != nil {
					return nil, visited, err
				}
				rv.Add(oid)
				visited++
			}
		case io.EOF:
			return rv, visited, nil
		default:
			return nil, visited, err
		}
	}
}

func loadExistingHashes() (*hashset.Hashset, error) {
	b := backups{}
	err := couchbase.Get(backupKey, &b)
	if err != nil && !gomemcached.IsNotFound(err) {
		return nil, err
	}

	oids := make(chan string)
	hsch := make(chan *hashset.Hashset)
	visitch := make(chan int)
	errch := make(chan error)

	wg := sync.WaitGroup{}

	for i := 0; i < 4; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for o := range oids {
				h, v, e := loadBackupHashes(o)
				if e == nil {
					hsch <- h
				} else {
					errch <- e
				}
				visitch <- v
			}
		}()
	}
	go func() {
		wg.Wait()
		close(hsch)
		close(visitch)
		close(errch)
	}()

	go func() {
		for _, i := range b.Backups {
			log.Printf("Loading backups from %v / %v", i.Oid, i.Fn)
			oids <- i.Oid
		}
		close(oids)
	}()

	visited := 0
	hs := &hashset.Hashset{}
	for {
		// Done getting all the things
		if hsch == nil && visitch == nil && errch == nil {
			break
		}
		select {
		case v, ok := <-visitch:
			visited += v
			if !ok {
				visitch = nil
			}
		case e, ok := <-errch:
			err = e
			if !ok {
				errch = nil
			}
		case h, ok := <-hsch:
			if ok {
				log.Printf("Got %v hashes from a backup",
					h.Len())
				hs.AddAll(h)
			} else {
				hsch = nil
			}
		}
	}

	log.Printf("Visited %v obs, kept %v", visited, hs.Len())

	return hs, err
}
<commit_msg>Allow overriding to 0<commit_after>package main

import (
	"compress/gzip"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/dustin/go-hashset"
	"github.com/dustin/gomemcached"

	"encoding/hex"
	"github.com/couchbaselabs/cbfs/config"
	"sync"
)

const backupKey = "/@backup"

type backupItem struct {
	Fn   string                `json:"filename"`
	Oid  string                `json:"oid"`
	When time.Time             `json:"when"`
	Conf cbfsconfig.CBFSConfig `json:"conf"`
}

type backups struct {
	Latest  backupItem   `json:"latest"`
	Backups []backupItem `json:"backups"`
}

func logDuration(m string, startTime time.Time) {
	log.Printf("Completed %v in %v", m, time.Since(startTime))
}

func streamFileMeta(w io.Writer,
	fch chan *namedFile,
	ech chan error) error {

	enc := json.NewEncoder(w)
	for {
		select {
		case f, ok := <-fch:
			if !ok {
				return nil
			}
			err := enc.Encode(map[string]interface{}{
				"path": f.name,
				"meta": f.meta,
			})
			if err != nil {
				return err
			}
		case e, ok := <-ech:
			if ok {
				return e
			}
			ech = nil
		}
	}
}

func backupTo(w io.Writer) (err error) {
	fch := make(chan *namedFile)
	ech := make(chan error)
	qch := make(chan bool)

	defer close(qch)

	defer logDuration("backup", time.Now())

	go pathGenerator("", fch, ech, qch)

	gz := gzip.NewWriter(w)
	defer func() {
		e := gz.Close()
		if err != nil {
			err = e
		}
	}()

	return streamFileMeta(gz, fch, ech)
}

func recordBackupObject() error {
	b := backups{}
	err := couchbase.Get(backupKey, &b)
	if err != nil {
		return err
	}

	removeDeadBackups(&b)

	f, err := os.Create(filepath.Join(*root, ".backup.json"))
	if err != nil {
		return err
	}
	defer f.Close()
	e := json.NewEncoder(f)
	return e.Encode(&b)
}

func recordRemoteBackupObjects() {
	rn, err := findRemoteNodes()
	if err != nil {
		log.Printf("Error getting remote nodes for recording backup: %v",
			err)
		return
	}
	for _, n := range rn {
		u := fmt.Sprintf("http://%s%s",
			n.Address(), markBackupPrefix)
		c := n.Client()
		res, err := c.Post(u, "application/octet-stream", nil)
		if err != nil {
			log.Printf("Error posting to %v: %v", u, err)
			continue
		}
		res.Body.Close()
		if res.StatusCode != 204 {
			log.Printf("HTTP Error posting to %v: %v", u, res.Status)
		}
	}
}

func removeDeadBackups(b *backups) {
	// Keep only backups we're pretty sure still exist.
	obn := b.Backups
	b.Backups = nil
	for _, bi := range obn {
		fm := fileMeta{}
		err := couchbase.Get(shortName(bi.Fn), &fm)
		if gomemcached.IsNotFound(err) {
			log.Printf("Dropping previous (deleted) backup: %v",
				bi.Fn)
		} else {
			b.Backups = append(b.Backups, bi)
		}
	}

}

func storeBackupObject(fn, h string) error {
	b := backups{}
	err := couchbase.Get(backupKey, &b)
	if err != nil && !gomemcached.IsNotFound(err) {
		log.Printf("Weird: %v", err)
		// return err
	}

	removeDeadBackups(&b)

	ob := backupItem{fn, h, time.Now().UTC(), *globalConfig}

	b.Latest = ob
	b.Backups = append(b.Backups, ob)

	return couchbase.Set(backupKey, 0, &b)
}

func backupToCBFS(fn string) error {
	f, err := NewHashRecord(*root, "")
	if err != nil {
		return err
	}
	defer f.Close()

	pr, pw := io.Pipe()

	go func() { pw.CloseWithError(backupTo(pw)) }()

	h, length, err := f.Process(pr)
	if err != nil {
		return err
	}

	err = recordBlobOwnership(h, length, true)
	if err != nil {
		return err
	}

	fm := fileMeta{
		OID:      h,
		Length:   length,
		Modified: time.Now().UTC(),
	}

	err = storeMeta(fn, 0, fm, 1, nil)
	if err != nil {
		return err
	}

	err = storeBackupObject(fn, h)
	if err != nil {
		return err
	}

	err = recordBackupObject()
	if err != nil {
		log.Printf("Failed to record backup OID: %v", err)
	}

	go recordRemoteBackupObjects()

	log.Printf("Replicating backup %v.", h)
	go increaseReplicaCount(h, length, globalConfig.MinReplicas-1)

	return nil
}

func doMarkBackup(w http.ResponseWriter, req *http.Request) {
	if req.FormValue("all") == "true" {
		go recordRemoteBackupObjects()
	}
	err := recordBackupObject()
	if err != nil {
		http.Error(w, fmt.Sprintf("Error marking backup; %v", err), 500)
		return
	}
	w.WriteHeader(204)
}

func doBackupDocs(w http.ResponseWriter, req *http.Request) {
	fn := req.FormValue("fn")
	if fn == "" {
		http.Error(w, "Missing fn parameter", 400)
		return
	}

	if bg, _ := strconv.ParseBool(req.FormValue("bg")); bg {
		go func() {
			err := backupToCBFS(fn)
			if err != nil {
				log.Printf("Error performing bg backup: %v", err)
			}
		}()
		w.WriteHeader(202)
		return
	}

	err := backupToCBFS(fn)
	if err != nil {
		http.Error(w, fmt.Sprintf("Error performing backup: %v", err), 500)
		return
	}

	w.WriteHeader(201)
}

func doGetBackupInfo(w http.ResponseWriter, req *http.Request) {
	b := backups{}
	err := couchbase.Get(backupKey, &b)
	if err != nil {
		code := 500
		if gomemcached.IsNotFound(err) {
			code = 404
		}
		http.Error(w, err.Error(), code)
		return
	}

	removeDeadBackups(&b)

	sendJson(w, req, &b)
}

var errExists = errors.New("item exists")

func maybeStoreMeta(k string, fm fileMeta, exp int, force bool) error {
	if force {
		return couchbase.Set(k, exp, fm)
	}
	added, err := couchbase.Add(k, exp, fm)
	if err == nil && !added {
		err = errExists
	}
	return err
}

func doRestoreDocument(w http.ResponseWriter, req *http.Request, fn string) {
	d := json.NewDecoder(req.Body)
	fm := fileMeta{}
	err := d.Decode(&fm)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	for len(fn) > 0 && fn[0] == '/' {
		fn = fn[1:]
	}

	if fn == "" {
		http.Error(w, "No filename", 400)
		return
	}

	if strings.Contains(fn, "//") {
		http.Error(w,
			fmt.Sprintf("Too many slashes in the path name: %v", fn), 400)
		return
	}

	_, err = referenceBlob(fm.OID)
	if err != nil {
		log.Printf("Missing blob %v while restoring %v - restoring anyway",
			fm.OID, fn)
	}

	exp := getExpiration(req.Header)
	if exp == -1 {
		exp = getExpiration(fm.Headers)
		if exp > 0 && exp < 60*60*24*30 {
			exp = int(fm.Modified.Add(time.Second * time.Duration(exp)).Unix())
		}
	}

	if exp < 0 {
		log.Printf("Attempt to restore expired file: %v", fn)
		w.WriteHeader(201)
		return
	}

	force := false
	err = maybeStoreMeta(fn, fm, exp, force)
	switch err {
	case errExists:
		http.Error(w, err.Error(), 409)
		return
	case nil:
	default:
		log.Printf("Error storing file meta of %v -> %v: %v",
			fn, fm.OID, err)
		http.Error(w,
			fmt.Sprintf("Error recording file meta: %v", err), 500)
		return
	}

	log.Printf("Restored %v -> %v (exp=%v)", fn, fm.OID, exp)

	w.WriteHeader(201)
}

func loadBackupHashes(oid string) (*hashset.Hashset, int, error) {
	rv := &hashset.Hashset{}

	r := blobReader(oid)
	defer r.Close()
	gz, err := gzip.NewReader(r)
	if err != nil {
		return nil, 0, err
	}
	defer gz.Close()

	d := json.NewDecoder(gz)

	hashlen := getHash().Size()

	visited := 0
	for {
		ob := struct {
			Meta struct {
				OID   string
				Older []struct {
					OID string
				}
			}
		}{}

		err := d.Decode(&ob)
		switch err {
		case nil:
			oid, err := hex.DecodeString(ob.Meta.OID)
			if err != nil {
				return nil, visited, err
			}
			if len(oid) != hashlen {
				log.Printf("Invalid hash from %#v", ob)
				continue
			}
			rv.Add(oid)
			visited++
			for _, obs := range ob.Meta.Older {
				oid, err = hex.DecodeString(obs.OID)
				if err != nil {
					return nil, visited, err
				}
				rv.Add(oid)
				visited++
			}
		case io.EOF:
			return rv, visited, nil
		default:
			return nil, visited, err
		}
	}
}

func loadExistingHashes() (*hashset.Hashset, error) {
	b := backups{}
	err := couchbase.Get(backupKey, &b)
	if err != nil && !gomemcached.IsNotFound(err) {
		return nil, err
	}

	oids := make(chan string)
	hsch := make(chan *hashset.Hashset)
	visitch := make(chan int)
	errch := make(chan error)

	wg := sync.WaitGroup{}

	for i := 0; i < 4; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for o := range oids {
				h, v, e := loadBackupHashes(o)
				if e == nil {
					hsch <- h
				} else {
					errch <- e
				}
				visitch <- v
			}
		}()
	}
	go func() {
		wg.Wait()
		close(hsch)
		close(visitch)
		close(errch)
	}()

	go func() {
		for _, i := range b.Backups {
			log.Printf("Loading backups from %v / %v", i.Oid, i.Fn)
			oids <- i.Oid
		}
		close(oids)
	}()

	visited := 0
	hs := &hashset.Hashset{}
	for {
		// Done getting all the things
		if hsch == nil && visitch == nil && errch == nil {
			break
		}
		select {
		case v, ok := <-visitch:
			visited += v
			if !ok {
				visitch = nil
			}
		case e, ok := <-errch:
			err = e
			if !ok {
				errch = nil
			}
		case h, ok := <-hsch:
			if ok {
				log.Printf("Got %v hashes from a backup",
					h.Len())
				hs.AddAll(h)
			} else {
				hsch = nil
			}
		}
	}

	log.Printf("Visited %v obs, kept %v", visited, hs.Len())

	return hs, err
}
<|endoftext|>


<commit_before>package ghttp

import (
	"github.com/gogf/gf/errors/gerror"
	"net/http"
)

const gfHTTPClientMiddlewareKey = "__gfHttpClientMiddlewareKey"

// Use Add middleware to client
func (c *Client) Use(handlers ...ClientHandlerFunc) *Client {
	newClient := c
	if c.parent == nil {
		newClient = c.Clone()
	}

	newClient.middlewareHandler = append(newClient.middlewareHandler, handlers...)
	return newClient
}

// MiddlewareNext call next middleware
// this is should only be call in ClientHandlerFunc
func (c *Client) MiddlewareNext(req *http.Request) (*ClientResponse, error) {
	m, ok := req.Context().Value(gfHTTPClientMiddlewareKey).(*clientMiddleware)
	if ok {
		resp, err := m.Next(req)
		return resp, err
	}
	return c.callRequest(req)
}

// MiddlewareAbort stop call after all middleware, so it will not send http request
// this is should only be call in ClientHandlerFunc
func (c *Client) MiddlewareAbort(req *http.Request) (*ClientResponse, error) {
	m, ok := req.Context().Value(gfHTTPClientMiddlewareKey).(*clientMiddleware)
	if ok {
		m.Abort()
		return m.resp, m.err
	}
	return nil, gerror.New("http request abort")
}

// ClientHandlerFunc middleware handler func
type ClientHandlerFunc = func(c *Client, r *http.Request) (*ClientResponse, error)

// clientMiddleware is the plugin for http client request workflow management.
type clientMiddleware struct {
	client       *Client             // http client
	handlers     []ClientHandlerFunc // mdl handlers
	handlerIndex int                 // current handler index
	abort        bool                // abort call after handlers
	resp         *ClientResponse     // save resp
	err          error               // save err
}

// Next call next middleware handler, if abort,
func (m *clientMiddleware) Next(req *http.Request) (resp *ClientResponse, err error) {
	if m.abort || m.err != nil {
		return m.resp, m.err
	}
	if m.handlerIndex < len(m.handlers) {
		m.handlerIndex++
		resp, err = m.handlers[m.handlerIndex](m.client, req)
		m.resp = resp
		m.err = err
	}
	return
}

func (m *clientMiddleware) Abort() {
	m.abort = true
	if m.err == nil {
		m.err = gerror.New("http request abort")
	}
}
<commit_msg>remove abort, actually the abort is unuse<commit_after>package ghttp

import (
	"net/http"
)

const gfHTTPClientMiddlewareKey = "__gfHttpClientMiddlewareKey"

// Use Add middleware to client
func (c *Client) Use(handlers ...ClientHandlerFunc) *Client {
	newClient := c
	if c.parent == nil {
		newClient = c.Clone()
	}

	newClient.middlewareHandler = append(newClient.middlewareHandler, handlers...)
	return newClient
}

// MiddlewareNext call next middleware
// this is should only be call in ClientHandlerFunc
func (c *Client) MiddlewareNext(req *http.Request) (*ClientResponse, error) {
	m, ok := req.Context().Value(gfHTTPClientMiddlewareKey).(*clientMiddleware)
	if ok {
		resp, err := m.Next(req)
		return resp, err
	}
	return c.callRequest(req)
}

// ClientHandlerFunc middleware handler func
type ClientHandlerFunc = func(c *Client, r *http.Request) (*ClientResponse, error)

// clientMiddleware is the plugin for http client request workflow management.
type clientMiddleware struct {
	client       *Client             // http client
	handlers     []ClientHandlerFunc // mdl handlers
	handlerIndex int                 // current handler index
	resp         *ClientResponse     // save resp
	err          error               // save err
}

// Next call next middleware handler, if abort,
func (m *clientMiddleware) Next(req *http.Request) (resp *ClientResponse, err error) {
	if m.err != nil {
		return m.resp, m.err
	}
	if m.handlerIndex < len(m.handlers) {
		m.handlerIndex++
		resp, err = m.handlers[m.handlerIndex](m.client, req)
		m.resp = resp
		m.err = err
	}
	return
}
<|endoftext|>


<commit_before>package transfer

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"sync"

	"github.com/github/git-lfs/api"
	"github.com/github/git-lfs/config"
	"github.com/github/git-lfs/errutil"
	"github.com/github/git-lfs/httputil"
	"github.com/github/git-lfs/progress"
	"github.com/github/git-lfs/tools"
	"github.com/github/git-lfs/vendor/_nuts/github.com/rubyist/tracerx"
)

const (
	BasicAdapterName = "basic"
)

// Base implementation of basic all-or-nothing HTTP upload / download adapter
type basicAdapter struct {
	direction Direction
	jobChan   chan *Transfer
	cb        progress.CopyCallback
	outChan   chan TransferResult
	// WaitGroup to sync the completion of all workers
	workerWait sync.WaitGroup
	// WaitGroup to serialise the first transfer response to perform login if needed
	authWait sync.WaitGroup
}

func newBasicAdapter(d Direction) *basicAdapter {
	return &basicAdapter{
		direction: d,
	}
}

func (a *basicAdapter) Direction() Direction {
	return a.direction
}

func (a *basicAdapter) Name() string {
	return BasicAdapterName
}

func (a *basicAdapter) Begin(cb progress.CopyCallback, completion chan TransferResult) error {
	a.cb = cb
	a.outChan = completion
	a.jobChan = make(chan *Transfer, 100)

	tracerx.Printf("xfer: adapter %q Begin()", a.Name())

	numworkers := config.Config.ConcurrentTransfers()
	a.workerWait.Add(numworkers)
	a.authWait.Add(1)
	for i := 0; i < numworkers; i++ {
		go a.worker(i)
	}
	return nil
}

func (a *basicAdapter) Add(t *Transfer) {
	tracerx.Printf("xfer: adapter %q Add() for %q", a.Name(), t.Object.Oid)
	a.jobChan <- t
}

func (a *basicAdapter) End() {
	tracerx.Printf("xfer: adapter %q End()", a.Name())
	close(a.jobChan)
	// wait for all transfers to complete
	a.workerWait.Wait()
}

func (a *basicAdapter) ClearTempStorage() error {
	// Should be empty already but also remove dir
	return os.RemoveAll(a.tempDir())
}

// worker function, many of these run per adapter
func (a *basicAdapter) worker(workerNum int) {

	tracerx.Printf("xfer: adapter %q worker %d starting", a.Name(), workerNum)
	waitForAuth := workerNum > 0
	signalAuthOnResponse := workerNum == 0

	// First worker is the only one allowed to start immediately
	// The rest wait until successful response from 1st worker to
	// make sure only 1 login prompt is presented if necessary
	// Deliberately outside jobChan processing so we know worker 0 will process 1st item
	if waitForAuth {
		tracerx.Printf("xfer: adapter %q worker %d waiting for Auth", a.Name(), workerNum)
		a.authWait.Wait()
		tracerx.Printf("xfer: adapter %q worker %d auth signal received", a.Name(), workerNum)
	}

	for t := range a.jobChan {
		tracerx.Printf("xfer: adapter %q worker %d processing job for %q", a.Name(), workerNum, t.Object.Oid)
		var err error
		switch a.Direction() {
		case Download:
			err = a.download(t, signalAuthOnResponse)
		case Upload:
			err = a.upload(t, signalAuthOnResponse)
		}

		res := TransferResult{t, err}
		a.outChan <- res

		signalAuthOnResponse = false
		tracerx.Printf("xfer: adapter %q worker %d finished job for %q", a.Name(), workerNum, t.Object.Oid)
	}
	a.workerWait.Done()
}

func (a *basicAdapter) tempDir() string {
	// Must be dedicated to this adapter as deleted by ClearTempStorage
	d := filepath.Join(os.TempDir(), "git-lfs-basic-temp")
	if err := os.MkdirAll(d, 0755); err != nil {
		return os.TempDir()
	}
	return d
}

func (a *basicAdapter) download(t *Transfer, signalAuthOnResponse bool) error {
	rel, ok := t.Object.Rel("download")
	if !ok {
		return errors.New("Object not found on the server.")
	}

	req, err := httputil.NewHttpRequest("GET", rel.Href, rel.Header)
	if err != nil {
		return err
	}

	res, err := httputil.DoHttpRequest(req, true)
	if err != nil {
		return errutil.NewRetriableError(err)
	}
	httputil.LogTransfer("lfs.data.download", res)
	defer res.Body.Close()

	// Signal auth OK on success response, before starting download to free up
	// other workers immediately
	if signalAuthOnResponse {
		a.authWait.Done()
	}

	// Now do transfer of content
	f, err := ioutil.TempFile(a.tempDir(), t.Object.Oid+"-")
	if err != nil {
		return fmt.Errorf("cannot create temp file: %v", err)
	}

	defer func() {
		if err != nil {
			// Don't leave the temp file lying around on error.
			_ = os.Remove(f.Name()) // yes, ignore the error, not much we can do about it.
		}
	}()

	hasher := tools.NewHashingReader(res.Body)

	// ensure we always close f. Note that this does not conflict with  the
	// close below, as close is idempotent.
	defer f.Close()
	tempfilename := f.Name()
	written, err := tools.CopyWithCallback(f, hasher, res.ContentLength, a.cb)
	if err != nil {
		return fmt.Errorf("cannot write data to tempfile %q: %v", tempfilename, err)
	}
	if err := f.Close(); err != nil {
		return fmt.Errorf("can't close tempfile %q: %v", tempfilename, err)
	}

	if actual := hasher.Hash(); actual != t.Object.Oid {
		return fmt.Errorf("Expected OID %s, got %s after %d bytes written", t.Object.Oid, actual, written)
	}

	return tools.RenameFileCopyPermissions(tempfilename, t.Path)

}
func (a *basicAdapter) upload(t *Transfer, signalAuthOnResponse bool) error {
	rel, ok := t.Object.Rel("upload")
	if !ok {
		return fmt.Errorf("No upload action for this object.")
	}

	req, err := httputil.NewHttpRequest("PUT", rel.Href, rel.Header)
	if err != nil {
		return err
	}

	if len(req.Header.Get("Content-Type")) == 0 {
		req.Header.Set("Content-Type", "application/octet-stream")
	}

	if req.Header.Get("Transfer-Encoding") == "chunked" {
		req.TransferEncoding = []string{"chunked"}
	} else {
		req.Header.Set("Content-Length", strconv.FormatInt(t.Object.Size, 10))
	}

	req.ContentLength = t.Object.Size

	f, err := os.OpenFile(t.Path, os.O_RDONLY, 0644)
	if err != nil {
		return errutil.Error(err)
	}
	defer f.Close()

	// Ensure progress callbacks made while uploading
	var reader io.Reader
	reader = &progress.CallbackReader{
		C:         a.cb,
		TotalSize: t.Object.Size,
		Reader:    f,
	}

	if signalAuthOnResponse {
		// Signal auth was ok on first read; this frees up other workers to start
		reader = newStartCallbackReader(reader, func(*startCallbackReader) {
			a.authWait.Done()
		})
	}

	req.Body = ioutil.NopCloser(reader)

	res, err := httputil.DoHttpRequest(req, true)
	if err != nil {
		return errutil.NewRetriableError(err)
	}
	httputil.LogTransfer("lfs.data.upload", res)

	// A status code of 403 likely means that an authentication token for the
	// upload has expired. This can be safely retried.
	if res.StatusCode == 403 {
		return errutil.NewRetriableError(err)
	}

	if res.StatusCode > 299 {
		return errutil.Errorf(nil, "Invalid status for %s: %d", httputil.TraceHttpReq(req), res.StatusCode)
	}

	io.Copy(ioutil.Discard, res.Body)
	res.Body.Close()

	return api.VerifyUpload(t.Object)
}

// startCallbackReader is a reader wrapper which calls a function as soon as the
// first Read() call is made. This callback is only made once
type startCallbackReader struct {
	r      io.Reader
	cb     func(*startCallbackReader)
	cbDone bool
}

func (s *startCallbackReader) Read(p []byte) (n int, err error) {
	if !s.cbDone && s.cb != nil {
		s.cb(s)
		s.cbDone = true
	}
	return s.r.Read(p)
}
func newStartCallbackReader(r io.Reader, cb func(*startCallbackReader)) *startCallbackReader {
	return &startCallbackReader{r, cb, false}
}

func init() {
	ul := newBasicAdapter(Upload)
	RegisterAdapter(ul)
	dl := newBasicAdapter(Download)
	RegisterAdapter(dl)
}
<commit_msg>Make sure all workers stop if no jobs are added<commit_after>package transfer

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"sync"

	"github.com/github/git-lfs/api"
	"github.com/github/git-lfs/config"
	"github.com/github/git-lfs/errutil"
	"github.com/github/git-lfs/httputil"
	"github.com/github/git-lfs/progress"
	"github.com/github/git-lfs/tools"
	"github.com/github/git-lfs/vendor/_nuts/github.com/rubyist/tracerx"
)

const (
	BasicAdapterName = "basic"
)

// Base implementation of basic all-or-nothing HTTP upload / download adapter
type basicAdapter struct {
	direction Direction
	jobChan   chan *Transfer
	cb        progress.CopyCallback
	outChan   chan TransferResult
	// WaitGroup to sync the completion of all workers
	workerWait sync.WaitGroup
	// WaitGroup to serialise the first transfer response to perform login if needed
	authWait sync.WaitGroup
}

func newBasicAdapter(d Direction) *basicAdapter {
	return &basicAdapter{
		direction: d,
	}
}

func (a *basicAdapter) Direction() Direction {
	return a.direction
}

func (a *basicAdapter) Name() string {
	return BasicAdapterName
}

func (a *basicAdapter) Begin(cb progress.CopyCallback, completion chan TransferResult) error {
	a.cb = cb
	a.outChan = completion
	a.jobChan = make(chan *Transfer, 100)

	tracerx.Printf("xfer: adapter %q Begin()", a.Name())

	numworkers := config.Config.ConcurrentTransfers()
	a.workerWait.Add(numworkers)
	a.authWait.Add(1)
	for i := 0; i < numworkers; i++ {
		go a.worker(i)
	}
	return nil
}

func (a *basicAdapter) Add(t *Transfer) {
	tracerx.Printf("xfer: adapter %q Add() for %q", a.Name(), t.Object.Oid)
	a.jobChan <- t
}

func (a *basicAdapter) End() {
	tracerx.Printf("xfer: adapter %q End()", a.Name())
	close(a.jobChan)
	// wait for all transfers to complete
	a.workerWait.Wait()
}

func (a *basicAdapter) ClearTempStorage() error {
	// Should be empty already but also remove dir
	return os.RemoveAll(a.tempDir())
}

// worker function, many of these run per adapter
func (a *basicAdapter) worker(workerNum int) {

	tracerx.Printf("xfer: adapter %q worker %d starting", a.Name(), workerNum)
	waitForAuth := workerNum > 0
	signalAuthOnResponse := workerNum == 0

	// First worker is the only one allowed to start immediately
	// The rest wait until successful response from 1st worker to
	// make sure only 1 login prompt is presented if necessary
	// Deliberately outside jobChan processing so we know worker 0 will process 1st item
	if waitForAuth {
		tracerx.Printf("xfer: adapter %q worker %d waiting for Auth", a.Name(), workerNum)
		a.authWait.Wait()
		tracerx.Printf("xfer: adapter %q worker %d auth signal received", a.Name(), workerNum)
	}

	for t := range a.jobChan {
		tracerx.Printf("xfer: adapter %q worker %d processing job for %q", a.Name(), workerNum, t.Object.Oid)
		var err error
		switch a.Direction() {
		case Download:
			err = a.download(t, signalAuthOnResponse)
		case Upload:
			err = a.upload(t, signalAuthOnResponse)
		}

		res := TransferResult{t, err}
		a.outChan <- res

		// Only need to signal for auth once
		signalAuthOnResponse = false

		tracerx.Printf("xfer: adapter %q worker %d finished job for %q", a.Name(), workerNum, t.Object.Oid)
	}
	// This will only happen if no jobs were submitted; just wake up all workers to finish
	if signalAuthOnResponse {
		a.authWait.Done()
	}
	a.workerWait.Done()
}

func (a *basicAdapter) tempDir() string {
	// Must be dedicated to this adapter as deleted by ClearTempStorage
	d := filepath.Join(os.TempDir(), "git-lfs-basic-temp")
	if err := os.MkdirAll(d, 0755); err != nil {
		return os.TempDir()
	}
	return d
}

func (a *basicAdapter) download(t *Transfer, signalAuthOnResponse bool) error {
	rel, ok := t.Object.Rel("download")
	if !ok {
		return errors.New("Object not found on the server.")
	}

	req, err := httputil.NewHttpRequest("GET", rel.Href, rel.Header)
	if err != nil {
		return err
	}

	res, err := httputil.DoHttpRequest(req, true)
	if err != nil {
		return errutil.NewRetriableError(err)
	}
	httputil.LogTransfer("lfs.data.download", res)
	defer res.Body.Close()

	// Signal auth OK on success response, before starting download to free up
	// other workers immediately
	if signalAuthOnResponse {
		a.authWait.Done()
	}

	// Now do transfer of content
	f, err := ioutil.TempFile(a.tempDir(), t.Object.Oid+"-")
	if err != nil {
		return fmt.Errorf("cannot create temp file: %v", err)
	}

	defer func() {
		if err != nil {
			// Don't leave the temp file lying around on error.
			_ = os.Remove(f.Name()) // yes, ignore the error, not much we can do about it.
		}
	}()

	hasher := tools.NewHashingReader(res.Body)

	// ensure we always close f. Note that this does not conflict with  the
	// close below, as close is idempotent.
	defer f.Close()
	tempfilename := f.Name()
	written, err := tools.CopyWithCallback(f, hasher, res.ContentLength, a.cb)
	if err != nil {
		return fmt.Errorf("cannot write data to tempfile %q: %v", tempfilename, err)
	}
	if err := f.Close(); err != nil {
		return fmt.Errorf("can't close tempfile %q: %v", tempfilename, err)
	}

	if actual := hasher.Hash(); actual != t.Object.Oid {
		return fmt.Errorf("Expected OID %s, got %s after %d bytes written", t.Object.Oid, actual, written)
	}

	return tools.RenameFileCopyPermissions(tempfilename, t.Path)

}
func (a *basicAdapter) upload(t *Transfer, signalAuthOnResponse bool) error {
	rel, ok := t.Object.Rel("upload")
	if !ok {
		return fmt.Errorf("No upload action for this object.")
	}

	req, err := httputil.NewHttpRequest("PUT", rel.Href, rel.Header)
	if err != nil {
		return err
	}

	if len(req.Header.Get("Content-Type")) == 0 {
		req.Header.Set("Content-Type", "application/octet-stream")
	}

	if req.Header.Get("Transfer-Encoding") == "chunked" {
		req.TransferEncoding = []string{"chunked"}
	} else {
		req.Header.Set("Content-Length", strconv.FormatInt(t.Object.Size, 10))
	}

	req.ContentLength = t.Object.Size

	f, err := os.OpenFile(t.Path, os.O_RDONLY, 0644)
	if err != nil {
		return errutil.Error(err)
	}
	defer f.Close()

	// Ensure progress callbacks made while uploading
	var reader io.Reader
	reader = &progress.CallbackReader{
		C:         a.cb,
		TotalSize: t.Object.Size,
		Reader:    f,
	}

	if signalAuthOnResponse {
		// Signal auth was ok on first read; this frees up other workers to start
		reader = newStartCallbackReader(reader, func(*startCallbackReader) {
			a.authWait.Done()
		})
	}

	req.Body = ioutil.NopCloser(reader)

	res, err := httputil.DoHttpRequest(req, true)
	if err != nil {
		return errutil.NewRetriableError(err)
	}
	httputil.LogTransfer("lfs.data.upload", res)

	// A status code of 403 likely means that an authentication token for the
	// upload has expired. This can be safely retried.
	if res.StatusCode == 403 {
		return errutil.NewRetriableError(err)
	}

	if res.StatusCode > 299 {
		return errutil.Errorf(nil, "Invalid status for %s: %d", httputil.TraceHttpReq(req), res.StatusCode)
	}

	io.Copy(ioutil.Discard, res.Body)
	res.Body.Close()

	return api.VerifyUpload(t.Object)
}

// startCallbackReader is a reader wrapper which calls a function as soon as the
// first Read() call is made. This callback is only made once
type startCallbackReader struct {
	r      io.Reader
	cb     func(*startCallbackReader)
	cbDone bool
}

func (s *startCallbackReader) Read(p []byte) (n int, err error) {
	if !s.cbDone && s.cb != nil {
		s.cb(s)
		s.cbDone = true
	}
	return s.r.Read(p)
}
func newStartCallbackReader(r io.Reader, cb func(*startCallbackReader)) *startCallbackReader {
	return &startCallbackReader{r, cb, false}
}

func init() {
	ul := newBasicAdapter(Upload)
	RegisterAdapter(ul)
	dl := newBasicAdapter(Download)
	RegisterAdapter(dl)
}
<|endoftext|>


<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package archive

import (
	"compress/gzip"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/juju/errors"
	"github.com/juju/utils/tar"

	"github.com/juju/juju/state/backups/metadata"
)

// Workspace is a wrapper around backup archive info that has a concrete
// root directory and an archive unpacked in it.
type Workspace struct {
	Archive
	rootDir string
}

func newWorkspace(filename string) (*Workspace, error) {
	dirName, err := ioutil.TempDir("", "juju-backups-")
	if err != nil {
		return nil, errors.Annotate(err, "while creating workspace dir")
	}

	ar := NewArchive(filename, dirName)
	ws := Workspace{
		Archive: *ar,
		rootDir: dirName,
	}
	return &ws, nil
}

// NewWorkspace creates a new workspace for backup archives. The
// workspace is based in a new directory that will be deleted when the
// workspace is closed.
func NewWorkspace(filename string) (*Workspace, error) {
	ws, err := newWorkspace(filename)
	if err != nil {
		return nil, errors.Trace(err)
	}

	if archiveFile, err := os.Open(ws.Filename); err != nil {
		if !os.IsNotExist(err) {
			return nil, errors.Trace(err)
		}
	} else {
		if err := unpack(archiveFile, ws.rootDir); err != nil {
			return nil, errors.Trace(err)
		}
	}

	return ws, nil
}

// NewWorkspace returns a new archive workspace with a new workspace dir
// populated from the archive file.
func NewWorkspaceFromFile(archiveFile io.Reader) (*Workspace, error) {
	ws, err := newWorkspace("")
	if err != nil {
		return nil, errors.Trace(err)
	}
	err = unpack(archiveFile, ws.rootDir)
	return ws, errors.Trace(err)
}

func unpack(tarFile io.Reader, targetDir string) error {
	tarFile, err := gzip.NewReader(tarFile)
	if err != nil {
		return errors.Annotate(err, "while uncompressing archive file")
	}
	if err := tar.UntarFiles(tarFile, targetDir); err != nil {
		return errors.Annotate(err, "while extracting files from archive")
	}
	return nil
}

// Close cleans up the workspace dir.
func (ws *Workspace) Close() error {
	err := os.RemoveAll(ws.rootDir)
	return errors.Trace(err)
}

// UnpackFiles unpacks the archived files bundle into the targeted dir.
func (ws *Workspace) UnpackFiles(targetRoot string) error {
	tarFile, err := os.Open(ws.FilesBundle())
	if err != nil {
		return errors.Trace(err)
	}
	defer tarFile.Close()

	if err := tar.UntarFiles(tarFile, targetRoot); err != nil {
		return errors.Annotate(err, "while unpacking system files")
	}
	return nil
}

// OpenFile returns an open ReadCloser for the corresponding file in
// the archived files bundle.
func (ws *Workspace) OpenFile(filename string) (io.Reader, error) {
	if filepath.IsAbs(filename) {
		return nil, errors.Errorf("filename must be relative, got %q", filename)
	}

	tarFile, err := os.Open(ws.FilesBundle())
	if err != nil {
		return nil, errors.Trace(err)
	}

	_, file, err := tar.FindFile(tarFile, filename)
	if err != nil {
		tarFile.Close()
		return nil, errors.Trace(err)
	}
	return file, nil
}

// Metadata returns the metadata derived from the JSON file in the archive.
func (ws *Workspace) Metadata() (*metadata.Metadata, error) {
	metaFile, err := os.Open(ws.MetadataFile())
	if err != nil {
		return nil, errors.Trace(err)
	}
	defer metaFile.Close()

	meta, err := metadata.NewFromJSONBuffer(metaFile)
	return meta, errors.Trace(err)
}
<commit_msg>Archive -> *Archive.<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package archive

import (
	"compress/gzip"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/juju/errors"
	"github.com/juju/utils/tar"

	"github.com/juju/juju/state/backups/metadata"
)

// Workspace is a wrapper around backup archive info that has a concrete
// root directory and an archive unpacked in it.
type Workspace struct {
	*Archive
	rootDir string
}

func newWorkspace(filename string) (*Workspace, error) {
	dirName, err := ioutil.TempDir("", "juju-backups-")
	if err != nil {
		return nil, errors.Annotate(err, "while creating workspace dir")
	}

	ar := NewArchive(filename, dirName)
	ws := Workspace{
		Archive: ar,
		rootDir: dirName,
	}
	return &ws, nil
}

// NewWorkspace creates a new workspace for backup archives. The
// workspace is based in a new directory that will be deleted when the
// workspace is closed.
func NewWorkspace(filename string) (*Workspace, error) {
	ws, err := newWorkspace(filename)
	if err != nil {
		return nil, errors.Trace(err)
	}

	if archiveFile, err := os.Open(ws.Filename); err != nil {
		if !os.IsNotExist(err) {
			return nil, errors.Trace(err)
		}
	} else {
		if err := unpack(archiveFile, ws.rootDir); err != nil {
			return nil, errors.Trace(err)
		}
	}

	return ws, nil
}

// NewWorkspace returns a new archive workspace with a new workspace dir
// populated from the archive file.
func NewWorkspaceFromFile(archiveFile io.Reader) (*Workspace, error) {
	ws, err := newWorkspace("")
	if err != nil {
		return nil, errors.Trace(err)
	}
	err = unpack(archiveFile, ws.rootDir)
	return ws, errors.Trace(err)
}

func unpack(tarFile io.Reader, targetDir string) error {
	tarFile, err := gzip.NewReader(tarFile)
	if err != nil {
		return errors.Annotate(err, "while uncompressing archive file")
	}
	if err := tar.UntarFiles(tarFile, targetDir); err != nil {
		return errors.Annotate(err, "while extracting files from archive")
	}
	return nil
}

// Close cleans up the workspace dir.
func (ws *Workspace) Close() error {
	err := os.RemoveAll(ws.rootDir)
	return errors.Trace(err)
}

// UnpackFiles unpacks the archived files bundle into the targeted dir.
func (ws *Workspace) UnpackFiles(targetRoot string) error {
	tarFile, err := os.Open(ws.FilesBundle())
	if err != nil {
		return errors.Trace(err)
	}
	defer tarFile.Close()

	if err := tar.UntarFiles(tarFile, targetRoot); err != nil {
		return errors.Annotate(err, "while unpacking system files")
	}
	return nil
}

// OpenFile returns an open ReadCloser for the corresponding file in
// the archived files bundle.
func (ws *Workspace) OpenFile(filename string) (io.Reader, error) {
	if filepath.IsAbs(filename) {
		return nil, errors.Errorf("filename must be relative, got %q", filename)
	}

	tarFile, err := os.Open(ws.FilesBundle())
	if err != nil {
		return nil, errors.Trace(err)
	}

	_, file, err := tar.FindFile(tarFile, filename)
	if err != nil {
		tarFile.Close()
		return nil, errors.Trace(err)
	}
	return file, nil
}

// Metadata returns the metadata derived from the JSON file in the archive.
func (ws *Workspace) Metadata() (*metadata.Metadata, error) {
	metaFile, err := os.Open(ws.MetadataFile())
	if err != nil {
		return nil, errors.Trace(err)
	}
	defer metaFile.Close()

	meta, err := metadata.NewFromJSONBuffer(metaFile)
	return meta, errors.Trace(err)
}
<|endoftext|>


<commit_before>package services

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"

	datatypes "github.com/maximilien/softlayer-go/data_types"
	softlayer "github.com/maximilien/softlayer-go/softlayer"
)

type softLayer_Virtual_Guest_Block_Device_Template_Group_Service struct {
	client softlayer.Client
}

func NewSoftLayer_Virtual_Guest_Block_Device_Template_Group_Service(client softlayer.Client) *softLayer_Virtual_Guest_Block_Device_Template_Group_Service {
	return &softLayer_Virtual_Guest_Block_Device_Template_Group_Service{
		client: client,
	}
}

func (slvgs *softLayer_Virtual_Guest_Block_Device_Template_Group_Service) GetName() string {
	return "SoftLayer_Virtual_Guest_Block_Device_Template_Group"
}

func (slvgbdtg *softLayer_Virtual_Guest_Block_Device_Template_Group_Service) GetObject(id int) (datatypes.SoftLayer_Virtual_Guest_Block_Device_Template_Group, error) {
	response, err := slvgbdtg.client.DoRawHttpRequest(fmt.Sprintf("%s/%d/getObject.json", slvgbdtg.GetName(), id), "GET", new(bytes.Buffer))
	if err != nil {
		return datatypes.SoftLayer_Virtual_Guest_Block_Device_Template_Group{}, err
	}

	vgbdtGroup := datatypes.SoftLayer_Virtual_Guest_Block_Device_Template_Group{}
	err = json.Unmarshal(response, &vgbdtGroup)
	if err != nil {
		return datatypes.SoftLayer_Virtual_Guest_Block_Device_Template_Group{}, err
	}

	return vgbdtGroup, nil
}

func (slvgbdtg *softLayer_Virtual_Guest_Block_Device_Template_Group_Service) DeleteObject(id int) (datatypes.SoftLayer_Provisioning_Version1_Transaction, error) {
	response, err := slvgbdtg.client.DoRawHttpRequest(fmt.Sprintf("%s/%d.json", slvgbdtg.GetName(), id), "DELETE", new(bytes.Buffer))
	if err != nil {
		return datatypes.SoftLayer_Provisioning_Version1_Transaction{}, err
	}

	transaction := datatypes.SoftLayer_Provisioning_Version1_Transaction{}
	err = json.Unmarshal(response, &transaction)
	if err != nil {
		return datatypes.SoftLayer_Provisioning_Version1_Transaction{}, err
	}

	return transaction, nil
}

func (slvgbdtg *softLayer_Virtual_Guest_Block_Device_Template_Group_Service) GetDatacenters(id int) ([]datatypes.SoftLayer_Location, error) {
	response, err := slvgbdtg.client.DoRawHttpRequest(fmt.Sprintf("%s/%d/getDatacenters.json", slvgbdtg.GetName(), id), "GET", new(bytes.Buffer))
	if err != nil {
		return []datatypes.SoftLayer_Location{}, err
	}

	locations := []datatypes.SoftLayer_Location{}
	err = json.Unmarshal(response, &locations)
	if err != nil {
		return []datatypes.SoftLayer_Location{}, err
	}

	return locations, nil
}

func (slvgbdtg *softLayer_Virtual_Guest_Block_Device_Template_Group_Service) GetSshKeys(id int) ([]datatypes.SoftLayer_Security_Ssh_Key, error) {
	response, err := slvgbdtg.client.DoRawHttpRequest(fmt.Sprintf("%s/%d/getSshKeys.json", slvgbdtg.GetName(), id), "GET", new(bytes.Buffer))
	if err != nil {
		return []datatypes.SoftLayer_Security_Ssh_Key{}, err
	}

	sshKeys := []datatypes.SoftLayer_Security_Ssh_Key{}
	err = json.Unmarshal(response, &sshKeys)
	if err != nil {
		return []datatypes.SoftLayer_Security_Ssh_Key{}, err
	}

	return sshKeys, nil
}

func (slvgbdtg *softLayer_Virtual_Guest_Block_Device_Template_Group_Service) GetStatus(id int) (datatypes.SoftLayer_Virtual_Guest_Block_Device_Template_Group_Status, error) {
	response, err := slvgbdtg.client.DoRawHttpRequest(fmt.Sprintf("%s/%d/getStatus.json", slvgbdtg.GetName(), id), "GET", new(bytes.Buffer))
	if err != nil {
		return datatypes.SoftLayer_Virtual_Guest_Block_Device_Template_Group_Status{}, err
	}

	status := datatypes.SoftLayer_Virtual_Guest_Block_Device_Template_Group_Status{}
	err = json.Unmarshal(response, &status)
	if err != nil {
		return datatypes.SoftLayer_Virtual_Guest_Block_Device_Template_Group_Status{}, err
	}

	return status, nil
}

func (slvgbdtg *softLayer_Virtual_Guest_Block_Device_Template_Group_Service) GetImageType(id int) (datatypes.SoftLayer_Image_Type, error) {
	response, err := slvgbdtg.client.DoRawHttpRequest(fmt.Sprintf("%s/%d/getImageType.json", slvgbdtg.GetName(), id), "GET", new(bytes.Buffer))
	if err != nil {
		return datatypes.SoftLayer_Image_Type{}, err
	}

	imageType := datatypes.SoftLayer_Image_Type{}
	err = json.Unmarshal(response, &imageType)
	if err != nil {
		return datatypes.SoftLayer_Image_Type{}, err
	}

	return imageType, nil
}

func (slvgbdtg *softLayer_Virtual_Guest_Block_Device_Template_Group_Service) GetStorageLocations(id int) ([]datatypes.SoftLayer_Location, error) {
	response, err := slvgbdtg.client.DoRawHttpRequest(fmt.Sprintf("%s/%d/getStorageLocations.json", slvgbdtg.GetName(), id), "GET", new(bytes.Buffer))
	if err != nil {
		return []datatypes.SoftLayer_Location{}, err
	}

	locations := []datatypes.SoftLayer_Location{}
	err = json.Unmarshal(response, &locations)
	if err != nil {
		return []datatypes.SoftLayer_Location{}, err
	}

	return locations, nil
}

func (slvgbdtg *softLayer_Virtual_Guest_Block_Device_Template_Group_Service) CreateFromExternalSource(configuration datatypes.SoftLayer_Container_Virtual_Guest_Block_Device_Template_Configuration) (datatypes.SoftLayer_Virtual_Guest_Block_Device_Template_Group, error) {
	parameters := datatypes.SoftLayer_Container_Virtual_Guest_Block_Device_Template_Configuration_Parameters{
		Parameters: []datatypes.SoftLayer_Container_Virtual_Guest_Block_Device_Template_Configuration{configuration},
	}

	requestBody, err := json.Marshal(parameters)
	if err != nil {
		return datatypes.SoftLayer_Virtual_Guest_Block_Device_Template_Group{}, err
	}

	response, err := slvgbdtg.client.DoRawHttpRequest(fmt.Sprintf("%s/CreateFromExternalSource.json", slvgbdtg.GetName()), "POST", bytes.NewBuffer(requestBody))
	if err != nil {
		return datatypes.SoftLayer_Virtual_Guest_Block_Device_Template_Group{}, err
	}

	vgbdtGroup := datatypes.SoftLayer_Virtual_Guest_Block_Device_Template_Group{}
	err = json.Unmarshal(response, &vgbdtGroup)
	if err != nil {
		return datatypes.SoftLayer_Virtual_Guest_Block_Device_Template_Group{}, err
	}

	return vgbdtGroup, err
}

func (slvgbdtg *softLayer_Virtual_Guest_Block_Device_Template_Group_Service) CopyToExternalSource(configuration datatypes.SoftLayer_Container_Virtual_Guest_Block_Device_Template_Configuration) (bool, error) {
	parameters := datatypes.SoftLayer_Container_Virtual_Guest_Block_Device_Template_Configuration_Parameters{
		Parameters: []datatypes.SoftLayer_Container_Virtual_Guest_Block_Device_Template_Configuration{configuration},
	}

	requestBody, err := json.Marshal(parameters)
	if err != nil {
		return false, err
	}

	response, err := slvgbdtg.client.DoRawHttpRequest(fmt.Sprintf("%s/CopyToExternalSource.json", slvgbdtg.GetName()), "POST", bytes.NewBuffer(requestBody))
	if err != nil {
		return false, err
	}

	if res := string(response[:]); res != "true" {
		return false, errors.New(fmt.Sprintf("Failed to create virtual guest block device template group, got '%s' as response from the API.", res))
	}

	return true, nil
}

func (slvgbdtg *softLayer_Virtual_Guest_Block_Device_Template_Group_Service) GetImageTypeKeyName(id int) (string, error) {
	response, err := slvgbdtg.client.DoRawHttpRequest(fmt.Sprintf("%s/%d/GetImageTypeKeyName.json", slvgbdtg.GetName(), id), "GET", new(bytes.Buffer))

	return string(response), err
}
<commit_msg>fix vgbdtgService methods REST paths<commit_after>package services

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"

	datatypes "github.com/maximilien/softlayer-go/data_types"
	softlayer "github.com/maximilien/softlayer-go/softlayer"
)

type softLayer_Virtual_Guest_Block_Device_Template_Group_Service struct {
	client softlayer.Client
}

func NewSoftLayer_Virtual_Guest_Block_Device_Template_Group_Service(client softlayer.Client) *softLayer_Virtual_Guest_Block_Device_Template_Group_Service {
	return &softLayer_Virtual_Guest_Block_Device_Template_Group_Service{
		client: client,
	}
}

func (slvgs *softLayer_Virtual_Guest_Block_Device_Template_Group_Service) GetName() string {
	return "SoftLayer_Virtual_Guest_Block_Device_Template_Group"
}

func (slvgbdtg *softLayer_Virtual_Guest_Block_Device_Template_Group_Service) GetObject(id int) (datatypes.SoftLayer_Virtual_Guest_Block_Device_Template_Group, error) {
	response, err := slvgbdtg.client.DoRawHttpRequest(fmt.Sprintf("%s/%d/getObject.json", slvgbdtg.GetName(), id), "GET", new(bytes.Buffer))
	if err != nil {
		return datatypes.SoftLayer_Virtual_Guest_Block_Device_Template_Group{}, err
	}

	vgbdtGroup := datatypes.SoftLayer_Virtual_Guest_Block_Device_Template_Group{}
	err = json.Unmarshal(response, &vgbdtGroup)
	if err != nil {
		return datatypes.SoftLayer_Virtual_Guest_Block_Device_Template_Group{}, err
	}

	return vgbdtGroup, nil
}

func (slvgbdtg *softLayer_Virtual_Guest_Block_Device_Template_Group_Service) DeleteObject(id int) (datatypes.SoftLayer_Provisioning_Version1_Transaction, error) {
	response, err := slvgbdtg.client.DoRawHttpRequest(fmt.Sprintf("%s/%d.json", slvgbdtg.GetName(), id), "DELETE", new(bytes.Buffer))
	if err != nil {
		return datatypes.SoftLayer_Provisioning_Version1_Transaction{}, err
	}

	transaction := datatypes.SoftLayer_Provisioning_Version1_Transaction{}
	err = json.Unmarshal(response, &transaction)
	if err != nil {
		return datatypes.SoftLayer_Provisioning_Version1_Transaction{}, err
	}

	return transaction, nil
}

func (slvgbdtg *softLayer_Virtual_Guest_Block_Device_Template_Group_Service) GetDatacenters(id int) ([]datatypes.SoftLayer_Location, error) {
	response, err := slvgbdtg.client.DoRawHttpRequest(fmt.Sprintf("%s/%d/getDatacenters.json", slvgbdtg.GetName(), id), "GET", new(bytes.Buffer))
	if err != nil {
		return []datatypes.SoftLayer_Location{}, err
	}

	locations := []datatypes.SoftLayer_Location{}
	err = json.Unmarshal(response, &locations)
	if err != nil {
		return []datatypes.SoftLayer_Location{}, err
	}

	return locations, nil
}

func (slvgbdtg *softLayer_Virtual_Guest_Block_Device_Template_Group_Service) GetSshKeys(id int) ([]datatypes.SoftLayer_Security_Ssh_Key, error) {
	response, err := slvgbdtg.client.DoRawHttpRequest(fmt.Sprintf("%s/%d/getSshKeys.json", slvgbdtg.GetName(), id), "GET", new(bytes.Buffer))
	if err != nil {
		return []datatypes.SoftLayer_Security_Ssh_Key{}, err
	}

	sshKeys := []datatypes.SoftLayer_Security_Ssh_Key{}
	err = json.Unmarshal(response, &sshKeys)
	if err != nil {
		return []datatypes.SoftLayer_Security_Ssh_Key{}, err
	}

	return sshKeys, nil
}

func (slvgbdtg *softLayer_Virtual_Guest_Block_Device_Template_Group_Service) GetStatus(id int) (datatypes.SoftLayer_Virtual_Guest_Block_Device_Template_Group_Status, error) {
	response, err := slvgbdtg.client.DoRawHttpRequest(fmt.Sprintf("%s/%d/getStatus.json", slvgbdtg.GetName(), id), "GET", new(bytes.Buffer))
	if err != nil {
		return datatypes.SoftLayer_Virtual_Guest_Block_Device_Template_Group_Status{}, err
	}

	status := datatypes.SoftLayer_Virtual_Guest_Block_Device_Template_Group_Status{}
	err = json.Unmarshal(response, &status)
	if err != nil {
		return datatypes.SoftLayer_Virtual_Guest_Block_Device_Template_Group_Status{}, err
	}

	return status, nil
}

func (slvgbdtg *softLayer_Virtual_Guest_Block_Device_Template_Group_Service) GetImageType(id int) (datatypes.SoftLayer_Image_Type, error) {
	response, err := slvgbdtg.client.DoRawHttpRequest(fmt.Sprintf("%s/%d/getImageType.json", slvgbdtg.GetName(), id), "GET", new(bytes.Buffer))
	if err != nil {
		return datatypes.SoftLayer_Image_Type{}, err
	}

	imageType := datatypes.SoftLayer_Image_Type{}
	err = json.Unmarshal(response, &imageType)
	if err != nil {
		return datatypes.SoftLayer_Image_Type{}, err
	}

	return imageType, nil
}

func (slvgbdtg *softLayer_Virtual_Guest_Block_Device_Template_Group_Service) GetStorageLocations(id int) ([]datatypes.SoftLayer_Location, error) {
	response, err := slvgbdtg.client.DoRawHttpRequest(fmt.Sprintf("%s/%d/getStorageLocations.json", slvgbdtg.GetName(), id), "GET", new(bytes.Buffer))
	if err != nil {
		return []datatypes.SoftLayer_Location{}, err
	}

	locations := []datatypes.SoftLayer_Location{}
	err = json.Unmarshal(response, &locations)
	if err != nil {
		return []datatypes.SoftLayer_Location{}, err
	}

	return locations, nil
}

func (slvgbdtg *softLayer_Virtual_Guest_Block_Device_Template_Group_Service) CreateFromExternalSource(configuration datatypes.SoftLayer_Container_Virtual_Guest_Block_Device_Template_Configuration) (datatypes.SoftLayer_Virtual_Guest_Block_Device_Template_Group, error) {
	parameters := datatypes.SoftLayer_Container_Virtual_Guest_Block_Device_Template_Configuration_Parameters{
		Parameters: []datatypes.SoftLayer_Container_Virtual_Guest_Block_Device_Template_Configuration{configuration},
	}

	requestBody, err := json.Marshal(parameters)
	if err != nil {
		return datatypes.SoftLayer_Virtual_Guest_Block_Device_Template_Group{}, err
	}

	response, err := slvgbdtg.client.DoRawHttpRequest(fmt.Sprintf("%s/createFromExternalSource.json", slvgbdtg.GetName()), "POST", bytes.NewBuffer(requestBody))
	if err != nil {
		return datatypes.SoftLayer_Virtual_Guest_Block_Device_Template_Group{}, err
	}

	vgbdtGroup := datatypes.SoftLayer_Virtual_Guest_Block_Device_Template_Group{}
	err = json.Unmarshal(response, &vgbdtGroup)
	if err != nil {
		return datatypes.SoftLayer_Virtual_Guest_Block_Device_Template_Group{}, err
	}

	return vgbdtGroup, err
}

func (slvgbdtg *softLayer_Virtual_Guest_Block_Device_Template_Group_Service) CopyToExternalSource(configuration datatypes.SoftLayer_Container_Virtual_Guest_Block_Device_Template_Configuration) (bool, error) {
	parameters := datatypes.SoftLayer_Container_Virtual_Guest_Block_Device_Template_Configuration_Parameters{
		Parameters: []datatypes.SoftLayer_Container_Virtual_Guest_Block_Device_Template_Configuration{configuration},
	}

	requestBody, err := json.Marshal(parameters)
	if err != nil {
		return false, err
	}

	response, err := slvgbdtg.client.DoRawHttpRequest(fmt.Sprintf("%s/copyToExternalSource.json", slvgbdtg.GetName()), "POST", bytes.NewBuffer(requestBody))
	if err != nil {
		return false, err
	}

	if res := string(response[:]); res != "true" {
		return false, errors.New(fmt.Sprintf("Failed to create virtual guest block device template group, got '%s' as response from the API.", res))
	}

	return true, nil
}

func (slvgbdtg *softLayer_Virtual_Guest_Block_Device_Template_Group_Service) GetImageTypeKeyName(id int) (string, error) {
	response, err := slvgbdtg.client.DoRawHttpRequest(fmt.Sprintf("%s/%d/getImageTypeKeyName.json", slvgbdtg.GetName(), id), "GET", new(bytes.Buffer))

	return string(response), err
}
<|endoftext|>


<commit_before>package main

import (
	aonui "github.com/rjw57/aonui"
	"fmt"
)

type ByteCount int64

func (bytes ByteCount) String() string {
	switch {
	case bytes < 2<<10:
		return fmt.Sprintf("%dB", bytes)
	case bytes < 2<<20:
		return fmt.Sprintf("%dKiB", bytes>>10)
	case bytes < 2<<30:
		return fmt.Sprintf("%dMiB", bytes>>20)
	default:
		return fmt.Sprintf("%dGiB", bytes>>30)
	}
}

// Sorting runs by date
type ByDate []*aonui.Run

func (d ByDate) Len() int {
	return len(d)
}

func (d ByDate) Swap(i, j int) {
	d[i], d[j] = d[j], d[i]
}

func (d ByDate) Less(i, j int) bool {
	return d[i].When.Before(d[j].When)
}
<commit_msg>aonuisync: fix imports to be idiomatic<commit_after>package main

import (
	"fmt"

	"github.com/rjw57/aonui"
)

type ByteCount int64

func (bytes ByteCount) String() string {
	switch {
	case bytes < 2<<10:
		return fmt.Sprintf("%dB", bytes)
	case bytes < 2<<20:
		return fmt.Sprintf("%dKiB", bytes>>10)
	case bytes < 2<<30:
		return fmt.Sprintf("%dMiB", bytes>>20)
	default:
		return fmt.Sprintf("%dGiB", bytes>>30)
	}
}

// Sorting runs by date
type ByDate []*aonui.Run

func (d ByDate) Len() int {
	return len(d)
}

func (d ByDate) Swap(i, j int) {
	d[i], d[j] = d[j], d[i]
}

func (d ByDate) Less(i, j int) bool {
	return d[i].When.Before(d[j].When)
}
<|endoftext|>


<commit_before>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Dirserver is a wrapper for a directory implementation that presents it as a grpc interface.
package main

import (
	"net/http"
	"os"

	"upspin.io/auth"
	"upspin.io/auth/grpcauth"
	"upspin.io/cloud/https"
	"upspin.io/context"
	"upspin.io/dir/filesystem"
	"upspin.io/dir/gcp"
	"upspin.io/dir/inprocess"
	"upspin.io/errors"
	"upspin.io/flags"
	"upspin.io/log"
	"upspin.io/metric"
	"upspin.io/upspin"
	"upspin.io/upspin/proto"

	gContext "golang.org/x/net/context"

	// TODO: Which of these are actually needed?

	// Load useful packers
	_ "upspin.io/pack/debug"
	_ "upspin.io/pack/ee"
	_ "upspin.io/pack/plain"

	// Load required transports
	_ "upspin.io/dir/transports"
	_ "upspin.io/key/transports"
	_ "upspin.io/store/transports"
)

// Server is a SecureServer that talks to a DirServer interface and serves gRPC requests.
type Server struct {
	context upspin.Context

	// What this server reports itself as through its Endpoint method.
	endpoint upspin.Endpoint

	// The underlying dirserver implementation.
	dir upspin.DirServer

	// Automatically handles authentication by implementing the Authenticate server method.
	grpcauth.SecureServer
}

const serverName = "dirserver"

func main() {
	flags.Parse("addr", "config", "context", "https", "kind", "log", "project")

	if flags.Project != "" {
		log.Connect(flags.Project, serverName)
		svr, err := metric.NewGCPSaver(flags.Project, "serverName", serverName)
		if err != nil {
			log.Fatalf("Can't start a metric saver for GCP project %q: %s", flags.Project, err)
		} else {
			metric.RegisterSaver(svr)
		}
	}

	// Load context and keys for this server. It needs a real upspin username and keys.
	ctxfd, err := os.Open(flags.Context)
	if err != nil {
		log.Fatal(err)
	}
	defer ctxfd.Close()
	ctx, err := context.InitContext(ctxfd)
	if err != nil {
		log.Fatal(err)
	}

	// Create a new store implementation.
	var dir upspin.DirServer
	err = nil
	switch flags.ServerKind {
	case "inprocess":
		dir = inprocess.New(ctx)
	case "gcp":
		dir, err = gcp.New(ctx, flags.Config...)
	case "filesystem":
		dir, err = filesystem.New(ctx, flags.Config...)
	default:
		err = errors.Errorf("bad -kind %q", flags.ServerKind)
	}
	if err != nil {
		log.Fatalf("Setting up DirServer: %v", err)
	}

	config := auth.Config{Lookup: auth.PublicUserKeyService(ctx)}
	grpcSecureServer, err := grpcauth.NewSecureServer(config)
	if err != nil {
		log.Fatal(err)
	}
	s := &Server{
		context: ctx,
		endpoint: upspin.Endpoint{
			Transport: upspin.Remote,
			NetAddr:   upspin.NetAddr(flags.NetAddr),
		},
		dir:          dir,
		SecureServer: grpcSecureServer,
	}
	proto.RegisterDirServer(grpcSecureServer.GRPCServer(), s)

	http.Handle("/", grpcSecureServer.GRPCServer())
	https.ListenAndServe(serverName, flags.HTTPSAddr, nil)
}

var (
	// Empty structs we can allocate just once.
	putResponse       proto.DirPutResponse
	deleteResponse    proto.DirDeleteResponse
	configureResponse proto.ConfigureResponse
)

// dirFor returns a DirServer instance bound to the user specified in the context.
func (s *Server) dirFor(ctx gContext.Context) (upspin.DirServer, error) {
	// Validate that we have a session. If not, it's an auth error.
	session, err := s.GetSessionFromContext(ctx)
	if err != nil {
		return nil, err
	}
	svc, err := s.dir.Dial(s.context.Copy().SetUserName(session.User()), s.dir.Endpoint())
	if err != nil {
		return nil, err
	}
	return svc.(upspin.DirServer), nil
}

// Lookup implements upspin.DirServer.
func (s *Server) Lookup(ctx gContext.Context, req *proto.DirLookupRequest) (*proto.DirLookupResponse, error) {
	log.Printf("Lookup %q", req.Name)

	dir, err := s.dirFor(ctx)
	if err != nil {
		return nil, err
	}
	entry, err := dir.Lookup(upspin.PathName(req.Name))
	if err != nil {
		log.Printf("Lookup %q failed: %v", req.Name, err)
		return &proto.DirLookupResponse{Error: errors.MarshalError(err)}, nil
	}
	b, err := entry.Marshal()
	if err != nil {
		return nil, err
	}

	resp := &proto.DirLookupResponse{
		Entry: b,
	}
	return resp, nil
}

// Put implements upspin.DirServer.
func (s *Server) Put(ctx gContext.Context, req *proto.DirPutRequest) (*proto.DirPutResponse, error) {
	log.Printf("Put")

	entry, err := proto.UpspinDirEntry(req.Entry)
	if err != nil {
		return &proto.DirPutResponse{Error: errors.MarshalError(err)}, nil
	}
	log.Printf("Put %q", entry.Name)
	dir, err := s.dirFor(ctx)
	if err != nil {
		return nil, err
	}
	_, err = dir.Put(entry)
	if err != nil {
		// TODO: implement links.
		log.Printf("Put %q failed: %v", entry.Name, err)
		return &proto.DirPutResponse{Error: errors.MarshalError(err)}, nil
	}
	return &putResponse, nil
}

// MakeDirectory implements upspin.DirServer.
func (s *Server) MakeDirectory(ctx gContext.Context, req *proto.DirMakeDirectoryRequest) (*proto.DirMakeDirectoryResponse, error) {
	log.Printf("MakeDirectory %q", req.Name)

	dir, err := s.dirFor(ctx)
	if err != nil {
		return nil, err
	}
	entry, err := dir.MakeDirectory(upspin.PathName(req.Name))
	if err != nil {
		log.Printf("MakeDirectory %q failed: %v", req.Name, err)
		return &proto.DirMakeDirectoryResponse{Error: errors.MarshalError(err)}, nil
	}
	b, err := entry.Marshal()
	if err != nil {
		return nil, err
	}
	resp := &proto.DirMakeDirectoryResponse{
		Entry: b,
	}
	return resp, nil
}

// Glob implements upspin.DirServer.
func (s *Server) Glob(ctx gContext.Context, req *proto.DirGlobRequest) (*proto.DirGlobResponse, error) {
	log.Printf("Glob %q", req.Pattern)

	dir, err := s.dirFor(ctx)
	if err != nil {
		return nil, err
	}
	entries, err := dir.Glob(req.Pattern)
	if err != nil {
		log.Printf("Glob %q failed: %v", req.Pattern, err)
		return &proto.DirGlobResponse{Error: errors.MarshalError(err)}, nil
	}
	data, err := proto.DirEntryBytes(entries)
	resp := &proto.DirGlobResponse{
		Entries: data,
	}
	return resp, err
}

// Delete implements upspin.DirServer.
func (s *Server) Delete(ctx gContext.Context, req *proto.DirDeleteRequest) (*proto.DirDeleteResponse, error) {
	log.Printf("Delete %q", req.Name)

	dir, err := s.dirFor(ctx)
	if err != nil {
		return nil, err
	}
	_, err = dir.Delete(upspin.PathName(req.Name))
	if err != nil {
		// TODO: implement links.
		log.Printf("Delete %q failed: %v", req.Name, err)
		return &proto.DirDeleteResponse{Error: errors.MarshalError(err)}, nil
	}
	return &deleteResponse, nil
}

// WhichAccess implements upspin.DirServer.
func (s *Server) WhichAccess(ctx gContext.Context, req *proto.DirWhichAccessRequest) (*proto.DirWhichAccessResponse, error) {
	log.Printf("WhichAccess %q", req.Name)

	dir, err := s.dirFor(ctx)
	if err != nil {
		return nil, err
	}
	entry, err := dir.WhichAccess(upspin.PathName(req.Name))
	if err != nil {
		// TODO: implement links.
		log.Printf("WhichAccess %q failed: %v", req.Name, err)
	}
	var b []byte
	if entry != nil {
		b, err = entry.Marshal()
		if err != nil {
			return nil, err
		}
	}
	resp := &proto.DirWhichAccessResponse{
		Entry: b,
		Error: errors.MarshalError(err),
	}
	return resp, nil
}

// Configure implements upspin.Service
func (s *Server) Configure(ctx gContext.Context, req *proto.ConfigureRequest) (*proto.ConfigureResponse, error) {
	log.Printf("Configure %q", req.Options)
	dir, err := s.dirFor(ctx)
	if err != nil {
		return nil, err
	}
	err = dir.Configure(req.Options...)
	if err != nil {
		log.Printf("Configure %q failed: %v", req.Options, err)
	}
	return &configureResponse, err
}

// Endpoint implements upspin.Service
func (s *Server) Endpoint(ctx gContext.Context, req *proto.EndpointRequest) (*proto.EndpointResponse, error) {
	log.Print("Endpoint")
	dir, err := s.dirFor(ctx)
	if err != nil {
		return nil, err
	}
	endpoint := dir.Endpoint()
	resp := &proto.EndpointResponse{
		Endpoint: &proto.Endpoint{
			Transport: int32(endpoint.Transport),
			NetAddr:   string(endpoint.NetAddr),
		},
	}
	return resp, nil
}
<commit_msg>cmd/dirserver: handle links<commit_after>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Dirserver is a wrapper for a directory implementation that presents it as a grpc interface.
package main

import (
	"net/http"
	"os"

	"upspin.io/auth"
	"upspin.io/auth/grpcauth"
	"upspin.io/cloud/https"
	"upspin.io/context"
	"upspin.io/dir/filesystem"
	"upspin.io/dir/gcp"
	"upspin.io/dir/inprocess"
	"upspin.io/errors"
	"upspin.io/flags"
	"upspin.io/log"
	"upspin.io/metric"
	"upspin.io/upspin"
	"upspin.io/upspin/proto"

	gContext "golang.org/x/net/context"

	// TODO: Which of these are actually needed?

	// Load useful packers
	_ "upspin.io/pack/debug"
	_ "upspin.io/pack/ee"
	_ "upspin.io/pack/plain"

	// Load required transports
	_ "upspin.io/dir/transports"
	_ "upspin.io/key/transports"
	_ "upspin.io/store/transports"
)

// Server is a SecureServer that talks to a DirServer interface and serves gRPC requests.
type Server struct {
	context upspin.Context

	// What this server reports itself as through its Endpoint method.
	endpoint upspin.Endpoint

	// The underlying dirserver implementation.
	dir upspin.DirServer

	// Automatically handles authentication by implementing the Authenticate server method.
	grpcauth.SecureServer
}

const serverName = "dirserver"

func main() {
	flags.Parse("addr", "config", "context", "https", "kind", "log", "project")

	if flags.Project != "" {
		log.Connect(flags.Project, serverName)
		svr, err := metric.NewGCPSaver(flags.Project, "serverName", serverName)
		if err != nil {
			log.Fatalf("Can't start a metric saver for GCP project %q: %s", flags.Project, err)
		} else {
			metric.RegisterSaver(svr)
		}
	}

	// Load context and keys for this server. It needs a real upspin username and keys.
	ctxfd, err := os.Open(flags.Context)
	if err != nil {
		log.Fatal(err)
	}
	defer ctxfd.Close()
	ctx, err := context.InitContext(ctxfd)
	if err != nil {
		log.Fatal(err)
	}

	// Create a new store implementation.
	var dir upspin.DirServer
	err = nil
	switch flags.ServerKind {
	case "inprocess":
		dir = inprocess.New(ctx)
	case "gcp":
		dir, err = gcp.New(ctx, flags.Config...)
	case "filesystem":
		dir, err = filesystem.New(ctx, flags.Config...)
	default:
		err = errors.Errorf("bad -kind %q", flags.ServerKind)
	}
	if err != nil {
		log.Fatalf("Setting up DirServer: %v", err)
	}

	config := auth.Config{Lookup: auth.PublicUserKeyService(ctx)}
	grpcSecureServer, err := grpcauth.NewSecureServer(config)
	if err != nil {
		log.Fatal(err)
	}
	s := &Server{
		context: ctx,
		endpoint: upspin.Endpoint{
			Transport: upspin.Remote,
			NetAddr:   upspin.NetAddr(flags.NetAddr),
		},
		dir:          dir,
		SecureServer: grpcSecureServer,
	}
	proto.RegisterDirServer(grpcSecureServer.GRPCServer(), s)

	http.Handle("/", grpcSecureServer.GRPCServer())
	https.ListenAndServe(serverName, flags.HTTPSAddr, nil)
}

var (
	// Empty structs we can allocate just once.
	putResponse       proto.DirPutResponse
	deleteResponse    proto.DirDeleteResponse
	configureResponse proto.ConfigureResponse
)

// dirFor returns a DirServer instance bound to the user specified in the context.
func (s *Server) dirFor(ctx gContext.Context) (upspin.DirServer, error) {
	// Validate that we have a session. If not, it's an auth error.
	session, err := s.GetSessionFromContext(ctx)
	if err != nil {
		return nil, err
	}
	svc, err := s.dir.Dial(s.context.Copy().SetUserName(session.User()), s.dir.Endpoint())
	if err != nil {
		return nil, err
	}
	return svc.(upspin.DirServer), nil
}

// TODO: We can probably simplify a lot of this by declaring one DirEntryError proto
// message and having all the functions return it, and then just writing one helper that
// takes entry, err and marshals it appropriately.

// Lookup implements upspin.DirServer.
func (s *Server) Lookup(ctx gContext.Context, req *proto.DirLookupRequest) (*proto.DirLookupResponse, error) {
	log.Printf("Lookup %q", req.Name)

	dir, err := s.dirFor(ctx)
	if err != nil {
		return nil, err
	}
	entry, lookupErr := dir.Lookup(upspin.PathName(req.Name))
	if lookupErr != nil && lookupErr != upspin.ErrFollowLink {
		log.Printf("Lookup %q failed: %v", req.Name, lookupErr)
		return &proto.DirLookupResponse{Error: errors.MarshalError(lookupErr)}, nil
	}

	// Fall through OK for ErrFollowLink.
	b, err := entry.Marshal()
	if err != nil {
		return nil, err
	}

	resp := &proto.DirLookupResponse{
		Entry: b,
		Error: errors.MarshalError(lookupErr),
	}
	return resp, nil
}

// Put implements upspin.DirServer.
func (s *Server) Put(ctx gContext.Context, req *proto.DirPutRequest) (*proto.DirPutResponse, error) {
	log.Printf("Put")

	entry, err := proto.UpspinDirEntry(req.Entry)
	if err != nil {
		log.Printf("Put %q failed: %v", entry.Name, err)
		return &proto.DirPutResponse{Error: errors.MarshalError(err)}, nil
	}
	log.Printf("Put %q", entry.Name)

	dir, err := s.dirFor(ctx)
	if err != nil {
		return nil, err
	}

	entry, putErr := dir.Put(entry)
	if putErr != nil && putErr != upspin.ErrFollowLink {
		log.Printf("Put %q failed: %v", entry.Name, putErr)
		return &proto.DirPutResponse{Error: errors.MarshalError(putErr)}, nil
	}

	if putErr == upspin.ErrFollowLink {
		b, err := entry.Marshal()
		if err != nil {
			return nil, err
		}
		resp := &proto.DirPutResponse{
			Entry: b,
			Error: errors.MarshalError(putErr),
		}
		return resp, nil
	}
	return &putResponse, nil
}

// MakeDirectory implements upspin.DirServer.
func (s *Server) MakeDirectory(ctx gContext.Context, req *proto.DirMakeDirectoryRequest) (*proto.DirMakeDirectoryResponse, error) {
	log.Printf("MakeDirectory %q", req.Name)

	dir, err := s.dirFor(ctx)
	if err != nil {
		return nil, err
	}
	entry, mkDirErr := dir.MakeDirectory(upspin.PathName(req.Name))
	if mkDirErr != nil && mkDirErr != upspin.ErrFollowLink {
		log.Printf("MakeDirectory %q failed: %v", req.Name, mkDirErr)
		return &proto.DirMakeDirectoryResponse{Error: errors.MarshalError(mkDirErr)}, nil
	}

	// Fall through OK for ErrFollowLink.
	b, err := entry.Marshal()
	if err != nil {
		return nil, err
	}
	resp := &proto.DirMakeDirectoryResponse{
		Entry: b,
		Error: errors.MarshalError(mkDirErr),
	}
	return resp, nil
}

// Glob implements upspin.DirServer.
func (s *Server) Glob(ctx gContext.Context, req *proto.DirGlobRequest) (*proto.DirGlobResponse, error) {
	log.Printf("Glob %q", req.Pattern)

	dir, err := s.dirFor(ctx)
	if err != nil {
		return nil, err
	}
	entries, globErr := dir.Glob(req.Pattern)
	if globErr != nil && globErr != upspin.ErrFollowLink {
		log.Printf("Glob %q failed: %v", req.Pattern, globErr)
		return &proto.DirGlobResponse{Error: errors.MarshalError(globErr)}, nil
	}

	// Fall through OK for ErrFollowLink.
	b, err := proto.DirEntryBytes(entries)
	if err != nil {
		return nil, err
	}
	resp := &proto.DirGlobResponse{
		Entries: b,
		Error:   errors.MarshalError(globErr),
	}
	return resp, err
}

// Delete implements upspin.DirServer.
func (s *Server) Delete(ctx gContext.Context, req *proto.DirDeleteRequest) (*proto.DirDeleteResponse, error) {
	log.Printf("Delete %q", req.Name)

	dir, err := s.dirFor(ctx)
	if err != nil {
		return nil, err
	}
	entry, deleteErr := dir.Delete(upspin.PathName(req.Name))
	if deleteErr != nil && deleteErr != upspin.ErrFollowLink {
		log.Printf("Delete %q failed: %v", req.Name, deleteErr)
		return &proto.DirDeleteResponse{Error: errors.MarshalError(deleteErr)}, nil
	}
	if err == upspin.ErrFollowLink {
		b, err := entry.Marshal()
		if err != nil {
			return nil, err
		}
		resp := &proto.DirDeleteResponse{
			Entry: b,
			Error: errors.MarshalError(deleteErr),
		}
		return resp, nil
	}
	return &deleteResponse, nil
}

// WhichAccess implements upspin.DirServer.
func (s *Server) WhichAccess(ctx gContext.Context, req *proto.DirWhichAccessRequest) (*proto.DirWhichAccessResponse, error) {
	log.Printf("WhichAccess %q", req.Name)

	dir, err := s.dirFor(ctx)
	if err != nil {
		return nil, err
	}
	entry, whichAccessErr := dir.WhichAccess(upspin.PathName(req.Name))
	if whichAccessErr != nil && whichAccessErr != upspin.ErrFollowLink {
		log.Printf("WhichAccess %q failed: %v", req.Name, whichAccessErr)
		return &proto.DirWhichAccessResponse{Error: errors.MarshalError(whichAccessErr)}, nil
	}

	// Entry might be nil, representing no Access file present.
	var b []byte
	if entry != nil {
		b, err = entry.Marshal()
		if err != nil {
			return nil, err
		}
	}
	resp := &proto.DirWhichAccessResponse{
		Entry: b,
		Error: errors.MarshalError(whichAccessErr),
	}
	return resp, nil
}

// Configure implements upspin.Service
func (s *Server) Configure(ctx gContext.Context, req *proto.ConfigureRequest) (*proto.ConfigureResponse, error) {
	log.Printf("Configure %q", req.Options)
	dir, err := s.dirFor(ctx)
	if err != nil {
		return nil, err
	}
	err = dir.Configure(req.Options...)
	if err != nil {
		log.Printf("Configure %q failed: %v", req.Options, err)
	}
	return &configureResponse, err
}

// Endpoint implements upspin.Service
func (s *Server) Endpoint(ctx gContext.Context, req *proto.EndpointRequest) (*proto.EndpointResponse, error) {
	log.Print("Endpoint")
	dir, err := s.dirFor(ctx)
	if err != nil {
		return nil, err
	}
	endpoint := dir.Endpoint()
	resp := &proto.EndpointResponse{
		Endpoint: &proto.Endpoint{
			Transport: int32(endpoint.Transport),
			NetAddr:   string(endpoint.NetAddr),
		},
	}
	return resp, nil
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/spf13/cobra"
	"github.com/weaveworks/flux"
	"github.com/weaveworks/flux/policy"
	"github.com/weaveworks/flux/update"
)

type servicePolicyOpts struct {
	*rootOpts
	outputOpts
	service   string
	container string
	tag       string

	automate, deautomate bool
	lock, unlock         bool

	cause update.Cause
}

func newServicePolicy(parent *rootOpts) *servicePolicyOpts {
	return &servicePolicyOpts{rootOpts: parent}
}

func (opts *servicePolicyOpts) Command() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "policy",
		Short: "Manage policies for a service.",
		Example: makeExample(
			"fluxctl policy --service=foo --automate",
			"fluxctl policy --service=foo --lock",
			"fluxctl policy --service=foo --container=bar --tag=1.2.*",
		),
		RunE: opts.RunE,
	}

	AddOutputFlags(cmd, &opts.outputOpts)
	AddCauseFlags(cmd, &opts.cause)
	flags := cmd.Flags()
	flags.StringVarP(&opts.service, "service", "s", "", "Service to modify")
	flags.StringVar(&opts.container, "container", "", "Container to set tag filter")
	flags.StringVar(&opts.tag, "tag", "", "Tag filter pattern")
	flags.BoolVar(&opts.automate, "automate", false, "Automate service")
	flags.BoolVar(&opts.deautomate, "deautomate", false, "Deautomate for service")
	flags.BoolVar(&opts.lock, "lock", false, "Lock service")
	flags.BoolVar(&opts.unlock, "unlock", false, "Unlock service")

	return cmd
}

func (opts *servicePolicyOpts) RunE(cmd *cobra.Command, args []string) error {
	if len(args) > 0 {
		return errorWantedNoArgs
	}
	if opts.service == "" {
		return newUsageError("-s, --service is required")
	}
	if opts.automate && opts.deautomate {
		return newUsageError("automate and deautomate both specified")
	}
	if opts.lock && opts.unlock {
		return newUsageError("lock and unlock both specified")
	}
	if opts.container != "" && opts.tag == "" {
		return newUsageError("container specified without a tag pattern")
	}
	if opts.tag != "" && opts.container == "" {
		return newUsageError("tag pattern specified without a container")
	}

	serviceID, err := flux.ParseServiceID(opts.service)
	if err != nil {
		return err
	}

	update := calculatePolicyChanges(opts)
	jobID, err := opts.API.UpdatePolicies(noInstanceID, policy.Updates{
		serviceID: update,
	}, opts.cause)
	if err != nil {
		return err
	}
	return await(cmd.OutOrStdout(), cmd.OutOrStderr(), opts.API, jobID, false, opts.verbose)
}

func calculatePolicyChanges(opts *servicePolicyOpts) policy.Update {
	add := policy.Set{}
	if opts.automate {
		add = add.Add(policy.Automated)
	}
	if opts.lock {
		add = add.Add(policy.Locked)
	}
	if opts.tag != "" && opts.tag != "*" {
		add = add.Set(policy.TagPrefix(opts.container), "glob:"+opts.tag)
	}

	remove := policy.Set{}
	if opts.deautomate {
		remove = remove.Add(policy.Automated)
	}
	if opts.unlock {
		remove = remove.Add(policy.Locked)
	}
	if opts.tag == "*" {
		remove = remove.Add(policy.TagPrefix(opts.container))
	}

	update := policy.Update{}
	if len(add) > 0 {
		update.Add = add
	}
	if len(remove) > 0 {
		update.Remove = remove
	}

	return update
}
<commit_msg>Support arbitrarily many pairs of containers and tags<commit_after>package main

import (
	"github.com/spf13/cobra"
	"github.com/weaveworks/flux"
	"github.com/weaveworks/flux/policy"
	"github.com/weaveworks/flux/update"
)

type servicePolicyOpts struct {
	*rootOpts
	outputOpts

	service    string
	containers []string
	tags       []string

	automate, deautomate bool
	lock, unlock         bool

	cause update.Cause
}

func newServicePolicy(parent *rootOpts) *servicePolicyOpts {
	return &servicePolicyOpts{rootOpts: parent}
}

func (opts *servicePolicyOpts) Command() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "policy",
		Short: "Manage policies for a service.",
		Example: makeExample(
			"fluxctl policy --service=foo --automate",
			"fluxctl policy --service=foo --lock",
			"fluxctl policy --service=foo --container=bar --tag=1.2.*",
		),
		RunE: opts.RunE,
	}

	AddOutputFlags(cmd, &opts.outputOpts)
	AddCauseFlags(cmd, &opts.cause)
	flags := cmd.Flags()
	flags.StringVarP(&opts.service, "service", "s", "", "Service to modify")
	flags.StringSliceVar(&opts.containers, "container", nil, "Containers to set tag filter")
	flags.StringSliceVar(&opts.tags, "tag", nil, "Tag filter patterns")
	flags.BoolVar(&opts.automate, "automate", false, "Automate service")
	flags.BoolVar(&opts.deautomate, "deautomate", false, "Deautomate for service")
	flags.BoolVar(&opts.lock, "lock", false, "Lock service")
	flags.BoolVar(&opts.unlock, "unlock", false, "Unlock service")

	return cmd
}

func (opts *servicePolicyOpts) RunE(cmd *cobra.Command, args []string) error {
	if len(args) > 0 {
		return errorWantedNoArgs
	}
	if opts.service == "" {
		return newUsageError("-s, --service is required")
	}
	if opts.automate && opts.deautomate {
		return newUsageError("automate and deautomate both specified")
	}
	if opts.lock && opts.unlock {
		return newUsageError("lock and unlock both specified")
	}
	if len(opts.containers) != len(opts.tags) {
		return newUsageError("mismatched container name and tag filter count")
	}

	serviceID, err := flux.ParseServiceID(opts.service)
	if err != nil {
		return err
	}

	update := calculatePolicyChanges(opts)
	jobID, err := opts.API.UpdatePolicies(noInstanceID, policy.Updates{
		serviceID: update,
	}, opts.cause)
	if err != nil {
		return err
	}
	return await(cmd.OutOrStdout(), cmd.OutOrStderr(), opts.API, jobID, false, opts.verbose)
}

func calculatePolicyChanges(opts *servicePolicyOpts) policy.Update {
	add := policy.Set{}
	if opts.automate {
		add = add.Add(policy.Automated)
	}
	if opts.lock {
		add = add.Add(policy.Locked)
	}

	remove := policy.Set{}
	if opts.deautomate {
		remove = remove.Add(policy.Automated)
	}
	if opts.unlock {
		remove = remove.Add(policy.Locked)
	}

	for i, container := range opts.containers {
		tag := opts.tags[i]
		if tag != "" && tag != "*" {
			add = add.Set(policy.TagPrefix(container), "glob:"+tag)
		} else if tag == "*" {
			remove = remove.Add(policy.TagPrefix(container))
		}
	}

	update := policy.Update{}
	if len(add) > 0 {
		update.Add = add
	}
	if len(remove) > 0 {
		update.Remove = remove
	}

	return update
}
<|endoftext|>