<commit_before>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"bytes"
	"fmt"
	"github.com/globocom/config"
	"github.com/globocom/tsuru/db"
	"github.com/globocom/tsuru/exec"
	"github.com/globocom/tsuru/log"
	"github.com/globocom/tsuru/provision"
	"github.com/globocom/tsuru/router"
	_ "github.com/globocom/tsuru/router/nginx"
	_ "github.com/globocom/tsuru/router/testing"
	"io"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

func init() {
	provision.Register("docker", &DockerProvisioner{})
}

var execut exec.Executor

func executor() exec.Executor {
	if execut == nil {
		execut = exec.OsExecutor{}
	}
	return execut
}

func (p *DockerProvisioner) router() (router.Router, error) {
	r, err := config.GetString("router")
	if err != nil {
		return nil, err
	}
	return router.Get(r)
}

type DockerProvisioner struct{}

func (p *DockerProvisioner) setup(ip, framework string) error {
	formulasPath, err := config.GetString("docker:formulas-path")
	if err != nil {
		return err
	}
	log.Printf("Creating hooks dir for %s", ip)
	args := []string{"-q", "-o", "StrictHostKeyChecking no", "-l", "ubuntu", ip, "sudo mkdir -p /var/lib/tsuru/hooks"}
	err = executor().Execute("ssh", args, nil, nil, nil)
	if err != nil {
		log.Printf("error on creating hooks dir for %s", ip)
		log.Print(err)
		return err
	}
	log.Printf("Permissons on hooks dir for %s", ip)
	args = []string{"-q", "-o", "StrictHostKeyChecking no", "-l", "ubuntu", ip, "sudo chown -R ubuntu /var/lib/tsuru/hooks"}
	err = executor().Execute("ssh", args, nil, nil, nil)
	if err != nil {
		log.Printf("error on permissions for %s", ip)
		log.Print(err)
		return err
	}
	log.Printf("coping hooks to %s", ip)
	output := bytes.Buffer{}
	args = []string{"-q", "-o", "StrictHostKeyChecking no", "-r", formulasPath + "/" + framework + "/hooks", "ubuntu@" + ip + ":/var/lib/tsuru"}
	err = executor().Execute("scp", args, nil, &output, &output)
	if err != nil {
		log.Printf("error on execute scp with the args: %#v", args)
		log.Print(output.String())
		log.Print(err)
		return err
	}
	return nil
}

func (p *DockerProvisioner) install(ip string) error {
	log.Printf("executing the install hook for %s", ip)
	args := []string{"-q", "-o", "StrictHostKeyChecking no", "-l", "ubuntu", ip, "sudo /var/lib/tsuru/hooks/install"}
	err := executor().Execute("ssh", args, nil, nil, nil)
	if err != nil {
		log.Printf("error on install for %s", ip)
		log.Print(err)
		return err
	}
	return nil
}

func (p *DockerProvisioner) start(ip string) error {
	args := []string{"-q", "-o", "StrictHostKeyChecking no", "-l", "ubuntu", ip, "sudo /var/lib/tsuru/hooks/start"}
	err := executor().Execute("ssh", args, nil, nil, nil)
	if err != nil {
		log.Printf("error on start for %s", ip)
		log.Print(err)
		return err
	}
	return nil
}

// Provision creates a container and install its dependencies
//
// TODO (flaviamissi): make this atomic
func (p *DockerProvisioner) Provision(app provision.App) error {
	go func(p *DockerProvisioner, app provision.App) {
		c := container{name: app.GetName()}
		log.Printf("creating container %s", c.name)
		u := provision.Unit{
			Name:       app.GetName(),
			AppName:    app.GetName(),
			Type:       app.GetFramework(),
			Machine:    0,
			InstanceId: app.GetName(),
			Status:     provision.StatusCreating,
			Ip:         "",
		}
		log.Printf("inserting container unit %s in the database", app.GetName())
		if err := p.collection().Insert(u); err != nil {
			log.Print(err)
			return
		}
		id, err := c.create()
		if err != nil {
			log.Printf("error on create container %s", app.GetName())
			log.Print(err)
			return
		}
		c.id = id
		u.InstanceId = id
		if err := c.start(); err != nil {
			log.Printf("error on start container %s", app.GetName())
			log.Print(err)
			return
		}
		ip, err := c.ip() // handle this error
		u.Ip = ip
		u.Status = provision.StatusInstalling
		if err := p.collection().Update(bson.M{"name": u.Name}, u); err != nil {
			log.Print(err)
			return
		}
		if err := p.setup(ip, app.GetFramework()); err != nil {
			log.Printf("error on setup container %s", app.GetName())
			log.Print(err)
			return
		}
		if err := p.install(ip); err != nil {
			log.Printf("error on install container %s", app.GetName())
			log.Print(err)
			return
		}
		log.Printf("running provisioning start() for container %s", c.id)
		if err := p.start(ip); err != nil {
			log.Printf("error on start app for container %s", app.GetName())
			log.Print(err)
			return
		}
		r, err := p.router()
		if err != nil {
			log.Print(err)
			return
		}
		err = r.AddRoute(app.GetName(), ip)
		if err != nil {
			log.Printf("error on add route for %s with ip %s", app.GetName(), ip)
			log.Print(err)
		}
		u.Status = provision.StatusStarted
		if err := p.collection().Update(bson.M{"name": u.Name}, u); err != nil {
			log.Print(err)
			return
		}
		log.Printf("Successfuly updated unit: %s", app.GetName())
	}(p, app)
	return nil
}

func (p *DockerProvisioner) Restart(app provision.App) error {
	var buf bytes.Buffer
	err := p.ExecuteCommand(&buf, &buf, app, "/var/lib/tsuru/hooks/restart")
	if err != nil {
		msg := fmt.Sprintf("Failed to restart the app (%s): %s", err, buf.String())
		app.Log(msg, "tsuru-provisioner")
		return &provision.Error{Reason: buf.String(), Err: err}
	}
	return nil
}

func (p *DockerProvisioner) Deploy(app provision.App, w io.Writer) error {
	return nil
}

func (p *DockerProvisioner) Destroy(app provision.App) error {
	units := app.ProvisionUnits()
	for _, u := range units {
		go func(u provision.AppUnit) {
			c := container{
				name: app.GetName(),
				// TODO: get actual c.id
				id: u.GetInstanceId(),
			}
			log.Printf("stoping container %s", u.GetInstanceId())
			if err := c.stop(); err != nil {
				log.Print("Could not stop container. Trying to remove it anyway.")
				log.Print(err.Error())
			}

			log.Printf("removeing container %s", u.GetInstanceId())
			if err := c.remove(); err != nil {
				log.Print("Could not remove container. Aborting...")
				log.Print(err.Error())
				return
			}

			log.Printf("removing container %s from the database", u.GetName())
			if err := p.collection().Remove(bson.M{"name": u.GetName()}); err != nil {
				log.Printf("Could not remove container from database. Error %s", err.Error())
			}
			log.Print("Units successfuly removed.")
		}(u)
	}
	return nil
}

func (*DockerProvisioner) Addr(app provision.App) (string, error) {
	units := app.ProvisionUnits()
	return units[0].GetIp(), nil
}

func (*DockerProvisioner) AddUnits(app provision.App, units uint) ([]provision.Unit, error) {
	return []provision.Unit{}, nil
}

func (*DockerProvisioner) RemoveUnit(app provision.App, unitName string) error {
	return nil
}

func (*DockerProvisioner) InstallDeps(app provision.App, w io.Writer) error {
	return nil
}

func (*DockerProvisioner) ExecuteCommand(stdout, stderr io.Writer, app provision.App, cmd string, args ...string) error {
	arguments := []string{"-l", "ubuntu", "-q", "-o", "StrictHostKeyChecking no"}
	arguments = append(arguments, app.ProvisionUnits()[0].GetIp())
	arguments = append(arguments, cmd)
	arguments = append(arguments, args...)
	err := executor().Execute("ssh", arguments, nil, stdout, stderr)
	if err != nil {
		return err
	}
	return nil
}

func (p *DockerProvisioner) CollectStatus() ([]provision.Unit, error) {
	var units []provision.Unit
	err := p.collection().Find(nil).All(&units)
	if err != nil {
		return []provision.Unit{}, err
	}
	return units, nil
}

func (p *DockerProvisioner) collection() *mgo.Collection {
	name, err := config.GetString("docker:collection")
	if err != nil {
		log.Fatalf("FATAL: %s.", err)
	}
	conn, err := db.Conn()
	if err != nil {
		log.Printf("Failed to connect to the database: %s", err)
	}
	return conn.Collection(name)
}
<commit_msg>provision/docker/provisioner.go: fixed typo in log msg<commit_after>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"bytes"
	"fmt"
	"github.com/globocom/config"
	"github.com/globocom/tsuru/db"
	"github.com/globocom/tsuru/exec"
	"github.com/globocom/tsuru/log"
	"github.com/globocom/tsuru/provision"
	"github.com/globocom/tsuru/router"
	_ "github.com/globocom/tsuru/router/nginx"
	_ "github.com/globocom/tsuru/router/testing"
	"io"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

func init() {
	provision.Register("docker", &DockerProvisioner{})
}

var execut exec.Executor

func executor() exec.Executor {
	if execut == nil {
		execut = exec.OsExecutor{}
	}
	return execut
}

func (p *DockerProvisioner) router() (router.Router, error) {
	r, err := config.GetString("router")
	if err != nil {
		return nil, err
	}
	return router.Get(r)
}

type DockerProvisioner struct{}

func (p *DockerProvisioner) setup(ip, framework string) error {
	formulasPath, err := config.GetString("docker:formulas-path")
	if err != nil {
		return err
	}
	log.Printf("Creating hooks dir for %s", ip)
	args := []string{"-q", "-o", "StrictHostKeyChecking no", "-l", "ubuntu", ip, "sudo mkdir -p /var/lib/tsuru/hooks"}
	err = executor().Execute("ssh", args, nil, nil, nil)
	if err != nil {
		log.Printf("error on creating hooks dir for %s", ip)
		log.Print(err)
		return err
	}
	log.Printf("Permissons on hooks dir for %s", ip)
	args = []string{"-q", "-o", "StrictHostKeyChecking no", "-l", "ubuntu", ip, "sudo chown -R ubuntu /var/lib/tsuru/hooks"}
	err = executor().Execute("ssh", args, nil, nil, nil)
	if err != nil {
		log.Printf("error on permissions for %s", ip)
		log.Print(err)
		return err
	}
	log.Printf("coping hooks to %s", ip)
	output := bytes.Buffer{}
	args = []string{"-q", "-o", "StrictHostKeyChecking no", "-r", formulasPath + "/" + framework + "/hooks", "ubuntu@" + ip + ":/var/lib/tsuru"}
	err = executor().Execute("scp", args, nil, &output, &output)
	if err != nil {
		log.Printf("error on execute scp with the args: %#v", args)
		log.Print(output.String())
		log.Print(err)
		return err
	}
	return nil
}

func (p *DockerProvisioner) install(ip string) error {
	log.Printf("executing the install hook for %s", ip)
	args := []string{"-q", "-o", "StrictHostKeyChecking no", "-l", "ubuntu", ip, "sudo /var/lib/tsuru/hooks/install"}
	err := executor().Execute("ssh", args, nil, nil, nil)
	if err != nil {
		log.Printf("error on install for %s", ip)
		log.Print(err)
		return err
	}
	return nil
}

func (p *DockerProvisioner) start(ip string) error {
	args := []string{"-q", "-o", "StrictHostKeyChecking no", "-l", "ubuntu", ip, "sudo /var/lib/tsuru/hooks/start"}
	err := executor().Execute("ssh", args, nil, nil, nil)
	if err != nil {
		log.Printf("error on start for %s", ip)
		log.Print(err)
		return err
	}
	return nil
}

// Provision creates a container and install its dependencies
//
// TODO (flaviamissi): make this atomic
func (p *DockerProvisioner) Provision(app provision.App) error {
	go func(p *DockerProvisioner, app provision.App) {
		c := container{name: app.GetName()}
		log.Printf("creating container %s", c.name)
		u := provision.Unit{
			Name:       app.GetName(),
			AppName:    app.GetName(),
			Type:       app.GetFramework(),
			Machine:    0,
			InstanceId: app.GetName(),
			Status:     provision.StatusCreating,
			Ip:         "",
		}
		log.Printf("inserting container unit %s in the database", app.GetName())
		if err := p.collection().Insert(u); err != nil {
			log.Print(err)
			return
		}
		id, err := c.create()
		if err != nil {
			log.Printf("error on create container %s", app.GetName())
			log.Print(err)
			return
		}
		c.id = id
		u.InstanceId = id
		if err := c.start(); err != nil {
			log.Printf("error on start container %s", app.GetName())
			log.Print(err)
			return
		}
		ip, err := c.ip() // handle this error
		u.Ip = ip
		u.Status = provision.StatusInstalling
		if err := p.collection().Update(bson.M{"name": u.Name}, u); err != nil {
			log.Print(err)
			return
		}
		if err := p.setup(ip, app.GetFramework()); err != nil {
			log.Printf("error on setup container %s", app.GetName())
			log.Print(err)
			return
		}
		if err := p.install(ip); err != nil {
			log.Printf("error on install container %s", app.GetName())
			log.Print(err)
			return
		}
		log.Printf("running provisioning start() for container %s", c.id)
		if err := p.start(ip); err != nil {
			log.Printf("error on start app for container %s", app.GetName())
			log.Print(err)
			return
		}
		r, err := p.router()
		if err != nil {
			log.Print(err)
			return
		}
		err = r.AddRoute(app.GetName(), ip)
		if err != nil {
			log.Printf("error on add route for %s with ip %s", app.GetName(), ip)
			log.Print(err)
		}
		u.Status = provision.StatusStarted
		if err := p.collection().Update(bson.M{"name": u.Name}, u); err != nil {
			log.Print(err)
			return
		}
		log.Printf("Successfuly updated unit: %s", app.GetName())
	}(p, app)
	return nil
}

func (p *DockerProvisioner) Restart(app provision.App) error {
	var buf bytes.Buffer
	err := p.ExecuteCommand(&buf, &buf, app, "/var/lib/tsuru/hooks/restart")
	if err != nil {
		msg := fmt.Sprintf("Failed to restart the app (%s): %s", err, buf.String())
		app.Log(msg, "tsuru-provisioner")
		return &provision.Error{Reason: buf.String(), Err: err}
	}
	return nil
}

func (p *DockerProvisioner) Deploy(app provision.App, w io.Writer) error {
	return nil
}

func (p *DockerProvisioner) Destroy(app provision.App) error {
	units := app.ProvisionUnits()
	for _, u := range units {
		go func(u provision.AppUnit) {
			c := container{
				name: app.GetName(),
				// TODO: get actual c.id
				id: u.GetInstanceId(),
			}
			log.Printf("stoping container %s", u.GetInstanceId())
			if err := c.stop(); err != nil {
				log.Print("Could not stop container. Trying to remove it anyway.")
				log.Print(err.Error())
			}

			log.Printf("removing container %s", u.GetInstanceId())
			if err := c.remove(); err != nil {
				log.Print("Could not remove container. Aborting...")
				log.Print(err.Error())
				return
			}

			log.Printf("removing container %s from the database", u.GetName())
			if err := p.collection().Remove(bson.M{"name": u.GetName()}); err != nil {
				log.Printf("Could not remove container from database. Error %s", err.Error())
			}
			log.Print("Units successfuly removed.")
		}(u)
	}
	return nil
}

func (*DockerProvisioner) Addr(app provision.App) (string, error) {
	units := app.ProvisionUnits()
	return units[0].GetIp(), nil
}

func (*DockerProvisioner) AddUnits(app provision.App, units uint) ([]provision.Unit, error) {
	return []provision.Unit{}, nil
}

func (*DockerProvisioner) RemoveUnit(app provision.App, unitName string) error {
	return nil
}

func (*DockerProvisioner) InstallDeps(app provision.App, w io.Writer) error {
	return nil
}

func (*DockerProvisioner) ExecuteCommand(stdout, stderr io.Writer, app provision.App, cmd string, args ...string) error {
	arguments := []string{"-l", "ubuntu", "-q", "-o", "StrictHostKeyChecking no"}
	arguments = append(arguments, app.ProvisionUnits()[0].GetIp())
	arguments = append(arguments, cmd)
	arguments = append(arguments, args...)
	err := executor().Execute("ssh", arguments, nil, stdout, stderr)
	if err != nil {
		return err
	}
	return nil
}

func (p *DockerProvisioner) CollectStatus() ([]provision.Unit, error) {
	var units []provision.Unit
	err := p.collection().Find(nil).All(&units)
	if err != nil {
		return []provision.Unit{}, err
	}
	return units, nil
}

func (p *DockerProvisioner) collection() *mgo.Collection {
	name, err := config.GetString("docker:collection")
	if err != nil {
		log.Fatalf("FATAL: %s.", err)
	}
	conn, err := db.Conn()
	if err != nil {
		log.Printf("Failed to connect to the database: %s", err)
	}
	return conn.Collection(name)
}
<|endoftext|>


<commit_before>package main

import (
	"sync"
)

func main() {
	const (
		max       = 4
		noWorkers = 10
	)
	ch := make(chan int, 1)

	wg := startWorkers(noWorkers, ch)
	for i := 0; i < max; i++ {
		select {
		case ch <- i:
			println("send", i)
		default:
			println("*** block", i)
			ch <- i
		}
	}
	close(ch)
	println("sending done")
	wg.Wait()
	println("the end")
}

func startWorkers(max int, ch chan int) *sync.WaitGroup {
	initWG := &sync.WaitGroup{}
	initWG.Add(max)
	wg := &sync.WaitGroup{}
	wg.Add(max)
	for i := 0; i < max; i++ {
		go worker(initWG, wg, ch)
	}
	initWG.Wait()
	println("workers init")
	return wg
}

func worker(initWG *sync.WaitGroup, wg *sync.WaitGroup, ch chan int) {
	initWG.Done()
	for v := range ch {
		//time.Sleep(1e9)
		println("receive", v)
	}
	wg.Done()
	println("end worker")
}
<commit_msg>channels/workers: clean worker func<commit_after>package main

import (
	"sync"
)

func main() {
	const (
		max       = 4
		noWorkers = 10
	)
	ch := make(chan int, 1)

	wg := startWorkers(noWorkers, ch)
	for i := 0; i < max; i++ {
		select {
		case ch <- i:
			println("send", i)
		default:
			println("*** block", i)
			ch <- i
		}
	}
	close(ch)
	println("sending done")
	wg.Wait()
	println("the end")
}

func startWorkers(max int, ch chan int) *sync.WaitGroup {
	initWG := &sync.WaitGroup{}
	initWG.Add(max)
	wg := &sync.WaitGroup{}
	wg.Add(max)
	for i := 0; i < max; i++ {
		go worker(initWG, wg, ch)
	}
	initWG.Wait()
	println("workers init")
	return wg
}

func worker(initWG *sync.WaitGroup, wg *sync.WaitGroup, ch chan int) {
	initWG.Done()
	for v := range ch {
		println("receive", v)
	}
	wg.Done()
	println("end worker")
}
<|endoftext|>


<commit_before>package repository

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"restic/backend"
	"restic/crypto"
	"restic/debug"
	"restic/pack"
	"restic/worker"
)

// Repack takes a list of packs together with a list of blobs contained in
// these packs. Each pack is loaded and the blobs listed in keepBlobs is saved
// into a new pack. Afterwards, the packs are removed. This operation requires
// an exclusive lock on the repo.
func Repack(repo *Repository, packs, keepBlobs backend.IDSet) (err error) {
	debug.Log("Repack", "repacking %d packs while keeping %d blobs", len(packs), len(keepBlobs))

	buf := make([]byte, 0, maxPackSize)
	for packID := range packs {
		// load the complete blob
		h := backend.Handle{Type: backend.Data, Name: packID.String()}

		l, err := repo.Backend().Load(h, buf[:cap(buf)], 0)
		if err == io.ErrUnexpectedEOF {
			err = nil
			buf = buf[:l]
		}

		if err != nil {
			return err
		}

		debug.Log("Repack", "pack %v loaded (%d bytes)", packID.Str(), len(buf))

		unpck, err := pack.NewUnpacker(repo.Key(), bytes.NewReader(buf))
		if err != nil {
			return err
		}

		debug.Log("Repack", "processing pack %v, blobs: %v", packID.Str(), len(unpck.Entries))
		var plaintext []byte
		for _, entry := range unpck.Entries {
			if !keepBlobs.Has(entry.ID) {
				continue
			}

			ciphertext := buf[entry.Offset : entry.Offset+entry.Length]

			if cap(plaintext) < len(ciphertext) {
				plaintext = make([]byte, len(ciphertext))
			}

			plaintext, err = crypto.Decrypt(repo.Key(), plaintext, ciphertext)
			if err != nil {
				return err
			}

			_, err = repo.SaveAndEncrypt(entry.Type, plaintext, &entry.ID)
			if err != nil {
				return err
			}

			debug.Log("Repack", "  saved blob %v", entry.ID.Str())

			keepBlobs.Delete(entry.ID)
		}
	}

	if err := repo.Flush(); err != nil {
		return err
	}

	for packID := range packs {
		err := repo.Backend().Remove(backend.Data, packID.String())
		if err != nil {
			debug.Log("Repack", "error removing pack %v: %v", packID.Str(), err)
			return err
		}
		debug.Log("Repack", "removed pack %v", packID.Str())
	}

	return nil
}

const rebuildIndexWorkers = 10

type loadBlobsResult struct {
	packID  backend.ID
	entries []pack.Blob
}

// loadBlobsFromAllPacks sends the contents of all packs to ch.
func loadBlobsFromAllPacks(repo *Repository, ch chan<- worker.Job, done <-chan struct{}) {
	f := func(job worker.Job, done <-chan struct{}) (interface{}, error) {
		packID := job.Data.(backend.ID)
		entries, err := repo.ListPack(packID)
		return loadBlobsResult{
			packID:  packID,
			entries: entries,
		}, err
	}

	jobCh := make(chan worker.Job)
	wp := worker.New(rebuildIndexWorkers, f, jobCh, ch)

	go func() {
		for id := range repo.List(backend.Data, done) {
			jobCh <- worker.Job{Data: id}
		}
		close(jobCh)
	}()

	wp.Wait()
}

// RebuildIndex lists all packs in the repo, writes a new index and removes all
// old indexes. This operation should only be done with an exclusive lock in
// place.
func RebuildIndex(repo *Repository) error {
	debug.Log("RebuildIndex", "start rebuilding index")

	done := make(chan struct{})
	defer close(done)

	ch := make(chan worker.Job)
	go loadBlobsFromAllPacks(repo, ch, done)

	idx := NewIndex()
	for job := range ch {
		id := job.Data.(backend.ID)

		if job.Error != nil {
			fmt.Fprintf(os.Stderr, "error for pack %v: %v\n", id, job.Error)
			continue
		}

		res := job.Result.(loadBlobsResult)

		for _, entry := range res.entries {
			pb := PackedBlob{
				ID:     entry.ID,
				Type:   entry.Type,
				Length: entry.Length,
				Offset: entry.Offset,
				PackID: res.packID,
			}
			idx.Store(pb)
		}
	}

	oldIndexes := backend.NewIDSet()
	for id := range repo.List(backend.Index, done) {
		idx.AddToSupersedes(id)
		oldIndexes.Insert(id)
	}

	id, err := SaveIndex(repo, idx)
	if err != nil {
		debug.Log("RebuildIndex.RebuildIndex", "error saving index: %v", err)
		return err
	}
	debug.Log("RebuildIndex.RebuildIndex", "new index saved as %v", id.Str())

	for indexID := range oldIndexes {
		err := repo.Backend().Remove(backend.Index, indexID.String())
		if err != nil {
			fmt.Fprintf(os.Stderr, "unable to remove index %v: %v\n", indexID.Str(), err)
		}
	}

	return nil
}
<commit_msg>Correct comment<commit_after>package repository

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"restic/backend"
	"restic/crypto"
	"restic/debug"
	"restic/pack"
	"restic/worker"
)

// Repack takes a list of packs together with a list of blobs contained in
// these packs. Each pack is loaded and the blobs listed in keepBlobs is saved
// into a new pack. Afterwards, the packs are removed. This operation requires
// an exclusive lock on the repo.
func Repack(repo *Repository, packs, keepBlobs backend.IDSet) (err error) {
	debug.Log("Repack", "repacking %d packs while keeping %d blobs", len(packs), len(keepBlobs))

	buf := make([]byte, 0, maxPackSize)
	for packID := range packs {
		// load the complete pack
		h := backend.Handle{Type: backend.Data, Name: packID.String()}

		l, err := repo.Backend().Load(h, buf[:cap(buf)], 0)
		if err == io.ErrUnexpectedEOF {
			err = nil
			buf = buf[:l]
		}

		if err != nil {
			return err
		}

		debug.Log("Repack", "pack %v loaded (%d bytes)", packID.Str(), len(buf))

		unpck, err := pack.NewUnpacker(repo.Key(), bytes.NewReader(buf))
		if err != nil {
			return err
		}

		debug.Log("Repack", "processing pack %v, blobs: %v", packID.Str(), len(unpck.Entries))
		var plaintext []byte
		for _, entry := range unpck.Entries {
			if !keepBlobs.Has(entry.ID) {
				continue
			}

			ciphertext := buf[entry.Offset : entry.Offset+entry.Length]

			if cap(plaintext) < len(ciphertext) {
				plaintext = make([]byte, len(ciphertext))
			}

			plaintext, err = crypto.Decrypt(repo.Key(), plaintext, ciphertext)
			if err != nil {
				return err
			}

			_, err = repo.SaveAndEncrypt(entry.Type, plaintext, &entry.ID)
			if err != nil {
				return err
			}

			debug.Log("Repack", "  saved blob %v", entry.ID.Str())

			keepBlobs.Delete(entry.ID)
		}
	}

	if err := repo.Flush(); err != nil {
		return err
	}

	for packID := range packs {
		err := repo.Backend().Remove(backend.Data, packID.String())
		if err != nil {
			debug.Log("Repack", "error removing pack %v: %v", packID.Str(), err)
			return err
		}
		debug.Log("Repack", "removed pack %v", packID.Str())
	}

	return nil
}

const rebuildIndexWorkers = 10

type loadBlobsResult struct {
	packID  backend.ID
	entries []pack.Blob
}

// loadBlobsFromAllPacks sends the contents of all packs to ch.
func loadBlobsFromAllPacks(repo *Repository, ch chan<- worker.Job, done <-chan struct{}) {
	f := func(job worker.Job, done <-chan struct{}) (interface{}, error) {
		packID := job.Data.(backend.ID)
		entries, err := repo.ListPack(packID)
		return loadBlobsResult{
			packID:  packID,
			entries: entries,
		}, err
	}

	jobCh := make(chan worker.Job)
	wp := worker.New(rebuildIndexWorkers, f, jobCh, ch)

	go func() {
		for id := range repo.List(backend.Data, done) {
			jobCh <- worker.Job{Data: id}
		}
		close(jobCh)
	}()

	wp.Wait()
}

// RebuildIndex lists all packs in the repo, writes a new index and removes all
// old indexes. This operation should only be done with an exclusive lock in
// place.
func RebuildIndex(repo *Repository) error {
	debug.Log("RebuildIndex", "start rebuilding index")

	done := make(chan struct{})
	defer close(done)

	ch := make(chan worker.Job)
	go loadBlobsFromAllPacks(repo, ch, done)

	idx := NewIndex()
	for job := range ch {
		id := job.Data.(backend.ID)

		if job.Error != nil {
			fmt.Fprintf(os.Stderr, "error for pack %v: %v\n", id, job.Error)
			continue
		}

		res := job.Result.(loadBlobsResult)

		for _, entry := range res.entries {
			pb := PackedBlob{
				ID:     entry.ID,
				Type:   entry.Type,
				Length: entry.Length,
				Offset: entry.Offset,
				PackID: res.packID,
			}
			idx.Store(pb)
		}
	}

	oldIndexes := backend.NewIDSet()
	for id := range repo.List(backend.Index, done) {
		idx.AddToSupersedes(id)
		oldIndexes.Insert(id)
	}

	id, err := SaveIndex(repo, idx)
	if err != nil {
		debug.Log("RebuildIndex.RebuildIndex", "error saving index: %v", err)
		return err
	}
	debug.Log("RebuildIndex.RebuildIndex", "new index saved as %v", id.Str())

	for indexID := range oldIndexes {
		err := repo.Backend().Remove(backend.Index, indexID.String())
		if err != nil {
			fmt.Fprintf(os.Stderr, "unable to remove index %v: %v\n", indexID.Str(), err)
		}
	}

	return nil
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package helper

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net/http"
	"os"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/ernestio/ernest-cli/model"
	"github.com/fatih/color"
	"github.com/gosuri/uilive"
	"github.com/hokaccha/go-prettyjson"
	"github.com/mitchellh/mapstructure"
	"github.com/r3labs/sse"
)

type print func([]byte)

var green = color.New(color.FgGreen).SprintFunc()
var yellow = color.New(color.FgYellow).SprintFunc()
var red = color.New(color.FgRed).SprintFunc()

// Monitorize opens a websocket connection to get input messages
func Monitorize(host, endpoint, token, stream string, resc chan string) {
	var s model.ServiceEvent
	var c model.ComponentEvent
	var format string
	var args []interface{}
	var resourceErr error

	writer := uilive.New()
	writer.Start()

	sseSubscribe(host, endpoint, token, stream, func(event []byte) {
		// clean msg body of any null characters
		cleanedInput := bytes.Trim(event, "\x00")

		msg := make(map[string]interface{})

		err := json.Unmarshal(cleanedInput, &msg)
		if err != nil {
			fmt.Println(err)
		}

		subject := msg["_subject"].(string)

		switch subject {
		case "service.create", "service.delete", "service.import":
			s = processServiceEvent(msg)
			format, args = renderOutput(s)
		case "service.create.done", "service.create.error", "service.delete.done", "service.delete.error", "service.import.done", "service.import.error":
			s = processServiceEvent(msg)
			err = renderUpdate(s, model.ComponentEvent{}, args)
			if err != nil {
				log.Println(err)
			}
		default:
			c = processComponentEvent(msg)
			err = renderUpdate(model.ServiceEvent{}, c, args)
			if err != nil {
				resourceErr = err
			}
		}

		time.Sleep(time.Millisecond * 100)
		fmt.Fprintf(writer, format, args...)

		switch subject {
		case "service.create.done":
			writer.Stop()
			resc <- s.Name
		case "service.delete.done", "service.import.done":
			writer.Stop()
			os.Exit(0)
		case "service.create.error", "service.delete.error", "service.import.error":
			writer.Stop()
			fmt.Printf("Message: %s\n\n", red(resourceErr))
			os.Exit(1)
		}
	})
}

func renderUpdate(s model.ServiceEvent, c model.ComponentEvent, a []interface{}) error {
	// component status
	for i, v := range a {
		t := formatType(c.Type)
		if v == t || v == t+"s" {
			if c.Action == "create" && c.State == "running" {
				a[i+3] = yellow("Creating")
			} else if c.Action == "create" && c.State == "completed" {
				a[i+1] = a[i+1].(int) + 1
				if a[i+1] == a[i+2] {
					a[i+3] = green("Created")
				}
			}
			if c.Action == "find" && c.State == "running" {
				a[i+3] = yellow("Searching")
			} else if c.Action == "find" && c.State == "completed" {
				if len(c.Components) > 0 {
					a[i+1] = len(c.Components)
					a[i+2] = len(c.Components)
					a[i+3] = green("Found")
				} else {
					a[i+3] = yellow("None")
				}
			}
			if c.Action == "delete" && c.State == "running" {
				a[i+3] = yellow("Deleting")
			} else if c.Action == "delete" && c.State == "completed" {
				a[i+1] = a[i+1].(int) + 1
				if a[i+1] == a[i+2] {
					a[i+3] = green("Deleted")
				}
			}
			if c.State == "errored" {
				a[i+3] = red("Error")
				return errors.New(c.Error)
			}
		}
	}

	// overall status
	switch s.Subject {
	case "service.delete.done":
		a[len(a)-1] = green("Destroyed")
	case "service.create.done":
		a[len(a)-1] = green("Applied")
	case "service.import.done":
		a[len(a)-1] = green("Imported")
	case "service.create.error", "service.delete.error", "service.import.error":
		a[len(a)-1] = red("Error")
	}

	return nil
}

func renderOutput(s model.ServiceEvent) (string, []interface{}) {
	var blue = color.New(color.FgBlue).SprintFunc()

	f := "\nService Name: %s\n"
	a := []interface{}{blue(s.Name)}
	f = f + "Service ID: %s\n\n"
	a = append(a, blue(s.ID))

	if len(s.Changes) == 0 {
		f = f + green("No changes detected\n")
	} else {
		changes := ParseChanges(s.Changes)

		keys := []string{}
		for key := range changes {
			keys = append(keys, key)
		}
		sort.Strings(keys)

		longestKey := 0
		for _, k := range keys {
			if len(k) > longestKey {
				longestKey = len(k)
			}
		}

		for _, k := range keys {
			f = f + "%-" + strconv.Itoa(longestKey+1) + "s %3d/%-3d %s\n"
			t := formatType(k)
			if s.Subject == "service.import" {
				a = append(a, t, 0, 0, "")
			} else {
				a = append(a, t+"s", 0, changes[k], "")
			}
		}
	}

	f = f + "\nStatus: %s\n\n"
	var status string
	switch s.Subject {
	case "service.create":
		status = yellow("Applying")
	case "service.delete":
		status = yellow("Destroying")
	case "service.import":
		status = yellow("Importing")
	default:
		status = yellow("Unknown")
	}
	a = append(a, status)

	return f, a
}

func ParseChanges(c []model.ComponentEvent) map[string]int {
	seen := map[string]int{}
	for _, v := range c {
		seen[v.Type] += 1
	}
	return seen
}

func formatType(t string) string {
	t = strings.Replace(t, "_", " ", -1)
	return strings.Title(t)
}

func processServiceEvent(s map[string]interface{}) model.ServiceEvent {
	m := model.ServiceEvent{}

	config := &mapstructure.DecoderConfig{
		Metadata: nil,
		Result:   &m,
		TagName:  "json",
	}

	decoder, err := mapstructure.NewDecoder(config)
	if err != nil {
		panic(err)
	}

	err = decoder.Decode(s)
	if err != nil {
		panic(err)
	}

	return m
}

func processComponentEvent(c map[string]interface{}) model.ComponentEvent {
	m := model.ComponentEvent{}
	config := &mapstructure.DecoderConfig{
		Metadata: nil,
		Result:   &m,
		TagName:  "json",
	}

	decoder, err := mapstructure.NewDecoder(config)
	if err != nil {
		panic(err)
	}

	err = decoder.Decode(c)
	if err != nil {
		panic(err)
	}

	return m
}

// PrintLogs : prints logs inline
func PrintLogs(host, endpoint, token, stream string) {
	sseSubscribe(host, endpoint, token, stream, func(body []byte) {
		m := model.Message{}

		// clean msg body of any null characters
		cleanedInput := bytes.Trim(body, "\x00")

		if err := json.Unmarshal(cleanedInput, &m); err != nil {
			fmt.Println(err)
		}

		color.Yellow(m.Subject)
		if len(m.Body) > 0 {
			message, _ := prettyjson.Format([]byte(m.Body))
			fmt.Println(string(message))
		} else {
			fmt.Println("-- Empty string --")
		}
	})
}

// PrintRawLogs : prints logs inline
func PrintRawLogs(host, endpoint, token, stream string) {
	sseSubscribe(host, endpoint, token, stream, func(body []byte) {
		m := model.Message{}

		// clean msg body of any null characters
		cleanedInput := bytes.Trim(body, "\x00")

		if err := json.Unmarshal(cleanedInput, &m); err != nil {
			fmt.Println(err)
		}

		fmt.Println("[" + m.Subject + "] : " + m.Body)
	})
}

func sseSubscribe(host, endpoint, token, stream string, fn print) {
	url := host + endpoint
	client := sse.NewClient(url)
	client.EncodingBase64 = true
	client.Connection.Transport = &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}

	client.Headers["Authorization"] = fmt.Sprintf("Bearer %s", token)

	err := client.Subscribe(stream, func(msg *sse.Event) {
		if msg.Data != nil {
			fn(msg.Data)
		}
	})
	if err != nil {
		log.Println("Failed with: " + err.Error())
		os.Exit(1)
	}
}
<commit_msg>Fix error capture<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package helper

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net/http"
	"os"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/ernestio/ernest-cli/model"
	"github.com/fatih/color"
	"github.com/gosuri/uilive"
	"github.com/hokaccha/go-prettyjson"
	"github.com/mitchellh/mapstructure"
	"github.com/r3labs/sse"
)

type print func([]byte)

var (
	green  = color.New(color.FgGreen).SprintFunc()
	yellow = color.New(color.FgYellow).SprintFunc()
	red    = color.New(color.FgRed).SprintFunc()
)

// Monitorize opens a websocket connection to get input messages
func Monitorize(host, endpoint, token, stream string, resc chan string) {
	var (
		s           model.ServiceEvent
		c           model.ComponentEvent
		format      string
		args        []interface{}
		resourceErr error
	)

	writer := uilive.New()
	writer.Start()

	sseSubscribe(host, endpoint, token, stream, func(event []byte) {
		// clean msg body of any null characters
		cleanedInput := bytes.Trim(event, "\x00")

		msg := make(map[string]interface{})

		err := json.Unmarshal(cleanedInput, &msg)
		if err != nil {
			fmt.Println(err)
		}

		subject := msg["_subject"].(string)

		switch subject {
		case "service.create", "service.delete", "service.import":
			s = processServiceEvent(msg)
			format, args = renderOutput(s)
		case "service.create.done", "service.create.error", "service.delete.done", "service.delete.error", "service.import.done", "service.import.error":
			s = processServiceEvent(msg)
			err = renderUpdate(s, model.ComponentEvent{}, args)
			if err != nil {
				fmt.Println(err)
			}
		default:
			c = processComponentEvent(msg)
			err = renderUpdate(model.ServiceEvent{}, c, args)
			if err != nil {
				resourceErr = err
			}
		}

		time.Sleep(time.Millisecond * 100)
		fmt.Fprintf(writer, format, args...)

		switch subject {
		case "service.create.done":
			writer.Stop()
			resc <- s.Name
		case "service.delete.done", "service.import.done":
			writer.Stop()
			os.Exit(0)
		case "service.create.error", "service.delete.error", "service.import.error":
			writer.Stop()
			fmt.Printf("Message: %s\n\n", red(resourceErr))
			os.Exit(1)
		}
	})
}

func renderUpdate(s model.ServiceEvent, c model.ComponentEvent, a []interface{}) error {
	// component status
	for i, v := range a {
		t := formatType(c.Type)
		if v == t || v == t+"s" {
			if c.Action == "create" && c.State == "running" {
				a[i+3] = yellow("Creating")
			} else if c.Action == "create" && c.State == "completed" {
				a[i+1] = a[i+1].(int) + 1
				if a[i+1] == a[i+2] {
					a[i+3] = green("Created")
				}
			}
			if c.Action == "find" && c.State == "running" {
				a[i+3] = yellow("Searching")
			} else if c.Action == "find" && c.State == "completed" {
				if len(c.Components) > 0 {
					a[i+1] = len(c.Components)
					a[i+2] = len(c.Components)
					a[i+3] = green("Found")
				} else {
					a[i+3] = yellow("None")
				}
			}
			if c.Action == "delete" && c.State == "running" {
				a[i+3] = yellow("Deleting")
			} else if c.Action == "delete" && c.State == "completed" {
				a[i+1] = a[i+1].(int) + 1
				if a[i+1] == a[i+2] {
					a[i+3] = green("Deleted")
				}
			}
			if c.State == "errored" {
				a[i+3] = red("Error")
				return errors.New(c.Error)
			}
		}
	}

	// overall status
	switch s.Subject {
	case "service.delete.done":
		a[len(a)-1] = green("Destroyed")
	case "service.create.done":
		a[len(a)-1] = green("Applied")
	case "service.import.done":
		a[len(a)-1] = green("Imported")
	case "service.create.error", "service.delete.error", "service.import.error":
		a[len(a)-1] = red("Error")
	}

	return nil
}

func renderOutput(s model.ServiceEvent) (string, []interface{}) {
	var blue = color.New(color.FgBlue).SprintFunc()

	f := "\nService Name: %s\n"
	a := []interface{}{blue(s.Name)}
	f = f + "Service ID: %s\n\n"
	a = append(a, blue(s.ID))

	if len(s.Changes) == 0 {
		f = f + green("No changes detected\n")
	} else {
		changes := ParseChanges(s.Changes)

		keys := []string{}
		for key := range changes {
			keys = append(keys, key)
		}
		sort.Strings(keys)

		longestKey := 0
		for _, k := range keys {
			if len(k) > longestKey {
				longestKey = len(k)
			}
		}

		for _, k := range keys {
			f = f + "%-" + strconv.Itoa(longestKey+1) + "s %3d/%-3d %s\n"
			t := formatType(k)
			if s.Subject == "service.import" {
				a = append(a, t, 0, 0, "")
			} else {
				a = append(a, t+"s", 0, changes[k], "")
			}
		}
	}

	f = f + "\nStatus: %s\n\n"
	var status string
	switch s.Subject {
	case "service.create":
		status = yellow("Applying")
	case "service.delete":
		status = yellow("Destroying")
	case "service.import":
		status = yellow("Importing")
	default:
		status = yellow("Unknown")
	}
	a = append(a, status)

	return f, a
}

func ParseChanges(c []model.ComponentEvent) map[string]int {
	seen := map[string]int{}
	for _, v := range c {
		seen[v.Type] += 1
	}
	return seen
}

func formatType(t string) string {
	t = strings.Replace(t, "_", " ", -1)
	return strings.Title(t)
}

func processServiceEvent(s map[string]interface{}) model.ServiceEvent {
	m := model.ServiceEvent{}

	config := &mapstructure.DecoderConfig{
		Metadata: nil,
		Result:   &m,
		TagName:  "json",
	}

	decoder, err := mapstructure.NewDecoder(config)
	if err != nil {
		panic(err)
	}

	err = decoder.Decode(s)
	if err != nil {
		panic(err)
	}

	return m
}

func processComponentEvent(c map[string]interface{}) model.ComponentEvent {
	m := model.ComponentEvent{}
	config := &mapstructure.DecoderConfig{
		Metadata: nil,
		Result:   &m,
		TagName:  "json",
	}

	decoder, err := mapstructure.NewDecoder(config)
	if err != nil {
		panic(err)
	}

	err = decoder.Decode(c)
	if err != nil {
		panic(err)
	}

	return m
}

// PrintLogs : prints logs inline
func PrintLogs(host, endpoint, token, stream string) {
	sseSubscribe(host, endpoint, token, stream, func(body []byte) {
		m := model.Message{}

		// clean msg body of any null characters
		cleanedInput := bytes.Trim(body, "\x00")

		if err := json.Unmarshal(cleanedInput, &m); err != nil {
			fmt.Println(err)
		}

		color.Yellow(m.Subject)
		if len(m.Body) > 0 {
			message, _ := prettyjson.Format([]byte(m.Body))
			fmt.Println(string(message))
		} else {
			fmt.Println("-- Empty string --")
		}
	})
}

// PrintRawLogs : prints logs inline
func PrintRawLogs(host, endpoint, token, stream string) {
	sseSubscribe(host, endpoint, token, stream, func(body []byte) {
		m := model.Message{}

		// clean msg body of any null characters
		cleanedInput := bytes.Trim(body, "\x00")

		if err := json.Unmarshal(cleanedInput, &m); err != nil {
			fmt.Println(err)
		}

		fmt.Println("[" + m.Subject + "] : " + m.Body)
	})
}

func sseSubscribe(host, endpoint, token, stream string, fn print) {
	url := host + endpoint
	client := sse.NewClient(url)
	client.EncodingBase64 = true
	client.Connection.Transport = &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}

	client.Headers["Authorization"] = fmt.Sprintf("Bearer %s", token)

	err := client.Subscribe(stream, func(msg *sse.Event) {
		if msg.Data != nil {
			fn(msg.Data)
		}
	})
	if err != nil {
		log.Println("Failed with: " + err.Error())
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package controllers

import (
	"go-git-webhook/models"
	"github.com/astaxie/beego/logs"
	"go-git-webhook/tasks"
)

type PayloadController struct {
	BaseController
}

func (c *PayloadController) Index() {
	c.Prepare()
	c.TplName = ""

	key := c.Ctx.Input.Param(":key")
	if key == ""{
		c.Abort("404")
	}

	webHook := models.NewWebHook()

	if err := webHook.FindByKey(key);err != nil {
		c.Ctx.WriteString("Git WebHook no found.")
		c.StopRun()
	}

	body := string(c.Ctx.Input.RequestBody[0:])

	if body == ""{
		c.Ctx.WriteString("Request body error.")
		c.StopRun()
	}
	hookData ,err := models.ResolveHookRequest(body)
	if err != nil {
		c.Ctx.WriteString(err.Error())
		c.StopRun()
	}

	branch,err := hookData.BranchName()

	if err != nil {
		c.Ctx.WriteString("Request is not valid Git webhook: " + err.Error())
		c.StopRun()
	}

	repo ,err := hookData.RepositoryName()

	if err != nil {
		c.Ctx.WriteString("Request is not valid Git webhook: " + err.Error())
		c.StopRun()
	}

	if webHook.RepositoryName != repo || webHook.BranchName != branch {
		c.Ctx.WriteString( "Not match the Repo and Branch.")
		//c.StopRun()
	}

	scheduler := models.NewScheduler()

	scheduler.WebHookId = webHook.WebHookId
	scheduler.Data = body

	if push,err := hookData.PushUser() ;err == nil {
		scheduler.PushUser = push
	}
	if value,err := hookData.PushSha(); err == nil{
		scheduler.ShaValue = value
	}else{
		logs.Error("%s",err)
		scheduler.ShaValue = "无"
	}

	scheduler.Status = "wait"

	relationDetailed,err := models.FindRelationDetailedByWhere("AND relation.web_hook_id = ?",webHook.WebHookId)

	if err != nil {
		logs.Error(5001,err.Error())

		c.Ctx.WriteString("Data error")
		c.StopRun()
	}
	if len(relationDetailed) < 0 {
		c.Ctx.WriteString("Data is empty")
		c.StopRun()
	}

	schedulerList := make([]models.Scheduler,len(relationDetailed))

	for i,relation := range relationDetailed {

		scheduler.SchedulerId = 0
		scheduler.ServerId = relation.ServerId
		scheduler.WebHookId = relation.WebHookId
		scheduler.RelationId = relation.RelationId

		schedulerList[i] = *scheduler
	}

	nums,err := models.NewScheduler().InsertMulti(schedulerList)

	if nums <=0 || err != nil {
		if err != nil {
			logs.Error(0,err.Error())

		}
		c.Ctx.WriteString("Data error")
	}
	for _, scheduler := range schedulerList {
		tasks.Add(tasks.Task{ SchedulerId : scheduler.SchedulerId ,ServerId:scheduler.ServerId,WebHookId:scheduler.WebHookId})
	}

	c.Ctx.WriteString("Work put into Queue.")
	c.StopRun()
}

<commit_msg>修复分支判断错误的问题<commit_after>package controllers

import (
	"go-git-webhook/models"
	"github.com/astaxie/beego/logs"
	"go-git-webhook/tasks"
)

type PayloadController struct {
	BaseController
}

func (c *PayloadController) Index() {
	c.Prepare()
	c.TplName = ""

	key := c.Ctx.Input.Param(":key")
	if key == ""{
		c.Abort("404")
	}

	webHook := models.NewWebHook()

	if err := webHook.FindByKey(key);err != nil {
		c.Ctx.WriteString("Git WebHook no found.")
		c.StopRun()
	}

	body := string(c.Ctx.Input.RequestBody[0:])

	if body == ""{
		c.Ctx.WriteString("Request body error.")
		c.StopRun()
	}
	hookData ,err := models.ResolveHookRequest(body)
	if err != nil {
		c.Ctx.WriteString(err.Error())
		c.StopRun()
	}

	branch,err := hookData.BranchName()

	if err != nil {
		c.Ctx.WriteString("Request is not valid Git webhook: " + err.Error())
		c.StopRun()
	}

	repo ,err := hookData.RepositoryName()

	if err != nil {
		c.Ctx.WriteString("Request is not valid Git webhook: " + err.Error())
		c.StopRun()
	}

	if webHook.RepositoryName != repo || (webHook.BranchName != branch && "heads/"+ webHook.BranchName != branch){
		c.Ctx.WriteString( "Not match the Repo and Branch.")
		//c.StopRun()
	}

	scheduler := models.NewScheduler()

	scheduler.WebHookId = webHook.WebHookId
	scheduler.Data = body

	if push,err := hookData.PushUser() ;err == nil {
		scheduler.PushUser = push
	}
	if value,err := hookData.PushSha(); err == nil{
		scheduler.ShaValue = value
	}else{
		logs.Error("%s",err)
		scheduler.ShaValue = "无"
	}

	scheduler.Status = "wait"

	relationDetailed,err := models.FindRelationDetailedByWhere("AND relation.web_hook_id = ?",webHook.WebHookId)

	if err != nil {
		logs.Error(5001,err.Error())

		c.Ctx.WriteString("Data error")
		c.StopRun()
	}
	if len(relationDetailed) < 0 {
		c.Ctx.WriteString("Data is empty")
		c.StopRun()
	}

	schedulerList := make([]models.Scheduler,len(relationDetailed))

	for i,relation := range relationDetailed {

		scheduler.SchedulerId = 0
		scheduler.ServerId = relation.ServerId
		scheduler.WebHookId = relation.WebHookId
		scheduler.RelationId = relation.RelationId

		schedulerList[i] = *scheduler
	}

	nums,err := models.NewScheduler().InsertMulti(schedulerList)

	if nums <=0 || err != nil {
		if err != nil {
			logs.Error(0,err.Error())

		}
		c.Ctx.WriteString("Data error")
	}
	for _, scheduler := range schedulerList {
		tasks.Add(tasks.Task{ SchedulerId : scheduler.SchedulerId ,ServerId:scheduler.ServerId,WebHookId:scheduler.WebHookId})
	}

	c.Ctx.WriteString("Work put into Queue.")
	c.StopRun()
}

<|endoftext|>


<commit_before>package component

import (
	"time"

	"github.com/TheThingsNetwork/ttn/utils/errors"
	"github.com/apex/log"
	"github.com/mwitkow/go-grpc-middleware"
	"golang.org/x/net/context" // See https://github.com/grpc/grpc-go/issues/711"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/metadata"
	"google.golang.org/grpc/peer"
)

func (c *Component) ServerOptions() []grpc.ServerOption {
	unary := func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
		var peerAddr string
		peer, ok := peer.FromContext(ctx)
		if ok {
			peerAddr = peer.Addr.String()
		}
		var peerID string
		meta, ok := metadata.FromContext(ctx)
		if ok {
			id, ok := meta["id"]
			if ok && len(id) > 0 {
				peerID = id[0]
			}
		}
		logCtx := c.Ctx.WithFields(log.Fields{
			"CallerID": peerID,
			"CallerIP": peerAddr,
			"Method":   info.FullMethod,
		})
		t := time.Now()
		iface, err := handler(ctx, req)
		logCtx = logCtx.WithField("Duration", time.Now().Sub(t))
		if err != nil {
			err := errors.FromGRPCError(err)
			logCtx.WithField("error", err.Error()).Warn("Could not handle Request")
		} else {
			logCtx.Info("Handled request")
		}
		return iface, err
	}

	stream := func(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
		var peerAddr string
		peer, ok := peer.FromContext(stream.Context())
		if ok {
			peerAddr = peer.Addr.String()
		}
		var peerID string
		meta, ok := metadata.FromContext(stream.Context())
		if ok {
			id, ok := meta["id"]
			if ok && len(id) > 0 {
				peerID = id[0]
			}
		}
		c.Ctx.WithFields(log.Fields{
			"CallerID": peerID,
			"CallerIP": peerAddr,
			"Method":   info.FullMethod,
		}).Info("Start stream")
		return handler(srv, stream)
	}

	opts := []grpc.ServerOption{
		grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(unary)),
		grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(stream)),
	}

	if c.tlsConfig != nil {
		opts = append(opts, grpc.Creds(credentials.NewTLS(c.tlsConfig)))
	}

	return opts
}
<commit_msg>Add more details to gRPC stream logs<commit_after>package component

import (
	"time"

	"github.com/apex/log"
	"github.com/mwitkow/go-grpc-middleware"
	"golang.org/x/net/context" // See https://github.com/grpc/grpc-go/issues/711"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/metadata"
	"google.golang.org/grpc/peer"
)

func (c *Component) ServerOptions() []grpc.ServerOption {
	unary := func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
		var peerAddr string
		peer, ok := peer.FromContext(ctx)
		if ok {
			peerAddr = peer.Addr.String()
		}
		var peerID string
		meta, ok := metadata.FromContext(ctx)
		if ok {
			id, ok := meta["id"]
			if ok && len(id) > 0 {
				peerID = id[0]
			}
		}
		logCtx := c.Ctx.WithFields(log.Fields{
			"CallerID": peerID,
			"CallerIP": peerAddr,
			"Method":   info.FullMethod,
		})
		t := time.Now()
		iface, err := handler(ctx, req)
		logCtx = logCtx.WithField("Duration", time.Now().Sub(t))
		if err != nil {
			logCtx.WithField("ErrCode", grpc.Code(err)).WithError(err).Warn("Could not handle Request")
		} else {
			logCtx.Debug("Handled request")
		}
		return iface, err
	}

	stream := func(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
		var peerAddr string
		peer, ok := peer.FromContext(stream.Context())
		if ok {
			peerAddr = peer.Addr.String()
		}
		var peerID string
		meta, ok := metadata.FromContext(stream.Context())
		if ok {
			id, ok := meta["id"]
			if ok && len(id) > 0 {
				peerID = id[0]
			}
		}
		logCtx := c.Ctx.WithFields(log.Fields{
			"CallerID": peerID,
			"CallerIP": peerAddr,
			"Method":   info.FullMethod,
		})
		t := time.Now()
		logCtx.Debug("Start stream")
		err := handler(srv, stream)
		logCtx = logCtx.WithField("Duration", time.Now().Sub(t))
		switch err {
		case nil, context.Canceled:
			logCtx.Debug("End stream")
		default:
			logCtx.WithField("ErrCode", grpc.Code(err)).WithError(err).Warn("End stream")
		}
		return err
	}

	opts := []grpc.ServerOption{
		grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(unary)),
		grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(stream)),
	}

	if c.tlsConfig != nil {
		opts = append(opts, grpc.Creds(credentials.NewTLS(c.tlsConfig)))
	}

	return opts
}
<|endoftext|>


<commit_before>package servicehttp

import (
	"net/http"

	"github.com/Comcast/webpa-common/logging"
	"github.com/Comcast/webpa-common/service"
	"github.com/go-kit/kit/log/level"
)

// HashOptions are the configuration parameters for a Hash constructor.
type HashOptions struct {
	// Accessor is the required service Accessor used to hash request keys
	Accessor service.Accessor

	// KeyFunc is used to extract a key from a request.  device.IDHashParser is typically used,
	// although this field has no default.
	KeyFunc KeyFunc

	// RejectCode is the HTTP response code returned when a request does not hash to any of the Self values.
	// If unset (or less than 400), http.StatusCone is used by default.
	RejectCode int

	// ErrorCode is the HTTP response code returned when an error in hashing occurs.  If unset (or less than 400),
	// http.StatusInternalServerError is used by default.
	ErrorCode int

	// Self is the set of instances that refer to the enclosing server.  If empty, no check is performed.  However,
	// the hash is still executed as a verification that the hash does work against the key.
	Self []string
}

// Hash produces an Alice-style constructor that decorates http Handlers with hash enforcement:  requests that do not
// hash to a list of "self" instances are rejected.
func Hash(o HashOptions) func(http.Handler) http.Handler {
	self := make(map[string]bool, len(o.Self))
	for _, i := range o.Self {
		self[i] = true
	}

	if o.RejectCode < 400 {
		o.RejectCode = http.StatusGone
	}

	if o.ErrorCode < 400 {
		o.ErrorCode = http.StatusInternalServerError
	}

	return func(next http.Handler) http.Handler {
		return http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
			key, err := o.KeyFunc(request)
			if err != nil {
				logging.GetLogger(request.Context()).Log(level.Key(), level.ErrorValue(), logging.MessageKey(), "Unable to extract device key", logging.ErrorKey(), err)
				response.WriteHeader(o.ErrorCode)
				return
			}

			hashed, err := o.Accessor.Get(key)
			if err != nil {
				logging.GetLogger(request.Context()).Log(level.Key(), level.ErrorValue(), logging.MessageKey(), "Unable to hash key", logging.ErrorKey(), err)
				response.WriteHeader(o.ErrorCode)
				return
			}

			if len(self) > 0 && !self[hashed] {
				logging.GetLogger(request.Context()).Log(level.Key(), level.ErrorValue(), logging.MessageKey(), "Device does not hash to this server", "key", string(key))
				response.WriteHeader(o.RejectCode)
				return
			}

			next.ServeHTTP(response, request)
		})
	}
}
<commit_msg>Fixed a log message<commit_after>package servicehttp

import (
	"net/http"

	"github.com/Comcast/webpa-common/logging"
	"github.com/Comcast/webpa-common/service"
	"github.com/go-kit/kit/log/level"
)

// HashOptions are the configuration parameters for a Hash constructor.
type HashOptions struct {
	// Accessor is the required service Accessor used to hash request keys
	Accessor service.Accessor

	// KeyFunc is used to extract a key from a request.  device.IDHashParser is typically used,
	// although this field has no default.
	KeyFunc KeyFunc

	// RejectCode is the HTTP response code returned when a request does not hash to any of the Self values.
	// If unset (or less than 400), http.StatusCone is used by default.
	RejectCode int

	// ErrorCode is the HTTP response code returned when an error in hashing occurs.  If unset (or less than 400),
	// http.StatusInternalServerError is used by default.
	ErrorCode int

	// Self is the set of instances that refer to the enclosing server.  If empty, no check is performed.  However,
	// the hash is still executed as a verification that the hash does work against the key.
	Self []string
}

// Hash produces an Alice-style constructor that decorates http Handlers with hash enforcement:  requests that do not
// hash to a list of "self" instances are rejected.
func Hash(o HashOptions) func(http.Handler) http.Handler {
	self := make(map[string]bool, len(o.Self))
	for _, i := range o.Self {
		self[i] = true
	}

	if o.RejectCode < 400 {
		o.RejectCode = http.StatusGone
	}

	if o.ErrorCode < 400 {
		o.ErrorCode = http.StatusInternalServerError
	}

	return func(next http.Handler) http.Handler {
		return http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
			key, err := o.KeyFunc(request)
			if err != nil {
				logging.GetLogger(request.Context()).Log(level.Key(), level.ErrorValue(), logging.MessageKey(), "Unable to extract device key", logging.ErrorKey(), err)
				response.WriteHeader(o.ErrorCode)
				return
			}

			hashed, err := o.Accessor.Get(key)
			if err != nil {
				logging.GetLogger(request.Context()).Log(level.Key(), level.ErrorValue(), logging.MessageKey(), "Unable to hash key", logging.ErrorKey(), err)
				response.WriteHeader(o.ErrorCode)
				return
			}

			if len(self) > 0 && !self[hashed] {
				logging.GetLogger(request.Context()).Log(level.Key(), level.ErrorValue(), logging.MessageKey(), "Request does not hash to this server", "key", string(key))
				response.WriteHeader(o.RejectCode)
				return
			}

			next.ServeHTTP(response, request)
		})
	}
}
<|endoftext|>


<commit_before>package testing

var FluxEndToEndSkipList = map[string]map[string]string{
	"universe": {
		// TODO(adam) determine the reason for these test failures.
		"cov":                      "Reason TBD",
		"covariance":               "Reason TBD",
		"cumulative_sum":           "Reason TBD",
		"cumulative_sum_default":   "Reason TBD",
		"cumulative_sum_noop":      "Reason TBD",
		"drop_non_existent":        "Reason TBD",
		"first":                    "Reason TBD",
		"highestAverage":           "Reason TBD",
		"highestMax":               "Reason TBD",
		"histogram":                "Reason TBD",
		"histogram_normalize":      "Reason TBD",
		"histogram_quantile":       "Reason TBD",
		"join":                     "Reason TBD",
		"join_across_measurements": "Reason TBD",
		"join_agg":                 "Reason TBD",
		"keep_non_existent":        "Reason TBD",
		"key_values":               "Reason TBD",
		"key_values_host_name":     "Reason TBD",
		"last":                     "Reason TBD",
		"lowestAverage":            "Reason TBD",
		"max":                      "Reason TBD",
		"min":                      "Reason TBD",
		"sample":                   "Reason TBD",
		"selector_preserve_time":   "Reason TBD",
		"shift":                    "Reason TBD",
		"shift_negative_duration":  "Reason TBD",
		"task_per_line":            "Reason TBD",
		"top":                      "Reason TBD",
		"union":                    "Reason TBD",
		"union_heterogeneous":      "Reason TBD",
		"unique":                   "Reason TBD",
		"distinct":                 "Reason TBD",

		// it appears these occur when writing the input data.  `to` may not be null safe.
		"fill_bool":   "failed to read meta data: panic: interface conversion: interface {} is nil, not uint64",
		"fill_float":  "failed to read meta data: panic: interface conversion: interface {} is nil, not uint64",
		"fill_int":    "failed to read meta data: panic: interface conversion: interface {} is nil, not uint64",
		"fill_string": "failed to read meta data: panic: interface conversion: interface {} is nil, not uint64",
		"fill_time":   "failed to read meta data: panic: interface conversion: interface {} is nil, not uint64",
		"fill_uint":   "failed to read meta data: panic: interface conversion: interface {} is nil, not uint64",
		"window_null": "failed to read meta data: panic: interface conversion: interface {} is nil, not float64",

		// these may just be missing calls to range() in the tests.  easy to fix in a new PR.
		"group_nulls":         "unbounded test",
		"integral":            "unbounded test",
		"integral_columns":    "unbounded test",
		"map":                 "unbounded test",
		"join_missing_on_col": "unbounded test",
		"rowfn_with_import":   "unbounded test",

		// the following tests have a difference between the CSV-decoded input table, and the storage-retrieved version of that table
		"columns":            "group key mismatch",
		"set":                "column order mismatch",
		"simple_max":         "_stop missing from expected output",
		"derivative":         "time bounds mismatch (engine uses now() instead of bounds on input table)",
		"difference_columns": "data write/read path loses columns x and y",
		"keys":               "group key mismatch",

		// failed to read meta data errors: the CSV encoding is incomplete probably due to data schema errors.  needs more detailed investigation to find root cause of error
		// "filter_by_regex":             "failed to read metadata",
		// "filter_by_tags":              "failed to read metadata",
		"group":                       "failed to read metadata",
		"group_except":                "failed to read metadata",
		"group_ungroup":               "failed to read metadata",
		"pivot_mean":                  "failed to read metadata",
		"histogram_quantile_minvalue": "failed to read meta data: no column with label _measurement exists",
		"increase":                    "failed to read meta data: table has no _value column",

		"string_max":                  "error: invalid use of function: *functions.MaxSelector has no implementation for type string (https://github.com/influxdata/platform/issues/224)",
		"null_as_value":               "null not supported as value in influxql (https://github.com/influxdata/platform/issues/353)",
		"string_interp":               "string interpolation not working as expected in flux (https://github.com/influxdata/platform/issues/404)",
		"to":                          "to functions are not supported in the testing framework (https://github.com/influxdata/flux/issues/77)",
		"covariance_missing_column_1": "need to support known errors in new test framework (https://github.com/influxdata/flux/issues/536)",
		"covariance_missing_column_2": "need to support known errors in new test framework (https://github.com/influxdata/flux/issues/536)",
		"drop_before_rename":          "need to support known errors in new test framework (https://github.com/influxdata/flux/issues/536)",
		"drop_referenced":             "need to support known errors in new test framework (https://github.com/influxdata/flux/issues/536)",
		"yield":                       "yield requires special test case (https://github.com/influxdata/flux/issues/535)",

		"window_group_mean_ungroup": "window trigger optimization modifies sort order of its output tables (https://github.com/influxdata/flux/issues/1067)",

		"median_column": "failing in different ways (https://github.com/influxdata/influxdb/issues/13909)",
		"dynamic_query": "tableFind does not work in e2e tests: https://github.com/influxdata/influxdb/issues/13975",

		"to_int":  "dateTime conversion issue: https://github.com/influxdata/influxdb/issues/14575",
		"to_uint": "dateTime conversion issue: https://github.com/influxdata/influxdb/issues/14575",

		"holt_winters_panic": "Expected output is an empty table which breaks the testing framework (https://github.com/influxdata/influxdb/issues/14749)",
	},
	"experimental": {
		"set":       "Reason TBD",
		"join":      "unbounded test",
		"alignTime": "unbounded test",
	},
	"experimental/geo": {
		"filterRowsNotStrict": "tableFind does not work in e2e tests: https://github.com/influxdata/influxdb/issues/13975",
		"filterRowsStrict":    "tableFind does not work in e2e tests: https://github.com/influxdata/influxdb/issues/13975",
		"gridFilterLevel":     "tableFind does not work in e2e tests: https://github.com/influxdata/influxdb/issues/13975",
		"gridFilter":          "tableFind does not work in e2e tests: https://github.com/influxdata/influxdb/issues/13975",
		"groupByArea":         "tableFind does not work in e2e tests: https://github.com/influxdata/influxdb/issues/13975",
		"filterRowsPivoted":   "tableFind does not work in e2e tests: https://github.com/influxdata/influxdb/issues/13975",
		"shapeDataWithFilter": "tableFind does not work in e2e tests: https://github.com/influxdata/influxdb/issues/13975",
		"shapeData":           "test run before to() is finished: https://github.com/influxdata/influxdb/issues/13975",
	},
	"regexp": {
		"replaceAllString": "Reason TBD",
	},
	"http": {
		"http_endpoint": "need ability to test side effects in e2e tests: (https://github.com/influxdata/flux/issues/1723)",
	},
	"influxdata/influxdb/v1": {
		"show_measurements": "flaky test (https://github.com/influxdata/influxdb/issues/15450)",
		"show_tag_values":   "flaky test (https://github.com/influxdata/influxdb/issues/15450)",
		"show_tag_keys":     "flaky test (https://github.com/influxdata/influxdb/issues/15450)",
	},
	"influxdata/influxdb/monitor": {
		"state_changes_big_any_to_any":     "unbounded test",
		"state_changes_big_info_to_ok":     "unbounded test",
		"state_changes_big_ok_to_info":     "unbounded test",
		"state_changes_any_to_any":         "test run before to() is finished: https://github.com/influxdata/influxdb/issues/13975",
		"state_changes_info_to_any":        "test run before to() is finished: https://github.com/influxdata/influxdb/issues/13975",
		"state_changes_invalid_any_to_any": "test run before to() is finished: https://github.com/influxdata/influxdb/issues/13975",
		"state_changes":                    "test run before to() is finished: https://github.com/influxdata/influxdb/issues/13975",
	},
	"influxdata/influxdb/secrets": {
		"secrets": "Cannot inject custom deps into the test framework so the secrets don't lookup correctly",
	},
	"internal/promql": {
		"join": "unbounded test",
	},
	"testing/chronograf": {
		"buckets": "unbounded test",
	},
	"testing/kapacitor": {
		"fill_default": "unknown field type for f1",
	},
	"testing/pandas": {
		"extract_regexp_findStringIndex": "pandas. map does not correctly handled returned arrays (https://github.com/influxdata/flux/issues/1387)",
		"partition_strings_splitN":       "pandas. map does not correctly handled returned arrays (https://github.com/influxdata/flux/issues/1387)",
	},
	"testing/promql": {
		"emptyTable": "tests a source",
		"year":       "flakey test: https://github.com/influxdata/influxdb/issues/15667",
	},
}
<commit_msg>test: skip flux end to end test for writing null values<commit_after>package testing

var FluxEndToEndSkipList = map[string]map[string]string{
	"universe": {
		// TODO(adam) determine the reason for these test failures.
		"cov":                      "Reason TBD",
		"covariance":               "Reason TBD",
		"cumulative_sum":           "Reason TBD",
		"cumulative_sum_default":   "Reason TBD",
		"cumulative_sum_noop":      "Reason TBD",
		"drop_non_existent":        "Reason TBD",
		"first":                    "Reason TBD",
		"highestAverage":           "Reason TBD",
		"highestMax":               "Reason TBD",
		"histogram":                "Reason TBD",
		"histogram_normalize":      "Reason TBD",
		"histogram_quantile":       "Reason TBD",
		"join":                     "Reason TBD",
		"join_across_measurements": "Reason TBD",
		"join_agg":                 "Reason TBD",
		"keep_non_existent":        "Reason TBD",
		"key_values":               "Reason TBD",
		"key_values_host_name":     "Reason TBD",
		"last":                     "Reason TBD",
		"lowestAverage":            "Reason TBD",
		"max":                      "Reason TBD",
		"min":                      "Reason TBD",
		"sample":                   "Reason TBD",
		"selector_preserve_time":   "Reason TBD",
		"shift":                    "Reason TBD",
		"shift_negative_duration":  "Reason TBD",
		"task_per_line":            "Reason TBD",
		"top":                      "Reason TBD",
		"union":                    "Reason TBD",
		"union_heterogeneous":      "Reason TBD",
		"unique":                   "Reason TBD",
		"distinct":                 "Reason TBD",

		// it appears these occur when writing the input data.  `to` may not be null safe.
		"fill_bool":   "failed to read meta data: panic: interface conversion: interface {} is nil, not uint64",
		"fill_float":  "failed to read meta data: panic: interface conversion: interface {} is nil, not uint64",
		"fill_int":    "failed to read meta data: panic: interface conversion: interface {} is nil, not uint64",
		"fill_string": "failed to read meta data: panic: interface conversion: interface {} is nil, not uint64",
		"fill_time":   "failed to read meta data: panic: interface conversion: interface {} is nil, not uint64",
		"fill_uint":   "failed to read meta data: panic: interface conversion: interface {} is nil, not uint64",
		"window_null": "failed to read meta data: panic: interface conversion: interface {} is nil, not float64",

		// these may just be missing calls to range() in the tests.  easy to fix in a new PR.
		"group_nulls":         "unbounded test",
		"integral":            "unbounded test",
		"integral_columns":    "unbounded test",
		"map":                 "unbounded test",
		"join_missing_on_col": "unbounded test",
		"rowfn_with_import":   "unbounded test",

		// the following tests have a difference between the CSV-decoded input table, and the storage-retrieved version of that table
		"columns":            "group key mismatch",
		"set":                "column order mismatch",
		"simple_max":         "_stop missing from expected output",
		"derivative":         "time bounds mismatch (engine uses now() instead of bounds on input table)",
		"difference_columns": "data write/read path loses columns x and y",
		"keys":               "group key mismatch",

		// failed to read meta data errors: the CSV encoding is incomplete probably due to data schema errors.  needs more detailed investigation to find root cause of error
		// "filter_by_regex":             "failed to read metadata",
		// "filter_by_tags":              "failed to read metadata",
		"group":                       "failed to read metadata",
		"group_except":                "failed to read metadata",
		"group_ungroup":               "failed to read metadata",
		"pivot_mean":                  "failed to read metadata",
		"histogram_quantile_minvalue": "failed to read meta data: no column with label _measurement exists",
		"increase":                    "failed to read meta data: table has no _value column",

		"string_max":                  "error: invalid use of function: *functions.MaxSelector has no implementation for type string (https://github.com/influxdata/platform/issues/224)",
		"null_as_value":               "null not supported as value in influxql (https://github.com/influxdata/platform/issues/353)",
		"string_interp":               "string interpolation not working as expected in flux (https://github.com/influxdata/platform/issues/404)",
		"to":                          "to functions are not supported in the testing framework (https://github.com/influxdata/flux/issues/77)",
		"covariance_missing_column_1": "need to support known errors in new test framework (https://github.com/influxdata/flux/issues/536)",
		"covariance_missing_column_2": "need to support known errors in new test framework (https://github.com/influxdata/flux/issues/536)",
		"drop_before_rename":          "need to support known errors in new test framework (https://github.com/influxdata/flux/issues/536)",
		"drop_referenced":             "need to support known errors in new test framework (https://github.com/influxdata/flux/issues/536)",
		"yield":                       "yield requires special test case (https://github.com/influxdata/flux/issues/535)",

		"window_group_mean_ungroup": "window trigger optimization modifies sort order of its output tables (https://github.com/influxdata/flux/issues/1067)",

		"median_column": "failing in different ways (https://github.com/influxdata/influxdb/issues/13909)",
		"dynamic_query": "tableFind does not work in e2e tests: https://github.com/influxdata/influxdb/issues/13975",

		"to_int":  "dateTime conversion issue: https://github.com/influxdata/influxdb/issues/14575",
		"to_uint": "dateTime conversion issue: https://github.com/influxdata/influxdb/issues/14575",

		"holt_winters_panic": "Expected output is an empty table which breaks the testing framework (https://github.com/influxdata/influxdb/issues/14749)",
		"map_nulls":          "to cannot write null values",
	},
	"experimental": {
		"set":       "Reason TBD",
		"join":      "unbounded test",
		"alignTime": "unbounded test",
	},
	"experimental/geo": {
		"filterRowsNotStrict": "tableFind does not work in e2e tests: https://github.com/influxdata/influxdb/issues/13975",
		"filterRowsStrict":    "tableFind does not work in e2e tests: https://github.com/influxdata/influxdb/issues/13975",
		"gridFilterLevel":     "tableFind does not work in e2e tests: https://github.com/influxdata/influxdb/issues/13975",
		"gridFilter":          "tableFind does not work in e2e tests: https://github.com/influxdata/influxdb/issues/13975",
		"groupByArea":         "tableFind does not work in e2e tests: https://github.com/influxdata/influxdb/issues/13975",
		"filterRowsPivoted":   "tableFind does not work in e2e tests: https://github.com/influxdata/influxdb/issues/13975",
		"shapeDataWithFilter": "tableFind does not work in e2e tests: https://github.com/influxdata/influxdb/issues/13975",
		"shapeData":           "test run before to() is finished: https://github.com/influxdata/influxdb/issues/13975",
	},
	"regexp": {
		"replaceAllString": "Reason TBD",
	},
	"http": {
		"http_endpoint": "need ability to test side effects in e2e tests: (https://github.com/influxdata/flux/issues/1723)",
	},
	"influxdata/influxdb/v1": {
		"show_measurements": "flaky test (https://github.com/influxdata/influxdb/issues/15450)",
		"show_tag_values":   "flaky test (https://github.com/influxdata/influxdb/issues/15450)",
		"show_tag_keys":     "flaky test (https://github.com/influxdata/influxdb/issues/15450)",
	},
	"influxdata/influxdb/monitor": {
		"state_changes_big_any_to_any":     "unbounded test",
		"state_changes_big_info_to_ok":     "unbounded test",
		"state_changes_big_ok_to_info":     "unbounded test",
		"state_changes_any_to_any":         "test run before to() is finished: https://github.com/influxdata/influxdb/issues/13975",
		"state_changes_info_to_any":        "test run before to() is finished: https://github.com/influxdata/influxdb/issues/13975",
		"state_changes_invalid_any_to_any": "test run before to() is finished: https://github.com/influxdata/influxdb/issues/13975",
		"state_changes":                    "test run before to() is finished: https://github.com/influxdata/influxdb/issues/13975",
	},
	"influxdata/influxdb/secrets": {
		"secrets": "Cannot inject custom deps into the test framework so the secrets don't lookup correctly",
	},
	"internal/promql": {
		"join": "unbounded test",
	},
	"testing/chronograf": {
		"buckets": "unbounded test",
	},
	"testing/kapacitor": {
		"fill_default": "unknown field type for f1",
	},
	"testing/pandas": {
		"extract_regexp_findStringIndex": "pandas. map does not correctly handled returned arrays (https://github.com/influxdata/flux/issues/1387)",
		"partition_strings_splitN":       "pandas. map does not correctly handled returned arrays (https://github.com/influxdata/flux/issues/1387)",
	},
	"testing/promql": {
		"emptyTable": "tests a source",
		"year":       "flakey test: https://github.com/influxdata/influxdb/issues/15667",
	},
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"github.com/ckpt/backend-services/caterings"
	"github.com/m4rw3r/uuid"
	"github.com/zenazn/goji/web"
	"net/http"
)

func createNewCatering(c web.C, w http.ResponseWriter, r *http.Request) *appError {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	nCatering := new(caterings.Catering)
	decoder := json.NewDecoder(r.Body)
	if err := decoder.Decode(nCatering); err != nil {
		return &appError{err, "Invalid JSON", 400}
	}
	nCatering, err := caterings.NewCatering(nCatering.Tournament, nCatering.Info)
	if err != nil {
		return &appError{err, "Failed to create new catering", 500}
	}
	w.Header().Set("Catering", "/caterings/"+nCatering.UUID.String())
	w.WriteHeader(201)
	return nil
}

func listAllCaterings(c web.C, w http.ResponseWriter, r *http.Request) *appError {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	list, err := caterings.AllCaterings()
	if err != nil {
		return &appError{err, "Cant load caterings", 500}
	}
	encoder := json.NewEncoder(w)
	encoder.Encode(list)
	return nil
}

func getCatering(c web.C, w http.ResponseWriter, r *http.Request) *appError {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	uuid, err := uuid.FromString(c.URLParams["uuid"])
	catering, err := caterings.CateringByUUID(uuid)
	if err != nil {
		return &appError{err, "Cant find catering", 404}
	}
	encoder := json.NewEncoder(w)
	encoder.Encode(catering)
	return nil
}

func updateCateringInfo(c web.C, w http.ResponseWriter, r *http.Request) *appError {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	uuid, err := uuid.FromString(c.URLParams["uuid"])
	catering, err := caterings.CateringByUUID(uuid)
	if err != nil {
		return &appError{err, "Cant find catering", 404}
	}
	tempInfo := new(caterings.Info)
	decoder := json.NewDecoder(r.Body)
	if err := decoder.Decode(tempInfo); err != nil {
		return &appError{err, "Invalid JSON", 400}
	}

	if err := catering.UpdateInfo(*tempInfo); err != nil {
		return &appError{err, "Failed to update catering info", 500}
	}
	w.WriteHeader(204)
	return nil
}

func addCateringVote(c web.C, w http.ResponseWriter, r *http.Request) *appError {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	uuid, err := uuid.FromString(c.URLParams["uuid"])
	catering, err := caterings.CateringByUUID(uuid)
	if err != nil {
		return &appError{err, "Cant find catering", 404}
	}
	tempInfo := new(caterings.Vote)
	decoder := json.NewDecoder(r.Body)
	if err := decoder.Decode(tempInfo); err != nil {
		return &appError{err, "Invalid JSON", 400}
	}

	if err := catering.AddVote(tempInfo.Player, tempInfo.Score); err != nil {
		return &appError{err, "Failed to add catering vote", 500}
	}
	w.WriteHeader(204)
	return nil
}

func updateCateringVote(c web.C, w http.ResponseWriter, r *http.Request) *appError {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	cateringuuid, err := uuid.FromString(c.URLParams["uuid"])
	playeruuid, err := uuid.FromString(c.URLParams["playeruuid"])
	catering, err := caterings.CateringByUUID(cateringuuid)
	if err != nil {
		return &appError{err, "Cant find catering", 404}
	}

	tempInfo := new(caterings.Vote)
	decoder := json.NewDecoder(r.Body)
	if err := decoder.Decode(tempInfo); err != nil {
		return &appError{err, "Invalid JSON", 400}
	}

	if err := catering.RemoveVote(playeruuid); err != nil {
		return &appError{err, "Failed to remove old catering vote when updating", 500}
	}

	if err := catering.AddVote(playeruuid, tempInfo.Score); err != nil {
		return &appError{err, "Failed to add updated catering vote", 500}
	}
	w.WriteHeader(204)
	return nil
}
<commit_msg>Fix broker location header in catering creation<commit_after>package main

import (
	"encoding/json"
	"github.com/ckpt/backend-services/caterings"
	"github.com/m4rw3r/uuid"
	"github.com/zenazn/goji/web"
	"net/http"
)

func createNewCatering(c web.C, w http.ResponseWriter, r *http.Request) *appError {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	nCatering := new(caterings.Catering)
	decoder := json.NewDecoder(r.Body)
	if err := decoder.Decode(nCatering); err != nil {
		return &appError{err, "Invalid JSON", 400}
	}
	nCatering, err := caterings.NewCatering(nCatering.Tournament, nCatering.Info)
	if err != nil {
		return &appError{err, "Failed to create new catering", 500}
	}
	w.Header().Set("Location", "/caterings/"+nCatering.UUID.String())
	w.WriteHeader(201)
	return nil
}

func listAllCaterings(c web.C, w http.ResponseWriter, r *http.Request) *appError {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	list, err := caterings.AllCaterings()
	if err != nil {
		return &appError{err, "Cant load caterings", 500}
	}
	encoder := json.NewEncoder(w)
	encoder.Encode(list)
	return nil
}

func getCatering(c web.C, w http.ResponseWriter, r *http.Request) *appError {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	uuid, err := uuid.FromString(c.URLParams["uuid"])
	catering, err := caterings.CateringByUUID(uuid)
	if err != nil {
		return &appError{err, "Cant find catering", 404}
	}
	encoder := json.NewEncoder(w)
	encoder.Encode(catering)
	return nil
}

func updateCateringInfo(c web.C, w http.ResponseWriter, r *http.Request) *appError {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	uuid, err := uuid.FromString(c.URLParams["uuid"])
	catering, err := caterings.CateringByUUID(uuid)
	if err != nil {
		return &appError{err, "Cant find catering", 404}
	}
	tempInfo := new(caterings.Info)
	decoder := json.NewDecoder(r.Body)
	if err := decoder.Decode(tempInfo); err != nil {
		return &appError{err, "Invalid JSON", 400}
	}

	if err := catering.UpdateInfo(*tempInfo); err != nil {
		return &appError{err, "Failed to update catering info", 500}
	}
	w.WriteHeader(204)
	return nil
}

func addCateringVote(c web.C, w http.ResponseWriter, r *http.Request) *appError {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	uuid, err := uuid.FromString(c.URLParams["uuid"])
	catering, err := caterings.CateringByUUID(uuid)
	if err != nil {
		return &appError{err, "Cant find catering", 404}
	}
	tempInfo := new(caterings.Vote)
	decoder := json.NewDecoder(r.Body)
	if err := decoder.Decode(tempInfo); err != nil {
		return &appError{err, "Invalid JSON", 400}
	}

	if err := catering.AddVote(tempInfo.Player, tempInfo.Score); err != nil {
		return &appError{err, "Failed to add catering vote", 500}
	}
	w.WriteHeader(204)
	return nil
}

func updateCateringVote(c web.C, w http.ResponseWriter, r *http.Request) *appError {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	cateringuuid, err := uuid.FromString(c.URLParams["uuid"])
	playeruuid, err := uuid.FromString(c.URLParams["playeruuid"])
	catering, err := caterings.CateringByUUID(cateringuuid)
	if err != nil {
		return &appError{err, "Cant find catering", 404}
	}

	tempInfo := new(caterings.Vote)
	decoder := json.NewDecoder(r.Body)
	if err := decoder.Decode(tempInfo); err != nil {
		return &appError{err, "Invalid JSON", 400}
	}

	if err := catering.RemoveVote(playeruuid); err != nil {
		return &appError{err, "Failed to remove old catering vote when updating", 500}
	}

	if err := catering.AddVote(playeruuid, tempInfo.Score); err != nil {
		return &appError{err, "Failed to add updated catering vote", 500}
	}
	w.WriteHeader(204)
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	log "github.com/cihub/seelog"
	"github.com/goamz/goamz/aws"
	"github.com/goamz/goamz/ec2"
	"github.com/goamz/goamz/route53"
	"os"
	"time"
)

// Store multi-part file (avoids blowing the memory by loading a huge file):
func hostInventoryUpdater() {

	log.Infof("[hostInventoryUpdater] Starting up")

	// Run forever:
	for {

		// Authenticate with AWS:
		awsAuth, err := aws.GetAuth("", "", "", time.Now())
		if err != nil {
			log.Errorf("[hostInventoryUpdater] Unable to authenticate to AWS! (%s) ...\n", err)
			os.Exit(1)

		} else {
			log.Debugf("[hostInventoryUpdater] Authenticated to AWS")
		}

		// Make a new EC2 connection:
		log.Debugf("[hostInventoryUpdater] Connecting to EC2...")
		ec2Connection := ec2.New(awsAuth, aws.Regions[awsRegion])

		// Prepare a filter:
		filter := ec2.NewFilter()
		filter.Add("instance-state-name", "running")

		// Make a "DescribeInstances" call (lists ALL instances in your account):
		describeInstancesResponse, err := ec2Connection.DescribeInstances([]string{}, filter)
		if err != nil {
			log.Errorf("[hostInventoryUpdater] Failed to make describe-instances call: %v", err)

		} else {
			log.Debugf("[hostInventoryUpdater] Found %v instances running in your account", len(describeInstancesResponse.Reservations))

			// Lock the host-list (so we don't change it while another goroutine is using it):
			hostInventoryMutex.Lock()

			// Clear out the existing host-inventory:
			hostInventory = HostInventoryDNSRecords{
				Environments: make(map[string]Environment),
			}

			// Re-populate it from the describe instances response:
			for _, reservation := range describeInstancesResponse.Reservations {

				// Search for our role and environment tags:
				var role, environment string
				for _, tag := range reservation.Instances[0].Tags {
					if tag.Key == roleTag {
						role = tag.Value
					}
					if tag.Key == environmentTag {
						environment = tag.Value
					}
				}

				// Make sure we have environment and role tags:
				if environment == "" || role == "" {
					log.Debugf("Instance (%v) must have both 'environment' and 'role' tags in order for DNS records to be creted...", reservation.Instances[0].InstanceId)

					// Continue with the next instance:
					continue
				}

				// Either create or add to the environment record:
				if _, ok := hostInventory.Environments[environment]; !ok {
					hostInventory.Environments[environment] = Environment{
						DNSRecords: make(map[string][]route53.ResourceRecordValue),
					}
				}

				// Either create or add to the per-role record:
				roleRecord := fmt.Sprintf("%v.%v", role, awsRegion)
				if _, ok := hostInventory.Environments[environment].DNSRecords[roleRecord]; !ok {
					hostInventory.Environments[environment].DNSRecords[roleRecord] = []route53.ResourceRecordValue{{Value: reservation.Instances[0].PrivateIPAddress}}
				} else {
					hostInventory.Environments[environment].DNSRecords[roleRecord] = append(hostInventory.Environments[environment].DNSRecords[roleRecord], route53.ResourceRecordValue{Value: reservation.Instances[0].PrivateIPAddress})
				}

				// Either create or add to the role-per-az record:
				azRecord := fmt.Sprintf("%v.%v", role, reservation.Instances[0].AvailabilityZone)
				if _, ok := hostInventory.Environments[environment].DNSRecords[azRecord]; !ok {
					hostInventory.Environments[environment].DNSRecords[azRecord] = []route53.ResourceRecordValue{{Value: reservation.Instances[0].PrivateIPAddress}}
				} else {
					hostInventory.Environments[environment].DNSRecords[azRecord] = append(hostInventory.Environments[environment].DNSRecords[azRecord], route53.ResourceRecordValue{Value: reservation.Instances[0].PrivateIPAddress})
				}

			}

			// Unlock:
			hostInventoryMutex.Unlock()

		}

		// Sleep until the next run:
		log.Debugf("[hostInventoryUpdater] Sleeping for %vs...", hostInventoryUpdateFrequencySeconds)
		time.Sleep(time.Duration(hostInventoryUpdateFrequencySeconds) * time.Second)

	}

}
<commit_msg>Now supporting public IPs too<commit_after>package main

import (
	"fmt"
	log "github.com/cihub/seelog"
	"github.com/goamz/goamz/aws"
	"github.com/goamz/goamz/ec2"
	"github.com/goamz/goamz/route53"
	"os"
	"time"
)

// Store multi-part file (avoids blowing the memory by loading a huge file):
func hostInventoryUpdater() {

	log.Infof("[hostInventoryUpdater] Starting up")

	// Run forever:
	for {

		// Authenticate with AWS:
		awsAuth, err := aws.GetAuth("", "", "", time.Now())
		if err != nil {
			log.Errorf("[hostInventoryUpdater] Unable to authenticate to AWS! (%s) ...\n", err)
			os.Exit(1)

		} else {
			log.Debugf("[hostInventoryUpdater] Authenticated to AWS")
		}

		// Make a new EC2 connection:
		log.Debugf("[hostInventoryUpdater] Connecting to EC2...")
		ec2Connection := ec2.New(awsAuth, aws.Regions[awsRegion])

		// Prepare a filter:
		filter := ec2.NewFilter()
		filter.Add("instance-state-name", "running")

		// Make a "DescribeInstances" call (lists ALL instances in your account):
		describeInstancesResponse, err := ec2Connection.DescribeInstances([]string{}, filter)
		if err != nil {
			log.Errorf("[hostInventoryUpdater] Failed to make describe-instances call: %v", err)

		} else {
			log.Debugf("[hostInventoryUpdater] Found %v instances running in your account", len(describeInstancesResponse.Reservations))

			// Lock the host-list (so we don't change it while another goroutine is using it):
			hostInventoryMutex.Lock()

			// Clear out the existing host-inventory:
			hostInventory = HostInventoryDNSRecords{
				Environments: make(map[string]Environment),
			}

			// Re-populate it from the describe instances response:
			for _, reservation := range describeInstancesResponse.Reservations {

				// Search for our role and environment tags:
				var role, environment string
				for _, tag := range reservation.Instances[0].Tags {
					if tag.Key == roleTag {
						role = tag.Value
					}
					if tag.Key == environmentTag {
						environment = tag.Value
					}
				}

				// Make sure we have environment and role tags:
				if environment == "" || role == "" {
					log.Debugf("Instance (%v) must have both 'environment' and 'role' tags in order for DNS records to be creted...", reservation.Instances[0].InstanceId)

					// Continue with the next instance:
					continue
				}

				// Either create or add to the environment record:
				if _, ok := hostInventory.Environments[environment]; !ok {
					hostInventory.Environments[environment] = Environment{
						DNSRecords: make(map[string][]route53.ResourceRecordValue),
					}
				}

				// Either create or add to the per-role records:
				internalRoleRecord := fmt.Sprintf("%v.%v.i", role, awsRegion)
				if _, ok := hostInventory.Environments[environment].DNSRecords[internalRoleRecord]; !ok {
					hostInventory.Environments[environment].DNSRecords[internalRoleRecord] = []route53.ResourceRecordValue{{Value: reservation.Instances[0].PrivateIPAddress}}
				} else {
					hostInventory.Environments[environment].DNSRecords[internalRoleRecord] = append(hostInventory.Environments[environment].DNSRecords[internalRoleRecord], route53.ResourceRecordValue{Value: reservation.Instances[0].PrivateIPAddress})
				}

				// Also make a per-role record with the public IP address (if we have one):
				if reservation.Instances[0].IPAddress != "" {
					externalRoleRecord := fmt.Sprintf("%v.%v.e", role, awsRegion)
					if _, ok := hostInventory.Environments[environment].DNSRecords[externalRoleRecord]; !ok {
						hostInventory.Environments[environment].DNSRecords[externalRoleRecord] = []route53.ResourceRecordValue{{Value: reservation.Instances[0].IPAddress}}
					} else {
						hostInventory.Environments[environment].DNSRecords[externalRoleRecord] = append(hostInventory.Environments[environment].DNSRecords[externalRoleRecord], route53.ResourceRecordValue{Value: reservation.Instances[0].IPAddress})
					}
				}

				// Either create or add to the role-per-az record:
				internalAZRecord := fmt.Sprintf("%v.%v.i", role, reservation.Instances[0].AvailabilityZone)
				if _, ok := hostInventory.Environments[environment].DNSRecords[internalAZRecord]; !ok {
					hostInventory.Environments[environment].DNSRecords[internalAZRecord] = []route53.ResourceRecordValue{{Value: reservation.Instances[0].PrivateIPAddress}}
				} else {
					hostInventory.Environments[environment].DNSRecords[internalAZRecord] = append(hostInventory.Environments[environment].DNSRecords[internalAZRecord], route53.ResourceRecordValue{Value: reservation.Instances[0].PrivateIPAddress})
				}

			}

			// Unlock:
			hostInventoryMutex.Unlock()

		}

		// Sleep until the next run:
		log.Debugf("[hostInventoryUpdater] Sleeping for %vs...", hostInventoryUpdateFrequencySeconds)
		time.Sleep(time.Duration(hostInventoryUpdateFrequencySeconds) * time.Second)

	}

}
<|endoftext|>


<commit_before>package cbfsclient

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"sync"
	"time"
)

type FetchCallback func(oid string, r io.Reader) error

type blobInfo struct {
	Nodes map[string]time.Time
}

func (c Client) getBlobInfos(oids ...string) (map[string]blobInfo, error) {
	u := c.Path("/.cbfs/blob/info/")
	form := url.Values{"blob": oids}
	res, err := http.PostForm(u, form)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		return nil, fmt.Errorf("HTTP error fetching blob info: %v",
			res.Status)
	}

	d := json.NewDecoder(res.Body)
	rv := map[string]blobInfo{}
	err = d.Decode(&rv)
	return rv, err
}

type fetchWork struct {
	oid string
	bi  blobInfo
}

type brokenReader struct{ err error }

func (b brokenReader) Read([]byte) (int, error) {
	return 0, b.err
}

func fetchOne(oid string, si StorageNode, cb FetchCallback) error {
	res, err := http.Get(si.BlobURL(oid))
	if err != nil {
		return err
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		return fmt.Errorf("HTTP error: %v", res.Status)
	}
	return cb(oid, res.Body)
}

func fetchWorker(cb FetchCallback, nodes map[string]StorageNode,
	ch chan fetchWork, errch chan<- error, wg *sync.WaitGroup) {

	defer wg.Done()
	for w := range ch {
		var err error
		for n := range w.bi.Nodes {
			err = fetchOne(w.oid, nodes[n], cb)
			if err == nil {
				break
			}
		}
		if err != nil {
			select {
			case errch <- err:
			default:
			}
			cb(w.oid,
				brokenReader{fmt.Errorf("couldn't find %v", w.oid)})
		}
	}
}

// Fetch many blobs in bulk.
func (c Client) Blobs(concurrency int,
	cb FetchCallback, oids ...string) error {

	nodes, err := c.Nodes()
	if err != nil {
		return err
	}

	infos, err := c.getBlobInfos(oids...)
	if err != nil {
		return err
	}

	workch := make(chan fetchWork)
	errch := make(chan error, 1)

	wg := &sync.WaitGroup{}
	for i := 0; i < concurrency; i++ {
		wg.Add(1)
		go fetchWorker(cb, nodes, workch, errch, wg)
	}

	for oid, info := range infos {
		workch <- fetchWork{oid, info}
	}
	close(workch)

	go func() {
		wg.Wait()
		close(errch)
	}()

	return <-errch
}

// Grab a file.
//
// This ensures the request is coming directly from a node that
// already has the blob vs. proxying.
func (c Client) Get(path string) (io.ReadCloser, error) {
	req, err := http.NewRequest("GET", c.Path(path), nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("X-CBFS-LocalOnly", "true")

	res, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}

	if res.StatusCode == 300 {
		defer res.Body.Close()

		sources := []string{}
		d := json.NewDecoder(res.Body)
		err = d.Decode(&sources)
		if err != nil {
			return nil, err
		}

		if len(sources) == 0 {
			return nil, fmt.Errorf("No sources found")
		}

		res, err = http.Get(sources[0])
		if err != nil {
			return nil, err
		}
	}

	return res.Body, nil
}
<commit_msg>Use location header in direct Gets.<commit_after>package cbfsclient

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"sync"
	"time"
)

type FetchCallback func(oid string, r io.Reader) error

type blobInfo struct {
	Nodes map[string]time.Time
}

func (c Client) getBlobInfos(oids ...string) (map[string]blobInfo, error) {
	u := c.Path("/.cbfs/blob/info/")
	form := url.Values{"blob": oids}
	res, err := http.PostForm(u, form)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		return nil, fmt.Errorf("HTTP error fetching blob info: %v",
			res.Status)
	}

	d := json.NewDecoder(res.Body)
	rv := map[string]blobInfo{}
	err = d.Decode(&rv)
	return rv, err
}

type fetchWork struct {
	oid string
	bi  blobInfo
}

type brokenReader struct{ err error }

func (b brokenReader) Read([]byte) (int, error) {
	return 0, b.err
}

func fetchOne(oid string, si StorageNode, cb FetchCallback) error {
	res, err := http.Get(si.BlobURL(oid))
	if err != nil {
		return err
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		return fmt.Errorf("HTTP error: %v", res.Status)
	}
	return cb(oid, res.Body)
}

func fetchWorker(cb FetchCallback, nodes map[string]StorageNode,
	ch chan fetchWork, errch chan<- error, wg *sync.WaitGroup) {

	defer wg.Done()
	for w := range ch {
		var err error
		for n := range w.bi.Nodes {
			err = fetchOne(w.oid, nodes[n], cb)
			if err == nil {
				break
			}
		}
		if err != nil {
			select {
			case errch <- err:
			default:
			}
			cb(w.oid,
				brokenReader{fmt.Errorf("couldn't find %v", w.oid)})
		}
	}
}

// Fetch many blobs in bulk.
func (c Client) Blobs(concurrency int,
	cb FetchCallback, oids ...string) error {

	nodes, err := c.Nodes()
	if err != nil {
		return err
	}

	infos, err := c.getBlobInfos(oids...)
	if err != nil {
		return err
	}

	workch := make(chan fetchWork)
	errch := make(chan error, 1)

	wg := &sync.WaitGroup{}
	for i := 0; i < concurrency; i++ {
		wg.Add(1)
		go fetchWorker(cb, nodes, workch, errch, wg)
	}

	for oid, info := range infos {
		workch <- fetchWork{oid, info}
	}
	close(workch)

	go func() {
		wg.Wait()
		close(errch)
	}()

	return <-errch
}

// Grab a file.
//
// This ensures the request is coming directly from a node that
// already has the blob vs. proxying.
func (c Client) Get(path string) (io.ReadCloser, error) {
	req, err := http.NewRequest("GET", c.Path(path), nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("X-CBFS-LocalOnly", "true")

	res, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}

	if res.StatusCode == 300 {
		defer res.Body.Close()
		nu := res.Header.Get("Location")
		res, err = http.Get(nu)
		if err != nil {
			return nil, err
		}
	}

	return res.Body, nil
}
<|endoftext|>


<commit_before>// Copyright 2017 The go-vgo Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// https://github.com/go-vgo/gt/blob/master/LICENSE
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

package hset

import (
	"testing"

	"github.com/vcaesar/tt"
)

func TestSetAdd(t *testing.T) {
	set := New()
	set.Add()
	set.Add(1)
	set.Add(2)
	set.Add(2, 3)
	set.Add()

	actualValue := set.Empty()
	tt.False(t, actualValue)

	actualVal := set.Len()
	tt.Equal(t, 3, actualVal)
}

func TestSetContains(t *testing.T) {
	set := New()
	set.Add(3, 1, 2)
	set.Add(2, 3)
	set.Add()

	actualValue := set.Contains()
	tt.True(t, actualValue)

	actualValue = set.Contains(1)
	tt.True(t, actualValue)

	actualValue = set.Contains(1, 2, 3)
	tt.True(t, actualValue)

	actualValue = set.Contains(1, 2, 3, 4)
	tt.False(t, actualValue)
}

func TestSetRemove(t *testing.T) {
	set := New()
	set.Add(3, 1, 2)
	set.Remove()
	actualValue := set.Len()
	tt.Equal(t, 3, actualValue)

	set.Remove(1)
	actualValue = set.Len()
	tt.Equal(t, 2, actualValue)

	set.Remove(3)
	set.Remove(3)
	set.Remove()
	set.Remove(2)

	actualValue = set.Len()
	tt.Equal(t, 0, actualValue)
}

func TestSetString(t *testing.T) {
	set := New()
	set.Add("11")
	set.Add(1)

	tt.Equal(t, "11, 1", set.String())
}
<commit_msg>Update hash set test code<commit_after>// Copyright 2017 The go-vgo Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// https://github.com/go-vgo/gt/blob/master/LICENSE
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

package hset

import (
	"log"
	"testing"

	"github.com/vcaesar/tt"
)

func TestSetAdd(t *testing.T) {
	set := New()
	set.Add()
	set.Add(1)
	set.Add(2)
	set.Add(2, 3)
	set.Add()
	log.Println("set: ", set)

	actualValue := set.Empty()
	tt.False(t, actualValue)

	actualVal := set.Len()
	tt.Equal(t, 3, actualVal)
}

func TestSetContains(t *testing.T) {
	set := New()
	set.Add(3, 1, 2)
	set.Add(2, 3)
	set.Add()
	log.Println("set: ", set)

	actualValue := set.Contains()
	tt.True(t, actualValue)

	actualValue = set.Contains(1)
	tt.True(t, actualValue)

	actualValue = set.Contains(1, 2, 3)
	tt.True(t, actualValue)

	actualValue = set.Contains(1, 2, 3, 4)
	tt.False(t, actualValue)
}

func TestSetRemove(t *testing.T) {
	set := New()
	set.Add(3, 1, 2)
	set.Remove()
	actualValue := set.Len()
	tt.Equal(t, 3, actualValue)

	set.Remove(1)
	actualValue = set.Len()
	tt.Equal(t, 2, actualValue)

	set.Remove(3)
	set.Remove(3)
	set.Remove()
	set.Remove(2)

	actualValue = set.Len()
	tt.Equal(t, 0, actualValue)
}

func TestSetString(t *testing.T) {
	set := New()
	set.Add("11")
	set.Add(1)
	s1 := set.String()
	log.Println("set: ", s1)

	tt.Equal(t, 5, len(s1))
}
<|endoftext|>


<commit_before><commit_msg>add IsBool() check in lisp.Object type assertion; also reorder objectTypeName if/else order based on the use frequency<commit_after><|endoftext|>