<commit_before>package main

import (
	"common"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"
	"time"
)

type orchestrator struct {
	repoip      chan string
	deploystate chan map[string]common.DockerInfo
	addip       chan string
}

func (o *orchestrator) pollDocker(ip string, update chan common.DockerInfo) {
	for ; ; time.Sleep(60 * time.Second) {
		c, err := common.ListContainers(ip)
		if err != nil {
			log.Print(err)
			continue
		}
		d := common.DockerInfo{ip, c, nil, time.Now()}
		update <- d
	}
}

func (o *orchestrator) StartState() {
	d := make(map[string]common.DockerInfo)
	o.deploystate = make(chan map[string]common.DockerInfo)
	o.addip = make(chan string)
	updatechan := make(chan common.DockerInfo)
	for {
		select {
		case o.deploystate <- d:

		case ip := <-o.addip:
			_, exist := d[ip]
			if !exist {
				d[ip] = common.DockerInfo{}
			}
			go o.pollDocker(ip, updatechan)

		case up := <-updatechan:
			d[up.Ip] = up
		}
	}
}

func (o *orchestrator) WaitRefresh(t time.Time) {
	for {
		s := <-o.deploystate
		good := false
		for _, v := range s {
			if v.Updated.After(t) {
				good = true
				break

			}
		}
		if good {
			break
		}
		time.Sleep(10 * time.Second)
	}
}

func (o *orchestrator) StartRepository() {
	log.Print("index setup")
	registry_name := "samalba/docker-registry"
	host := os.Getenv("HOST")
	// So that id is passed out of the function
	id := ""
	var err error
	var running bool
	for ; ; time.Sleep(10 * time.Second) {
		running, id, err = common.ImageRunning(host, registry_name)
		if err != nil {
			log.Print(err)
			continue
		}
		if !running {
			log.Print("index not running")
			err := common.LoadImage(host, registry_name)
			if err != nil {
				log.Print(err)
				continue
			}
			id, err = common.RunImage(host, registry_name, false)
			if err != nil {
				log.Print(err)
				continue
			}
		}
		break
	}
	log.Print("index running id: ", id)
	config, err := common.InspectContainer(host, id)
	log.Print("fetched config")
	ip := config.NetworkSettings.PortMapping.Tcp["5000"]

	host = host + ":" + ip

	if err != nil {
		log.Print(err)
	}

	o.repoip = make(chan string)
	for {
		o.repoip <- host
	}

}

func (o *orchestrator) handleImage(w http.ResponseWriter, r *http.Request) {
	repoip := <-o.repoip
	io.WriteString(w, "Recieved\n")
	tag := r.URL.Query()["name"]
	if len(tag) > 0 {
		io.WriteString(w, "Building image\n")
		err := common.BuildImage(os.Getenv("HOST"), r.Body, tag[0])
		if err != nil {
			io.WriteString(w, err.Error()+"\n")
			return
		}

		io.WriteString(w, "Tagging\n")
		repo_tag := repoip + "/" + tag[0]
		err = common.TagImage(os.Getenv("HOST"), tag[0], repo_tag)
		if err != nil {
			io.WriteString(w, err.Error())
			return
		}
		io.WriteString(w, "Pushing to index\n")
		err = common.PushImage(os.Getenv("HOST"), w, repo_tag)
		if err != nil {
			io.WriteString(w, err.Error())
			return
		}
	}
	io.WriteString(w, "built\n")
}

func (o *orchestrator) calcUpdate(w io.Writer, desired common.SkeletonDeployment, current map[string]common.DockerInfo) (update map[string][]string) {
	c := fmt.Sprint(current)
	io.WriteString(w, c)
	io.WriteString(w, "\n")
	// Maps IP's to lists of containers to deploy
	update = make(map[string][]string)
	// For each container we want to deploy
	for container, _ := range desired.Containers {
		// Assuming granularity machine

		// For each machine check for container
		for ip, mInfo := range current {

			//Have we found the container
			found := false

			//Check if the container is running
			for _, checkContainer := range mInfo.Containers {

				imageName := checkContainer.Image

				//Get the actual name
				io.WriteString(w, "Image name before proc: ")
				io.WriteString(w, imageName)
				io.WriteString(w, "\n")
				if strings.Contains(imageName, "/") {
					imageName = strings.SplitN(imageName, "/", 2)[1]
				}
				if strings.Contains(imageName, ":") {
					imageName = strings.SplitN(imageName, ":", 2)[0]
				}
				io.WriteString(w, "Image name after proc: ")
				io.WriteString(w, imageName)
				io.WriteString(w, "\n")

				if imageName == container {
					found = true
					break
				}
			}

			//Do we need to deploy a image?
			if !found {
				update[ip] = append(update[ip], container)
			}

		}
	}

	return update

}

func (o *orchestrator) deploy(w http.ResponseWriter, r *http.Request) {

	io.WriteString(w, "Starting deploy\n")
	d := &common.SkeletonDeployment{}
	c, err := ioutil.ReadAll(r.Body)

	if err != nil {
		io.WriteString(w, err.Error())
		return
	}
	err = json.Unmarshal(c, d)
	if err != nil {
		io.WriteString(w, err.Error())
		return
	}

	for _, ip := range d.Machines.Ip {
		io.WriteString(w, "Adding ip\n")
		io.WriteString(w, ip)
		io.WriteString(w, "\n")
		o.addip <- ip
	}

	io.WriteString(w, "Waiting for image refreshes\n")
	o.WaitRefresh(time.Now())
	io.WriteString(w, "waited\n")

	current := <-o.deploystate

	diff := o.calcUpdate(w, *d, current)

	sdiff := fmt.Sprint(diff)
	io.WriteString(w, sdiff)
	io.WriteString(w, "\n")

	indexip := <-o.repoip

	io.WriteString(w, "Deploying diff\n")
	for ip, images := range diff {
		for _, container := range images {
			io.WriteString(w, "Deploying "+container+" on "+ip+"\n")
			err := common.LoadImage(ip, indexip+"/"+container)
			if err != nil {
				io.WriteString(w, err.Error())
				continue
			}
			id, err := common.RunImage(ip, indexip+"/"+container, false)
			io.WriteString(w, "Deployed \n")
			io.WriteString(w, id)
			io.WriteString(w, "\n")
			if err != nil {
				io.WriteString(w, err.Error())
			}
			io.WriteString(w, "\n")
		}
	}
}

func NewOrchestrator() (o *orchestrator) {
	o = new(orchestrator)
	go o.StartRepository()
	go o.StartState()
	return o
}

func main() {

	o := NewOrchestrator()

	http.HandleFunc("/version", func(w http.ResponseWriter, r *http.Request) {
		io.WriteString(w, "orchestrator v0")
	})

	http.HandleFunc("/image", o.handleImage)

	http.HandleFunc("/deploy", o.deploy)

	log.Fatal(http.ListenAndServe(":900", nil))
}
<commit_msg>orchestrator: Add warning when having to download repo image<commit_after>package main

import (
	"common"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"
	"time"
)

type orchestrator struct {
	repoip      chan string
	deploystate chan map[string]common.DockerInfo
	addip       chan string
}

func (o *orchestrator) pollDocker(ip string, update chan common.DockerInfo) {
	for ; ; time.Sleep(60 * time.Second) {
		c, err := common.ListContainers(ip)
		if err != nil {
			log.Print(err)
			continue
		}
		d := common.DockerInfo{ip, c, nil, time.Now()}
		update <- d
	}
}

func (o *orchestrator) StartState() {
	d := make(map[string]common.DockerInfo)
	o.deploystate = make(chan map[string]common.DockerInfo)
	o.addip = make(chan string)
	updatechan := make(chan common.DockerInfo)
	for {
		select {
		case o.deploystate <- d:

		case ip := <-o.addip:
			_, exist := d[ip]
			if !exist {
				d[ip] = common.DockerInfo{}
			}
			go o.pollDocker(ip, updatechan)

		case up := <-updatechan:
			d[up.Ip] = up
		}
	}
}

func (o *orchestrator) WaitRefresh(t time.Time) {
	for {
		s := <-o.deploystate
		good := false
		for _, v := range s {
			if v.Updated.After(t) {
				good = true
				break

			}
		}
		if good {
			break
		}
		time.Sleep(10 * time.Second)
	}
}

func (o *orchestrator) StartRepository() {
	log.Print("index setup")
	registry_name := "samalba/docker-registry"
	host := os.Getenv("HOST")
	// So that id is passed out of the function
	id := ""
	var err error
	var running bool
	for ; ; time.Sleep(10 * time.Second) {
		running, id, err = common.ImageRunning(host, registry_name)
		if err != nil {
			log.Print(err)
			continue
		}
		if !running {
			log.Print("index not running")
			err := common.LoadImage(host, registry_name)
			if err != nil {
				log.Print(err)
				continue
			}
			id, err = common.RunImage(host, registry_name, false)
			if err != nil {
				log.Print(err)
				continue
			}
		}
		break
	}
	log.Print("index running id: ", id)
	config, err := common.InspectContainer(host, id)
	log.Print("fetched config")
	ip := config.NetworkSettings.PortMapping.Tcp["5000"]

	host = host + ":" + ip

	if err != nil {
		log.Print(err)
	}

	o.repoip = make(chan string)
	for {
		o.repoip <- host
	}

}

func (o *orchestrator) handleImage(w http.ResponseWriter, r *http.Request) {
	if o.repoip == nil {
		io.WriteString(w, "Waiting for index to be downloaded, this may take a while")
	}
	repoip := <-o.repoip
	io.WriteString(w, "Recieved\n")
	tag := r.URL.Query()["name"]
	if len(tag) > 0 {
		io.WriteString(w, "Building image\n")
		err := common.BuildImage(os.Getenv("HOST"), r.Body, tag[0])
		if err != nil {
			io.WriteString(w, err.Error()+"\n")
			return
		}

		io.WriteString(w, "Tagging\n")
		repo_tag := repoip + "/" + tag[0]
		err = common.TagImage(os.Getenv("HOST"), tag[0], repo_tag)
		if err != nil {
			io.WriteString(w, err.Error())
			return
		}
		io.WriteString(w, "Pushing to index\n")
		err = common.PushImage(os.Getenv("HOST"), w, repo_tag)
		if err != nil {
			io.WriteString(w, err.Error())
			return
		}
	}
	io.WriteString(w, "built\n")
}

func (o *orchestrator) calcUpdate(w io.Writer, desired common.SkeletonDeployment, current map[string]common.DockerInfo) (update map[string][]string) {
	c := fmt.Sprint(current)
	io.WriteString(w, c)
	io.WriteString(w, "\n")
	// Maps IP's to lists of containers to deploy
	update = make(map[string][]string)
	// For each container we want to deploy
	for container, _ := range desired.Containers {
		// Assuming granularity machine

		// For each machine check for container
		for ip, mInfo := range current {

			//Have we found the container
			found := false

			//Check if the container is running
			for _, checkContainer := range mInfo.Containers {

				imageName := checkContainer.Image

				//Get the actual name
				io.WriteString(w, "Image name before proc: ")
				io.WriteString(w, imageName)
				io.WriteString(w, "\n")
				if strings.Contains(imageName, "/") {
					imageName = strings.SplitN(imageName, "/", 2)[1]
				}
				if strings.Contains(imageName, ":") {
					imageName = strings.SplitN(imageName, ":", 2)[0]
				}
				io.WriteString(w, "Image name after proc: ")
				io.WriteString(w, imageName)
				io.WriteString(w, "\n")

				if imageName == container {
					found = true
					break
				}
			}

			//Do we need to deploy a image?
			if !found {
				update[ip] = append(update[ip], container)
			}

		}
	}

	return update

}

func (o *orchestrator) deploy(w http.ResponseWriter, r *http.Request) {

	io.WriteString(w, "Starting deploy\n")
	d := &common.SkeletonDeployment{}
	c, err := ioutil.ReadAll(r.Body)

	if err != nil {
		io.WriteString(w, err.Error())
		return
	}
	err = json.Unmarshal(c, d)
	if err != nil {
		io.WriteString(w, err.Error())
		return
	}

	for _, ip := range d.Machines.Ip {
		io.WriteString(w, "Adding ip\n")
		io.WriteString(w, ip)
		io.WriteString(w, "\n")
		o.addip <- ip
	}

	io.WriteString(w, "Waiting for image refreshes\n")
	o.WaitRefresh(time.Now())
	io.WriteString(w, "waited\n")

	current := <-o.deploystate

	diff := o.calcUpdate(w, *d, current)

	sdiff := fmt.Sprint(diff)
	io.WriteString(w, sdiff)
	io.WriteString(w, "\n")

	indexip := <-o.repoip

	io.WriteString(w, "Deploying diff\n")
	for ip, images := range diff {
		for _, container := range images {
			io.WriteString(w, "Deploying "+container+" on "+ip+"\n")
			err := common.LoadImage(ip, indexip+"/"+container)
			if err != nil {
				io.WriteString(w, err.Error())
				continue
			}
			id, err := common.RunImage(ip, indexip+"/"+container, false)
			io.WriteString(w, "Deployed \n")
			io.WriteString(w, id)
			io.WriteString(w, "\n")
			if err != nil {
				io.WriteString(w, err.Error())
			}
			io.WriteString(w, "\n")
		}
	}
}

func NewOrchestrator() (o *orchestrator) {
	o = new(orchestrator)
	go o.StartRepository()
	go o.StartState()
	return o
}

func main() {

	o := NewOrchestrator()

	http.HandleFunc("/version", func(w http.ResponseWriter, r *http.Request) {
		io.WriteString(w, "orchestrator v0")
	})

	http.HandleFunc("/image", o.handleImage)

	http.HandleFunc("/deploy", o.deploy)

	log.Fatal(http.ListenAndServe(":900", nil))
}
<|endoftext|>


<commit_before>/*

Package mirror implement the mirror backend designed to store blobs to multiple backends.

Get/Exists/Enumerate requests are performed on the first BlobHandler.

*/
package mirror

import (
	"expvar"
	"fmt"
	"log"
	"strings"

	"github.com/tsileo/blobstash/backend"
)

var (
	bytesUploaded   = expvar.NewMap("mirror-bytes-uploaded")
	bytesDownloaded = expvar.NewMap("mirror-bytes-downloaded")
	blobsUploaded   = expvar.NewMap("mirror-blobs-uploaded")
	blobsDownloaded = expvar.NewMap("mirror-blobs-downloaded")
)

type MirrorBackend struct {
	backends []backend.BlobHandler
}

func New(backends ...backend.BlobHandler) *MirrorBackend {
	log.Println("MirrorBackend: starting")
	b := &MirrorBackend{[]backend.BlobHandler{}}
	for _, mBackend := range backends {
		log.Printf("MirrorBackend: adding backend %v", mBackend.String())
		b.backends = append(b.backends, mBackend)
	}
	return b
}

func (backend *MirrorBackend) String() string {
	backends := []string{}
	for _, b := range backend.backends {
		backends = append(backends, b.String())
	}
	return fmt.Sprintf("mirror-%v", strings.Join(backends, "-"))
}

func (backend *MirrorBackend) Close() {
	for _, b := range backend.backends {
		b.Close()
	}
}

func (backend *MirrorBackend) Done() error {
	for _, b := range backend.backends {
		if err := b.Done(); err != nil {
			return err
		}
	}
	return nil
}

func (backend *MirrorBackend) Put(hash string, data []byte) (err error) {
	for _, b := range backend.backends {
		if err := b.Put(hash, data); err != nil {
			return err
		}
		bytesUploaded.Add("total", int64(len(data)))
		blobsUploaded.Add("total", 1)
		bytesUploaded.Add(b.String(), int64(len(data)))
		blobsUploaded.Add(b.String(), 1)
	}
	return
}

func (backend *MirrorBackend) Exists(hash string) bool {
	for _, b := range backend.backends {
		return b.Exists(hash)
	}
	return false
}

func (backend *MirrorBackend) Get(hash string) (data []byte, err error) {
	for _, b := range backend.backends {
		data, err = b.Get(hash)
		if err == nil {
			blobsDownloaded.Add("total", 1)
			bytesDownloaded.Add("total", int64(len(data)))
			blobsDownloaded.Add(b.String(), 1)
			bytesDownloaded.Add(b.String(), int64(len(data)))
			return
		} else {
			log.Printf("MirrorBackend: error fetching blob %v from backend %b", hash, b.String())
		}
	}
	return
}

func (backen *MirrorBackend) Enumerate(blobs chan<- string) error {
	defer close(blobs)
	for _, b := range backen.backends {
		tblobs := make(chan string)
		err := b.Enumerate(tblobs)
		go func() {
			for bl := range tblobs {
				blobs <- bl
			}
		}()
		switch err {
		case backend.ErrWriteOnly:
			continue
		default:
			return err
		}
	}
	return fmt.Errorf("shouldn't happen")
}
<commit_msg>mirror backend, better Enumerate design<commit_after>/*

Package mirror implement the mirror backend designed to store blobs to multiple backends.

Get/Exists/Enumerate requests are performed on the first BlobHandler.

*/
package mirror

import (
	"expvar"
	"fmt"
	"log"
	"strings"

	"github.com/tsileo/blobstash/backend"
)

var (
	bytesUploaded   = expvar.NewMap("mirror-bytes-uploaded")
	bytesDownloaded = expvar.NewMap("mirror-bytes-downloaded")
	blobsUploaded   = expvar.NewMap("mirror-blobs-uploaded")
	blobsDownloaded = expvar.NewMap("mirror-blobs-downloaded")
)

type MirrorBackend struct {
	backends []backend.BlobHandler
}

func New(backends ...backend.BlobHandler) *MirrorBackend {
	log.Println("MirrorBackend: starting")
	b := &MirrorBackend{[]backend.BlobHandler{}}
	for _, mBackend := range backends {
		log.Printf("MirrorBackend: adding backend %v", mBackend.String())
		b.backends = append(b.backends, mBackend)
	}
	return b
}

func (backend *MirrorBackend) String() string {
	backends := []string{}
	for _, b := range backend.backends {
		backends = append(backends, b.String())
	}
	return fmt.Sprintf("mirror-%v", strings.Join(backends, "-"))
}

func (backend *MirrorBackend) Close() {
	for _, b := range backend.backends {
		b.Close()
	}
}

func (backend *MirrorBackend) Done() error {
	for _, b := range backend.backends {
		if err := b.Done(); err != nil {
			return err
		}
	}
	return nil
}

func (backend *MirrorBackend) Put(hash string, data []byte) (err error) {
	for _, b := range backend.backends {
		if err := b.Put(hash, data); err != nil {
			return err
		}
		bytesUploaded.Add("total", int64(len(data)))
		blobsUploaded.Add("total", 1)
		bytesUploaded.Add(b.String(), int64(len(data)))
		blobsUploaded.Add(b.String(), 1)
	}
	return
}

func (backend *MirrorBackend) Exists(hash string) bool {
	for _, b := range backend.backends {
		return b.Exists(hash)
	}
	return false
}

func (backend *MirrorBackend) Get(hash string) (data []byte, err error) {
	for _, b := range backend.backends {
		data, err = b.Get(hash)
		if err == nil {
			blobsDownloaded.Add("total", 1)
			bytesDownloaded.Add("total", int64(len(data)))
			blobsDownloaded.Add(b.String(), 1)
			bytesDownloaded.Add(b.String(), int64(len(data)))
			return
		} else {
			log.Printf("MirrorBackend: error fetching blob %v from backend %b", hash, b.String())
		}
	}
	return
}

func (backen *MirrorBackend) Enumerate(blobs chan<- string) error {
	defer close(blobs)
	for _, b := range backen.backends {
		errc := make(chan error)
		tblobs := make(chan string)
		go func() {
			errc <- b.Enumerate(tblobs)
		}()
		for bl := range tblobs {
			blobs <- bl
		}
		err := <-errc
		switch err {
		case backend.ErrWriteOnly:
			continue
		default:
			return err
		}
	}
	return fmt.Errorf("shouldn't happen")
}
<|endoftext|>


<commit_before>package tsz

import (
	"math/rand"
	"testing"
)

var T uint32
var V float64

func BenchmarkPushSeries4h(b *testing.B) {
	s := NewSeries4h(0)
	N := uint32(b.N)
	for i := uint32(1); i <= N; i++ {
		if i%10 == 0 {
			s.Push(i, 0)
		} else if i%10 == 1 {
			s.Push(i, 1)
		} else {
			s.Push(i, float64(i)+123.45)
		}
	}
	s.Finish()
	b.Logf("Series4h size: %dB", len(s.Bytes()))
}

func BenchmarkPushSeriesLong(b *testing.B) {
	s := NewSeriesLong(0)
	N := uint32(b.N)
	for i := uint32(1); i <= N; i++ {
		if i%10 == 0 {
			s.Push(i, 0)
		} else if i%10 == 1 {
			s.Push(i, 1)
		} else {
			s.Push(i, float64(i)+123.45)
		}
	}
	s.Finish()
	b.Logf("SeriesLong size: %dB", len(s.Bytes()))
}

func BenchmarkIterSeries4h(b *testing.B) {
	s := NewSeries4h(0)
	N := uint32(b.N)
	for i := uint32(1); i <= N; i++ {
		s.Push(i, 123.45)
	}
	b.ResetTimer()
	iter := s.Iter(1)
	var t uint32
	var v float64
	for iter.Next() {
		t, v = iter.Values()
	}
	err := iter.Err()
	if err != nil {
		panic(err)
	}
	T = t
	V = v
}

func BenchmarkIterSeriesLong(b *testing.B) {
	s := NewSeriesLong(0)
	N := uint32(b.N)
	for i := uint32(1); i <= N; i++ {
		s.Push(i, 123.45)
	}
	b.ResetTimer()
	iter := s.Iter()
	var t uint32
	var v float64
	for iter.Next() {
		t, v = iter.Values()
	}
	err := iter.Err()
	if err != nil {
		panic(err)
	}
	T = t
	V = v
}

func BenchmarkIterSeriesLongInterface(b *testing.B) {
	s := NewSeriesLong(0)
	N := uint32(b.N)
	for i := uint32(1); i <= N; i++ {
		s.Push(i, 123.45)
	}
	b.ResetTimer()
	var t uint32
	var v float64
	var iter Iter
	// avoid compiler optimization where it can statically assign the right type
	// and skip the overhead of the interface
	if rand.Intn(1) == 0 {
		iter = s.Iter()
	}
	for iter.Next() {
		t, v = iter.Values()
	}
	err := iter.Err()
	if err != nil {
		panic(err)
	}
	T = t
	V = v
}
<commit_msg>Add more tszlong  benchmarks<commit_after>package tsz

import (
	"math/rand"
	"testing"
)

var T uint32
var V float64

func BenchmarkPushSeries4h(b *testing.B) {
	s := NewSeries4h(0)
	N := uint32(b.N)
	for i := uint32(1); i <= N; i++ {
		if i%10 == 0 {
			s.Push(i, 0)
		} else if i%10 == 1 {
			s.Push(i, 1)
		} else {
			s.Push(i, float64(i)+123.45)
		}
	}
	s.Finish()
	b.Logf("Series4h size: %dB", len(s.Bytes()))
}

func benchmarkSeriesLong(b *testing.B, generator func(uint32) float64) {
	s := NewSeriesLong(0)
	N := uint32(b.N)
	for i := uint32(1); i <= N; i++ {
		s.Push(i, generator(i))
	}
	s.Finish()
	b.Logf("SeriesLong size: %d points in %dB, avg %.2f bytes/point", N, len(s.Bytes()), float64(len(s.Bytes()))/float64(N))
}

func BenchmarkPushSeriesLong(b *testing.B) {
	benchmarkSeriesLong(b, func(i uint32) float64 {
		if i%10 == 0 {
			return 0
		} else if i%10 == 1 {
			return 1
		}
		return float64(i) + 123.45
	})
}

func BenchmarkPushSeriesLongMonotonicIncrease(b *testing.B) {
	benchmarkSeriesLong(b, func(i uint32) float64 {
		return float64(i) + 123.45
	})
}

func BenchmarkPushSeriesLongSawtooth(b *testing.B) {
	benchmarkSeriesLong(b, func(i uint32) float64 {
		multiplier := 1.0
		if i%2 == 0 {
			multiplier = -1.0
		}
		return multiplier*123.45 + float64(i)/1000
	})
}

func BenchmarkPushSeriesLongSawtoothWithFlats(b *testing.B) {
	benchmarkSeriesLong(b, func(i uint32) float64 {
		multiplier := 1.0
		if i%2 == 0 && i%100 != 0 {
			multiplier = -1.0
		}
		return multiplier*123.45 + float64(i)/1000
	})
}

func BenchmarkPushSeriesLongSteps(b *testing.B) {
	benchmarkSeriesLong(b, func(i uint32) float64 {
		multiplier := 1.0
		if (i/100)%2 == 0 {
			multiplier = -1.0
		}
		return multiplier*123.45 + float64(i)/1000
	})
}

func BenchmarkPushSeriesLongRealWorldCPU(b *testing.B) {
	values := []float64{95.3, 95.7, 86.2, 95.0, 94.7, 95.4, 94.5, 94.0, 94.7, 95.0, 95.0, 93.8, 95.3, 95.4, 94.6, 83.8, 94.5, 94.5, 94.6, 92.0, 95.0, 89.6, 72.8, 72.1, 86.5, 94.9, 94.9, 93.9, 94.4, 95.4, 95.1, 93.7, 95.5, 95.4, 94.4, 93.2, 94.6, 95.5, 94.9, 94.1, 95.0, 95.5, 94.7, 93.7, 95.1, 96.6, 95.3, 94.0, 95.0, 95.2, 93.3, 94.2, 95.2, 94.9, 94.5, 95.3, 93.2, 95.4, 95.0, 95.2, 93.7}
	benchmarkSeriesLong(b, func(i uint32) float64 {
		return values[int(i)%len(values)]
	})
}

func BenchmarkIterSeries4h(b *testing.B) {
	s := NewSeries4h(0)
	N := uint32(b.N)
	for i := uint32(1); i <= N; i++ {
		s.Push(i, 123.45)
	}
	b.ResetTimer()
	iter := s.Iter(1)
	var t uint32
	var v float64
	for iter.Next() {
		t, v = iter.Values()
	}
	err := iter.Err()
	if err != nil {
		panic(err)
	}
	T = t
	V = v
}

func BenchmarkIterSeriesLong(b *testing.B) {
	s := NewSeriesLong(0)
	N := uint32(b.N)
	for i := uint32(1); i <= N; i++ {
		s.Push(i, 123.45)
	}
	b.ResetTimer()
	iter := s.Iter()
	var t uint32
	var v float64
	for iter.Next() {
		t, v = iter.Values()
	}
	err := iter.Err()
	if err != nil {
		panic(err)
	}
	T = t
	V = v
}

func BenchmarkIterSeriesLongInterface(b *testing.B) {
	s := NewSeriesLong(0)
	N := uint32(b.N)
	for i := uint32(1); i <= N; i++ {
		s.Push(i, 123.45)
	}
	b.ResetTimer()
	var t uint32
	var v float64
	var iter Iter
	// avoid compiler optimization where it can statically assign the right type
	// and skip the overhead of the interface
	if rand.Intn(1) == 0 {
		iter = s.Iter()
	}
	for iter.Next() {
		t, v = iter.Values()
	}
	err := iter.Err()
	if err != nil {
		panic(err)
	}
	T = t
	V = v
}
<|endoftext|>


<commit_before>// +build !as_performance

// Copyright 2013-2016 Aerospike, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package aerospike

import (
	"reflect"

	Buffer "github.com/aerospike/aerospike-client-go/utils/buffer"
)

// this function will only be set if the performance flag is not passed for build
func init() {
	multiObjectParser = batchParseObject
	prepareReflectionData = concretePrepareReflectionData
}

func concretePrepareReflectionData(cmd *baseMultiCommand) {
	// if a channel is assigned, assign its value type
	if cmd.recordset != nil && !cmd.recordset.objChan.IsNil() {
		// this channel must be of type chan *T
		cmd.resObjType = cmd.recordset.objChan.Type().Elem().Elem()
		cmd.resObjMappings = objectMappings.getMapping(cmd.recordset.objChan.Type().Elem().Elem())

		cmd.selectCases = []reflect.SelectCase{
			reflect.SelectCase{Dir: reflect.SelectSend, Chan: cmd.recordset.objChan},
			reflect.SelectCase{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(cmd.recordset.cancelled)},
		}
	}
}

func batchParseObject(
	cmd *baseMultiCommand,
	obj reflect.Value,
	opCount int,
	fieldCount int,
	generation uint32,
	expiration uint32,
) error {
	for i := 0; i < opCount; i++ {
		if err := cmd.readBytes(8); err != nil {
			cmd.recordset.Errors <- newNodeError(cmd.node, err)
			return err
		}

		opSize := int(Buffer.BytesToUint32(cmd.dataBuffer, 0))
		particleType := int(cmd.dataBuffer[5])
		nameSize := int(cmd.dataBuffer[7])

		if err := cmd.readBytes(nameSize); err != nil {
			cmd.recordset.Errors <- newNodeError(cmd.node, err)
			return err
		}
		name := string(cmd.dataBuffer[:nameSize])

		particleBytesSize := int((opSize - (4 + nameSize)))
		if err := cmd.readBytes(particleBytesSize); err != nil {
			cmd.recordset.Errors <- newNodeError(cmd.node, err)
			return err
		}
		value, err := bytesToParticle(particleType, cmd.dataBuffer, 0, particleBytesSize)
		if err != nil {
			cmd.recordset.Errors <- newNodeError(cmd.node, err)
			return err
		}

		iobj := indirect(obj)
		if err := setObjectField(cmd.resObjMappings, iobj, name, value); err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>Moved conflicted code<commit_after>// +build !as_performance

// Copyright 2013-2016 Aerospike, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package aerospike

import (
	"reflect"

	Buffer "github.com/aerospike/aerospike-client-go/utils/buffer"
)

// this function will only be set if the performance flag is not passed for build
func init() {
	multiObjectParser = batchParseObject
	prepareReflectionData = concretePrepareReflectionData
}

func concretePrepareReflectionData(cmd *baseMultiCommand) {
	// if a channel is assigned, assign its value type
	if cmd.recordset != nil && !cmd.recordset.objChan.IsNil() {
		// this channel must be of type chan *T
		cmd.resObjType = cmd.recordset.objChan.Type().Elem().Elem()
		cmd.resObjMappings = objectMappings.getMapping(cmd.recordset.objChan.Type().Elem().Elem())

		cmd.selectCases = []reflect.SelectCase{
			{Dir: reflect.SelectSend, Chan: cmd.recordset.objChan},
			{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(cmd.recordset.cancelled)},
		}
	}
}

func batchParseObject(
	cmd *baseMultiCommand,
	obj reflect.Value,
	opCount int,
	fieldCount int,
	generation uint32,
	expiration uint32,
) error {
	for i := 0; i < opCount; i++ {
		if err := cmd.readBytes(8); err != nil {
			cmd.recordset.Errors <- newNodeError(cmd.node, err)
			return err
		}

		opSize := int(Buffer.BytesToUint32(cmd.dataBuffer, 0))
		particleType := int(cmd.dataBuffer[5])
		nameSize := int(cmd.dataBuffer[7])

		if err := cmd.readBytes(nameSize); err != nil {
			cmd.recordset.Errors <- newNodeError(cmd.node, err)
			return err
		}
		name := string(cmd.dataBuffer[:nameSize])

		particleBytesSize := int((opSize - (4 + nameSize)))
		if err := cmd.readBytes(particleBytesSize); err != nil {
			cmd.recordset.Errors <- newNodeError(cmd.node, err)
			return err
		}
		value, err := bytesToParticle(particleType, cmd.dataBuffer, 0, particleBytesSize)
		if err != nil {
			cmd.recordset.Errors <- newNodeError(cmd.node, err)
			return err
		}

		iobj := indirect(obj)
		if err := setObjectField(cmd.resObjMappings, iobj, name, value); err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package wok_test

import (
	"encoding/json"
	"fmt"
	"github.com/andviro/noodle"
	mw "github.com/andviro/noodle/middleware"
	"github.com/andviro/noodle/render"
	"github.com/andviro/wok"
	"golang.org/x/net/context"
	"gopkg.in/tylerb/is.v1"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"testing"
	"time"
)

func mwFactory(tag string) noodle.Middleware {
	return func(next noodle.Handler) noodle.Handler {
		return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
			fmt.Fprintf(w, "%s>", tag)
			return next(ctx, w, r)
		}
	}
}

func handlerFactory(tag string) noodle.Handler {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
		fmt.Fprintf(w, "[%s]", tag)
		return nil
	}
}

func TestNew(t *testing.T) {
	is := is.New(t)
	wok := wok.New()
	is.NotNil(wok)
}

func testRequest(wok *wok.Wok, method, path string) string {
	w := httptest.NewRecorder()
	r, _ := http.NewRequest(method, "http://localhost"+path, nil)
	wok.ServeHTTP(w, r)
	return w.Body.String()
}

func TestHandle(t *testing.T) {
	is := is.New(t)
	wk := wok.New(mwFactory("A"))
	wk.GET("/")(handlerFactory("B"))
	is.Equal(testRequest(wk, "GET", "/"), "A>[B]")
}

func TestGroup(t *testing.T) {
	is := is.New(t)
	wk := wok.New(mwFactory("A"))
	g1 := wk.Group("/g1", mwFactory("G1"))
	g2 := wk.Group("/g2", mwFactory("G2"))
	g1.GET("/", mwFactory("G11"))(handlerFactory("B"))
	g2.GET("/", mwFactory("G21"))(handlerFactory("C"))

	is.Equal(testRequest(wk, "GET", "/g1"), "A>G1>G11>[B]")
	is.Equal(testRequest(wk, "GET", "/g2"), "A>G2>G21>[C]")
}

func TestRouterVars(t *testing.T) {
	is := is.New(t)
	mw := func(next noodle.Handler) noodle.Handler {
		return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
			fmt.Fprint(w, "MW>")
			return next(context.WithValue(ctx, 0, "testValue"), w, r)
		}
	}
	wk := wok.New(mw)
	wk.GET("/:varA/:varB")(func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
		fmt.Fprintf(w, "[%s][%s][%s]", wok.Var(ctx, "varA"), wok.Var(ctx, "varB"), ctx.Value(0).(string))
		return nil
	})
	is.Equal(testRequest(wk, "GET", "/1/2"), "MW>[1][2][testValue]")
}

func ExampleApplication() {
	// globalErrorHandler receives all errors from all handlers
	// and tries to return meaningful HTTP status and message
	globalErrorHandler := func(next noodle.Handler) noodle.Handler {
		return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
			err := next(ctx, w, r)
			switch err {
			case mw.UnauthorizedRequest:
				w.WriteHeader(401)
				fmt.Fprint(w, "Please provide credentials")
			case nil:
				return nil
			default:
				w.WriteHeader(500)
				fmt.Fprintf(w, "There was an error: %v", err)
			}
			return err
		}
	}

	// apiErrorHandler is a specific error catcher that renders its messages into JSON
	apiErrorHandler := func(next noodle.Handler) noodle.Handler {
		return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
			err := next(ctx, w, r)
			switch err {
			case mw.UnauthorizedRequest:
				render.Yield(ctx, 401, map[string]interface{}{
					"error": err,
				})
			case nil:
				return nil
			default:
				render.Yield(ctx, 500, map[string]interface{}{
					"error": err,
				})
			}
			return err
		}
	}

	// apiAuth guards access to api group
	apiAuth := mw.HTTPAuth("API", func(user, pass string) bool {
		return pass == "Secret"
	})
	// apiAuth guards access to dashboard group
	dashboardAuth := mw.HTTPAuth("Dashboard", func(user, pass string) bool {
		return pass == "Password"
	})

	// w is the root router
	w := wok.Default(globalErrorHandler)

	// Handle index page
	w.GET("/")(func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
		fmt.Fprint(w, "Index page")
		return nil
	})

	// api is a group of routes with common authentication, result rendering and error handling
	api := w.Group("/api", render.JSON, apiErrorHandler, apiAuth)
	{
		api.GET("/")(func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
			res := []int{1, 2, 3, 4, 5}
			return render.Yield(ctx, 200, res)
		})
		api.GET("/:id")(func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
			id := wok.Var(ctx, "id")
			res := struct {
				ID string
			}{id}
			return render.Yield(ctx, 201, res)
		})
	}

	// dash is an example of another separate route group
	dash := w.Group("/dash", dashboardAuth)
	{
		dash.GET("/")(func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
			fmt.Fprintf(w, "Hello %s", mw.GetUser(ctx))
			return nil
		})
	}

	go http.ListenAndServe(":8989", w)
	time.Sleep(300 * time.Millisecond) // let it settle down

	// Here we will test webapp responses

	// index
	resp, _ := http.Get("http://localhost:8989/")
	data, _ := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	fmt.Println(string(data))

	// dashboard
	resp, _ = http.Get("http://user:Password@localhost:8989/dash")
	data, _ = ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	fmt.Println(string(data))

	// api index
	resp, _ = http.Get("http://user:Secret@localhost:8989/api")
	var lst []int
	json.NewDecoder(resp.Body).Decode(&lst)
	fmt.Println(lst)

	// api with parameter
	resp, _ = http.Get("http://user:Secret@localhost:8989/api/12")
	var obj map[string]interface{}
	json.NewDecoder(resp.Body).Decode(&obj)
	fmt.Println(obj)

	// Output: Index page
	// Hello user
	// [1 2 3 4 5]
	// map[ID:12]
}
<commit_msg>fixed test<commit_after>package wok_test

import (
	"encoding/json"
	"fmt"
	"github.com/andviro/noodle"
	mw "github.com/andviro/noodle/middleware"
	"github.com/andviro/noodle/render"
	"github.com/andviro/noodle/wok"
	"golang.org/x/net/context"
	"gopkg.in/tylerb/is.v1"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"testing"
	"time"
)

func mwFactory(tag string) noodle.Middleware {
	return func(next noodle.Handler) noodle.Handler {
		return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
			fmt.Fprintf(w, "%s>", tag)
			return next(ctx, w, r)
		}
	}
}

func handlerFactory(tag string) noodle.Handler {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
		fmt.Fprintf(w, "[%s]", tag)
		return nil
	}
}

func TestNew(t *testing.T) {
	is := is.New(t)
	wok := wok.New()
	is.NotNil(wok)
}

func testRequest(wok *wok.Wok, method, path string) string {
	w := httptest.NewRecorder()
	r, _ := http.NewRequest(method, "http://localhost"+path, nil)
	wok.ServeHTTP(w, r)
	return w.Body.String()
}

func TestHandle(t *testing.T) {
	is := is.New(t)
	wk := wok.New(mwFactory("A"))
	wk.GET("/")(handlerFactory("B"))
	is.Equal(testRequest(wk, "GET", "/"), "A>[B]")
}

func TestGroup(t *testing.T) {
	is := is.New(t)
	wk := wok.New(mwFactory("A"))
	g1 := wk.Group("/g1", mwFactory("G1"))
	g2 := wk.Group("/g2", mwFactory("G2"))
	g1.GET("/", mwFactory("G11"))(handlerFactory("B"))
	g2.GET("/", mwFactory("G21"))(handlerFactory("C"))

	is.Equal(testRequest(wk, "GET", "/g1"), "A>G1>G11>[B]")
	is.Equal(testRequest(wk, "GET", "/g2"), "A>G2>G21>[C]")
}

func TestRouterVars(t *testing.T) {
	is := is.New(t)
	mw := func(next noodle.Handler) noodle.Handler {
		return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
			fmt.Fprint(w, "MW>")
			return next(context.WithValue(ctx, 0, "testValue"), w, r)
		}
	}
	wk := wok.New(mw)
	wk.GET("/:varA/:varB")(func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
		fmt.Fprintf(w, "[%s][%s][%s]", wok.Var(ctx, "varA"), wok.Var(ctx, "varB"), ctx.Value(0).(string))
		return nil
	})
	is.Equal(testRequest(wk, "GET", "/1/2"), "MW>[1][2][testValue]")
}

func ExampleApplication() {
	// globalErrorHandler receives all errors from all handlers
	// and tries to return meaningful HTTP status and message
	globalErrorHandler := func(next noodle.Handler) noodle.Handler {
		return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
			err := next(ctx, w, r)
			switch err {
			case mw.UnauthorizedRequest:
				w.WriteHeader(401)
				fmt.Fprint(w, "Please provide credentials")
			case nil:
				return nil
			default:
				w.WriteHeader(500)
				fmt.Fprintf(w, "There was an error: %v", err)
			}
			return err
		}
	}

	// apiErrorHandler is a specific error catcher that renders its messages into JSON
	apiErrorHandler := func(next noodle.Handler) noodle.Handler {
		return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
			err := next(ctx, w, r)
			switch err {
			case mw.UnauthorizedRequest:
				render.Yield(ctx, 401, map[string]interface{}{
					"error": err,
				})
			case nil:
				return nil
			default:
				render.Yield(ctx, 500, map[string]interface{}{
					"error": err,
				})
			}
			return err
		}
	}

	// apiAuth guards access to api group
	apiAuth := mw.HTTPAuth("API", func(user, pass string) bool {
		return pass == "Secret"
	})
	// apiAuth guards access to dashboard group
	dashboardAuth := mw.HTTPAuth("Dashboard", func(user, pass string) bool {
		return pass == "Password"
	})

	// w is the root router
	w := wok.Default(globalErrorHandler)

	// Handle index page
	w.GET("/")(func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
		fmt.Fprint(w, "Index page")
		return nil
	})

	// api is a group of routes with common authentication, result rendering and error handling
	api := w.Group("/api", render.JSON, apiErrorHandler, apiAuth)
	{
		api.GET("/")(func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
			res := []int{1, 2, 3, 4, 5}
			return render.Yield(ctx, 200, res)
		})
		api.GET("/:id")(func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
			id := wok.Var(ctx, "id")
			res := struct {
				ID string
			}{id}
			return render.Yield(ctx, 201, res)
		})
	}

	// dash is an example of another separate route group
	dash := w.Group("/dash", dashboardAuth)
	{
		dash.GET("/")(func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
			fmt.Fprintf(w, "Hello %s", mw.GetUser(ctx))
			return nil
		})
	}

	go http.ListenAndServe(":8989", w)
	time.Sleep(300 * time.Millisecond) // let it settle down

	// Here we will test webapp responses

	// index
	resp, _ := http.Get("http://localhost:8989/")
	data, _ := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	fmt.Println(string(data))

	// dashboard
	resp, _ = http.Get("http://user:Password@localhost:8989/dash")
	data, _ = ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	fmt.Println(string(data))

	// api index
	resp, _ = http.Get("http://user:Secret@localhost:8989/api")
	var lst []int
	json.NewDecoder(resp.Body).Decode(&lst)
	fmt.Println(lst)

	// api with parameter
	resp, _ = http.Get("http://user:Secret@localhost:8989/api/12")
	var obj map[string]interface{}
	json.NewDecoder(resp.Body).Decode(&obj)
	fmt.Println(obj)

	// Output: Index page
	// Hello user
	// [1 2 3 4 5]
	// map[ID:12]
}
<|endoftext|>


<commit_before>// Copyright 2015-2017 trivago GmbH
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package router

import (
	"github.com/trivago/gollum/core"
)

// Distribute stream plugin
// Messages will be routed to all streams configured. Each target stream can
// hold another stream configuration, too, so this is not directly sending to
// the producers attached to the target streams.
// Configuration example
//
// 	myrouter":
//    Type: "router.Distribute"
//    Stream: "mystream"
// 	  Routes:
//      - "foo"
//      - "bar"
//
// Routes defines a 1:n stream remapping.
// Messages are reassigned to all of stream(s) in this list.
// If no route is set messages are forwarded on the incoming router.
// When routing to multiple streams, the incoming stream has to be listed explicitly to be used.
type Distribute struct {
	Broadcast
	streams []core.Router
}

func init() {
	core.TypeRegistry.Register(Distribute{})
}

// Configure initializes this distributor with values from a plugin config.
func (router *Distribute) Configure(conf core.PluginConfigReader) error {
	router.Broadcast.Configure(conf)

	boundStreamIDs := conf.GetStreamArray("Streams", []core.MessageStreamID{})
	for _, streamID := range boundStreamIDs {
		route := core.StreamRegistry.GetRouterOrFallback(streamID)
		router.streams = append(router.streams, route)
	}

	return conf.Errors.OrNil()
}

func (router *Distribute) route(msg *core.Message, route core.Router) {
	if router.StreamID() == router.StreamID() {
		router.Broadcast.Enqueue(msg)
	} else {
		msg.SetStreamID(route.StreamID())
		core.Route(msg, route)
	}
}

// Enqueue enques a message to the router
func (router *Distribute) Enqueue(msg *core.Message) error {
	numStreams := len(router.streams)

	switch numStreams {
	case 0:
		return core.NewModulateResultError("No producers configured for stream %s", router.GetID())

	case 1:
		router.route(msg, router.streams[0])

	default:
		lastStreamIdx := numStreams - 1
		for streamIdx := 0; streamIdx < lastStreamIdx; streamIdx++ {
			router.route(msg.Clone(), router.streams[streamIdx])
		}
		router.route(msg, router.streams[lastStreamIdx])
	}

	return nil
}
<commit_msg>fixed Distribute router<commit_after>// Copyright 2015-2017 trivago GmbH
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package router

import (
	"github.com/trivago/gollum/core"
)

// Distribute stream plugin
// Messages will be routed to all streams configured. Each target stream can
// hold another stream configuration, too, so this is not directly sending to
// the producers attached to the target streams.
// Configuration example
//
// "myrouter":
//    Type: "router.Distribute"
//    Stream: "mystream"
//    TargetStreams:
//    	- "foo"
//      - "bar"
//
// Routes defines a 1:n stream remapping.
// Messages are reassigned to all of stream(s) in this list.
// If no route is set messages are forwarded on the incoming router.
// When routing to multiple streams, the incoming stream has to be listed explicitly to be used.
type Distribute struct {
	Broadcast
	streams []core.Router
}

func init() {
	core.TypeRegistry.Register(Distribute{})
}

// Configure initializes this distributor with values from a plugin config.
func (router *Distribute) Configure(conf core.PluginConfigReader) error {
	router.Broadcast.Configure(conf)

	boundStreamIDs := conf.GetStreamArray("TargetStreams", []core.MessageStreamID{})
	for _, streamID := range boundStreamIDs {
		targetRouter := core.StreamRegistry.GetRouterOrFallback(streamID)
		router.streams = append(router.streams, targetRouter)
	}

	return conf.Errors.OrNil()
}

func (router *Distribute) route(msg *core.Message, targetRouter core.Router) {
	if router.StreamID() == targetRouter.StreamID() {
		router.Broadcast.Enqueue(msg)
	} else {
		msg.SetStreamID(targetRouter.StreamID())
		core.Route(msg, targetRouter)
	}
}

// Enqueue enques a message to the router
func (router *Distribute) Enqueue(msg *core.Message) error {
	numStreams := len(router.streams)

	switch numStreams {
	case 0:
		return core.NewModulateResultError("No producers configured for stream %s", router.GetID())

	case 1:
		router.route(msg, router.streams[0])

	default:
		lastStreamIdx := numStreams - 1
		for streamIdx := 0; streamIdx < lastStreamIdx; streamIdx++ {
			router.route(msg.Clone(), router.streams[streamIdx])
			router.Log.Debug.Printf("routed to StreamID '%v'", router.streams[streamIdx].StreamID())
		}
		router.route(msg, router.streams[lastStreamIdx])
		router.Log.Debug.Printf("routed to StreamID '%v'", router.streams[lastStreamIdx].StreamID())
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// maintserve is a program that serves Go issues over HTTP, so they
// can be viewed in a browser. It uses x/build/maintner/godata as
// its backing source of data.
//
// It statically embeds all the resources it uses, so it's possible to use
// it when offline. During that time, the corpus will not be able to update,
// and GitHub user profile pictures won't load.
package main

import (
	"context"
	"flag"
	"fmt"
	"html/template"
	"log"
	"mime"
	"net/http"
	"net/url"
	"sort"
	"strings"
	"time"

	"github.com/shurcooL/gofontwoff"
	"github.com/shurcooL/httpgzip"
	"github.com/shurcooL/issues"
	maintnerissues "github.com/shurcooL/issues/maintner"
	"github.com/shurcooL/issuesapp"
	"golang.org/x/build/maintner"
	"golang.org/x/build/maintner/godata"
)

var httpFlag = flag.String("http", ":8080", "Listen for HTTP connections on this address.")

func main() {
	flag.Parse()

	err := run()
	if err != nil {
		log.Fatalln(err)
	}
}

func run() error {
	if err := mime.AddExtensionType(".woff2", "application/font-woff"); err != nil {
		return err
	}

	corpus, err := godata.Get(context.Background())
	if err != nil {
		return err
	}
	issuesService := maintnerissues.NewService(corpus)
	issuesApp := issuesapp.New(issuesService, nil, issuesapp.Options{
		HeadPre: `<meta name="viewport" content="width=device-width">
<link href="/assets/fonts/fonts.css" rel="stylesheet" type="text/css">
<link href="/assets/style.css" rel="stylesheet" type="text/css">`,
		HeadPost: ``,
		BodyPre: `

{{/* Override new comment component to link to original issue for leaving comments. */}}
{{define "new-comment"}}

	View original issue to comment.

{{end}}`,
		DisableReactions: true,
	})

	// TODO: Implement background updates for corpus while the appliation is running.
	//       Right now, it only updates at startup.
	//       It's likely just a matter of calling RLock/RUnlock before all read operations,
	//       and launching a background goroutine that occasionally calls corpus.Update()
	//       or corpus.Sync() or something.

	printServingAt(*httpFlag)
	err = http.ListenAndServe(*httpFlag, &handler{
		c:             corpus,
		fontsHandler:  httpgzip.FileServer(gofontwoff.Assets, httpgzip.FileServerOptions{}),
		issuesHandler: issuesApp,
	})
	return err
}

func printServingAt(addr string) {
	hostPort := addr
	if strings.HasPrefix(hostPort, ":") {
		hostPort = "localhost" + hostPort
	}
	fmt.Printf("serving at http://%s/\n", hostPort)
}

// handler handles all requests to maintserve. It acts like a request multiplexer,
// choosing from various endpoints and parsing the repository ID from URL.
type handler struct {
	c             *maintner.Corpus
	fontsHandler  http.Handler
	issuesHandler http.Handler
}

func (h *handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	// Handle "/".
	if req.URL.Path == "/" {
		h.serveIndex(w, req)
		return
	}

	// Handle "/assets/fonts/...".
	if strings.HasPrefix(req.URL.Path, "/assets/fonts") {
		req = stripPrefix(req, len("/assets/fonts"))
		h.fontsHandler.ServeHTTP(w, req)
		return
	}

	// Handle "/assets/style.css".
	if req.URL.Path == "/assets/style.css" {
		http.ServeContent(w, req, "style.css", time.Time{}, strings.NewReader(styleCSS))
		return
	}

	// Handle "/owner/repo/..." URLs.
	elems := strings.SplitN(req.URL.Path[1:], "/", 3)
	if len(elems) < 2 {
		http.Error(w, "404 Not Found", http.StatusNotFound)
		return
	}
	owner, repo := elems[0], elems[1]
	baseURLLen := 1 + len(owner) + 1 + len(repo) // Base URL is "/owner/repo".
	if baseURL := req.URL.Path[:baseURLLen]; req.URL.Path == baseURL+"/" {
		// Redirect "/owner/repo/" to "/owner/repo".
		if req.URL.RawQuery != "" {
			baseURL += "?" + req.URL.RawQuery
		}
		http.Redirect(w, req, baseURL, http.StatusFound)
		return
	}
	req = stripPrefix(req, baseURLLen)
	h.serveIssues(w, req, maintner.GitHubRepoID{Owner: owner, Repo: repo})
}

var indexHTML = template.Must(template.New("").Parse(`

		<title>maintserve</title>
		<meta name="viewport" content="width=device-width">
		<link href="/assets/fonts/fonts.css" rel="stylesheet" type="text/css">
		<link href="/assets/style.css" rel="stylesheet" type="text/css">

maintserve

Repos

{{range .}}

{{.RepoID}} ({{.Count}} issues)

				{{- end}}

`))

// serveIndex serves the index page, which lists all available repositories.
func (h *handler) serveIndex(w http.ResponseWriter, req *http.Request) {
	type repo struct {
		RepoID maintner.GitHubRepoID
		Count  uint64 // Issues count.
	}
	var repos []repo
	err := h.c.GitHub().ForeachRepo(func(r *maintner.GitHubRepo) error {
		issues, err := countIssues(r)
		if err != nil {
			return err
		}
		repos = append(repos, repo{
			RepoID: r.ID(),
			Count:  issues,
		})
		return nil
	})
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	sort.Slice(repos, func(i, j int) bool {
		return repos[i].RepoID.String() < repos[j].RepoID.String()
	})

	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	err = indexHTML.Execute(w, repos)
	if err != nil {
		log.Println(err)
	}
}

// countIssues reports the number of issues in a GitHubRepo r.
func countIssues(r *maintner.GitHubRepo) (uint64, error) {
	var issues uint64
	err := r.ForeachIssue(func(i *maintner.GitHubIssue) error {
		if i.NotExist {
			return nil
		}
		issues++
		return nil
	})
	return issues, err
}

// serveIssues serves issues for repository id.
func (h *handler) serveIssues(w http.ResponseWriter, req *http.Request, id maintner.GitHubRepoID) {
	if h.c.GitHub().Repo(id.Owner, id.Repo) == nil {
		http.Error(w, fmt.Sprintf("404 Not Found\n\nrepository %q not found", id), http.StatusNotFound)
		return
	}

	req = req.WithContext(context.WithValue(req.Context(),
		issuesapp.RepoSpecContextKey, issues.RepoSpec{URI: fmt.Sprintf("%s/%s", id.Owner, id.Repo)}))
	req = req.WithContext(context.WithValue(req.Context(),
		issuesapp.BaseURIContextKey, fmt.Sprintf("/%s/%s", id.Owner, id.Repo)))
	h.issuesHandler.ServeHTTP(w, req)
}

// stripPrefix returns request r with prefix of length prefixLen stripped from r.URL.Path.
// prefixLen must not be longer than len(r.URL.Path), otherwise stripPrefix panics.
// If r.URL.Path is empty after the prefix is stripped, the path is changed to "/".
func stripPrefix(r *http.Request, prefixLen int) *http.Request {
	r2 := new(http.Request)
	*r2 = *r
	r2.URL = new(url.URL)
	*r2.URL = *r.URL
	r2.URL.Path = r.URL.Path[prefixLen:]
	if r2.URL.Path == "" {
		r2.URL.Path = "/"
	}
	return r2
}

const styleCSS = `body {
	margin: 20px;
	font-family: Go;
	font-size: 14px;
	line-height: initial;
	color: #373a3c;
}
a {
	color: #0275d8;
	text-decoration: none;
}
a:focus, a:hover {
	color: #014c8c;
	text-decoration: underline;
}
.btn {
	font-family: inherit;
	font-size: 11px;
	line-height: 11px;
	height: 18px;
	border-radius: 4px;
	border: solid #d2d2d2 1px;
	background-color: #fff;
	box-shadow: 0 1px 1px rgba(0, 0, 0, .05);
}`
<commit_msg>maintner/cmd/maintserve: don't count PRs as issues<commit_after>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// maintserve is a program that serves Go issues over HTTP, so they
// can be viewed in a browser. It uses x/build/maintner/godata as
// its backing source of data.
//
// It statically embeds all the resources it uses, so it's possible to use
// it when offline. During that time, the corpus will not be able to update,
// and GitHub user profile pictures won't load.
package main

import (
	"context"
	"flag"
	"fmt"
	"html/template"
	"log"
	"mime"
	"net/http"
	"net/url"
	"sort"
	"strings"
	"time"

	"github.com/shurcooL/gofontwoff"
	"github.com/shurcooL/httpgzip"
	"github.com/shurcooL/issues"
	maintnerissues "github.com/shurcooL/issues/maintner"
	"github.com/shurcooL/issuesapp"
	"golang.org/x/build/maintner"
	"golang.org/x/build/maintner/godata"
)

var httpFlag = flag.String("http", ":8080", "Listen for HTTP connections on this address.")

func main() {
	flag.Parse()

	err := run()
	if err != nil {
		log.Fatalln(err)
	}
}

func run() error {
	if err := mime.AddExtensionType(".woff2", "application/font-woff"); err != nil {
		return err
	}

	corpus, err := godata.Get(context.Background())
	if err != nil {
		return err
	}
	issuesService := maintnerissues.NewService(corpus)
	issuesApp := issuesapp.New(issuesService, nil, issuesapp.Options{
		HeadPre: `<meta name="viewport" content="width=device-width">
<link href="/assets/fonts/fonts.css" rel="stylesheet" type="text/css">
<link href="/assets/style.css" rel="stylesheet" type="text/css">`,
		HeadPost: ``,
		BodyPre: `

{{/* Override new comment component to link to original issue for leaving comments. */}}
{{define "new-comment"}}

	View original issue to comment.

{{end}}`,
		DisableReactions: true,
	})

	// TODO: Implement background updates for corpus while the appliation is running.
	//       Right now, it only updates at startup.
	//       It's likely just a matter of calling RLock/RUnlock before all read operations,
	//       and launching a background goroutine that occasionally calls corpus.Update()
	//       or corpus.Sync() or something.

	printServingAt(*httpFlag)
	err = http.ListenAndServe(*httpFlag, &handler{
		c:             corpus,
		fontsHandler:  httpgzip.FileServer(gofontwoff.Assets, httpgzip.FileServerOptions{}),
		issuesHandler: issuesApp,
	})
	return err
}

func printServingAt(addr string) {
	hostPort := addr
	if strings.HasPrefix(hostPort, ":") {
		hostPort = "localhost" + hostPort
	}
	fmt.Printf("serving at http://%s/\n", hostPort)
}

// handler handles all requests to maintserve. It acts like a request multiplexer,
// choosing from various endpoints and parsing the repository ID from URL.
type handler struct {
	c             *maintner.Corpus
	fontsHandler  http.Handler
	issuesHandler http.Handler
}

func (h *handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	// Handle "/".
	if req.URL.Path == "/" {
		h.serveIndex(w, req)
		return
	}

	// Handle "/assets/fonts/...".
	if strings.HasPrefix(req.URL.Path, "/assets/fonts") {
		req = stripPrefix(req, len("/assets/fonts"))
		h.fontsHandler.ServeHTTP(w, req)
		return
	}

	// Handle "/assets/style.css".
	if req.URL.Path == "/assets/style.css" {
		http.ServeContent(w, req, "style.css", time.Time{}, strings.NewReader(styleCSS))
		return
	}

	// Handle "/owner/repo/..." URLs.
	elems := strings.SplitN(req.URL.Path[1:], "/", 3)
	if len(elems) < 2 {
		http.Error(w, "404 Not Found", http.StatusNotFound)
		return
	}
	owner, repo := elems[0], elems[1]
	baseURLLen := 1 + len(owner) + 1 + len(repo) // Base URL is "/owner/repo".
	if baseURL := req.URL.Path[:baseURLLen]; req.URL.Path == baseURL+"/" {
		// Redirect "/owner/repo/" to "/owner/repo".
		if req.URL.RawQuery != "" {
			baseURL += "?" + req.URL.RawQuery
		}
		http.Redirect(w, req, baseURL, http.StatusFound)
		return
	}
	req = stripPrefix(req, baseURLLen)
	h.serveIssues(w, req, maintner.GitHubRepoID{Owner: owner, Repo: repo})
}

var indexHTML = template.Must(template.New("").Parse(`

		<title>maintserve</title>
		<meta name="viewport" content="width=device-width">
		<link href="/assets/fonts/fonts.css" rel="stylesheet" type="text/css">
		<link href="/assets/style.css" rel="stylesheet" type="text/css">

maintserve

Repos

{{range .}}

{{.RepoID}} ({{.Count}} issues)

				{{- end}}

`))

// serveIndex serves the index page, which lists all available repositories.
func (h *handler) serveIndex(w http.ResponseWriter, req *http.Request) {
	type repo struct {
		RepoID maintner.GitHubRepoID
		Count  uint64 // Issues count.
	}
	var repos []repo
	err := h.c.GitHub().ForeachRepo(func(r *maintner.GitHubRepo) error {
		issues, err := countIssues(r)
		if err != nil {
			return err
		}
		repos = append(repos, repo{
			RepoID: r.ID(),
			Count:  issues,
		})
		return nil
	})
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	sort.Slice(repos, func(i, j int) bool {
		return repos[i].RepoID.String() < repos[j].RepoID.String()
	})

	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	err = indexHTML.Execute(w, repos)
	if err != nil {
		log.Println(err)
	}
}

// countIssues reports the number of issues in a GitHubRepo r.
func countIssues(r *maintner.GitHubRepo) (uint64, error) {
	var issues uint64
	err := r.ForeachIssue(func(i *maintner.GitHubIssue) error {
		if i.NotExist || i.PullRequest {
			return nil
		}
		issues++
		return nil
	})
	return issues, err
}

// serveIssues serves issues for repository id.
func (h *handler) serveIssues(w http.ResponseWriter, req *http.Request, id maintner.GitHubRepoID) {
	if h.c.GitHub().Repo(id.Owner, id.Repo) == nil {
		http.Error(w, fmt.Sprintf("404 Not Found\n\nrepository %q not found", id), http.StatusNotFound)
		return
	}

	req = req.WithContext(context.WithValue(req.Context(),
		issuesapp.RepoSpecContextKey, issues.RepoSpec{URI: fmt.Sprintf("%s/%s", id.Owner, id.Repo)}))
	req = req.WithContext(context.WithValue(req.Context(),
		issuesapp.BaseURIContextKey, fmt.Sprintf("/%s/%s", id.Owner, id.Repo)))
	h.issuesHandler.ServeHTTP(w, req)
}

// stripPrefix returns request r with prefix of length prefixLen stripped from r.URL.Path.
// prefixLen must not be longer than len(r.URL.Path), otherwise stripPrefix panics.
// If r.URL.Path is empty after the prefix is stripped, the path is changed to "/".
func stripPrefix(r *http.Request, prefixLen int) *http.Request {
	r2 := new(http.Request)
	*r2 = *r
	r2.URL = new(url.URL)
	*r2.URL = *r.URL
	r2.URL.Path = r.URL.Path[prefixLen:]
	if r2.URL.Path == "" {
		r2.URL.Path = "/"
	}
	return r2
}

const styleCSS = `body {
	margin: 20px;
	font-family: Go;
	font-size: 14px;
	line-height: initial;
	color: #373a3c;
}
a {
	color: #0275d8;
	text-decoration: none;
}
a:focus, a:hover {
	color: #014c8c;
	text-decoration: underline;
}
.btn {
	font-family: inherit;
	font-size: 11px;
	line-height: 11px;
	height: 18px;
	border-radius: 4px;
	border: solid #d2d2d2 1px;
	background-color: #fff;
	box-shadow: 0 1px 1px rgba(0, 0, 0, .05);
}`
<|endoftext|>


<commit_before>package buildinfo

import (
	"path/filepath"

	"github.com/admpub/nging/v4/application/initialize/backend"
	"github.com/admpub/nging/v4/application/library/module"
	"github.com/webx-top/echo/middleware/render/driver"
)

func SetNgingDir(ngingDir string) {
	backend.AssetsDir = filepath.Join(ngingDir, backend.DefaultAssetsDir)
	backend.TemplateDir = filepath.Join(ngingDir, backend.DefaultTemplateDir)
}

func SetNgingPluginsDir(ngingPluginsDir string) {
	module.NgingPluginDir = ngingPluginsDir
}

func WatchTemplateDir(templateDirs ...string) {
	rendererDo := backend.RendererDo
	backend.RendererDo = func(renderer driver.Driver) {
		rendererDo(renderer)
		for _, templateDir := range templateDirs {
			renderer.Manager().AddWatchDir(templateDir)
		}
	}
}
<commit_msg>update<commit_after>package nginginfo

import (
	"path/filepath"

	"github.com/admpub/nging/v4/application/initialize/backend"
	"github.com/admpub/nging/v4/application/library/module"
	"github.com/webx-top/echo/middleware/render/driver"
)

func SetNgingDir(ngingDir string) {
	backend.AssetsDir = filepath.Join(ngingDir, backend.DefaultAssetsDir)
	backend.TemplateDir = filepath.Join(ngingDir, backend.DefaultTemplateDir)
}

func SetNgingPluginsDir(ngingPluginsDir string) {
	module.NgingPluginDir = ngingPluginsDir
}

func WatchTemplateDir(templateDirs ...string) {
	rendererDo := backend.RendererDo
	backend.RendererDo = func(renderer driver.Driver) {
		rendererDo(renderer)
		for _, templateDir := range templateDirs {
			renderer.Manager().AddWatchDir(templateDir)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 Pulcy.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package backend

import (
	"encoding/json"
	"fmt"
	"net/url"
	"path"
	"strconv"
	"strings"

	"github.com/coreos/etcd/client"
	"github.com/op/go-logging"
	"golang.org/x/net/context"
)

const (
	servicePrefix  = "service"
	frontEndPrefix = "frontend"
)

type etcdBackend struct {
	client  client.Client
	watcher client.Watcher
	Logger  *logging.Logger
	prefix  string
}

func NewEtcdBackend(logger *logging.Logger, uri *url.URL) (Backend, error) {
	cfg := client.Config{
		Transport: client.DefaultTransport,
	}
	if uri.Host != "" {
		cfg.Endpoints = append(cfg.Endpoints, "http://"+uri.Host)
	}
	c, err := client.New(cfg)
	if err != nil {
		return nil, maskAny(err)
	}
	kAPI := client.NewKeysAPI(c)
	options := &client.WatcherOptions{
		Recursive: true,
	}
	watcher := kAPI.Watcher(uri.Path, options)
	return &etcdBackend{
		client:  c,
		watcher: watcher,
		prefix:  uri.Path,
		Logger:  logger,
	}, nil
}

// Watch for changes on a path and return where there is a change.
func (eb *etcdBackend) Watch() error {
	_, err := eb.watcher.Next(context.Background())
	if err != nil {
		return maskAny(err)
	}
	return nil
}

// Load all registered services
func (eb *etcdBackend) Services() (ServiceRegistrations, error) {
	servicesTree, err := eb.readServicesTree()
	if err != nil {
		return nil, maskAny(err)
	}
	frontEndTree, err := eb.readFrontEndsTree()
	if err != nil {
		return nil, maskAny(err)
	}
	result, err := eb.mergeTrees(servicesTree, frontEndTree)
	if err != nil {
		return nil, maskAny(err)
	}
	return result, nil
}

// Load all registered services
func (eb *etcdBackend) readServicesTree() (ServiceRegistrations, error) {
	etcdPath := path.Join(eb.prefix, servicePrefix)
	kAPI := client.NewKeysAPI(eb.client)
	options := &client.GetOptions{
		Recursive: true,
		Sort:      false,
	}
	resp, err := kAPI.Get(context.Background(), etcdPath, options)
	if err != nil {
		return nil, maskAny(err)
	}
	list := ServiceRegistrations{}
	if resp.Node == nil {
		return list, nil
	}
	for _, serviceNode := range resp.Node.Nodes {
		serviceName := path.Base(serviceNode.Key)
		registrations := make(map[int]*ServiceRegistration)
		for _, instanceNode := range serviceNode.Nodes {
			uniqueID := path.Base(instanceNode.Key)
			parts := strings.Split(uniqueID, ":")
			if len(parts) < 3 {
				eb.Logger.Warning("UniqueID malformed: '%s'", uniqueID)
				continue
			}
			port, err := strconv.Atoi(parts[2])
			if err != nil {
				eb.Logger.Warning("Failed to parse port: '%s'", parts[2])
				continue
			}
			instance, err := eb.parseServiceInstance(instanceNode.Value)
			if err != nil {
				eb.Logger.Warning("Failed to parse instance '%s': %#v", instanceNode.Value, err)
				continue
			}
			sr, ok := registrations[port]
			if !ok {
				sr = &ServiceRegistration{ServiceName: serviceName, ServicePort: port}
				registrations[port] = sr
			}
			sr.Instances = append(sr.Instances, instance)

			// Register instance as separate service
			instanceName := parts[1]
			if strings.HasPrefix(instanceName, serviceName+"-") {
				sr := ServiceRegistration{ServiceName: instanceName, ServicePort: port}
				sr.Instances = append(sr.Instances, instance)
				list = append(list, sr)
			}
		}
		for _, v := range registrations {
			list = append(list, *v)
		}
	}

	return list, nil
}

// parseServiceInstance parses a string in the format of "<ip>':'<port>" into a ServiceInstance.
func (eb *etcdBackend) parseServiceInstance(s string) (ServiceInstance, error) {
	parts := strings.Split(s, ":")
	if len(parts) != 2 {
		return ServiceInstance{}, maskAny(fmt.Errorf("Invalid service instance '%s'", s))
	}
	port, err := strconv.Atoi(parts[1])
	if err != nil {
		return ServiceInstance{}, maskAny(fmt.Errorf("Invalid service instance port '%s' in '%s'", parts[1], s))
	}
	return ServiceInstance{
		IP:   parts[0],
		Port: port,
	}, nil
}

type frontendRecord struct {
	Selectors     []frontendSelectorRecord `json:"selectors"`
	Service       string                   `json:"service,omitempty"`
	Mode          string                   `json:"mode,omitempty"` // http|tcp
	HttpCheckPath string                   `json:"http-check-path,omitempty"`
}

type frontendSelectorRecord struct {
	Weight     int          `json:"weight,omitempty"`
	Domain     string       `json:"domain,omitempty"`
	SslCert    string       `json:"ssl-cert,omitempty"`
	PathPrefix string       `json:"path-prefix,omitempty"`
	Port       int          `json:"port,omitempty"`
	Private    bool         `json:"private,omitempty"`
	Users      []userRecord `json:"users,omitempty"`
}

type userRecord struct {
	Name         string `json:"user"`
	PasswordHash string `json:"pwhash"`
}

// Load all registered front-ends
func (eb *etcdBackend) readFrontEndsTree() ([]frontendRecord, error) {
	etcdPath := path.Join(eb.prefix, frontEndPrefix)
	kAPI := client.NewKeysAPI(eb.client)
	options := &client.GetOptions{
		Recursive: false,
		Sort:      false,
	}
	resp, err := kAPI.Get(context.Background(), etcdPath, options)
	if err != nil {
		return nil, maskAny(err)
	}
	list := []frontendRecord{}
	if resp.Node == nil {
		return list, nil
	}
	for _, frontEndNode := range resp.Node.Nodes {
		rawJson := frontEndNode.Value
		record := frontendRecord{}
		if err := json.Unmarshal([]byte(rawJson), &record); err != nil {
			eb.Logger.Errorf("Cannot unmarshal registration of %s", frontEndNode.Key)
			continue
		}
		list = append(list, record)
	}

	return list, nil
}

// mergeTrees merges the 2 trees into a single list of registrations.
func (eb *etcdBackend) mergeTrees(services ServiceRegistrations, frontends []frontendRecord) (ServiceRegistrations, error) {
	result := ServiceRegistrations{}
	for _, service := range services {
		for _, fr := range frontends {
			if service.ServiceName != fr.Service {
				continue
			}
			if fr.HttpCheckPath != "" && service.HttpCheckPath == "" {
				service.HttpCheckPath = fr.HttpCheckPath
			}
			if fr.Mode != "" && service.Mode == "" {
				service.Mode = fr.Mode
			}
			for _, sel := range fr.Selectors {
				if sel.Port != 0 && sel.Port != service.ServicePort {
					continue
				}
				srSel := ServiceSelector{
					Weight:      sel.Weight,
					Domain:      sel.Domain,
					SslCertName: sel.SslCert,
					PathPrefix:  sel.PathPrefix,
					Private:     sel.Private,
				}
				for _, user := range sel.Users {
					srSel.Users = append(srSel.Users, User{
						Name:         user.Name,
						PasswordHash: user.PasswordHash,
					})
				}
				if !service.Selectors.Contains(srSel) {
					service.Selectors = append(service.Selectors, srSel)
				}
			}
		}
		if len(service.Selectors) > 0 {
			if service.Mode == "" {
				service.Mode = "http"
			}
			result = append(result, service)
		}
	}
	return result, nil
}
<commit_msg>Reset ETCD watcher after too many errors<commit_after>// Copyright (c) 2016 Pulcy.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package backend

import (
	"encoding/json"
	"fmt"
	"net/url"
	"path"
	"strconv"
	"strings"

	"github.com/coreos/etcd/client"
	"github.com/op/go-logging"
	"golang.org/x/net/context"
)

const (
	servicePrefix        = "service"
	frontEndPrefix       = "frontend"
	recentWatchErrorsMax = 5
)

type etcdBackend struct {
	client            client.Client
	watcher           client.Watcher
	Logger            *logging.Logger
	prefix            string
	recentWatchErrors int
}

func NewEtcdBackend(logger *logging.Logger, uri *url.URL) (Backend, error) {
	cfg := client.Config{
		Transport: client.DefaultTransport,
	}
	if uri.Host != "" {
		cfg.Endpoints = append(cfg.Endpoints, "http://"+uri.Host)
	}
	c, err := client.New(cfg)
	if err != nil {
		return nil, maskAny(err)
	}
	kAPI := client.NewKeysAPI(c)
	options := &client.WatcherOptions{
		Recursive: true,
	}
	watcher := kAPI.Watcher(uri.Path, options)
	return &etcdBackend{
		client:  c,
		watcher: watcher,
		prefix:  uri.Path,
		Logger:  logger,
	}, nil
}

// Watch for changes on a path and return where there is a change.
func (eb *etcdBackend) Watch() error {
	if eb.watcher == nil || eb.recentWatchErrors > recentWatchErrorsMax {
		eb.recentWatchErrors = 0
		kAPI := client.NewKeysAPI(eb.client)
		options := &client.WatcherOptions{
			Recursive: true,
		}
		eb.watcher = kAPI.Watcher(eb.prefix, options)
	}
	_, err := eb.watcher.Next(context.Background())
	if err != nil {
		return maskAny(err)
	}
	eb.recentWatchErrors = 0
	return nil
}

// Load all registered services
func (eb *etcdBackend) Services() (ServiceRegistrations, error) {
	servicesTree, err := eb.readServicesTree()
	if err != nil {
		return nil, maskAny(err)
	}
	frontEndTree, err := eb.readFrontEndsTree()
	if err != nil {
		return nil, maskAny(err)
	}
	result, err := eb.mergeTrees(servicesTree, frontEndTree)
	if err != nil {
		return nil, maskAny(err)
	}
	return result, nil
}

// Load all registered services
func (eb *etcdBackend) readServicesTree() (ServiceRegistrations, error) {
	etcdPath := path.Join(eb.prefix, servicePrefix)
	kAPI := client.NewKeysAPI(eb.client)
	options := &client.GetOptions{
		Recursive: true,
		Sort:      false,
	}
	resp, err := kAPI.Get(context.Background(), etcdPath, options)
	if err != nil {
		return nil, maskAny(err)
	}
	list := ServiceRegistrations{}
	if resp.Node == nil {
		return list, nil
	}
	for _, serviceNode := range resp.Node.Nodes {
		serviceName := path.Base(serviceNode.Key)
		registrations := make(map[int]*ServiceRegistration)
		for _, instanceNode := range serviceNode.Nodes {
			uniqueID := path.Base(instanceNode.Key)
			parts := strings.Split(uniqueID, ":")
			if len(parts) < 3 {
				eb.Logger.Warning("UniqueID malformed: '%s'", uniqueID)
				continue
			}
			port, err := strconv.Atoi(parts[2])
			if err != nil {
				eb.Logger.Warning("Failed to parse port: '%s'", parts[2])
				continue
			}
			instance, err := eb.parseServiceInstance(instanceNode.Value)
			if err != nil {
				eb.Logger.Warning("Failed to parse instance '%s': %#v", instanceNode.Value, err)
				continue
			}
			sr, ok := registrations[port]
			if !ok {
				sr = &ServiceRegistration{ServiceName: serviceName, ServicePort: port}
				registrations[port] = sr
			}
			sr.Instances = append(sr.Instances, instance)

			// Register instance as separate service
			instanceName := parts[1]
			if strings.HasPrefix(instanceName, serviceName+"-") {
				sr := ServiceRegistration{ServiceName: instanceName, ServicePort: port}
				sr.Instances = append(sr.Instances, instance)
				list = append(list, sr)
			}
		}
		for _, v := range registrations {
			list = append(list, *v)
		}
	}

	return list, nil
}

// parseServiceInstance parses a string in the format of "<ip>':'<port>" into a ServiceInstance.
func (eb *etcdBackend) parseServiceInstance(s string) (ServiceInstance, error) {
	parts := strings.Split(s, ":")
	if len(parts) != 2 {
		return ServiceInstance{}, maskAny(fmt.Errorf("Invalid service instance '%s'", s))
	}
	port, err := strconv.Atoi(parts[1])
	if err != nil {
		return ServiceInstance{}, maskAny(fmt.Errorf("Invalid service instance port '%s' in '%s'", parts[1], s))
	}
	return ServiceInstance{
		IP:   parts[0],
		Port: port,
	}, nil
}

type frontendRecord struct {
	Selectors     []frontendSelectorRecord `json:"selectors"`
	Service       string                   `json:"service,omitempty"`
	Mode          string                   `json:"mode,omitempty"` // http|tcp
	HttpCheckPath string                   `json:"http-check-path,omitempty"`
}

type frontendSelectorRecord struct {
	Weight     int          `json:"weight,omitempty"`
	Domain     string       `json:"domain,omitempty"`
	SslCert    string       `json:"ssl-cert,omitempty"`
	PathPrefix string       `json:"path-prefix,omitempty"`
	Port       int          `json:"port,omitempty"`
	Private    bool         `json:"private,omitempty"`
	Users      []userRecord `json:"users,omitempty"`
}

type userRecord struct {
	Name         string `json:"user"`
	PasswordHash string `json:"pwhash"`
}

// Load all registered front-ends
func (eb *etcdBackend) readFrontEndsTree() ([]frontendRecord, error) {
	etcdPath := path.Join(eb.prefix, frontEndPrefix)
	kAPI := client.NewKeysAPI(eb.client)
	options := &client.GetOptions{
		Recursive: false,
		Sort:      false,
	}
	resp, err := kAPI.Get(context.Background(), etcdPath, options)
	if err != nil {
		return nil, maskAny(err)
	}
	list := []frontendRecord{}
	if resp.Node == nil {
		return list, nil
	}
	for _, frontEndNode := range resp.Node.Nodes {
		rawJson := frontEndNode.Value
		record := frontendRecord{}
		if err := json.Unmarshal([]byte(rawJson), &record); err != nil {
			eb.Logger.Errorf("Cannot unmarshal registration of %s", frontEndNode.Key)
			continue
		}
		list = append(list, record)
	}

	return list, nil
}

// mergeTrees merges the 2 trees into a single list of registrations.
func (eb *etcdBackend) mergeTrees(services ServiceRegistrations, frontends []frontendRecord) (ServiceRegistrations, error) {
	result := ServiceRegistrations{}
	for _, service := range services {
		for _, fr := range frontends {
			if service.ServiceName != fr.Service {
				continue
			}
			if fr.HttpCheckPath != "" && service.HttpCheckPath == "" {
				service.HttpCheckPath = fr.HttpCheckPath
			}
			if fr.Mode != "" && service.Mode == "" {
				service.Mode = fr.Mode
			}
			for _, sel := range fr.Selectors {
				if sel.Port != 0 && sel.Port != service.ServicePort {
					continue
				}
				srSel := ServiceSelector{
					Weight:      sel.Weight,
					Domain:      sel.Domain,
					SslCertName: sel.SslCert,
					PathPrefix:  sel.PathPrefix,
					Private:     sel.Private,
				}
				for _, user := range sel.Users {
					srSel.Users = append(srSel.Users, User{
						Name:         user.Name,
						PasswordHash: user.PasswordHash,
					})
				}
				if !service.Selectors.Contains(srSel) {
					service.Selectors = append(service.Selectors, srSel)
				}
			}
		}
		if len(service.Selectors) > 0 {
			if service.Mode == "" {
				service.Mode = "http"
			}
			result = append(result, service)
		}
	}
	return result, nil
}
<|endoftext|>


<commit_before>// Copyright 2013 Hui Chen
// Copyright 2016 ego authors
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package types

import (
	"github.com/go-ego/riot/utils"
)

// BaseResp search response options
type BaseResp struct {
	// 搜索用到的关键词
	Tokens []string

	// 类别
	// Class string

	// 搜索到的文档，已排序
	// Docs []ScoredDoc
	// Docs interface{}

	// 搜索是否超时。超时的情况下也可能会返回部分结果
	Timeout bool

	// 搜索到的文档个数。注意这是全部文档中满足条件的个数，可能比返回的文档数要大
	NumDocs int
}

// SearchResp search response options
type SearchResp struct {
	BaseResp
	// 搜索到的文档，已排序
	Docs interface{}
}

// SearchDoc search response options
type SearchDoc struct {
	BaseResp
	// 搜索到的文档，已排序
	Docs []ScoredDoc
}

// SearchID search response options
type SearchID struct {
	BaseResp
	// 搜索到的文档，已排序
	Docs []ScoredID
}

// Content search content
type Content struct {
	// new Content
	Content string

	// new 属性 Attri
	Attri interface{}

	// new 返回评分字段
	Fields interface{}
}

// ScoredDoc scored the document
type ScoredDoc struct {
	ScoredID

	// new 返回文档 Content
	Content string
	// new 返回文档属性 Attri
	Attri interface{}
	// new 返回评分字段
	Fields interface{}
}

// ScoredDocs 为了方便排序
type ScoredDocs []ScoredDoc

func (docs ScoredDocs) Len() int {
	return len(docs)
}

func (docs ScoredDocs) Swap(i, j int) {
	docs[i], docs[j] = docs[j], docs[i]
}

func (docs ScoredDocs) Less(i, j int) bool {
	// 为了从大到小排序，这实际上实现的是 More 的功能
	for iScore := 0; iScore < utils.MinInt(len(docs[i].Scores), len(docs[j].Scores)); iScore++ {
		if docs[i].Scores[iScore] > docs[j].Scores[iScore] {
			return true
		} else if docs[i].Scores[iScore] < docs[j].Scores[iScore] {
			return false
		}
	}
	return len(docs[i].Scores) > len(docs[j].Scores)
}

/*
  ______   .__   __.  __      ____    ____  __   _______
 /  __  \  |  \ |  | |  |     \   \  /   / |  | |       \
|  |  |  | |   \|  | |  |      \   \/   /  |  | |  .--.  |
|  |  |  | |  . `  | |  |       \_    _/   |  | |  |  |  |
|  `--'  | |  |\   | |  `----.    |  |     |  | |  '--'  |
 \______/  |__| \__| |_______|    |__|     |__| |_______/

*/

// ScoredID scored doc only id
type ScoredID struct {
	DocId uint64

	// 文档的打分值
	// 搜索结果按照 Scores 的值排序，先按照第一个数排，
	// 如果相同则按照第二个数排序，依次类推。
	Scores []float32

	// 用于生成摘要的关键词在文本中的字节位置，
	// 该切片长度和 SearchResp.Tokens 的长度一样
	// 只有当 IndexType == LocsIndex 时不为空
	TokenSnippetLocs []int

	// 关键词出现的位置
	// 只有当 IndexType == LocsIndex 时不为空
	TokenLocs [][]int
}

// ScoredIDs 为了方便排序
type ScoredIDs []ScoredID

func (docs ScoredIDs) Len() int {
	return len(docs)
}

func (docs ScoredIDs) Swap(i, j int) {
	docs[i], docs[j] = docs[j], docs[i]
}

func (docs ScoredIDs) Less(i, j int) bool {
	// 为了从大到小排序，这实际上实现的是 More 的功能
	for iScore := 0; iScore < utils.MinInt(len(docs[i].Scores), len(docs[j].Scores)); iScore++ {
		if docs[i].Scores[iScore] > docs[j].Scores[iScore] {
			return true
		} else if docs[i].Scores[iScore] < docs[j].Scores[iScore] {
			return false
		}
	}
	return len(docs[i].Scores) > len(docs[j].Scores)
}
<commit_msg>optimize search sort code<commit_after>// Copyright 2013 Hui Chen
// Copyright 2016 ego authors
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package types

import (
	"github.com/go-ego/riot/utils"
)

// BaseResp search response options
type BaseResp struct {
	// 搜索用到的关键词
	Tokens []string

	// 类别
	// Class string

	// 搜索到的文档，已排序
	// Docs []ScoredDoc
	// Docs interface{}

	// 搜索是否超时。超时的情况下也可能会返回部分结果
	Timeout bool

	// 搜索到的文档个数。注意这是全部文档中满足条件的个数，可能比返回的文档数要大
	NumDocs int
}

// SearchResp search response options
type SearchResp struct {
	BaseResp
	// 搜索到的文档，已排序
	Docs interface{}
}

// SearchDoc search response options
type SearchDoc struct {
	BaseResp
	// 搜索到的文档，已排序
	Docs []ScoredDoc
}

// SearchID search response options
type SearchID struct {
	BaseResp
	// 搜索到的文档，已排序
	Docs []ScoredID
}

// Content search content
type Content struct {
	// new Content
	Content string

	// new 属性 Attri
	Attri interface{}

	// new 返回评分字段
	Fields interface{}
}

// ScoredDoc scored the document
type ScoredDoc struct {
	ScoredID

	// new 返回文档 Content
	Content string
	// new 返回文档属性 Attri
	Attri interface{}
	// new 返回评分字段
	Fields interface{}
}

// ScoredDocs 为了方便排序
type ScoredDocs []ScoredDoc

func (docs ScoredDocs) Len() int {
	return len(docs)
}

func (docs ScoredDocs) Swap(i, j int) {
	docs[i], docs[j] = docs[j], docs[i]
}

func (docs ScoredDocs) Less(i, j int) bool {
	// 为了从大到小排序，这实际上实现的是 More 的功能
	min := utils.MinInt(len(docs[i].Scores), len(docs[j].Scores))
	for iScore := 0; iScore < min; iScore++ {
		if docs[i].Scores[iScore] > docs[j].Scores[iScore] {
			return true
		} else if docs[i].Scores[iScore] < docs[j].Scores[iScore] {
			return false
		}
	}
	return len(docs[i].Scores) > len(docs[j].Scores)
}

/*
  ______   .__   __.  __      ____    ____  __   _______
 /  __  \  |  \ |  | |  |     \   \  /   / |  | |       \
|  |  |  | |   \|  | |  |      \   \/   /  |  | |  .--.  |
|  |  |  | |  . `  | |  |       \_    _/   |  | |  |  |  |
|  `--'  | |  |\   | |  `----.    |  |     |  | |  '--'  |
 \______/  |__| \__| |_______|    |__|     |__| |_______/

*/

// ScoredID scored doc only id
type ScoredID struct {
	DocId uint64

	// 文档的打分值
	// 搜索结果按照 Scores 的值排序，先按照第一个数排，
	// 如果相同则按照第二个数排序，依次类推。
	Scores []float32

	// 用于生成摘要的关键词在文本中的字节位置，
	// 该切片长度和 SearchResp.Tokens 的长度一样
	// 只有当 IndexType == LocsIndex 时不为空
	TokenSnippetLocs []int

	// 关键词出现的位置
	// 只有当 IndexType == LocsIndex 时不为空
	TokenLocs [][]int
}

// ScoredIDs 为了方便排序
type ScoredIDs []ScoredID

func (docs ScoredIDs) Len() int {
	return len(docs)
}

func (docs ScoredIDs) Swap(i, j int) {
	docs[i], docs[j] = docs[j], docs[i]
}

func (docs ScoredIDs) Less(i, j int) bool {
	// 为了从大到小排序，这实际上实现的是 More 的功能
	min := utils.MinInt(len(docs[i].Scores), len(docs[j].Scores))
	for iScore := 0; iScore < min; iScore++ {
		if docs[i].Scores[iScore] > docs[j].Scores[iScore] {
			return true
		} else if docs[i].Scores[iScore] < docs[j].Scores[iScore] {
			return false
		}
	}
	return len(docs[i].Scores) > len(docs[j].Scores)
}
<|endoftext|>


<commit_before>// Copyright 2012 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Author: jsing@google.com (Joel Sing)

package watchdog

import (
	"fmt"
	"io"
	"os"
	"os/user"
	"path"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"

	log "github.com/golang/glog"
)

const logDir = "/var/log/seesaw"

const prioProcess = 0

// Service contains the data needed to manage a service.
type Service struct {
	name   string
	binary string
	path   string
	args   []string

	uid      uint32
	gid      uint32
	priority int

	dependencies map[string]*Service
	dependents   map[string]*Service

	termTimeout time.Duration

	lock    sync.Mutex
	process *os.Process

	done     chan bool
	shutdown chan bool
	started  chan bool
	stopped  chan bool

	failures uint64
	restarts uint64

	lastFailure time.Time
	lastRestart time.Time
}

// newService returns an initialised service.
func newService(name, binary string) *Service {
	return &Service{
		name:         name,
		binary:       binary,
		args:         make([]string, 0),
		dependencies: make(map[string]*Service),
		dependents:   make(map[string]*Service),

		done:     make(chan bool),
		shutdown: make(chan bool, 1),
		started:  make(chan bool, 1),
		stopped:  make(chan bool, 1),

		termTimeout: 5 * time.Second,
	}
}

// AddDependency registers a dependency for this service.
func (svc *Service) AddDependency(name string) {
	svc.dependencies[name] = nil
}

// AddArgs adds the given string as arguments.
func (svc *Service) AddArgs(args string) {
	svc.args = strings.Fields(args)
}

// SetPriority sets the process priority for a service.
func (svc *Service) SetPriority(priority int) error {
	if priority < -20 || priority > 19 {
		return fmt.Errorf("Invalid priority %d - must be between -20 and 19", priority)
	}
	svc.priority = priority
	return nil
}

// SetTermTimeout sets the termination timeout for a service.
func (svc *Service) SetTermTimeout(tt time.Duration) {
	svc.termTimeout = tt
}

// SetUser sets the user for a service.
func (svc *Service) SetUser(username string) error {
	u, err := user.Lookup(username)
	if err != nil {
		return err
	}
	uid, err := strconv.Atoi(u.Uid)
	if err != nil {
		return err
	}
	gid, err := strconv.Atoi(u.Gid)
	if err != nil {
		return err
	}
	svc.uid = uint32(uid)
	svc.gid = uint32(gid)
	return nil
}

// run runs a service and restarts it upon termination, unless a shutdown
// notification has been received.
func (svc *Service) run() {

	// Wait for dependencies to start.
	for _, dep := range svc.dependencies {
		log.Infof("Service %s waiting for %s to start", svc.name, dep.name)
		select {
		case started := <-dep.started:
			dep.started <- started
		case <-svc.shutdown:
			goto done
		}
	}

	for {
		if svc.failures > 0 {
			delay := time.Duration(svc.failures) * restartBackoff
			if delay > restartBackoffMax {
				delay = restartBackoffMax
			}
			log.Infof("Service %s has failed %d times - delaying %s before restart",
				svc.name, svc.failures, delay)

			select {
			case <-time.After(delay):
			case <-svc.shutdown:
				goto done
			}
		}

		svc.restarts++
		svc.lastRestart = time.Now()
		svc.runOnce()

		select {
		case <-time.After(restartDelay):
		case <-svc.shutdown:
			goto done
		}
	}
done:
	svc.done <- true
}

// logFile creates a log file for this service.
func (svc *Service) logFile() (*os.File, error) {
	name := "seesaw_" + svc.name
	t := time.Now()
	logName := fmt.Sprintf("%s.log.%04d%02d%02d-%02d%02d%02d", name,
		t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second())

	f, err := os.Create(path.Join(logDir, logName))
	if err != nil {
		return nil, err
	}

	logLink := path.Join(logDir, name+".log")
	os.Remove(logLink)
	os.Symlink(logName, logLink)

	fmt.Fprintf(f, "Log file for %s (stdout/stderr)\n", name)
	fmt.Fprintf(f, "Created at: %s\n", t.Format("2006/01/02 15:04:05"))

	return f, nil
}

// logSink copies output from the given reader to a log file, before closing
// both the reader and the log file.
func (svc *Service) logSink(f *os.File, r io.ReadCloser) {
	_, err := io.Copy(f, r)
	if err != nil {
		log.Warningf("Service %s - log sink failed: %v", svc.name, err)
	}
	f.Close()
	r.Close()
}

// runOnce runs a service once, returning once an error occurs or the process
// has exited.
func (svc *Service) runOnce() {
	args := make([]string, len(svc.args)+1)
	args[0] = "seesaw_" + svc.name
	copy(args[1:], svc.args)

	null, err := os.Open(os.DevNull)
	if err != nil {
		log.Warningf("Service %s - failed to open %s: %v", svc.name, os.DevNull, err)
		return
	}

	pr, pw, err := os.Pipe()
	if err != nil {
		log.Warningf("Service %s - failed to create pipes: %v", svc.name, err)
		null.Close()
		return
	}

	f, err := svc.logFile()
	if err != nil {
		log.Warningf("Service %s - failed to create log file: %v", svc.name, err)
		null.Close()
		pr.Close()
		pw.Close()
		return
	}

	go svc.logSink(f, pr)

	attr := &os.ProcAttr{
		Dir:   svc.path,
		Files: []*os.File{null, pw, pw},
		Sys: &syscall.SysProcAttr{
			Credential: &syscall.Credential{
				Uid: svc.uid,
				Gid: svc.gid,
			},
			Setpgid: true,
		},
	}

	log.Infof("Starting service %s...", svc.name)
	proc, err := os.StartProcess(svc.binary, args, attr)
	if err != nil {
		log.Warningf("Service %s failed to start: %v", svc.name, err)
		svc.lastFailure = time.Now()
		svc.failures++
		null.Close()
		pw.Close()
		return
	}
	null.Close()
	pw.Close()
	svc.lock.Lock()
	svc.process = proc
	svc.lock.Unlock()

	if _, _, err := syscall.Syscall(syscall.SYS_SETPRIORITY, uintptr(prioProcess), uintptr(proc.Pid), uintptr(svc.priority)); err != 0 {
		log.Warningf("Failed to set priority to %d for service %s: %v", svc.priority, svc.name, err)
	}

	select {
	case svc.started <- true:
	default:
	}

	state, err := svc.process.Wait()
	if err != nil {
		log.Warningf("Service %s wait failed with %v", svc.name, err)
		svc.lastFailure = time.Now()
		svc.failures++
		return
	}
	if !state.Success() {
		log.Warningf("Service %s exited with %v", svc.name, state)
		svc.lastFailure = time.Now()
		svc.failures++
		return
	}
	// TODO(jsing): Reset failures after process has been running for some
	// given duration, so that failures with large intervals do not result
	// in backoff. However, we also want to count the total number of
	// failures and export it for monitoring purposes.
	svc.failures = 0
	log.Infof("Service %s exited normally.", svc.name)
}

// signal sends a signal to the service.
func (svc *Service) signal(sig os.Signal) error {
	svc.lock.Lock()
	defer svc.lock.Unlock()
	if svc.process == nil {
		return nil
	}
	return svc.process.Signal(sig)
}

// stop stops a running service.
func (svc *Service) stop() {
	// TODO(jsing): Check if it is actually running?
	log.Infof("Stopping service %s...", svc.name)

	// Wait for dependents to shutdown.
	for _, dep := range svc.dependents {
		log.Infof("Service %s waiting for %s to stop", svc.name, dep.name)
		stopped := <-dep.stopped
		dep.stopped <- stopped
	}

	svc.shutdown <- true
	svc.signal(syscall.SIGTERM)
	select {
	case <-svc.done:
	case <-time.After(svc.termTimeout):
		svc.signal(syscall.SIGKILL)
		<-svc.done
	}
	log.Infof("Service %s stopped", svc.name)
	svc.stopped <- true
}
<commit_msg>watchdog: use time.Time's Format method<commit_after>// Copyright 2012 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Author: jsing@google.com (Joel Sing)

package watchdog

import (
	"fmt"
	"io"
	"os"
	"os/user"
	"path"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"

	log "github.com/golang/glog"
)

const logDir = "/var/log/seesaw"

const prioProcess = 0

// Service contains the data needed to manage a service.
type Service struct {
	name   string
	binary string
	path   string
	args   []string

	uid      uint32
	gid      uint32
	priority int

	dependencies map[string]*Service
	dependents   map[string]*Service

	termTimeout time.Duration

	lock    sync.Mutex
	process *os.Process

	done     chan bool
	shutdown chan bool
	started  chan bool
	stopped  chan bool

	failures uint64
	restarts uint64

	lastFailure time.Time
	lastRestart time.Time
}

// newService returns an initialised service.
func newService(name, binary string) *Service {
	return &Service{
		name:         name,
		binary:       binary,
		args:         make([]string, 0),
		dependencies: make(map[string]*Service),
		dependents:   make(map[string]*Service),

		done:     make(chan bool),
		shutdown: make(chan bool, 1),
		started:  make(chan bool, 1),
		stopped:  make(chan bool, 1),

		termTimeout: 5 * time.Second,
	}
}

// AddDependency registers a dependency for this service.
func (svc *Service) AddDependency(name string) {
	svc.dependencies[name] = nil
}

// AddArgs adds the given string as arguments.
func (svc *Service) AddArgs(args string) {
	svc.args = strings.Fields(args)
}

// SetPriority sets the process priority for a service.
func (svc *Service) SetPriority(priority int) error {
	if priority < -20 || priority > 19 {
		return fmt.Errorf("Invalid priority %d - must be between -20 and 19", priority)
	}
	svc.priority = priority
	return nil
}

// SetTermTimeout sets the termination timeout for a service.
func (svc *Service) SetTermTimeout(tt time.Duration) {
	svc.termTimeout = tt
}

// SetUser sets the user for a service.
func (svc *Service) SetUser(username string) error {
	u, err := user.Lookup(username)
	if err != nil {
		return err
	}
	uid, err := strconv.Atoi(u.Uid)
	if err != nil {
		return err
	}
	gid, err := strconv.Atoi(u.Gid)
	if err != nil {
		return err
	}
	svc.uid = uint32(uid)
	svc.gid = uint32(gid)
	return nil
}

// run runs a service and restarts it upon termination, unless a shutdown
// notification has been received.
func (svc *Service) run() {

	// Wait for dependencies to start.
	for _, dep := range svc.dependencies {
		log.Infof("Service %s waiting for %s to start", svc.name, dep.name)
		select {
		case started := <-dep.started:
			dep.started <- started
		case <-svc.shutdown:
			goto done
		}
	}

	for {
		if svc.failures > 0 {
			delay := time.Duration(svc.failures) * restartBackoff
			if delay > restartBackoffMax {
				delay = restartBackoffMax
			}
			log.Infof("Service %s has failed %d times - delaying %s before restart",
				svc.name, svc.failures, delay)

			select {
			case <-time.After(delay):
			case <-svc.shutdown:
				goto done
			}
		}

		svc.restarts++
		svc.lastRestart = time.Now()
		svc.runOnce()

		select {
		case <-time.After(restartDelay):
		case <-svc.shutdown:
			goto done
		}
	}
done:
	svc.done <- true
}

// logFile creates a log file for this service.
func (svc *Service) logFile() (*os.File, error) {
	name := "seesaw_" + svc.name
	t := time.Now()
	logName := fmt.Sprintf("%s.log.%s", name, t.Format("20060102-150405"))

	f, err := os.Create(path.Join(logDir, logName))
	if err != nil {
		return nil, err
	}

	logLink := path.Join(logDir, name+".log")
	os.Remove(logLink)
	os.Symlink(logName, logLink)

	fmt.Fprintf(f, "Log file for %s (stdout/stderr)\n", name)
	fmt.Fprintf(f, "Created at: %s\n", t.Format("2006/01/02 15:04:05"))

	return f, nil
}

// logSink copies output from the given reader to a log file, before closing
// both the reader and the log file.
func (svc *Service) logSink(f *os.File, r io.ReadCloser) {
	_, err := io.Copy(f, r)
	if err != nil {
		log.Warningf("Service %s - log sink failed: %v", svc.name, err)
	}
	f.Close()
	r.Close()
}

// runOnce runs a service once, returning once an error occurs or the process
// has exited.
func (svc *Service) runOnce() {
	args := make([]string, len(svc.args)+1)
	args[0] = "seesaw_" + svc.name
	copy(args[1:], svc.args)

	null, err := os.Open(os.DevNull)
	if err != nil {
		log.Warningf("Service %s - failed to open %s: %v", svc.name, os.DevNull, err)
		return
	}

	pr, pw, err := os.Pipe()
	if err != nil {
		log.Warningf("Service %s - failed to create pipes: %v", svc.name, err)
		null.Close()
		return
	}

	f, err := svc.logFile()
	if err != nil {
		log.Warningf("Service %s - failed to create log file: %v", svc.name, err)
		null.Close()
		pr.Close()
		pw.Close()
		return
	}

	go svc.logSink(f, pr)

	attr := &os.ProcAttr{
		Dir:   svc.path,
		Files: []*os.File{null, pw, pw},
		Sys: &syscall.SysProcAttr{
			Credential: &syscall.Credential{
				Uid: svc.uid,
				Gid: svc.gid,
			},
			Setpgid: true,
		},
	}

	log.Infof("Starting service %s...", svc.name)
	proc, err := os.StartProcess(svc.binary, args, attr)
	if err != nil {
		log.Warningf("Service %s failed to start: %v", svc.name, err)
		svc.lastFailure = time.Now()
		svc.failures++
		null.Close()
		pw.Close()
		return
	}
	null.Close()
	pw.Close()
	svc.lock.Lock()
	svc.process = proc
	svc.lock.Unlock()

	if _, _, err := syscall.Syscall(syscall.SYS_SETPRIORITY, uintptr(prioProcess), uintptr(proc.Pid), uintptr(svc.priority)); err != 0 {
		log.Warningf("Failed to set priority to %d for service %s: %v", svc.priority, svc.name, err)
	}

	select {
	case svc.started <- true:
	default:
	}

	state, err := svc.process.Wait()
	if err != nil {
		log.Warningf("Service %s wait failed with %v", svc.name, err)
		svc.lastFailure = time.Now()
		svc.failures++
		return
	}
	if !state.Success() {
		log.Warningf("Service %s exited with %v", svc.name, state)
		svc.lastFailure = time.Now()
		svc.failures++
		return
	}
	// TODO(jsing): Reset failures after process has been running for some
	// given duration, so that failures with large intervals do not result
	// in backoff. However, we also want to count the total number of
	// failures and export it for monitoring purposes.
	svc.failures = 0
	log.Infof("Service %s exited normally.", svc.name)
}

// signal sends a signal to the service.
func (svc *Service) signal(sig os.Signal) error {
	svc.lock.Lock()
	defer svc.lock.Unlock()
	if svc.process == nil {
		return nil
	}
	return svc.process.Signal(sig)
}

// stop stops a running service.
func (svc *Service) stop() {
	// TODO(jsing): Check if it is actually running?
	log.Infof("Stopping service %s...", svc.name)

	// Wait for dependents to shutdown.
	for _, dep := range svc.dependents {
		log.Infof("Service %s waiting for %s to stop", svc.name, dep.name)
		stopped := <-dep.stopped
		dep.stopped <- stopped
	}

	svc.shutdown <- true
	svc.signal(syscall.SIGTERM)
	select {
	case <-svc.done:
	case <-time.After(svc.termTimeout):
		svc.signal(syscall.SIGKILL)
		<-svc.done
	}
	log.Infof("Service %s stopped", svc.name)
	svc.stopped <- true
}
<|endoftext|>