<commit_before>package main

import (
	"flag"
	"fmt"
	"net/http"
	"os"
	"os/signal"
	"sync"
	"syscall"
	"time"

	stdprometheus "github.com/prometheus/client_golang/prometheus"
	"golang.org/x/net/context"

	"github.com/go-kit/kit/log"
	kitprometheus "github.com/go-kit/kit/metrics/prometheus"

	"github.com/go-kit/kit/examples/shipping/booking"
	"github.com/go-kit/kit/examples/shipping/cargo"
	"github.com/go-kit/kit/examples/shipping/handling"
	"github.com/go-kit/kit/examples/shipping/inspection"
	"github.com/go-kit/kit/examples/shipping/location"
	"github.com/go-kit/kit/examples/shipping/repository"
	"github.com/go-kit/kit/examples/shipping/routing"
	"github.com/go-kit/kit/examples/shipping/tracking"
)

const (
	defaultPort              = "8080"
	defaultRoutingServiceURL = "http://localhost:7878"
)

func main() {
	var (
		addr  = envString("PORT", defaultPort)
		rsurl = envString("ROUTINGSERVICE_URL", defaultRoutingServiceURL)

		httpAddr          = flag.String("http.addr", ":"+addr, "HTTP listen address")
		routingServiceURL = flag.String("service.routing", rsurl, "routing service URL")

		ctx = context.Background()
	)

	flag.Parse()

	var logger log.Logger
	logger = log.NewLogfmtLogger(os.Stderr)
	logger = &serializedLogger{Logger: logger}
	logger = log.NewContext(logger).With("ts", log.DefaultTimestampUTC)

	var (
		cargos         = repository.NewCargo()
		locations      = repository.NewLocation()
		voyages        = repository.NewVoyage()
		handlingEvents = repository.NewHandlingEvent()
	)

	// Configure some questionable dependencies.
	var (
		handlingEventFactory = cargo.HandlingEventFactory{
			CargoRepository:    cargos,
			VoyageRepository:   voyages,
			LocationRepository: locations,
		}
		handlingEventHandler = handling.NewEventHandler(
			inspection.NewService(cargos, handlingEvents, nil),
		)
	)

	// Facilitate testing by adding some cargos.
	storeTestData(cargos)

	fieldKeys := []string{"method"}

	var rs routing.Service
	rs = routing.NewProxyingMiddleware(*routingServiceURL, ctx)(rs)

	var bs booking.Service
	bs = booking.NewService(cargos, locations, handlingEvents, rs)
	bs = booking.NewLoggingService(log.NewContext(logger).With("component", "booking"), bs)
	bs = booking.NewInstrumentingService(
		kitprometheus.NewCounterFrom(stdprometheus.CounterOpts{
			Namespace: "api",
			Subsystem: "booking_service",
			Name:      "request_count",
			Help:      "Number of requests received.",
		}, fieldKeys),
		kitprometheus.NewSummaryFrom(stdprometheus.SummaryOpts{
			Namespace: "api",
			Subsystem: "booking_service",
			Name:      "request_latency_microseconds",
			Help:      "Total duration of requests in microseconds.",
		}, fieldKeys),
		bs,
	)

	var ts tracking.Service
	ts = tracking.NewService(cargos, handlingEvents)
	ts = tracking.NewLoggingService(log.NewContext(logger).With("component", "tracking"), ts)
	ts = tracking.NewInstrumentingService(
		kitprometheus.NewCounterFrom(stdprometheus.CounterOpts{
			Namespace: "api",
			Subsystem: "tracking_service",
			Name:      "request_count",
			Help:      "Number of requests received.",
		}, fieldKeys),
		kitprometheus.NewSummaryFrom(stdprometheus.SummaryOpts{
			Namespace: "api",
			Subsystem: "tracking_service",
			Name:      "request_latency_microseconds",
			Help:      "Total duration of requests in microseconds.",
		}, fieldKeys),
		ts,
	)

	var hs handling.Service
	hs = handling.NewService(handlingEvents, handlingEventFactory, handlingEventHandler)
	hs = handling.NewLoggingService(log.NewContext(logger).With("component", "handling"), hs)
	hs = handling.NewInstrumentingService(
		kitprometheus.NewCounterFrom(stdprometheus.CounterOpts{
			Namespace: "api",
			Subsystem: "handling_service",
			Name:      "request_count",
			Help:      "Number of requests received.",
		}, fieldKeys),
		kitprometheus.NewSummaryFrom(stdprometheus.SummaryOpts{
			Namespace: "api",
			Subsystem: "handling_service",
			Name:      "request_latency_microseconds",
			Help:      "Total duration of requests in microseconds.",
		}, fieldKeys),
		hs,
	)

	httpLogger := log.NewContext(logger).With("component", "http")

	mux := http.NewServeMux()

	mux.Handle("/booking/v1/", booking.MakeHandler(ctx, bs, httpLogger))
	mux.Handle("/tracking/v1/", tracking.MakeHandler(ctx, ts, httpLogger))
	mux.Handle("/handling/v1/", handling.MakeHandler(ctx, hs, httpLogger))

	http.Handle("/", accessControl(mux))
	http.Handle("/metrics", stdprometheus.Handler())

	errs := make(chan error, 2)
	go func() {
		logger.Log("transport", "http", "address", *httpAddr, "msg", "listening")
		errs <- http.ListenAndServe(*httpAddr, nil)
	}()
	go func() {
		c := make(chan os.Signal)
		signal.Notify(c, syscall.SIGINT)
		errs <- fmt.Errorf("%s", <-c)
	}()

	logger.Log("terminated", <-errs)
}

func accessControl(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Access-Control-Allow-Origin", "*")
		w.Header().Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
		w.Header().Set("Access-Control-Allow-Headers", "Origin, Content-Type")

		if r.Method == "OPTIONS" {
			return
		}

		h.ServeHTTP(w, r)
	})
}

func envString(env, fallback string) string {
	e := os.Getenv(env)
	if e == "" {
		return fallback
	}
	return e
}

func storeTestData(r cargo.Repository) {
	test1 := cargo.New("FTL456", cargo.RouteSpecification{
		Origin:          location.AUMEL,
		Destination:     location.SESTO,
		ArrivalDeadline: time.Now().AddDate(0, 0, 7),
	})
	_ = r.Store(test1)

	test2 := cargo.New("ABC123", cargo.RouteSpecification{
		Origin:          location.SESTO,
		Destination:     location.CNHKG,
		ArrivalDeadline: time.Now().AddDate(0, 0, 14),
	})
	_ = r.Store(test2)
}

type serializedLogger struct {
	mtx sync.Mutex
	log.Logger
}

func (l *serializedLogger) Log(keyvals ...interface{}) error {
	l.mtx.Lock()
	defer l.mtx.Unlock()
	return l.Logger.Log(keyvals...)
}
<commit_msg>examples: shipping: use pcp<commit_after>package main

import (
	"flag"
	"fmt"
	"net/http"
	"os"
	"os/signal"
	"sync"
	"syscall"
	"time"

	"golang.org/x/net/context"

	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/metrics/pcp"

	"github.com/go-kit/kit/examples/shipping/booking"
	"github.com/go-kit/kit/examples/shipping/cargo"
	"github.com/go-kit/kit/examples/shipping/handling"
	"github.com/go-kit/kit/examples/shipping/inspection"
	"github.com/go-kit/kit/examples/shipping/location"
	"github.com/go-kit/kit/examples/shipping/repository"
	"github.com/go-kit/kit/examples/shipping/routing"
	"github.com/go-kit/kit/examples/shipping/tracking"
)

const (
	defaultPort              = "8080"
	defaultRoutingServiceURL = "http://localhost:7878"
)

func main() {
	var (
		addr  = envString("PORT", defaultPort)
		rsurl = envString("ROUTINGSERVICE_URL", defaultRoutingServiceURL)

		httpAddr          = flag.String("http.addr", ":"+addr, "HTTP listen address")
		routingServiceURL = flag.String("service.routing", rsurl, "routing service URL")

		ctx = context.Background()
	)

	flag.Parse()

	var logger log.Logger
	logger = log.NewLogfmtLogger(os.Stderr)
	logger = &serializedLogger{Logger: logger}
	logger = log.NewContext(logger).With("ts", log.DefaultTimestampUTC)

	var (
		cargos         = repository.NewCargo()
		locations      = repository.NewLocation()
		voyages        = repository.NewVoyage()
		handlingEvents = repository.NewHandlingEvent()
	)

	// Configure some questionable dependencies.
	var (
		handlingEventFactory = cargo.HandlingEventFactory{
			CargoRepository:    cargos,
			VoyageRepository:   voyages,
			LocationRepository: locations,
		}
		handlingEventHandler = handling.NewEventHandler(
			inspection.NewService(cargos, handlingEvents, nil),
		)
	)

	// Facilitate testing by adding some cargos.
	storeTestData(cargos)

	// fieldKeys := []string{"method"}

	var rs routing.Service
	rs = routing.NewProxyingMiddleware(*routingServiceURL, ctx)(rs)

	var bs booking.Service
	bs = booking.NewService(cargos, locations, handlingEvents, rs)
	bs = booking.NewLoggingService(log.NewContext(logger).With("component", "booking"), bs)
	bs = booking.NewInstrumentingService(
		pcp.NewCounter("api.booking_service.request_count", "Number of requests received."),
		pcp.NewHistogram("api.booking_service.request_latency_microseconds", "Total duration of requests in microseconds."),
		bs,
	)

	var ts tracking.Service
	ts = tracking.NewService(cargos, handlingEvents)
	ts = tracking.NewLoggingService(log.NewContext(logger).With("component", "tracking"), ts)
	ts = tracking.NewInstrumentingService(
		pcp.NewCounter("api.tracking_service.request_count", "Number of requests received."),
		pcp.NewHistogram("api.tracking_service.request_latency_microseconds", "Total duration of requests in microseconds."),
		ts,
	)

	var hs handling.Service
	hs = handling.NewService(handlingEvents, handlingEventFactory, handlingEventHandler)
	hs = handling.NewLoggingService(log.NewContext(logger).With("component", "handling"), hs)
	hs = handling.NewInstrumentingService(
		pcp.NewCounter("api.handling_service.request_count", "Number of requests received."),
		pcp.NewHistogram("api.handling_service.request_latency_microseconds", "Total duration of requests in microseconds."),
		hs,
	)

	httpLogger := log.NewContext(logger).With("component", "http")

	mux := http.NewServeMux()

	mux.Handle("/booking/v1/", booking.MakeHandler(ctx, bs, httpLogger))
	mux.Handle("/tracking/v1/", tracking.MakeHandler(ctx, ts, httpLogger))
	mux.Handle("/handling/v1/", handling.MakeHandler(ctx, hs, httpLogger))

	http.Handle("/", accessControl(mux))

	pcp.StartReporting("shipping")
	defer pcp.StopReporting()

	errs := make(chan error, 2)
	go func() {
		logger.Log("transport", "http", "address", *httpAddr, "msg", "listening")
		errs <- http.ListenAndServe(*httpAddr, nil)
	}()
	go func() {
		c := make(chan os.Signal)
		signal.Notify(c, syscall.SIGINT)
		errs <- fmt.Errorf("%s", <-c)
	}()

	logger.Log("terminated", <-errs)
}

func accessControl(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Access-Control-Allow-Origin", "*")
		w.Header().Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
		w.Header().Set("Access-Control-Allow-Headers", "Origin, Content-Type")

		if r.Method == "OPTIONS" {
			return
		}

		h.ServeHTTP(w, r)
	})
}

func envString(env, fallback string) string {
	e := os.Getenv(env)
	if e == "" {
		return fallback
	}
	return e
}

func storeTestData(r cargo.Repository) {
	test1 := cargo.New("FTL456", cargo.RouteSpecification{
		Origin:          location.AUMEL,
		Destination:     location.SESTO,
		ArrivalDeadline: time.Now().AddDate(0, 0, 7),
	})
	_ = r.Store(test1)

	test2 := cargo.New("ABC123", cargo.RouteSpecification{
		Origin:          location.SESTO,
		Destination:     location.CNHKG,
		ArrivalDeadline: time.Now().AddDate(0, 0, 14),
	})
	_ = r.Store(test2)
}

type serializedLogger struct {
	mtx sync.Mutex
	log.Logger
}

func (l *serializedLogger) Log(keyvals ...interface{}) error {
	l.mtx.Lock()
	defer l.mtx.Unlock()
	return l.Logger.Log(keyvals...)
}
<|endoftext|>


<commit_before>// Copyright 2015 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !js,!windows

package audio

import (
	"fmt"
	"io"
	"runtime"
	"sync"
	"time"

	"golang.org/x/mobile/exp/audio/al"
)

type alSourceCacheEntry struct {
	source     al.Source
	sampleRate int
}

const maxSourceNum = 32

var alSourceCache = []alSourceCacheEntry{}

type player struct {
	alSource   al.Source
	alBuffers  []al.Buffer
	source     io.ReadSeeker
	sampleRate int
}

var m sync.Mutex

func newAlSource(sampleRate int) (al.Source, error) {
	for _, e := range alSourceCache {
		if e.sampleRate != sampleRate {
			continue
		}
		s := e.source.State()
		if s != al.Initial && s != al.Stopped {
			continue
		}
		return e.source, nil
	}
	if maxSourceNum <= len(alSourceCache) {
		return 0, ErrTooManyPlayers
	}
	s := al.GenSources(1)
	if err := al.Error(); err != 0 {
		panic(fmt.Sprintf("audio: al.GenSources error: %d", err))
	}
	e := alSourceCacheEntry{
		source:     s[0],
		sampleRate: sampleRate,
	}
	alSourceCache = append(alSourceCache, e)
	return s[0], nil
}

func newPlayer(src io.ReadSeeker, sampleRate int) (*Player, error) {
	m.Lock()
	if e := al.OpenDevice(); e != nil {
		m.Unlock()
		return nil, fmt.Errorf("audio: OpenAL initialization failed: %v", e)
	}
	s, err := newAlSource(sampleRate)
	if err != nil {
		m.Unlock()
		return nil, err
	}
	m.Unlock()
	p := &player{
		alSource:   s,
		alBuffers:  []al.Buffer{},
		source:     src,
		sampleRate: sampleRate,
	}
	runtime.SetFinalizer(p, (*player).close)
	return &Player{p}, nil
}

const bufferSize = 1024

func (p *player) proceed() error {
	m.Lock()
	processedNum := p.alSource.BuffersProcessed()
	if 0 < processedNum {
		bufs := make([]al.Buffer, processedNum)
		p.alSource.UnqueueBuffers(bufs...)
		p.alBuffers = append(p.alBuffers, bufs...)
	}
	m.Unlock()
	for 0 < len(p.alBuffers) {
		b := make([]byte, bufferSize)
		n, err := p.source.Read(b)
		if 0 < n {
			m.Lock()
			buf := p.alBuffers[0]
			p.alBuffers = p.alBuffers[1:]
			buf.BufferData(al.FormatStereo16, b[:n], int32(p.sampleRate))
			p.alSource.QueueBuffers(buf)
			m.Unlock()
		}
		if err != nil {
			return err
		}
	}
	m.Lock()
	if p.alSource.State() == al.Stopped {
		al.RewindSources(p.alSource)
		al.PlaySources(p.alSource)
	}
	m.Unlock()

	return nil
}

func (p *player) play() error {
	const bufferMaxNum = 8
	// TODO: What if play is already called?
	m.Lock()
	n := bufferMaxNum - int(p.alSource.BuffersQueued()) - len(p.alBuffers)
	if 0 < n {
		p.alBuffers = append(p.alBuffers, al.GenBuffers(n)...)
	}
	if 0 < len(p.alBuffers) {
		emptyBytes := make([]byte, bufferSize)
		for _, buf := range p.alBuffers {
			// Note that the third argument of only the first buffer is used.
			buf.BufferData(al.FormatStereo16, emptyBytes, int32(p.sampleRate))
			p.alSource.QueueBuffers(buf)
		}
		p.alBuffers = []al.Buffer{}
	}
	al.PlaySources(p.alSource)
	m.Unlock()
	go func() {
		defer p.close()
		for {
			err := p.proceed()
			if err == io.EOF {
				break
			}
			if err != nil {
				// TODO: Record the last error
				panic(err)
			}
			time.Sleep(1)
		}
	}()
	return nil
}

func (p *player) close() error {
	m.Lock()
	var bs []al.Buffer
	if p.alSource != 0 {
		al.RewindSources(p.alSource)
		al.StopSources(p.alSource)
		n := p.alSource.BuffersQueued()
		if 0 < n {
			bs = make([]al.Buffer, n)
			p.alSource.UnqueueBuffers(bs...)
		}
		p.alSource = 0
	}
	if 0 < len(p.alBuffers) {
		al.DeleteBuffers(p.alBuffers...)
		p.alBuffers = []al.Buffer{}
	}
	if bs != nil {
		p.alBuffers = append(p.alBuffers, bs...)
	}
	if err := al.Error(); err != 0 {
		panic(fmt.Sprintf("audio: closing error: %d", err))
	}
	m.Unlock()
	runtime.SetFinalizer(p, nil)
	return nil
}

// TODO: Implement Close method
<commit_msg>audio: Bug fix: sources were not reused properly<commit_after>// Copyright 2015 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !js,!windows

package audio

import (
	"fmt"
	"io"
	"runtime"
	"sync"
	"time"

	"golang.org/x/mobile/exp/audio/al"
)

type alSourceCacheEntry struct {
	source     al.Source
	sampleRate int
	isClosed   bool
}

const maxSourceNum = 32

var alSourceCache = []*alSourceCacheEntry{}

type player struct {
	alSource   al.Source
	alBuffers  []al.Buffer
	source     io.ReadSeeker
	sampleRate int
}

var m sync.Mutex

func newAlSource(sampleRate int) (al.Source, error) {
	for _, e := range alSourceCache {
		if e.sampleRate != sampleRate {
			continue
		}
		if !e.isClosed {
			continue
		}
		e.isClosed = false
		return e.source, nil
	}
	if maxSourceNum <= len(alSourceCache) {
		return 0, ErrTooManyPlayers
	}
	s := al.GenSources(1)
	if err := al.Error(); err != 0 {
		panic(fmt.Sprintf("audio: al.GenSources error: %d", err))
	}
	e := &alSourceCacheEntry{
		source:     s[0],
		sampleRate: sampleRate,
	}
	alSourceCache = append(alSourceCache, e)
	return s[0], nil
}

func newPlayer(src io.ReadSeeker, sampleRate int) (*Player, error) {
	m.Lock()
	if e := al.OpenDevice(); e != nil {
		m.Unlock()
		return nil, fmt.Errorf("audio: OpenAL initialization failed: %v", e)
	}
	s, err := newAlSource(sampleRate)
	if err != nil {
		m.Unlock()
		return nil, err
	}
	m.Unlock()
	p := &player{
		alSource:   s,
		alBuffers:  []al.Buffer{},
		source:     src,
		sampleRate: sampleRate,
	}
	runtime.SetFinalizer(p, (*player).close)
	return &Player{p}, nil
}

const bufferSize = 1024

func (p *player) proceed() error {
	m.Lock()
	processedNum := p.alSource.BuffersProcessed()
	if 0 < processedNum {
		bufs := make([]al.Buffer, processedNum)
		p.alSource.UnqueueBuffers(bufs...)
		p.alBuffers = append(p.alBuffers, bufs...)
	}
	m.Unlock()
	for 0 < len(p.alBuffers) {
		b := make([]byte, bufferSize)
		n, err := p.source.Read(b)
		if 0 < n {
			m.Lock()
			buf := p.alBuffers[0]
			p.alBuffers = p.alBuffers[1:]
			buf.BufferData(al.FormatStereo16, b[:n], int32(p.sampleRate))
			p.alSource.QueueBuffers(buf)
			m.Unlock()
		}
		if err != nil {
			return err
		}
	}
	m.Lock()
	if p.alSource.State() == al.Stopped {
		al.RewindSources(p.alSource)
		al.PlaySources(p.alSource)
	}
	m.Unlock()

	return nil
}

func (p *player) play() error {
	const bufferMaxNum = 8
	// TODO: What if play is already called?
	m.Lock()
	n := bufferMaxNum - int(p.alSource.BuffersQueued()) - len(p.alBuffers)
	if 0 < n {
		p.alBuffers = append(p.alBuffers, al.GenBuffers(n)...)
	}
	if 0 < len(p.alBuffers) {
		emptyBytes := make([]byte, bufferSize)
		for _, buf := range p.alBuffers {
			// Note that the third argument of only the first buffer is used.
			buf.BufferData(al.FormatStereo16, emptyBytes, int32(p.sampleRate))
			p.alSource.QueueBuffers(buf)
		}
		p.alBuffers = []al.Buffer{}
	}
	al.PlaySources(p.alSource)
	m.Unlock()
	go func() {
		defer p.close()
		for {
			err := p.proceed()
			if err == io.EOF {
				break
			}
			if err != nil {
				// TODO: Record the last error
				panic(err)
			}
			time.Sleep(1)
		}
	}()
	return nil
}

func (p *player) close() error {
	m.Lock()
	var bs []al.Buffer
	s := p.alSource
	if p.alSource == 0 {
		al.RewindSources(p.alSource)
		al.StopSources(p.alSource)
		n := p.alSource.BuffersQueued()
		if 0 < n {
			bs = make([]al.Buffer, n)
			p.alSource.UnqueueBuffers(bs...)
		}
		p.alSource = 0
	}
	// TODO: Is this needed?
	if 0 < len(p.alBuffers) {
		al.DeleteBuffers(p.alBuffers...)
		p.alBuffers = []al.Buffer{}
	}
	if bs != nil {
		p.alBuffers = append(p.alBuffers, bs...)
	}
	if err := al.Error(); err != 0 {
		panic(fmt.Sprintf("audio: closing error: %d", err))
	}
	if s != 0 {
		found := false
		for _, e := range alSourceCache {
			if e.source != s {
				continue
			}
			if e.isClosed {
				panic("audio: cache state is invalid: source is already closed?")
			}
			e.isClosed = true
			found = true
			break
		}
		if !found {
			panic("audio: cache state is invalid: source is not cached?")
		}
	}
	m.Unlock()
	runtime.SetFinalizer(p, nil)
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !js,!windows

package audio

import (
	"fmt"
	"io"
	"runtime"
	"time"

	"golang.org/x/mobile/exp/audio/al"
)

const (
	maxBufferNum = 8
)

// TODO: This should be in player
var totalBufferNum = 0

type player struct {
	alSource   al.Source
	alBuffers  []al.Buffer
	source     io.Reader
	sampleRate int
	isClosed   bool
}

func newPlayer(src io.Reader, sampleRate int) (*player, error) {
	if e := al.OpenDevice(); e != nil {
		return nil, fmt.Errorf("audio: OpenAL initialization failed: %v", e)
	}
	s := al.GenSources(1)
	if err := al.Error(); err != 0 {
		return nil, fmt.Errorf("audio: al.GenSources error: %d", err)
	}
	p := &player{
		alSource:   s[0],
		alBuffers:  []al.Buffer{},
		source:     src,
		sampleRate: sampleRate,
	}
	runtime.SetFinalizer(p, (*player).close)

	n := maxBufferNum - int(p.alSource.BuffersQueued()) - len(p.alBuffers)
	if 0 < n {
		p.alBuffers = append(p.alBuffers, al.GenBuffers(n)...)
		totalBufferNum += n
		if maxBufferNum < totalBufferNum {
			panic("audio: not reach: too many buffers are created")
		}
	}
	if 0 < len(p.alBuffers) {
		emptyBytes := make([]byte, bufferSize)
		for _, buf := range p.alBuffers {
			// Note that the third argument of only the first buffer is used.
			buf.BufferData(al.FormatStereo16, emptyBytes, int32(p.sampleRate))
			p.alSource.QueueBuffers(buf)
		}
		p.alBuffers = []al.Buffer{}
	}
	al.PlaySources(p.alSource)
	return p, nil
}

const (
	bufferSize = 1024
)

var (
	tmpBuffer    = make([]byte, bufferSize)
	tmpAlBuffers = make([]al.Buffer, maxBufferNum)
)

func (p *player) proceed() error {
	if err := al.Error(); err != 0 {
		return fmt.Errorf("audio: before proceed: %d", err)
	}
	processedNum := p.alSource.BuffersProcessed()
	if 0 < processedNum {
		bufs := tmpAlBuffers[:processedNum]
		p.alSource.UnqueueBuffers(bufs...)
		if err := al.Error(); err != 0 {
			return fmt.Errorf("audio: Unqueue in process: %d", err)
		}
		p.alBuffers = append(p.alBuffers, bufs...)
	}

	for 0 < len(p.alBuffers) {
		n, err := p.source.Read(tmpBuffer)
		if 0 < n {
			buf := p.alBuffers[0]
			p.alBuffers = p.alBuffers[1:]
			buf.BufferData(al.FormatStereo16, tmpBuffer[:n], int32(p.sampleRate))
			p.alSource.QueueBuffers(buf)
			if err := al.Error(); err != 0 {
				return fmt.Errorf("audio: Queue in process: %d", err)
			}
		}
		if err != nil {
			return err
		}
		if n == 0 {
			time.Sleep(1 * time.Millisecond)
		}
	}

	if p.alSource.State() == al.Stopped {
		al.RewindSources(p.alSource)
		al.PlaySources(p.alSource)
		if err := al.Error(); err != 0 {
			return fmt.Errorf("audio: PlaySource in process: %d", err)
		}
	}

	return nil
}

func (p *player) close() error {
	if err := al.Error(); err != 0 {
		return fmt.Errorf("audio: error before closing: %d", err)
	}
	if p.isClosed {
		return nil
	}
	var bs []al.Buffer
	al.RewindSources(p.alSource)
	al.StopSources(p.alSource)
	if n := p.alSource.BuffersQueued(); 0 < n {
		bs = make([]al.Buffer, n)
		p.alSource.UnqueueBuffers(bs...)
		p.alBuffers = append(p.alBuffers, bs...)
	}
	p.isClosed = true
	if err := al.Error(); err != 0 {
		return fmt.Errorf("audio: error after closing: %d", err)
	}
	runtime.SetFinalizer(p, nil)
	return nil
}
<commit_msg>audio: Refactoring<commit_after>// Copyright 2015 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !js,!windows

package audio

import (
	"fmt"
	"io"
	"runtime"
	"time"

	"golang.org/x/mobile/exp/audio/al"
)

const (
	maxBufferNum = 8
)

type player struct {
	alSource   al.Source
	alBuffers  []al.Buffer
	source     io.Reader
	sampleRate int
	isClosed   bool
}

func newPlayer(src io.Reader, sampleRate int) (*player, error) {
	if e := al.OpenDevice(); e != nil {
		return nil, fmt.Errorf("audio: OpenAL initialization failed: %v", e)
	}
	s := al.GenSources(1)
	if err := al.Error(); err != 0 {
		return nil, fmt.Errorf("audio: al.GenSources error: %d", err)
	}
	p := &player{
		alSource:   s[0],
		alBuffers:  []al.Buffer{},
		source:     src,
		sampleRate: sampleRate,
	}
	runtime.SetFinalizer(p, (*player).close)

	bs := al.GenBuffers(maxBufferNum)
	emptyBytes := make([]byte, bufferSize)
	for _, b := range bs {
		// Note that the third argument of only the first buffer is used.
		b.BufferData(al.FormatStereo16, emptyBytes, int32(p.sampleRate))
		p.alSource.QueueBuffers(b)
	}
	al.PlaySources(p.alSource)
	return p, nil
}

const (
	bufferSize = 1024
)

var (
	tmpBuffer    = make([]byte, bufferSize)
	tmpAlBuffers = make([]al.Buffer, maxBufferNum)
)

func (p *player) proceed() error {
	if err := al.Error(); err != 0 {
		return fmt.Errorf("audio: before proceed: %d", err)
	}
	processedNum := p.alSource.BuffersProcessed()
	if 0 < processedNum {
		bufs := tmpAlBuffers[:processedNum]
		p.alSource.UnqueueBuffers(bufs...)
		if err := al.Error(); err != 0 {
			return fmt.Errorf("audio: Unqueue in process: %d", err)
		}
		p.alBuffers = append(p.alBuffers, bufs...)
	}

	for 0 < len(p.alBuffers) {
		n, err := p.source.Read(tmpBuffer)
		if 0 < n {
			buf := p.alBuffers[0]
			p.alBuffers = p.alBuffers[1:]
			buf.BufferData(al.FormatStereo16, tmpBuffer[:n], int32(p.sampleRate))
			p.alSource.QueueBuffers(buf)
			if err := al.Error(); err != 0 {
				return fmt.Errorf("audio: Queue in process: %d", err)
			}
		}
		if err != nil {
			return err
		}
		if n == 0 {
			time.Sleep(1 * time.Millisecond)
		}
	}

	if p.alSource.State() == al.Stopped {
		al.RewindSources(p.alSource)
		al.PlaySources(p.alSource)
		if err := al.Error(); err != 0 {
			return fmt.Errorf("audio: PlaySource in process: %d", err)
		}
	}

	return nil
}

func (p *player) close() error {
	if err := al.Error(); err != 0 {
		return fmt.Errorf("audio: error before closing: %d", err)
	}
	if p.isClosed {
		return nil
	}
	var bs []al.Buffer
	al.RewindSources(p.alSource)
	al.StopSources(p.alSource)
	if n := p.alSource.BuffersQueued(); 0 < n {
		bs = make([]al.Buffer, n)
		p.alSource.UnqueueBuffers(bs...)
		p.alBuffers = append(p.alBuffers, bs...)
	}
	p.isClosed = true
	if err := al.Error(); err != 0 {
		return fmt.Errorf("audio: error after closing: %d", err)
	}
	runtime.SetFinalizer(p, nil)
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !js,!windows

package audio

import (
	"fmt"
	"io"
	"runtime"
	"time"

	"github.com/hajimehoshi/ebiten"
	"golang.org/x/mobile/exp/audio/al"
)

const (
	maxBufferNum = 8
)

// TODO: This should be in player
var totalBufferNum = 0

type player struct {
	alSource   al.Source
	alBuffers  []al.Buffer
	source     io.Reader
	sampleRate int
	isClosed   bool
}

func startPlaying(src io.Reader, sampleRate int) (*player, error) {
	if e := al.OpenDevice(); e != nil {
		return nil, fmt.Errorf("audio: OpenAL initialization failed: %v", e)
	}
	s := al.GenSources(1)
	if err := al.Error(); err != 0 {
		panic(fmt.Sprintf("audio: al.GenSources error: %d", err))
	}
	p := &player{
		alSource:   s[0],
		alBuffers:  []al.Buffer{},
		source:     src,
		sampleRate: sampleRate,
	}
	runtime.SetFinalizer(p, (*player).close)
	if err := p.start(); err != nil {
		return nil, err
	}
	return p, nil
}

const (
	bufferSize = 1024
)

var (
	tmpBuffer    = make([]byte, bufferSize)
	tmpAlBuffers = make([]al.Buffer, maxBufferNum)
)

func (p *player) proceed() error {
	if err := al.Error(); err != 0 {
		panic(fmt.Sprintf("audio: before proceed: %d", err))
	}
	processedNum := p.alSource.BuffersProcessed()
	if 0 < processedNum {
		bufs := tmpAlBuffers[:processedNum]
		p.alSource.UnqueueBuffers(bufs...)
		if err := al.Error(); err != 0 {
			panic(fmt.Sprintf("audio: Unqueue in process: %d", err))
		}
		p.alBuffers = append(p.alBuffers, bufs...)
	}

	for 0 < len(p.alBuffers) {
		n, err := p.source.Read(tmpBuffer)
		if 0 < n {
			buf := p.alBuffers[0]
			p.alBuffers = p.alBuffers[1:]
			buf.BufferData(al.FormatStereo16, tmpBuffer[:n], int32(p.sampleRate))
			p.alSource.QueueBuffers(buf)
			if err := al.Error(); err != 0 {
				panic(fmt.Sprintf("audio: Queue in process: %d", err))
			}
		}
		if err != nil {
			return err
		}
		if n == 0 {
			time.Sleep(1)
		}
	}

	if p.alSource.State() == al.Stopped {
		al.RewindSources(p.alSource)
		al.PlaySources(p.alSource)
		if err := al.Error(); err != 0 {
			panic(fmt.Sprintf("audio: PlaySource in process: %d", err))
		}
	}

	return nil
}

func (p *player) start() error {
	n := maxBufferNum - int(p.alSource.BuffersQueued()) - len(p.alBuffers)
	if 0 < n {
		p.alBuffers = append(p.alBuffers, al.GenBuffers(n)...)
		totalBufferNum += n
		if maxBufferNum < totalBufferNum {
			panic("audio: too many buffers are created")
		}
	}
	if 0 < len(p.alBuffers) {
		emptyBytes := make([]byte, bufferSize)
		for _, buf := range p.alBuffers {
			// Note that the third argument of only the first buffer is used.
			buf.BufferData(al.FormatStereo16, emptyBytes, int32(p.sampleRate))
			p.alSource.QueueBuffers(buf)
		}
		p.alBuffers = []al.Buffer{}
	}
	al.PlaySources(p.alSource)

	go func() {
		// TODO: Is it OK to close asap?
		defer p.close()
		for {
			err := p.proceed()
			if err == io.EOF {
				break
			}
			if err != nil {
				// TODO: Record the last error
				panic(err)
			}
			time.Sleep(1 * time.Second / ebiten.FPS / 2)
		}
	}()
	return nil
}

// TODO: When is this called? Can we remove this?
func (p *player) close() error {
	if err := al.Error(); err != 0 {
		panic(fmt.Sprintf("audio: error before closing: %d", err))
	}
	if p.isClosed {
		return nil
	}
	var bs []al.Buffer
	al.RewindSources(p.alSource)
	al.StopSources(p.alSource)
	n := p.alSource.BuffersQueued()
	if 0 < n {
		bs = make([]al.Buffer, n)
		p.alSource.UnqueueBuffers(bs...)
		p.alBuffers = append(p.alBuffers, bs...)
	}
	p.isClosed = true
	if err := al.Error(); err != 0 {
		panic(fmt.Sprintf("audio: error after closing: %d", err))
	}
	runtime.SetFinalizer(p, nil)
	return nil
}
<commit_msg>audio: More sleeping (#183)<commit_after>// Copyright 2015 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !js,!windows

package audio

import (
	"fmt"
	"io"
	"runtime"
	"time"

	"golang.org/x/mobile/exp/audio/al"
)

const (
	maxBufferNum = 8
)

// TODO: This should be in player
var totalBufferNum = 0

type player struct {
	alSource   al.Source
	alBuffers  []al.Buffer
	source     io.Reader
	sampleRate int
	isClosed   bool
}

func startPlaying(src io.Reader, sampleRate int) (*player, error) {
	if e := al.OpenDevice(); e != nil {
		return nil, fmt.Errorf("audio: OpenAL initialization failed: %v", e)
	}
	s := al.GenSources(1)
	if err := al.Error(); err != 0 {
		panic(fmt.Sprintf("audio: al.GenSources error: %d", err))
	}
	p := &player{
		alSource:   s[0],
		alBuffers:  []al.Buffer{},
		source:     src,
		sampleRate: sampleRate,
	}
	runtime.SetFinalizer(p, (*player).close)
	if err := p.start(); err != nil {
		return nil, err
	}
	return p, nil
}

const (
	bufferSize = 1024
)

var (
	tmpBuffer    = make([]byte, bufferSize)
	tmpAlBuffers = make([]al.Buffer, maxBufferNum)
)

func (p *player) proceed() error {
	if err := al.Error(); err != 0 {
		panic(fmt.Sprintf("audio: before proceed: %d", err))
	}
	processedNum := p.alSource.BuffersProcessed()
	if 0 < processedNum {
		bufs := tmpAlBuffers[:processedNum]
		p.alSource.UnqueueBuffers(bufs...)
		if err := al.Error(); err != 0 {
			panic(fmt.Sprintf("audio: Unqueue in process: %d", err))
		}
		p.alBuffers = append(p.alBuffers, bufs...)
	}

	for 0 < len(p.alBuffers) {
		n, err := p.source.Read(tmpBuffer)
		if 0 < n {
			buf := p.alBuffers[0]
			p.alBuffers = p.alBuffers[1:]
			buf.BufferData(al.FormatStereo16, tmpBuffer[:n], int32(p.sampleRate))
			p.alSource.QueueBuffers(buf)
			if err := al.Error(); err != 0 {
				panic(fmt.Sprintf("audio: Queue in process: %d", err))
			}
		}
		if err != nil {
			return err
		}
		if n == 0 {
			time.Sleep(1 * time.Millisecond)
		}
	}

	if p.alSource.State() == al.Stopped {
		al.RewindSources(p.alSource)
		al.PlaySources(p.alSource)
		if err := al.Error(); err != 0 {
			panic(fmt.Sprintf("audio: PlaySource in process: %d", err))
		}
	}

	return nil
}

func (p *player) start() error {
	n := maxBufferNum - int(p.alSource.BuffersQueued()) - len(p.alBuffers)
	if 0 < n {
		p.alBuffers = append(p.alBuffers, al.GenBuffers(n)...)
		totalBufferNum += n
		if maxBufferNum < totalBufferNum {
			panic("audio: too many buffers are created")
		}
	}
	if 0 < len(p.alBuffers) {
		emptyBytes := make([]byte, bufferSize)
		for _, buf := range p.alBuffers {
			// Note that the third argument of only the first buffer is used.
			buf.BufferData(al.FormatStereo16, emptyBytes, int32(p.sampleRate))
			p.alSource.QueueBuffers(buf)
		}
		p.alBuffers = []al.Buffer{}
	}
	al.PlaySources(p.alSource)

	go func() {
		// TODO: Is it OK to close asap?
		defer p.close()
		for {
			err := p.proceed()
			if err == io.EOF {
				break
			}
			if err != nil {
				// TODO: Record the last error
				panic(err)
			}
			//time.Sleep(1 * time.Second / ebiten.FPS / 2)
			time.Sleep(1 * time.Millisecond)
		}
	}()
	return nil
}

// TODO: When is this called? Can we remove this?
func (p *player) close() error {
	if err := al.Error(); err != 0 {
		panic(fmt.Sprintf("audio: error before closing: %d", err))
	}
	if p.isClosed {
		return nil
	}
	var bs []al.Buffer
	al.RewindSources(p.alSource)
	al.StopSources(p.alSource)
	n := p.alSource.BuffersQueued()
	if 0 < n {
		bs = make([]al.Buffer, n)
		p.alSource.UnqueueBuffers(bs...)
		p.alBuffers = append(p.alBuffers, bs...)
	}
	p.isClosed = true
	if err := al.Error(); err != 0 {
		panic(fmt.Sprintf("audio: error after closing: %d", err))
	}
	runtime.SetFinalizer(p, nil)
	return nil
}
<|endoftext|>


<commit_before>package eth

import (
	"bytes"
	"math"
	"math/big"

	"github.com/ethereum/go-ethereum/core/types"
	"github.com/ethereum/go-ethereum/ethutil"
	"github.com/ethereum/go-ethereum/p2p"
	"github.com/ethereum/go-ethereum/rlp"
)

const (
	ProtocolVersion    = 49
	NetworkId          = 0
	ProtocolLength     = uint64(8)
	ProtocolMaxMsgSize = 10 * 1024 * 1024
)

// eth protocol message codes
const (
	StatusMsg = iota
	GetTxMsg  // unused
	TxMsg
	GetBlockHashesMsg
	BlockHashesMsg
	GetBlocksMsg
	BlocksMsg
	NewBlockMsg
)

// ethProtocol represents the ethereum wire protocol
// instance is running on each peer
type ethProtocol struct {
	txPool       txPool
	chainManager chainManager
	blockPool    blockPool
	peer         *p2p.Peer
	id           string
	rw           p2p.MsgReadWriter
}

// backend is the interface the ethereum protocol backend should implement
// used as an argument to EthProtocol
type txPool interface {
	AddTransactions([]*types.Transaction)
}

type chainManager interface {
	GetBlockHashesFromHash(hash []byte, amount uint64) (hashes [][]byte)
	GetBlock(hash []byte) (block *types.Block)
	Status() (td *big.Int, currentBlock []byte, genesisBlock []byte)
}

type blockPool interface {
	AddBlockHashes(next func() ([]byte, bool), peerId string)
	AddBlock(block *types.Block, peerId string)
	AddPeer(td *big.Int, currentBlock []byte, peerId string, requestHashes func([]byte) error, requestBlocks func([][]byte) error, peerError func(int, string, ...interface{})) (best bool)
	RemovePeer(peerId string)
}

// message structs used for rlp decoding
type newBlockMsgData struct {
	Block *types.Block
	TD    *big.Int
}

type getBlockHashesMsgData struct {
	Hash   []byte
	Amount uint64
}

// main entrypoint, wrappers starting a server running the eth protocol
// use this constructor to attach the protocol ("class") to server caps
// the Dev p2p layer then runs the protocol instance on each peer
func EthProtocol(txPool txPool, chainManager chainManager, blockPool blockPool) p2p.Protocol {
	return p2p.Protocol{
		Name:    "eth",
		Version: ProtocolVersion,
		Length:  ProtocolLength,
		Run: func(peer *p2p.Peer, rw p2p.MsgReadWriter) error {
			return runEthProtocol(txPool, chainManager, blockPool, peer, rw)
		},
	}
}

// the main loop that handles incoming messages
// note RemovePeer in the post-disconnect hook
func runEthProtocol(txPool txPool, chainManager chainManager, blockPool blockPool, peer *p2p.Peer, rw p2p.MsgReadWriter) (err error) {
	self := &ethProtocol{
		txPool:       txPool,
		chainManager: chainManager,
		blockPool:    blockPool,
		rw:           rw,
		peer:         peer,
		id:           (string)(peer.Identity().Pubkey()),
	}
	err = self.handleStatus()
	if err == nil {
		go func() {
			for {
				err = self.handle()
				if err != nil {
					self.blockPool.RemovePeer(self.id)
					break
				}
			}
		}()
	}
	return
}

func (self *ethProtocol) handle() error {
	msg, err := self.rw.ReadMsg()
	if err != nil {
		return err
	}
	if msg.Size > ProtocolMaxMsgSize {
		return ProtocolError(ErrMsgTooLarge, "%v > %v", msg.Size, ProtocolMaxMsgSize)
	}
	// make sure that the payload has been fully consumed
	defer msg.Discard()

	switch msg.Code {

	case StatusMsg:
		return ProtocolError(ErrExtraStatusMsg, "")

	case TxMsg:
		// TODO: rework using lazy RLP stream
		var txs []*types.Transaction
		if err := msg.Decode(&txs); err != nil {
			return ProtocolError(ErrDecode, "%v", err)
		}
		self.txPool.AddTransactions(txs)

	case GetBlockHashesMsg:
		var request getBlockHashesMsgData
		if err := msg.Decode(&request); err != nil {
			return ProtocolError(ErrDecode, "%v", err)
		}
		hashes := self.chainManager.GetBlockHashesFromHash(request.Hash, request.Amount)
		return self.rw.EncodeMsg(BlockHashesMsg, ethutil.ByteSliceToInterface(hashes)...)

	case BlockHashesMsg:
		// TODO: redo using lazy decode , this way very inefficient on known chains
		msgStream := rlp.NewListStream(msg.Payload, uint64(msg.Size))
		var err error
		iter := func() (hash []byte, ok bool) {
			hash, err = msgStream.Bytes()
			if err == nil {
				ok = true
			}
			return
		}
		self.blockPool.AddBlockHashes(iter, self.id)
		if err != nil && err != rlp.EOL {
			return ProtocolError(ErrDecode, "%v", err)
		}

	case GetBlocksMsg:
		var blockHashes [][]byte
		if err := msg.Decode(&blockHashes); err != nil {
			return ProtocolError(ErrDecode, "%v", err)
		}
		max := int(math.Min(float64(len(blockHashes)), blockHashesBatchSize))
		var blocks []interface{}
		for i, hash := range blockHashes {
			if i >= max {
				break
			}
			block := self.chainManager.GetBlock(hash)
			if block != nil {
				blocks = append(blocks, block.Value().Raw())
			}
		}
		return self.rw.EncodeMsg(BlocksMsg, blocks...)

	case BlocksMsg:
		msgStream := rlp.NewListStream(msg.Payload, uint64(msg.Size))
		for {
			var block *types.Block
			if err := msgStream.Decode(&block); err != nil {
				if err == rlp.EOL {
					break
				} else {
					return ProtocolError(ErrDecode, "%v", err)
				}
			}
			self.blockPool.AddBlock(block, self.id)
		}

	case NewBlockMsg:
		var request newBlockMsgData
		if err := msg.Decode(&request); err != nil {
			return ProtocolError(ErrDecode, "%v", err)
		}
		hash := request.Block.Hash()
		// to simplify backend interface adding a new block
		// uses AddPeer followed by AddHashes, AddBlock only if peer is the best peer
		// (or selected as new best peer)
		if self.blockPool.AddPeer(request.TD, hash, self.id, self.requestBlockHashes, self.requestBlocks, self.protoErrorDisconnect) {
			called := true
			iter := func() (hash []byte, ok bool) {
				if called {
					called = false
					return hash, true
				} else {
					return
				}
			}
			self.blockPool.AddBlockHashes(iter, self.id)
			self.blockPool.AddBlock(request.Block, self.id)
		}

	default:
		return ProtocolError(ErrInvalidMsgCode, "%v", msg.Code)
	}
	return nil
}

type statusMsgData struct {
	ProtocolVersion uint
	NetworkId       uint
	TD              *big.Int
	CurrentBlock    []byte
	GenesisBlock    []byte
}

func (self *ethProtocol) statusMsg() p2p.Msg {
	td, currentBlock, genesisBlock := self.chainManager.Status()

	return p2p.NewMsg(StatusMsg,
		uint32(ProtocolVersion),
		uint32(NetworkId),
		td,
		currentBlock,
		genesisBlock,
	)
}

func (self *ethProtocol) handleStatus() error {
	// send precanned status message
	if err := self.rw.WriteMsg(self.statusMsg()); err != nil {
		return err
	}

	// read and handle remote status
	msg, err := self.rw.ReadMsg()
	if err != nil {
		return err
	}

	if msg.Code != StatusMsg {
		return ProtocolError(ErrNoStatusMsg, "first msg has code %x (!= %x)", msg.Code, StatusMsg)
	}

	if msg.Size > ProtocolMaxMsgSize {
		return ProtocolError(ErrMsgTooLarge, "%v > %v", msg.Size, ProtocolMaxMsgSize)
	}

	var status statusMsgData
	if err := msg.Decode(&status); err != nil {
		return ProtocolError(ErrDecode, "%v", err)
	}

	_, _, genesisBlock := self.chainManager.Status()

	if bytes.Compare(status.GenesisBlock, genesisBlock) != 0 {
		return ProtocolError(ErrGenesisBlockMismatch, "%x (!= %x)", status.GenesisBlock, genesisBlock)
	}

	if status.NetworkId != NetworkId {
		return ProtocolError(ErrNetworkIdMismatch, "%d (!= %d)", status.NetworkId, NetworkId)
	}

	if ProtocolVersion != status.ProtocolVersion {
		return ProtocolError(ErrProtocolVersionMismatch, "%d (!= %d)", status.ProtocolVersion, ProtocolVersion)
	}

	self.peer.Infof("Peer is [eth] capable (%d/%d). TD=%v H=%x\n", status.ProtocolVersion, status.NetworkId, status.TD, status.CurrentBlock[:4])

	//self.blockPool.AddPeer(status.TD, status.CurrentBlock, self.id, self.requestBlockHashes, self.requestBlocks, self.protoErrorDisconnect)
	self.peer.Infoln("AddPeer(IGNORED)")

	return nil
}

func (self *ethProtocol) requestBlockHashes(from []byte) error {
	self.peer.Debugf("fetching hashes (%d) %x...\n", blockHashesBatchSize, from[0:4])
	return self.rw.EncodeMsg(GetBlockHashesMsg, from, blockHashesBatchSize)
}

func (self *ethProtocol) requestBlocks(hashes [][]byte) error {
	self.peer.Debugf("fetching %v blocks", len(hashes))
	return self.rw.EncodeMsg(GetBlocksMsg, ethutil.ByteSliceToInterface(hashes))
}

func (self *ethProtocol) protoError(code int, format string, params ...interface{}) (err *protocolError) {
	err = ProtocolError(code, format, params...)
	if err.Fatal() {
		self.peer.Errorln(err)
	} else {
		self.peer.Debugln(err)
	}
	return
}

func (self *ethProtocol) protoErrorDisconnect(code int, format string, params ...interface{}) {
	err := ProtocolError(code, format, params...)
	if err.Fatal() {
		self.peer.Errorln(err)
		// disconnect
	} else {
		self.peer.Debugln(err)
	}

}
<commit_msg>Removed goroutine from "Run"<commit_after>package eth

import (
	"bytes"
	"fmt"
	"math"
	"math/big"

	"github.com/ethereum/go-ethereum/core/types"
	"github.com/ethereum/go-ethereum/ethutil"
	"github.com/ethereum/go-ethereum/p2p"
	"github.com/ethereum/go-ethereum/rlp"
)

const (
	ProtocolVersion    = 49
	NetworkId          = 0
	ProtocolLength     = uint64(8)
	ProtocolMaxMsgSize = 10 * 1024 * 1024
)

// eth protocol message codes
const (
	StatusMsg = iota
	GetTxMsg  // unused
	TxMsg
	GetBlockHashesMsg
	BlockHashesMsg
	GetBlocksMsg
	BlocksMsg
	NewBlockMsg
)

// ethProtocol represents the ethereum wire protocol
// instance is running on each peer
type ethProtocol struct {
	txPool       txPool
	chainManager chainManager
	blockPool    blockPool
	peer         *p2p.Peer
	id           string
	rw           p2p.MsgReadWriter
}

// backend is the interface the ethereum protocol backend should implement
// used as an argument to EthProtocol
type txPool interface {
	AddTransactions([]*types.Transaction)
}

type chainManager interface {
	GetBlockHashesFromHash(hash []byte, amount uint64) (hashes [][]byte)
	GetBlock(hash []byte) (block *types.Block)
	Status() (td *big.Int, currentBlock []byte, genesisBlock []byte)
}

type blockPool interface {
	AddBlockHashes(next func() ([]byte, bool), peerId string)
	AddBlock(block *types.Block, peerId string)
	AddPeer(td *big.Int, currentBlock []byte, peerId string, requestHashes func([]byte) error, requestBlocks func([][]byte) error, peerError func(int, string, ...interface{})) (best bool)
	RemovePeer(peerId string)
}

// message structs used for rlp decoding
type newBlockMsgData struct {
	Block *types.Block
	TD    *big.Int
}

type getBlockHashesMsgData struct {
	Hash   []byte
	Amount uint64
}

// main entrypoint, wrappers starting a server running the eth protocol
// use this constructor to attach the protocol ("class") to server caps
// the Dev p2p layer then runs the protocol instance on each peer
func EthProtocol(txPool txPool, chainManager chainManager, blockPool blockPool) p2p.Protocol {
	return p2p.Protocol{
		Name:    "eth",
		Version: ProtocolVersion,
		Length:  ProtocolLength,
		Run: func(peer *p2p.Peer, rw p2p.MsgReadWriter) error {
			return runEthProtocol(txPool, chainManager, blockPool, peer, rw)
		},
	}
}

// the main loop that handles incoming messages
// note RemovePeer in the post-disconnect hook
func runEthProtocol(txPool txPool, chainManager chainManager, blockPool blockPool, peer *p2p.Peer, rw p2p.MsgReadWriter) (err error) {
	self := &ethProtocol{
		txPool:       txPool,
		chainManager: chainManager,
		blockPool:    blockPool,
		rw:           rw,
		peer:         peer,
		id:           (string)(peer.Identity().Pubkey()),
	}
	err = self.handleStatus()
	if err == nil {
		for {
			err = self.handle()
			if err != nil {
				fmt.Println(err)
				self.blockPool.RemovePeer(self.id)
				break
			}
		}
	}
	return
}

func (self *ethProtocol) handle() error {
	msg, err := self.rw.ReadMsg()
	if err != nil {
		return err
	}
	if msg.Size > ProtocolMaxMsgSize {
		return ProtocolError(ErrMsgTooLarge, "%v > %v", msg.Size, ProtocolMaxMsgSize)
	}
	// make sure that the payload has been fully consumed
	defer msg.Discard()

	switch msg.Code {

	case StatusMsg:
		return ProtocolError(ErrExtraStatusMsg, "")

	case TxMsg:
		// TODO: rework using lazy RLP stream
		var txs []*types.Transaction
		if err := msg.Decode(&txs); err != nil {
			return ProtocolError(ErrDecode, "%v", err)
		}
		self.txPool.AddTransactions(txs)

	case GetBlockHashesMsg:
		var request getBlockHashesMsgData
		if err := msg.Decode(&request); err != nil {
			return ProtocolError(ErrDecode, "%v", err)
		}
		hashes := self.chainManager.GetBlockHashesFromHash(request.Hash, request.Amount)
		return self.rw.EncodeMsg(BlockHashesMsg, ethutil.ByteSliceToInterface(hashes)...)

	case BlockHashesMsg:
		// TODO: redo using lazy decode , this way very inefficient on known chains
		msgStream := rlp.NewListStream(msg.Payload, uint64(msg.Size))
		var err error
		iter := func() (hash []byte, ok bool) {
			hash, err = msgStream.Bytes()
			if err == nil {
				ok = true
			}
			return
		}
		self.blockPool.AddBlockHashes(iter, self.id)
		if err != nil && err != rlp.EOL {
			return ProtocolError(ErrDecode, "%v", err)
		}

	case GetBlocksMsg:
		var blockHashes [][]byte
		if err := msg.Decode(&blockHashes); err != nil {
			return ProtocolError(ErrDecode, "%v", err)
		}
		max := int(math.Min(float64(len(blockHashes)), blockHashesBatchSize))
		var blocks []interface{}
		for i, hash := range blockHashes {
			if i >= max {
				break
			}
			block := self.chainManager.GetBlock(hash)
			if block != nil {
				blocks = append(blocks, block.Value().Raw())
			}
		}
		return self.rw.EncodeMsg(BlocksMsg, blocks...)

	case BlocksMsg:
		msgStream := rlp.NewListStream(msg.Payload, uint64(msg.Size))
		for {
			var block *types.Block
			if err := msgStream.Decode(&block); err != nil {
				if err == rlp.EOL {
					break
				} else {
					return ProtocolError(ErrDecode, "%v", err)
				}
			}
			self.blockPool.AddBlock(block, self.id)
		}

	case NewBlockMsg:
		var request newBlockMsgData
		if err := msg.Decode(&request); err != nil {
			return ProtocolError(ErrDecode, "%v", err)
		}
		hash := request.Block.Hash()
		// to simplify backend interface adding a new block
		// uses AddPeer followed by AddHashes, AddBlock only if peer is the best peer
		// (or selected as new best peer)
		if self.blockPool.AddPeer(request.TD, hash, self.id, self.requestBlockHashes, self.requestBlocks, self.protoErrorDisconnect) {
			called := true
			iter := func() (hash []byte, ok bool) {
				if called {
					called = false
					return hash, true
				} else {
					return
				}
			}
			self.blockPool.AddBlockHashes(iter, self.id)
			self.blockPool.AddBlock(request.Block, self.id)
		}

	default:
		return ProtocolError(ErrInvalidMsgCode, "%v", msg.Code)
	}
	return nil
}

type statusMsgData struct {
	ProtocolVersion uint
	NetworkId       uint
	TD              *big.Int
	CurrentBlock    []byte
	GenesisBlock    []byte
}

func (self *ethProtocol) statusMsg() p2p.Msg {
	td, currentBlock, genesisBlock := self.chainManager.Status()

	return p2p.NewMsg(StatusMsg,
		uint32(ProtocolVersion),
		uint32(NetworkId),
		td,
		currentBlock,
		genesisBlock,
	)
}

func (self *ethProtocol) handleStatus() error {
	// send precanned status message
	if err := self.rw.WriteMsg(self.statusMsg()); err != nil {
		return err
	}

	// read and handle remote status
	msg, err := self.rw.ReadMsg()
	if err != nil {
		return err
	}

	if msg.Code != StatusMsg {
		return ProtocolError(ErrNoStatusMsg, "first msg has code %x (!= %x)", msg.Code, StatusMsg)
	}

	if msg.Size > ProtocolMaxMsgSize {
		return ProtocolError(ErrMsgTooLarge, "%v > %v", msg.Size, ProtocolMaxMsgSize)
	}

	var status statusMsgData
	if err := msg.Decode(&status); err != nil {
		return ProtocolError(ErrDecode, "%v", err)
	}

	_, _, genesisBlock := self.chainManager.Status()

	if bytes.Compare(status.GenesisBlock, genesisBlock) != 0 {
		return ProtocolError(ErrGenesisBlockMismatch, "%x (!= %x)", status.GenesisBlock, genesisBlock)
	}

	if status.NetworkId != NetworkId {
		return ProtocolError(ErrNetworkIdMismatch, "%d (!= %d)", status.NetworkId, NetworkId)
	}

	if ProtocolVersion != status.ProtocolVersion {
		return ProtocolError(ErrProtocolVersionMismatch, "%d (!= %d)", status.ProtocolVersion, ProtocolVersion)
	}

	self.peer.Infof("Peer is [eth] capable (%d/%d). TD=%v H=%x\n", status.ProtocolVersion, status.NetworkId, status.TD, status.CurrentBlock[:4])

	//self.blockPool.AddPeer(status.TD, status.CurrentBlock, self.id, self.requestBlockHashes, self.requestBlocks, self.protoErrorDisconnect)
	self.peer.Infoln("AddPeer(IGNORED)")

	return nil
}

func (self *ethProtocol) requestBlockHashes(from []byte) error {
	self.peer.Debugf("fetching hashes (%d) %x...\n", blockHashesBatchSize, from[0:4])
	return self.rw.EncodeMsg(GetBlockHashesMsg, from, blockHashesBatchSize)
}

func (self *ethProtocol) requestBlocks(hashes [][]byte) error {
	self.peer.Debugf("fetching %v blocks", len(hashes))
	return self.rw.EncodeMsg(GetBlocksMsg, ethutil.ByteSliceToInterface(hashes))
}

func (self *ethProtocol) protoError(code int, format string, params ...interface{}) (err *protocolError) {
	err = ProtocolError(code, format, params...)
	if err.Fatal() {
		self.peer.Errorln(err)
	} else {
		self.peer.Debugln(err)
	}
	return
}

func (self *ethProtocol) protoErrorDisconnect(code int, format string, params ...interface{}) {
	err := ProtocolError(code, format, params...)
	if err.Fatal() {
		self.peer.Errorln(err)
		// disconnect
	} else {
		self.peer.Debugln(err)
	}

}
<|endoftext|>


<commit_before>package ethutil

import "reflect"

// The list type is an anonymous slice handler which can be used
// for containing any slice type to use in an environment which
// does not support slice types (e.g., JavaScript, QML)
type List struct {
	list   reflect.Value
	Length int
}

// Initialise a new list. Panics if non-slice type is given.
func NewList(t interface{}) *List {
	list := reflect.ValueOf(t)
	if list.Kind() != reflect.Slice {
		panic("list container initialized with a non-slice type")
	}

	return &List{list, list.Len()}
}

func EmptyList() *List {
	return NewList([]interface{}{})
}

// Get N element from the embedded slice. Returns nil if OOB.
func (self *List) Get(i int) interface{} {
	if self.list.Len() > i {
		return self.list.Index(i).Interface()
	}

	return nil
}

// Appends value at the end of the slice. Panics when incompatible value
// is given.
func (self *List) Append(v interface{}) {
	self.list = reflect.Append(self.list, reflect.ValueOf(v))
	self.Length = self.list.Len()
}

// Returns the underlying slice as interface.
func (self *List) Interface() interface{} {
	return self.list.Interface()
}
<commit_msg>Added JavaScript JSON helper<commit_after>package ethutil

import (
	"encoding/json"
	"reflect"
)

// The list type is an anonymous slice handler which can be used
// for containing any slice type to use in an environment which
// does not support slice types (e.g., JavaScript, QML)
type List struct {
	list   reflect.Value
	Length int
}

// Initialise a new list. Panics if non-slice type is given.
func NewList(t interface{}) *List {
	list := reflect.ValueOf(t)
	if list.Kind() != reflect.Slice {
		panic("list container initialized with a non-slice type")
	}

	return &List{list, list.Len()}
}

func EmptyList() *List {
	return NewList([]interface{}{})
}

// Get N element from the embedded slice. Returns nil if OOB.
func (self *List) Get(i int) interface{} {
	if self.list.Len() > i {
		return self.list.Index(i).Interface()
	}

	return nil
}

// Appends value at the end of the slice. Panics when incompatible value
// is given.
func (self *List) Append(v interface{}) {
	self.list = reflect.Append(self.list, reflect.ValueOf(v))
	self.Length = self.list.Len()
}

// Returns the underlying slice as interface.
func (self *List) Interface() interface{} {
	return self.list.Interface()
}

// For JavaScript <3
func (self *List) ToJSON() string {
	var list []interface{}
	for i := 0; i < self.Length; i++ {
		list = append(list, self.Get(i))
	}

	data, _ := json.Marshal(list)

	return string(data)
}
<|endoftext|>


<commit_before>package orchestrators

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"regexp"
	"sort"
	"strings"
	"time"
	"unicode/utf8"

	log "github.com/Sirupsen/logrus"
	"github.com/rancher/go-rancher-metadata/metadata"
	"github.com/rancher/go-rancher/v2"
	"golang.org/x/net/websocket"

	"github.com/camptocamp/bivac/handler"
	"github.com/camptocamp/bivac/volume"
)

// CattleOrchestrator implements a container orchestrator for Cattle
type CattleOrchestrator struct {
	Handler *handler.Bivac
	Client  *client.RancherClient
}

// NewCattleOrchestrator creates a Cattle client
func NewCattleOrchestrator(c *handler.Bivac) (o *CattleOrchestrator) {
	var err error
	o = &CattleOrchestrator{
		Handler: c,
	}

	o.Client, err = client.NewRancherClient(&client.ClientOpts{
		Url:       o.Handler.Config.Cattle.URL,
		AccessKey: o.Handler.Config.Cattle.AccessKey,
		SecretKey: o.Handler.Config.Cattle.SecretKey,
		Timeout:   30 * time.Second,
	})
	if err != nil {
		log.Errorf("failed to create a new Rancher client: %s", err)
	}

	return
}

// GetName returns the orchestrator name
func (*CattleOrchestrator) GetName() string {
	return "Cattle"
}

// GetPath returns the path of the backup
func (*CattleOrchestrator) GetPath(v *volume.Volume) string {
	return v.Hostname + "/" + v.Name
}

// GetHandler returns the Orchestrator's handler
func (o *CattleOrchestrator) GetHandler() *handler.Bivac {
	return o.Handler
}

// GetVolumes returns the Cattle volumes
func (o *CattleOrchestrator) GetVolumes() (volumes []*volume.Volume, err error) {
	c := o.Handler

	vs, err := o.Client.Volume.List(&client.ListOpts{
		Filters: map[string]interface{}{
			"limit": -2,
			"all":   true,
		},
	})
	if err != nil {
		log.Errorf("failed to list volumes: %s", err)
	}

	var mountpoint string
	for _, v := range vs.Data {
		if len(v.Mounts) < 1 {
			mountpoint = "/data"
		} else {
			mountpoint = v.Mounts[0].Path
		}

		var hostID, hostname string
		var spc *client.StoragePoolCollection
		err := o.rawAPICall("GET", v.Links["storagePools"], &spc)
		if err != nil {
			log.Errorf("failed to retrieve storage pool from volume %s: %s", v.Name, err)
			continue
		}

		if len(spc.Data) == 0 {
			log.Errorf("no storage pool for the volume %s: %s", v.Name, err)
			continue
		}

		if len(spc.Data[0].HostIds) == 0 {
			log.Errorf("no host for the volume %s: %s", v.Name, err)
			continue
		}

		hostID = spc.Data[0].HostIds[0]

		h, err := o.Client.Host.ById(hostID)
		if err != nil {
			log.Errorf("failed to retrieve host from id %s: %s", hostID, err)
			hostname = ""
		} else {
			hostname = h.Hostname
		}

		nv := &volume.Volume{
			Config:     &volume.Config{},
			Mountpoint: mountpoint,
			ID:         v.Id,
			Name:       v.Name,
			HostBind:   hostID,
			Hostname:   hostname,
		}

		v := volume.NewVolume(nv, c.Config, hostname)
		if b, r, s := o.blacklistedVolume(v); b {
			log.WithFields(log.Fields{
				"volume": v.Name,
				"reason": r,
				"source": s,
			}).Info("Ignoring volume")
			continue
		}
		volumes = append(volumes, v)
	}
	return
}

func createWorkerName() string {
	var letter = []rune("abcdefghijklmnopqrstuvwxyz0123456789")
	b := make([]rune, 10)
	for i := range b {
		b[i] = letter[rand.Intn(len(letter))]
	}
	return "bivac-worker-" + string(b)
}

// LaunchContainer starts a containe using the Cattle orchestrator
func (o *CattleOrchestrator) LaunchContainer(image string, env map[string]string, cmd []string, volumes []*volume.Volume) (state int, stdout string, err error) {
	environment := make(map[string]interface{}, len(env))
	for envKey, envVal := range env {
		environment[envKey] = envVal
	}

	var hostbind string
	if len(volumes) > 0 {
		hostbind = volumes[0].HostBind
	} else {
		hostbind = ""
	}

	cvs := []string{}
	for _, v := range volumes {
		cvs = append(cvs, v.Name+":"+v.Mountpoint)
	}

	metadataClient, err := metadata.NewClientAndWait("http://rancher-metadata/latest/")
	if err != nil {
		log.Errorf("Error initiating metadata client: %v", err)
		err = fmt.Errorf("can't build client")
		return
	}
	managerCont, err := metadataClient.GetSelfContainer()
	if err != nil {
		log.Errorf("failed to get current container: %s", err)
		err = fmt.Errorf("can't inspect current container")
		return
	}
	containers, err := o.Client.Container.List(&client.ListOpts{
		Filters: map[string]interface{}{
			"limit": -2,
			"all":   true,
		},
	})
	if err != nil {
		log.Errorf("failed to get container list: %s", err)
		err = fmt.Errorf("can't get container list")
		return
	}
	var managerContainer *client.Container
	for _, container := range containers.Data {
		if container.Name == managerCont.Name {
			managerContainer = &container
			break
		}
	}
	if managerContainer == nil {
		log.Errorf("failed to get manager container: %v", err)
		return
	}
	for envKey, envVal := range managerContainer.Environment {
		environment[envKey] = envVal
	}

	container, err := o.Client.Container.Create(&client.Container{
		Name:            createWorkerName(),
		RequestedHostId: hostbind,
		ImageUuid:       "docker:" + image,
		Command:         cmd,
		Environment:     environment,
		RestartPolicy: &client.RestartPolicy{
			MaximumRetryCount: 1,
			Name:              "on-failure",
		},
		DataVolumes: cvs,
	})
	if err != nil {
		log.Errorf("failed to create worker container: %s", err)
		err = fmt.Errorf("can't create worker container")
		return
	}

	defer o.DeleteWorker(container)

	stopped := false
	terminated := false
	for !terminated {
		container, err := o.Client.Container.ById(container.Id)
		if err != nil {
			log.Errorf("failed to inspect worker: %s", err)
			err = fmt.Errorf("can't inspect worker")
			return 1, "", err
		}

		// This workaround is awful but it's the only way to know if the container failed.
		if container.State == "stopped" {
			if container.StartCount == 1 {
				if stopped == false {
					stopped = true
					time.Sleep(5 * time.Second)
				} else {
					terminated = true
					state = 0
				}
			} else {
				state = 1
				terminated = true
			}
		}
	}

	var hostAccess *client.HostAccess
	err = o.rawAPICall("POST", container.Links["self"]+"/?action=logs", &hostAccess)
	if err != nil {
		log.Errorf("failed to read response from rancher: %s", err)
		err = fmt.Errorf("can't access worker logs")
		return
	}

	origin := o.Handler.Config.Cattle.URL

	u, err := url.Parse(hostAccess.Url)
	if err != nil {
		log.Errorf("failed to parse rancher server url: %s", err)
		err = fmt.Errorf("can't access worker logs")
	}
	q := u.Query()
	q.Set("token", hostAccess.Token)
	u.RawQuery = q.Encode()

	ws, err := websocket.Dial(u.String(), "", origin)
	if err != nil {
		log.Errorf("failed to open websocket with rancher server: %s", err)
		err = fmt.Errorf("can't access worker logs")
		return
	}

	defer ws.Close()

	var data bytes.Buffer
	io.Copy(&data, ws)

	re := regexp.MustCompile(`(?m)[0-9]{2,} [ZT\-\:\.0-9]+ (.*)`)
	for _, line := range re.FindAllStringSubmatch(data.String(), -1) {
		stdout = strings.Join([]string{stdout, line[1]}, "\n")
	}

	log.WithFields(log.Fields{
		"container": container.Id,
		"volumes":   strings.Join(cvs[:], ","),
		"cmd":       strings.Join(cmd[:], " "),
	}).Debug(stdout)
	return
}

// DeleteWorker deletes a worker
func (o *CattleOrchestrator) DeleteWorker(container *client.Container) {
	err := o.Client.Container.Delete(container)
	if err != nil {
		log.Errorf("failed to delete worker: %s", err)
	}
	removed := false
	for !removed {
		container, err := o.Client.Container.ById(container.Id)
		if err != nil {
			log.Errorf("failed to inspect worker: %s", err)
		}
		if container.Removed != "" {
			removed = true
		}
	}
	return
}

// GetContainersMountingVolume returns containers mounting a volume
func (o *CattleOrchestrator) GetContainersMountingVolume(v *volume.Volume) (containers []*volume.MountedVolume, err error) {
	vol, err := o.Client.Volume.ById(v.ID)

	if err != nil {
		log.Errorf("failed to get volume: %s", err)
	}

	for _, mount := range vol.Mounts {
		instance, err := o.Client.Container.ById(mount.InstanceId)
		if err != nil {
			log.Errorf("failed to inspect container %s", mount.InstanceId)
			continue
		}
		if instance.State != "running" {
			continue
		}

		mv := &volume.MountedVolume{
			ContainerID: mount.InstanceId,
			Volume:      v,
			Path:        mount.Path,
		}
		containers = append(containers, mv)
	}
	return
}

// ContainerExec executes a command in a container
func (o *CattleOrchestrator) ContainerExec(mountedVolumes *volume.MountedVolume, command []string) (stdout string, err error) {

	container, err := o.Client.Container.ById(mountedVolumes.ContainerID)
	if err != nil {
		log.Errorf("failed to retrieve container: %s", err)
		return
	}

	hostAccess, err := o.Client.Container.ActionExecute(container, &client.ContainerExec{
		AttachStdin:  false,
		AttachStdout: true,
		Command:      command,
		Tty:          false,
	})
	if err != nil {
		log.Errorf("failed to prepare command execution in container: %s", err)
		return
	}

	origin := o.Handler.Config.Cattle.URL

	u, err := url.Parse(hostAccess.Url)
	if err != nil {
		log.Errorf("failed to parse rancher server url: %s", err)
	}
	q := u.Query()
	q.Set("token", hostAccess.Token)
	u.RawQuery = q.Encode()

	ws, err := websocket.Dial(u.String(), "", origin)
	if err != nil {
		log.Errorf("failed to open websocket with rancher server: %s", err)
	}

	var data bytes.Buffer
	io.Copy(&data, ws)

	rawStdout, _ := base64.StdEncoding.DecodeString(data.String())
	stdout = string(rawStdout)

	log.WithFields(log.Fields{
		"container": mountedVolumes.ContainerID,
		"cmd":       strings.Join(command[:], " "),
	}).Debug(stdout)
	return
}

func (o *CattleOrchestrator) blacklistedVolume(vol *volume.Volume) (bool, string, string) {
	if utf8.RuneCountInString(vol.Name) == 64 || utf8.RuneCountInString(vol.Name) == 0 {
		return true, "unnamed", ""
	}

	if strings.Contains(vol.Name, "/") {
		return true, "blacklisted", "path"
	}

	// Use whitelist if defined
	if l := o.Handler.Config.VolumesWhitelist; len(l) > 0 && l[0] != "" {
		sort.Strings(l)
		i := sort.SearchStrings(l, vol.Name)
		if i < len(l) && l[i] == vol.Name {
			return false, "", ""
		}
		return true, "blacklisted", "whitelist config"
	}

	list := o.Handler.Config.VolumesBlacklist
	sort.Strings(list)
	i := sort.SearchStrings(list, vol.Name)
	if i < len(list) && list[i] == vol.Name {
		return true, "blacklisted", "blacklist config"
	}

	if vol.Config.Ignore {
		return true, "blacklisted", "volume config"
	}

	return false, "", ""
}

func (o *CattleOrchestrator) rawAPICall(method, endpoint string, object interface{}) (err error) {
	// TODO: Use go-rancher.
	// It was impossible to use it, maybe a problem in go-rancher or a lack of documentation.
	clientHTTP := &http.Client{}
	v := url.Values{}
	req, err := http.NewRequest(method, endpoint, strings.NewReader(v.Encode()))
	req.SetBasicAuth(o.Handler.Config.Cattle.AccessKey, o.Handler.Config.Cattle.SecretKey)
	resp, err := clientHTTP.Do(req)
	if err != nil {
		log.Errorf("failed to execute POST request: %s", err)
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Errorf("failed to read response from rancher: %s", err)
	}
	err = json.Unmarshal(body, object)
	if err != nil {
		log.Errorf("failed to unmarshal: %s", err)
	}
	return
}

func detectCattle() bool {
	_, err := net.LookupHost("rancher-metadata")
	if err != nil {
		return false
	}
	return true
}
<commit_msg>cattle: reload container config before reading logs<commit_after>package orchestrators

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"regexp"
	"sort"
	"strings"
	"time"
	"unicode/utf8"

	log "github.com/Sirupsen/logrus"
	"github.com/rancher/go-rancher-metadata/metadata"
	"github.com/rancher/go-rancher/v2"
	"golang.org/x/net/websocket"

	"github.com/camptocamp/bivac/handler"
	"github.com/camptocamp/bivac/volume"
)

// CattleOrchestrator implements a container orchestrator for Cattle
type CattleOrchestrator struct {
	Handler *handler.Bivac
	Client  *client.RancherClient
}

// NewCattleOrchestrator creates a Cattle client
func NewCattleOrchestrator(c *handler.Bivac) (o *CattleOrchestrator) {
	var err error
	o = &CattleOrchestrator{
		Handler: c,
	}

	o.Client, err = client.NewRancherClient(&client.ClientOpts{
		Url:       o.Handler.Config.Cattle.URL,
		AccessKey: o.Handler.Config.Cattle.AccessKey,
		SecretKey: o.Handler.Config.Cattle.SecretKey,
		Timeout:   30 * time.Second,
	})
	if err != nil {
		log.Errorf("failed to create a new Rancher client: %s", err)
	}

	return
}

// GetName returns the orchestrator name
func (*CattleOrchestrator) GetName() string {
	return "Cattle"
}

// GetPath returns the path of the backup
func (*CattleOrchestrator) GetPath(v *volume.Volume) string {
	return v.Hostname + "/" + v.Name
}

// GetHandler returns the Orchestrator's handler
func (o *CattleOrchestrator) GetHandler() *handler.Bivac {
	return o.Handler
}

// GetVolumes returns the Cattle volumes
func (o *CattleOrchestrator) GetVolumes() (volumes []*volume.Volume, err error) {
	c := o.Handler

	vs, err := o.Client.Volume.List(&client.ListOpts{
		Filters: map[string]interface{}{
			"limit": -2,
			"all":   true,
		},
	})
	if err != nil {
		log.Errorf("failed to list volumes: %s", err)
	}

	var mountpoint string
	for _, v := range vs.Data {
		if len(v.Mounts) < 1 {
			mountpoint = "/data"
		} else {
			mountpoint = v.Mounts[0].Path
		}

		var hostID, hostname string
		var spc *client.StoragePoolCollection
		err := o.rawAPICall("GET", v.Links["storagePools"], &spc)
		if err != nil {
			log.Errorf("failed to retrieve storage pool from volume %s: %s", v.Name, err)
			continue
		}

		if len(spc.Data) == 0 {
			log.Errorf("no storage pool for the volume %s: %s", v.Name, err)
			continue
		}

		if len(spc.Data[0].HostIds) == 0 {
			log.Errorf("no host for the volume %s: %s", v.Name, err)
			continue
		}

		hostID = spc.Data[0].HostIds[0]

		h, err := o.Client.Host.ById(hostID)
		if err != nil {
			log.Errorf("failed to retrieve host from id %s: %s", hostID, err)
			hostname = ""
		} else {
			hostname = h.Hostname
		}

		nv := &volume.Volume{
			Config:     &volume.Config{},
			Mountpoint: mountpoint,
			ID:         v.Id,
			Name:       v.Name,
			HostBind:   hostID,
			Hostname:   hostname,
		}

		v := volume.NewVolume(nv, c.Config, hostname)
		if b, r, s := o.blacklistedVolume(v); b {
			log.WithFields(log.Fields{
				"volume": v.Name,
				"reason": r,
				"source": s,
			}).Info("Ignoring volume")
			continue
		}
		volumes = append(volumes, v)
	}
	return
}

func createWorkerName() string {
	var letter = []rune("abcdefghijklmnopqrstuvwxyz0123456789")
	b := make([]rune, 10)
	for i := range b {
		b[i] = letter[rand.Intn(len(letter))]
	}
	return "bivac-worker-" + string(b)
}

// LaunchContainer starts a containe using the Cattle orchestrator
func (o *CattleOrchestrator) LaunchContainer(image string, env map[string]string, cmd []string, volumes []*volume.Volume) (state int, stdout string, err error) {
	environment := make(map[string]interface{}, len(env))
	for envKey, envVal := range env {
		environment[envKey] = envVal
	}

	var hostbind string
	if len(volumes) > 0 {
		hostbind = volumes[0].HostBind
	} else {
		hostbind = ""
	}

	cvs := []string{}
	for _, v := range volumes {
		cvs = append(cvs, v.Name+":"+v.Mountpoint)
	}

	metadataClient, err := metadata.NewClientAndWait("http://rancher-metadata/latest/")
	if err != nil {
		log.Errorf("Error initiating metadata client: %v", err)
		err = fmt.Errorf("can't build client")
		return
	}
	managerCont, err := metadataClient.GetSelfContainer()
	if err != nil {
		log.Errorf("failed to get current container: %s", err)
		err = fmt.Errorf("can't inspect current container")
		return
	}
	containers, err := o.Client.Container.List(&client.ListOpts{
		Filters: map[string]interface{}{
			"limit": -2,
			"all":   true,
		},
	})
	if err != nil {
		log.Errorf("failed to get container list: %s", err)
		err = fmt.Errorf("can't get container list")
		return
	}
	var managerContainer *client.Container
	for _, container := range containers.Data {
		if container.Name == managerCont.Name {
			managerContainer = &container
			break
		}
	}
	if managerContainer == nil {
		log.Errorf("failed to get manager container: %v", err)
		return
	}
	for envKey, envVal := range managerContainer.Environment {
		environment[envKey] = envVal
	}

	container, err := o.Client.Container.Create(&client.Container{
		Name:            createWorkerName(),
		RequestedHostId: hostbind,
		ImageUuid:       "docker:" + image,
		Command:         cmd,
		Environment:     environment,
		RestartPolicy: &client.RestartPolicy{
			MaximumRetryCount: 1,
			Name:              "on-failure",
		},
		DataVolumes: cvs,
	})
	if err != nil {
		log.Errorf("failed to create worker container: %s", err)
		err = fmt.Errorf("can't create worker container")
		return
	}

	defer o.DeleteWorker(container)

	stopped := false
	terminated := false
	for !terminated {
		container, err := o.Client.Container.ById(container.Id)
		if err != nil {
			log.Errorf("failed to inspect worker: %s", err)
			err = fmt.Errorf("can't inspect worker")
			return 1, "", err
		}

		// This workaround is awful but it's the only way to know if the container failed.
		if container.State == "stopped" {
			if container.StartCount == 1 {
				if stopped == false {
					stopped = true
					time.Sleep(5 * time.Second)
				} else {
					terminated = true
					state = 0
				}
			} else {
				state = 1
				terminated = true
			}
		}
	}

	container, err = o.Client.Container.ById(container.Id)
	if err != nil {
		log.Errorf("failed to inspect worker before retrieving logs: %s", err)
		err = fmt.Errorf("can't inspect worker")
		return 1, "", err
	}

	var hostAccess *client.HostAccess
	err = o.rawAPICall("POST", container.Links["self"]+"/?action=logs", &hostAccess)
	if err != nil {
		log.Errorf("failed to read response from rancher: %s", err)
		err = fmt.Errorf("can't access worker logs")
		return
	}

	origin := o.Handler.Config.Cattle.URL

	u, err := url.Parse(hostAccess.Url)
	if err != nil {
		log.Errorf("failed to parse rancher server url: %s", err)
		err = fmt.Errorf("can't access worker logs")
	}
	q := u.Query()
	q.Set("token", hostAccess.Token)
	u.RawQuery = q.Encode()

	ws, err := websocket.Dial(u.String(), "", origin)
	if err != nil {
		log.Errorf("failed to open websocket with rancher server: %s", err)
		err = fmt.Errorf("can't access worker logs")
		return
	}

	defer ws.Close()

	var data bytes.Buffer
	io.Copy(&data, ws)

	re := regexp.MustCompile(`(?m)[0-9]{2,} [ZT\-\:\.0-9]+ (.*)`)
	for _, line := range re.FindAllStringSubmatch(data.String(), -1) {
		stdout = strings.Join([]string{stdout, line[1]}, "\n")
	}

	log.WithFields(log.Fields{
		"container": container.Id,
		"volumes":   strings.Join(cvs[:], ","),
		"cmd":       strings.Join(cmd[:], " "),
	}).Debug(stdout)
	return
}

// DeleteWorker deletes a worker
func (o *CattleOrchestrator) DeleteWorker(container *client.Container) {
	err := o.Client.Container.Delete(container)
	if err != nil {
		log.Errorf("failed to delete worker: %s", err)
	}
	removed := false
	for !removed {
		container, err := o.Client.Container.ById(container.Id)
		if err != nil {
			log.Errorf("failed to inspect worker: %s", err)
		}
		if container.Removed != "" {
			removed = true
		}
	}
	return
}

// GetContainersMountingVolume returns containers mounting a volume
func (o *CattleOrchestrator) GetContainersMountingVolume(v *volume.Volume) (containers []*volume.MountedVolume, err error) {
	vol, err := o.Client.Volume.ById(v.ID)

	if err != nil {
		log.Errorf("failed to get volume: %s", err)
	}

	for _, mount := range vol.Mounts {
		instance, err := o.Client.Container.ById(mount.InstanceId)
		if err != nil {
			log.Errorf("failed to inspect container %s", mount.InstanceId)
			continue
		}
		if instance.State != "running" {
			continue
		}

		mv := &volume.MountedVolume{
			ContainerID: mount.InstanceId,
			Volume:      v,
			Path:        mount.Path,
		}
		containers = append(containers, mv)
	}
	return
}

// ContainerExec executes a command in a container
func (o *CattleOrchestrator) ContainerExec(mountedVolumes *volume.MountedVolume, command []string) (stdout string, err error) {

	container, err := o.Client.Container.ById(mountedVolumes.ContainerID)
	if err != nil {
		log.Errorf("failed to retrieve container: %s", err)
		return
	}

	hostAccess, err := o.Client.Container.ActionExecute(container, &client.ContainerExec{
		AttachStdin:  false,
		AttachStdout: true,
		Command:      command,
		Tty:          false,
	})
	if err != nil {
		log.Errorf("failed to prepare command execution in container: %s", err)
		return
	}

	origin := o.Handler.Config.Cattle.URL

	u, err := url.Parse(hostAccess.Url)
	if err != nil {
		log.Errorf("failed to parse rancher server url: %s", err)
	}
	q := u.Query()
	q.Set("token", hostAccess.Token)
	u.RawQuery = q.Encode()

	ws, err := websocket.Dial(u.String(), "", origin)
	if err != nil {
		log.Errorf("failed to open websocket with rancher server: %s", err)
	}

	var data bytes.Buffer
	io.Copy(&data, ws)

	rawStdout, _ := base64.StdEncoding.DecodeString(data.String())
	stdout = string(rawStdout)

	log.WithFields(log.Fields{
		"container": mountedVolumes.ContainerID,
		"cmd":       strings.Join(command[:], " "),
	}).Debug(stdout)
	return
}

func (o *CattleOrchestrator) blacklistedVolume(vol *volume.Volume) (bool, string, string) {
	if utf8.RuneCountInString(vol.Name) == 64 || utf8.RuneCountInString(vol.Name) == 0 {
		return true, "unnamed", ""
	}

	if strings.Contains(vol.Name, "/") {
		return true, "blacklisted", "path"
	}

	// Use whitelist if defined
	if l := o.Handler.Config.VolumesWhitelist; len(l) > 0 && l[0] != "" {
		sort.Strings(l)
		i := sort.SearchStrings(l, vol.Name)
		if i < len(l) && l[i] == vol.Name {
			return false, "", ""
		}
		return true, "blacklisted", "whitelist config"
	}

	list := o.Handler.Config.VolumesBlacklist
	sort.Strings(list)
	i := sort.SearchStrings(list, vol.Name)
	if i < len(list) && list[i] == vol.Name {
		return true, "blacklisted", "blacklist config"
	}

	if vol.Config.Ignore {
		return true, "blacklisted", "volume config"
	}

	return false, "", ""
}

func (o *CattleOrchestrator) rawAPICall(method, endpoint string, object interface{}) (err error) {
	// TODO: Use go-rancher.
	// It was impossible to use it, maybe a problem in go-rancher or a lack of documentation.
	clientHTTP := &http.Client{}
	v := url.Values{}
	req, err := http.NewRequest(method, endpoint, strings.NewReader(v.Encode()))
	req.SetBasicAuth(o.Handler.Config.Cattle.AccessKey, o.Handler.Config.Cattle.SecretKey)
	resp, err := clientHTTP.Do(req)
	if err != nil {
		log.Errorf("failed to execute POST request: %s", err)
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Errorf("failed to read response from rancher: %s", err)
	}
	err = json.Unmarshal(body, object)
	if err != nil {
		log.Errorf("failed to unmarshal: %s", err)
	}
	return
}

func detectCattle() bool {
	_, err := net.LookupHost("rancher-metadata")
	if err != nil {
		return false
	}
	return true
}
<|endoftext|>