<commit_before>// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package rafthttp

import (
	"context"
	"sync"
	"time"

	"go.etcd.io/etcd/client/pkg/v3/types"
	"go.etcd.io/etcd/raft/v3"
	"go.etcd.io/etcd/raft/v3/raftpb"
	"go.etcd.io/etcd/server/v3/etcdserver/api/snap"
	stats "go.etcd.io/etcd/server/v3/etcdserver/api/v2stats"

	"go.uber.org/zap"
	"golang.org/x/time/rate"
)

const (
	// ConnReadTimeout and ConnWriteTimeout are the i/o timeout set on each connection rafthttp pkg creates.
	// A 5 seconds timeout is good enough for recycling bad connections. Or we have to wait for
	// tcp keepalive failing to detect a bad connection, which is at minutes level.
	// For long term streaming connections, rafthttp pkg sends application level linkHeartbeatMessage
	// to keep the connection alive.
	// For short term pipeline connections, the connection MUST be killed to avoid it being
	// put back to http pkg connection pool.
	DefaultConnReadTimeout  = 5 * time.Second
	DefaultConnWriteTimeout = 5 * time.Second

	recvBufSize = 4096
	// maxPendingProposals holds the proposals during one leader election process.
	// Generally one leader election takes at most 1 sec. It should have
	// 0-2 election conflicts, and each one takes 0.5 sec.
	// We assume the number of concurrent proposers is smaller than 4096.
	// One client blocks on its proposal for at least 1 sec, so 4096 is enough
	// to hold all proposals.
	maxPendingProposals = 4096

	streamAppV2 = "streamMsgAppV2"
	streamMsg   = "streamMsg"
	pipelineMsg = "pipeline"
	sendSnap    = "sendMsgSnap"
)

var (
	ConnReadTimeout  = DefaultConnReadTimeout
	ConnWriteTimeout = DefaultConnWriteTimeout
)

type Peer interface {
	// send sends the message to the remote peer. The function is non-blocking
	// and has no promise that the message will be received by the remote.
	// When it fails to send message out, it will report the status to underlying
	// raft.
	send(m raftpb.Message)

	// sendSnap sends the merged snapshot message to the remote peer. Its behavior
	// is similar to send.
	sendSnap(m snap.Message)

	// update updates the urls of remote peer.
	update(urls types.URLs)

	// attachOutgoingConn attaches the outgoing connection to the peer for
	// stream usage. After the call, the ownership of the outgoing
	// connection hands over to the peer. The peer will close the connection
	// when it is no longer used.
	attachOutgoingConn(conn *outgoingConn)
	// activeSince returns the time that the connection with the
	// peer becomes active.
	activeSince() time.Time
	// stop performs any necessary finalization and terminates the peer
	// elegantly.
	stop()
}

// peer is the representative of a remote raft node. Local raft node sends
// messages to the remote through peer.
// Each peer has two underlying mechanisms to send out a message: stream and
// pipeline.
// A stream is a receiver initialized long-polling connection, which
// is always open to transfer messages. Besides general stream, peer also has
// a optimized stream for sending msgApp since msgApp accounts for large part
// of all messages. Only raft leader uses the optimized stream to send msgApp
// to the remote follower node.
// A pipeline is a series of http clients that send http requests to the remote.
// It is only used when the stream has not been established.
type peer struct {
	lg *zap.Logger

	localID types.ID
	// id of the remote raft peer node
	id types.ID

	r Raft

	status *peerStatus

	picker *urlPicker

	msgAppV2Writer *streamWriter
	writer         *streamWriter
	pipeline       *pipeline
	snapSender     *snapshotSender // snapshot sender to send v3 snapshot messages
	msgAppV2Reader *streamReader
	msgAppReader   *streamReader

	recvc chan raftpb.Message
	propc chan raftpb.Message

	mu     sync.Mutex
	paused bool

	cancel context.CancelFunc // cancel pending works in go routine created by peer.
	stopc  chan struct{}
}

func startPeer(t *Transport, urls types.URLs, peerID types.ID, fs *stats.FollowerStats) *peer {
	if t.Logger != nil {
		t.Logger.Info("starting remote peer", zap.String("remote-peer-id", peerID.String()))
	}
	defer func() {
		if t.Logger != nil {
			t.Logger.Info("started remote peer", zap.String("remote-peer-id", peerID.String()))
		}
	}()

	status := newPeerStatus(t.Logger, t.ID, peerID)
	picker := newURLPicker(urls)
	errorc := t.ErrorC
	r := t.Raft
	pipeline := &pipeline{
		peerID:        peerID,
		tr:            t,
		picker:        picker,
		status:        status,
		followerStats: fs,
		raft:          r,
		errorc:        errorc,
	}
	pipeline.start()

	p := &peer{
		lg:             t.Logger,
		localID:        t.ID,
		id:             peerID,
		r:              r,
		status:         status,
		picker:         picker,
		msgAppV2Writer: startStreamWriter(t.Logger, t.ID, peerID, status, fs, r),
		writer:         startStreamWriter(t.Logger, t.ID, peerID, status, fs, r),
		pipeline:       pipeline,
		snapSender:     newSnapshotSender(t, picker, peerID, status),
		recvc:          make(chan raftpb.Message, recvBufSize),
		propc:          make(chan raftpb.Message, maxPendingProposals),
		stopc:          make(chan struct{}),
	}

	ctx, cancel := context.WithCancel(context.Background())
	p.cancel = cancel
	go func() {
		for {
			select {
			case mm := <-p.recvc:
				if err := r.Process(ctx, mm); err != nil {
					if t.Logger != nil {
						t.Logger.Warn("failed to process Raft message", zap.Error(err))
					}
				}
			case <-p.stopc:
				return
			}
		}
	}()

	// r.Process might block for processing proposal when there is no leader.
	// Thus propc must be put into a separate routine with recvc to avoid blocking
	// processing other raft messages.
	go func() {
		for {
			select {
			case mm := <-p.propc:
				if err := r.Process(ctx, mm); err != nil {
					if t.Logger != nil {
						t.Logger.Warn("failed to process Raft message", zap.Error(err))
					}
				}
			case <-p.stopc:
				return
			}
		}
	}()

	p.msgAppV2Reader = &streamReader{
		lg:     t.Logger,
		peerID: peerID,
		typ:    streamTypeMsgAppV2,
		tr:     t,
		picker: picker,
		status: status,
		recvc:  p.recvc,
		propc:  p.propc,
		rl:     rate.NewLimiter(t.DialRetryFrequency, 1),
	}
	p.msgAppReader = &streamReader{
		lg:     t.Logger,
		peerID: peerID,
		typ:    streamTypeMessage,
		tr:     t,
		picker: picker,
		status: status,
		recvc:  p.recvc,
		propc:  p.propc,
		rl:     rate.NewLimiter(t.DialRetryFrequency, 1),
	}

	p.msgAppV2Reader.start()
	p.msgAppReader.start()

	return p
}

func (p *peer) send(m raftpb.Message) {
	p.mu.Lock()
	paused := p.paused
	p.mu.Unlock()

	if paused {
		return
	}

	writec, name := p.pick(m)
	select {
	case writec <- m:
	default:
		p.r.ReportUnreachable(m.To)
		if isMsgSnap(m) {
			p.r.ReportSnapshot(m.To, raft.SnapshotFailure)
		}
		if p.status.isActive() {
			if p.lg != nil {
				p.lg.Warn(
					"dropped internal Raft message since sending buffer is full (overloaded network)",
					zap.String("message-type", m.Type.String()),
					zap.String("local-member-id", p.localID.String()),
					zap.String("from", types.ID(m.From).String()),
					zap.String("remote-peer-id", p.id.String()),
					zap.String("remote-peer-name", name),
					zap.Bool("remote-peer-active", p.status.isActive()),
				)
			}
		} else {
			if p.lg != nil {
				p.lg.Warn(
					"dropped internal Raft message since sending buffer is full (overloaded network)",
					zap.String("message-type", m.Type.String()),
					zap.String("local-member-id", p.localID.String()),
					zap.String("from", types.ID(m.From).String()),
					zap.String("remote-peer-id", p.id.String()),
					zap.String("remote-peer-name", name),
					zap.Bool("remote-peer-active", p.status.isActive()),
				)
			}
		}
		sentFailures.WithLabelValues(types.ID(m.To).String()).Inc()
	}
}

func (p *peer) sendSnap(m snap.Message) {
	go p.snapSender.send(m)
}

func (p *peer) update(urls types.URLs) {
	p.picker.update(urls)
}

func (p *peer) attachOutgoingConn(conn *outgoingConn) {
	var ok bool
	switch conn.t {
	case streamTypeMsgAppV2:
		ok = p.msgAppV2Writer.attach(conn)
	case streamTypeMessage:
		ok = p.writer.attach(conn)
	default:
		if p.lg != nil {
			p.lg.Panic("unknown stream type", zap.String("type", conn.t.String()))
		}
	}
	if !ok {
		conn.Close()
	}
}

func (p *peer) activeSince() time.Time { return p.status.activeSince() }

// Pause pauses the peer. The peer will simply drops all incoming
// messages without returning an error.
func (p *peer) Pause() {
	p.mu.Lock()
	defer p.mu.Unlock()
	p.paused = true
	p.msgAppReader.pause()
	p.msgAppV2Reader.pause()
}

// Resume resumes a paused peer.
func (p *peer) Resume() {
	p.mu.Lock()
	defer p.mu.Unlock()
	p.paused = false
	p.msgAppReader.resume()
	p.msgAppV2Reader.resume()
}

func (p *peer) stop() {
	if p.lg != nil {
		p.lg.Info("stopping remote peer", zap.String("remote-peer-id", p.id.String()))
	}

	defer func() {
		if p.lg != nil {
			p.lg.Info("stopped remote peer", zap.String("remote-peer-id", p.id.String()))
		}
	}()

	close(p.stopc)
	p.cancel()
	p.msgAppV2Writer.stop()
	p.writer.stop()
	p.pipeline.stop()
	p.snapSender.stop()
	p.msgAppV2Reader.stop()
	p.msgAppReader.stop()
}

// pick picks a chan for sending the given message. The picked chan and the picked chan
// string name are returned.
func (p *peer) pick(m raftpb.Message) (writec chan<- raftpb.Message, picked string) {
	var ok bool
	// Considering MsgSnap may have a big size, e.g., 1G, and will block
	// stream for a long time, only use one of the N pipelines to send MsgSnap.
	if isMsgSnap(m) {
		return p.pipeline.msgc, pipelineMsg
	} else if writec, ok = p.msgAppV2Writer.writec(); ok && isMsgApp(m) {
		return writec, streamAppV2
	} else if writec, ok = p.writer.writec(); ok {
		return writec, streamMsg
	}
	return p.pipeline.msgc, pipelineMsg
}

func isMsgApp(m raftpb.Message) bool { return m.Type == raftpb.MsgApp }

func isMsgSnap(m raftpb.Message) bool { return m.Type == raftpb.MsgSnap }
<commit_msg>etcdserver: remove code duplication from the peer.send func<commit_after>// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package rafthttp

import (
	"context"
	"sync"
	"time"

	"go.etcd.io/etcd/client/pkg/v3/types"
	"go.etcd.io/etcd/raft/v3"
	"go.etcd.io/etcd/raft/v3/raftpb"
	"go.etcd.io/etcd/server/v3/etcdserver/api/snap"
	stats "go.etcd.io/etcd/server/v3/etcdserver/api/v2stats"

	"go.uber.org/zap"
	"golang.org/x/time/rate"
)

const (
	// ConnReadTimeout and ConnWriteTimeout are the i/o timeout set on each connection rafthttp pkg creates.
	// A 5 seconds timeout is good enough for recycling bad connections. Or we have to wait for
	// tcp keepalive failing to detect a bad connection, which is at minutes level.
	// For long term streaming connections, rafthttp pkg sends application level linkHeartbeatMessage
	// to keep the connection alive.
	// For short term pipeline connections, the connection MUST be killed to avoid it being
	// put back to http pkg connection pool.
	DefaultConnReadTimeout  = 5 * time.Second
	DefaultConnWriteTimeout = 5 * time.Second

	recvBufSize = 4096
	// maxPendingProposals holds the proposals during one leader election process.
	// Generally one leader election takes at most 1 sec. It should have
	// 0-2 election conflicts, and each one takes 0.5 sec.
	// We assume the number of concurrent proposers is smaller than 4096.
	// One client blocks on its proposal for at least 1 sec, so 4096 is enough
	// to hold all proposals.
	maxPendingProposals = 4096

	streamAppV2 = "streamMsgAppV2"
	streamMsg   = "streamMsg"
	pipelineMsg = "pipeline"
	sendSnap    = "sendMsgSnap"
)

var (
	ConnReadTimeout  = DefaultConnReadTimeout
	ConnWriteTimeout = DefaultConnWriteTimeout
)

type Peer interface {
	// send sends the message to the remote peer. The function is non-blocking
	// and has no promise that the message will be received by the remote.
	// When it fails to send message out, it will report the status to underlying
	// raft.
	send(m raftpb.Message)

	// sendSnap sends the merged snapshot message to the remote peer. Its behavior
	// is similar to send.
	sendSnap(m snap.Message)

	// update updates the urls of remote peer.
	update(urls types.URLs)

	// attachOutgoingConn attaches the outgoing connection to the peer for
	// stream usage. After the call, the ownership of the outgoing
	// connection hands over to the peer. The peer will close the connection
	// when it is no longer used.
	attachOutgoingConn(conn *outgoingConn)
	// activeSince returns the time that the connection with the
	// peer becomes active.
	activeSince() time.Time
	// stop performs any necessary finalization and terminates the peer
	// elegantly.
	stop()
}

// peer is the representative of a remote raft node. Local raft node sends
// messages to the remote through peer.
// Each peer has two underlying mechanisms to send out a message: stream and
// pipeline.
// A stream is a receiver initialized long-polling connection, which
// is always open to transfer messages. Besides general stream, peer also has
// a optimized stream for sending msgApp since msgApp accounts for large part
// of all messages. Only raft leader uses the optimized stream to send msgApp
// to the remote follower node.
// A pipeline is a series of http clients that send http requests to the remote.
// It is only used when the stream has not been established.
type peer struct {
	lg *zap.Logger

	localID types.ID
	// id of the remote raft peer node
	id types.ID

	r Raft

	status *peerStatus

	picker *urlPicker

	msgAppV2Writer *streamWriter
	writer         *streamWriter
	pipeline       *pipeline
	snapSender     *snapshotSender // snapshot sender to send v3 snapshot messages
	msgAppV2Reader *streamReader
	msgAppReader   *streamReader

	recvc chan raftpb.Message
	propc chan raftpb.Message

	mu     sync.Mutex
	paused bool

	cancel context.CancelFunc // cancel pending works in go routine created by peer.
	stopc  chan struct{}
}

func startPeer(t *Transport, urls types.URLs, peerID types.ID, fs *stats.FollowerStats) *peer {
	if t.Logger != nil {
		t.Logger.Info("starting remote peer", zap.String("remote-peer-id", peerID.String()))
	}
	defer func() {
		if t.Logger != nil {
			t.Logger.Info("started remote peer", zap.String("remote-peer-id", peerID.String()))
		}
	}()

	status := newPeerStatus(t.Logger, t.ID, peerID)
	picker := newURLPicker(urls)
	errorc := t.ErrorC
	r := t.Raft
	pipeline := &pipeline{
		peerID:        peerID,
		tr:            t,
		picker:        picker,
		status:        status,
		followerStats: fs,
		raft:          r,
		errorc:        errorc,
	}
	pipeline.start()

	p := &peer{
		lg:             t.Logger,
		localID:        t.ID,
		id:             peerID,
		r:              r,
		status:         status,
		picker:         picker,
		msgAppV2Writer: startStreamWriter(t.Logger, t.ID, peerID, status, fs, r),
		writer:         startStreamWriter(t.Logger, t.ID, peerID, status, fs, r),
		pipeline:       pipeline,
		snapSender:     newSnapshotSender(t, picker, peerID, status),
		recvc:          make(chan raftpb.Message, recvBufSize),
		propc:          make(chan raftpb.Message, maxPendingProposals),
		stopc:          make(chan struct{}),
	}

	ctx, cancel := context.WithCancel(context.Background())
	p.cancel = cancel
	go func() {
		for {
			select {
			case mm := <-p.recvc:
				if err := r.Process(ctx, mm); err != nil {
					if t.Logger != nil {
						t.Logger.Warn("failed to process Raft message", zap.Error(err))
					}
				}
			case <-p.stopc:
				return
			}
		}
	}()

	// r.Process might block for processing proposal when there is no leader.
	// Thus propc must be put into a separate routine with recvc to avoid blocking
	// processing other raft messages.
	go func() {
		for {
			select {
			case mm := <-p.propc:
				if err := r.Process(ctx, mm); err != nil {
					if t.Logger != nil {
						t.Logger.Warn("failed to process Raft message", zap.Error(err))
					}
				}
			case <-p.stopc:
				return
			}
		}
	}()

	p.msgAppV2Reader = &streamReader{
		lg:     t.Logger,
		peerID: peerID,
		typ:    streamTypeMsgAppV2,
		tr:     t,
		picker: picker,
		status: status,
		recvc:  p.recvc,
		propc:  p.propc,
		rl:     rate.NewLimiter(t.DialRetryFrequency, 1),
	}
	p.msgAppReader = &streamReader{
		lg:     t.Logger,
		peerID: peerID,
		typ:    streamTypeMessage,
		tr:     t,
		picker: picker,
		status: status,
		recvc:  p.recvc,
		propc:  p.propc,
		rl:     rate.NewLimiter(t.DialRetryFrequency, 1),
	}

	p.msgAppV2Reader.start()
	p.msgAppReader.start()

	return p
}

func (p *peer) send(m raftpb.Message) {
	p.mu.Lock()
	paused := p.paused
	p.mu.Unlock()

	if paused {
		return
	}

	writec, name := p.pick(m)
	select {
	case writec <- m:
	default:
		p.r.ReportUnreachable(m.To)
		if isMsgSnap(m) {
			p.r.ReportSnapshot(m.To, raft.SnapshotFailure)
		}
		if p.lg != nil {
			p.lg.Warn(
				"dropped internal Raft message since sending buffer is full",
				zap.String("message-type", m.Type.String()),
				zap.String("local-member-id", p.localID.String()),
				zap.String("from", types.ID(m.From).String()),
				zap.String("remote-peer-id", p.id.String()),
				zap.String("remote-peer-name", name),
				zap.Bool("remote-peer-active", p.status.isActive()),
			)
		}
		sentFailures.WithLabelValues(types.ID(m.To).String()).Inc()
	}
}

func (p *peer) sendSnap(m snap.Message) {
	go p.snapSender.send(m)
}

func (p *peer) update(urls types.URLs) {
	p.picker.update(urls)
}

func (p *peer) attachOutgoingConn(conn *outgoingConn) {
	var ok bool
	switch conn.t {
	case streamTypeMsgAppV2:
		ok = p.msgAppV2Writer.attach(conn)
	case streamTypeMessage:
		ok = p.writer.attach(conn)
	default:
		if p.lg != nil {
			p.lg.Panic("unknown stream type", zap.String("type", conn.t.String()))
		}
	}
	if !ok {
		conn.Close()
	}
}

func (p *peer) activeSince() time.Time { return p.status.activeSince() }

// Pause pauses the peer. The peer will simply drops all incoming
// messages without returning an error.
func (p *peer) Pause() {
	p.mu.Lock()
	defer p.mu.Unlock()
	p.paused = true
	p.msgAppReader.pause()
	p.msgAppV2Reader.pause()
}

// Resume resumes a paused peer.
func (p *peer) Resume() {
	p.mu.Lock()
	defer p.mu.Unlock()
	p.paused = false
	p.msgAppReader.resume()
	p.msgAppV2Reader.resume()
}

func (p *peer) stop() {
	if p.lg != nil {
		p.lg.Info("stopping remote peer", zap.String("remote-peer-id", p.id.String()))
	}

	defer func() {
		if p.lg != nil {
			p.lg.Info("stopped remote peer", zap.String("remote-peer-id", p.id.String()))
		}
	}()

	close(p.stopc)
	p.cancel()
	p.msgAppV2Writer.stop()
	p.writer.stop()
	p.pipeline.stop()
	p.snapSender.stop()
	p.msgAppV2Reader.stop()
	p.msgAppReader.stop()
}

// pick picks a chan for sending the given message. The picked chan and the picked chan
// string name are returned.
func (p *peer) pick(m raftpb.Message) (writec chan<- raftpb.Message, picked string) {
	var ok bool
	// Considering MsgSnap may have a big size, e.g., 1G, and will block
	// stream for a long time, only use one of the N pipelines to send MsgSnap.
	if isMsgSnap(m) {
		return p.pipeline.msgc, pipelineMsg
	} else if writec, ok = p.msgAppV2Writer.writec(); ok && isMsgApp(m) {
		return writec, streamAppV2
	} else if writec, ok = p.writer.writec(); ok {
		return writec, streamMsg
	}
	return p.pipeline.msgc, pipelineMsg
}

func isMsgApp(m raftpb.Message) bool { return m.Type == raftpb.MsgApp }

func isMsgSnap(m raftpb.Message) bool { return m.Type == raftpb.MsgSnap }
<|endoftext|>


<commit_before>package webservice

import (
	ae "appengine"
	"data"
	"encoding/json"
	"errors"
	"googlebooks"
	"isbn13"
	"net/http"
	"persistence"
)

type Call struct {
	Context    ae.Context
	Request    *http.Request
	Response   http.ResponseWriter
	StatusCode int
}

func (call *Call) writeError(err error, strings []string) []string {
	if me, ok := err.(ae.MultiError); ok {
		for _, next := range me {
			strings = call.writeError(next, strings)
		}
	} else {
		str := err.Error()
		call.Context.Errorf("Error reported: %s", str)
		strings = append(strings, str)
	}

	return strings
}

func (call *Call) DetermineCountry() string {
	header := call.Request.Header["X-Appengine-Country"]
	if len(header) > 0 {
		country := header[0]

		if country != "ZZ" {
			return country
		}
		call.Context.Debugf("Falling back to user-supplied country in dev server")
	} else {
		call.Context.Warningf("Could not auto-detect country (available headers: %v), falling back to query parameter", call.Request.Header)
	}

	query := call.Request.URL.Query()["country"]

	if len(query) > 0 {
		return query[0]
	}

	return "unknown"
}

type CallHandler func(*Call) (interface{}, error)

func (function CallHandler) ServeHTTP(w http.ResponseWriter, rq *http.Request) {
	call := Call{
		Context:  ae.NewContext(rq),
		Request:  rq,
		Response: w,
	}

	call.Response.Header().Add("Content-Type", "application/json;charset=UTF-8")
	result, err := function(&call)

	if err != nil {
		if call.StatusCode == 0 {
			call.StatusCode = http.StatusInternalServerError
		}
		errors := call.writeError(err, nil)
		result = map[string][]string{"errors": errors}
	} else {
		if call.StatusCode == 0 {
			call.StatusCode = http.StatusOK
		}
	}

	encoder := json.NewEncoder(w)
	call.Response.WriteHeader(call.StatusCode)
	call.Context.Debugf("Returning %v to client", result)
	encoder.Encode(result)
}

func init() {
	http.Handle("/volumes/", CallHandler(serveVolumeSingle))
	http.Handle("/volumes", CallHandler(serveVolumeBulk))
}

func serveVolumeBulk(call *Call) (interface{}, error) {
	var shelf *data.LookupReply
	var err error

	switch call.Request.Method {
	case "GET":
		shelf, err = getVolumeBulk(call)
	case "PUT":
		shelf, err = putVolumeBulk(call)
	default:
		call.StatusCode = http.StatusMethodNotAllowed

		call.Response.Header().Add("Allow", "GET, PUT")
		err = errors.New("Unsupported operation. Only GET and PUT methods are allowed")
	}

	return shelf, err
}

func getVolumeBulk(call *Call) (reply *data.LookupReply, err error) {
	var shelf *data.Bookshelf

	if shelf, err = persistence.LookupBookshelf(call.Context); err == nil {
		call.Context.Debugf("Bookshelf contains %d volumes", len(shelf.Books))

		for _, str := range call.Request.URL.Query()["search"] {
			matches := shelf.Search(str)

			shelf = &data.Bookshelf{make([]data.BookMetaData, len(matches))}
			for i, ptr := range matches {
				shelf.Books[i] = *ptr
			}

			call.Context.Debugf("Filtered by \"%s\", down to %d entries: %v", str, len(shelf.Books), shelf.Books)
		}

		reply = &data.LookupReply{
			Count:     len(shelf.Books),
			BookInfos: shelf.Books,
		}
	}

	return
}

func putVolumeBulk(call *Call) (reply *data.LookupReply, err error) {
	decode := json.NewDecoder(call.Request.Body)
	shelf := new(data.Bookshelf)
	if err = decode.Decode(shelf); err == nil {
		for i := range shelf.Books {
			book := &shelf.Books[i]
			normalize(call, book)
		}

		if err = persistence.StoreBookshelf(call.Context, shelf); err == nil {
			reply = &data.LookupReply{
				Count:     len(shelf.Books),
				BookInfos: shelf.Books,
			}
		}
	}

	return
}

func serveVolumeSingle(call *Call) (interface{}, error) {
	isbn, err := isbn13.New(call.Request.URL.Path[9:])
	var book *data.BookMetaData

	if err == nil {
		switch call.Request.Method {
		case "GET":
			book, err = compositeISBNLookup(call.Context, call.DetermineCountry(), isbn)
		case "PUT":
			book, err = putVolumeSingle(call, isbn)
		case "DELETE":
			book, err = deleteVolumeSingle(call, isbn)
		default:
			call.StatusCode = http.StatusMethodNotAllowed
			call.Response.Header().Add("Allow", "GET, PUT, DELETE")
			err = errors.New("Unsupported operation. Only GET, PUT and DELETE methods are allowed")
		}
	}

	return book, err
}

func deleteVolumeSingle(call *Call, isbn isbn13.ISBN13) (info *data.BookMetaData, err error) {
	call.Context.Infof("Deleting ISBN %d", isbn)
	err = persistence.UpdateBookshelf(call.Context, func(tx *persistence.Transaction, shelf *data.Bookshelf) error {
		for i := range shelf.Books {
			if ptr := &shelf.Books[i]; ptr.ISBN == isbn.String() {
				tx.Delete = []data.KeyStringer{ptr}
				return nil
			}

		}
		return errors.New("ISBN not found")
	})

	return
}

func putVolumeSingle(call *Call, isbn isbn13.ISBN13) (info *data.BookMetaData, err error) {
	call.Context.Debugf("Updating %d", isbn)
	decode := json.NewDecoder(call.Request.Body)
	info = new(data.BookMetaData)
	if err = decode.Decode(info); err == nil {
		call.Context.Debugf("Raw decoded entity: %v", info)
		info.ISBN = isbn.String()
		normalize(call, info)
		call.Context.Debugf("Normalized decoded entity: %v", info)

		persistence.UpdateBookshelf(call.Context, func(tx *persistence.Transaction, shelf *data.Bookshelf) error {

			if ptr := shelf.LookupInfo(isbn); ptr != nil {
				*ptr = *info
				tx.Put = []data.KeyStringer{ptr}
			} else {
				shelf.Books = append(shelf.Books, *info)
				tx.Put = []data.KeyStringer{info}
			}

			return nil
		})
	}
	return info, err
}

func normalize(call *Call, info *data.BookMetaData) {
	var isbn isbn13.ISBN13
	var err error
	if isbn, err = isbn13.New(info.ISBN); err == nil {
		var base *data.BookMetaData
		if base, err = persistence.LookupISBN(call.Context, call.DetermineCountry(), isbn); err == nil {
			info.Volume.Images = base.Volume.Images
			return
		} else {
			info.Volume.Images = data.ImageLinks{}
		}
	} else {
		call.Context.Errorf("Could not normalize data for %s: %s", info.ISBN, err.Error())
	}
}

func compositeISBNLookup(ctx ae.Context, country string, isbn isbn13.ISBN13) (resp *data.BookMetaData, err error) {
	// look in my own library first - if this fails badly, abort anything further
	if shelf, err := persistence.LookupBookshelf(ctx); err == nil {
		if book := shelf.LookupInfo(isbn); book != nil {
			return book, nil
		}
	} else {
		return nil, err
	}

	var errors ae.MultiError
	if book, err := persistence.LookupISBN(ctx, country, isbn); err == nil {
		return book, nil
	} else {
		errors = append(errors, err)
	}

	if r, err := googlebooks.LookupISBN(ctx, country, isbn); err == nil {
		persistence.StoreISBNResult(ctx, country, isbn, r)
		return r, nil
	} else {
		errors = append(errors, err)
	}

	return nil, errors
}
<commit_msg>FIX: Delete from memcache too<commit_after>package webservice

import (
	ae "appengine"
	"data"
	"encoding/json"
	"errors"
	"googlebooks"
	"isbn13"
	"net/http"
	"persistence"
)

type Call struct {
	Context    ae.Context
	Request    *http.Request
	Response   http.ResponseWriter
	StatusCode int
}

func (call *Call) writeError(err error, strings []string) []string {
	if me, ok := err.(ae.MultiError); ok {
		for _, next := range me {
			strings = call.writeError(next, strings)
		}
	} else {
		str := err.Error()
		call.Context.Errorf("Error reported: %s", str)
		strings = append(strings, str)
	}

	return strings
}

func (call *Call) DetermineCountry() string {
	header := call.Request.Header["X-Appengine-Country"]
	if len(header) > 0 {
		country := header[0]

		if country != "ZZ" {
			return country
		}
		call.Context.Debugf("Falling back to user-supplied country in dev server")
	} else {
		call.Context.Warningf("Could not auto-detect country (available headers: %v), falling back to query parameter", call.Request.Header)
	}

	query := call.Request.URL.Query()["country"]

	if len(query) > 0 {
		return query[0]
	}

	return "unknown"
}

type CallHandler func(*Call) (interface{}, error)

func (function CallHandler) ServeHTTP(w http.ResponseWriter, rq *http.Request) {
	call := Call{
		Context:  ae.NewContext(rq),
		Request:  rq,
		Response: w,
	}

	call.Response.Header().Add("Content-Type", "application/json;charset=UTF-8")
	result, err := function(&call)

	if err != nil {
		if call.StatusCode == 0 {
			call.StatusCode = http.StatusInternalServerError
		}
		errors := call.writeError(err, nil)
		result = map[string][]string{"errors": errors}
	} else {
		if call.StatusCode == 0 {
			call.StatusCode = http.StatusOK
		}
	}

	encoder := json.NewEncoder(w)
	call.Response.WriteHeader(call.StatusCode)
	call.Context.Debugf("Returning %v to client", result)
	encoder.Encode(result)
}

func init() {
	http.Handle("/volumes/", CallHandler(serveVolumeSingle))
	http.Handle("/volumes", CallHandler(serveVolumeBulk))
}

func serveVolumeBulk(call *Call) (interface{}, error) {
	var shelf *data.LookupReply
	var err error

	switch call.Request.Method {
	case "GET":
		shelf, err = getVolumeBulk(call)
	case "PUT":
		shelf, err = putVolumeBulk(call)
	default:
		call.StatusCode = http.StatusMethodNotAllowed

		call.Response.Header().Add("Allow", "GET, PUT")
		err = errors.New("Unsupported operation. Only GET and PUT methods are allowed")
	}

	return shelf, err
}

func getVolumeBulk(call *Call) (reply *data.LookupReply, err error) {
	var shelf *data.Bookshelf

	if shelf, err = persistence.LookupBookshelf(call.Context); err == nil {
		call.Context.Debugf("Bookshelf contains %d volumes", len(shelf.Books))

		for _, str := range call.Request.URL.Query()["search"] {
			matches := shelf.Search(str)

			shelf = &data.Bookshelf{make([]data.BookMetaData, len(matches))}
			for i, ptr := range matches {
				shelf.Books[i] = *ptr
			}

			call.Context.Debugf("Filtered by \"%s\", down to %d entries: %v", str, len(shelf.Books), shelf.Books)
		}

		reply = &data.LookupReply{
			Count:     len(shelf.Books),
			BookInfos: shelf.Books,
		}
	}

	return
}

func putVolumeBulk(call *Call) (reply *data.LookupReply, err error) {
	decode := json.NewDecoder(call.Request.Body)
	shelf := new(data.Bookshelf)
	if err = decode.Decode(shelf); err == nil {
		for i := range shelf.Books {
			book := &shelf.Books[i]
			normalize(call, book)
		}

		if err = persistence.StoreBookshelf(call.Context, shelf); err == nil {
			reply = &data.LookupReply{
				Count:     len(shelf.Books),
				BookInfos: shelf.Books,
			}
		}
	}

	return
}

func serveVolumeSingle(call *Call) (interface{}, error) {
	isbn, err := isbn13.New(call.Request.URL.Path[9:])
	var book *data.BookMetaData

	if err == nil {
		switch call.Request.Method {
		case "GET":
			book, err = compositeISBNLookup(call.Context, call.DetermineCountry(), isbn)
		case "PUT":
			book, err = putVolumeSingle(call, isbn)
		case "DELETE":
			book, err = deleteVolumeSingle(call, isbn)
		default:
			call.StatusCode = http.StatusMethodNotAllowed
			call.Response.Header().Add("Allow", "GET, PUT, DELETE")
			err = errors.New("Unsupported operation. Only GET, PUT and DELETE methods are allowed")
		}
	}

	return book, err
}

func deleteVolumeSingle(call *Call, isbn isbn13.ISBN13) (info *data.BookMetaData, err error) {
	call.Context.Infof("Deleting ISBN %d", isbn)
	err = persistence.UpdateBookshelf(call.Context, func(tx *persistence.Transaction, shelf *data.Bookshelf) error {
		for i := range shelf.Books {
			if ptr := &shelf.Books[i]; ptr.ISBN == isbn.String() {
				shelf.Books = append(shelf.Books[:i], shelf.Books[i+1:]...)
				tx.Delete = []data.KeyStringer{ptr}
				return nil
			}

		}
		return errors.New("ISBN not found")
	})

	return
}

func putVolumeSingle(call *Call, isbn isbn13.ISBN13) (info *data.BookMetaData, err error) {
	call.Context.Debugf("Updating %d", isbn)
	decode := json.NewDecoder(call.Request.Body)
	info = new(data.BookMetaData)
	if err = decode.Decode(info); err == nil {
		call.Context.Debugf("Raw decoded entity: %v", info)
		info.ISBN = isbn.String()
		normalize(call, info)
		call.Context.Debugf("Normalized decoded entity: %v", info)

		persistence.UpdateBookshelf(call.Context, func(tx *persistence.Transaction, shelf *data.Bookshelf) error {

			if ptr := shelf.LookupInfo(isbn); ptr != nil {
				*ptr = *info
				tx.Put = []data.KeyStringer{ptr}
			} else {
				shelf.Books = append(shelf.Books, *info)
				tx.Put = []data.KeyStringer{info}
			}

			return nil
		})
	}
	return info, err
}

func normalize(call *Call, info *data.BookMetaData) {
	var isbn isbn13.ISBN13
	var err error
	if isbn, err = isbn13.New(info.ISBN); err == nil {
		var base *data.BookMetaData
		if base, err = persistence.LookupISBN(call.Context, call.DetermineCountry(), isbn); err == nil {
			info.Volume.Images = base.Volume.Images
			return
		} else {
			info.Volume.Images = data.ImageLinks{}
		}
	} else {
		call.Context.Errorf("Could not normalize data for %s: %s", info.ISBN, err.Error())
	}
}

func compositeISBNLookup(ctx ae.Context, country string, isbn isbn13.ISBN13) (resp *data.BookMetaData, err error) {
	// look in my own library first - if this fails badly, abort anything further
	if shelf, err := persistence.LookupBookshelf(ctx); err == nil {
		if book := shelf.LookupInfo(isbn); book != nil {
			return book, nil
		}
	} else {
		return nil, err
	}

	var errors ae.MultiError
	if book, err := persistence.LookupISBN(ctx, country, isbn); err == nil {
		return book, nil
	} else {
		errors = append(errors, err)
	}

	if r, err := googlebooks.LookupISBN(ctx, country, isbn); err == nil {
		persistence.StoreISBNResult(ctx, country, isbn, r)
		return r, nil
	} else {
		errors = append(errors, err)
	}

	return nil, errors
}
<|endoftext|>


<commit_before>// Package wikimg can pull the latest image URLs from Wikimedia Commons
// https://commons.wikimedia.org and map image colors to an xterm256 color
// palette.
package wikimg

import (
	"encoding/json"
	"errors"
	"fmt"
	"image"
	"image/color"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"

	// We define which image formats we support by importing decoder packages
	_ "image/gif"
	_ "image/jpeg"
	_ "image/png"
)

var (
	// EndOfResults is returned by Next() when no more results are available
	EndOfResults = errors.New("end of results")

	// Canceled may be returned by Next() and FirstColor() when the client
	// closes the Cancel channel on a Puller
	Canceled = errors.New("wikimg: canceled image processing")
)

const (
	// queryURL is the API we are querying
	queryURL = "https://commons.wikimedia.org/w/api.php"

	// apiMax is the max results we can request from the API at one time
	apiMax = 500

	// cancelCheckpoint is the number of pixels between checking whether the
	// request was canceled when running FirstColor()
	cancelCheckpoint = 10000
)

// queryResp mirrors the JSON structure returned by queryURL, specifying only
// the info we're interested in.
type queryResp struct {

	// Continue contains strings we need to pass back into the API to
	// continue where we left off
	Continue struct {
		Continue   string
		AIContinue string
	}

	// Query contains the actual results
	Query struct {
		AllImages []struct {
			URL string
		}
	}
}

// Puller is an image puller that retrieves the most recent image URLs that
// have been uploaded to Wikimedia Commons https://commons.wikimedia.org
type Puller struct {
	// qr is the most recent response from the API
	qr *queryResp

	// i is the current index into qr.Query.AllImages
	i int

	// count is the total number of images we've collected
	count int

	// max is the maximum number of images we want to collect
	max int

	// Cancel is an optional channel. Setting this value on Puller
	// and closing the channel signals to the Puller that any
	// in process operations (i.e, retrieving an image or computing
	// its first color) should be canceled. Any future
	// calls to Next() or FirstColor() will return a Canceled
	// error.
	Cancel <-chan struct{}
}

// NewPuller creates a puller that can return at most max images when calls to
// Next() are made
func NewPuller(max int) *Puller {
	return &Puller{
		max: max,
	}
}

// Next returns the next most recent image URL. If no more results are
// available EndOfResults is returned as an error.
func (p *Puller) Next() (string, error) {
	// If we've exceeded that max we want to get, then stop
	if p.count >= p.max {
		return "", EndOfResults
	}

	// Ensure we haven't been canceled yet
	select {
	case <-p.Cancel:
		// If p.Cancel has been closed, this will be triggered
		return "", Canceled

	default:
		// Otherwise we'll just do nothing immediately
	}

	// If we're within the length of our current request,
	// return right away and increment our counters
	if p.qr != nil && p.i < len(p.qr.Query.AllImages) {
		img := p.qr.Query.AllImages[p.i].URL
		p.i++
		p.count++
		return img, nil
	}

	// Otherwise, we need to create a new request. Recreate our request params
	// and reset per-request counter.
	p.i = 0
	params := url.Values{}
	params.Set("action", "query")
	params.Set("format", "json")
	params.Set("list", "allimages")
	params.Set("aidir", "descending")
	params.Set("aisort", "timestamp")

	// 500 is the most allowed by the API per request, but we may want less.
	if p.count+apiMax > p.max {
		params.Set("ailimit", strconv.Itoa(p.max-p.count))
	} else {
		params.Set("ailimit", strconv.Itoa(p.max))
	}

	// If we have a previous request with continue values, use them
	if p.qr != nil &&
		len(p.qr.Continue.Continue) > 0 &&
		len(p.qr.Continue.AIContinue) > 0 {
		params.Set("continue", p.qr.Continue.Continue)
		params.Set("aicontinue", p.qr.Continue.AIContinue)
	}

	// Call the wikimedia API
	resp, err := http.Get(queryURL + "?" + params.Encode())
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	// Read the contents of the response as bytes
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}

	// Parse the bytes into a struct
	p.qr = &queryResp{}
	err = json.Unmarshal(b, p.qr)
	if err != nil {
		return "", err
	}

	// If there's no more images, then return
	if len(p.qr.Query.AllImages) < 1 {
		return "", EndOfResults
	}

	// Return first value of the new request
	p.count++
	return p.qr.Query.AllImages[p.i].URL, nil
}

// FirstColor tries to return the first non-gray color in the image. A gray
// color is one that, when mapped to an xterm256 palette, has the same value
// for red, green and blue. Ok, we've defined "gray". What does "first" mean?
// We iterate through pixels starting with 0,0 (top left) and move to the
// bottom right one by one. In the worst case (a grayscale image), we iterate
// through every pixel, give up, and return the final pixel color even though
// it's gray. Both the xtermColor (an integer between 0-255) and a hex string
// (e.g., "#bb00cc") is returned.
func (p *Puller) FirstColor(imgURL string) (xtermColor int, hex string, err error) {
	// Create a request so we can use req.Cancel
	req, err := http.NewRequest("GET", imgURL, nil)
	if err != nil {
		return
	}

	// Set up cancellation pipeline, link request to puller
	req.Cancel = p.Cancel

	// Call the image server
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return
	}
	defer resp.Body.Close()

	// Decode into an object
	img, _, err := image.Decode(resp.Body)
	if err != nil {
		return
	}

	// Use our XTerm256 as a color.Palette so we can map the colors of the
	// image to our palette.
	pal := color.Palette(XTerm256)

	// Iterate through every pixel and try to find a color. If we don't find a
	// color (i.e., the image is grayscale) we'll default to the last pixel in
	// the image.
	rect := img.Bounds()
	i := 0
	for x := 0; x < rect.Dx(); x++ {
		for y := 0; y < rect.Dy(); y++ {

			// Check if p.Cancel has been closed once every cancelCheckpoint
			// iterations
			if i%cancelCheckpoint == 0 {
				select {

				case <-p.Cancel:
					// If p.Cancel has been closed, this will be triggered
					err = Canceled
					return

				default:
					// Otherwise we'll just do nothing immediately
				}
			}

			// xtermColor is the index in the palette which this
			// actual color maps to. It is also (by design) the
			// xterm256 value that maps to this color.
			xtermColor = pal.Index(img.At(x, y))

			// Get the color.RGBA value for this color. Not great to do a type
			// assertion here but easiest way to get 8-bit values without bit
			// fiddling.
			rgba, ok := pal[xtermColor].(color.RGBA)
			if !ok {
				err = errors.New("can't assert to color.RGBA")
				return
			}

			// Compute the hex value of the color
			hex = fmt.Sprintf("#%02x%02x%02x", rgba.R, rgba.G, rgba.B)

			// If any of the RGB values differ, it's a color, so we can stop.
			if !(rgba.R == rgba.G && rgba.G == rgba.B) {
				break
			}
		}
	}

	return
}
<commit_msg>update doc<commit_after>// Package wikimg can pull the latest image URLs from Wikimedia Commons
// https://commons.wikimedia.org and map image colors to an xterm256 color
// palette.
package wikimg

import (
	"encoding/json"
	"errors"
	"fmt"
	"image"
	"image/color"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"

	// We define which image formats we support by importing decoder packages
	_ "image/gif"
	_ "image/jpeg"
	_ "image/png"
)

var (
	// EndOfResults is returned by Next() when no more results are available
	EndOfResults = errors.New("end of results")

	// Canceled may be returned by Next() and FirstColor() when the client
	// closes the Cancel channel on a Puller
	Canceled = errors.New("wikimg: canceled image processing")
)

const (
	// queryURL is the API we are querying
	queryURL = "https://commons.wikimedia.org/w/api.php"

	// apiMax is the max results we can request from the API at one time
	apiMax = 500

	// cancelCheckpoint is the number of pixels between checking whether the
	// request was canceled when running FirstColor()
	cancelCheckpoint = 10000
)

// queryResp mirrors the JSON structure returned by queryURL, specifying only
// the info we're interested in.
type queryResp struct {

	// Continue contains strings we need to pass back into the API to
	// continue where we left off
	Continue struct {
		Continue   string
		AIContinue string
	}

	// Query contains the actual results
	Query struct {
		AllImages []struct {
			URL string
		}
	}
}

// Puller is an image puller that retrieves the most recent image URLs that
// have been uploaded to Wikimedia Commons https://commons.wikimedia.org
type Puller struct {
	// qr is the most recent response from the API
	qr *queryResp

	// i is the current index into qr.Query.AllImages
	i int

	// count is the total number of images we've collected
	count int

	// max is the maximum number of images we want to collect
	max int

	// Cancel is an optional channel. Setting this value on Puller
	// and closing the channel signals to the Puller that any
	// in process operations (i.e, retrieving an image or computing
	// its first color) should be canceled. Any future
	// calls to Next() or FirstColor() will return a Canceled
	// error.
	Cancel <-chan struct{}
}

// NewPuller creates a puller that can return at most max images when calls to
// Next() are made
func NewPuller(max int) *Puller {
	return &Puller{
		max: max,
	}
}

// Next returns the next most recent image URL. If no more results are
// available EndOfResults is returned as an error.
func (p *Puller) Next() (string, error) {
	// If we've exceeded that max we want to get, then stop
	if p.count >= p.max {
		return "", EndOfResults
	}

	// Ensure we haven't been canceled yet
	select {
	case <-p.Cancel:
		// If p.Cancel has been closed, this will be triggered
		return "", Canceled

	default:
		// Otherwise we'll just do nothing immediately
	}

	// If we're within the length of our current request,
	// return right away and increment our counters
	if p.qr != nil && p.i < len(p.qr.Query.AllImages) {
		img := p.qr.Query.AllImages[p.i].URL
		p.i++
		p.count++
		return img, nil
	}

	// Otherwise, we need to create a new request. Recreate our request params
	// and reset per-request counter.
	p.i = 0
	params := url.Values{}
	params.Set("action", "query")
	params.Set("format", "json")
	params.Set("list", "allimages")
	params.Set("aidir", "descending")
	params.Set("aisort", "timestamp")

	// 500 is the most allowed by the API per request, but we may want less.
	if p.count+apiMax > p.max {
		params.Set("ailimit", strconv.Itoa(p.max-p.count))
	} else {
		params.Set("ailimit", strconv.Itoa(p.max))
	}

	// If we have a previous request with continue values, use them
	if p.qr != nil &&
		len(p.qr.Continue.Continue) > 0 &&
		len(p.qr.Continue.AIContinue) > 0 {
		params.Set("continue", p.qr.Continue.Continue)
		params.Set("aicontinue", p.qr.Continue.AIContinue)
	}

	// Call the wikimedia API
	resp, err := http.Get(queryURL + "?" + params.Encode())
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	// Read the contents of the response as bytes
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}

	// Parse the bytes into a struct
	p.qr = &queryResp{}
	err = json.Unmarshal(b, p.qr)
	if err != nil {
		return "", err
	}

	// If there's no more images, then return
	if len(p.qr.Query.AllImages) < 1 {
		return "", EndOfResults
	}

	// Return first value of the new request
	p.count++
	return p.qr.Query.AllImages[p.i].URL, nil
}

// FirstColor tries to return the first non-gray color in the image. A gray
// color is one that, when mapped to an xterm256 palette, has the same value
// for red, green and blue. We iterate through pixels starting with 0,0 and
// through each x and y value. In the worst case (a grayscale image), we
// iterate through every pixel, give up, and return the final pixel color even
// though it's gray. Both the xtermColor (an integer between 0-255) and a hex
// string (e.g., "#bb00cc") is returned.
func (p *Puller) FirstColor(imgURL string) (xtermColor int, hex string, err error) {
	// Create a request so we can use req.Cancel
	req, err := http.NewRequest("GET", imgURL, nil)
	if err != nil {
		return
	}

	// Set up cancellation pipeline, link request to puller
	req.Cancel = p.Cancel

	// Call the image server
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return
	}
	defer resp.Body.Close()

	// Decode into an object
	img, _, err := image.Decode(resp.Body)
	if err != nil {
		return
	}

	// Use our XTerm256 as a color.Palette so we can map the colors of the
	// image to our palette.
	pal := color.Palette(XTerm256)

	// Iterate through every pixel and try to find a color. If we don't find a
	// color (i.e., the image is grayscale) we'll default to the last pixel in
	// the image.
	rect := img.Bounds()
	i := 0
	for x := 0; x < rect.Dx(); x++ {
		for y := 0; y < rect.Dy(); y++ {

			// Check if p.Cancel has been closed once every cancelCheckpoint
			// iterations
			if i%cancelCheckpoint == 0 {
				select {

				case <-p.Cancel:
					// If p.Cancel has been closed, this will be triggered
					err = Canceled
					return

				default:
					// Otherwise we'll just do nothing immediately
				}
			}

			// xtermColor is the index in the palette which this
			// actual color maps to. It is also (by design) the
			// xterm256 value that maps to this color.
			xtermColor = pal.Index(img.At(x, y))

			// Get the color.RGBA value for this color. Not great to do a type
			// assertion here but easiest way to get 8-bit values without bit
			// fiddling.
			rgba, ok := pal[xtermColor].(color.RGBA)
			if !ok {
				err = errors.New("can't assert to color.RGBA")
				return
			}

			// Compute the hex value of the color
			hex = fmt.Sprintf("#%02x%02x%02x", rgba.R, rgba.G, rgba.B)

			// If any of the RGB values differ, it's a color, so we can stop.
			if !(rgba.R == rgba.G && rgba.G == rgba.B) {
				break
			}
		}
	}

	return
}
<|endoftext|>


<commit_before>// Copyright 2018 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package convert_test

import (
	"io/ioutil"
	"math"
	"testing"

	"github.com/hajimehoshi/ebiten/audio"
	. "github.com/hajimehoshi/ebiten/audio/internal/convert"
)

func soundAt(timeInSecond float64) float64 {
	const freq = 220

	amp := []float64{1.0, 0.8, 0.6, 0.4, 0.2}
	v := 0.0
	for j := 0; j < len(amp); j++ {
		v += amp[j] * math.Sin(2.0*math.Pi*timeInSecond*freq*float64(j+1))
	}
	if v > 1 {
		v = 1
	}
	if v < -1 {
		v = -1
	}
	return v
}

func newSoundBytes(sampleRate int) []byte {
	b := make([]byte, sampleRate*4) // 1 second
	for i := 0; i < len(b)/4; i++ {
		v := soundAt(float64(i) / float64(sampleRate))
		v16 := int16(v * (1<<15 - 1))
		b[4*i] = byte(v16)
		b[4*i+1] = byte(v16 >> 8)
		b[4*i+2] = byte(v16)
		b[4*i+3] = byte(v16 >> 8)
	}
	return b
}

func abs(x int) int {
	if x < 0 {
		return -x
	}
	return x
}

func TestResampling(t *testing.T) {
	cases := []struct {
		In  int
		Out int
	}{
		{
			In:  44100,
			Out: 48000,
		},
		{
			In:  48000,
			Out: 44100,
		},
	}
	for _, c := range cases {
		inB := newSoundBytes(c.In)
		outS := NewResampling(audio.BytesReadSeekCloser(inB), int64(len(inB)), c.In, c.Out)
		gotB, err := ioutil.ReadAll(outS)
		if err != nil {
			t.Fatal(err)
		}
		wantB := newSoundBytes(c.Out)
		if len(gotB) != len(wantB) {
			t.Errorf("len(gotB) == %d but len(wantB) == %d", len(gotB), len(wantB))
		}
		for i := 0; i < len(gotB)/2; i++ {
			got := float64(int16(gotB[2*i])|(int16(gotB[2*i+1])<<8)) / (1<<15 - 1)
			want := float64(int16(wantB[2*i])|(int16(wantB[2*i+1])<<8)) / (1<<15 - 1)
			if math.Abs(got-want) > 0.025 {
				t.Errorf("sample rate: %d, index: %d: got: %f, want: %f", c.Out, i, got, want)
			}
		}
	}
}
<commit_msg>audio/internal/convert: Remove unused functions<commit_after>// Copyright 2018 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package convert_test

import (
	"io/ioutil"
	"math"
	"testing"

	"github.com/hajimehoshi/ebiten/audio"
	. "github.com/hajimehoshi/ebiten/audio/internal/convert"
)

func soundAt(timeInSecond float64) float64 {
	const freq = 220

	amp := []float64{1.0, 0.8, 0.6, 0.4, 0.2}
	v := 0.0
	for j := 0; j < len(amp); j++ {
		v += amp[j] * math.Sin(2.0*math.Pi*timeInSecond*freq*float64(j+1))
	}
	if v > 1 {
		v = 1
	}
	if v < -1 {
		v = -1
	}
	return v
}

func newSoundBytes(sampleRate int) []byte {
	b := make([]byte, sampleRate*4) // 1 second
	for i := 0; i < len(b)/4; i++ {
		v := soundAt(float64(i) / float64(sampleRate))
		v16 := int16(v * (1<<15 - 1))
		b[4*i] = byte(v16)
		b[4*i+1] = byte(v16 >> 8)
		b[4*i+2] = byte(v16)
		b[4*i+3] = byte(v16 >> 8)
	}
	return b
}

func TestResampling(t *testing.T) {
	cases := []struct {
		In  int
		Out int
	}{
		{
			In:  44100,
			Out: 48000,
		},
		{
			In:  48000,
			Out: 44100,
		},
	}
	for _, c := range cases {
		inB := newSoundBytes(c.In)
		outS := NewResampling(audio.BytesReadSeekCloser(inB), int64(len(inB)), c.In, c.Out)
		gotB, err := ioutil.ReadAll(outS)
		if err != nil {
			t.Fatal(err)
		}
		wantB := newSoundBytes(c.Out)
		if len(gotB) != len(wantB) {
			t.Errorf("len(gotB) == %d but len(wantB) == %d", len(gotB), len(wantB))
		}
		for i := 0; i < len(gotB)/2; i++ {
			got := float64(int16(gotB[2*i])|(int16(gotB[2*i+1])<<8)) / (1<<15 - 1)
			want := float64(int16(wantB[2*i])|(int16(wantB[2*i+1])<<8)) / (1<<15 - 1)
			if math.Abs(got-want) > 0.025 {
				t.Errorf("sample rate: %d, index: %d: got: %f, want: %f", c.Out, i, got, want)
			}
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 Travis Keep. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or
// at http://opensource.org/licenses/BSD-3-Clause.

//This library is a modified version of the library at http://godoc.org/github.com/keep94/sunrise

// Package sunrise computes sunrises and sunsets using wikipedia article
// http://en.wikipedia.org/wiki/Sunrise_equation. Testing at my
// latitude and longitude in California shows that computed sunrises and
// sunsets can vary by as much as 2 minutes from those that NOAA reports
// at http://www.esrl.noaa.gov/gmd/grad/solcalc/sunrise.html.
package sunrise

import (
	"fmt"
	"math"
	"time"
)

const (
	jepoch = float64(2451545.0)
	uepoch = int64(946728000.0)
)

//Location gives sunrise and sunset times.
type Location struct {
	location        *time.Location
	latitude        float64
	longitude       float64
	jstar           float64
	solarNoon       float64
	hourAngleInDays float64
}

// NewLocation computes the sunrise and sunset times for latitude and longitude
// around currentTime. Generally, the computed sunrise will be no earlier
// than 24 hours before currentTime and the computed sunset will be no later
// than 24 hours after currentTime. However, these differences may exceed 24
// hours on days with more than 23 hours of daylight.
// The latitude is positive for north and negative for south. Longitude is
// positive for east and negative for west.
func NewLocation(latitude float64, longitude float64) *Location {
	l := &Location{
		location:  time.Now().Location(),
		latitude:  latitude,
		longitude: longitude,
		jstar:     jStar(longitude),
	}

	l.computeSolarNoonHourAngle()

	return l
}

//String interface to show location details
func (l *Location) String() string {
	return fmt.Sprintf("Calculation Details: Lat %.3f, Long %.3f", l.latitude, l.longitude)
}

//Today updates instance for calculation of today's sunrise and sunset
func (l *Location) Today() {
	l.jstar = jStar(l.longitude)

	l.computeSolarNoonHourAngle()
}

// AddDays computes the sunrise and sunset numDays after
// (or before if numDays is negative) the current sunrise and sunset at the
// same latitude and longitude.
func (l *Location) AddDays(numDays int) {
	l.jstar += float64(numDays)
	l.computeSolarNoonHourAngle()
}

// Sunrise returns the current computed sunrise. Returned sunrise has the same
// location as the time passed to Around.
func (l *Location) Sunrise() time.Time {
	return goTime(l.solarNoon-l.hourAngleInDays, l.location)
}

// Sunset returns the current computed sunset. Returned sunset has the same
// location as the time passed to Around.
func (l *Location) Sunset() time.Time {
	return goTime(l.solarNoon+l.hourAngleInDays, l.location)
}

func (l *Location) computeSolarNoonHourAngle() {
	ma := mod360(357.5291 + 0.98560028*(l.jstar-jepoch))
	center := 1.9148*sin(ma) + 0.02*sin(2.0*ma) + 0.0003*sin(3.0*ma)
	el := mod360(ma + 102.9372 + center + 180.0)
	l.solarNoon = l.jstar + 0.0053*sin(ma) - 0.0069*sin(2.0*el)
	declination := asin(sin(el) * sin(23.45))
	l.hourAngleInDays = acos((sin(-0.83)-sin(l.latitude)*sin(declination))/(cos(l.latitude)*cos(declination))) / 360.0
}

func julianDay(unix int64) float64 {
	return float64(unix-uepoch)/86400.0 + jepoch
}

func jStar(longitude float64) float64 {
	return math.Floor(
		julianDay(time.Now().Unix())-0.0009+longitude/360.0+0.5) + 0.0009 - longitude/360.0
}

func goTime(julianDay float64, loc *time.Location) time.Time {
	unix := uepoch + int64((julianDay-jepoch)*86400.0)
	return time.Unix(unix, 0).In(loc)
}

func sin(degrees float64) float64 {
	return math.Sin(degrees * math.Pi / 180.0)
}

func cos(degrees float64) float64 {
	return math.Cos(degrees * math.Pi / 180.0)
}

func asin(x float64) float64 {
	return math.Asin(x) * 180.0 / math.Pi
}

func acos(x float64) float64 {
	if x >= 1.0 {
		return 0.0
	}
	if x <= -1.0 {
		return 180.0
	}
	return math.Acos(x) * 180.0 / math.Pi
}

func mod360(x float64) float64 {
	return x - 360.0*math.Floor(x/360.0)
}
<commit_msg>Added String interface<commit_after>// Copyright 2013 Travis Keep. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or
// at http://opensource.org/licenses/BSD-3-Clause.

//This library is a modified version of the library at http://godoc.org/github.com/keep94/sunrise

// Package sunrise computes sunrises and sunsets using wikipedia article
// http://en.wikipedia.org/wiki/Sunrise_equation. Testing at my
// latitude and longitude in California shows that computed sunrises and
// sunsets can vary by as much as 2 minutes from those that NOAA reports
// at http://www.esrl.noaa.gov/gmd/grad/solcalc/sunrise.html.
package sunrise

import (
	"fmt"
	"math"
	"time"
)

const (
	jepoch = float64(2451545.0)
	uepoch = int64(946728000.0)
)

//Location gives sunrise and sunset times.
type Location struct {
	location        *time.Location
	latitude        float64
	longitude       float64
	jstar           float64
	solarNoon       float64
	hourAngleInDays float64
}

// NewLocation computes the sunrise and sunset times for latitude and longitude
// around currentTime. Generally, the computed sunrise will be no earlier
// than 24 hours before currentTime and the computed sunset will be no later
// than 24 hours after currentTime. However, these differences may exceed 24
// hours on days with more than 23 hours of daylight.
// The latitude is positive for north and negative for south. Longitude is
// positive for east and negative for west.
func NewLocation(latitude float64, longitude float64) *Location {
	l := &Location{
		location:  time.Now().Location(),
		latitude:  latitude,
		longitude: longitude,
		jstar:     jStar(longitude),
	}

	l.computeSolarNoonHourAngle()

	return l
}

//String interface to show location details
func (l *Location) String() string {
	return fmt.Sprintf("Calculation Details - Lat: %.3f, Long: %.3f", l.latitude, l.longitude)
}

//Today updates instance for calculation of today's sunrise and sunset
func (l *Location) Today() {
	l.jstar = jStar(l.longitude)

	l.computeSolarNoonHourAngle()
}

// AddDays computes the sunrise and sunset numDays after
// (or before if numDays is negative) the current sunrise and sunset at the
// same latitude and longitude.
func (l *Location) AddDays(numDays int) {
	l.jstar += float64(numDays)
	l.computeSolarNoonHourAngle()
}

// Sunrise returns the current computed sunrise. Returned sunrise has the same
// location as the time passed to Around.
func (l *Location) Sunrise() time.Time {
	return goTime(l.solarNoon-l.hourAngleInDays, l.location)
}

// Sunset returns the current computed sunset. Returned sunset has the same
// location as the time passed to Around.
func (l *Location) Sunset() time.Time {
	return goTime(l.solarNoon+l.hourAngleInDays, l.location)
}

func (l *Location) computeSolarNoonHourAngle() {
	ma := mod360(357.5291 + 0.98560028*(l.jstar-jepoch))
	center := 1.9148*sin(ma) + 0.02*sin(2.0*ma) + 0.0003*sin(3.0*ma)
	el := mod360(ma + 102.9372 + center + 180.0)
	l.solarNoon = l.jstar + 0.0053*sin(ma) - 0.0069*sin(2.0*el)
	declination := asin(sin(el) * sin(23.45))
	l.hourAngleInDays = acos((sin(-0.83)-sin(l.latitude)*sin(declination))/(cos(l.latitude)*cos(declination))) / 360.0
}

func julianDay(unix int64) float64 {
	return float64(unix-uepoch)/86400.0 + jepoch
}

func jStar(longitude float64) float64 {
	return math.Floor(
		julianDay(time.Now().Unix())-0.0009+longitude/360.0+0.5) + 0.0009 - longitude/360.0
}

func goTime(julianDay float64, loc *time.Location) time.Time {
	unix := uepoch + int64((julianDay-jepoch)*86400.0)
	return time.Unix(unix, 0).In(loc)
}

func sin(degrees float64) float64 {
	return math.Sin(degrees * math.Pi / 180.0)
}

func cos(degrees float64) float64 {
	return math.Cos(degrees * math.Pi / 180.0)
}

func asin(x float64) float64 {
	return math.Asin(x) * 180.0 / math.Pi
}

func acos(x float64) float64 {
	if x >= 1.0 {
		return 0.0
	}
	if x <= -1.0 {
		return 180.0
	}
	return math.Acos(x) * 180.0 / math.Pi
}

func mod360(x float64) float64 {
	return x - 360.0*math.Floor(x/360.0)
}
<|endoftext|>


<commit_before>package command

import "testing"

type testcase struct {
	in   *exportData
	want bool
}

func innerTest(t *testing.T, cases []testcase) {
	for _, c := range cases {
		if got := c.in.IsValid(); got != c.want {
			t.Errorf("IsValid(%q) == %q, want %q", c.in, got, c.want)
		}
	}
}

func TestDefaultExportDataIsValid(t *testing.T) {
	cases := []testcase{
		{
			in: &exportData{
				projectId: 1,
				limit:     1,
				from:      0,
				timeFmt:   "msec",
				output:    "json",
			},
			want: true,
		},
		{
			in: &exportData{
				projectId: 0, // must be > 0
				limit:     1,
				from:      0,
				timeFmt:   "msec",
				output:    "json",
			},
			want: false,
		},
		{
			in: &exportData{
				projectId: 1,
				limit:     0, // must be > 0
				from:      0,
				timeFmt:   "msec",
				output:    "json",
			},
			want: false,
		},
	}

	innerTest(t, cases)
}

func TestTimeRangeExportDataIsValid(t *testing.T) {
	cases := []testcase{
		{
			in: &exportData{
				projectId: 1,
				limit:     1,
				timeFmt:   "msec",
				output:    "json",
				from:      0,
			},
			want: true,
		},
		{
			in: &exportData{
				projectId: 1,
				limit:     1,
				timeFmt:   "msec",
				output:    "json",
				to:        10,
			},
			want: true,
		},
		{
			in: &exportData{
				projectId: 1,
				limit:     1,
				timeFmt:   "msec",
				output:    "json",
				from:      0,
				to:        10,
			},
			want: true,
		},
		{
			in: &exportData{
				projectId: 1,
				limit:     1,
				timeFmt:   "msec",
				output:    "json",
				from:      0,
				to:        0,
			},
			want: true,
		},
		{
			in: &exportData{
				projectId: 1,
				limit:     1,
				timeFmt:   "msec",
				output:    "json",
				from:      10,
				to:        0,
			},
			want: false,
		},
	}

	innerTest(t, cases)
}

func TestValRangeExportDataIsValid(t *testing.T) {
	cases := []testcase{
		{
			in: &exportData{
				projectId:   1,
				limit:       1,
				from:        0,
				timeFmt:     "msec",
				output:      "json",
				greaterThan: 0,
			},
			want: true,
		},
		{
			in: &exportData{
				projectId: 1,
				limit:     1,
				from:      0,
				timeFmt:   "msec",
				output:    "json",
				lessThan:  10,
			},
			want: true,
		},
		{
			in: &exportData{
				projectId:   1,
				limit:       1,
				from:        0,
				timeFmt:     "msec",
				output:      "json",
				greaterThan: 0,
				lessThan:    10,
			},
			want: true,
		},
		{
			in: &exportData{
				projectId:   1,
				limit:       1,
				from:        0,
				timeFmt:     "msec",
				output:      "json",
				greaterThan: 0,
				lessThan:    0,
			},
			want: true,
		},
		{
			in: &exportData{
				projectId:   1,
				limit:       1,
				from:        0,
				timeFmt:     "msec",
				output:      "json",
				greaterThan: 10,
				lessThan:    0,
			},
			want: false,
		},
	}

	innerTest(t, cases)
}
<commit_msg>command: add a bunch more exports tests<commit_after>package command

import "testing"

type testcase struct {
	in   *exportData
	want bool
}

func innerTest(t *testing.T, cases []testcase) {
	for _, c := range cases {
		if got := c.in.IsValid(); got != c.want {
			t.Errorf("IsValid(%q) == %q, want %q", c.in, got, c.want)
		}
	}
}

func TestDefaultExportDataIsValid(t *testing.T) {
	cases := []testcase{
		{
			in: &exportData{
				projectId: 1,
				limit:     1,
				from:      0,
				timeFmt:   "msec",
				output:    "json",
			},
			want: true,
		},
		{
			in: &exportData{
				projectId: 0, // must be > 0
				limit:     1,
				from:      0,
				timeFmt:   "msec",
				output:    "json",
			},
			want: false,
		},
		{
			in: &exportData{
				projectId: 1,
				limit:     0, // must be > 0
				from:      0,
				timeFmt:   "msec",
				output:    "json",
			},
			want: false,
		},
	}

	innerTest(t, cases)
}

// TestTimeRangeExportDataIsValid tests if the validity checks for
// time ranges are working.
func TestTimeRangeExportDataIsValid(t *testing.T) {
	cases := []testcase{
		{
			in: &exportData{
				projectId: 1,
				limit:     1,
				timeFmt:   "msec",
				output:    "json",
				from:      0,
			},
			want: true,
		},
		{
			in: &exportData{
				projectId: 1,
				limit:     1,
				timeFmt:   "msec",
				output:    "json",
				to:        10,
			},
			want: true,
		},
		{
			in: &exportData{
				projectId: 1,
				limit:     1,
				timeFmt:   "msec",
				output:    "json",
				from:      0,
				to:        10,
			},
			want: true,
		},
		{
			in: &exportData{
				projectId: 1,
				limit:     1,
				timeFmt:   "msec",
				output:    "json",
				from:      0,
				to:        0,
			},
			want: true,
		},
		{
			in: &exportData{
				projectId: 1,
				limit:     1,
				timeFmt:   "msec",
				output:    "json",
				from:      10,
				to:        0,
			},
			want: false,
		},
	}

	innerTest(t, cases)
}

// TestValRangeExportDataIsValid tests if the validity checks for
// value ranges are working.
func TestValRangeExportDataIsValid(t *testing.T) {
	cases := []testcase{
		{
			in: &exportData{
				projectId:   1,
				limit:       1,
				from:        0,
				timeFmt:     "msec",
				output:      "json",
				greaterThan: 0,
			},
			want: true,
		},
		{
			in: &exportData{
				projectId: 1,
				limit:     1,
				from:      0,
				timeFmt:   "msec",
				output:    "json",
				lessThan:  10,
			},
			want: true,
		},
		{
			in: &exportData{
				projectId:   1,
				limit:       1,
				from:        0,
				timeFmt:     "msec",
				output:      "json",
				greaterThan: 0,
				lessThan:    10,
			},
			want: true,
		},
		{
			in: &exportData{
				projectId:   1,
				limit:       1,
				from:        0,
				timeFmt:     "msec",
				output:      "json",
				greaterThan: 0,
				lessThan:    0,
			},
			want: true,
		},
		{
			in: &exportData{
				projectId:   1,
				limit:       1,
				from:        0,
				timeFmt:     "msec",
				output:      "json",
				greaterThan: 10,
				lessThan:    0,
			},
			want: false,
		},
	}

	innerTest(t, cases)
}

func TestEqualExportDataIsValid(t *testing.T) {
	cases := []testcase{
		{
			in: &exportData{
				projectId: 1,
				limit:     1,
				from:      0,
				timeFmt:   "msec",
				output:    "json",
				equal:     "0",
			},
			want: true,
		},
		{
			in: &exportData{
				projectId: 1,
				limit:     1,
				from:      0,
				timeFmt:   "msec",
				output:    "json",
				equal:     "-10",
			},
			want: true,
		},
		{
			in: &exportData{
				projectId: 1,
				limit:     1,
				from:      0,
				timeFmt:   "msec",
				output:    "json",
				equal:     "10",
			},
			want: true,
		},
		{
			in: &exportData{
				projectId: 1,
				limit:     1,
				from:      0,
				timeFmt:   "msec",
				output:    "json",
				equal:     "not a number",
			},
			want: false,
		},
	}

	innerTest(t, cases)
}

func TestOpExportDataIsValid(t *testing.T) {
	cases := []testcase{
		{
			in: &exportData{
				projectId: 1,
				limit:     1,
				from:      0,
				timeFmt:   "msec",
				output:    "json",
				operator:  "sum",
			},
			want: true,
		},
		{
			in: &exportData{
				projectId: 1,
				limit:     1,
				from:      0,
				timeFmt:   "msec",
				output:    "json",
				operator:  "count",
			},
			want: true,
		},
		{
			in: &exportData{
				projectId: 1,
				limit:     1,
				from:      0,
				timeFmt:   "msec",
				output:    "json",
				operator:  "min",
			},
			want: true,
		},
		{
			in: &exportData{
				projectId: 1,
				limit:     1,
				from:      0,
				timeFmt:   "msec",
				output:    "json",
				operator:  "max",
			},
			want: true,
		},
		{
			in: &exportData{
				projectId: 1,
				limit:     1,
				from:      0,
				timeFmt:   "msec",
				output:    "json",
				operator:  "mean",
			},
			want: true,
		},
		{
			in: &exportData{
				projectId: 1,
				limit:     1,
				from:      0,
				timeFmt:   "msec",
				output:    "json",
				operator:  "not an op",
			},
			want: false,
		},
	}

	innerTest(t, cases)
}

func TestGroupExportDataIsValid(t *testing.T) {
	cases := []testcase{
		{
			in: &exportData{
				projectId: 1,
				limit:     1,
				from:      0,
				timeFmt:   "msec",
				output:    "json",
				operator:  "sum",
				groupBy:   "1h",
			},
			want: true,
		},
		{
			in: &exportData{
				projectId: 1,
				limit:     1,
				from:      0,
				timeFmt:   "msec",
				output:    "json",
				operator:  "count",
				groupBy:   "12q", // q not a valid unit
			},
			want: false,
		},
		{
			in: &exportData{
				projectId: 1,
				limit:     1,
				from:      0,
				timeFmt:   "msec",
				output:    "json",
				groupBy:   "1h", // no operator
			},
			want: false,
		},
	}

	innerTest(t, cases)
}

func TestTimeFmtExportDataIsValid(t *testing.T) {
	cases := []testcase{
		{
			in: &exportData{
				projectId: 1,
				limit:     1,
				from:      0,
				timeFmt:   "sec",
				output:    "json",
			},
			want: true,
		},
		{
			in: &exportData{
				projectId: 1,
				limit:     1,
				from:      0,
				timeFmt:   "msec",
				output:    "json",
			},
			want: true,
		},
		{
			in: &exportData{
				projectId: 1,
				limit:     1,
				from:      0,
				timeFmt:   "usec",
				output:    "json",
			},
			want: true,
		},
		{
			in: &exportData{
				projectId: 1,
				limit:     1,
				from:      0,
				timeFmt:   "timeval",
				output:    "json",
			},
			want: true,
		},
		{
			in: &exportData{
				projectId: 1,
				limit:     1,
				from:      0,
				timeFmt:   "not valid",
				output:    "json",
			},
			want: false,
		},
	}

	innerTest(t, cases)
}

func TestOutputExportDataIsValid(t *testing.T) {
	cases := []testcase{
		{
			in: &exportData{
				projectId: 1,
				limit:     1,
				from:      0,
				timeFmt:   "msec",
				output:    "json",
			},
			want: true,
		},
		{
			in: &exportData{
				projectId: 1,
				limit:     1,
				from:      0,
				timeFmt:   "msec",
				output:    "csv",
			},
			want: true,
		},
		{
			in: &exportData{
				projectId: 1,
				limit:     1,
				from:      0,
				timeFmt:   "msec",
				output:    "not valid",
			},
			want: false,
		},
	}

	innerTest(t, cases)
}
<|endoftext|>


<commit_before>package commands

import (
	"testing"

	"github.com/Skarlso/go-furnace/config"
	"github.com/aws/aws-sdk-go/service/cloudformation"
	"github.com/aws/aws-sdk-go/service/cloudformation/cloudformationiface"
)

type fakeCFClient struct {
	cloudformationiface.CloudFormationAPI
	err error
}

func (fc *fakeCFClient) ValidateTemplate(input *cloudformation.ValidateTemplateInput) (*cloudformation.ValidateTemplateOutput, error) {
	return &cloudformation.ValidateTemplateOutput{}, fc.err
}

func (fc *fakeCFClient) CreateStack(input *cloudformation.CreateStackInput) (*cloudformation.CreateStackOutput, error) {
	return &cloudformation.CreateStackOutput{}, fc.err
}

func (fc *fakeCFClient) WaitUntilStackCreateComplete(input *cloudformation.DescribeStacksInput) error {
	return fc.err
}

func (fc *fakeCFClient) DescribeStacks(input *cloudformation.DescribeStacksInput) (*cloudformation.DescribeStacksOutput, error) {
	return &cloudformation.DescribeStacksOutput{}, fc.err
}

func TestCreateCommand(t *testing.T) {
	config.WAITFREQUENCY = 0
	client := new(CFClient)
	client.Client = &fakeCFClient{err: nil}
	config := []byte("{}")
	stackname := "TestStack"
	create(stackname, config, client)
}
<commit_msg>Actually testing something. :P<commit_after>package commands

import (
	"testing"

	"github.com/Skarlso/go-furnace/config"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/cloudformation"
	"github.com/aws/aws-sdk-go/service/cloudformation/cloudformationiface"
)

var notEmptyStack = &cloudformation.DescribeStacksOutput{
	Stacks: []*cloudformation.Stack{
		&cloudformation.Stack{StackName: aws.String("TestStack")},
	},
}

type fakeCFClient struct {
	cloudformationiface.CloudFormationAPI
	stackname string
	err       error
}

func (fc *fakeCFClient) ValidateTemplate(input *cloudformation.ValidateTemplateInput) (*cloudformation.ValidateTemplateOutput, error) {
	return &cloudformation.ValidateTemplateOutput{}, fc.err
}

func (fc *fakeCFClient) CreateStack(input *cloudformation.CreateStackInput) (*cloudformation.CreateStackOutput, error) {
	if fc.stackname == "NotEmptyStack" {
		return &cloudformation.CreateStackOutput{StackId: aws.String("DummyID")}, fc.err
	}
	return &cloudformation.CreateStackOutput{}, fc.err
}

func (fc *fakeCFClient) WaitUntilStackCreateComplete(input *cloudformation.DescribeStacksInput) error {
	return fc.err
}

func (fc *fakeCFClient) DescribeStacks(input *cloudformation.DescribeStacksInput) (*cloudformation.DescribeStacksOutput, error) {
	if fc.stackname == "NotEmptyStack" {
		return notEmptyStack, fc.err
	}
	return &cloudformation.DescribeStacksOutput{}, fc.err
}

func TestCreateProcedure(t *testing.T) {
	config.WAITFREQUENCY = 0
	client := new(CFClient)
	stackname := "NotEmptyStack"
	client.Client = &fakeCFClient{err: nil, stackname: stackname}
	config := []byte("{}")
	stacks := create(stackname, config, client)
	if len(stacks) == 0 {
		t.Fatal("Stack was not returned by create.")
	}
	if *stacks[0].StackName != "TestStack" {
		t.Fatal("Not the correct stack returned. Returned was:", stacks)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2016 caicloud authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package worker

import (
	"fmt"
	"strings"
	"time"

	"github.com/caicloud/cyclone/api"
	"github.com/caicloud/cyclone/cloud"
	"github.com/caicloud/cyclone/docker"
	"github.com/caicloud/cyclone/pkg/osutil"
	"github.com/caicloud/cyclone/websocket"
	"github.com/caicloud/cyclone/worker/ci"
	"github.com/caicloud/cyclone/worker/ci/parser"
	"github.com/caicloud/cyclone/worker/handler"
	"github.com/caicloud/cyclone/worker/helper"
	worker_log "github.com/caicloud/cyclone/worker/log"
	"github.com/caicloud/cyclone/worker/vcs"
	"github.com/zoumo/logdog"
)

const (
	WorkerTimeout = 7200 * time.Second
	WaitTimes     = 5

	DefaultCaicloudYaml = "build:\n  dockerfile_name: Dockerfile"
)

// Worker ...
type Worker struct {
	Config *Config
	Envs   *cloud.WorkerEnvs
}

// Run starts the worker
func (worker *Worker) Run() error {

	logdog.Info("worker start with", logdog.Fields{"config": worker.Config, "envs": worker.Envs})

	// Get event for cyclone server
	event, err := worker.getEvent()
	if err != nil {
		setEventFailStatus(&event, err.Error())
		sendErr := worker.sendEvent(event)
		if sendErr != nil {
			logdog.Errorf("set event result err: %v", err)
		}
		return err
	}
	logdog.Info("get event success", logdog.Fields{"event": event})

	// Handle event
	logdog.Info("handleEvent ...")
	worker.handleEvent(&event)

	// Sent event for cyclone server
	err = worker.sendEvent(event)
	if err != nil {
		logdog.Errorf("set event result err: %v", err)
		return err
	}
	logdog.Info("send event to server", logdog.Fields{"event": event})
	return nil
}

// getEvent used for getting event for cyclone server
func (worker *Worker) getEvent() (api.Event, error) {
	eventID := worker.Config.ID
	serverHost := worker.Envs.CycloneServer

	BaseURL := fmt.Sprintf("%s/api/%s", serverHost, api.APIVersion)
	httpHandler := handler.NewHTTPHandler(BaseURL)

	var getResponse api.GetEventResponse
	err := httpHandler.GetEvent(eventID, &getResponse)
	if err != nil {
		return api.Event{}, err
	}

	event := getResponse.Event

	return event, nil
}

// handleEvent analysize the the operation in event, and do the relate operation
func (worker *Worker) handleEvent(event *api.Event) {
	vcsManager := vcs.NewManager()

	logServer := worker.Envs.LogServer
	err := worker_log.DialLogServer(logServer)
	if err != nil {
		logdog.Errorf("dail log server err: %v", err)
	} else {
		go worker_log.SendHeartBeat()
		defer worker_log.Disconnect()
	}

	switch event.Operation {
	case "create-service":
		worker.createService(vcsManager, event)

	case "create-version":
		worker.createVersion(vcsManager, event)

	default:
		setEventFailStatus(event, "unkwon operation")
	}
}

// createService verify repo validity for creating service
func (worker *Worker) createService(vcsManager *vcs.Manager, event *api.Event) {
	// err := vcsManager.CloneServiceRepository(event)
	err := vcsManager.CheckRepoValid(event)
	if err != nil {
		setEventFailStatus(event, err.Error())
	} else {
		event.Status = api.EventStatusSuccess
	}

}

// createVersion create version for service and push log to server via websocket
// Step1: clone code from VCS
// Step2: integretion
// Step3: publish
// Step4: deploy
func (worker *Worker) createVersion(vcsManager *vcs.Manager, event *api.Event) {
	registryLocation := worker.Envs.RegistryLocation
	registryUsername := worker.Envs.RegistryUsername
	registryPassword := worker.Envs.RegistryPassword

	dockerManager, err := docker.NewManager(worker.Config.DockerHost, "",
		api.RegistryCompose{registryLocation, registryUsername, registryPassword})
	if err != nil {
		setEventFailStatus(event, err.Error())
		return
	}
	ciManager, err := ci.NewManager(dockerManager)
	if err != nil {
		setEventFailStatus(event, err.Error())
		return
	}

	err = worker_log.CreateFileBuffer(event.EventID)
	if err != nil {
		setEventFailStatus(event, err.Error())
		return
	}
	output := worker_log.Output
	ch := make(chan interface{})
	defer func() {
		// Push log file to cyclone.
		if err := helper.PushLogToCyclone(event); err != nil {
			setEventFailStatus(event, err.Error())
		}
		output.Close()
		worker_log.SetWatchLogFileSwitch(output.Name(), false)
		// wait util the send the log to kafka throuth cyclone server totally.
		for i := 0; i < WaitTimes; i++ {
			if isChanClosed(ch) {
				break
			}
			time.Sleep(time.Second * 2)
		}
	}()

	topicLog := websocket.CreateTopicName(string(event.Operation), event.Service.UserID,
		event.Service.ServiceID, event.Version.VersionID)
	go worker_log.WatchLogFile(output.Name(), topicLog, ch)

	destDir := vcsManager.GetCloneDir(&event.Service, &event.Version)
	event.Data["context-dir"] = destDir
	event.Data["image-name"] = fmt.Sprintf("%s/%s/%s", dockerManager.Registry,
		strings.ToLower(event.Service.Username), strings.ToLower(event.Service.Name))
	event.Data["tag-name"] = event.Version.Name

	if err = vcsManager.CloneVersionRepository(event); err != nil {
		setEventFailStatus(event, err.Error())
		return
	}

	replaceDockerfile(event, destDir)
	replaceCaicloudYaml(event, destDir)

	// Get the execution tree from the caicloud.yml.
	tree, err := ciManager.Parse(event)
	if err != nil {
		setEventFailStatus(event, err.Error())
		return

	}
	worker.yamlBuild(event, tree, dockerManager, ciManager)
}

// replaceDockerfile create new dockerfile to replace the dockerfile in repo
func replaceDockerfile(event *api.Event, destDir string) {

	// Create dockerfile and caicloud.yml if need
	if event.Service.Dockerfile != "" {
		path := destDir + "/Dockerfile"
		err := osutil.ReplaceFile(path, strings.NewReader(event.Service.Dockerfile))
		if err != nil {
			worker_log.InsertStepLog(event, worker_log.CloneRepository, worker_log.Stop, err)
			return
		}
	}
}

// replaceCaicloudYaml create new yaml to replace the yaml in repo
// default yaml file is caicloud.yml
// if you set your own yaml config name, cyclone will use it.
// if there is no yaml found, cyclone will create a default yaml file with only build step.
func replaceCaicloudYaml(event *api.Event, destDir string) {

	yamlFile := destDir + "/" + ci.DefaultYamlFile
	if event.Service.YAMLConfigName != "" {
		yamlFile = destDir + "/" + event.Service.YAMLConfigName
	}

	if event.Service.CaicloudYaml != "" || !osutil.IsFileExists(yamlFile) {
		content := DefaultCaicloudYaml
		if event.Service.CaicloudYaml != "" {
			content = event.Service.CaicloudYaml
		}
		err := osutil.ReplaceFile(yamlFile, strings.NewReader(content))
		if err != nil {
			worker_log.InsertStepLog(event, worker_log.CloneRepository, worker_log.Stop, err)
			return
		}
	}
}

// yamlBuild func uses for build with yaml file.
func (worker *Worker) yamlBuild(event *api.Event, tree *parser.Tree, dockerManager *docker.Manager, ciManager *ci.Manager) {
	operation := string(event.Version.Operation)
	// Load the tree to a runner.Build.
	r, err := ciManager.LoadTree(event, tree)
	if err != nil {
		setEventFailStatus(event, err.Error())
		return
	}

	// Build image
	if err = helper.ExecBuild(ciManager, r); err != nil {
		setEventFailStatus(event, err.Error())
		return
	}

	// If need integration
	if strings.Contains(operation, "integration") {
		// Integration
		if err = helper.ExecIntegration(ciManager, r); err != nil {
			setEventFailStatus(event, err.Error())
			return
		}
	}

	// If need publish
	if strings.Contains(operation, "publish") {
		// Publish
		if err = helper.ExecPublish(ciManager, r); err != nil {
			setEventFailStatus(event, err.Error())
			return
		}
	}

	// If need deploy
	if strings.Contains(operation, "deploy") {
		// Deploy
		if err = helper.ExecDeploy(event, dockerManager, r, tree); err != nil {
			setEventFailStatus(event, err.Error())
			return
		}

		err = worker.sendEvent(*event)
		if err != nil {
			logdog.Errorf("set event result err: %v", err)
		}

		// Deploy Check
		helper.ExecDeployCheck(event, tree)

	}
	event.Status = api.EventStatusSuccess
}

// sendEvent used for setting event for circe server
func (worker *Worker) sendEvent(event api.Event) error {
	eventID := worker.Config.ID
	serverHost := worker.Envs.CycloneServer

	BaseURL := fmt.Sprintf("%s/api/%s", serverHost, api.APIVersion)
	httpHandler := handler.NewHTTPHandler(BaseURL)
	result := &api.SetEvent{
		Event: event,
	}
	var setResponse api.SetEventResponse
	var err error
	DueTime := time.Now().Add(time.Duration(WorkerTimeout))
	for DueTime.After(time.Now()) == true {
		err = httpHandler.SetEvent(eventID, result, &setResponse)
		if err != nil {
			if strings.Contains(err.Error(), "connection refused") {
				time.Sleep(time.Minute)
				continue
			}
			return err
		}
		logdog.Infof("set event result: %v", setResponse)
		return nil
	}
	return err
}

func isChanClosed(ch chan interface{}) bool {
	if len(ch) == 0 {
		select {
		case _, ok := <-ch:
			return !ok
		}
	}
	return false
}

// setEventFailStatus sets the fail status of the event.
func setEventFailStatus(event *api.Event, ErrorMessage string) {
	event.Status = api.EventStatusFail
	event.ErrorMessage = ErrorMessage
	logdog.Error("Operation failed", logdog.Fields{"event": event})
}
<commit_msg>fix(214): fix panice error when dial websocket gets error<commit_after>/*
Copyright 2016 caicloud authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package worker

import (
	"fmt"
	"strings"
	"time"

	"github.com/caicloud/cyclone/api"
	"github.com/caicloud/cyclone/cloud"
	"github.com/caicloud/cyclone/docker"
	"github.com/caicloud/cyclone/pkg/osutil"
	"github.com/caicloud/cyclone/websocket"
	"github.com/caicloud/cyclone/worker/ci"
	"github.com/caicloud/cyclone/worker/ci/parser"
	"github.com/caicloud/cyclone/worker/handler"
	"github.com/caicloud/cyclone/worker/helper"
	worker_log "github.com/caicloud/cyclone/worker/log"
	"github.com/caicloud/cyclone/worker/vcs"
	"github.com/zoumo/logdog"
)

const (
	WorkerTimeout = 7200 * time.Second
	WaitTimes     = 5

	DefaultCaicloudYaml = "build:\n  dockerfile_name: Dockerfile"
)

// Worker ...
type Worker struct {
	Config *Config
	Envs   *cloud.WorkerEnvs
}

// Run starts the worker
func (worker *Worker) Run() error {

	logdog.Info("worker start with", logdog.Fields{"config": worker.Config, "envs": worker.Envs})

	// Get event for cyclone server
	event, err := worker.getEvent()
	if err != nil {
		setEventFailStatus(&event, err.Error())
		sendErr := worker.sendEvent(event)
		if sendErr != nil {
			logdog.Errorf("set event result err: %v", err)
		}
		return err
	}
	logdog.Info("get event success", logdog.Fields{"event": event})

	// Handle event
	logdog.Info("handleEvent ...")
	worker.handleEvent(&event)

	// Sent event for cyclone server
	err = worker.sendEvent(event)
	if err != nil {
		logdog.Errorf("set event result err: %v", err)
		return err
	}
	logdog.Info("send event to server", logdog.Fields{"event": event})
	return nil
}

// getEvent used for getting event for cyclone server
func (worker *Worker) getEvent() (api.Event, error) {
	eventID := worker.Config.ID
	serverHost := worker.Envs.CycloneServer

	BaseURL := fmt.Sprintf("%s/api/%s", serverHost, api.APIVersion)
	httpHandler := handler.NewHTTPHandler(BaseURL)

	var getResponse api.GetEventResponse
	err := httpHandler.GetEvent(eventID, &getResponse)
	if err != nil {
		return api.Event{}, err
	}

	event := getResponse.Event

	return event, nil
}

// handleEvent analysize the the operation in event, and do the relate operation
func (worker *Worker) handleEvent(event *api.Event) {
	vcsManager := vcs.NewManager()

	logServer := worker.Envs.LogServer

	if err := worker_log.DialLogServer(logServer); err != nil {
		setEventFailStatus(event, err.Error())
		return
	}

	go worker_log.SendHeartBeat()
	defer worker_log.Disconnect()

	switch event.Operation {
	case "create-service":
		worker.createService(vcsManager, event)

	case "create-version":
		worker.createVersion(vcsManager, event)

	default:
		setEventFailStatus(event, "unkwon operation")
	}
}

// createService verify repo validity for creating service
func (worker *Worker) createService(vcsManager *vcs.Manager, event *api.Event) {
	// err := vcsManager.CloneServiceRepository(event)
	err := vcsManager.CheckRepoValid(event)
	if err != nil {
		setEventFailStatus(event, err.Error())
	} else {
		event.Status = api.EventStatusSuccess
	}

}

// createVersion create version for service and push log to server via websocket
// Step1: clone code from VCS
// Step2: integretion
// Step3: publish
// Step4: deploy
func (worker *Worker) createVersion(vcsManager *vcs.Manager, event *api.Event) {
	registryLocation := worker.Envs.RegistryLocation
	registryUsername := worker.Envs.RegistryUsername
	registryPassword := worker.Envs.RegistryPassword

	dockerManager, err := docker.NewManager(worker.Config.DockerHost, "",
		api.RegistryCompose{registryLocation, registryUsername, registryPassword})
	if err != nil {
		setEventFailStatus(event, err.Error())
		return
	}
	ciManager, err := ci.NewManager(dockerManager)
	if err != nil {
		setEventFailStatus(event, err.Error())
		return
	}

	err = worker_log.CreateFileBuffer(event.EventID)
	if err != nil {
		setEventFailStatus(event, err.Error())
		return
	}
	output := worker_log.Output
	ch := make(chan interface{})
	defer func() {
		// Push log file to cyclone.
		if err := helper.PushLogToCyclone(event); err != nil {
			setEventFailStatus(event, err.Error())
		}
		output.Close()
		worker_log.SetWatchLogFileSwitch(output.Name(), false)
		// wait util the send the log to kafka throuth cyclone server totally.
		for i := 0; i < WaitTimes; i++ {
			if isChanClosed(ch) {
				break
			}
			time.Sleep(time.Second * 2)
		}
	}()

	topicLog := websocket.CreateTopicName(string(event.Operation), event.Service.UserID,
		event.Service.ServiceID, event.Version.VersionID)
	go worker_log.WatchLogFile(output.Name(), topicLog, ch)

	destDir := vcsManager.GetCloneDir(&event.Service, &event.Version)
	event.Data["context-dir"] = destDir
	event.Data["image-name"] = fmt.Sprintf("%s/%s/%s", dockerManager.Registry,
		strings.ToLower(event.Service.Username), strings.ToLower(event.Service.Name))
	event.Data["tag-name"] = event.Version.Name

	if err = vcsManager.CloneVersionRepository(event); err != nil {
		setEventFailStatus(event, err.Error())
		return
	}

	replaceDockerfile(event, destDir)
	replaceCaicloudYaml(event, destDir)

	// Get the execution tree from the caicloud.yml.
	tree, err := ciManager.Parse(event)
	if err != nil {
		setEventFailStatus(event, err.Error())
		return

	}
	worker.yamlBuild(event, tree, dockerManager, ciManager)
}

// replaceDockerfile create new dockerfile to replace the dockerfile in repo
func replaceDockerfile(event *api.Event, destDir string) {

	// Create dockerfile and caicloud.yml if need
	if event.Service.Dockerfile != "" {
		path := destDir + "/Dockerfile"
		err := osutil.ReplaceFile(path, strings.NewReader(event.Service.Dockerfile))
		if err != nil {
			worker_log.InsertStepLog(event, worker_log.CloneRepository, worker_log.Stop, err)
			return
		}
	}
}

// replaceCaicloudYaml create new yaml to replace the yaml in repo
// default yaml file is caicloud.yml
// if you set your own yaml config name, cyclone will use it.
// if there is no yaml found, cyclone will create a default yaml file with only build step.
func replaceCaicloudYaml(event *api.Event, destDir string) {

	yamlFile := destDir + "/" + ci.DefaultYamlFile
	if event.Service.YAMLConfigName != "" {
		yamlFile = destDir + "/" + event.Service.YAMLConfigName
	}

	if event.Service.CaicloudYaml != "" || !osutil.IsFileExists(yamlFile) {
		content := DefaultCaicloudYaml
		if event.Service.CaicloudYaml != "" {
			content = event.Service.CaicloudYaml
		}
		err := osutil.ReplaceFile(yamlFile, strings.NewReader(content))
		if err != nil {
			worker_log.InsertStepLog(event, worker_log.CloneRepository, worker_log.Stop, err)
			return
		}
	}
}

// yamlBuild func uses for build with yaml file.
func (worker *Worker) yamlBuild(event *api.Event, tree *parser.Tree, dockerManager *docker.Manager, ciManager *ci.Manager) {
	operation := string(event.Version.Operation)
	// Load the tree to a runner.Build.
	r, err := ciManager.LoadTree(event, tree)
	if err != nil {
		setEventFailStatus(event, err.Error())
		return
	}

	// Build image
	if err = helper.ExecBuild(ciManager, r); err != nil {
		setEventFailStatus(event, err.Error())
		return
	}

	// If need integration
	if strings.Contains(operation, "integration") {
		// Integration
		if err = helper.ExecIntegration(ciManager, r); err != nil {
			setEventFailStatus(event, err.Error())
			return
		}
	}

	// If need publish
	if strings.Contains(operation, "publish") {
		// Publish
		if err = helper.ExecPublish(ciManager, r); err != nil {
			setEventFailStatus(event, err.Error())
			return
		}
	}

	// If need deploy
	if strings.Contains(operation, "deploy") {
		// Deploy
		if err = helper.ExecDeploy(event, dockerManager, r, tree); err != nil {
			setEventFailStatus(event, err.Error())
			return
		}

		err = worker.sendEvent(*event)
		if err != nil {
			logdog.Errorf("set event result err: %v", err)
		}

		// Deploy Check
		helper.ExecDeployCheck(event, tree)

	}
	event.Status = api.EventStatusSuccess
}

// sendEvent used for setting event for circe server
func (worker *Worker) sendEvent(event api.Event) error {
	eventID := worker.Config.ID
	serverHost := worker.Envs.CycloneServer

	BaseURL := fmt.Sprintf("%s/api/%s", serverHost, api.APIVersion)
	httpHandler := handler.NewHTTPHandler(BaseURL)
	result := &api.SetEvent{
		Event: event,
	}
	var setResponse api.SetEventResponse
	var err error
	DueTime := time.Now().Add(time.Duration(WorkerTimeout))
	for DueTime.After(time.Now()) == true {
		err = httpHandler.SetEvent(eventID, result, &setResponse)
		if err != nil {
			if strings.Contains(err.Error(), "connection refused") {
				time.Sleep(time.Minute)
				continue
			}
			return err
		}
		logdog.Infof("set event result: %v", setResponse)
		return nil
	}
	return err
}

func isChanClosed(ch chan interface{}) bool {
	if len(ch) == 0 {
		select {
		case _, ok := <-ch:
			return !ok
		}
	}
	return false
}

// setEventFailStatus sets the fail status of the event.
func setEventFailStatus(event *api.Event, ErrorMessage string) {
	event.Status = api.EventStatusFail
	event.ErrorMessage = ErrorMessage
	logdog.Error("Operation failed", logdog.Fields{"event": event})
}
<|endoftext|>