<commit_before>// Copyright 2017 Apcera Inc. All rights reserved.

package server

import (
	"encoding/binary"
	"fmt"
	"io"
	"sync"

	"github.com/hashicorp/raft"
	"github.com/nats-io/go-nats-streaming/pb"

	"github.com/nats-io/nats-streaming-server/spb"
)

const snapshotBatchSize = 200

var (
	fragmentPool     = &sync.Pool{New: func() interface{} { return &spb.RaftSnapshotFragment{} }}
	batchPool        = &sync.Pool{New: func() interface{} { return &spb.Batch{} }}
	msgsSnapshotPool = &sync.Pool{New: func() interface{} { return &spb.MessagesSnapshot{} }}
	subSnapshotPool  = &sync.Pool{New: func() interface{} { return &spb.SubSnapshot{} }}
)

// channelSnapshot implements the raft.FSMSnapshot interface.
type channelSnapshot struct {
	*channel
}

func newChannelSnapshot(c *channel) raft.FSMSnapshot {
	return &channelSnapshot{channel: c}
}

// Persist should dump all necessary state to the WriteCloser 'sink',
// and call sink.Close() when finished or call sink.Cancel() on error.
func (c *channelSnapshot) Persist(sink raft.SnapshotSink) (err error) {
	defer func() {
		if err != nil {
			sink.Cancel()
		}
	}()

	if err := c.snapshotMessages(sink); err != nil {
		return err
	}

	if err := c.snapshotSubscriptions(sink); err != nil {
		return err
	}

	return sink.Close()
}

// Release is a no-op.
func (c *channelSnapshot) Release() {}

func (c *channelSnapshot) snapshotMessages(sink raft.SnapshotSink) error {
	// TODO: this is very expensive and might repeatedly fail if messages are
	// constantly being truncated. Is there a way we can optimize this, e.g.
	// handling Restore() out-of-band from Raft?
	// See issue #410.

	first, last, err := c.store.Msgs.FirstAndLastSequence()
	if err != nil {
		return err
	}

	var (
		buf              [4]byte
		batch            = batchPool.Get().(*spb.Batch)
		writeMsgFragment = func(b *spb.Batch) error {
			fragment := newFragment()
			fragment.FragmentType = spb.RaftSnapshotFragment_Messages
			msgsSnap := msgsSnapshotPool.Get().(*spb.MessagesSnapshot)
			msgsSnap.MessageBatch = b
			fragment.Msgs = msgsSnap
			return writeFragment(sink, fragment, buf)
		}
	)
	batch.Messages = make([]*pb.MsgProto, 0, snapshotBatchSize)

	for seq := first; seq <= last; seq++ {
		msg, err := c.store.Msgs.Lookup(seq)
		if err != nil {
			batchPool.Put(batch)
			return err
		}
		// If msg is nil, channel truncation has occurred while snapshotting.
		if msg == nil {
			// Channel truncation has occurred while snapshotting.
			batchPool.Put(batch)
			return fmt.Errorf("channel %q was truncated while snapshotting", c.name)
		}

		// Previous batch is full, ship it.
		if len(batch.Messages) == snapshotBatchSize {
			if err := writeMsgFragment(batch); err != nil {
				return err
			}

			// Create a new batch.
			batch = batchPool.Get().(*spb.Batch)
			batch.Messages = make([]*pb.MsgProto, 0, snapshotBatchSize)
		}

		batch.Messages = append(batch.Messages, msg)
	}

	// Ship any partial batch.
	if len(batch.Messages) > 0 {
		if err := writeMsgFragment(batch); err != nil {
			return err
		}
	}

	return nil
}

func (c *channelSnapshot) snapshotSubscriptions(sink raft.SnapshotSink) error {
	var buf [4]byte
	for _, sub := range c.ss.getAllSubs() {
		fragment := newFragment()
		fragment.FragmentType = spb.RaftSnapshotFragment_Subscription
		subSnap := subSnapshotPool.Get().(*spb.SubSnapshot)
		sub.Lock()
		subSnap.Subject = sub.subject
		subSnap.State = &sub.SubState
		subSnap.AcksPending = make([]uint64, len(sub.acksPending))
		i := 0
		for seq, _ := range sub.acksPending {
			subSnap.AcksPending[i] = seq
			i++
		}
		sub.Unlock()
		fragment.Sub = subSnap
		if err := writeFragment(sink, fragment, buf); err != nil {
			return err
		}
	}
	return nil
}

// writeFragment writes the marshaled RaftSnapshotFragment to the
// SnapshotSink. The fragment should not be used after this is called.
func writeFragment(sink raft.SnapshotSink, fragment *spb.RaftSnapshotFragment, sizeBuf [4]byte) error {
	data, err := fragment.Marshal()
	if fragment.Msgs != nil {
		batchPool.Put(fragment.Msgs.MessageBatch)
		msgsSnapshotPool.Put(fragment.Msgs)
	}
	if fragment.Sub != nil {
		subSnapshotPool.Put(fragment.Sub)
	}
	fragmentPool.Put(fragment)
	if err != nil {
		return err
	}
	binary.BigEndian.PutUint32(sizeBuf[:], uint32(len(data)))
	_, err = sink.Write(sizeBuf[:])
	if err != nil {
		return err
	}
	_, err = sink.Write(data)
	return err
}

func newFragment() *spb.RaftSnapshotFragment {
	fragment := fragmentPool.Get().(*spb.RaftSnapshotFragment)
	fragment.Msgs = nil
	return fragment
}

// restoreFromSnapshot restores a channel from a snapshot. This is not called
// concurrently with any other Raft commands.
func (c *channel) restoreFromSnapshot(snapshot io.ReadCloser) error {
	defer snapshot.Close()

	// Drop all existing subs. These will be restored from the snapshot.
	for _, sub := range c.ss.getAllSubs() {
		c.stan.unsubscribe(c, sub.ClientID, sub, true)
	}

	sizeBuf := make([]byte, 4)
	for {
		// Read the fragment size.
		if _, err := io.ReadFull(snapshot, sizeBuf); err != nil {
			if err == io.EOF {
				break
			}
			return err
		}

		// Read the fragment.
		size := binary.BigEndian.Uint32(sizeBuf)
		buf := make([]byte, size)
		if _, err := io.ReadFull(snapshot, buf); err != nil {
			return err
		}

		// Unmarshal the fragment.
		fragment := newFragment()
		if err := fragment.Unmarshal(buf); err != nil {
			return err
		}

		// Apply the fragment.
		switch fragment.FragmentType {
		case spb.RaftSnapshotFragment_Messages:
			// Channel messages.
			err := c.restoreMsgsFromSnapshot(fragment.Msgs)
			batchPool.Put(fragment.Msgs.MessageBatch)
			msgsSnapshotPool.Put(fragment.Msgs)
			fragmentPool.Put(fragment)
			if err != nil {
				return err
			}
		case spb.RaftSnapshotFragment_Subscription:
			// Channel subscription.
			err := c.restoreSubFromSnapshot(fragment.Sub)
			subSnapshotPool.Put(fragment.Sub)
			fragmentPool.Put(fragment)
			if err != nil {
				return err
			}
		default:
			panic(fmt.Sprintf("unknown snapshot fragment type %s", fragment.FragmentType))
		}
	}
	return c.store.Msgs.Flush()
}

func (c *channel) restoreMsgsFromSnapshot(msgsSnapshot *spb.MessagesSnapshot) error {
	for _, msg := range msgsSnapshot.MessageBatch.Messages {
		if _, err := c.store.Msgs.Store(msg); err != nil {
			return err
		}
	}
	return nil
}

func (c *channel) restoreSubFromSnapshot(subSnapshot *spb.SubSnapshot) error {
	acksPending := make(map[uint64]int64, len(subSnapshot.AcksPending))
	for _, seq := range subSnapshot.AcksPending {
		// Set 0 for expiration time. This will be computed
		// when the follower becomes leader and attempts to
		// redeliver messages.
		acksPending[seq] = 0
	}
	sub := &subState{
		SubState:    *subSnapshot.State,
		subject:     subSnapshot.Subject,
		ackWait:     computeAckWait(subSnapshot.State.AckWaitInSecs),
		acksPending: acksPending,
		store:       c.store.Subs,
	}
	// Store the subscription.
	if err := c.stan.addSubscription(c.ss, sub); err != nil {
		return err
	}
	// Store pending acks for the subscription.
	for seq, _ := range sub.acksPending {
		if err := sub.store.AddSeqPending(sub.ID, seq); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>Nil out sub when creating new snapshot fragment<commit_after>// Copyright 2017 Apcera Inc. All rights reserved.

package server

import (
	"encoding/binary"
	"fmt"
	"io"
	"sync"

	"github.com/hashicorp/raft"
	"github.com/nats-io/go-nats-streaming/pb"

	"github.com/nats-io/nats-streaming-server/spb"
)

const snapshotBatchSize = 200

var (
	fragmentPool     = &sync.Pool{New: func() interface{} { return &spb.RaftSnapshotFragment{} }}
	batchPool        = &sync.Pool{New: func() interface{} { return &spb.Batch{} }}
	msgsSnapshotPool = &sync.Pool{New: func() interface{} { return &spb.MessagesSnapshot{} }}
	subSnapshotPool  = &sync.Pool{New: func() interface{} { return &spb.SubSnapshot{} }}
)

// channelSnapshot implements the raft.FSMSnapshot interface.
type channelSnapshot struct {
	*channel
}

func newChannelSnapshot(c *channel) raft.FSMSnapshot {
	return &channelSnapshot{channel: c}
}

// Persist should dump all necessary state to the WriteCloser 'sink',
// and call sink.Close() when finished or call sink.Cancel() on error.
func (c *channelSnapshot) Persist(sink raft.SnapshotSink) (err error) {
	defer func() {
		if err != nil {
			sink.Cancel()
		}
	}()

	if err := c.snapshotMessages(sink); err != nil {
		return err
	}

	if err := c.snapshotSubscriptions(sink); err != nil {
		return err
	}

	return sink.Close()
}

// Release is a no-op.
func (c *channelSnapshot) Release() {}

func (c *channelSnapshot) snapshotMessages(sink raft.SnapshotSink) error {
	// TODO: this is very expensive and might repeatedly fail if messages are
	// constantly being truncated. Is there a way we can optimize this, e.g.
	// handling Restore() out-of-band from Raft?
	// See issue #410.

	first, last, err := c.store.Msgs.FirstAndLastSequence()
	if err != nil {
		return err
	}

	var (
		buf              [4]byte
		batch            = batchPool.Get().(*spb.Batch)
		writeMsgFragment = func(b *spb.Batch) error {
			fragment := newFragment()
			fragment.FragmentType = spb.RaftSnapshotFragment_Messages
			msgsSnap := msgsSnapshotPool.Get().(*spb.MessagesSnapshot)
			msgsSnap.MessageBatch = b
			fragment.Msgs = msgsSnap
			return writeFragment(sink, fragment, buf)
		}
	)
	batch.Messages = make([]*pb.MsgProto, 0, snapshotBatchSize)

	for seq := first; seq <= last; seq++ {
		msg, err := c.store.Msgs.Lookup(seq)
		if err != nil {
			batchPool.Put(batch)
			return err
		}
		// If msg is nil, channel truncation has occurred while snapshotting.
		if msg == nil {
			// Channel truncation has occurred while snapshotting.
			batchPool.Put(batch)
			return fmt.Errorf("channel %q was truncated while snapshotting", c.name)
		}

		// Previous batch is full, ship it.
		if len(batch.Messages) == snapshotBatchSize {
			if err := writeMsgFragment(batch); err != nil {
				return err
			}

			// Create a new batch.
			batch = batchPool.Get().(*spb.Batch)
			batch.Messages = make([]*pb.MsgProto, 0, snapshotBatchSize)
		}

		batch.Messages = append(batch.Messages, msg)
	}

	// Ship any partial batch.
	if len(batch.Messages) > 0 {
		if err := writeMsgFragment(batch); err != nil {
			return err
		}
	}

	return nil
}

func (c *channelSnapshot) snapshotSubscriptions(sink raft.SnapshotSink) error {
	var buf [4]byte
	for _, sub := range c.ss.getAllSubs() {
		fragment := newFragment()
		fragment.FragmentType = spb.RaftSnapshotFragment_Subscription
		subSnap := subSnapshotPool.Get().(*spb.SubSnapshot)
		sub.Lock()
		subSnap.Subject = sub.subject
		subSnap.State = &sub.SubState
		subSnap.AcksPending = make([]uint64, len(sub.acksPending))
		i := 0
		for seq, _ := range sub.acksPending {
			subSnap.AcksPending[i] = seq
			i++
		}
		sub.Unlock()
		fragment.Sub = subSnap
		if err := writeFragment(sink, fragment, buf); err != nil {
			return err
		}
	}
	return nil
}

// writeFragment writes the marshaled RaftSnapshotFragment to the
// SnapshotSink. The fragment should not be used after this is called.
func writeFragment(sink raft.SnapshotSink, fragment *spb.RaftSnapshotFragment, sizeBuf [4]byte) error {
	data, err := fragment.Marshal()
	if fragment.Msgs != nil {
		batchPool.Put(fragment.Msgs.MessageBatch)
		msgsSnapshotPool.Put(fragment.Msgs)
	}
	if fragment.Sub != nil {
		subSnapshotPool.Put(fragment.Sub)
	}
	fragmentPool.Put(fragment)
	if err != nil {
		return err
	}
	binary.BigEndian.PutUint32(sizeBuf[:], uint32(len(data)))
	_, err = sink.Write(sizeBuf[:])
	if err != nil {
		return err
	}
	_, err = sink.Write(data)
	return err
}

func newFragment() *spb.RaftSnapshotFragment {
	fragment := fragmentPool.Get().(*spb.RaftSnapshotFragment)
	fragment.Msgs = nil
	fragment.Sub = nil
	return fragment
}

// restoreFromSnapshot restores a channel from a snapshot. This is not called
// concurrently with any other Raft commands.
func (c *channel) restoreFromSnapshot(snapshot io.ReadCloser) error {
	defer snapshot.Close()

	// Drop all existing subs. These will be restored from the snapshot.
	for _, sub := range c.ss.getAllSubs() {
		c.stan.unsubscribe(c, sub.ClientID, sub, true)
	}

	sizeBuf := make([]byte, 4)
	for {
		// Read the fragment size.
		if _, err := io.ReadFull(snapshot, sizeBuf); err != nil {
			if err == io.EOF {
				break
			}
			return err
		}

		// Read the fragment.
		size := binary.BigEndian.Uint32(sizeBuf)
		buf := make([]byte, size)
		if _, err := io.ReadFull(snapshot, buf); err != nil {
			return err
		}

		// Unmarshal the fragment.
		fragment := newFragment()
		if err := fragment.Unmarshal(buf); err != nil {
			return err
		}

		// Apply the fragment.
		switch fragment.FragmentType {
		case spb.RaftSnapshotFragment_Messages:
			// Channel messages.
			err := c.restoreMsgsFromSnapshot(fragment.Msgs)
			batchPool.Put(fragment.Msgs.MessageBatch)
			msgsSnapshotPool.Put(fragment.Msgs)
			fragmentPool.Put(fragment)
			if err != nil {
				return err
			}
		case spb.RaftSnapshotFragment_Subscription:
			// Channel subscription.
			err := c.restoreSubFromSnapshot(fragment.Sub)
			subSnapshotPool.Put(fragment.Sub)
			fragmentPool.Put(fragment)
			if err != nil {
				return err
			}
		default:
			panic(fmt.Sprintf("unknown snapshot fragment type %s", fragment.FragmentType))
		}
	}
	return c.store.Msgs.Flush()
}

func (c *channel) restoreMsgsFromSnapshot(msgsSnapshot *spb.MessagesSnapshot) error {
	for _, msg := range msgsSnapshot.MessageBatch.Messages {
		if _, err := c.store.Msgs.Store(msg); err != nil {
			return err
		}
	}
	return nil
}

func (c *channel) restoreSubFromSnapshot(subSnapshot *spb.SubSnapshot) error {
	acksPending := make(map[uint64]int64, len(subSnapshot.AcksPending))
	for _, seq := range subSnapshot.AcksPending {
		// Set 0 for expiration time. This will be computed
		// when the follower becomes leader and attempts to
		// redeliver messages.
		acksPending[seq] = 0
	}
	sub := &subState{
		SubState:    *subSnapshot.State,
		subject:     subSnapshot.Subject,
		ackWait:     computeAckWait(subSnapshot.State.AckWaitInSecs),
		acksPending: acksPending,
		store:       c.store.Subs,
	}
	// Store the subscription.
	if err := c.stan.addSubscription(c.ss, sub); err != nil {
		return err
	}
	// Store pending acks for the subscription.
	for seq, _ := range sub.acksPending {
		if err := sub.store.AddSeqPending(sub.ID, seq); err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package service

/*
The service.go defines what to do for each API-call. This part of the service
runs on the node.
*/

import (
	"time"

	"errors"
	"sync"

	"github.com/dedis/cothority_template"
	"github.com/dedis/cothority_template/protocol"
	"gopkg.in/dedis/onet.v1"
	"gopkg.in/dedis/onet.v1/log"
	"gopkg.in/dedis/onet.v1/network"
)

// Used for tests
var templateID onet.ServiceID

func init() {
	var err error
	templateID, err = onet.RegisterNewService(template.ServiceName, newService)
	log.ErrFatal(err)
	network.RegisterMessage(&storage{})
}

// Service is our template-service
type Service struct {
	// We need to embed the ServiceProcessor, so that incoming messages
	// are correctly handled.
	*onet.ServiceProcessor
	path string

	storage *storage
}

// storageID reflects the data we're storing - we could store more
// than one structure.
const storageID = "main"

// storage is used to save our data.
type storage struct {
	Count int
	sync.Mutex
}

// ClockRequest starts a template-protocol and returns the run-time.
func (s *Service) ClockRequest(req *template.ClockRequest) (*template.ClockResponse, onet.ClientError) {
	s.storage.Lock()
	s.storage.Count++
	s.storage.Unlock()
	s.save()
	tree := req.Roster.GenerateNaryTreeWithRoot(2, s.ServerIdentity())
	pi, err := s.CreateProtocol(protocol.Name, tree)
	if err != nil {
		return nil, onet.NewClientError(err)
	}
	start := time.Now()
	pi.Start()
	resp := &template.ClockResponse{
		Children: <-pi.(*protocol.Template).ChildCount,
	}
	resp.Time = time.Now().Sub(start).Seconds()
	return resp, nil
}

// CountRequest returns the number of instantiations of the protocol.
func (s *Service) CountRequest(req *template.CountRequest) (*template.CountResponse, onet.ClientError) {
	s.storage.Lock()
	defer s.storage.Unlock()
	return &template.CountResponse{Count: s.storage.Count}, nil
}

// NewProtocol is called on all nodes of a Tree (except the root, since it is
// the one starting the protocol) so it's the Service that will be called to
// generate the PI on all others node.
// If you use CreateProtocolOnet, this will not be called, as the Onet will
// instantiate the protocol on its own. If you need more control at the
// instantiation of the protocol, use CreateProtocolService, and you can
// give some extra-configuration to your protocol in here.
func (s *Service) NewProtocol(tn *onet.TreeNodeInstance, conf *onet.GenericConfig) (onet.ProtocolInstance, error) {
	log.Lvl3("Not templated yet")
	return nil, nil
}

// saves all skipblocks.
func (s *Service) save() {
	s.storage.Lock()
	defer s.storage.Unlock()
	err := s.Save(storageID, s.storage)
	if err != nil {
		log.Error("Couldn't save file:", err)
	}
}

// Tries to load the configuration and updates the data in the service
// if it finds a valid config-file.
func (s *Service) tryLoad() error {
	s.storage = &storage{}
	if !s.DataAvailable(storageID) {
		return nil
	}
	msg, err := s.Load(storageID)
	if err != nil {
		return err
	}
	var ok bool
	s.storage, ok = msg.(*storage)
	if !ok {
		return errors.New("Data of wrong type")
	}
	return nil
}

// newService receives the context and a path where it can write its
// configuration, if desired. As we don't know when the service will exit,
// we need to save the configuration on our own from time to time.
func newService(c *onet.Context) onet.Service {
	s := &Service{
		ServiceProcessor: onet.NewServiceProcessor(c),
	}
	if err := s.RegisterHandlers(s.ClockRequest, s.CountRequest); err != nil {
		log.ErrFatal(err, "Couldn't register messages")
	}
	if err := s.tryLoad(); err != nil {
		log.Error(err)
	}
	return s
}
<commit_msg>Update service.go (#21)<commit_after>package service

/*
The service.go defines what to do for each API-call. This part of the service
runs on the node.
*/

import (
	"time"

	"errors"
	"sync"

	"github.com/dedis/cothority_template"
	"github.com/dedis/cothority_template/protocol"
	"gopkg.in/dedis/onet.v1"
	"gopkg.in/dedis/onet.v1/log"
	"gopkg.in/dedis/onet.v1/network"
)

// Used for tests
var templateID onet.ServiceID

func init() {
	var err error
	templateID, err = onet.RegisterNewService(template.ServiceName, newService)
	log.ErrFatal(err)
	network.RegisterMessage(&storage{})
}

// Service is our template-service
type Service struct {
	// We need to embed the ServiceProcessor, so that incoming messages
	// are correctly handled.
	*onet.ServiceProcessor

	storage *storage
}

// storageID reflects the data we're storing - we could store more
// than one structure.
const storageID = "main"

// storage is used to save our data.
type storage struct {
	Count int
	sync.Mutex
}

// ClockRequest starts a template-protocol and returns the run-time.
func (s *Service) ClockRequest(req *template.ClockRequest) (*template.ClockResponse, onet.ClientError) {
	s.storage.Lock()
	s.storage.Count++
	s.storage.Unlock()
	s.save()
	tree := req.Roster.GenerateNaryTreeWithRoot(2, s.ServerIdentity())
	pi, err := s.CreateProtocol(protocol.Name, tree)
	if err != nil {
		return nil, onet.NewClientError(err)
	}
	start := time.Now()
	pi.Start()
	resp := &template.ClockResponse{
		Children: <-pi.(*protocol.Template).ChildCount,
	}
	resp.Time = time.Now().Sub(start).Seconds()
	return resp, nil
}

// CountRequest returns the number of instantiations of the protocol.
func (s *Service) CountRequest(req *template.CountRequest) (*template.CountResponse, onet.ClientError) {
	s.storage.Lock()
	defer s.storage.Unlock()
	return &template.CountResponse{Count: s.storage.Count}, nil
}

// NewProtocol is called on all nodes of a Tree (except the root, since it is
// the one starting the protocol) so it's the Service that will be called to
// generate the PI on all others node.
// If you use CreateProtocolOnet, this will not be called, as the Onet will
// instantiate the protocol on its own. If you need more control at the
// instantiation of the protocol, use CreateProtocolService, and you can
// give some extra-configuration to your protocol in here.
func (s *Service) NewProtocol(tn *onet.TreeNodeInstance, conf *onet.GenericConfig) (onet.ProtocolInstance, error) {
	log.Lvl3("Not templated yet")
	return nil, nil
}

// saves all skipblocks.
func (s *Service) save() {
	s.storage.Lock()
	defer s.storage.Unlock()
	err := s.Save(storageID, s.storage)
	if err != nil {
		log.Error("Couldn't save file:", err)
	}
}

// Tries to load the configuration and updates the data in the service
// if it finds a valid config-file.
func (s *Service) tryLoad() error {
	s.storage = &storage{}
	if !s.DataAvailable(storageID) {
		return nil
	}
	msg, err := s.Load(storageID)
	if err != nil {
		return err
	}
	var ok bool
	s.storage, ok = msg.(*storage)
	if !ok {
		return errors.New("Data of wrong type")
	}
	return nil
}

// newService receives the context that holds information about the node it's
// running on. Saving and loading can be done using the context. The data will
// be stored in memory for tests and simulations, and on disk for real deployments.
func newService(c *onet.Context) onet.Service {
	s := &Service{
		ServiceProcessor: onet.NewServiceProcessor(c),
	}
	if err := s.RegisterHandlers(s.ClockRequest, s.CountRequest); err != nil {
		log.ErrFatal(err, "Couldn't register messages")
	}
	if err := s.tryLoad(); err != nil {
		log.Error(err)
	}
	return s
}
<|endoftext|>


<commit_before>package sessions

import (
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/kataras/iris/context"
)

var (
	// CookieExpireDelete may be set on Cookie.Expire for expiring the given cookie.
	CookieExpireDelete = time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC)

	// CookieExpireUnlimited indicates that the cookie doesn't expire.
	CookieExpireUnlimited = time.Now().AddDate(24, 10, 10)
)

// GetCookie returns cookie's value by it's name
// returns empty string if nothing was found
func GetCookie(ctx context.Context, name string) string {
	c, err := ctx.Request().Cookie(name)
	if err != nil {
		return ""
	}

	return c.Value

	// return ctx.GetCookie(name)
}

// AddCookie adds a cookie
func AddCookie(ctx context.Context, cookie *http.Cookie, reclaim bool) {
	// http.SetCookie(ctx.ResponseWriter(), cookie)
	if reclaim {
		ctx.Request().AddCookie(cookie)
	}
	ctx.SetCookie(cookie)
}

// RemoveCookie deletes a cookie by it's name/key
// If "purge" is true then it removes the, temp, cookie from the request as well.
func RemoveCookie(ctx context.Context, name string, purge bool) {
	c, err := ctx.Request().Cookie(name)
	if err != nil {
		return
	}

	c.Expires = CookieExpireDelete
	// MaxAge<0 means delete cookie now, equivalently 'Max-Age: 0'
	c.MaxAge = -1
	c.Value = ""
	c.Path = "/"
	AddCookie(ctx, c, purge)
}

// IsValidCookieDomain returns true if the receiver is a valid domain to set
// valid means that is recognised as 'domain' by the browser, so it(the cookie) can be shared with subdomains also
func IsValidCookieDomain(domain string) bool {
	if domain == "0.0.0.0" || domain == "127.0.0.1" {
		// for these type of hosts, we can't allow subdomains persistence,
		// the web browser doesn't understand the mysubdomain.0.0.0.0 and mysubdomain.127.0.0.1 mysubdomain.32.196.56.181. as scorrectly ubdomains because of the many dots
		// so don't set a cookie domain here, let browser handle this
		return false
	}

	dotLen := strings.Count(domain, ".")
	if dotLen == 0 {
		// we don't have a domain, maybe something like 'localhost', browser doesn't see the .localhost as wildcard subdomain+domain
		return false
	}
	if dotLen >= 3 {
		if lastDotIdx := strings.LastIndexByte(domain, '.'); lastDotIdx != -1 {
			// chekc the last part, if it's number then propably it's ip
			if len(domain) > lastDotIdx+1 {
				_, err := strconv.Atoi(domain[lastDotIdx+1:])
				if err == nil {
					return false
				}
			}
		}
	}

	return true
}
<commit_msg>need a better way to replace an existed cookie<commit_after>package sessions

import (
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/kataras/iris/context"
)

var (
	// CookieExpireDelete may be set on Cookie.Expire for expiring the given cookie.
	CookieExpireDelete = time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC)

	// CookieExpireUnlimited indicates that the cookie doesn't expire.
	CookieExpireUnlimited = time.Now().AddDate(24, 10, 10)
)

// GetCookie returns cookie's value by it's name
// returns empty string if nothing was found
func GetCookie(ctx context.Context, name string) string {
	c, err := ctx.Request().Cookie(name)
	if err != nil {
		return ""
	}

	return c.Value

	// return ctx.GetCookie(name)
}

// AddCookie adds a cookie
func AddCookie(ctx context.Context, cookie *http.Cookie, reclaim bool) {
	// http.SetCookie(ctx.ResponseWriter(), cookie)
	if reclaim {
		ctx.Request().AddCookie(cookie)
	}
	ctx.SetCookie(cookie)
}

// RemoveCookie deletes a cookie by it's name/key
// If "purge" is true then it removes the, temp, cookie from the request as well.
func RemoveCookie(ctx context.Context, name string, purge bool) {
	c, err := ctx.Request().Cookie(name)
	if err != nil {
		return
	}

	c.Expires = CookieExpireDelete
	// MaxAge<0 means delete cookie now, equivalently 'Max-Age: 0'
	c.MaxAge = -1
	c.Value = ""
	c.Path = "/"
	AddCookie(ctx, c, purge)

	if purge {
		// delete request's cookie also, which is temporary available.
		ctx.Request().Header.Set("Cookie", "")
	}
}

// IsValidCookieDomain returns true if the receiver is a valid domain to set
// valid means that is recognised as 'domain' by the browser, so it(the cookie) can be shared with subdomains also
func IsValidCookieDomain(domain string) bool {
	if domain == "0.0.0.0" || domain == "127.0.0.1" {
		// for these type of hosts, we can't allow subdomains persistence,
		// the web browser doesn't understand the mysubdomain.0.0.0.0 and mysubdomain.127.0.0.1 mysubdomain.32.196.56.181. as scorrectly ubdomains because of the many dots
		// so don't set a cookie domain here, let browser handle this
		return false
	}

	dotLen := strings.Count(domain, ".")
	if dotLen == 0 {
		// we don't have a domain, maybe something like 'localhost', browser doesn't see the .localhost as wildcard subdomain+domain
		return false
	}
	if dotLen >= 3 {
		if lastDotIdx := strings.LastIndexByte(domain, '.'); lastDotIdx != -1 {
			// chekc the last part, if it's number then propably it's ip
			if len(domain) > lastDotIdx+1 {
				_, err := strconv.Atoi(domain[lastDotIdx+1:])
				if err == nil {
					return false
				}
			}
		}
	}

	return true
}
<|endoftext|>


<commit_before>package repository

import (
	"bytes"
	"crypto/rand"
	"crypto/rsa"
	"fmt"
	"testing"
)

var (
	testAes = []byte("AESKEY256-32-Character1234567890")
	testIv  = []byte("1234567890123456")
	testKey = generateTestKey(1024)
)

func generateTestKey(bits int) *rsa.PrivateKey {
	key, err := rsa.GenerateKey(rand.Reader, bits)
	if err != nil {
		panic(fmt.Sprintf("Failed to generate random key with %d bits", bits))
	}

	return key
}

func TestKeylength(t *testing.T) {
	repo := &Repository{privateKey: testKey}
	if repo.keylength() != 128 {
		t.Errorf("Expected key length of 128 but got %d", repo.keylength())
	}

	repo = &Repository{publicKey: &testKey.PublicKey}
	if repo.keylength() != 128 {
		t.Errorf("Expected key length of 128 but got %d", repo.keylength())
	}
}

func TestWriteHeader(t *testing.T) {
	repo := &WriteRepository{}
	repo.AesKey = testAes
	repo.iv = testIv
	repo.publicKey = &testKey.PublicKey

	buffer := new(bytes.Buffer)
	repo.writeHeader(buffer)

	if len(buffer.Bytes()) != repo.keylength() {
		t.Errorf("Expected %d bytes but got %d", repo.keylength(), len(buffer.Bytes()))
	}
}

func TestWriteSignature(t *testing.T) {
	repo := &WriteRepository{}
	repo.AesKey = testAes
	repo.iv = testIv
	repo.privateKey = testKey
	repo.publicKey = &testKey.PublicKey

	buffer := new(bytes.Buffer)
	repo.writeSignature(buffer)

	if len(buffer.Bytes()) != repo.keylength() {
		t.Errorf("Expected %d bytes but got %d", repo.keylength(), len(buffer.Bytes()))
	}

}
<commit_msg>Adding test while deciding next ticket<commit_after>package repository

import (
	"bytes"
	"crypto/rand"
	"crypto/rsa"
	"fmt"
	"log"
	"testing"
)

var (
	testAes = []byte("AESKEY256-32-Character1234567890")
	testIv  = []byte("1234567890123456")
	testKey = generateTestKey(1024)
)

func generateTestKey(bits int) *rsa.PrivateKey {
	key, err := rsa.GenerateKey(rand.Reader, bits)
	if err != nil {
		panic(fmt.Sprintf("Failed to generate random key with %d bits", bits))
	}

	return key
}

func TestKeylength(t *testing.T) {
	repo := &Repository{privateKey: testKey}
	if repo.keylength() != 128 {
		t.Errorf("Expected key length of 128 but got %d", repo.keylength())
	}

	repo = &Repository{publicKey: &testKey.PublicKey}
	if repo.keylength() != 128 {
		t.Errorf("Expected key length of 128 but got %d", repo.keylength())
	}
}

func TestWriteHeader(t *testing.T) {
	repo := &WriteRepository{}
	repo.AesKey = testAes
	repo.iv = testIv
	repo.publicKey = &testKey.PublicKey

	buffer := new(bytes.Buffer)
	repo.writeHeader(buffer)

	if len(buffer.Bytes()) != repo.keylength() {
		t.Errorf("Expected %d bytes but got %d", repo.keylength(), len(buffer.Bytes()))
	}
}

func TestWriteSignature(t *testing.T) {
	repo := &WriteRepository{}
	repo.AesKey = testAes
	repo.iv = testIv
	repo.privateKey = testKey
	repo.publicKey = &testKey.PublicKey

	buffer := new(bytes.Buffer)
	repo.writeSignature(buffer)

	if len(buffer.Bytes()) != repo.keylength() {
		t.Errorf("Expected %d bytes but got %d", repo.keylength(), len(buffer.Bytes()))
	}

}

func TestCreateRepository(t *testing.T) {
	buffer := new(bytes.Buffer)
	_, err := CreateRepository(&testKey.PublicKey, testKey, buffer)
	if err != nil {
		log.Fatalf("Unable to write out repository: %v", err)
	}

	reader := bytes.NewReader(buffer.Bytes())
	_, err = OpenRepository(testKey, &testKey.PublicKey, reader)
	if err != nil {
		log.Fatalf("Failed to open repository: %v", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"time"
	"net/http"
	"log"
	"fmt"
	"strconv"
	"encoding/json"
	"math"

	"github.com/tarm/serial"
)

type PixelServer struct {
	Serial        *serial.Port
	LastPixelData PixelData
}

type PixelData struct {
	Value      int
	Message    string
	Blink      int
	Brightness int
}

func (ps PixelServer) statusHandler(w http.ResponseWriter, r *http.Request) {
	if(r.Method != "POST") {
		http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
		return
	}
	r.ParseForm()

	var pd PixelData

	pd.Value, _ = strconv.Atoi(r.FormValue("value"))
	pd.Message = r.FormValue("message")
	pd.Blink, _ = strconv.Atoi(r.FormValue("blink"))
	pd.Brightness, _ = strconv.Atoi(r.FormValue("brightness"))

	pixelServer.setStatus(pd)
}

func (ps PixelServer) kapacitorHandler(w http.ResponseWriter, r *http.Request) {
	if(r.Method != "POST") {
		http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
		return
	}
	ad := KapacitorAlertData{}
	decoder := json.NewDecoder(r.Body)
	err := decoder.Decode(&ad)
	if err != nil {
		log.Fatalf("Could not decode kapacitor AlertData, %s", err)
	}

	var pd PixelData
	pd.Brightness = 100

	switch ad.Level {
	case OKAlert:
		pd.Value = 100
		//pd.Blink = 1
	case InfoAlert:
		pd.Value = -1
	case WarnAlert:
		pd.Value = 50
	case CritAlert:
		pd.Value = 1
		pd.Blink = 2
	}

	data := ad.Data.Series[0]
	pd.Message = fmt.Sprintf("%s\\%s: %v", data.Tags.Host, data.Name, data.Values[0][1]) // data.Values[1]

	pixelServer.setStatus(pd)
}

func (ps *PixelServer) setStatus (pd PixelData){
	animationDuration := 1000 // ms

	switch pd.Blink {
	case 1:
		pd.Value += 100
	case 2:
		pd.Value += 200
	}

	delta := pd.Value - ps.LastPixelData.Value
	stepTime := float64(animationDuration) / math.Abs(float64(delta))

	var step int
	if delta > 0{
		step = 1
	} else {
		step = -1
	}

	if(pd.Blink == 0 && ps.LastPixelData.Value > 0 && pd.Value > 0) {
		for i := ps.LastPixelData.Value; i != pd.Value; i += step {
			ps.sendSerial(PixelData{i, "", 0, pd.Brightness })
			time.Sleep(time.Millisecond * time.Duration(stepTime))
		}
	}

	time.Sleep(100 * time.Duration(stepTime))
	log.Printf("setStatus: %v\n", pd)
	ps.sendSerial(pd)

	// if success value, turn off led
	if pd.Value == 100{
		time.Sleep(time.Millisecond * 5000)
		ps.sendSerial(PixelData{ -1, "", 0, 100 })
	}

	time.Sleep(1000 * time.Millisecond)
	ps.LastPixelData = pd
}

func (ps PixelServer) sendSerial (pd PixelData) (int, error){
	command := fmt.Sprintf("%d|%s|%d\n",pd.Value, pd.Message, pd.Brightness)
	//fmt.Println(command)
	n, err := ps.Serial.Write([]byte(command))
	if err != nil {
		log.Fatalf("Could not write to port, %s", err)
	}
	return n, err
}
<commit_msg>fix(serial): change animations, remove blink on kapacitor critical<commit_after>package main

import (
	"time"
	"net/http"
	"log"
	"fmt"
	"strconv"
	"encoding/json"
	"math"

	"github.com/tarm/serial"
)

type PixelServer struct {
	Serial        *serial.Port
	LastPixelData PixelData
}

type PixelData struct {
	Value      int
	Message    string
	Blink      int
	Brightness int
}

func (ps PixelServer) statusHandler(w http.ResponseWriter, r *http.Request) {
	if(r.Method != "POST") {
		http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
		return
	}
	r.ParseForm()

	var pd PixelData

	pd.Value, _ = strconv.Atoi(r.FormValue("value"))
	pd.Message = r.FormValue("message")
	pd.Blink, _ = strconv.Atoi(r.FormValue("blink"))
	pd.Brightness, _ = strconv.Atoi(r.FormValue("brightness"))

	pixelServer.setStatus(pd)
}

func (ps PixelServer) kapacitorHandler(w http.ResponseWriter, r *http.Request) {
	if(r.Method != "POST") {
		http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
		return
	}
	ad := KapacitorAlertData{}
	decoder := json.NewDecoder(r.Body)
	err := decoder.Decode(&ad)
	if err != nil {
		log.Fatalf("Could not decode kapacitor AlertData, %s", err)
	}

	var pd PixelData
	pd.Brightness = 100

	switch ad.Level {
	case OKAlert:
		pd.Value = 100
		//pd.Blink = 1
	case InfoAlert:
		pd.Value = -1
	case WarnAlert:
		pd.Value = 50
	case CritAlert:
		pd.Value = 1
		//pd.Blink = 2
	}

	data := ad.Data.Series[0]
	pd.Message = fmt.Sprintf("%s\\%s: %v", data.Tags.Host, data.Name, data.Values[0][1]) // data.Values[1]

	pixelServer.setStatus(pd)
}

func (ps *PixelServer) setStatus (pd PixelData){
	animationDuration := 3000 // ms

	switch pd.Blink {
	case 1:
		pd.Value += 100
	case 2:
		pd.Value += 200
	}

	delta := pd.Value - ps.LastPixelData.Value
	stepTime := float64(animationDuration) / math.Abs(float64(delta))

	var step int
	if delta > 0{
		step = 1
	} else {
		step = -1
	}

	if delta > 0{
		// smooth switch color
		if(pd.Blink == 0 && ps.LastPixelData.Value > 0 && pd.Value > 0) {
			for i := ps.LastPixelData.Value; i != pd.Value; i += step {
				ps.sendSerial(PixelData{i, "", 0, pd.Brightness })
				time.Sleep(time.Millisecond * time.Duration(stepTime))
			}
		}
	} else {
		// sharp switch color
		if(pd.Blink == 0 && ps.LastPixelData.Value > 0 && pd.Value > 0) {
			for i := 0; i < 3; i++ {
				ps.sendSerial(PixelData{pd.Value, "", 0, pd.Brightness })
				time.Sleep(time.Millisecond * 250)
				ps.sendSerial(PixelData{ps.LastPixelData.Value, "", 0, pd.Brightness })
				time.Sleep(time.Millisecond * 250)
			}
			ps.sendSerial(PixelData{pd.Value, "", 0, pd.Brightness })
		}
	}

	time.Sleep(100 * time.Duration(stepTime))
	log.Printf("setStatus: %v\n", pd)
	ps.sendSerial(pd)

	ps.LastPixelData = pd

	// if success value, turn off led
	if pd.Value == 100{
		time.Sleep(time.Millisecond * 5000)
		pixelServer.setStatus(PixelData{ -1, "", 0, 100 })
		//ps.sendSerial(PixelData{ -1, "", 0, 100 })
	}

	time.Sleep(1000 * time.Millisecond)
}

func (ps PixelServer) sendSerial (pd PixelData) (int, error){
	command := fmt.Sprintf("%d|%s|%d\n",pd.Value, pd.Message, pd.Brightness)
	//fmt.Println(command)
	n, err := ps.Serial.Write([]byte(command))
	if err != nil {
		log.Fatalf("Could not write to port, %s", err)
	}
	return n, err
}
<|endoftext|>


<commit_before>package webhooks

import "net/http"

// Provider defines the type of webhook
type Provider int

func (p Provider) String() string {
	switch p {
	case GitHub:
		return "GitHub"
	default:
		return "Unknown"
	}
}

// webhooks available providers
const (
	GitHub Provider = iota
)

// Webhook interface defines a webhook to recieve events
type Webhook interface {
	Provider() Provider
	ParsePayload(w http.ResponseWriter, r *http.Request)
}

type server struct {
	hook Webhook
	path string
}

// ProcessPayloadFunc is a common function for payload return values
type ProcessPayloadFunc func(payload interface{})

// Run runs a server
func Run(hook Webhook, addr string, path string) error {
	srv := &server{
		hook: hook,
		path: path,
	}

	s := &http.Server{Addr: addr, Handler: srv}

	return s.ListenAndServe()
}

// RunServer runs a custom server.
func RunServer(s *http.Server, hook Webhook, path string) error {

	srv := &server{
		hook: hook,
		path: path,
	}

	s.Handler = srv

	return s.ListenAndServe()
}

// RunTLSServer runs a custom server with TLS configuration.
// NOTE: http.Server Handler will be overridden by this library, just set it to nil.
// Setting the Certificates can be done in the http.Server.TLSConfig.Certificates
// see example here:
func RunTLSServer(s *http.Server, hook Webhook, path string) error {

	// var err error

	srv := &server{
		hook: hook,
		path: path,
	}

	s.Handler = srv

	return s.ListenAndServeTLS("", "")
}

func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	defer r.Body.Close()

	if r.Method != "POST" {
		http.Error(w, "405 Method not allowed", http.StatusMethodNotAllowed)
		return
	}
	if r.URL.Path != s.path {
		http.Error(w, "404 Not found", http.StatusNotFound)
		return
	}

	s.hook.ParsePayload(w, r)
}
<commit_msg>Completed webhooks package documentation<commit_after>package webhooks

import "net/http"

// Provider defines the type of webhook
type Provider int

func (p Provider) String() string {
	switch p {
	case GitHub:
		return "GitHub"
	default:
		return "Unknown"
	}
}

// webhooks available providers
const (
	GitHub Provider = iota
)

// Webhook interface defines a webhook to recieve events
type Webhook interface {
	Provider() Provider
	ParsePayload(w http.ResponseWriter, r *http.Request)
}

type server struct {
	hook Webhook
	path string
}

// ProcessPayloadFunc is a common function for payload return values
type ProcessPayloadFunc func(payload interface{})

// Run runs a server
func Run(hook Webhook, addr string, path string) error {
	srv := &server{
		hook: hook,
		path: path,
	}

	s := &http.Server{Addr: addr, Handler: srv}

	return s.ListenAndServe()
}

// RunServer runs a custom server.
func RunServer(s *http.Server, hook Webhook, path string) error {

	srv := &server{
		hook: hook,
		path: path,
	}

	s.Handler = srv

	return s.ListenAndServe()
}

// RunTLSServer runs a custom server with TLS configuration.
// NOTE: http.Server Handler will be overridden by this library, just set it to nil.
// Setting the Certificates can be done in the http.Server.TLSConfig.Certificates
// see example here: https://github.com/joeybloggs/webhooks/blob/master/webhooks_test.go#L178
func RunTLSServer(s *http.Server, hook Webhook, path string) error {

	srv := &server{
		hook: hook,
		path: path,
	}

	s.Handler = srv

	return s.ListenAndServeTLS("", "")
}

// ServeHTTP is the Handler for every posted WebHook Event
func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	defer r.Body.Close()

	if r.Method != "POST" {
		http.Error(w, "405 Method not allowed", http.StatusMethodNotAllowed)
		return
	}
	if r.URL.Path != s.path {
		http.Error(w, "404 Not found", http.StatusNotFound)
		return
	}

	s.hook.ParsePayload(w, r)
}
<|endoftext|>


<commit_before>package command

import (
	"fmt"
	"net/http"
	httppprof "net/http/pprof"
	"os"
	"runtime"
	"runtime/pprof"
	"strconv"
	"strings"
	"time"

	"github.com/spf13/viper"
	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/util/grace"

	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/security"
	"github.com/chrislusf/seaweedfs/weed/util/httpdown"

	"google.golang.org/grpc/reflection"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
	"github.com/chrislusf/seaweedfs/weed/server"
	"github.com/chrislusf/seaweedfs/weed/storage"
	"github.com/chrislusf/seaweedfs/weed/util"
)

var (
	v VolumeServerOptions
)

type VolumeServerOptions struct {
	port                  *int
	publicPort            *int
	folders               []string
	folderMaxLimits       []int
	ip                    *string
	publicUrl             *string
	bindIp                *string
	masters               *string
	idleConnectionTimeout *int
	dataCenter            *string
	rack                  *string
	whiteList             []string
	indexType             *string
	fixJpgOrientation     *bool
	readRedirect          *bool
	cpuProfile            *string
	memProfile            *string
	compactionMBPerSecond *int
	fileSizeLimitMB       *int
	minFreeSpacePercents  []float32
	pprof                 *bool
	preStopSeconds        *int
	// pulseSeconds          *int
}

func init() {
	cmdVolume.Run = runVolume // break init cycle
	v.port = cmdVolume.Flag.Int("port", 8080, "http listen port")
	v.publicPort = cmdVolume.Flag.Int("port.public", 0, "port opened to public")
	v.ip = cmdVolume.Flag.String("ip", util.DetectedHostAddress(), "ip or server name")
	v.publicUrl = cmdVolume.Flag.String("publicUrl", "", "Publicly accessible address")
	v.bindIp = cmdVolume.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
	v.masters = cmdVolume.Flag.String("mserver", "localhost:9333", "comma-separated master servers")
	v.preStopSeconds = cmdVolume.Flag.Int("preStopSeconds", 10, "number of seconds between stop send heartbeats and stop volume server")
	// v.pulseSeconds = cmdVolume.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats, must be smaller than or equal to the master's setting")
	v.idleConnectionTimeout = cmdVolume.Flag.Int("idleTimeout", 30, "connection idle seconds")
	v.dataCenter = cmdVolume.Flag.String("dataCenter", "", "current volume server's data center name")
	v.rack = cmdVolume.Flag.String("rack", "", "current volume server's rack name")
	v.indexType = cmdVolume.Flag.String("index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
	v.fixJpgOrientation = cmdVolume.Flag.Bool("images.fix.orientation", false, "Adjust jpg orientation when uploading.")
	v.readRedirect = cmdVolume.Flag.Bool("read.redirect", true, "Redirect moved or non-local volumes.")
	v.cpuProfile = cmdVolume.Flag.String("cpuprofile", "", "cpu profile output file")
	v.memProfile = cmdVolume.Flag.String("memprofile", "", "memory profile output file")
	v.compactionMBPerSecond = cmdVolume.Flag.Int("compactionMBps", 0, "limit background compaction or copying speed in mega bytes per second")
	v.fileSizeLimitMB = cmdVolume.Flag.Int("fileSizeLimitMB", 1024, "limit file size to avoid out of memory")
	v.pprof = cmdVolume.Flag.Bool("pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile")
}

var cmdVolume = &Command{
	UsageLine: "volume -port=8080 -dir=/tmp -max=5 -ip=server_name -mserver=localhost:9333",
	Short:     "start a volume server",
	Long: `start a volume server to provide storage spaces

  `,
}

var (
	volumeFolders         = cmdVolume.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
	maxVolumeCounts       = cmdVolume.Flag.String("max", "8", "maximum numbers of volumes, count[,count]... If set to zero, the limit will be auto configured.")
	volumeWhiteListOption = cmdVolume.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
	minFreeSpacePercent   = cmdVolume.Flag.String("minFreeSpacePercent", "1", "minimum free disk space (default to 1%). Low disk space will mark all volumes as ReadOnly.")
)

func runVolume(cmd *Command, args []string) bool {

	util.LoadConfiguration("security", false)

	runtime.GOMAXPROCS(runtime.NumCPU())

	// If --pprof is set we assume the caller wants to be able to collect
	// cpu and memory profiles via go tool pprof
	if !*v.pprof {
		grace.SetupProfiling(*v.cpuProfile, *v.memProfile)
	}

	v.startVolumeServer(*volumeFolders, *maxVolumeCounts, *volumeWhiteListOption, *minFreeSpacePercent)

	return true
}

func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, volumeWhiteListOption, minFreeSpacePercent string) {

	// Set multiple folders and each folder's max volume count limit'
	v.folders = strings.Split(volumeFolders, ",")
	for _, folder := range v.folders {
		if err := util.TestFolderWritable(util.ResolvePath(folder)); err != nil {
			glog.Fatalf("Check Data Folder(-dir) Writable %s : %s", folder, err)
		}
	}

	// set max
	maxCountStrings := strings.Split(maxVolumeCounts, ",")
	for _, maxString := range maxCountStrings {
		if max, e := strconv.Atoi(maxString); e == nil {
			v.folderMaxLimits = append(v.folderMaxLimits, max)
		} else {
			glog.Fatalf("The max specified in -max not a valid number %s", maxString)
		}
	}
	if len(v.folders) != len(v.folderMaxLimits) {
		glog.Fatalf("%d directories by -dir, but only %d max is set by -max", len(v.folders), len(v.folderMaxLimits))
	}

	// set minFreeSpacePercent
	minFreeSpacePercentStrings := strings.Split(minFreeSpacePercent, ",")
	for _, freeString := range minFreeSpacePercentStrings {
		if value, e := strconv.ParseFloat(freeString, 32); e == nil {
			v.minFreeSpacePercents = append(v.minFreeSpacePercents, float32(value))
		} else {
			glog.Fatalf("The value specified in -minFreeSpacePercent not a valid value %s", freeString)
		}
	}
	if len(v.minFreeSpacePercents) == 1 && len(v.folders) > 1 {
		for i := 0; i < len(v.folders)-1; i++ {
			v.minFreeSpacePercents = append(v.minFreeSpacePercents, v.minFreeSpacePercents[0])
		}
	}
	if len(v.folders) != len(v.minFreeSpacePercents) {
		glog.Fatalf("%d directories by -dir, but only %d minFreeSpacePercent is set by -minFreeSpacePercent", len(v.folders), len(v.minFreeSpacePercents))
	}

	// security related white list configuration
	if volumeWhiteListOption != "" {
		v.whiteList = strings.Split(volumeWhiteListOption, ",")
	}

	if *v.ip == "" {
		*v.ip = util.DetectedHostAddress()
		glog.V(0).Infof("detected volume server ip address: %v", *v.ip)
	}

	if *v.publicPort == 0 {
		*v.publicPort = *v.port
	}
	if *v.publicUrl == "" {
		*v.publicUrl = *v.ip + ":" + strconv.Itoa(*v.publicPort)
	}

	volumeMux := http.NewServeMux()
	publicVolumeMux := volumeMux
	if v.isSeparatedPublicPort() {
		publicVolumeMux = http.NewServeMux()
	}

	if *v.pprof {
		volumeMux.HandleFunc("/debug/pprof/", httppprof.Index)
		volumeMux.HandleFunc("/debug/pprof/cmdline", httppprof.Cmdline)
		volumeMux.HandleFunc("/debug/pprof/profile", httppprof.Profile)
		volumeMux.HandleFunc("/debug/pprof/symbol", httppprof.Symbol)
		volumeMux.HandleFunc("/debug/pprof/trace", httppprof.Trace)
	}

	volumeNeedleMapKind := storage.NeedleMapInMemory
	switch *v.indexType {
	case "leveldb":
		volumeNeedleMapKind = storage.NeedleMapLevelDb
	case "leveldbMedium":
		volumeNeedleMapKind = storage.NeedleMapLevelDbMedium
	case "leveldbLarge":
		volumeNeedleMapKind = storage.NeedleMapLevelDbLarge
	}

	masters := *v.masters

	volumeServer := weed_server.NewVolumeServer(volumeMux, publicVolumeMux,
		*v.ip, *v.port, *v.publicUrl,
		v.folders, v.folderMaxLimits, v.minFreeSpacePercents,
		volumeNeedleMapKind,
		strings.Split(masters, ","), 5, *v.dataCenter, *v.rack,
		v.whiteList,
		*v.fixJpgOrientation, *v.readRedirect,
		*v.compactionMBPerSecond,
		*v.fileSizeLimitMB,
	)
	// starting grpc server
	grpcS := v.startGrpcService(volumeServer)

	// starting public http server
	var publicHttpDown httpdown.Server
	if v.isSeparatedPublicPort() {
		publicHttpDown = v.startPublicHttpService(publicVolumeMux)
		if nil == publicHttpDown {
			glog.Fatalf("start public http service failed")
		}
	}

	// starting the cluster http server
	clusterHttpServer := v.startClusterHttpService(volumeMux)

	stopChain := make(chan struct{})
	grace.OnInterrupt(func() {
		fmt.Println("volume server has be killed")
		var startTime time.Time

		// Stop heartbeats
		glog.V(0).Infof("stop send heartbeat and wait %d seconds until shutdown ...", *v.preStopSeconds)
		volumeServer.SendHeartbeat = false
		time.Sleep(time.Duration(*v.preStopSeconds) * time.Second)
		glog.V(0).Infof("end sleep %d sec", *v.preStopSeconds)
		// firstly, stop the public http service to prevent from receiving new user request
		if nil != publicHttpDown {
			startTime = time.Now()
			if err := publicHttpDown.Stop(); err != nil {
				glog.Warningf("stop the public http server failed, %v", err)
			}
			delta := time.Now().Sub(startTime).Nanoseconds() / 1e6
			glog.V(0).Infof("stop public http server, elapsed %dms", delta)
		}

		startTime = time.Now()
		if err := clusterHttpServer.Stop(); err != nil {
			glog.Warningf("stop the cluster http server failed, %v", err)
		}
		delta := time.Now().Sub(startTime).Nanoseconds() / 1e6
		glog.V(0).Infof("graceful stop cluster http server, elapsed [%d]", delta)

		startTime = time.Now()
		grpcS.GracefulStop()
		delta = time.Now().Sub(startTime).Nanoseconds() / 1e6
		glog.V(0).Infof("graceful stop gRPC, elapsed [%d]", delta)

		startTime = time.Now()
		volumeServer.Shutdown()
		delta = time.Now().Sub(startTime).Nanoseconds() / 1e6
		glog.V(0).Infof("stop volume server, elapsed [%d]", delta)

		pprof.StopCPUProfile()

		close(stopChain) // notify exit
	})

	select {
	case <-stopChain:
	}
	glog.Warningf("the volume server exit.")
}

// check whether configure the public port
func (v VolumeServerOptions) isSeparatedPublicPort() bool {
	return *v.publicPort != *v.port
}

func (v VolumeServerOptions) startGrpcService(vs volume_server_pb.VolumeServerServer) *grpc.Server {
	grpcPort := *v.port + 10000
	grpcL, err := util.NewListener(*v.bindIp+":"+strconv.Itoa(grpcPort), 0)
	if err != nil {
		glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err)
	}
	grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.volume"))
	volume_server_pb.RegisterVolumeServerServer(grpcS, vs)
	reflection.Register(grpcS)
	go func() {
		if err := grpcS.Serve(grpcL); err != nil {
			glog.Fatalf("start gRPC service failed, %s", err)
		}
	}()
	return grpcS
}

func (v VolumeServerOptions) startPublicHttpService(handler http.Handler) httpdown.Server {
	publicListeningAddress := *v.bindIp + ":" + strconv.Itoa(*v.publicPort)
	glog.V(0).Infoln("Start Seaweed volume server", util.Version(), "public at", publicListeningAddress)
	publicListener, e := util.NewListener(publicListeningAddress, time.Duration(*v.idleConnectionTimeout)*time.Second)
	if e != nil {
		glog.Fatalf("Volume server listener error:%v", e)
	}

	pubHttp := httpdown.HTTP{StopTimeout: 5 * time.Minute, KillTimeout: 5 * time.Minute}
	publicHttpDown := pubHttp.Serve(&http.Server{Handler: handler}, publicListener)
	go func() {
		if err := publicHttpDown.Wait(); err != nil {
			glog.Errorf("public http down wait failed, %v", err)
		}
	}()

	return publicHttpDown
}

func (v VolumeServerOptions) startClusterHttpService(handler http.Handler) httpdown.Server {
	var (
		certFile, keyFile string
	)
	if viper.GetString("https.volume.key") != "" {
		certFile = viper.GetString("https.volume.cert")
		keyFile = viper.GetString("https.volume.key")
	}

	listeningAddress := *v.bindIp + ":" + strconv.Itoa(*v.port)
	glog.V(0).Infof("Start Seaweed volume server %s at %s", util.Version(), listeningAddress)
	listener, e := util.NewListener(listeningAddress, time.Duration(*v.idleConnectionTimeout)*time.Second)
	if e != nil {
		glog.Fatalf("Volume server listener error:%v", e)
	}

	httpDown := httpdown.HTTP{
		KillTimeout: 5 * time.Minute,
		StopTimeout: 5 * time.Minute,
		CertFile:    certFile,
		KeyFile:     keyFile}
	clusterHttpServer := httpDown.Serve(&http.Server{Handler: handler}, listener)
	go func() {
		if e := clusterHttpServer.Wait(); e != nil {
			glog.Fatalf("Volume server fail to serve: %v", e)
		}
	}()
	return clusterHttpServer
}
<commit_msg>refactoring<commit_after>package command

import (
	"fmt"
	"net/http"
	httppprof "net/http/pprof"
	"os"
	"runtime"
	"runtime/pprof"
	"strconv"
	"strings"
	"time"

	"github.com/spf13/viper"
	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/util/grace"

	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/security"
	"github.com/chrislusf/seaweedfs/weed/util/httpdown"

	"google.golang.org/grpc/reflection"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
	"github.com/chrislusf/seaweedfs/weed/server"
	"github.com/chrislusf/seaweedfs/weed/storage"
	"github.com/chrislusf/seaweedfs/weed/util"
)

var (
	v VolumeServerOptions
)

type VolumeServerOptions struct {
	port                  *int
	publicPort            *int
	folders               []string
	folderMaxLimits       []int
	ip                    *string
	publicUrl             *string
	bindIp                *string
	masters               *string
	idleConnectionTimeout *int
	dataCenter            *string
	rack                  *string
	whiteList             []string
	indexType             *string
	fixJpgOrientation     *bool
	readRedirect          *bool
	cpuProfile            *string
	memProfile            *string
	compactionMBPerSecond *int
	fileSizeLimitMB       *int
	minFreeSpacePercents  []float32
	pprof                 *bool
	preStopSeconds        *int
	// pulseSeconds          *int
}

func init() {
	cmdVolume.Run = runVolume // break init cycle
	v.port = cmdVolume.Flag.Int("port", 8080, "http listen port")
	v.publicPort = cmdVolume.Flag.Int("port.public", 0, "port opened to public")
	v.ip = cmdVolume.Flag.String("ip", util.DetectedHostAddress(), "ip or server name")
	v.publicUrl = cmdVolume.Flag.String("publicUrl", "", "Publicly accessible address")
	v.bindIp = cmdVolume.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
	v.masters = cmdVolume.Flag.String("mserver", "localhost:9333", "comma-separated master servers")
	v.preStopSeconds = cmdVolume.Flag.Int("preStopSeconds", 10, "number of seconds between stop send heartbeats and stop volume server")
	// v.pulseSeconds = cmdVolume.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats, must be smaller than or equal to the master's setting")
	v.idleConnectionTimeout = cmdVolume.Flag.Int("idleTimeout", 30, "connection idle seconds")
	v.dataCenter = cmdVolume.Flag.String("dataCenter", "", "current volume server's data center name")
	v.rack = cmdVolume.Flag.String("rack", "", "current volume server's rack name")
	v.indexType = cmdVolume.Flag.String("index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
	v.fixJpgOrientation = cmdVolume.Flag.Bool("images.fix.orientation", false, "Adjust jpg orientation when uploading.")
	v.readRedirect = cmdVolume.Flag.Bool("read.redirect", true, "Redirect moved or non-local volumes.")
	v.cpuProfile = cmdVolume.Flag.String("cpuprofile", "", "cpu profile output file")
	v.memProfile = cmdVolume.Flag.String("memprofile", "", "memory profile output file")
	v.compactionMBPerSecond = cmdVolume.Flag.Int("compactionMBps", 0, "limit background compaction or copying speed in mega bytes per second")
	v.fileSizeLimitMB = cmdVolume.Flag.Int("fileSizeLimitMB", 1024, "limit file size to avoid out of memory")
	v.pprof = cmdVolume.Flag.Bool("pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile")
}

var cmdVolume = &Command{
	UsageLine: "volume -port=8080 -dir=/tmp -max=5 -ip=server_name -mserver=localhost:9333",
	Short:     "start a volume server",
	Long: `start a volume server to provide storage spaces

  `,
}

var (
	volumeFolders         = cmdVolume.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
	maxVolumeCounts       = cmdVolume.Flag.String("max", "8", "maximum numbers of volumes, count[,count]... If set to zero, the limit will be auto configured.")
	volumeWhiteListOption = cmdVolume.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
	minFreeSpacePercent   = cmdVolume.Flag.String("minFreeSpacePercent", "1", "minimum free disk space (default to 1%). Low disk space will mark all volumes as ReadOnly.")
)

func runVolume(cmd *Command, args []string) bool {

	util.LoadConfiguration("security", false)

	runtime.GOMAXPROCS(runtime.NumCPU())

	// If --pprof is set we assume the caller wants to be able to collect
	// cpu and memory profiles via go tool pprof
	if !*v.pprof {
		grace.SetupProfiling(*v.cpuProfile, *v.memProfile)
	}

	v.startVolumeServer(*volumeFolders, *maxVolumeCounts, *volumeWhiteListOption, *minFreeSpacePercent)

	return true
}

func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, volumeWhiteListOption, minFreeSpacePercent string) {

	// Set multiple folders and each folder's max volume count limit'
	v.folders = strings.Split(volumeFolders, ",")
	for _, folder := range v.folders {
		if err := util.TestFolderWritable(util.ResolvePath(folder)); err != nil {
			glog.Fatalf("Check Data Folder(-dir) Writable %s : %s", folder, err)
		}
	}

	// set max
	maxCountStrings := strings.Split(maxVolumeCounts, ",")
	for _, maxString := range maxCountStrings {
		if max, e := strconv.Atoi(maxString); e == nil {
			v.folderMaxLimits = append(v.folderMaxLimits, max)
		} else {
			glog.Fatalf("The max specified in -max not a valid number %s", maxString)
		}
	}
	if len(v.folders) != len(v.folderMaxLimits) {
		glog.Fatalf("%d directories by -dir, but only %d max is set by -max", len(v.folders), len(v.folderMaxLimits))
	}

	// set minFreeSpacePercent
	minFreeSpacePercentStrings := strings.Split(minFreeSpacePercent, ",")
	for _, freeString := range minFreeSpacePercentStrings {
		if value, e := strconv.ParseFloat(freeString, 32); e == nil {
			v.minFreeSpacePercents = append(v.minFreeSpacePercents, float32(value))
		} else {
			glog.Fatalf("The value specified in -minFreeSpacePercent not a valid value %s", freeString)
		}
	}
	if len(v.minFreeSpacePercents) == 1 && len(v.folders) > 1 {
		for i := 0; i < len(v.folders)-1; i++ {
			v.minFreeSpacePercents = append(v.minFreeSpacePercents, v.minFreeSpacePercents[0])
		}
	}
	if len(v.folders) != len(v.minFreeSpacePercents) {
		glog.Fatalf("%d directories by -dir, but only %d minFreeSpacePercent is set by -minFreeSpacePercent", len(v.folders), len(v.minFreeSpacePercents))
	}

	// security related white list configuration
	if volumeWhiteListOption != "" {
		v.whiteList = strings.Split(volumeWhiteListOption, ",")
	}

	if *v.ip == "" {
		*v.ip = util.DetectedHostAddress()
		glog.V(0).Infof("detected volume server ip address: %v", *v.ip)
	}

	if *v.publicPort == 0 {
		*v.publicPort = *v.port
	}
	if *v.publicUrl == "" {
		*v.publicUrl = *v.ip + ":" + strconv.Itoa(*v.publicPort)
	}

	volumeMux := http.NewServeMux()
	publicVolumeMux := volumeMux
	if v.isSeparatedPublicPort() {
		publicVolumeMux = http.NewServeMux()
	}

	if *v.pprof {
		volumeMux.HandleFunc("/debug/pprof/", httppprof.Index)
		volumeMux.HandleFunc("/debug/pprof/cmdline", httppprof.Cmdline)
		volumeMux.HandleFunc("/debug/pprof/profile", httppprof.Profile)
		volumeMux.HandleFunc("/debug/pprof/symbol", httppprof.Symbol)
		volumeMux.HandleFunc("/debug/pprof/trace", httppprof.Trace)
	}

	volumeNeedleMapKind := storage.NeedleMapInMemory
	switch *v.indexType {
	case "leveldb":
		volumeNeedleMapKind = storage.NeedleMapLevelDb
	case "leveldbMedium":
		volumeNeedleMapKind = storage.NeedleMapLevelDbMedium
	case "leveldbLarge":
		volumeNeedleMapKind = storage.NeedleMapLevelDbLarge
	}

	masters := *v.masters

	volumeServer := weed_server.NewVolumeServer(volumeMux, publicVolumeMux,
		*v.ip, *v.port, *v.publicUrl,
		v.folders, v.folderMaxLimits, v.minFreeSpacePercents,
		volumeNeedleMapKind,
		strings.Split(masters, ","), 5, *v.dataCenter, *v.rack,
		v.whiteList,
		*v.fixJpgOrientation, *v.readRedirect,
		*v.compactionMBPerSecond,
		*v.fileSizeLimitMB,
	)
	// starting grpc server
	grpcS := v.startGrpcService(volumeServer)

	// starting public http server
	var publicHttpDown httpdown.Server
	if v.isSeparatedPublicPort() {
		publicHttpDown = v.startPublicHttpService(publicVolumeMux)
		if nil == publicHttpDown {
			glog.Fatalf("start public http service failed")
		}
	}

	// starting the cluster http server
	clusterHttpServer := v.startClusterHttpService(volumeMux)

	grace.OnInterrupt(func() {
		fmt.Println("volume server has be killed")

		// Stop heartbeatsZ
		glog.V(0).Infof("stop send heartbeat and wait %d seconds until shutdown ...", *v.preStopSeconds)
		volumeServer.SendHeartbeat = false
		time.Sleep(time.Duration(*v.preStopSeconds) * time.Second)

		v.shutdown(publicHttpDown, clusterHttpServer, grpcS, volumeServer)
	})

	select {}

}

func (v VolumeServerOptions) shutdown(publicHttpDown httpdown.Server, clusterHttpServer httpdown.Server, grpcS *grpc.Server, volumeServer *weed_server.VolumeServer) {

	// firstly, stop the public http service to prevent from receiving new user request
	if nil != publicHttpDown {
		glog.V(0).Infof("stop public http server ... ")
		if err := publicHttpDown.Stop(); err != nil {
			glog.Warningf("stop the public http server failed, %v", err)
		}
	}

	glog.V(0).Infof("graceful stop cluster http server ... ")
	if err := clusterHttpServer.Stop(); err != nil {
		glog.Warningf("stop the cluster http server failed, %v", err)
	}

	glog.V(0).Infof("graceful stop gRPC ...")
	grpcS.GracefulStop()

	volumeServer.Shutdown()

	pprof.StopCPUProfile()

}

// check whether configure the public port
func (v VolumeServerOptions) isSeparatedPublicPort() bool {
	return *v.publicPort != *v.port
}

func (v VolumeServerOptions) startGrpcService(vs volume_server_pb.VolumeServerServer) *grpc.Server {
	grpcPort := *v.port + 10000
	grpcL, err := util.NewListener(*v.bindIp+":"+strconv.Itoa(grpcPort), 0)
	if err != nil {
		glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err)
	}
	grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.volume"))
	volume_server_pb.RegisterVolumeServerServer(grpcS, vs)
	reflection.Register(grpcS)
	go func() {
		if err := grpcS.Serve(grpcL); err != nil {
			glog.Fatalf("start gRPC service failed, %s", err)
		}
	}()
	return grpcS
}

func (v VolumeServerOptions) startPublicHttpService(handler http.Handler) httpdown.Server {
	publicListeningAddress := *v.bindIp + ":" + strconv.Itoa(*v.publicPort)
	glog.V(0).Infoln("Start Seaweed volume server", util.Version(), "public at", publicListeningAddress)
	publicListener, e := util.NewListener(publicListeningAddress, time.Duration(*v.idleConnectionTimeout)*time.Second)
	if e != nil {
		glog.Fatalf("Volume server listener error:%v", e)
	}

	pubHttp := httpdown.HTTP{StopTimeout: 5 * time.Minute, KillTimeout: 5 * time.Minute}
	publicHttpDown := pubHttp.Serve(&http.Server{Handler: handler}, publicListener)
	go func() {
		if err := publicHttpDown.Wait(); err != nil {
			glog.Errorf("public http down wait failed, %v", err)
		}
	}()

	return publicHttpDown
}

func (v VolumeServerOptions) startClusterHttpService(handler http.Handler) httpdown.Server {
	var (
		certFile, keyFile string
	)
	if viper.GetString("https.volume.key") != "" {
		certFile = viper.GetString("https.volume.cert")
		keyFile = viper.GetString("https.volume.key")
	}

	listeningAddress := *v.bindIp + ":" + strconv.Itoa(*v.port)
	glog.V(0).Infof("Start Seaweed volume server %s at %s", util.Version(), listeningAddress)
	listener, e := util.NewListener(listeningAddress, time.Duration(*v.idleConnectionTimeout)*time.Second)
	if e != nil {
		glog.Fatalf("Volume server listener error:%v", e)
	}

	httpDown := httpdown.HTTP{
		KillTimeout: 5 * time.Minute,
		StopTimeout: 5 * time.Minute,
		CertFile:    certFile,
		KeyFile:     keyFile}
	clusterHttpServer := httpDown.Serve(&http.Server{Handler: handler}, listener)
	go func() {
		if e := clusterHttpServer.Wait(); e != nil {
			glog.Fatalf("Volume server fail to serve: %v", e)
		}
	}()
	return clusterHttpServer
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"strings"
	"time"
)

const (
	defaultStoreTimeout = 5 * time.Second
)

// Team model struct
type Team struct {
	ID          string `json:"id"`
	DisplayName string `json:"displayName"`
}

// Period model struct
type Period struct {
	ID          string   `json:"id"`
	DisplayName string   `json:"displayName"`
	Unit        string   `json:"unit"`
	Buckets     []Bucket `json:"buckets"`
	People      []Person `json:"people"`
}

// Bucket model struct
type Bucket struct {
	DisplayName          string      `json:"displayName"`
	AllocationPercentage float64     `json:"allocationPercentage"`
	Objectives           []Objective `json:"objectives"`
}

// Objective model struct
type Objective struct {
	Name             string       `json:"name"`
	ResourceEstimate float64      `json:"resourceEstimate"`
	Assignments      []Assignment `json:"assignments"`
}

// Assignment model struct
type Assignment struct {
	PersonID   string  `json:"personId"`
	Commitment float64 `json:"commitment"`
}

// Person model struct
type Person struct {
	ID           string  `json:"id"`
	DisplayName  string  `json:"displayName"`
	Availability float64 `json:"availability"`
}

// StorageService to represent the persistent store
type StorageService interface {
	GetAllTeams(ctx context.Context) ([]Team, error)
	GetTeam(ctx context.Context, teamID string) (Team, bool, error)
	CreateTeam(ctx context.Context, team Team) error
	UpdateTeam(ctx context.Context, team Team) error
	GetAllPeriods(ctx context.Context, teamID string) ([]Period, bool, error)
	GetPeriod(ctx context.Context, teamID, periodID string) (Period, bool, error)
	CreatePeriod(ctx context.Context, teamID string, period Period) error
	UpdatePeriod(ctx context.Context, teamID string, period Period) error
	Close() error
}

// Server struct to handle incoming HTTP requests
type Server struct {
	store        StorageService
	storeTimeout time.Duration
}

func (s *Server) makeHandler() http.Handler {
	mux := http.NewServeMux()
	mux.HandleFunc("/api/team/", s.handleTeam)
	mux.HandleFunc("/api/period/", s.handlePeriod)
	return mux
}

func (s *Server) ensureTeamExistence(w http.ResponseWriter, r *http.Request, teamID string, expected bool) bool {
	ctx, cancel := context.WithTimeout(r.Context(), s.storeTimeout)
	defer cancel()
	_, exists, err := s.store.GetTeam(ctx, teamID)
	if err != nil {
		log.Printf("Could not validate existence of team '%s': error: %s", teamID, err)
		http.Error(w, fmt.Sprintf("Could not validate existence of team '%s' (see server log)", teamID), http.StatusInternalServerError)
		return false
	}
	if exists != expected {
		statusCode := http.StatusBadRequest
		if expected {
			statusCode = http.StatusNotFound
		}
		http.Error(w, fmt.Sprintf("Team with ID '%s' expected exists=%v, found %v", teamID, expected, exists), statusCode)
		return false
	}
	return true
}

func (s *Server) ensurePeriodExistence(w http.ResponseWriter, r *http.Request, teamID, periodID string, expected bool) bool {
	ctx, cancel := context.WithTimeout(r.Context(), s.storeTimeout)
	defer cancel()
	_, exists, err := s.store.GetPeriod(ctx, teamID, periodID)
	if err != nil {
		log.Printf("Could not validate existence of period '%s' for team '%s': error: %s", periodID, teamID, err)
		http.Error(w, fmt.Sprintf("Could not validate existence of period '%s' for team '%s' (see server log)", periodID, teamID), http.StatusInternalServerError)
		return false
	}
	if exists != expected {
		statusCode := http.StatusBadRequest
		if expected {
			statusCode = http.StatusNotFound
		}
		http.Error(w, fmt.Sprintf("Period '%s' for team '%s' expected exists=%v, found %v", periodID, teamID, expected, exists), statusCode)
		return false
	}
	return true
}

func (s *Server) handleTeam(w http.ResponseWriter, r *http.Request) {
	pathParts := strings.Split(r.URL.Path, "/")
	log.Printf("Team path: %q", pathParts)
	if len(pathParts) != 4 {
		http.NotFound(w, r)
		return
	}
	teamID := pathParts[3]
	if r.Method == http.MethodGet {
		s.handleGetTeam(w, r, teamID)
	} else if r.Method == http.MethodPost {
		s.handlePostTeam(w, r)
	} else if r.Method == http.MethodPut {
		s.handlePutTeam(w, r)
	} else {
		http.Error(w, fmt.Sprintf("Unsupported method '%s'", r.Method), http.StatusBadRequest)
	}
}

func (s *Server) handleGetTeam(w http.ResponseWriter, r *http.Request, teamID string) {
	ctx, cancel := context.WithTimeout(r.Context(), s.storeTimeout)
	defer cancel()
	if teamID == "" {
		teams, err := s.store.GetAllTeams(ctx)
		if err != nil {
			log.Printf("Could not retrieve teams: error: %s", err)
			http.Error(w, "Could not retrieve teams (see server log)", http.StatusInternalServerError)
			return
		}
		enc := json.NewEncoder(w)
		w.Header().Set("Content-Type", "application/json")
		enc.Encode(teams)
	} else {
		team, found, err := s.store.GetTeam(ctx, teamID)
		if err != nil {
			log.Printf("Could not retrieve team '%s': error: %s", teamID, err)
			http.Error(w, fmt.Sprintf("Could not retrieve team '%s' (see server log)", teamID), http.StatusInternalServerError)
			return
		}
		if !found {
			http.NotFound(w, r)
			return
		}
		enc := json.NewEncoder(w)
		w.Header().Set("Content-Type", "application/json")
		enc.Encode(team)
	}
}

func (s *Server) handlePostTeam(w http.ResponseWriter, r *http.Request) {
	team, ok := readTeamFromBody(w, r)
	if !ok {
		return
	}
	if !s.ensureTeamExistence(w, r, team.ID, false) {
		return
	}
	ctx, cancel := context.WithTimeout(r.Context(), s.storeTimeout)
	defer cancel()
	err := s.store.CreateTeam(ctx, team)
	if err != nil {
		log.Printf("Could not create team: error: %s", err)
		http.Error(w, "Could not create team (see server log)", http.StatusInternalServerError)
		return
	}
}

func (s *Server) handlePutTeam(w http.ResponseWriter, r *http.Request) {
	team, ok := readTeamFromBody(w, r)
	if !ok {
		return
	}
	if !s.ensureTeamExistence(w, r, team.ID, true) {
		return
	}
	ctx, cancel := context.WithTimeout(r.Context(), s.storeTimeout)
	defer cancel()
	err := s.store.UpdateTeam(ctx, team)
	if err != nil {
		log.Printf("Could not update team: error: %s", err)
		http.Error(w, "Could not update team (see server log)", http.StatusInternalServerError)
		return
	}
}

func readTeamFromBody(w http.ResponseWriter, r *http.Request) (Team, bool) {
	dec := json.NewDecoder(r.Body)
	team := Team{}
	err := dec.Decode(&team)
	if err != nil {
		http.Error(w, fmt.Sprintf("Could not decode body: %v", err), http.StatusBadRequest)
		return team, false
	}
	return team, true
}

func (s *Server) handlePeriod(w http.ResponseWriter, r *http.Request) {
	pathParts := strings.Split(r.URL.Path, "/")
	log.Printf("Period path: %q", pathParts)
	if len(pathParts) != 5 {
		http.NotFound(w, r)
		return
	}
	teamID := pathParts[3]
	periodID := pathParts[4]
	if r.Method == http.MethodGet {
		s.handleGetPeriod(w, r, teamID, periodID)
	} else if r.Method == http.MethodPost {
		s.handlePostPeriod(w, r, teamID)
	} else if r.Method == http.MethodPut {
		s.handlePutPeriod(w, r, teamID, periodID)
	} else {
		http.Error(w, fmt.Sprintf("Unsupported method '%s'", r.Method), http.StatusBadRequest)
	}
}

func (s *Server) handleGetPeriod(w http.ResponseWriter, r *http.Request, teamID, periodID string) {
	ctx, cancel := context.WithTimeout(r.Context(), s.storeTimeout)
	defer cancel()
	if periodID == "" {
		periods, found, err := s.store.GetAllPeriods(ctx, teamID)
		if err != nil {
			log.Printf("Could not retrieve periods for team '%s': error: %s", teamID, err)
			http.Error(w, fmt.Sprintf("Could not retrieve periods for team '%s' (see server log)", teamID), http.StatusInternalServerError)
			return
		}
		if !found {
			http.NotFound(w, r)
			return
		}
		enc := json.NewEncoder(w)
		w.Header().Set("Content-Type", "application/json")
		enc.Encode(periods)
	} else {
		period, found, err := s.store.GetPeriod(ctx, teamID, periodID)
		if err != nil {
			log.Printf("Could not retrieve period '%s' for team '%s': error: %s", periodID, teamID, err)
			http.Error(w, fmt.Sprintf("Could not retrieve period '%s' for team '%s' (see server log)", periodID, teamID), http.StatusInternalServerError)
			return
		}
		if !found {
			http.NotFound(w, r)
			return
		}
		enc := json.NewEncoder(w)
		w.Header().Set("Content-Type", "application/json")
		enc.Encode(period)
	}
}

func (s *Server) handlePostPeriod(w http.ResponseWriter, r *http.Request, teamID string) {
	period, ok := readPeriodFromBody(w, r)
	if !ok {
		return
	}
	if !s.ensureTeamExistence(w, r, teamID, true) {
		return
	}
	if !s.ensurePeriodExistence(w, r, teamID, period.ID, false) {
		return
	}
	ctx, cancel := context.WithTimeout(r.Context(), s.storeTimeout)
	defer cancel()
	err := s.store.CreatePeriod(ctx, teamID, period)
	if err != nil {
		log.Printf("Could not create period for team '%s': error: %s", teamID, err)
		http.Error(w, fmt.Sprintf("Could not create period for team '%s' (see server log)", teamID), http.StatusInternalServerError)
		return
	}
}

func (s *Server) handlePutPeriod(w http.ResponseWriter, r *http.Request, teamID, periodID string) {
	period, ok := readPeriodFromBody(w, r)
	if !ok {
		return
	}
	if !s.ensureTeamExistence(w, r, teamID, true) {
		return
	}
	if !s.ensurePeriodExistence(w, r, teamID, periodID, true) {
		return
	}
	ctx, cancel := context.WithTimeout(r.Context(), s.storeTimeout)
	defer cancel()
	err := s.store.UpdatePeriod(ctx, teamID, period)
	if err != nil {
		log.Printf("Could not update period '%s' for team '%s': error: %s", periodID, teamID, err)
		http.Error(w, fmt.Sprintf("Could not update period '%s' for team '%s' (see server log)", periodID, teamID), http.StatusInternalServerError)
		return
	}
}

func readPeriodFromBody(w http.ResponseWriter, r *http.Request) (Period, bool) {
	dec := json.NewDecoder(r.Body)
	period := Period{}
	err := dec.Decode(&period)
	if err != nil {
		http.Error(w, fmt.Sprintf("Could not decode body: %v", err), http.StatusBadRequest)
		return period, false
	}
	return period, true
}

func main() {
	var gcloudProject string
	flag.StringVar(&gcloudProject, "gcloud_project", "", "GCP project ID")
	flag.Parse()
	var store StorageService
	if gcloudProject == "" {
		log.Printf("Using in-memory store")
		store = makeInMemStore()
	} else {
		log.Printf("Using Cloud Datastore storage service; project='%s'", gcloudProject)
		log.Printf("To use the local emulator, see https://cloud.google.com/datastore/docs/tools/datastore-emulator")
		ctx := context.Background()
		var err error
		store, err = makeGoogleCDSStore(ctx, gcloudProject)
		if err != nil {
			log.Fatalf("Could not instantiate datastore: %s", err)
			return
		}
	}
	server := Server{store: store, storeTimeout: defaultStoreTimeout}
	handler := server.makeHandler()
	port := os.Getenv("PORT")
	if port == "" {
		port = "8080"
		log.Printf("Defaulting to port %s", port)
	}
	log.Printf("Listening on port %s", port)
	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", port), handler))
}
<commit_msg>Read GCP project from environment<commit_after>package main

import (
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"strings"
	"time"
)

const (
	defaultStoreTimeout = 5 * time.Second
)

// Team model struct
type Team struct {
	ID          string `json:"id"`
	DisplayName string `json:"displayName"`
}

// Period model struct
type Period struct {
	ID          string   `json:"id"`
	DisplayName string   `json:"displayName"`
	Unit        string   `json:"unit"`
	Buckets     []Bucket `json:"buckets"`
	People      []Person `json:"people"`
}

// Bucket model struct
type Bucket struct {
	DisplayName          string      `json:"displayName"`
	AllocationPercentage float64     `json:"allocationPercentage"`
	Objectives           []Objective `json:"objectives"`
}

// Objective model struct
type Objective struct {
	Name             string       `json:"name"`
	ResourceEstimate float64      `json:"resourceEstimate"`
	Assignments      []Assignment `json:"assignments"`
}

// Assignment model struct
type Assignment struct {
	PersonID   string  `json:"personId"`
	Commitment float64 `json:"commitment"`
}

// Person model struct
type Person struct {
	ID           string  `json:"id"`
	DisplayName  string  `json:"displayName"`
	Availability float64 `json:"availability"`
}

// StorageService to represent the persistent store
type StorageService interface {
	GetAllTeams(ctx context.Context) ([]Team, error)
	GetTeam(ctx context.Context, teamID string) (Team, bool, error)
	CreateTeam(ctx context.Context, team Team) error
	UpdateTeam(ctx context.Context, team Team) error
	GetAllPeriods(ctx context.Context, teamID string) ([]Period, bool, error)
	GetPeriod(ctx context.Context, teamID, periodID string) (Period, bool, error)
	CreatePeriod(ctx context.Context, teamID string, period Period) error
	UpdatePeriod(ctx context.Context, teamID string, period Period) error
	Close() error
}

// Server struct to handle incoming HTTP requests
type Server struct {
	store        StorageService
	storeTimeout time.Duration
}

func (s *Server) makeHandler() http.Handler {
	mux := http.NewServeMux()
	mux.HandleFunc("/api/team/", s.handleTeam)
	mux.HandleFunc("/api/period/", s.handlePeriod)
	return mux
}

func (s *Server) ensureTeamExistence(w http.ResponseWriter, r *http.Request, teamID string, expected bool) bool {
	ctx, cancel := context.WithTimeout(r.Context(), s.storeTimeout)
	defer cancel()
	_, exists, err := s.store.GetTeam(ctx, teamID)
	if err != nil {
		log.Printf("Could not validate existence of team '%s': error: %s", teamID, err)
		http.Error(w, fmt.Sprintf("Could not validate existence of team '%s' (see server log)", teamID), http.StatusInternalServerError)
		return false
	}
	if exists != expected {
		statusCode := http.StatusBadRequest
		if expected {
			statusCode = http.StatusNotFound
		}
		http.Error(w, fmt.Sprintf("Team with ID '%s' expected exists=%v, found %v", teamID, expected, exists), statusCode)
		return false
	}
	return true
}

func (s *Server) ensurePeriodExistence(w http.ResponseWriter, r *http.Request, teamID, periodID string, expected bool) bool {
	ctx, cancel := context.WithTimeout(r.Context(), s.storeTimeout)
	defer cancel()
	_, exists, err := s.store.GetPeriod(ctx, teamID, periodID)
	if err != nil {
		log.Printf("Could not validate existence of period '%s' for team '%s': error: %s", periodID, teamID, err)
		http.Error(w, fmt.Sprintf("Could not validate existence of period '%s' for team '%s' (see server log)", periodID, teamID), http.StatusInternalServerError)
		return false
	}
	if exists != expected {
		statusCode := http.StatusBadRequest
		if expected {
			statusCode = http.StatusNotFound
		}
		http.Error(w, fmt.Sprintf("Period '%s' for team '%s' expected exists=%v, found %v", periodID, teamID, expected, exists), statusCode)
		return false
	}
	return true
}

func (s *Server) handleTeam(w http.ResponseWriter, r *http.Request) {
	pathParts := strings.Split(r.URL.Path, "/")
	log.Printf("Team path: %q", pathParts)
	if len(pathParts) != 4 {
		http.NotFound(w, r)
		return
	}
	teamID := pathParts[3]
	if r.Method == http.MethodGet {
		s.handleGetTeam(w, r, teamID)
	} else if r.Method == http.MethodPost {
		s.handlePostTeam(w, r)
	} else if r.Method == http.MethodPut {
		s.handlePutTeam(w, r)
	} else {
		http.Error(w, fmt.Sprintf("Unsupported method '%s'", r.Method), http.StatusBadRequest)
	}
}

func (s *Server) handleGetTeam(w http.ResponseWriter, r *http.Request, teamID string) {
	ctx, cancel := context.WithTimeout(r.Context(), s.storeTimeout)
	defer cancel()
	if teamID == "" {
		teams, err := s.store.GetAllTeams(ctx)
		if err != nil {
			log.Printf("Could not retrieve teams: error: %s", err)
			http.Error(w, "Could not retrieve teams (see server log)", http.StatusInternalServerError)
			return
		}
		enc := json.NewEncoder(w)
		w.Header().Set("Content-Type", "application/json")
		enc.Encode(teams)
	} else {
		team, found, err := s.store.GetTeam(ctx, teamID)
		if err != nil {
			log.Printf("Could not retrieve team '%s': error: %s", teamID, err)
			http.Error(w, fmt.Sprintf("Could not retrieve team '%s' (see server log)", teamID), http.StatusInternalServerError)
			return
		}
		if !found {
			http.NotFound(w, r)
			return
		}
		enc := json.NewEncoder(w)
		w.Header().Set("Content-Type", "application/json")
		enc.Encode(team)
	}
}

func (s *Server) handlePostTeam(w http.ResponseWriter, r *http.Request) {
	team, ok := readTeamFromBody(w, r)
	if !ok {
		return
	}
	if !s.ensureTeamExistence(w, r, team.ID, false) {
		return
	}
	ctx, cancel := context.WithTimeout(r.Context(), s.storeTimeout)
	defer cancel()
	err := s.store.CreateTeam(ctx, team)
	if err != nil {
		log.Printf("Could not create team: error: %s", err)
		http.Error(w, "Could not create team (see server log)", http.StatusInternalServerError)
		return
	}
}

func (s *Server) handlePutTeam(w http.ResponseWriter, r *http.Request) {
	team, ok := readTeamFromBody(w, r)
	if !ok {
		return
	}
	if !s.ensureTeamExistence(w, r, team.ID, true) {
		return
	}
	ctx, cancel := context.WithTimeout(r.Context(), s.storeTimeout)
	defer cancel()
	err := s.store.UpdateTeam(ctx, team)
	if err != nil {
		log.Printf("Could not update team: error: %s", err)
		http.Error(w, "Could not update team (see server log)", http.StatusInternalServerError)
		return
	}
}

func readTeamFromBody(w http.ResponseWriter, r *http.Request) (Team, bool) {
	dec := json.NewDecoder(r.Body)
	team := Team{}
	err := dec.Decode(&team)
	if err != nil {
		http.Error(w, fmt.Sprintf("Could not decode body: %v", err), http.StatusBadRequest)
		return team, false
	}
	return team, true
}

func (s *Server) handlePeriod(w http.ResponseWriter, r *http.Request) {
	pathParts := strings.Split(r.URL.Path, "/")
	log.Printf("Period path: %q", pathParts)
	if len(pathParts) != 5 {
		http.NotFound(w, r)
		return
	}
	teamID := pathParts[3]
	periodID := pathParts[4]
	if r.Method == http.MethodGet {
		s.handleGetPeriod(w, r, teamID, periodID)
	} else if r.Method == http.MethodPost {
		s.handlePostPeriod(w, r, teamID)
	} else if r.Method == http.MethodPut {
		s.handlePutPeriod(w, r, teamID, periodID)
	} else {
		http.Error(w, fmt.Sprintf("Unsupported method '%s'", r.Method), http.StatusBadRequest)
	}
}

func (s *Server) handleGetPeriod(w http.ResponseWriter, r *http.Request, teamID, periodID string) {
	ctx, cancel := context.WithTimeout(r.Context(), s.storeTimeout)
	defer cancel()
	if periodID == "" {
		periods, found, err := s.store.GetAllPeriods(ctx, teamID)
		if err != nil {
			log.Printf("Could not retrieve periods for team '%s': error: %s", teamID, err)
			http.Error(w, fmt.Sprintf("Could not retrieve periods for team '%s' (see server log)", teamID), http.StatusInternalServerError)
			return
		}
		if !found {
			http.NotFound(w, r)
			return
		}
		enc := json.NewEncoder(w)
		w.Header().Set("Content-Type", "application/json")
		enc.Encode(periods)
	} else {
		period, found, err := s.store.GetPeriod(ctx, teamID, periodID)
		if err != nil {
			log.Printf("Could not retrieve period '%s' for team '%s': error: %s", periodID, teamID, err)
			http.Error(w, fmt.Sprintf("Could not retrieve period '%s' for team '%s' (see server log)", periodID, teamID), http.StatusInternalServerError)
			return
		}
		if !found {
			http.NotFound(w, r)
			return
		}
		enc := json.NewEncoder(w)
		w.Header().Set("Content-Type", "application/json")
		enc.Encode(period)
	}
}

func (s *Server) handlePostPeriod(w http.ResponseWriter, r *http.Request, teamID string) {
	period, ok := readPeriodFromBody(w, r)
	if !ok {
		return
	}
	if !s.ensureTeamExistence(w, r, teamID, true) {
		return
	}
	if !s.ensurePeriodExistence(w, r, teamID, period.ID, false) {
		return
	}
	ctx, cancel := context.WithTimeout(r.Context(), s.storeTimeout)
	defer cancel()
	err := s.store.CreatePeriod(ctx, teamID, period)
	if err != nil {
		log.Printf("Could not create period for team '%s': error: %s", teamID, err)
		http.Error(w, fmt.Sprintf("Could not create period for team '%s' (see server log)", teamID), http.StatusInternalServerError)
		return
	}
}

func (s *Server) handlePutPeriod(w http.ResponseWriter, r *http.Request, teamID, periodID string) {
	period, ok := readPeriodFromBody(w, r)
	if !ok {
		return
	}
	if !s.ensureTeamExistence(w, r, teamID, true) {
		return
	}
	if !s.ensurePeriodExistence(w, r, teamID, periodID, true) {
		return
	}
	ctx, cancel := context.WithTimeout(r.Context(), s.storeTimeout)
	defer cancel()
	err := s.store.UpdatePeriod(ctx, teamID, period)
	if err != nil {
		log.Printf("Could not update period '%s' for team '%s': error: %s", periodID, teamID, err)
		http.Error(w, fmt.Sprintf("Could not update period '%s' for team '%s' (see server log)", periodID, teamID), http.StatusInternalServerError)
		return
	}
}

func readPeriodFromBody(w http.ResponseWriter, r *http.Request) (Period, bool) {
	dec := json.NewDecoder(r.Body)
	period := Period{}
	err := dec.Decode(&period)
	if err != nil {
		http.Error(w, fmt.Sprintf("Could not decode body: %v", err), http.StatusBadRequest)
		return period, false
	}
	return period, true
}

func main() {
	var useInMemStore bool
	flag.BoolVar(&useInMemStore, "inmemstore", false, "Use in-memory datastore")
	flag.Parse()

	var store StorageService
	if useInMemStore {
		log.Printf("Using in-memory store per command-line flag")
		store = makeInMemStore()
	} else {
		gcloudProject := os.Getenv("GOOGLE_CLOUD_PROJECT")
		if gcloudProject == "" {
			log.Fatalf("GOOGLE_CLOUD_PROJECT not set")
			return
		}
		log.Printf("Using Cloud Datastore storage service; project='%s'", gcloudProject)
		log.Printf("To use the local emulator, see https://cloud.google.com/datastore/docs/tools/datastore-emulator")
		ctx := context.Background()
		var err error
		store, err = makeGoogleCDSStore(ctx, gcloudProject)
		if err != nil {
			log.Fatalf("Could not instantiate datastore: %s", err)
			return
		}
	}
	server := Server{store: store, storeTimeout: defaultStoreTimeout}
	handler := server.makeHandler()
	port := os.Getenv("PORT")
	if port == "" {
		port = "8080"
		log.Printf("Defaulting to port %s", port)
	}
	log.Printf("Listening on port %s", port)
	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", port), handler))
}
<|endoftext|>


<commit_before>package wishful

type Applicative interface {
	Of(v AnyVal) Applicative
	Ap(v Applicative) Applicative
}

func (o Laws) Identity() {
	f := func(v AnyVal) AnyVal {
		return o.Of(Identity).Ap(o.Of(v))
	}
	g := func(v AnyVal) AnyVal {
		return o.Of(v)
	}
	return f, g
}

func (o Laws) Composition() {
	f := func(v AnyVal) AnyVal {
		return o.Of(Compose).Ap(o.Of(Identity)).Ap(o.Of(Identity)).Ap(o.Of(v))
	}
	g := func(v AnyVal) AnyVal {
		return o.Of(Identity).Ap(o.Of(Identity).Ap(o.Of(v)))
	}
	return f, g
}

func (o Laws) Homomorphism() {
	f := func(v AnyVal) AnyVal {
		return o.Of(Identity).Ap(o.Of(v))
	}
	g := func(v AnyVal) AnyVal {
		return o.Of(Identity(v))
	}
	return f, g
}

func (o Laws) Interchange() {
	f := func(v AnyVal) AnyVal {
		return o.Of(Identity).Ap(o.Of(v))
	}
	g := func(v AnyVal) AnyVal {
		return o.Of(Thrush(v)).Ap(o.Of(Identity))
	}
	return f, g
}
<commit_msg>Adding applicative struct for the laws<commit_after>package wishful

type Applicative interface {
	Of(v AnyVal) Applicative
	Ap(v Applicative) Applicative
}

type Applicative struct{}

func (o Applicative) Identity() {
	f := func(v AnyVal) AnyVal {
		return o.Of(Identity).Ap(o.Of(v))
	}
	g := func(v AnyVal) AnyVal {
		return o.Of(v)
	}
	return f, g
}

func (o Applicative) Composition() {
	f := func(v AnyVal) AnyVal {
		return o.Of(Compose).Ap(o.Of(Identity)).Ap(o.Of(Identity)).Ap(o.Of(v))
	}
	g := func(v AnyVal) AnyVal {
		return o.Of(Identity).Ap(o.Of(Identity).Ap(o.Of(v)))
	}
	return f, g
}

func (o Applicative) Homomorphism() {
	f := func(v AnyVal) AnyVal {
		return o.Of(Identity).Ap(o.Of(v))
	}
	g := func(v AnyVal) AnyVal {
		return o.Of(Identity(v))
	}
	return f, g
}

func (o Applicative) Interchange() {
	f := func(v AnyVal) AnyVal {
		return o.Of(Identity).Ap(o.Of(v))
	}
	g := func(v AnyVal) AnyVal {
		return o.Of(Thrush(v)).Ap(o.Of(Identity))
	}
	return f, g
}
<|endoftext|>


<commit_before>package wood

import (
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestParseHumanSize(t *testing.T) {
	assert.Equal(t, int64(50*1000), parseHumanSize("50K"))
	assert.Equal(t, int64(5*1000*1000), parseHumanSize("5M"))
	assert.Equal(t, int64(100*1000*1000*1000), parseHumanSize("100G"))

	for _, str := range []string{"", "1", "K", "10", "KM"} {
		assert.Panics(t, func() {
			parseHumanSize(str)
		})
	}
}

func TestProtectorBodyOverflow(t *testing.T) {
	p := DefaultProtector()
	e := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		buf, err := ioutil.ReadAll(r.Body)
		assert.Equal(t, 4000, len(buf))
		assert.Error(t, err)

		w.WriteHeader(http.StatusContinue)
		w.Write([]byte("OK"))
	})
	h := p(e)

	r, err := http.NewRequest("GET", "/foo", randomReader(4001))
	assert.NoError(t, err)

	w := httptest.NewRecorder()

	h.ServeHTTP(w, r)
	assert.Equal(t, http.StatusContinue, w.Code)
	assert.Equal(t, "OK", w.Body.String())
}

func TestProtectorCORS(t *testing.T) {
	p := DefaultProtector()
	e := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusContinue)
		w.Write([]byte("OK"))
	})
	h := p(e)

	r, err := http.NewRequest("OPTIONS", "/foo", nil)
	assert.NoError(t, err)

	w := httptest.NewRecorder()

	h.ServeHTTP(w, r)
	assert.Equal(t, http.StatusOK, w.Code)
	assert.Equal(t, http.Header{
		"Vary": []string{
			"Origin",
			"Access-Control-Request-Method",
			"Access-Control-Request-Headers",
		},
	}, w.HeaderMap)
	assert.Empty(t, w.Body.String())
}
<commit_msg>fixed tests<commit_after>package wood

import (
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestParseHumanSize(t *testing.T) {
	assert.Equal(t, int64(50*1000), parseHumanSize("50K"))
	assert.Equal(t, int64(5*1000*1000), parseHumanSize("5M"))
	assert.Equal(t, int64(100*1000*1000*1000), parseHumanSize("100G"))

	for _, str := range []string{"", "1", "K", "10", "KM"} {
		assert.Panics(t, func() {
			parseHumanSize(str)
		})
	}
}

func TestProtectorBodyOverflow(t *testing.T) {
	p := DefaultProtector()
	e := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		buf, err := ioutil.ReadAll(r.Body)
		assert.Equal(t, 8000000, len(buf))
		assert.Error(t, err)

		w.WriteHeader(http.StatusContinue)
		w.Write([]byte("OK"))
	})
	h := p(e)

	r, err := http.NewRequest("GET", "/foo", randomReader(8000001))
	assert.NoError(t, err)

	w := httptest.NewRecorder()

	h.ServeHTTP(w, r)
	assert.Equal(t, http.StatusContinue, w.Code)
	assert.Equal(t, "OK", w.Body.String())
}

func TestProtectorCORS(t *testing.T) {
	p := DefaultProtector()
	e := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusContinue)
		w.Write([]byte("OK"))
	})
	h := p(e)

	r, err := http.NewRequest("OPTIONS", "/foo", nil)
	assert.NoError(t, err)

	w := httptest.NewRecorder()

	h.ServeHTTP(w, r)
	assert.Equal(t, http.StatusOK, w.Code)
	assert.Equal(t, http.Header{
		"Vary": []string{
			"Origin",
			"Access-Control-Request-Method",
			"Access-Control-Request-Headers",
		},
	}, w.HeaderMap)
	assert.Empty(t, w.Body.String())
}
<|endoftext|>


<commit_before>package datastore

import "fmt"

type LockError struct {
	Item string
}

func (e *LockError) Error() string {
	return fmt.Sprintf("%s is locked", e.Item)
}

type UpdatePreemptionError struct {
	OldSerial int64
	NewSerial int64
}

func (e *UpdatePreemptionError) Error() string {
	return fmt.Sprintf("%d subsequent update(s) have occurred", e.NewSerial-e.OldSerial)
}

type ValidationError struct {
	ErrorString string
}

func (e *ValidationError) Error() string {
	return e.ErrorString
}
<commit_msg>Removed datastore.LockError.<commit_after>package datastore

import "fmt"

type UpdatePreemptionError struct {
	OldSerial int64
	NewSerial int64
}

func (e *UpdatePreemptionError) Error() string {
	return fmt.Sprintf("%d subsequent update(s) have occurred", e.NewSerial-e.OldSerial)
}

type ValidationError struct {
	ErrorString string
}

func (e *ValidationError) Error() string {
	return e.ErrorString
}
<|endoftext|>


<commit_before>package datatype

/*
#cgo pkg-config: libqrencode
#include <stdlib.h>
#include <qrencode.h>
*/
import "C"

import "unsafe"

// QRcode struct represents a QR Code
type QRcode struct {
	version, width int
	data           []byte
}

// CreateQRCode creates a Go QR Code from a C QR Code
func CreateQRCode(cqrcode *C.QRcode) *QRcode {
	version := int(cqrcode.version)
	width := int(cqrcode.width)
	data := C.GoBytes(unsafe.Pointer(cqrcode.data), C.int(width*width))

	return &QRcode{version: version, width: width, data: data}
}
<commit_msg>Make QRCode fields public<commit_after>package datatype

/*
#cgo pkg-config: libqrencode
#include <stdlib.h>
#include <qrencode.h>
*/
import "C"

import "unsafe"

// QRcode struct represents a QR Code
type QRcode struct {
	Version, Width int
	Data           []byte
}

// CreateQRCode creates a Go QR Code from a C QR Code
func CreateQRCode(cqrcode *C.QRcode) *QRcode {
	if cqrcode == nil {
		return nil
	}

	version := int(cqrcode.version)
	width := int(cqrcode.width)
	data := C.GoBytes(unsafe.Pointer(cqrcode.data), C.int(width*width))

	return &QRcode{Version: version, Width: width, Data: data}
}
<|endoftext|>


<commit_before>package decor

import (
	"fmt"
	"io"
	"math"
	"strconv"
	"strings"
)

const (
	_ib   SizeB1024 = iota + 1
	_iKiB SizeB1024 = 1 << (iota * 10)
	_iMiB
	_iGiB
	_iTiB
)

//go:generate stringer -type=SizeB1024 -trimprefix=_i
type SizeB1024 int64

func (self SizeB1024) Format(st fmt.State, verb rune) {
	var prec int
	switch verb {
	case 'd':
	case 's':
		prec = -1
	default:
		if p, ok := st.Precision(); ok {
			prec = p
		} else {
			prec = 6
		}
	}

	var unit SizeB1024
	switch {
	case self < _iKiB:
		unit = _ib
	case self < _iMiB:
		unit = _iKiB
	case self < _iGiB:
		unit = _iMiB
	case self < _iTiB:
		unit = _iGiB
	case self <= math.MaxInt64:
		unit = _iTiB
	}

	var b strings.Builder
	b.WriteString(strconv.FormatFloat(float64(self)/float64(unit), 'f', prec, 64))

	if st.Flag(' ') {
		b.WriteString(" ")
	}
	b.WriteString(unit.String())

	if w, ok := st.Width(); ok {
		if l := b.Len(); l < w {
			pad := strings.Repeat(" ", w-l)
			if st.Flag('-') {
				b.WriteString(pad)
			} else {
				tmp := b.String()
				b.Reset()
				b.WriteString(pad)
				b.WriteString(tmp)
			}
		}
	}

	io.WriteString(st, b.String())
}

const (
	_b  SizeB1000 = 1
	_KB SizeB1000 = 1000
	_MB SizeB1000 = _KB * 1000
	_GB SizeB1000 = _MB * 1000
	_TB SizeB1000 = _GB * 1000
)

//go:generate stringer -type=SizeB1000 -trimprefix=_
type SizeB1000 int64

func (self SizeB1000) Format(st fmt.State, verb rune) {
	var prec int
	switch verb {
	case 'd':
	case 's':
		prec = -1
	default:
		if p, ok := st.Precision(); ok {
			prec = p
		} else {
			prec = 6
		}
	}

	var unit SizeB1000
	switch {
	case self < _KB:
		unit = _b
	case self < _MB:
		unit = _KB
	case self < _GB:
		unit = _MB
	case self < _TB:
		unit = _GB
	case self <= math.MaxInt64:
		unit = _TB
	}

	var b strings.Builder
	b.WriteString(strconv.FormatFloat(float64(self)/float64(unit), 'f', prec, 64))

	if st.Flag(' ') {
		b.WriteString(" ")
	}
	b.WriteString(unit.String())

	if w, ok := st.Width(); ok {
		if l := b.Len(); l < w {
			pad := strings.Repeat(" ", w-l)
			if st.Flag('-') {
				b.WriteString(pad)
			} else {
				tmp := b.String()
				b.Reset()
				b.WriteString(pad)
				b.WriteString(tmp)
			}
		}
	}

	io.WriteString(st, b.String())
}
<commit_msg>for consistency sake<commit_after>package decor

import (
	"fmt"
	"io"
	"math"
	"strconv"
	"strings"
)

const (
	_ib   SizeB1024 = iota + 1
	_iKiB SizeB1024 = 1 << (iota * 10)
	_iMiB
	_iGiB
	_iTiB
)

//go:generate stringer -type=SizeB1024 -trimprefix=_i
type SizeB1024 int64

func (self SizeB1024) Format(st fmt.State, verb rune) {
	var prec int
	switch verb {
	case 'd':
	case 's':
		prec = -1
	default:
		if p, ok := st.Precision(); ok {
			prec = p
		} else {
			prec = 6
		}
	}

	var unit SizeB1024
	switch {
	case self < _iKiB:
		unit = _ib
	case self < _iMiB:
		unit = _iKiB
	case self < _iGiB:
		unit = _iMiB
	case self < _iTiB:
		unit = _iGiB
	case self <= math.MaxInt64:
		unit = _iTiB
	}

	var b strings.Builder
	b.WriteString(strconv.FormatFloat(float64(self)/float64(unit), 'f', prec, 64))

	if st.Flag(' ') {
		b.WriteString(" ")
	}
	b.WriteString(unit.String())

	if w, ok := st.Width(); ok {
		if l := b.Len(); l < w {
			pad := strings.Repeat(" ", w-l)
			if st.Flag('-') {
				b.WriteString(pad)
			} else {
				tmp := b.String()
				b.Reset()
				b.WriteString(pad)
				b.WriteString(tmp)
			}
		}
	}

	io.WriteString(st, b.String())
}

const (
	_b  SizeB1000 = 1
	_KB SizeB1000 = _b * 1000
	_MB SizeB1000 = _KB * 1000
	_GB SizeB1000 = _MB * 1000
	_TB SizeB1000 = _GB * 1000
)

//go:generate stringer -type=SizeB1000 -trimprefix=_
type SizeB1000 int64

func (self SizeB1000) Format(st fmt.State, verb rune) {
	var prec int
	switch verb {
	case 'd':
	case 's':
		prec = -1
	default:
		if p, ok := st.Precision(); ok {
			prec = p
		} else {
			prec = 6
		}
	}

	var unit SizeB1000
	switch {
	case self < _KB:
		unit = _b
	case self < _MB:
		unit = _KB
	case self < _GB:
		unit = _MB
	case self < _TB:
		unit = _GB
	case self <= math.MaxInt64:
		unit = _TB
	}

	var b strings.Builder
	b.WriteString(strconv.FormatFloat(float64(self)/float64(unit), 'f', prec, 64))

	if st.Flag(' ') {
		b.WriteString(" ")
	}
	b.WriteString(unit.String())

	if w, ok := st.Width(); ok {
		if l := b.Len(); l < w {
			pad := strings.Repeat(" ", w-l)
			if st.Flag('-') {
				b.WriteString(pad)
			} else {
				tmp := b.String()
				b.Reset()
				b.WriteString(pad)
				b.WriteString(tmp)
			}
		}
	}

	io.WriteString(st, b.String())
}
<|endoftext|>