<commit_before>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package rafthttp

import (
	"net/http"
	"sync"
	"time"

	"github.com/coreos/etcd/etcdserver/stats"
	"github.com/coreos/etcd/pkg/logutil"
	"github.com/coreos/etcd/pkg/transport"
	"github.com/coreos/etcd/pkg/types"
	"github.com/coreos/etcd/raft"
	"github.com/coreos/etcd/raft/raftpb"
	"github.com/coreos/etcd/snap"
	"github.com/coreos/pkg/capnslog"
	"github.com/xiang90/probing"
	"golang.org/x/net/context"
)

var plog = logutil.NewMergeLogger(capnslog.NewPackageLogger("github.com/coreos/etcd", "rafthttp"))

type Raft interface {
	Process(ctx context.Context, m raftpb.Message) error
	IsIDRemoved(id uint64) bool
	ReportUnreachable(id uint64)
	ReportSnapshot(id uint64, status raft.SnapshotStatus)
}

type Transporter interface {
	// Start starts the given Transporter.
	// Start MUST be called before calling other functions in the interface.
	Start() error
	// Handler returns the HTTP handler of the transporter.
	// A transporter HTTP handler handles the HTTP requests
	// from remote peers.
	// The handler MUST be used to handle RaftPrefix(/raft)
	// endpoint.
	Handler() http.Handler
	// Send sends out the given messages to the remote peers.
	// Each message has a To field, which is an id that maps
	// to an existing peer in the transport.
	// If the id cannot be found in the transport, the message
	// will be ignored.
	Send(m []raftpb.Message)
	// SendSnapshot sends out the given snapshot message to a remote peer.
	// The behavior of SendSnapshot is similar to Send.
	SendSnapshot(m snap.Message)
	// AddRemote adds a remote with given peer urls into the transport.
	// A remote helps newly joined member to catch up the progress of cluster,
	// and will not be used after that.
	// It is the caller's responsibility to ensure the urls are all valid,
	// or it panics.
	AddRemote(id types.ID, urls []string)
	// AddPeer adds a peer with given peer urls into the transport.
	// It is the caller's responsibility to ensure the urls are all valid,
	// or it panics.
	// Peer urls are used to connect to the remote peer.
	AddPeer(id types.ID, urls []string)
	// RemovePeer removes the peer with given id.
	RemovePeer(id types.ID)
	// RemoveAllPeers removes all the existing peers in the transport.
	RemoveAllPeers()
	// UpdatePeer updates the peer urls of the peer with the given id.
	// It is the caller's responsibility to ensure the urls are all valid,
	// or it panics.
	UpdatePeer(id types.ID, urls []string)
	// ActiveSince returns the time that the connection with the peer
	// of the given id becomes active.
	// If the connection is active since peer was added, it returns the adding time.
	// If the connection is currently inactive, it returns zero time.
	ActiveSince(id types.ID) time.Time
	// Stop closes the connections and stops the transporter.
	Stop()
}

// Transport implements Transporter interface. It provides the functionality
// to send raft messages to peers, and receive raft messages from peers.
// User should call Handler method to get a handler to serve requests
// received from peerURLs.
// User needs to call Start before calling other functions, and call
// Stop when the Transport is no longer used.
type Transport struct {
	DialTimeout time.Duration     // maximum duration before timing out dial of the request
	TLSInfo     transport.TLSInfo // TLS information used when creating connection

	ID          types.ID   // local member ID
	URLs        types.URLs // local peer URLs
	ClusterID   types.ID   // raft cluster ID for request validation
	Raft        Raft       // raft state machine, to which the Transport forwards received messages and reports status
	Snapshotter *snap.Snapshotter
	ServerStats *stats.ServerStats // used to record general transportation statistics
	// used to record transportation statistics with followers when
	// performing as leader in raft protocol
	LeaderStats *stats.LeaderStats
	// ErrorC is used to report detected critical errors, e.g.,
	// the member has been permanently removed from the cluster
	// When an error is received from ErrorC, user should stop raft state
	// machine and thus stop the Transport.
	ErrorC chan error

	streamRt   http.RoundTripper // roundTripper used by streams
	pipelineRt http.RoundTripper // roundTripper used by pipelines

	mu      sync.RWMutex         // protect the remote and peer map
	remotes map[types.ID]*remote // remotes map that helps newly joined member to catch up
	peers   map[types.ID]Peer    // peers map

	prober probing.Prober
}

func (t *Transport) Start() error {
	var err error
	t.streamRt, err = newStreamRoundTripper(t.TLSInfo, t.DialTimeout)
	if err != nil {
		return err
	}
	t.pipelineRt, err = NewRoundTripper(t.TLSInfo, t.DialTimeout)
	if err != nil {
		return err
	}
	t.remotes = make(map[types.ID]*remote)
	t.peers = make(map[types.ID]Peer)
	t.prober = probing.NewProber(t.pipelineRt)
	return nil
}

func (t *Transport) Handler() http.Handler {
	pipelineHandler := newPipelineHandler(t, t.Raft, t.ClusterID)
	streamHandler := newStreamHandler(t, t, t.Raft, t.ID, t.ClusterID)
	snapHandler := newSnapshotHandler(t, t.Raft, t.Snapshotter, t.ClusterID)
	mux := http.NewServeMux()
	mux.Handle(RaftPrefix, pipelineHandler)
	mux.Handle(RaftStreamPrefix+"/", streamHandler)
	mux.Handle(RaftSnapshotPrefix, snapHandler)
	mux.Handle(ProbingPrefix, probing.NewHandler())
	return mux
}

func (t *Transport) Get(id types.ID) Peer {
	t.mu.RLock()
	defer t.mu.RUnlock()
	return t.peers[id]
}

func (t *Transport) Send(msgs []raftpb.Message) {
	for _, m := range msgs {
		if m.To == 0 {
			// ignore intentionally dropped message
			continue
		}
		to := types.ID(m.To)

		t.mu.RLock()
		p, pok := t.peers[to]
		g, rok := t.remotes[to]
		t.mu.RUnlock()

		if pok {
			if m.Type == raftpb.MsgApp {
				t.ServerStats.SendAppendReq(m.Size())
			}
			p.send(m)
			continue
		}

		if rok {
			g.send(m)
			continue
		}

		plog.Debugf("ignored message %s (sent to unknown peer %s)", m.Type, to)
	}
}

func (t *Transport) Stop() {
	t.mu.Lock()
	defer t.mu.Unlock()
	for _, r := range t.remotes {
		r.stop()
	}
	for _, p := range t.peers {
		p.stop()
	}
	t.prober.RemoveAll()
	if tr, ok := t.streamRt.(*http.Transport); ok {
		tr.CloseIdleConnections()
	}
	if tr, ok := t.pipelineRt.(*http.Transport); ok {
		tr.CloseIdleConnections()
	}
}

func (t *Transport) AddRemote(id types.ID, us []string) {
	t.mu.Lock()
	defer t.mu.Unlock()
	if _, ok := t.peers[id]; ok {
		return
	}
	if _, ok := t.remotes[id]; ok {
		return
	}
	urls, err := types.NewURLs(us)
	if err != nil {
		plog.Panicf("newURLs %+v should never fail: %+v", us, err)
	}
	t.remotes[id] = startRemote(t, urls, t.ID, id, t.ClusterID, t.Raft, t.ErrorC)
}

func (t *Transport) AddPeer(id types.ID, us []string) {
	t.mu.Lock()
	defer t.mu.Unlock()
	if _, ok := t.peers[id]; ok {
		return
	}
	urls, err := types.NewURLs(us)
	if err != nil {
		plog.Panicf("newURLs %+v should never fail: %+v", us, err)
	}
	fs := t.LeaderStats.Follower(id.String())
	t.peers[id] = startPeer(t, urls, t.ID, id, t.ClusterID, t.Raft, fs, t.ErrorC)
	addPeerToProber(t.prober, id.String(), us)
}

func (t *Transport) RemovePeer(id types.ID) {
	t.mu.Lock()
	defer t.mu.Unlock()
	t.removePeer(id)
}

func (t *Transport) RemoveAllPeers() {
	t.mu.Lock()
	defer t.mu.Unlock()
	for id := range t.peers {
		t.removePeer(id)
	}
}

// the caller of this function must have the peers mutex.
func (t *Transport) removePeer(id types.ID) {
	if peer, ok := t.peers[id]; ok {
		peer.stop()
	} else {
		plog.Panicf("unexpected removal of unknown peer '%d'", id)
	}
	delete(t.peers, id)
	delete(t.LeaderStats.Followers, id.String())
	t.prober.Remove(id.String())
}

func (t *Transport) UpdatePeer(id types.ID, us []string) {
	t.mu.Lock()
	defer t.mu.Unlock()
	// TODO: return error or just panic?
	if _, ok := t.peers[id]; !ok {
		return
	}
	urls, err := types.NewURLs(us)
	if err != nil {
		plog.Panicf("newURLs %+v should never fail: %+v", us, err)
	}
	t.peers[id].update(urls)

	t.prober.Remove(id.String())
	addPeerToProber(t.prober, id.String(), us)
}

func (t *Transport) ActiveSince(id types.ID) time.Time {
	t.mu.Lock()
	defer t.mu.Unlock()
	if p, ok := t.peers[id]; ok {
		return p.activeSince()
	}
	return time.Time{}
}

func (t *Transport) SendSnapshot(m snap.Message) {
	t.mu.Lock()
	defer t.mu.Unlock()
	p := t.peers[types.ID(m.To)]
	if p == nil {
		m.CloseWithError(errMemberNotFound)
		return
	}
	p.sendSnap(m)
}

// Pausable is a testing interface for pausing transport traffic.
type Pausable interface {
	Pause()
	Resume()
}

func (t *Transport) Pause() {
	for _, p := range t.peers {
		p.(Pausable).Pause()
	}
}

func (t *Transport) Resume() {
	for _, p := range t.peers {
		p.(Pausable).Resume()
	}
}

type nopTransporter struct{}

func NewNopTransporter() Transporter {
	return &nopTransporter{}
}

func (s *nopTransporter) Start() error                        { return nil }
func (s *nopTransporter) Handler() http.Handler               { return nil }
func (s *nopTransporter) Send(m []raftpb.Message)             {}
func (s *nopTransporter) SendSnapshot(m snap.Message)         {}
func (s *nopTransporter) AddRemote(id types.ID, us []string)  {}
func (s *nopTransporter) AddPeer(id types.ID, us []string)    {}
func (s *nopTransporter) RemovePeer(id types.ID)              {}
func (s *nopTransporter) RemoveAllPeers()                     {}
func (s *nopTransporter) UpdatePeer(id types.ID, us []string) {}
func (s *nopTransporter) ActiveSince(id types.ID) time.Time   { return time.Time{} }
func (s *nopTransporter) Stop()                               {}
func (s *nopTransporter) Pause()                              {}
func (s *nopTransporter) Resume()                             {}

type snapTransporter struct {
	nopTransporter
	snapDoneC chan snap.Message
	snapDir   string
}

func NewSnapTransporter(snapDir string) (Transporter, <-chan snap.Message) {
	ch := make(chan snap.Message, 1)
	tr := &snapTransporter{snapDoneC: ch, snapDir: snapDir}
	return tr, ch
}

func (s *snapTransporter) SendSnapshot(m snap.Message) {
	ss := snap.New(s.snapDir)
	ss.SaveDBFrom(m.ReadCloser, m.Snapshot.Metadata.Index+1)
	m.CloseWithError(nil)
	s.snapDoneC <- m
}
<commit_msg>rafthttp: do not create new connections after stopping transport<commit_after>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package rafthttp

import (
	"net/http"
	"sync"
	"time"

	"github.com/coreos/etcd/etcdserver/stats"
	"github.com/coreos/etcd/pkg/logutil"
	"github.com/coreos/etcd/pkg/transport"
	"github.com/coreos/etcd/pkg/types"
	"github.com/coreos/etcd/raft"
	"github.com/coreos/etcd/raft/raftpb"
	"github.com/coreos/etcd/snap"
	"github.com/coreos/pkg/capnslog"
	"github.com/xiang90/probing"
	"golang.org/x/net/context"
)

var plog = logutil.NewMergeLogger(capnslog.NewPackageLogger("github.com/coreos/etcd", "rafthttp"))

type Raft interface {
	Process(ctx context.Context, m raftpb.Message) error
	IsIDRemoved(id uint64) bool
	ReportUnreachable(id uint64)
	ReportSnapshot(id uint64, status raft.SnapshotStatus)
}

type Transporter interface {
	// Start starts the given Transporter.
	// Start MUST be called before calling other functions in the interface.
	Start() error
	// Handler returns the HTTP handler of the transporter.
	// A transporter HTTP handler handles the HTTP requests
	// from remote peers.
	// The handler MUST be used to handle RaftPrefix(/raft)
	// endpoint.
	Handler() http.Handler
	// Send sends out the given messages to the remote peers.
	// Each message has a To field, which is an id that maps
	// to an existing peer in the transport.
	// If the id cannot be found in the transport, the message
	// will be ignored.
	Send(m []raftpb.Message)
	// SendSnapshot sends out the given snapshot message to a remote peer.
	// The behavior of SendSnapshot is similar to Send.
	SendSnapshot(m snap.Message)
	// AddRemote adds a remote with given peer urls into the transport.
	// A remote helps newly joined member to catch up the progress of cluster,
	// and will not be used after that.
	// It is the caller's responsibility to ensure the urls are all valid,
	// or it panics.
	AddRemote(id types.ID, urls []string)
	// AddPeer adds a peer with given peer urls into the transport.
	// It is the caller's responsibility to ensure the urls are all valid,
	// or it panics.
	// Peer urls are used to connect to the remote peer.
	AddPeer(id types.ID, urls []string)
	// RemovePeer removes the peer with given id.
	RemovePeer(id types.ID)
	// RemoveAllPeers removes all the existing peers in the transport.
	RemoveAllPeers()
	// UpdatePeer updates the peer urls of the peer with the given id.
	// It is the caller's responsibility to ensure the urls are all valid,
	// or it panics.
	UpdatePeer(id types.ID, urls []string)
	// ActiveSince returns the time that the connection with the peer
	// of the given id becomes active.
	// If the connection is active since peer was added, it returns the adding time.
	// If the connection is currently inactive, it returns zero time.
	ActiveSince(id types.ID) time.Time
	// Stop closes the connections and stops the transporter.
	Stop()
}

// Transport implements Transporter interface. It provides the functionality
// to send raft messages to peers, and receive raft messages from peers.
// User should call Handler method to get a handler to serve requests
// received from peerURLs.
// User needs to call Start before calling other functions, and call
// Stop when the Transport is no longer used.
type Transport struct {
	DialTimeout time.Duration     // maximum duration before timing out dial of the request
	TLSInfo     transport.TLSInfo // TLS information used when creating connection

	ID          types.ID   // local member ID
	URLs        types.URLs // local peer URLs
	ClusterID   types.ID   // raft cluster ID for request validation
	Raft        Raft       // raft state machine, to which the Transport forwards received messages and reports status
	Snapshotter *snap.Snapshotter
	ServerStats *stats.ServerStats // used to record general transportation statistics
	// used to record transportation statistics with followers when
	// performing as leader in raft protocol
	LeaderStats *stats.LeaderStats
	// ErrorC is used to report detected critical errors, e.g.,
	// the member has been permanently removed from the cluster
	// When an error is received from ErrorC, user should stop raft state
	// machine and thus stop the Transport.
	ErrorC chan error

	streamRt   http.RoundTripper // roundTripper used by streams
	pipelineRt http.RoundTripper // roundTripper used by pipelines

	mu      sync.RWMutex         // protect the remote and peer map
	remotes map[types.ID]*remote // remotes map that helps newly joined member to catch up
	peers   map[types.ID]Peer    // peers map

	prober probing.Prober
}

func (t *Transport) Start() error {
	var err error
	t.streamRt, err = newStreamRoundTripper(t.TLSInfo, t.DialTimeout)
	if err != nil {
		return err
	}
	t.pipelineRt, err = NewRoundTripper(t.TLSInfo, t.DialTimeout)
	if err != nil {
		return err
	}
	t.remotes = make(map[types.ID]*remote)
	t.peers = make(map[types.ID]Peer)
	t.prober = probing.NewProber(t.pipelineRt)
	return nil
}

func (t *Transport) Handler() http.Handler {
	pipelineHandler := newPipelineHandler(t, t.Raft, t.ClusterID)
	streamHandler := newStreamHandler(t, t, t.Raft, t.ID, t.ClusterID)
	snapHandler := newSnapshotHandler(t, t.Raft, t.Snapshotter, t.ClusterID)
	mux := http.NewServeMux()
	mux.Handle(RaftPrefix, pipelineHandler)
	mux.Handle(RaftStreamPrefix+"/", streamHandler)
	mux.Handle(RaftSnapshotPrefix, snapHandler)
	mux.Handle(ProbingPrefix, probing.NewHandler())
	return mux
}

func (t *Transport) Get(id types.ID) Peer {
	t.mu.RLock()
	defer t.mu.RUnlock()
	return t.peers[id]
}

func (t *Transport) Send(msgs []raftpb.Message) {
	for _, m := range msgs {
		if m.To == 0 {
			// ignore intentionally dropped message
			continue
		}
		to := types.ID(m.To)

		t.mu.RLock()
		p, pok := t.peers[to]
		g, rok := t.remotes[to]
		t.mu.RUnlock()

		if pok {
			if m.Type == raftpb.MsgApp {
				t.ServerStats.SendAppendReq(m.Size())
			}
			p.send(m)
			continue
		}

		if rok {
			g.send(m)
			continue
		}

		plog.Debugf("ignored message %s (sent to unknown peer %s)", m.Type, to)
	}
}

func (t *Transport) Stop() {
	t.mu.Lock()
	defer t.mu.Unlock()
	for _, r := range t.remotes {
		r.stop()
	}
	for _, p := range t.peers {
		p.stop()
	}
	t.prober.RemoveAll()
	if tr, ok := t.streamRt.(*http.Transport); ok {
		tr.CloseIdleConnections()
	}
	if tr, ok := t.pipelineRt.(*http.Transport); ok {
		tr.CloseIdleConnections()
	}
	t.peers = nil
	t.remotes = nil
}

func (t *Transport) AddRemote(id types.ID, us []string) {
	t.mu.Lock()
	defer t.mu.Unlock()
	if t.remotes == nil {
		// there's no clean way to shutdown the golang http server
		// (see: https://github.com/golang/go/issues/4674) before
		// stopping the transport; ignore any new connections.
		return
	}
	if _, ok := t.peers[id]; ok {
		return
	}
	if _, ok := t.remotes[id]; ok {
		return
	}
	urls, err := types.NewURLs(us)
	if err != nil {
		plog.Panicf("newURLs %+v should never fail: %+v", us, err)
	}
	t.remotes[id] = startRemote(t, urls, t.ID, id, t.ClusterID, t.Raft, t.ErrorC)
}

func (t *Transport) AddPeer(id types.ID, us []string) {
	t.mu.Lock()
	defer t.mu.Unlock()
	if t.peers == nil {
		panic("transport stopped")
	}
	if _, ok := t.peers[id]; ok {
		return
	}
	urls, err := types.NewURLs(us)
	if err != nil {
		plog.Panicf("newURLs %+v should never fail: %+v", us, err)
	}
	fs := t.LeaderStats.Follower(id.String())
	t.peers[id] = startPeer(t, urls, t.ID, id, t.ClusterID, t.Raft, fs, t.ErrorC)
	addPeerToProber(t.prober, id.String(), us)
}

func (t *Transport) RemovePeer(id types.ID) {
	t.mu.Lock()
	defer t.mu.Unlock()
	t.removePeer(id)
}

func (t *Transport) RemoveAllPeers() {
	t.mu.Lock()
	defer t.mu.Unlock()
	for id := range t.peers {
		t.removePeer(id)
	}
}

// the caller of this function must have the peers mutex.
func (t *Transport) removePeer(id types.ID) {
	if peer, ok := t.peers[id]; ok {
		peer.stop()
	} else {
		plog.Panicf("unexpected removal of unknown peer '%d'", id)
	}
	delete(t.peers, id)
	delete(t.LeaderStats.Followers, id.String())
	t.prober.Remove(id.String())
}

func (t *Transport) UpdatePeer(id types.ID, us []string) {
	t.mu.Lock()
	defer t.mu.Unlock()
	// TODO: return error or just panic?
	if _, ok := t.peers[id]; !ok {
		return
	}
	urls, err := types.NewURLs(us)
	if err != nil {
		plog.Panicf("newURLs %+v should never fail: %+v", us, err)
	}
	t.peers[id].update(urls)

	t.prober.Remove(id.String())
	addPeerToProber(t.prober, id.String(), us)
}

func (t *Transport) ActiveSince(id types.ID) time.Time {
	t.mu.Lock()
	defer t.mu.Unlock()
	if p, ok := t.peers[id]; ok {
		return p.activeSince()
	}
	return time.Time{}
}

func (t *Transport) SendSnapshot(m snap.Message) {
	t.mu.Lock()
	defer t.mu.Unlock()
	p := t.peers[types.ID(m.To)]
	if p == nil {
		m.CloseWithError(errMemberNotFound)
		return
	}
	p.sendSnap(m)
}

// Pausable is a testing interface for pausing transport traffic.
type Pausable interface {
	Pause()
	Resume()
}

func (t *Transport) Pause() {
	for _, p := range t.peers {
		p.(Pausable).Pause()
	}
}

func (t *Transport) Resume() {
	for _, p := range t.peers {
		p.(Pausable).Resume()
	}
}

type nopTransporter struct{}

func NewNopTransporter() Transporter {
	return &nopTransporter{}
}

func (s *nopTransporter) Start() error                        { return nil }
func (s *nopTransporter) Handler() http.Handler               { return nil }
func (s *nopTransporter) Send(m []raftpb.Message)             {}
func (s *nopTransporter) SendSnapshot(m snap.Message)         {}
func (s *nopTransporter) AddRemote(id types.ID, us []string)  {}
func (s *nopTransporter) AddPeer(id types.ID, us []string)    {}
func (s *nopTransporter) RemovePeer(id types.ID)              {}
func (s *nopTransporter) RemoveAllPeers()                     {}
func (s *nopTransporter) UpdatePeer(id types.ID, us []string) {}
func (s *nopTransporter) ActiveSince(id types.ID) time.Time   { return time.Time{} }
func (s *nopTransporter) Stop()                               {}
func (s *nopTransporter) Pause()                              {}
func (s *nopTransporter) Resume()                             {}

type snapTransporter struct {
	nopTransporter
	snapDoneC chan snap.Message
	snapDir   string
}

func NewSnapTransporter(snapDir string) (Transporter, <-chan snap.Message) {
	ch := make(chan snap.Message, 1)
	tr := &snapTransporter{snapDoneC: ch, snapDir: snapDir}
	return tr, ch
}

func (s *snapTransporter) SendSnapshot(m snap.Message) {
	ss := snap.New(s.snapDir)
	ss.SaveDBFrom(m.ReadCloser, m.Snapshot.Metadata.Index+1)
	m.CloseWithError(nil)
	s.snapDoneC <- m
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"strings"
	"regexp"
	"io/ioutil"
)

const DEFAULT_BLOCK_SIZE = 1024

type SearchString struct {
	str string
	caseSensitive bool
}

type FileSearch struct {
	file *os.File
	path string
	buff1 []byte
	buff1Len int
	buff2 []byte
	buff2Len int
	blockSize int
	blocksRead int
	closed bool
	searchStrings []SearchString
	contains []SearchString
	read bool
	content string
}

func newFileSearch(searchStrings []SearchString, path string) *FileSearch {
	max := 0
	for i := 0; i < len(searchStrings); i++ {
		if len(searchStrings[i].str) > max {
			max = len(searchStrings[i].str)
		}
	}

	fs := new(FileSearch)
	fs.path = path
	fs.closed = false
	fs.searchStrings = searchStrings
	fs.blocksRead = 0
	fs.contains = make([]SearchString, 0, 5)
	fs.read = false
	if max > DEFAULT_BLOCK_SIZE {
		fs.blockSize = max
	} else {
		fs.blockSize = DEFAULT_BLOCK_SIZE
	}
	fs.buff1 = make([]byte, fs.blockSize)
	fs.buff2 = make([]byte, fs.blockSize)
	return fs
}

func updateSearch(fs *FileSearch, info os.FileInfo) {
	if fs.closed {
		// this search has been closed, return
		return
	} else if fs.file == nil {
		// the file has not been opened yet, open it
		var err error
		fs.file, err = os.Open(fs.path)
		if err != nil {
			// something went wrong, mark the search as closed
			fs.closed = true
			return
		}
	}

	// read bytes
	var bytesRead int
	var err error
	// alternate between byte buffers
	if fs.blocksRead % 2 == 1 {
		bytesRead, err = fs.file.Read(fs.buff1)
		fs.buff1Len = bytesRead
	} else {
		bytesRead, err = fs.file.Read(fs.buff2)
		fs.buff2Len = bytesRead
	}
	if err != nil {
		fs.closed = true
		return
	} else if int64(fs.blocksRead * fs.blockSize + bytesRead) == info.Size() {
		defer fs.file.Close()
		fs.closed = true
	}

	var sval string
	if fs.blocksRead % 2 == 0 {
		sval = string(fs.buff1[:fs.buff1Len]) + string(fs.buff2[:fs.buff2Len])
	} else {
		sval = string(fs.buff2[:fs.buff2Len]) + string(fs.buff1[:fs.buff1Len])
	}
	svalLower := strings.ToLower(sval)
	fs.blocksRead++

	// check for file containing target strings
	for i := 0; i < len(fs.searchStrings); i++ {
		if fs.searchStrings[i].caseSensitive {
			if strings.Contains(sval, fs.searchStrings[i].str) {
				fs.contains = append(fs.contains, fs.searchStrings[i])
			}
		} else {
			if strings.Contains(svalLower, strings.ToLower(fs.searchStrings[i].str)) {
				fs.contains = append(fs.contains, fs.searchStrings[i])
			}
		}
	}
}

func fileContainsString(fs *FileSearch, info os.FileInfo, target string, caseSensitive bool) bool {
	for !fs.closed && !searchStringExists(fs.contains, SearchString{target, caseSensitive}) {
		updateSearch(fs, info)
	}
	return searchStringExists(fs.contains, SearchString{target, caseSensitive})
}

func searchStringExists(searchSlice []SearchString, searchString SearchString) bool {
	for i := 0; i < len(searchSlice); i++ {
		if searchSlice[i].caseSensitive == searchString.caseSensitive &&
				searchSlice[i].str == searchString.str {
			return true
		}
	}
	return false
}

func fileMatchesString(fs *FileSearch, info os.FileInfo, re *regexp.Regexp) bool {
	if !fs.read {
		// read whole file into memory
		bytes, err := ioutil.ReadFile(fs.path)
		if err != nil {
			return false
		}
		fs.content = string(bytes)
		fs.read = true
	}

	return re.MatchString(fs.content)
}

func fileStartsWithString(path string, str string, caseSensitive bool) bool {
	f, err := os.Open(path)
	if err != nil {
		return false
	}
	defer f.Close()

	buff := make([]byte, len(str))
	_, readErr := f.Read(buff)
	if readErr != nil {
		return false
	}

	startValue := string(buff)
	if !caseSensitive {
		startValue = strings.ToLower(startValue)
	}

	return str == startValue
}

func fileEndsWithString(path string, info os.FileInfo, str string, caseSensitive bool) bool {
	f, err := os.Open(path)
	if err != nil {
		return false
	}
	defer f.Close()

	bsize := len(str)
	buff := make([]byte, bsize)
	cbuff := make([]byte, 1)

	j := bsize - 1
	for i := info.Size() - 1; i >= 0 && j >= 0; i-- {
		_, readErr := f.ReadAt(cbuff, i)
		if readErr != nil {
			return false
		}

		if cbuff[0] >= 32 || j < bsize-1 {
			buff[j] = cbuff[0]
			j--
		}
	}

	endValue := string(buff)
	if !caseSensitive {
		endValue = strings.ToLower(endValue)
	}

	return str == endValue
}
<commit_msg>File search optimization.<commit_after>package main

import (
	"os"
	"strings"
	"regexp"
	"io/ioutil"
)

const DEFAULT_BLOCK_SIZE = 1024

type SearchString struct {
	str string
	caseSensitive bool
}

type FileSearch struct {
	file *os.File
	path string
	buff1 []byte
	buff1Len int
	buff2 []byte
	buff2Len int
	blockSize int
	blocksRead int
	closed bool
	searchStrings []SearchString
	contains []SearchString
	read bool
	content string
}

func newFileSearch(searchStrings []SearchString, path string) *FileSearch {
	max := 0
	for i := 0; i < len(searchStrings); i++ {
		if len(searchStrings[i].str) > max {
			max = len(searchStrings[i].str)
		}
	}

	fs := new(FileSearch)
	fs.path = path
	fs.closed = false
	fs.searchStrings = searchStrings
	fs.blocksRead = 0
	fs.contains = make([]SearchString, 0, 5)
	fs.read = false
	if max > DEFAULT_BLOCK_SIZE {
		fs.blockSize = max
	} else {
		fs.blockSize = DEFAULT_BLOCK_SIZE
	}
	fs.buff1 = make([]byte, fs.blockSize)
	fs.buff2 = make([]byte, fs.blockSize)
	return fs
}

func updateSearch(fs *FileSearch, info os.FileInfo) {
	if fs.closed {
		// this search has been closed, return
		return
	} else if fs.file == nil {
		// the file has not been opened yet, open it
		var err error
		fs.file, err = os.Open(fs.path)
		if err != nil {
			// something went wrong, mark the search as closed
			fs.closed = true
			return
		}
	}

	// read bytes
	var bytesRead int
	var err error
	// alternate between byte buffers
	if fs.blocksRead % 2 == 1 {
		bytesRead, err = fs.file.Read(fs.buff1)
		fs.buff1Len = bytesRead
	} else {
		bytesRead, err = fs.file.Read(fs.buff2)
		fs.buff2Len = bytesRead
	}
	if err != nil {
		fs.closed = true
		return
	} else if int64(fs.blocksRead * fs.blockSize + bytesRead) == info.Size() {
		defer fs.file.Close()
		fs.closed = true
	}

	var sval string
	if fs.blocksRead % 2 == 0 {
		sval = string(fs.buff1[:fs.buff1Len]) + string(fs.buff2[:fs.buff2Len])
	} else {
		sval = string(fs.buff2[:fs.buff2Len]) + string(fs.buff1[:fs.buff1Len])
	}
	svalLower := strings.ToLower(sval)
	fs.blocksRead++

	// check for file containing target strings
	for i := 0; i < len(fs.searchStrings); i++ {
		if fs.searchStrings[i].caseSensitive {
			if strings.Contains(sval, fs.searchStrings[i].str) {
				fs.contains = append(fs.contains, fs.searchStrings[i])
			}
		} else {
			if strings.Contains(svalLower, strings.ToLower(fs.searchStrings[i].str)) {
				fs.contains = append(fs.contains, fs.searchStrings[i])
			}
		}
	}
}

func fileContainsString(fs *FileSearch, info os.FileInfo, target string, caseSensitive bool) bool {
	if fs.read {
		// file contents had to be fully read into memory previously, use
		// this instead of reading the file again incrementally
		if caseSensitive {
			return strings.Contains(fs.content, target)
		} else {
			return strings.Contains(strings.ToLower(fs.content), strings.ToLower(target))
		}
	}

	for !fs.closed && !searchStringExists(fs.contains, SearchString{target, caseSensitive}) {
		updateSearch(fs, info)
	}
	return searchStringExists(fs.contains, SearchString{target, caseSensitive})
}

func searchStringExists(searchSlice []SearchString, searchString SearchString) bool {
	for i := 0; i < len(searchSlice); i++ {
		if searchSlice[i].caseSensitive == searchString.caseSensitive &&
				searchSlice[i].str == searchString.str {
			return true
		}
	}
	return false
}

func fileMatchesString(fs *FileSearch, info os.FileInfo, re *regexp.Regexp) bool {
	if !fs.read {
		// read whole file into memory
		bytes, err := ioutil.ReadFile(fs.path)
		if err != nil {
			return false
		}
		fs.content = string(bytes)
		fs.read = true
	}

	return re.MatchString(fs.content)
}

func fileStartsWithString(path string, str string, caseSensitive bool) bool {
	f, err := os.Open(path)
	if err != nil {
		return false
	}
	defer f.Close()

	buff := make([]byte, len(str))
	_, readErr := f.Read(buff)
	if readErr != nil {
		return false
	}

	startValue := string(buff)
	if !caseSensitive {
		startValue = strings.ToLower(startValue)
	}

	return str == startValue
}

func fileEndsWithString(path string, info os.FileInfo, str string, caseSensitive bool) bool {
	f, err := os.Open(path)
	if err != nil {
		return false
	}
	defer f.Close()

	bsize := len(str)
	buff := make([]byte, bsize)
	cbuff := make([]byte, 1)

	j := bsize - 1
	for i := info.Size() - 1; i >= 0 && j >= 0; i-- {
		_, readErr := f.ReadAt(cbuff, i)
		if readErr != nil {
			return false
		}

		if cbuff[0] >= 32 || j < bsize-1 {
			buff[j] = cbuff[0]
			j--
		}
	}

	endValue := string(buff)
	if !caseSensitive {
		endValue = strings.ToLower(endValue)
	}

	return str == endValue
}
<|endoftext|>


<commit_before>/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	"google.golang.org/api/youtube/v3"
)

const ytDateLayout = "2006-01-02T15:04:05.000Z" // ISO 8601 (YYYY-MM-DDThh:mm:ss.sssZ)
const inputDateLayout = "2006-01-02"
const inputDatetimeLayout = "2006-01-02T15:04:05-07:00"

type Date struct {
	time.Time
}

func LoadVideoMeta(filename string, video *youtube.Video) (*VideoMeta, error) {
	videoMeta := &VideoMeta{}
	// attempt to load from meta JSON, otherwise use values specified from command line flags
	if filename != "" {
		file, e := ioutil.ReadFile(filename)
		if e != nil {
			e2 := fmt.Errorf("Error reading file '%s': %s\n", filename, e)
			return nil, e2
		}

		e = json.Unmarshal(file, &videoMeta)
		if e != nil {
			e2 := fmt.Errorf("Error parsing file '%s': %s\n", filename, e)
			return nil, e2
		}

		video.Status = &youtube.VideoStatus{}
		video.Snippet.Tags = videoMeta.Tags
		video.Snippet.Title = videoMeta.Title
		video.Snippet.Description = videoMeta.Description
		video.Snippet.CategoryId = videoMeta.CategoryId
		// Location has been deprecated by Google
		// see: https://developers.google.com/youtube/v3/revision_history#release_notes_06_01_2017
		/*
			if videoMeta.Location != nil {
				video.RecordingDetails.Location = videoMeta.Location
			}
			if videoMeta.LocationDescription != "" {
				video.RecordingDetails.LocationDescription = videoMeta.LocationDescription
			}
		*/
		if !videoMeta.RecordingDate.IsZero() {
			video.RecordingDetails.RecordingDate = videoMeta.RecordingDate.UTC().Format(ytDateLayout)
		}

		// status
		if videoMeta.PrivacyStatus != "" {
			video.Status.PrivacyStatus = videoMeta.PrivacyStatus
		}
		if videoMeta.MadeForKids {
			video.Status.SelfDeclaredMadeForKids = true
		}
		if videoMeta.Embeddable {
			video.Status.Embeddable = true
		}
		if videoMeta.License != "" {
			video.Status.License = videoMeta.License
		}
		if videoMeta.PublicStatsViewable {
			video.Status.PublicStatsViewable = videoMeta.PublicStatsViewable
		}
		if !videoMeta.PublishAt.IsZero() {
			if video.Status.PrivacyStatus != "private" {
				fmt.Printf("publishAt can only be used when privacyStatus is 'private'. Ignoring publishAt...\n")
			} else {
				if videoMeta.PublishAt.Before(time.Now()) {
					fmt.Printf("publishAt (%s) was in the past!? Publishing now instead...\n", videoMeta.PublishAt)
					video.Status.PublishAt = time.Now().UTC().Format(ytDateLayout)
				} else {
					video.Status.PublishAt = videoMeta.PublishAt.UTC().Format(ytDateLayout)
				}
			}
		}

		if videoMeta.Language != "" {
			video.Snippet.DefaultLanguage = videoMeta.Language
			video.Snippet.DefaultAudioLanguage = videoMeta.Language
		}
	}

	if video.Status.PrivacyStatus == "" {
		video.Status.PrivacyStatus = *privacy
	}
	if video.Snippet.Tags == nil && strings.Trim(*tags, "") != "" {
		video.Snippet.Tags = strings.Split(*tags, ",")
	}
	if video.Snippet.Title == "" {
		video.Snippet.Title = *title
	}
	if video.Snippet.Description == "" {
		// expand newlines
		descriptionExpanded, err := strconv.Unquote(`"` + *description + `"`)
		if err != nil {
			video.Snippet.Description = *description
		} else {
			video.Snippet.Description = descriptionExpanded
		}
	}
	if video.Snippet.CategoryId == "" && *categoryId != "" {
		video.Snippet.CategoryId = *categoryId
	}
	if video.Snippet.DefaultLanguage == "" && *language != "" {
		video.Snippet.DefaultLanguage = *language
	}
	if video.Snippet.DefaultAudioLanguage == "" && *language != "" {
		video.Snippet.DefaultAudioLanguage = *language
	}

	return videoMeta, nil
}

func Open(filename string) (io.ReadCloser, int64, error) {
	var reader io.ReadCloser
	var filesize int64
	var err error
	if strings.HasPrefix(filename, "http") {
		var resp *http.Response
		resp, err = http.Head(filename)
		if err != nil {
			return reader, 0, fmt.Errorf("error opening %s: %s", filename, err)
		}
		lenStr := resp.Header.Get("content-length")
		if lenStr != "" {
			filesize, err = strconv.ParseInt(lenStr, 10, 64)
			if err != nil {
				return reader, filesize, err
			}
		}

		resp, err = http.Get(filename)
		if err != nil {
			return reader, 0, fmt.Errorf("error opening %s: %s", filename, err)
		}
		if resp.ContentLength != 0 {
			filesize = resp.ContentLength
		}
		reader = resp.Body
	} else if filename == "-" {
		reader = os.Stdin
	} else {
		var file *os.File
		var fileInfo os.FileInfo
		file, err = os.Open(filename)
		if err != nil {
			return reader, 0, fmt.Errorf("error opening %s: %s", filename, err)
		}

		fileInfo, err = file.Stat()
		if err != nil {
			return reader, 0, fmt.Errorf("error stat'ing %s: %s", filename, err)
		}

		reader = file
		filesize = fileInfo.Size()
	}
	return reader, filesize, err
}

func (d *Date) UnmarshalJSON(b []byte) (err error) {
	s := string(b)
	s = s[1 : len(s)-1]
	// support ISO 8601 date only, and date + time
	if strings.ContainsAny(s, ":") {
		d.Time, err = time.Parse(inputDatetimeLayout, s)
	} else {
		d.Time, err = time.Parse(inputDateLayout, s)
	}
	return
}
<commit_msg>Warn if input file is not a video<commit_after>/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	"google.golang.org/api/youtube/v3"
)

const ytDateLayout = "2006-01-02T15:04:05.000Z" // ISO 8601 (YYYY-MM-DDThh:mm:ss.sssZ)
const inputDateLayout = "2006-01-02"
const inputDatetimeLayout = "2006-01-02T15:04:05-07:00"

type Date struct {
	time.Time
}

func LoadVideoMeta(filename string, video *youtube.Video) (*VideoMeta, error) {
	videoMeta := &VideoMeta{}
	// attempt to load from meta JSON, otherwise use values specified from command line flags
	if filename != "" {
		file, e := ioutil.ReadFile(filename)
		if e != nil {
			e2 := fmt.Errorf("Error reading file '%s': %s\n", filename, e)
			return nil, e2
		}

		e = json.Unmarshal(file, &videoMeta)
		if e != nil {
			e2 := fmt.Errorf("Error parsing file '%s': %s\n", filename, e)
			return nil, e2
		}

		video.Status = &youtube.VideoStatus{}
		video.Snippet.Tags = videoMeta.Tags
		video.Snippet.Title = videoMeta.Title
		video.Snippet.Description = videoMeta.Description
		video.Snippet.CategoryId = videoMeta.CategoryId
		// Location has been deprecated by Google
		// see: https://developers.google.com/youtube/v3/revision_history#release_notes_06_01_2017
		/*
			if videoMeta.Location != nil {
				video.RecordingDetails.Location = videoMeta.Location
			}
			if videoMeta.LocationDescription != "" {
				video.RecordingDetails.LocationDescription = videoMeta.LocationDescription
			}
		*/
		if !videoMeta.RecordingDate.IsZero() {
			video.RecordingDetails.RecordingDate = videoMeta.RecordingDate.UTC().Format(ytDateLayout)
		}

		// status
		if videoMeta.PrivacyStatus != "" {
			video.Status.PrivacyStatus = videoMeta.PrivacyStatus
		}
		if videoMeta.MadeForKids {
			video.Status.SelfDeclaredMadeForKids = true
		}
		if videoMeta.Embeddable {
			video.Status.Embeddable = true
		}
		if videoMeta.License != "" {
			video.Status.License = videoMeta.License
		}
		if videoMeta.PublicStatsViewable {
			video.Status.PublicStatsViewable = videoMeta.PublicStatsViewable
		}
		if !videoMeta.PublishAt.IsZero() {
			if video.Status.PrivacyStatus != "private" {
				fmt.Printf("publishAt can only be used when privacyStatus is 'private'. Ignoring publishAt...\n")
			} else {
				if videoMeta.PublishAt.Before(time.Now()) {
					fmt.Printf("publishAt (%s) was in the past!? Publishing now instead...\n", videoMeta.PublishAt)
					video.Status.PublishAt = time.Now().UTC().Format(ytDateLayout)
				} else {
					video.Status.PublishAt = videoMeta.PublishAt.UTC().Format(ytDateLayout)
				}
			}
		}

		if videoMeta.Language != "" {
			video.Snippet.DefaultLanguage = videoMeta.Language
			video.Snippet.DefaultAudioLanguage = videoMeta.Language
		}
	}

	if video.Status.PrivacyStatus == "" {
		video.Status.PrivacyStatus = *privacy
	}
	if video.Snippet.Tags == nil && strings.Trim(*tags, "") != "" {
		video.Snippet.Tags = strings.Split(*tags, ",")
	}
	if video.Snippet.Title == "" {
		video.Snippet.Title = *title
	}
	if video.Snippet.Description == "" {
		// expand newlines
		descriptionExpanded, err := strconv.Unquote(`"` + *description + `"`)
		if err != nil {
			video.Snippet.Description = *description
		} else {
			video.Snippet.Description = descriptionExpanded
		}
	}
	if video.Snippet.CategoryId == "" && *categoryId != "" {
		video.Snippet.CategoryId = *categoryId
	}
	if video.Snippet.DefaultLanguage == "" && *language != "" {
		video.Snippet.DefaultLanguage = *language
	}
	if video.Snippet.DefaultAudioLanguage == "" && *language != "" {
		video.Snippet.DefaultAudioLanguage = *language
	}

	return videoMeta, nil
}

func Open(filename string) (io.ReadCloser, int64, error) {
	var reader io.ReadCloser
	var filesize int64
	var err error
	if strings.HasPrefix(filename, "http") {
		var resp *http.Response
		resp, err = http.Head(filename)
		if err != nil {
			return reader, 0, fmt.Errorf("error opening %s: %s", filename, err)
		}
		lenStr := resp.Header.Get("content-length")
		if lenStr != "" {
			filesize, err = strconv.ParseInt(lenStr, 10, 64)
			if err != nil {
				return reader, filesize, err
			}
		}

		resp, err = http.Get(filename)
		if err != nil {
			return reader, 0, fmt.Errorf("error opening %s: %s", filename, err)
		}
		if resp.ContentLength != 0 {
			filesize = resp.ContentLength
		}
		reader = resp.Body
	} else if filename == "-" {
		reader = os.Stdin
	} else {
		var file *os.File
		var fileInfo os.FileInfo
		file, err = os.Open(filename)
		if err != nil {
			return reader, 0, fmt.Errorf("error opening %s: %s", filename, err)
		}

		fileInfo, err = file.Stat()
		if err != nil {
			return reader, 0, fmt.Errorf("error stat'ing %s: %s", filename, err)
		}

		// check the file looks like a video
		buf := make([]byte, 512)
		_, err = file.Read(buf)
		if err != nil {
			return reader, 0, fmt.Errorf("error reading %s: %s", filename, err)
		}
		_, err = file.Seek(0, 0)
		if err != nil {
			return reader, 0, fmt.Errorf("error reading %s: %s", filename, err)
		}
		contentType := http.DetectContentType(buf)
		if !strings.HasPrefix(contentType, "video") && contentType != "application/octet-stream" {
			fmt.Printf("WARNING: input file doesn't appear to be a video. It has content type '%s'\n", contentType)
		}

		reader = file
		filesize = fileInfo.Size()

	}
	return reader, filesize, err
}

func (d *Date) UnmarshalJSON(b []byte) (err error) {
	s := string(b)
	s = s[1 : len(s)-1]
	// support ISO 8601 date only, and date + time
	if strings.ContainsAny(s, ":") {
		d.Time, err = time.Parse(inputDatetimeLayout, s)
	} else {
		d.Time, err = time.Parse(inputDateLayout, s)
	}
	return
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 SUSE LLC. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"

	"github.com/codegangsta/cli"
)

// Returns the version string
func version() string {
	const (
		major = 0
		minor = 1
	)
	return fmt.Sprintf("%v.%v", major, minor)
}

// It returns an application with all the flags and subcommands already in
// place.
func newApp() *cli.App {
	app := cli.NewApp()

	app.Name = "zypper-docker"
	app.Usage = "Patching Docker images safely"
	app.Version = version()

	app.CommandNotFound = func(context *cli.Context, cmd string) {
		fmt.Printf("Incorrect usage: command '%v' does not exist.\n\n", cmd)
		cli.ShowAppHelp(context)
	}

	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "n, non-interactive",
			Usage: "Switches to non-interactive mode",
		},
		cli.BoolFlag{
			Name:  "no-gpg-checks",
			Usage: "Ignore GPG check failures and continue",
		},
		cli.BoolFlag{
			Name:  "gpg-auto-import-keys",
			Usage: "If new repository signing key is found, do not ask what to do; trust and import it automatically",
		},
		cli.BoolFlag{
			Name:  "f, force",
			Usage: "Ignore all the local caches",
		},
	}
	app.Commands = []cli.Command{
		{
			Name:   "images",
			Usage:  "List all the images based on either OpenSUSE or SLES",
			Action: imagesCmd,
		},
		{
			Name:    "list-updates",
			Aliases: []string{"lu"},
			Usage:   "List all the available updates",
			Action:  listUpdatesCmd,
		},
		{
			Name:    "list-updates-container",
			Aliases: []string{"luc"},
			Usage:   "List all the available updates for the given container",
			Action:  listUpdatesContainerCmd,
		},
		{
			Name:    "update",
			Aliases: []string{"up"},
			Usage:   "Install the available updates",
			Action:  updateCmd,
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "l, auto-agree-with-licenses",
					Usage: "Automatically say yes to third party license confirmation prompt. By using this option, you choose to agree with licenses of all third-party software this command will install.",
				},
				cli.BoolFlag{
					Name:  "no-recommends",
					Usage: "By default, zypper installs also packages recommended by the requested ones. This option causes the recommended packages to be ignored and only the required ones to be installed.",
				},
				cli.BoolFlag{
					Name:  "replacefiles",
					Usage: "Install the packages even if they replace files from other, already installed, packages. Default is to treat file conflicts as an error.",
				},
				cli.StringFlag{
					Name:   "author",
					EnvVar: "USERNAME",
					Usage:  "Commit author to associate with the new layer (e.g., \"John Doe <john.doe@example.com>\")",
				},
				cli.StringFlag{
					Name:  "message",
					Value: "[zypper-docker] update",
					Usage: "Commit message to associated with the new layer",
				},
			},
		},
		{
			Name:    "list-patches",
			Aliases: []string{"lp"},
			Usage:   "List all the available patches",
			Action:  listPatchesCmd,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "bugzilla",
					Value: "",
					Usage: "List available needed patches for all Bugzilla issues, or issues whose number matches the given string (--bugzilla=#).",
				},
				cli.StringFlag{
					Name:  "cve",
					Value: "",
					Usage: "List available needed patches for all CVE issues, or issues whose number matches the given string (--cve=#).",
				},
				cli.StringFlag{
					Name:  "date",
					Value: "",
					Usage: "List patches issued up to, but not including, the specified date (YYYY-MM-DD).",
				},
				cli.StringFlag{
					Name:  "issues",
					Value: "",
					Usage: "Look for issues whose number, summary, or description matches the specified string (--issue=string).",
				},
				cli.StringFlag{
					Name:  "g, category",
					Value: "",
					Usage: "List only patches with this category.",
				},
			},
		},
		{
			Name:    "list-patches-container",
			Aliases: []string{"lpc"},
			Usage:   "List all the available patches for the given container",
			Action:  listPatchesContainerCmd,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "b, bugzilla",
					Value: "",
					Usage: "List available needed patches for all Bugzilla issues, or issues whose number matches the given string (--bugzilla=#).",
				},
				cli.StringFlag{
					Name:  "cve",
					Value: "",
					Usage: "List available needed patches for all CVE issues, or issues whose number matches the given string (--cve=#).",
				},
				cli.StringFlag{
					Name:  "date",
					Value: "",
					Usage: "List patches issued up to, but not including, the specified date (YYYY-MM-DD).",
				},
				cli.StringFlag{
					Name:  "issues",
					Value: "",
					Usage: "Look for issues whose number, summary, or description matches the specified string (--issue=string).",
				},
				cli.StringFlag{
					Name:  "g, category",
					Value: "",
					Usage: "List only patches with this category.",
				},
			},
		},
		{
			Name:   "patch",
			Usage:  "Install the available patches",
			Action: patchCmd,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "bugzilla",
					Value: "",
					Usage: "Install available needed patches for all Bugzilla issues, or issues whose number matches the given string (--bugzilla=#).",
				},
				cli.StringFlag{
					Name:  "cve",
					Value: "",
					Usage: "Install available needed patches for all CVE issues, or issues whose number matches the given string (--cve=#).",
				},
				cli.StringFlag{
					Name:  "date",
					Value: "",
					Usage: "Install patches issued up to, but not including, the specified date (YYYY-MM-DD).",
				},
				cli.StringFlag{
					Name:  "g, category",
					Value: "",
					Usage: "Install only patches with this category.",
				},
				cli.BoolFlag{
					Name:  "l, auto-agree-with-licenses",
					Usage: "Automatically say yes to third party license confirmation prompt. By using this option, you choose to agree with licenses of all third-party software this command will install.",
				},
				cli.BoolFlag{
					Name:  "no-recommends",
					Usage: "By default, zypper installs also packages recommended by the requested ones. This option causes the recommended packages to be ignored and only the required ones to be installed.",
				},
				cli.BoolFlag{
					Name:  "replacefiles",
					Usage: "Install the packages even if they replace files from other, already installed, packages. Default is to treat file conflicts as an error.",
				},
				cli.StringFlag{
					Name:   "author",
					EnvVar: "USERNAME",
					Usage:  "Commit author to associate with the new layer (e.g., \"John Doe <john.doe@example.com>\")",
				},
				cli.StringFlag{
					Name:  "message",
					Value: "[zypper-docker] patch",
					Usage: "Commit message to associated with the new layer",
				},
			},
		},
		{
			Name:    "patch-check",
			Aliases: []string{"pchk"},
			Usage:   "Check for patches",
			Action:  patchCheckCmd,
		},
		{
			Name:    "patch-check-container",
			Aliases: []string{"pchkc"},
			Usage:   "Check for patches available for the given container",
			Action:  patchCheckContainerCmd,
		},
		{
			Name:   "ps",
			Usage:  "List all the containers that are outdated",
			Action: psCmd,
		},
	}
	return app
}
<commit_msg>Fix default commit author<commit_after>// Copyright (c) 2015 SUSE LLC. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"log"
	"os/user"

	"github.com/codegangsta/cli"
)

// Returns the version string
func version() string {
	const (
		major = 0
		minor = 1
	)
	return fmt.Sprintf("%v.%v", major, minor)
}

func defaultCommitAuthor() string {
	current, err := user.Current()
	if err != nil {
		log.Printf("Cannot determine current user: %s", err)
		return ""
	}

	if current.Name != "" {
		return current.Name
	}
	return current.Username
}

// It returns an application with all the flags and subcommands already in
// place.
func newApp() *cli.App {
	app := cli.NewApp()

	app.Name = "zypper-docker"
	app.Usage = "Patching Docker images safely"
	app.Version = version()

	app.CommandNotFound = func(context *cli.Context, cmd string) {
		fmt.Printf("Incorrect usage: command '%v' does not exist.\n\n", cmd)
		cli.ShowAppHelp(context)
	}

	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "n, non-interactive",
			Usage: "Switches to non-interactive mode",
		},
		cli.BoolFlag{
			Name:  "no-gpg-checks",
			Usage: "Ignore GPG check failures and continue",
		},
		cli.BoolFlag{
			Name:  "gpg-auto-import-keys",
			Usage: "If new repository signing key is found, do not ask what to do; trust and import it automatically",
		},
		cli.BoolFlag{
			Name:  "f, force",
			Usage: "Ignore all the local caches",
		},
	}
	app.Commands = []cli.Command{
		{
			Name:   "images",
			Usage:  "List all the images based on either OpenSUSE or SLES",
			Action: imagesCmd,
		},
		{
			Name:    "list-updates",
			Aliases: []string{"lu"},
			Usage:   "List all the available updates",
			Action:  listUpdatesCmd,
		},
		{
			Name:    "list-updates-container",
			Aliases: []string{"luc"},
			Usage:   "List all the available updates for the given container",
			Action:  listUpdatesContainerCmd,
		},
		{
			Name:    "update",
			Aliases: []string{"up"},
			Usage:   "Install the available updates",
			Action:  updateCmd,
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "l, auto-agree-with-licenses",
					Usage: "Automatically say yes to third party license confirmation prompt. By using this option, you choose to agree with licenses of all third-party software this command will install.",
				},
				cli.BoolFlag{
					Name:  "no-recommends",
					Usage: "By default, zypper installs also packages recommended by the requested ones. This option causes the recommended packages to be ignored and only the required ones to be installed.",
				},
				cli.BoolFlag{
					Name:  "replacefiles",
					Usage: "Install the packages even if they replace files from other, already installed, packages. Default is to treat file conflicts as an error.",
				},
				cli.StringFlag{
					Name:  "author",
					Value: defaultCommitAuthor(),
					Usage: "Commit author to associate with the new layer (e.g., \"John Doe <john.doe@example.com>\")",
				},
				cli.StringFlag{
					Name:  "message",
					Value: "[zypper-docker] update",
					Usage: "Commit message to associated with the new layer",
				},
			},
		},
		{
			Name:    "list-patches",
			Aliases: []string{"lp"},
			Usage:   "List all the available patches",
			Action:  listPatchesCmd,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "bugzilla",
					Value: "",
					Usage: "List available needed patches for all Bugzilla issues, or issues whose number matches the given string (--bugzilla=#).",
				},
				cli.StringFlag{
					Name:  "cve",
					Value: "",
					Usage: "List available needed patches for all CVE issues, or issues whose number matches the given string (--cve=#).",
				},
				cli.StringFlag{
					Name:  "date",
					Value: "",
					Usage: "List patches issued up to, but not including, the specified date (YYYY-MM-DD).",
				},
				cli.StringFlag{
					Name:  "issues",
					Value: "",
					Usage: "Look for issues whose number, summary, or description matches the specified string (--issue=string).",
				},
				cli.StringFlag{
					Name:  "g, category",
					Value: "",
					Usage: "List only patches with this category.",
				},
			},
		},
		{
			Name:    "list-patches-container",
			Aliases: []string{"lpc"},
			Usage:   "List all the available patches for the given container",
			Action:  listPatchesContainerCmd,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "b, bugzilla",
					Value: "",
					Usage: "List available needed patches for all Bugzilla issues, or issues whose number matches the given string (--bugzilla=#).",
				},
				cli.StringFlag{
					Name:  "cve",
					Value: "",
					Usage: "List available needed patches for all CVE issues, or issues whose number matches the given string (--cve=#).",
				},
				cli.StringFlag{
					Name:  "date",
					Value: "",
					Usage: "List patches issued up to, but not including, the specified date (YYYY-MM-DD).",
				},
				cli.StringFlag{
					Name:  "issues",
					Value: "",
					Usage: "Look for issues whose number, summary, or description matches the specified string (--issue=string).",
				},
				cli.StringFlag{
					Name:  "g, category",
					Value: "",
					Usage: "List only patches with this category.",
				},
			},
		},
		{
			Name:   "patch",
			Usage:  "Install the available patches",
			Action: patchCmd,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "bugzilla",
					Value: "",
					Usage: "Install available needed patches for all Bugzilla issues, or issues whose number matches the given string (--bugzilla=#).",
				},
				cli.StringFlag{
					Name:  "cve",
					Value: "",
					Usage: "Install available needed patches for all CVE issues, or issues whose number matches the given string (--cve=#).",
				},
				cli.StringFlag{
					Name:  "date",
					Value: "",
					Usage: "Install patches issued up to, but not including, the specified date (YYYY-MM-DD).",
				},
				cli.StringFlag{
					Name:  "g, category",
					Value: "",
					Usage: "Install only patches with this category.",
				},
				cli.BoolFlag{
					Name:  "l, auto-agree-with-licenses",
					Usage: "Automatically say yes to third party license confirmation prompt. By using this option, you choose to agree with licenses of all third-party software this command will install.",
				},
				cli.BoolFlag{
					Name:  "no-recommends",
					Usage: "By default, zypper installs also packages recommended by the requested ones. This option causes the recommended packages to be ignored and only the required ones to be installed.",
				},
				cli.BoolFlag{
					Name:  "replacefiles",
					Usage: "Install the packages even if they replace files from other, already installed, packages. Default is to treat file conflicts as an error.",
				},
				cli.StringFlag{
					Name:  "author",
					Value: defaultCommitAuthor(),
					Usage: "Commit author to associate with the new layer (e.g., \"John Doe <john.doe@example.com>\")",
				},
				cli.StringFlag{
					Name:  "message",
					Value: "[zypper-docker] patch",
					Usage: "Commit message to associated with the new layer",
				},
			},
		},
		{
			Name:    "patch-check",
			Aliases: []string{"pchk"},
			Usage:   "Check for patches",
			Action:  patchCheckCmd,
		},
		{
			Name:    "patch-check-container",
			Aliases: []string{"pchkc"},
			Usage:   "Check for patches available for the given container",
			Action:  patchCheckContainerCmd,
		},
		{
			Name:   "ps",
			Usage:  "List all the containers that are outdated",
			Action: psCmd,
		},
	}
	return app
}
<|endoftext|>


<commit_before>package vindinium

import (
	"encoding/json"
	"fmt"
)

// Board represents the current game board
type Board struct {
	Size int

	Tiles     []Tile
	HeroID    map[Position]int
	MineOwner map[Position]int
	Taverns   map[Position]struct{}
}

// TileAt returns the tile at the given board position. If the given position
// is outside of the board, WallTile is returned.
func (b Board) TileAt(pos Position) Tile {
	if pos.X < 0 || pos.X >= b.Size {
		return WallTile
	}

	if pos.Y < 0 || pos.Y >= b.Size {
		return WallTile
	}

	// The positions sent from the server follow a different convention, so
	// we can't use pos.Y*b.Size + pos.X here
	return b.Tiles[pos.X*b.Size+pos.Y]
}

// To is a convenience function that returns the position that lies in the
// direction `dir` of the tile at Position `pos`.
func (b Board) To(pos Position, dir Direction) Position {
	switch dir {
	case North:
		return Position{pos.X, pos.Y - 1}
	case East:
		return Position{pos.X + 1, pos.Y}
	case South:
		return Position{pos.X, pos.Y + 1}
	case West:
		return Position{pos.X - 1, pos.Y}
	default:
		return pos
	}
}

func newBoard(size int, tiles string) (Board, error) {
	b := Board{
		Size: size,

		Tiles:     make([]Tile, size*size),
		HeroID:    make(map[Position]int),
		MineOwner: make(map[Position]int),
		Taverns:   make(map[Position]struct{}),
	}

	if len(tiles) != size*size*2 {
		return Board{}, fmt.Errorf("Board: couldn't parse raw tiles, raw tiles string is of wrong size. Expected %v, got %v.", size*size*2, len(tiles))
	}

	for x := 0; x < b.Size; x++ {
		for y := 0; y < b.Size; y++ {
			idx := y*b.Size + x

			switch tiles[2*idx : 2*idx+2] {
			case "  ":
				b.Tiles[idx] = AirTile

			case "##":
				b.Tiles[idx] = WallTile

			case "[]":
				b.Tiles[idx] = TavernTile
				b.Taverns[Position{x, y}] = struct{}{}

			case "$-":
				b.Tiles[idx] = MineTile

			case "$1":
				b.Tiles[idx] = MineTile
				b.MineOwner[Position{x, y}] = 1

			case "$2":
				b.Tiles[idx] = MineTile
				b.MineOwner[Position{x, y}] = 2

			case "$3":
				b.Tiles[idx] = MineTile
				b.MineOwner[Position{x, y}] = 3

			case "$4":
				b.Tiles[idx] = MineTile
				b.MineOwner[Position{x, y}] = 4

			case "@1":
				b.Tiles[idx] = HeroTile
				b.HeroID[Position{x, y}] = 1

			case "@2":
				b.Tiles[idx] = HeroTile
				b.HeroID[Position{x, y}] = 2

			case "@3":
				b.Tiles[idx] = HeroTile
				b.HeroID[Position{x, y}] = 3

			case "@4":
				b.Tiles[idx] = HeroTile
				b.HeroID[Position{x, y}] = 4

			default:
				return Board{}, fmt.Errorf("Board: Could not parse tiles, unknown tile found: %q", tiles[2*idx:2*idx+2])
			}
		}
	}

	return b, nil
}

// jsonBoard is used to unmarshal the board sent by the server
type jsonBoard struct {
	Size  int
	Tiles string
}

// UnmarshalJSON is called by the JSON unmarshaller. It takes care to parse the
// tile-string sent by the server into the useable Board representation.
func (b *Board) UnmarshalJSON(text []byte) error {
	var jb jsonBoard

	err := json.Unmarshal(text, &jb)
	if err != nil {
		return err
	}

	*b, err = newBoard(jb.Size, jb.Tiles)

	return err
}
<commit_msg>Board: Make `tiles` a private member, to be accessed only through the TileAt func<commit_after>package vindinium

import (
	"encoding/json"
	"fmt"
)

// Board represents the current game board
type Board struct {
	Size int

	HeroID    map[Position]int
	MineOwner map[Position]int
	Taverns   map[Position]struct{}

	tiles []Tile
}

// TileAt returns the tile at the given board position. If the given position
// is outside of the board, WallTile is returned.
func (b Board) TileAt(pos Position) Tile {
	if pos.X < 0 || pos.X >= b.Size {
		return WallTile
	}

	if pos.Y < 0 || pos.Y >= b.Size {
		return WallTile
	}

	// The positions sent from the server follow a different convention, so
	// we can't use pos.Y*b.Size + pos.X here
	return b.tiles[pos.X*b.Size+pos.Y]
}

// To is a convenience function that returns the position that lies in the
// direction `dir` of the tile at Position `pos`.
func (b Board) To(pos Position, dir Direction) Position {
	switch dir {
	case North:
		return Position{pos.X, pos.Y - 1}
	case East:
		return Position{pos.X + 1, pos.Y}
	case South:
		return Position{pos.X, pos.Y + 1}
	case West:
		return Position{pos.X - 1, pos.Y}
	default:
		return pos
	}
}

func newBoard(size int, tiles string) (Board, error) {
	b := Board{
		Size: size,

		HeroID:    make(map[Position]int),
		MineOwner: make(map[Position]int),
		Taverns:   make(map[Position]struct{}),

		tiles: make([]Tile, size*size),
	}

	if len(tiles) != size*size*2 {
		return Board{}, fmt.Errorf("Board: couldn't parse raw tiles, raw tiles string is of wrong size. Expected %v, got %v.", size*size*2, len(tiles))
	}

	for x := 0; x < b.Size; x++ {
		for y := 0; y < b.Size; y++ {
			idx := y*b.Size + x

			switch tiles[2*idx : 2*idx+2] {
			case "  ":
				b.tiles[idx] = AirTile

			case "##":
				b.tiles[idx] = WallTile

			case "[]":
				b.tiles[idx] = TavernTile
				b.Taverns[Position{x, y}] = struct{}{}

			case "$-":
				b.tiles[idx] = MineTile

			case "$1":
				b.tiles[idx] = MineTile
				b.MineOwner[Position{x, y}] = 1

			case "$2":
				b.tiles[idx] = MineTile
				b.MineOwner[Position{x, y}] = 2

			case "$3":
				b.tiles[idx] = MineTile
				b.MineOwner[Position{x, y}] = 3

			case "$4":
				b.tiles[idx] = MineTile
				b.MineOwner[Position{x, y}] = 4

			case "@1":
				b.tiles[idx] = HeroTile
				b.HeroID[Position{x, y}] = 1

			case "@2":
				b.tiles[idx] = HeroTile
				b.HeroID[Position{x, y}] = 2

			case "@3":
				b.tiles[idx] = HeroTile
				b.HeroID[Position{x, y}] = 3

			case "@4":
				b.tiles[idx] = HeroTile
				b.HeroID[Position{x, y}] = 4

			default:
				return Board{}, fmt.Errorf("Board: Could not parse tiles, unknown tile found: %q", tiles[2*idx:2*idx+2])
			}
		}
	}

	return b, nil
}

// jsonBoard is used to unmarshal the board sent by the server
type jsonBoard struct {
	Size  int
	Tiles string
}

// UnmarshalJSON is called by the JSON unmarshaller. It takes care to parse the
// tile-string sent by the server into the useable Board representation.
func (b *Board) UnmarshalJSON(text []byte) error {
	var jb jsonBoard

	err := json.Unmarshal(text, &jb)
	if err != nil {
		return err
	}

	*b, err = newBoard(jb.Size, jb.Tiles)

	return err
}
<|endoftext|>


<commit_before>package ionic

import (
	"fmt"
	"net/http"
	"testing"

	. "github.com/franela/goblin"
	"github.com/gomicro/bogus"
	. "github.com/onsi/gomega"
)

const (
	testToken = "token"
)

func TestGetDeliveryDestinations(t *testing.T) {
	g := Goblin(t)
	RegisterFailHandler(func(m string, _ ...int) { g.Fail(m) })

	g.Describe("Get Delivery Destinations", func() {
		server := bogus.New()
		h, p := server.HostPort()
		client, _ := New(fmt.Sprintf("http://%v:%v", h, p))

		g.It("should unmarshal list of destinations", func() {
			server.AddPath("/v1/teams/getDeliveryDestinations").
				SetMethods("GET").
				SetPayload([]byte(SampleValidDeliveryDestinations)).
				SetStatus(http.StatusOK)

			deliveries, err := client.GetDeliveryDestinations("7660D469-45DA-4AA3-A421-4F65E9C0CEE9", "token")
			Expect(err).To(BeNil())
			Expect(deliveries).NotTo(BeNil())
			Expect(deliveries[0].ID).To(Equal("B3DFA2C7-6DE6-4629-9F19-B493BBE6F2DC"))
			Expect(deliveries[1].Name).To(Equal("location2Name"))
		})

		g.It("should return an error for an invalid action", func() {
			server.AddPath("/v1/teams/getDeliveryDestinations").
				SetMethods("GET").
				SetPayload([]byte(SampleInvalidDeliveryDestinations)).
				SetStatus(http.StatusOK)

			deliveries, err := client.GetDeliveryDestinations("7660D469-45DA-4AA3-A421-4F65E9C0CEE9", "token")
			Expect(err).NotTo(BeNil())
			Expect(deliveries).To(BeNil())
		})
	})
}

func TestDeleteDeliveryDestination(t *testing.T) {
	g := Goblin(t)
	RegisterFailHandler(func(m string, _ ...int) { g.Fail(m) })

	g.Describe("Delete Delivery Destinations", func() {
		server := bogus.New()
		h, p := server.HostPort()
		client, _ := New(fmt.Sprintf("http://%v:%v", h, p))

		g.It("should get get no error when sucessful", func() {
			server.AddPath("/v1/teams/deleteDeliveryDestination").
				SetMethods("DELETE").
				SetStatus(http.StatusNoContent)

			err := client.DeleteDeliveryDestination("7660D469-45DA-4AA3-A421-4F65E9C0CEE9", "token")
			if err != nil {
				fmt.Printf("\nHERE:\n%v\n", err.Error())
			}
			Expect(err).To(BeNil())
		})

		g.It("should get err when API returns bad request", func() {
			server.AddPath("/v1/teams/deleteDeliveryDestination").
				SetMethods("DELETE").
				SetStatus(http.StatusBadRequest)

			err := client.DeleteDeliveryDestination("7660D469-45DA-4AA3-A421-4F65E9C0CEE9", "token")
			Expect(err).NotTo(BeNil())
		})

		g.It("should get err when API returns not found", func() {
			server.AddPath("/v1/teams/deleteDeliveryDestination").
				SetMethods("DELETE").
				SetStatus(http.StatusNotFound)

			err := client.DeleteDeliveryDestination("7660D469-45DA-4AA3-A421-4F65E9C0CEE9", "token")
			Expect(err).NotTo(BeNil())
		})

		g.It("should get err when API returns internal server error", func() {
			server.AddPath("/v1/teams/deleteDeliveryDestination").
				SetMethods("DELETE").
				SetStatus(http.StatusInternalServerError)

			err := client.DeleteDeliveryDestination("7660D469-45DA-4AA3-A421-4F65E9C0CEE9", "token")
			Expect(err).NotTo(BeNil())
		})
	})
}

const (
	SampleValidDeliveryDestinations = `
	{
		"data": [
		  {
			"id": "B3DFA2C7-6DE6-4629-9F19-B493BBE6F2DC",
			"team_id": "7660D469-45DA-4AA3-A421-4F65E9C0CEE9",
			"location": "location1",
			"region": "us-east-1",
			"name": "location1Name",
			"type": "s3"
		  },
		  {
			"id": "0728CC78-868D-4C27-89DE-29C6E6FD11F5",
			"team_id": "7660D469-45DA-4AA3-A421-4F65E9C0CEE9",
			"location": "location2",
			"region": "us-east-1",
			"name": "location2Name",
			"type": "s3",
			"deleted_at": "2019-08-10T20:00:00.840678Z"
		  }
		]
	  }
	`

	SampleInvalidDeliveryDestinations = `{"analysis":"fooanalysis", "action":"foo_action"}`
)
<commit_msg>Adds DeleteDeliveryDestination Changes DeleteTeamUser to only return error<commit_after>package ionic

import (
	"fmt"
	"net/http"
	"testing"

	. "github.com/franela/goblin"
	"github.com/gomicro/bogus"
	. "github.com/onsi/gomega"
)

const (
	testToken = "token"
)

func TestGetDeliveryDestinations(t *testing.T) {
	g := Goblin(t)
	RegisterFailHandler(func(m string, _ ...int) { g.Fail(m) })

	g.Describe("Get Delivery Destinations", func() {
		server := bogus.New()
		h, p := server.HostPort()
		client, _ := New(fmt.Sprintf("http://%v:%v", h, p))

		g.It("should unmarshal list of destinations", func() {
			server.AddPath("/v1/teams/getDeliveryDestinations").
				SetMethods("GET").
				SetPayload([]byte(SampleValidDeliveryDestinations)).
				SetStatus(http.StatusOK)

			deliveries, err := client.GetDeliveryDestinations("7660D469-45DA-4AA3-A421-4F65E9C0CEE9", "token")
			Expect(err).To(BeNil())
			Expect(deliveries).NotTo(BeNil())
			Expect(deliveries[0].ID).To(Equal("B3DFA2C7-6DE6-4629-9F19-B493BBE6F2DC"))
			Expect(deliveries[1].Name).To(Equal("location2Name"))
		})

		g.It("should return an error for an invalid action", func() {
			server.AddPath("/v1/teams/getDeliveryDestinations").
				SetMethods("GET").
				SetPayload([]byte(SampleInvalidDeliveryDestinations)).
				SetStatus(http.StatusOK)

			deliveries, err := client.GetDeliveryDestinations("7660D469-45DA-4AA3-A421-4F65E9C0CEE9", "token")
			Expect(err).NotTo(BeNil())
			Expect(deliveries).To(BeNil())
		})
	})
}

func TestDeleteDeliveryDestination(t *testing.T) {
	g := Goblin(t)
	RegisterFailHandler(func(m string, _ ...int) { g.Fail(m) })

	g.Describe("Delete Delivery Destinations", func() {
		server := bogus.New()
		h, p := server.HostPort()
		client, _ := New(fmt.Sprintf("http://%v:%v", h, p))

		g.It("should get no error when sucessful", func() {
			server.AddPath("/v1/teams/deleteDeliveryDestination").
				SetMethods("DELETE").
				SetStatus(http.StatusNoContent)

			err := client.DeleteDeliveryDestination("7660D469-45DA-4AA3-A421-4F65E9C0CEE9", "token")
			if err != nil {
				fmt.Printf("\nHERE:\n%v\n", err.Error())
			}
			Expect(err).To(BeNil())
		})

		g.It("should get err when API returns bad request", func() {
			server.AddPath("/v1/teams/deleteDeliveryDestination").
				SetMethods("DELETE").
				SetStatus(http.StatusBadRequest)

			err := client.DeleteDeliveryDestination("7660D469-45DA-4AA3-A421-4F65E9C0CEE9", "token")
			Expect(err).NotTo(BeNil())
		})

		g.It("should get err when API returns not found", func() {
			server.AddPath("/v1/teams/deleteDeliveryDestination").
				SetMethods("DELETE").
				SetStatus(http.StatusNotFound)

			err := client.DeleteDeliveryDestination("7660D469-45DA-4AA3-A421-4F65E9C0CEE9", "token")
			Expect(err).NotTo(BeNil())
		})

		g.It("should get err when API returns internal server error", func() {
			server.AddPath("/v1/teams/deleteDeliveryDestination").
				SetMethods("DELETE").
				SetStatus(http.StatusInternalServerError)

			err := client.DeleteDeliveryDestination("7660D469-45DA-4AA3-A421-4F65E9C0CEE9", "token")
			Expect(err).NotTo(BeNil())
		})
	})
}

const (
	SampleValidDeliveryDestinations = `
	{
		"data": [
		  {
			"id": "B3DFA2C7-6DE6-4629-9F19-B493BBE6F2DC",
			"team_id": "7660D469-45DA-4AA3-A421-4F65E9C0CEE9",
			"location": "location1",
			"region": "us-east-1",
			"name": "location1Name",
			"type": "s3"
		  },
		  {
			"id": "0728CC78-868D-4C27-89DE-29C6E6FD11F5",
			"team_id": "7660D469-45DA-4AA3-A421-4F65E9C0CEE9",
			"location": "location2",
			"region": "us-east-1",
			"name": "location2Name",
			"type": "s3",
			"deleted_at": "2019-08-10T20:00:00.840678Z"
		  }
		]
	  }
	`

	SampleInvalidDeliveryDestinations = `{"analysis":"fooanalysis", "action":"foo_action"}`
)
<|endoftext|>


<commit_before>// Copyright 2012 Samuel Stauffer. All rights reserved.
// Use of this source code is governed by a 3-clause BSD
// license that can be found in the LICENSE file.

package metrics

import (
	"math"
	"sort"
	"sync"
)

const (
	mpElemSize = 8 // sizeof int64
)

var (
	DefaultPrecision = Precision{0.0001, 1000 * 1000}
	DefaultMaxMemory = 4 * 1024
)

// Precision expresses the maximum epsilon tolerated for a typical size of input
type Precision struct {
	Episilon float64
	N        int
}

type mpHistogram struct {
	buffer     [][]int64
	bufferPool [2][]int64
	indices    []int
	count      int64
	sum        int64
	min        int64
	max        int64
	leafCount  int // number of elements in the bottom two leaves
	currentTop int
	rootWeight int
	bufferSize int
	maxDepth   int
	mutex      sync.RWMutex
}

// An implemenation of the Munro-Paterson approximate histogram algorithm adapted from:
// https://github.com/twitter/commons/blob/master/src/java/com/twitter/common/stats/ApproximateHistogram.java
// http://szl.googlecode.com/svn-history/r36/trunk/src/emitters/szlquantile.cc
func NewMunroPatersonHistogram(bufSize, maxDepth int) Histogram {
	buffer := make([][]int64, maxDepth+1)
	for i := 0; i < len(buffer); i++ {
		buffer[i] = make([]int64, bufSize)
	}
	return &mpHistogram{
		buffer:     buffer,
		bufferPool: [2][]int64{make([]int64, bufSize), make([]int64, bufSize)},
		indices:    make([]int, maxDepth+1),
		rootWeight: 1,
		currentTop: 1,
		bufferSize: bufSize,
		maxDepth:   maxDepth,
	}
}

func NewDefaultMunroPatersonHistogram() Histogram {
	return NewMunroPatersonHistogramWithMaxMemory(DefaultMaxMemory)
}

func NewMunroPatersonHistogramWithMaxMemory(bytes int) Histogram {
	b := computeB(DefaultPrecision.Episilon, DefaultPrecision.N)
	bufSize := computeBufferSize(b, DefaultPrecision.N)
	maxDepth := computeMaxDepth(bytes, bufSize)
	return NewMunroPatersonHistogram(bufSize, maxDepth)
}

func NewMunroPatersonHistogramWithPrecision(p Precision) Histogram {
	b := computeB(p.Episilon, p.N)
	bufSize := computeBufferSize(b, p.N)
	return NewMunroPatersonHistogram(bufSize, b)
}

func (mp *mpHistogram) String() string {
	return histogramToJson(mp, DefaultPercentiles, DefaultPercentileNames)
}

func (mp *mpHistogram) Clear() {
	mp.mutex.Lock()
	mp.count = 0
	mp.sum = 0
	mp.leafCount = 0
	mp.rootWeight = 1
	mp.min = 0
	mp.max = 0
	mp.mutex.Unlock()
}

func (mp *mpHistogram) Count() uint64 {
	mp.mutex.RLock()
	count := uint64(mp.count)
	mp.mutex.RUnlock()
	return count
}

func (mp *mpHistogram) Mean() float64 {
	mp.mutex.RLock()
	mean := float64(mp.sum) / float64(mp.count)
	mp.mutex.RUnlock()
	return mean
}

func (mp *mpHistogram) Sum() int64 {
	mp.mutex.RLock()
	sum := mp.sum
	mp.mutex.RUnlock()
	return sum
}

func (mp *mpHistogram) Min() int64 {
	mp.mutex.RLock()
	min := mp.min
	mp.mutex.RUnlock()
	return min
}

func (mp *mpHistogram) Max() int64 {
	mp.mutex.RLock()
	max := mp.max
	mp.mutex.RUnlock()
	return max
}

func (mp *mpHistogram) Percentiles(qs []float64) []int64 {
	mp.mutex.Lock()
	defer mp.mutex.Unlock()

	output := make([]int64, len(qs))
	if mp.count == 0 {
		return output
	}

	// the two leaves are the only buffer that can be partially filled
	buf0Size := mp.leafCount
	buf1Size := 0
	if mp.leafCount > mp.bufferSize {
		buf0Size = mp.bufferSize
		buf1Size = mp.leafCount - mp.bufferSize
	}

	sort.Sort(int64Slice(mp.buffer[0][:buf0Size]))
	sort.Sort(int64Slice(mp.buffer[1][:buf1Size]))

	indices := mp.indices
	for i := 0; i < len(indices); i++ {
		indices[i] = 0
	}
	sum := int64(0)
	io := 0
	floatCount := float64(mp.count)
	for io < len(output) {
		i := mp.smallest(buf0Size, buf1Size, indices)
		id := indices[i]
		indices[i]++
		sum += int64(mp.weight(i))
		for io < len(qs) && int64(qs[io]*floatCount) <= sum {
			output[io] = mp.buffer[i][id]
			io++
		}
	}
	return output
}

// Return the level of the smallest element (using the indices array 'ids'
// to track which elements have been already returned). Every buffers has
// already been sorted at this point.
func (mp *mpHistogram) smallest(buf0Size, buf1Size int, ids []int) int {
	smallest := int64(math.MaxInt64)
	id0 := ids[0]
	id1 := ids[1]
	iSmallest := 0

	if mp.leafCount > 0 && id0 < buf0Size {
		smallest = mp.buffer[0][id0]
	}
	if mp.leafCount > mp.bufferSize && id1 < buf1Size {
		x := mp.buffer[1][id1]
		if x < smallest {
			smallest = x
			iSmallest = 1
		}
	}
	for i := 2; i <= mp.currentTop; i++ {
		if !mp.isBufferEmpty(i) && ids[i] < mp.bufferSize {
			x := mp.buffer[i][ids[i]]
			if x < smallest {
				smallest = x
				iSmallest = i
			}
		}
	}
	return iSmallest
}

func (mp *mpHistogram) Update(x int64) {
	mp.mutex.Lock()
	// if the leaves of the tree are full, "collapse" recursively the tree
	if mp.leafCount == 2*mp.bufferSize {
		sort.Sort(int64Slice(mp.buffer[0]))
		sort.Sort(int64Slice(mp.buffer[1]))
		mp.recCollapse(mp.buffer[0], 1)
		mp.leafCount = 0
	}

	// Now we're sure there is space for adding x
	if mp.leafCount < mp.bufferSize {
		mp.buffer[0][mp.leafCount] = x
	} else {
		mp.buffer[1][mp.leafCount-mp.bufferSize] = x
	}
	mp.leafCount++
	if mp.count == 0 {
		mp.min = x
		mp.max = x
	} else {
		if x < mp.min {
			mp.min = x
		}
		if x > mp.max {
			mp.max = x
		}
	}
	mp.count++
	mp.sum += x
	mp.mutex.Unlock()
}

func (mp *mpHistogram) recCollapse(buf []int64, level int) {
	// if we reach the root, we can't add more buffer
	if level == mp.maxDepth {
		// weight() returns the weight of the root, in that case we need the
		// weight of merge result
		mergeWeight := 1 << (uint(level) - 1)
		idx := level & 1
		merged := mp.bufferPool[idx]
		tmp := mp.buffer[level]
		if mergeWeight == mp.rootWeight {
			mp.collapse1(buf, mp.buffer[level], merged)
		} else {
			mp.collapse(buf, mergeWeight, mp.buffer[level], mp.rootWeight, merged)
		}
		mp.buffer[level] = merged
		mp.bufferPool[idx] = tmp
		mp.rootWeight += mergeWeight
	} else {
		if level == mp.currentTop {
			// if we reach the top, add a new buffer
			mp.collapse1(buf, mp.buffer[level], mp.buffer[level+1])
			mp.currentTop++
			mp.rootWeight *= 2
		} else if mp.isBufferEmpty(level + 1) {
			// if the upper buffer is empty, use it
			mp.collapse1(buf, mp.buffer[level], mp.buffer[level+1])
		} else {
			// if the upper buffer isn't empty, collapse with it
			merged := mp.bufferPool[level&1]
			mp.collapse1(buf, mp.buffer[level], merged)
			mp.recCollapse(merged, level+1)
		}
	}
}

// collapse two sorted Arrays of different weight
// ex: [2,5,7] weight 2 and [3,8,9] weight 3
//     weight x array + concat = [2,2,5,5,7,7,3,3,3,8,8,8,9,9,9]
//     sort = [2,2,3,3,3,5,5,7,7,8,8,8,9,9,9]
//     select every nth elems = [3,7,9]  (n = sum weight / 2)
func (mp *mpHistogram) collapse(left []int64, leftWeight int, right []int64, rightWeight int, output []int64) {
	totalWeight := leftWeight + rightWeight
	cnt0 := -totalWeight / 2
	cnt1 := totalWeight + cnt0
	i := 0
	j := 0
	k := 0
	cnt := cnt0

	var smallest int64
	var weight int

	for i < len(left) || j < len(right) {
		if i < len(left) && (j == len(right) || left[i] < right[j]) {
			smallest = left[i]
			weight = leftWeight
			i++
		} else {
			smallest = right[j]
			weight = rightWeight
			j++
		}

		if cnt <= 0 && cnt+weight > 0 {
			output[k] = smallest
			k++
		}

		cnt += weight
		if cnt >= cnt1 {
			cnt -= totalWeight
			if cnt > 0 {
				output[k] = smallest
				k++
			}
		}
	}
}

// Optimized version of collapse for collapsing two arrays of the
// same weight (which is what we want most of the time)
func (mp *mpHistogram) collapse1(left, right, output []int64) {
	i, j, k, cnt := 0, 0, 0, 0
	ll := len(left)
	lr := len(right)
	for i < ll || j < lr {
		var smallest int64
		if i < ll && (j == lr || left[i] < right[j]) {
			smallest = left[i]
			i++
		} else {
			smallest = right[j]
			j++
		}
		if cnt&1 == 1 {
			output[k] = smallest
			k++
		}
		cnt++
	}
}

func (mp *mpHistogram) isBufferEmpty(level int) bool {
	if level == mp.currentTop {
		return false // root buffer (is present) is always full
	}
	return (mp.count/int64(mp.bufferSize*mp.weight(level)))&1 == 1
}

// return the weight of the level ie. 2^(i-1) except for the two tree
// leaves (weight=1) and for the root
func (mp *mpHistogram) weight(level int) int {
	if level < 2 {
		return 1
	}
	if level == mp.maxDepth {
		return mp.rootWeight
	}
	return 1 << (uint(level) - 1)
}

//

// We compute the "smallest possible k" satisfying two inequalities:
//    1)   (b - 2) * (2 ^ (b - 2)) + 0.5 <= epsilon * N
//    2)   k * (2 ^ (b - 1)) >= N
//
// For an explanation of these inequalities, please read the Munro-Paterson or
// the Manku-Rajagopalan-Linday papers.
func computeB(epsilon float64, n int) int {
	b := uint(2)
	en := epsilon * float64(n)
	for float64((b-2)*(1<<(b-2)))+0.5 <= en {
		b++
	}
	return int(b)
}

func computeBufferSize(b int, n int) int {
	return int(n / (1 << (uint(b) - 1)))
}

// Return the maximum depth of the graph to comply with the memory constraint
func computeMaxDepth(maxMemoryBytes int, bufferSize int) int {
	bm := 0
	n := maxMemoryBytes - 100 - mpElemSize*bufferSize
	if n < 0 {
		bm = 2
	} else {
		bm = int(n / (16 + mpElemSize*bufferSize))
	}
	if bm < 2 {
		bm = 2
	}
	return bm
}
<commit_msg>Simplify some code<commit_after>// Copyright 2012 Samuel Stauffer. All rights reserved.
// Use of this source code is governed by a 3-clause BSD
// license that can be found in the LICENSE file.

package metrics

import (
	"math"
	"sort"
	"sync"
)

const (
	mpElemSize = 8 // sizeof int64
)

var (
	DefaultPrecision = Precision{0.0001, 1000 * 1000}
	DefaultMaxMemory = 4 * 1024
)

// Precision expresses the maximum epsilon tolerated for a typical size of input
type Precision struct {
	Episilon float64
	N        int
}

type mpHistogram struct {
	buffer     [][]int64
	bufferPool [2][]int64
	indices    []int
	count      int64
	sum        int64
	min        int64
	max        int64
	leafCount  int // number of elements in the bottom two leaves
	currentTop int
	rootWeight int
	bufferSize int
	maxDepth   int
	mutex      sync.RWMutex
}

// An implemenation of the Munro-Paterson approximate histogram algorithm adapted from:
// https://github.com/twitter/commons/blob/master/src/java/com/twitter/common/stats/ApproximateHistogram.java
// http://szl.googlecode.com/svn-history/r36/trunk/src/emitters/szlquantile.cc
func NewMunroPatersonHistogram(bufSize, maxDepth int) Histogram {
	buffer := make([][]int64, maxDepth+1)
	for i := 0; i < len(buffer); i++ {
		buffer[i] = make([]int64, bufSize)
	}
	return &mpHistogram{
		buffer:     buffer,
		bufferPool: [2][]int64{make([]int64, bufSize), make([]int64, bufSize)},
		indices:    make([]int, maxDepth+1),
		rootWeight: 1,
		currentTop: 1,
		bufferSize: bufSize,
		maxDepth:   maxDepth,
	}
}

func NewDefaultMunroPatersonHistogram() Histogram {
	return NewMunroPatersonHistogramWithMaxMemory(DefaultMaxMemory)
}

func NewMunroPatersonHistogramWithMaxMemory(bytes int) Histogram {
	b := computeB(DefaultPrecision.Episilon, DefaultPrecision.N)
	bufSize := computeBufferSize(b, DefaultPrecision.N)
	maxDepth := computeMaxDepth(bytes, bufSize)
	return NewMunroPatersonHistogram(bufSize, maxDepth)
}

func NewMunroPatersonHistogramWithPrecision(p Precision) Histogram {
	b := computeB(p.Episilon, p.N)
	bufSize := computeBufferSize(b, p.N)
	return NewMunroPatersonHistogram(bufSize, b)
}

func (mp *mpHistogram) String() string {
	return histogramToJson(mp, DefaultPercentiles, DefaultPercentileNames)
}

func (mp *mpHistogram) Clear() {
	mp.mutex.Lock()
	mp.count = 0
	mp.sum = 0
	mp.leafCount = 0
	mp.rootWeight = 1
	mp.min = 0
	mp.max = 0
	mp.mutex.Unlock()
}

func (mp *mpHistogram) Count() uint64 {
	mp.mutex.RLock()
	count := uint64(mp.count)
	mp.mutex.RUnlock()
	return count
}

func (mp *mpHistogram) Mean() float64 {
	mp.mutex.RLock()
	mean := float64(mp.sum) / float64(mp.count)
	mp.mutex.RUnlock()
	return mean
}

func (mp *mpHistogram) Sum() int64 {
	mp.mutex.RLock()
	sum := mp.sum
	mp.mutex.RUnlock()
	return sum
}

func (mp *mpHistogram) Min() int64 {
	mp.mutex.RLock()
	min := mp.min
	mp.mutex.RUnlock()
	return min
}

func (mp *mpHistogram) Max() int64 {
	mp.mutex.RLock()
	max := mp.max
	mp.mutex.RUnlock()
	return max
}

func (mp *mpHistogram) Percentiles(qs []float64) []int64 {
	mp.mutex.Lock()
	defer mp.mutex.Unlock()

	output := make([]int64, len(qs))
	if mp.count == 0 {
		return output
	}

	// the two leaves are the only buffer that can be partially filled
	buf0Size := mp.leafCount
	buf1Size := 0
	if mp.leafCount > mp.bufferSize {
		buf0Size = mp.bufferSize
		buf1Size = mp.leafCount - mp.bufferSize
	}

	sort.Sort(int64Slice(mp.buffer[0][:buf0Size]))
	sort.Sort(int64Slice(mp.buffer[1][:buf1Size]))

	indices := mp.indices
	for i := 0; i < len(indices); i++ {
		indices[i] = 0
	}
	sum := int64(0)
	io := 0
	floatCount := float64(mp.count)
	for io < len(output) {
		i := mp.smallest(buf0Size, buf1Size, indices)
		id := indices[i]
		indices[i]++
		sum += int64(mp.weight(i))
		for io < len(qs) && int64(qs[io]*floatCount) <= sum {
			output[io] = mp.buffer[i][id]
			io++
		}
	}
	return output
}

// Return the level of the smallest element (using the indices array 'ids'
// to track which elements have been already returned). Every buffers has
// already been sorted at this point.
func (mp *mpHistogram) smallest(buf0Size, buf1Size int, ids []int) int {
	smallest := int64(math.MaxInt64)
	id0 := ids[0]
	id1 := ids[1]
	iSmallest := 0

	if mp.leafCount > 0 && id0 < buf0Size {
		smallest = mp.buffer[0][id0]
	}
	if mp.leafCount > mp.bufferSize && id1 < buf1Size {
		x := mp.buffer[1][id1]
		if x < smallest {
			smallest = x
			iSmallest = 1
		}
	}
	for i := 2; i <= mp.currentTop; i++ {
		if !mp.isBufferEmpty(i) && ids[i] < mp.bufferSize {
			x := mp.buffer[i][ids[i]]
			if x < smallest {
				smallest = x
				iSmallest = i
			}
		}
	}
	return iSmallest
}

func (mp *mpHistogram) Update(x int64) {
	mp.mutex.Lock()
	// if the leaves of the tree are full, "collapse" recursively the tree
	if mp.leafCount == 2*mp.bufferSize {
		sort.Sort(int64Slice(mp.buffer[0]))
		sort.Sort(int64Slice(mp.buffer[1]))
		mp.recCollapse(mp.buffer[0], 1)
		mp.leafCount = 0
	}

	// Now we're sure there is space for adding x
	if mp.leafCount < mp.bufferSize {
		mp.buffer[0][mp.leafCount] = x
	} else {
		mp.buffer[1][mp.leafCount-mp.bufferSize] = x
	}
	mp.leafCount++
	if mp.count == 0 {
		mp.min = x
		mp.max = x
	} else {
		if x < mp.min {
			mp.min = x
		}
		if x > mp.max {
			mp.max = x
		}
	}
	mp.count++
	mp.sum += x
	mp.mutex.Unlock()
}

func (mp *mpHistogram) recCollapse(buf []int64, level int) {
	// if we reach the root, we can't add more buffer
	if level == mp.maxDepth {
		// weight() returns the weight of the root, in that case we need the
		// weight of merge result
		mergeWeight := 1 << (uint(level) - 1)
		idx := level & 1
		merged := mp.bufferPool[idx]
		tmp := mp.buffer[level]
		if mergeWeight == mp.rootWeight {
			mp.collapse1(buf, mp.buffer[level], merged)
		} else {
			mp.collapse(buf, mergeWeight, mp.buffer[level], mp.rootWeight, merged)
		}
		mp.buffer[level] = merged
		mp.bufferPool[idx] = tmp
		mp.rootWeight += mergeWeight
	} else {
		if level == mp.currentTop {
			// if we reach the top, add a new buffer
			mp.collapse1(buf, mp.buffer[level], mp.buffer[level+1])
			mp.currentTop++
			mp.rootWeight *= 2
		} else if mp.isBufferEmpty(level + 1) {
			// if the upper buffer is empty, use it
			mp.collapse1(buf, mp.buffer[level], mp.buffer[level+1])
		} else {
			// if the upper buffer isn't empty, collapse with it
			merged := mp.bufferPool[level&1]
			mp.collapse1(buf, mp.buffer[level], merged)
			mp.recCollapse(merged, level+1)
		}
	}
}

// collapse two sorted Arrays of different weight
// ex: [2,5,7] weight 2 and [3,8,9] weight 3
//     weight x array + concat = [2,2,5,5,7,7,3,3,3,8,8,8,9,9,9]
//     sort = [2,2,3,3,3,5,5,7,7,8,8,8,9,9,9]
//     select every nth elems = [3,7,9]  (n = sum weight / 2)
func (mp *mpHistogram) collapse(left []int64, leftWeight int, right []int64, rightWeight int, output []int64) {
	totalWeight := leftWeight + rightWeight
	i := 0
	j := 0
	k := 0
	cnt := 0

	var smallest int64
	var weight int

	for i < len(left) || j < len(right) {
		if i < len(left) && (j == len(right) || left[i] < right[j]) {
			smallest = left[i]
			weight = leftWeight
			i++
		} else {
			smallest = right[j]
			weight = rightWeight
			j++
		}

		cur := (cnt + (totalWeight >> 1) - 1) / totalWeight
		cnt += weight
		next := (cnt + (totalWeight >> 1) - 1) / totalWeight

		for ; cur < next; cur++ {
			output[k] = smallest
			k++
		}
	}
}

// Optimized version of collapse for collapsing two arrays of the
// same weight (which is what we want most of the time)
func (mp *mpHistogram) collapse1(left, right, output []int64) {
	i, j, k, cnt := 0, 0, 0, 0
	ll := len(left)
	lr := len(right)
	for i < ll || j < lr {
		var smallest int64
		if i < ll && (j == lr || left[i] < right[j]) {
			smallest = left[i]
			i++
		} else {
			smallest = right[j]
			j++
		}
		if cnt&1 == 1 {
			output[k] = smallest
			k++
		}
		cnt++
	}
}

func (mp *mpHistogram) isBufferEmpty(level int) bool {
	if level == mp.currentTop {
		return false // root buffer (is present) is always full
	}
	return (mp.count/int64(mp.bufferSize*mp.weight(level)))&1 == 1
}

// return the weight of the level ie. 2^(i-1) except for the two tree
// leaves (weight=1) and for the root
func (mp *mpHistogram) weight(level int) int {
	if level < 2 {
		return 1
	}
	if level == mp.maxDepth {
		return mp.rootWeight
	}
	return 1 << (uint(level) - 1)
}

//

// We compute the "smallest possible k" satisfying two inequalities:
//    1)   (b - 2) * (2 ^ (b - 2)) + 0.5 <= epsilon * N
//    2)   k * (2 ^ (b - 1)) >= N
//
// For an explanation of these inequalities, please read the Munro-Paterson or
// the Manku-Rajagopalan-Linday papers.
func computeB(epsilon float64, n int) int {
	b := uint(2)
	en := epsilon * float64(n)
	for float64((b-2)*(1<<(b-2)))+0.5 <= en {
		b++
	}
	return int(b)
}

func computeBufferSize(b int, n int) int {
	return int(n / (1 << (uint(b) - 1)))
}

// Return the maximum depth of the graph to comply with the memory constraint
func computeMaxDepth(maxMemoryBytes int, bufferSize int) int {
	bm := 0
	n := maxMemoryBytes - 100 - mpElemSize*bufferSize
	if n < 0 {
		bm = 2
	} else {
		bm = int(n / (16 + mpElemSize*bufferSize))
	}
	if bm < 2 {
		bm = 2
	}
	return bm
}
<|endoftext|>


<commit_before>package main

import "github.com/Sirupsen/logrus"
import logger "./logger"
import wschat "./wschat"
import "encoding/json"
import "io/ioutil"
import "./chat_sender"
import "runtime"

var log = logger.GetLogger()

type Config struct {
	Channels   []string  `json:"channels"`
	LogglyKey string  `json:"loggly-key"`
}

func main() {
	file, e := ioutil.ReadFile("./config.json")
	if e != nil {
		log.WithFields(logrus.Fields{
			"err": e,
		}).Fatal("Error opening config")
	}

  var config Config
  e = json.Unmarshal(file, &config)
	if e != nil {
		log.WithFields(logrus.Fields{
			"err": e,
		}).Fatal("Error parsing config")
	}

  log.WithFields(logrus.Fields{
    "channels": config.Channels,
    "loggly_key": config.LogglyKey,
  }).Info("Config loaded.")

  cs := chatsender.ChatSender{config.LogglyKey}

  for _, channel := range config.Channels{
    ws := wschat.WsIrc{channel, nil, cs.SendLine}
    ws.Start()
  }
  for {
    runtime.Gosched()
  }
}
<commit_msg>Ran go fmt<commit_after>package main

import "github.com/Sirupsen/logrus"
import logger "./logger"
import wschat "./wschat"
import "encoding/json"
import "io/ioutil"
import "./chat_sender"
import "runtime"

var log = logger.GetLogger()

type Config struct {
	Channels  []string `json:"channels"`
	LogglyKey string   `json:"loggly-key"`
}

func main() {
	file, e := ioutil.ReadFile("./config.json")
	if e != nil {
		log.WithFields(logrus.Fields{
			"err": e,
		}).Fatal("Error opening config")
	}

	var config Config
	e = json.Unmarshal(file, &config)
	if e != nil {
		log.WithFields(logrus.Fields{
			"err": e,
		}).Fatal("Error parsing config")
	}

	log.WithFields(logrus.Fields{
		"channels":   config.Channels,
		"loggly_key": config.LogglyKey,
	}).Info("Config loaded.")

	cs := chatsender.ChatSender{config.LogglyKey}

	for _, channel := range config.Channels {
		ws := wschat.WsIrc{channel, nil, cs.SendLine}
		ws.Start()
	}
	for {
		runtime.Gosched()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"strconv"

	"github.com/clarkduvall/hyperloglog"
	metro "github.com/dgryski/go-metro"
	"github.com/seiflotfy/hlltc"
)

type fakeHash64 uint64

func (f fakeHash64) Sum64() uint64 { return uint64(f) }

func estimateError(got, exp uint64) float64 {
	var delta uint64
	if got > exp {
		delta = got - exp
	} else {
		delta = exp - got
	}
	return float64(delta) / float64(exp)
}

func main() {
	hlltc, _ := hlltc.New(14)
	hllpp, _ := hyperloglog.NewPlus(14)

	step := 10
	unique := map[string]bool{}

	for i := 1; len(unique) <= 10000000; i++ {
		str := strconv.Itoa(i)
		hlltc.Insert([]byte(str))
		item := fakeHash64(metro.Hash64([]byte(str), 1337))
		hllpp.Add(item)
		unique[str] = true

		if len(unique)%step == 0 || len(unique) == 10000000 {
			step *= 5
			exact := uint64(len(unique))
			res := uint64(hlltc.Estimate())
			ratio := 100 * estimateError(res, exact)
			res2 := uint64(hllpp.Count())
			ratio2 := 100 * estimateError(res2, exact)
			fmt.Printf("Exact %d, got:\n\thlltc %d (%.4f%% off)\n\thllpp %d (%.4f%% off)\n", exact, res, ratio, res2, ratio2)
		}
	}
}
<commit_msg>Add mashalling to demo<commit_after>package main

import (
	"fmt"
	"strconv"

	hyperloglog "github.com/influxdata/influxdb/pkg/estimator/hll"
	"github.com/seiflotfy/hlltc"
)

type fakeHash64 uint64

func (f fakeHash64) Sum64() uint64 { return uint64(f) }

func estimateError(got, exp uint64) float64 {
	var delta uint64
	if got > exp {
		delta = got - exp
	} else {
		delta = exp - got
	}
	return float64(delta) / float64(exp)
}

func main() {
	hlltc, _ := hlltc.New(14)
	hllpp, _ := hyperloglog.NewPlus(14)

	step := 10
	unique := map[string]bool{}

	for i := 1; len(unique) <= 10000000; i++ {
		str := "stream-" + strconv.Itoa(i)
		hlltc.Insert([]byte(str))
		hllpp.Add([]byte(str))
		unique[str] = true

		if len(unique)%step == 0 || len(unique) == 10000000 {
			step *= 5
			exact := uint64(len(unique))
			res := uint64(hlltc.Estimate())
			ratio := 100 * estimateError(res, exact)
			res2 := uint64(hllpp.Count())
			ratio2 := 100 * estimateError(res2, exact)
			fmt.Printf("Exact %d, got:\n\t axiom hlltc %d (%.4f%% off)\n\tinflux hllpp %d (%.4f%% off)\n", exact, res, ratio, res2, ratio2)
		}
	}

	data1, _ := hllpp.MarshalBinary()
	data2, _ := hlltc.MarshalBinary()
	fmt.Println("AxiomHQ HLLTC total size", len(data2))
	fmt.Println("InfluxData HLLPP total size", len(data1))

}
<|endoftext|>