<commit_before>package testsuite

import (
	"crypto/tls"

	"github.com/kyma-project/kyma/tests/end-to-end/external-solution-integration/pkg/testkit"
)

// ConnectApplication is a step which connects application with client certificates and saves connected httpClient in the state
type ConnectApplication struct {
	connector *testkit.ConnectorClient
	state     ConnectApplicationState
	tenant    string
	group     string
}

// ConnectApplicationState allows ConnectApplication to save connected http.Client for further use by other steps
type ConnectApplicationState interface {
	SetGatewayClientCerts(certs []tls.Certificate)
}

// NewConnectApplication returns new ConnectApplication
func NewConnectApplication(connector *testkit.ConnectorClient, state ConnectApplicationState, tenant, group string) *ConnectApplication {
	return &ConnectApplication{
		connector: connector,
		state:     state,
		tenant:    tenant,
		group:     group,
	}
}

// Name returns name name of the step
func (s ConnectApplication) Name() string {
	return "Connect application"
}

// Run executes the step
func (s ConnectApplication) Run() error {
	infoURL, err := s.connector.GetToken(s.tenant, s.group)
	if err != nil {
		return err
	}

	certInfo, err := s.connector.GetInfo(infoURL)
	if err != nil {
		return err
	}

	privateKey, err := testkit.CreateKey()
	if err != nil {
		return err
	}

	csr, err := testkit.CreateCSR(certInfo.Certificate.Subject, privateKey)
	if err != nil {
		return err
	}

	chain, err := s.connector.GetCertificate(certInfo.CertUrl, csr)
	if err != nil {
		return err
	}

	rawChain := make([][]byte, 0, len(chain))
	for _, cert := range chain {
		rawChain = append(rawChain, cert.Raw)
	}
	cert := tls.Certificate{Certificate: rawChain, PrivateKey: privateKey}
	s.state.SetGatewayClientCerts([]tls.Certificate{cert})
	return nil
}

// Cleanup removes all resources that may possibly created by the step
func (s ConnectApplication) Cleanup() error {
	return nil
}
<commit_msg>Add retries inside of conenct application test step (#9098)<commit_after>package testsuite

import (
	"crypto/tls"
	"crypto/x509"

	"github.com/kyma-project/kyma/tests/end-to-end/external-solution-integration/pkg/retry"
	"github.com/kyma-project/kyma/tests/end-to-end/external-solution-integration/pkg/testkit"
)

// ConnectApplication is a step which connects application with client certificates and saves connected httpClient in the state
type ConnectApplication struct {
	connector *testkit.ConnectorClient
	state     ConnectApplicationState
	tenant    string
	group     string
}

// ConnectApplicationState allows ConnectApplication to save connected http.Client for further use by other steps
type ConnectApplicationState interface {
	SetGatewayClientCerts(certs []tls.Certificate)
}

// NewConnectApplication returns new ConnectApplication
func NewConnectApplication(connector *testkit.ConnectorClient, state ConnectApplicationState, tenant, group string) *ConnectApplication {
	return &ConnectApplication{
		connector: connector,
		state:     state,
		tenant:    tenant,
		group:     group,
	}
}

// Name returns name name of the step
func (s ConnectApplication) Name() string {
	return "Connect application"
}

// Run executes the step
func (s ConnectApplication) Run() error {
	infoURL, err := s.connector.GetToken(s.tenant, s.group)
	if err != nil {
		return err
	}

	certInfo, err := func() (info *testkit.InfoResponse, err error) {
		err = retry.Do(func() error {
			info, err = s.connector.GetInfo(infoURL)
			return err
		})
		return info, err
	}()
	if err != nil {
		return err
	}

	privateKey, err := testkit.CreateKey()
	if err != nil {
		return err
	}

	csr, err := testkit.CreateCSR(certInfo.Certificate.Subject, privateKey)
	if err != nil {
		return err
	}

	chain, err := func() (chain []*x509.Certificate, err error) {
		err = retry.Do(func() error {
			chain, err = s.connector.GetCertificate(certInfo.CertUrl, csr)
			return err
		})
		return chain, err
	}()
	if err != nil {
		return err
	}

	rawChain := make([][]byte, 0, len(chain))
	for _, cert := range chain {
		rawChain = append(rawChain, cert.Raw)
	}
	cert := tls.Certificate{Certificate: rawChain, PrivateKey: privateKey}
	s.state.SetGatewayClientCerts([]tls.Certificate{cert})
	return nil
}

// Cleanup removes all resources that may possibly created by the step
func (s ConnectApplication) Cleanup() error {
	return nil
}
<|endoftext|>


<commit_before>package v2action_test

import (
	"errors"

	"code.cloudfoundry.org/cli/actor/actionerror"
	. "code.cloudfoundry.org/cli/actor/v2action"
	"code.cloudfoundry.org/cli/actor/v2action/v2actionfakes"
	"code.cloudfoundry.org/cli/api/cloudcontroller/ccerror"
	"code.cloudfoundry.org/cli/api/cloudcontroller/ccv2"
	"code.cloudfoundry.org/cli/api/cloudcontroller/ccv2/constant"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Application Summary Actions", func() {
	Describe("ApplicationSummary", func() {
		Describe("StartingOrRunningInstanceCount", func() {
			It("only counts the running and starting instances", func() {
				app := ApplicationSummary{
					RunningInstances: []ApplicationInstanceWithStats{
						{State: ApplicationInstanceState(constant.ApplicationInstanceCrashed)},
						{State: ApplicationInstanceState(constant.ApplicationInstanceDown)},
						{State: ApplicationInstanceState(constant.ApplicationInstanceFlapping)},
						{State: ApplicationInstanceState(constant.ApplicationInstanceRunning)},
						{State: ApplicationInstanceState(constant.ApplicationInstanceStarting)},
						{State: ApplicationInstanceState(constant.ApplicationInstanceUnknown)},
					},
				}
				Expect(app.StartingOrRunningInstanceCount()).To(Equal(2))
			})
		})
	})

	Describe("GetApplicationSummaryByNameSpace", func() {
		var (
			actor                     *Actor
			fakeCloudControllerClient *v2actionfakes.FakeCloudControllerClient
			app                       ccv2.Application
		)

		BeforeEach(func() {
			fakeCloudControllerClient = new(v2actionfakes.FakeCloudControllerClient)
			actor = NewActor(fakeCloudControllerClient, nil, nil)
			app = ccv2.Application{
				GUID: "some-app-guid",
				Name: "some-app",
			}
		})

		Context("when the application does not exist", func() {
			BeforeEach(func() {
				fakeCloudControllerClient.GetApplicationsReturns(
					[]ccv2.Application{},
					ccv2.Warnings{"app-warning"},
					nil)
			})

			It("returns an ApplicationNotFoundError and all warnings", func() {
				_, warnings, err := actor.GetApplicationSummaryByNameAndSpace("some-app", "some-space-guid")
				Expect(err).To(MatchError(actionerror.ApplicationNotFoundError{Name: "some-app"}))
				Expect(warnings).To(ConsistOf("app-warning"))
			})
		})

		Context("when the application exists", func() {
			BeforeEach(func() {
				fakeCloudControllerClient.GetApplicationsReturns(
					[]ccv2.Application{app},
					ccv2.Warnings{"app-warning"},
					nil)
			})

			Context("when the application is STARTED", func() {
				BeforeEach(func() {
					app.State = constant.ApplicationStarted
					fakeCloudControllerClient.GetApplicationsReturns(
						[]ccv2.Application{app},
						ccv2.Warnings{"app-warning"},
						nil)
				})

				Context("when instance information is available", func() {
					BeforeEach(func() {
						fakeCloudControllerClient.GetApplicationApplicationInstanceStatusesReturns(
							map[int]ccv2.ApplicationInstanceStatus{
								0: {ID: 0, IsolationSegment: "isolation-segment-1"},
								1: {ID: 1, IsolationSegment: "isolation-segment-2"}, // should never happen; iso segs for 2 instances of the same app should match.
							},
							ccv2.Warnings{"stats-warning"},
							nil)
						fakeCloudControllerClient.GetApplicationApplicationInstancesReturns(
							map[int]ccv2.ApplicationInstance{
								0: {ID: 0},
								1: {ID: 1},
							},
							ccv2.Warnings{"instance-warning"},
							nil)
					})

					It("returns the application with instance information and warnings and populates isolation segment from the first instance", func() {
						app, warnings, err := actor.GetApplicationSummaryByNameAndSpace("some-app", "some-space-guid")
						Expect(err).ToNot(HaveOccurred())
						Expect(app).To(Equal(ApplicationSummary{
							Application: Application{
								GUID:  "some-app-guid",
								Name:  "some-app",
								State: constant.ApplicationStarted,
							},
							RunningInstances: []ApplicationInstanceWithStats{
								{ID: 0, IsolationSegment: "isolation-segment-1"},
								{ID: 1, IsolationSegment: "isolation-segment-2"},
							},
							IsolationSegment: "isolation-segment-1",
						}))
						Expect(warnings).To(ConsistOf("app-warning", "stats-warning", "instance-warning"))
					})
				})

				Context("when instance information is not available", func() {
					BeforeEach(func() {
						fakeCloudControllerClient.GetApplicationApplicationInstanceStatusesReturns(
							nil,
							ccv2.Warnings{"stats-warning"},
							ccerror.ApplicationStoppedStatsError{})
					})

					It("returns the empty list of instances and all warnings", func() {
						app, warnings, err := actor.GetApplicationSummaryByNameAndSpace("some-app", "some-space-guid")
						Expect(err).ToNot(HaveOccurred())
						Expect(app.RunningInstances).To(BeEmpty())
						Expect(warnings).To(ConsistOf("app-warning", "stats-warning"))
					})
				})
			})

			Context("when the application is not STARTED", func() {
				BeforeEach(func() {
					app.State = constant.ApplicationStopped
				})

				It("does not try and get application instance information", func() {
					app, _, err := actor.GetApplicationSummaryByNameAndSpace("some-app", "some-space-guid")
					Expect(err).ToNot(HaveOccurred())
					Expect(app.RunningInstances).To(BeEmpty())

					Expect(fakeCloudControllerClient.GetApplicationApplicationInstanceStatusesCallCount()).To(Equal(0))
					Expect(fakeCloudControllerClient.GetApplicationApplicationInstancesCallCount()).To(Equal(0))
				})
			})

			Context("when the app has routes", func() {
				BeforeEach(func() {
					fakeCloudControllerClient.GetApplicationRoutesReturns(
						[]ccv2.Route{
							{
								GUID: "some-route-1-guid",
								Host: "host-1",
							},
							{
								GUID: "some-route-2-guid",
								Host: "host-2",
							},
						},
						ccv2.Warnings{"get-application-routes-warning"},
						nil)
				})

				It("returns the routes and all warnings", func() {
					app, warnings, err := actor.GetApplicationSummaryByNameAndSpace("some-app", "some-space-guid")
					Expect(err).ToNot(HaveOccurred())
					Expect(warnings).To(ConsistOf("app-warning", "get-application-routes-warning"))
					Expect(app.Routes).To(ConsistOf(
						Route{
							GUID: "some-route-1-guid",
							Host: "host-1",
						},
						Route{
							GUID: "some-route-2-guid",
							Host: "host-2",
						},
					))
				})

				Context("when an error is encountered while getting routes", func() {
					var expectedErr error

					BeforeEach(func() {
						expectedErr = errors.New("get routes error")
						fakeCloudControllerClient.GetApplicationRoutesReturns(
							nil,
							ccv2.Warnings{"get-application-routes-warning"},
							expectedErr,
						)
					})

					It("returns the error and all warnings", func() {
						app, warnings, err := actor.GetApplicationSummaryByNameAndSpace("some-app", "some-space-guid")
						Expect(err).To(MatchError(expectedErr))
						Expect(app.Routes).To(BeEmpty())
						Expect(warnings).To(ConsistOf("app-warning", "get-application-routes-warning"))
					})
				})
			})

			Context("when the app has stack information", func() {
				BeforeEach(func() {
					fakeCloudControllerClient.GetStackReturns(
						ccv2.Stack{Name: "some-stack"},
						ccv2.Warnings{"get-application-stack-warning"},
						nil)
				})

				It("returns the stack information and all warnings", func() {
					app, warnings, err := actor.GetApplicationSummaryByNameAndSpace("some-app", "some-space-guid")
					Expect(err).ToNot(HaveOccurred())
					Expect(warnings).To(ConsistOf("app-warning", "get-application-stack-warning"))
					Expect(app.Stack).To(Equal(Stack{Name: "some-stack"}))
				})

				Context("when an error is encountered while getting stack", func() {
					var expectedErr error

					BeforeEach(func() {
						expectedErr = errors.New("get stack error")
						fakeCloudControllerClient.GetStackReturns(
							ccv2.Stack{},
							ccv2.Warnings{"get-application-stack-warning"},
							expectedErr,
						)
					})

					It("returns the error and all warnings", func() {
						app, warnings, err := actor.GetApplicationSummaryByNameAndSpace("some-app", "some-space-guid")
						Expect(err).To(MatchError(expectedErr))
						Expect(app.Stack).To(Equal(Stack{}))
						Expect(warnings).To(ConsistOf("app-warning", "get-application-stack-warning"))
					})
				})
			})
		})
	})
})
<commit_msg>removed shadow from test<commit_after>package v2action_test

import (
	"errors"

	"code.cloudfoundry.org/cli/actor/actionerror"
	. "code.cloudfoundry.org/cli/actor/v2action"
	"code.cloudfoundry.org/cli/actor/v2action/v2actionfakes"
	"code.cloudfoundry.org/cli/api/cloudcontroller/ccerror"
	"code.cloudfoundry.org/cli/api/cloudcontroller/ccv2"
	"code.cloudfoundry.org/cli/api/cloudcontroller/ccv2/constant"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Application Summary Actions", func() {
	Describe("ApplicationSummary", func() {
		Describe("StartingOrRunningInstanceCount", func() {
			It("only counts the running and starting instances", func() {
				app := ApplicationSummary{
					RunningInstances: []ApplicationInstanceWithStats{
						{State: ApplicationInstanceState(constant.ApplicationInstanceCrashed)},
						{State: ApplicationInstanceState(constant.ApplicationInstanceDown)},
						{State: ApplicationInstanceState(constant.ApplicationInstanceFlapping)},
						{State: ApplicationInstanceState(constant.ApplicationInstanceRunning)},
						{State: ApplicationInstanceState(constant.ApplicationInstanceStarting)},
						{State: ApplicationInstanceState(constant.ApplicationInstanceUnknown)},
					},
				}
				Expect(app.StartingOrRunningInstanceCount()).To(Equal(2))
			})
		})
	})

	Describe("GetApplicationSummaryByNameSpace", func() {
		var (
			actor                     *Actor
			fakeCloudControllerClient *v2actionfakes.FakeCloudControllerClient
			ccApp                     ccv2.Application
		)

		BeforeEach(func() {
			fakeCloudControllerClient = new(v2actionfakes.FakeCloudControllerClient)
			actor = NewActor(fakeCloudControllerClient, nil, nil)
			ccApp = ccv2.Application{
				GUID: "some-app-guid",
				Name: "some-app",
			}
		})

		Context("when the application does not exist", func() {
			BeforeEach(func() {
				fakeCloudControllerClient.GetApplicationsReturns(
					[]ccv2.Application{},
					ccv2.Warnings{"app-warning"},
					nil)
			})

			It("returns an ApplicationNotFoundError and all warnings", func() {
				_, warnings, err := actor.GetApplicationSummaryByNameAndSpace("some-app", "some-space-guid")
				Expect(err).To(MatchError(actionerror.ApplicationNotFoundError{Name: "some-app"}))
				Expect(warnings).To(ConsistOf("app-warning"))
			})
		})

		Context("when the application exists", func() {
			BeforeEach(func() {
				fakeCloudControllerClient.GetApplicationsReturns(
					[]ccv2.Application{ccApp},
					ccv2.Warnings{"app-warning"},
					nil)
			})

			Context("when the application is STARTED", func() {
				BeforeEach(func() {
					ccApp.State = constant.ApplicationStarted
					fakeCloudControllerClient.GetApplicationsReturns(
						[]ccv2.Application{ccApp},
						ccv2.Warnings{"app-warning"},
						nil)
				})

				Context("when instance information is available", func() {
					BeforeEach(func() {
						fakeCloudControllerClient.GetApplicationApplicationInstanceStatusesReturns(
							map[int]ccv2.ApplicationInstanceStatus{
								0: {ID: 0, IsolationSegment: "isolation-segment-1"},
								1: {ID: 1, IsolationSegment: "isolation-segment-2"}, // should never happen; iso segs for 2 instances of the same app should match.
							},
							ccv2.Warnings{"stats-warning"},
							nil)
						fakeCloudControllerClient.GetApplicationApplicationInstancesReturns(
							map[int]ccv2.ApplicationInstance{
								0: {ID: 0},
								1: {ID: 1},
							},
							ccv2.Warnings{"instance-warning"},
							nil)
					})

					It("returns the application with instance information and warnings and populates isolation segment from the first instance", func() {
						app, warnings, err := actor.GetApplicationSummaryByNameAndSpace("some-app", "some-space-guid")
						Expect(err).ToNot(HaveOccurred())
						Expect(app).To(Equal(ApplicationSummary{
							Application: Application{
								GUID:  "some-app-guid",
								Name:  "some-app",
								State: constant.ApplicationStarted,
							},
							RunningInstances: []ApplicationInstanceWithStats{
								{ID: 0, IsolationSegment: "isolation-segment-1"},
								{ID: 1, IsolationSegment: "isolation-segment-2"},
							},
							IsolationSegment: "isolation-segment-1",
						}))
						Expect(warnings).To(ConsistOf("app-warning", "stats-warning", "instance-warning"))
					})
				})

				Context("when instance information is not available", func() {
					BeforeEach(func() {
						fakeCloudControllerClient.GetApplicationApplicationInstanceStatusesReturns(
							nil,
							ccv2.Warnings{"stats-warning"},
							ccerror.ApplicationStoppedStatsError{})
					})

					It("returns the empty list of instances and all warnings", func() {
						app, warnings, err := actor.GetApplicationSummaryByNameAndSpace("some-app", "some-space-guid")
						Expect(err).ToNot(HaveOccurred())
						Expect(app.RunningInstances).To(BeEmpty())
						Expect(warnings).To(ConsistOf("app-warning", "stats-warning"))
					})
				})
			})

			Context("when the application is not STARTED", func() {
				BeforeEach(func() {
					ccApp.State = constant.ApplicationStopped
				})

				It("does not try and get application instance information", func() {
					app, _, err := actor.GetApplicationSummaryByNameAndSpace("some-app", "some-space-guid")
					Expect(err).ToNot(HaveOccurred())
					Expect(app.RunningInstances).To(BeEmpty())

					Expect(fakeCloudControllerClient.GetApplicationApplicationInstanceStatusesCallCount()).To(Equal(0))
					Expect(fakeCloudControllerClient.GetApplicationApplicationInstancesCallCount()).To(Equal(0))
				})
			})

			Context("when the app has routes", func() {
				BeforeEach(func() {
					fakeCloudControllerClient.GetApplicationRoutesReturns(
						[]ccv2.Route{
							{
								GUID: "some-route-1-guid",
								Host: "host-1",
							},
							{
								GUID: "some-route-2-guid",
								Host: "host-2",
							},
						},
						ccv2.Warnings{"get-application-routes-warning"},
						nil)
				})

				It("returns the routes and all warnings", func() {
					app, warnings, err := actor.GetApplicationSummaryByNameAndSpace("some-app", "some-space-guid")
					Expect(err).ToNot(HaveOccurred())
					Expect(warnings).To(ConsistOf("app-warning", "get-application-routes-warning"))
					Expect(app.Routes).To(ConsistOf(
						Route{
							GUID: "some-route-1-guid",
							Host: "host-1",
						},
						Route{
							GUID: "some-route-2-guid",
							Host: "host-2",
						},
					))
				})

				Context("when an error is encountered while getting routes", func() {
					var expectedErr error

					BeforeEach(func() {
						expectedErr = errors.New("get routes error")
						fakeCloudControllerClient.GetApplicationRoutesReturns(
							nil,
							ccv2.Warnings{"get-application-routes-warning"},
							expectedErr,
						)
					})

					It("returns the error and all warnings", func() {
						app, warnings, err := actor.GetApplicationSummaryByNameAndSpace("some-app", "some-space-guid")
						Expect(err).To(MatchError(expectedErr))
						Expect(app.Routes).To(BeEmpty())
						Expect(warnings).To(ConsistOf("app-warning", "get-application-routes-warning"))
					})
				})
			})

			Context("when the app has stack information", func() {
				BeforeEach(func() {
					fakeCloudControllerClient.GetStackReturns(
						ccv2.Stack{Name: "some-stack"},
						ccv2.Warnings{"get-application-stack-warning"},
						nil)
				})

				It("returns the stack information and all warnings", func() {
					app, warnings, err := actor.GetApplicationSummaryByNameAndSpace("some-app", "some-space-guid")
					Expect(err).ToNot(HaveOccurred())
					Expect(warnings).To(ConsistOf("app-warning", "get-application-stack-warning"))
					Expect(app.Stack).To(Equal(Stack{Name: "some-stack"}))
				})

				Context("when an error is encountered while getting stack", func() {
					var expectedErr error

					BeforeEach(func() {
						expectedErr = errors.New("get stack error")
						fakeCloudControllerClient.GetStackReturns(
							ccv2.Stack{},
							ccv2.Warnings{"get-application-stack-warning"},
							expectedErr,
						)
					})

					It("returns the error and all warnings", func() {
						app, warnings, err := actor.GetApplicationSummaryByNameAndSpace("some-app", "some-space-guid")
						Expect(err).To(MatchError(expectedErr))
						Expect(app.Stack).To(Equal(Stack{}))
						Expect(warnings).To(ConsistOf("app-warning", "get-application-stack-warning"))
					})
				})
			})
		})
	})
})
<|endoftext|>


<commit_before><commit_msg>fs: parseduration: fixed tests to use UTC time<commit_after><|endoftext|>


<commit_before>package centralserver

import (
	"distributed2048/rpc/centralrpc"
	"distributed2048/util"
	"encoding/json"
	"errors"
	"fmt"
	"math"
	"net"
	"net/http"
	"net/rpc"
	"os"
	"sync"
)

const (
	ERROR_LOG bool = true
	DEBUG_LOG bool = true
)

var LOGV = util.NewLogger(DEBUG_LOG, "DEBUG", os.Stdout)
var LOGE = util.NewLogger(ERROR_LOG, "ERROR", os.Stderr)

type gameServer struct {
	info        centralrpc.Node
	clientCount int
}

type centralServer struct {
	nextGameServerID     uint32
	gameServersLock      sync.Mutex
	gameServers          map[uint32]*gameServer
	hostPortToGameServer map[string]*gameServer
	gameServersSlice     []centralrpc.Node
	numGameServers       int
}

func NewCentralServer(port, numGameServers int) (CentralServer, error) {
	LOGV.Println("New Central Server is starting up")
	if numGameServers < 1 {
		return nil, errors.New("numGameServers must be at least 1")
	}

	cs := &centralServer{
		numGameServers:   numGameServers,
		gameServers:      make(map[uint32]*gameServer),
		gameServersSlice: nil,
	}

	// Serve up information for the game client
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		LOGV.Println("a new request was made with URI " + r.RequestURI)
		reply := HttpReply{}
//		cs.gameServersLock.Lock()
		if len(cs.gameServers) < cs.numGameServers {
			// Not all game servers have connected to the ring, so reply with NotReady
			reply.Status = "NotReady"
			reply.Hostport = ""
		} else {
			id := cs.getGameServerIDMinClients()
			cs.gameServers[id].clientCount++
			reply.Status = "OK"
			reply.Hostport = cs.gameServers[id].info.HostPort
		}
//		cs.gameServersLock.Unlock()
		LOGV.Println("testing if it reaches this point")
		buf, err := json.Marshal(reply)
		if err == nil {
			LOGV.Printf("sending back a response...")
			//		http.Error(w, fmt.Sprintln(err), 500)
			w.Header().Set("Content-Type", "text/plain; charset=utf-8")
			w.Header().Set("Access-Control-Allow-Origin", "*")
			w.Header().Set("Connection", "Keep-Alive")
			_, err = w.Write(buf)
		} else {
			LOGV.Printf("Error with marshalling reply: " + err.Error())
		}
	})
	go http.ListenAndServe(fmt.Sprintf(":%d", port), nil)
	LOGV.Printf("testing if it reaches here")

	rpc.RegisterName("CentralServer", centralrpc.Wrap(cs))
	rpc.HandleHTTP()
	l, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
	if err != nil {
		return nil, err
	}
	go http.Serve(l, nil)

	return cs, nil
}

func (cs *centralServer) GetGameServerForClient(args *centralrpc.GetGameServerForClientArgs, reply *centralrpc.GetGameServerForClientReply) error {
	cs.gameServersLock.Lock()
	if len(cs.gameServers) < cs.numGameServers {
		// Not all game servers have connected to the ring, so reply with NotReady
		reply.Status = centralrpc.NotReady
	} else {
		id := cs.getGameServerIDMinClients()
		cs.gameServers[id].clientCount++
		reply.Status = centralrpc.OK
		reply.HostPort = cs.gameServers[id].info.HostPort
	}
	cs.gameServersLock.Unlock()

	return nil
}

func (cs *centralServer) RegisterGameServer(args *centralrpc.RegisterGameServerArgs, reply *centralrpc.RegisterGameServerReply) error {
	cs.gameServersLock.Lock()

	var id uint32
	if gs, exists := cs.hostPortToGameServer[args.HostPort]; !exists {

		// Get a new ID
		id = cs.nextGameServerID
		cs.nextGameServerID++

		// Get host:port
		hostport := args.HostPort

		// Add new server object to map
		gs = &gameServer{centralrpc.Node{id, hostport}, 0}
		cs.gameServers[id] = gs
		cs.hostPortToGameServer[hostport] = gs
	} else {
		id = gs.info.NodeID
	}

	// Check if all the game servers in the ring have registered. If they
	// haven't, then reply with not ready. Otherwise, reply with OK, send back
	// to the unique ID, and the list of all game servers.
	if len(cs.gameServers) < cs.numGameServers {
		reply.Status = centralrpc.NotReady
	} else {
		reply.Status = centralrpc.OK
		reply.GameServerID = id
		// Check if the game servers sliced has been cached. If it hasn't, make it.
		if cs.gameServersSlice == nil {
			cs.gameServersSlice = make([]centralrpc.Node, len(cs.gameServers))
			i := 0
			for _, node := range cs.gameServers {
				cs.gameServersSlice[i] = node.info
				i++
			}
		}
		reply.Servers = cs.gameServersSlice
	}

	cs.gameServersLock.Unlock()

	return nil
}

type HttpReply struct {
	Status   string
	Hostport string
}

//func (cs *centralServer) gameClientViewHandler(w http.ResponseWriter, r *http.Request)

func (cs *centralServer) getGameServerIDMinClients() uint32 {
	// Must be called with the LOCK acquired
	min := math.MaxInt32
	var resultID uint32
	for _, gs := range cs.gameServers {
		if gs.clientCount < min {
			min = gs.clientCount
			resultID = gs.info.NodeID
		}
	}
	return resultID
}
<commit_msg>Cleaned up centralserver constructor<commit_after>package centralserver

import (
	"distributed2048/rpc/centralrpc"
	"distributed2048/util"
	"encoding/json"
	"errors"
	"fmt"
	"math"
	"net"
	"net/http"
	"net/rpc"
	"os"
	"sync"
)

const (
	ERROR_LOG bool = true
	DEBUG_LOG bool = true
)

var LOGV = util.NewLogger(DEBUG_LOG, "DEBUG", os.Stdout)
var LOGE = util.NewLogger(ERROR_LOG, "ERROR", os.Stderr)

type gameServer struct {
	info        centralrpc.Node
	clientCount int
}

type centralServer struct {
	nextGameServerID     uint32
	gameServersLock      sync.Mutex
	gameServers          map[uint32]*gameServer
	hostPortToGameServer map[string]*gameServer
	gameServersSlice     []centralrpc.Node
	numGameServers       int
}

func NewCentralServer(port, numGameServers int) (CentralServer, error) {
	LOGV.Println("New Central Server is starting up")
	if numGameServers < 1 {
		return nil, errors.New("numGameServers must be at least 1")
	}

	cs := &centralServer{
		numGameServers:   numGameServers,
		gameServers:      make(map[uint32]*gameServer),
		gameServersSlice: nil,
	}

	// Serve up information for the game client
	http.HandleFunc("/", cs.gameClientViewHandler)
	go http.ListenAndServe(fmt.Sprintf(":%d", port), nil)

	rpc.RegisterName("CentralServer", centralrpc.Wrap(cs))
	rpc.HandleHTTP()
	l, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
	if err != nil {
		return nil, err
	}
	go http.Serve(l, nil)

	return cs, nil
}

func (cs *centralServer) GetGameServerForClient(args *centralrpc.GetGameServerForClientArgs, reply *centralrpc.GetGameServerForClientReply) error {
	cs.gameServersLock.Lock()
	if len(cs.gameServers) < cs.numGameServers {
		// Not all game servers have connected to the ring, so reply with NotReady
		reply.Status = centralrpc.NotReady
	} else {
		id := cs.getGameServerIDMinClients()
		cs.gameServers[id].clientCount++
		reply.Status = centralrpc.OK
		reply.HostPort = cs.gameServers[id].info.HostPort
	}
	cs.gameServersLock.Unlock()

	return nil
}

func (cs *centralServer) RegisterGameServer(args *centralrpc.RegisterGameServerArgs, reply *centralrpc.RegisterGameServerReply) error {
	cs.gameServersLock.Lock()

	var id uint32
	if gs, exists := cs.hostPortToGameServer[args.HostPort]; !exists {

		// Get a new ID
		id = cs.nextGameServerID
		cs.nextGameServerID++

		// Get host:port
		hostport := args.HostPort

		// Add new server object to map
		gs = &gameServer{centralrpc.Node{id, hostport}, 0}
		cs.gameServers[id] = gs
		cs.hostPortToGameServer[hostport] = gs
	} else {
		id = gs.info.NodeID
	}

	// Check if all the game servers in the ring have registered. If they
	// haven't, then reply with not ready. Otherwise, reply with OK, send back
	// to the unique ID, and the list of all game servers.
	if len(cs.gameServers) < cs.numGameServers {
		reply.Status = centralrpc.NotReady
	} else {
		reply.Status = centralrpc.OK
		reply.GameServerID = id
		// Check if the game servers sliced has been cached. If it hasn't, make it.
		if cs.gameServersSlice == nil {
			cs.gameServersSlice = make([]centralrpc.Node, len(cs.gameServers))
			i := 0
			for _, node := range cs.gameServers {
				cs.gameServersSlice[i] = node.info
				i++
			}
		}
		reply.Servers = cs.gameServersSlice
	}

	cs.gameServersLock.Unlock()

	return nil
}

type HttpReply struct {
	Status   string
	Hostport string
}

func (cs *centralServer) gameClientViewHandler(w http.ResponseWriter, r *http.Request) {
	LOGV.Println("a new request was made with URI " + r.RequestURI)
	reply := HttpReply{}
	cs.gameServersLock.Lock()
	if len(cs.gameServers) < cs.numGameServers {
		// Not all game servers have connected to the ring, so reply with NotReady
		reply.Status = "NotReady"
		reply.Hostport = ""
	} else {
		id := cs.getGameServerIDMinClients()
		cs.gameServers[id].clientCount++
		reply.Status = "OK"
		reply.Hostport = cs.gameServers[id].info.HostPort
	}
	cs.gameServersLock.Unlock()
	buf, err := json.Marshal(reply)
	if err == nil {
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		w.Header().Set("Access-Control-Allow-Origin", "*")
		w.Header().Set("Connection", "Keep-Alive")
		_, err = w.Write(buf)
	} else {
		LOGE.Printf("Error with marshalling reply: " + err.Error())
	}
}

func (cs *centralServer) getGameServerIDMinClients() uint32 {
	// Must be called with the LOCK acquired
	min := math.MaxInt32
	var resultID uint32
	for _, gs := range cs.gameServers {
		if gs.clientCount < min {
			min = gs.clientCount
			resultID = gs.info.NodeID
		}
	}
	return resultID
}
<|endoftext|>


<commit_before><commit_msg>Remove some worthless comments.<commit_after><|endoftext|>


<commit_before>//   Copyright 2017 MSolution.IO
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package costs

import (
	"fmt"
	"net/http"
	"strings"
	"time"

	"github.com/trackit/jsonlog"
	"gopkg.in/olivere/elastic.v5"
)

var detailedLineItemFieldsName = map[string]func([]string) []paramAggrAndName{
	"product": createAggregationPerProduct,
	"region":  createAggregationPerRegion,
	"account": createAggregationPerAccount,
	"tag":     createAggregationPerTag,
	"cost":    createCostSumAggregation,
}

type paramAggrAndName struct {
	paramName string
	paramAggr elastic.Aggregation
}

const aggregationMaxSize = 0x7FFFFFFF

func createQueryAccountFilter(accountList []string) *elastic.TermsQuery {
	return elastic.NewTermsQuery("linked_account_id", accountList)
}

func createQueryTimeRange(durationBegin time.Time, durationEnd time.Time) *elastic.RangeQuery {
	return elastic.NewRangeQuery("usage_start_date").From(durationBegin).To(durationEnd)
}

func createAggregationPerProduct(paramSplit []string) []paramAggrAndName {
	res := make([]paramAggrAndName, 1)
	res[0] = paramAggrAndName{
		paramName: "product",
		paramAggr: elastic.NewTermsAggregation().Field("product_name").Size(aggregationMaxSize)}
	return res
}

func createAggregationPerRegion(paramSplit []string) []paramAggrAndName {
	res := make([]paramAggrAndName, 1)
	res[0] = paramAggrAndName{
		paramName: "region",
		paramAggr: elastic.NewTermsAggregation().Field("availability_zone").Size(aggregationMaxSize)}
	return res
}

func createAggregationPerAccount(paramSplit []string) []paramAggrAndName {
	res := make([]paramAggrAndName, 1)
	res[0] = paramAggrAndName{
		paramName: "account",
		paramAggr: elastic.NewTermsAggregation().Field("linked_account_id").Size(aggregationMaxSize)}
	return res
}

func createAggregationPerTag(paramSplit []string) []paramAggrAndName {
	res := make([]paramAggrAndName, 2)
	res[0] = paramAggrAndName{
		paramName: "tag_key",
		paramAggr: elastic.NewFilterAggregation().Filter(elastic.NewTermQuery("tag.key", fmt.Sprintf("user:%v", paramSplit[1])))}
	res[1] = paramAggrAndName{
		paramName: "tag_value",
		paramAggr: elastic.NewTermsAggregation().Field("tag.value").Size(aggregationMaxSize)}
	return res
}

func createCostSumAggregation(paramSplit []string) []paramAggrAndName {
	res := make([]paramAggrAndName, 1)
	res[0] = paramAggrAndName{
		paramName: "cost",
		paramAggr: elastic.NewSumAggregation().Field("cost")}
	return res
}

func reverseAggregationArray(aggregationArray []paramAggrAndName) []paramAggrAndName {
	for i := len(aggregationArray)/2 - 1; i >= 0; i-- {
		opp := len(aggregationArray) - 1 - i
		aggregationArray[i], aggregationArray[opp] = aggregationArray[opp], aggregationArray[i]
	}
	return aggregationArray
}

func nestAggregation(allAggrSlice []paramAggrAndName) elastic.Aggregation {
	allAggrSlice = reverseAggregationArray(allAggrSlice)
	aggrToNest := allAggrSlice[0]
	for _, baseAggr := range allAggrSlice[1:] {
		// fmt.Printf("aggrToNest.paramName = %v; baseAggr.paramName = %v\n", aggrToNest.paramName, baseAggr.paramName)
		switch assertedBaseAggr := baseAggr.paramAggr.(type) {
		case *elastic.TermsAggregation:
			aggrBuff := assertedBaseAggr.SubAggregation(aggrToNest.paramName, aggrToNest.paramAggr)
			aggrToNest = paramAggrAndName{paramName: baseAggr.paramName, paramAggr: aggrBuff}
		case *elastic.FilterAggregation:
			aggrBuff := assertedBaseAggr.SubAggregation(aggrToNest.paramName, aggrToNest.paramAggr)
			aggrToNest = paramAggrAndName{paramName: baseAggr.paramName, paramAggr: aggrBuff}
		case *elastic.SumAggregation:
			aggrBuff := assertedBaseAggr.SubAggregation(aggrToNest.paramName, aggrToNest.paramAggr)
			aggrToNest = paramAggrAndName{paramName: baseAggr.paramName, paramAggr: aggrBuff}
		}
	}
	return aggrToNest.paramAggr
}

// GetElasticSearchParams is used to construct an ElasticSearch *elastic.SearchService used to perform a request on ES
// It takes as paramters :
// 	- accountList []string : A slice of strings representing aws account number, in the format of the field 'awsdetailedlineitem.linked_account_id'
//	- durationBeing time.TIme : A time.Time struct representing the begining of the time range in the query
//	- durationEnd time.Time : A time.Time struct representing the end of the time range in the query
//	- param []string : A slice of strings representing the different parameters, in the nesting order, that will create aggregations.
//	  Those can be :
//		- "product" : It will create a TermsAggregation on the field 'product_name'
//		- "region" : It will create a TermsAggregation on the field 'availability_zone'
//		- "account" : It will create a TermsAggregation on the field 'linked_account_id'
//		- "tag:<TAG_KEY>" : It will create a FilterAggregation on the field 'tag.key', filtering on the value 'user:<TAG_KEY>'. It will then create a TermsAggregation on the field 'tag.value'
//	- client *elastic.Client : an instance of *elastic.Client that represent an Elastic Search client. It needs to be fully configured and ready to execute a client.Search()
//	- index string : The Elastic Search index on wich to execute the query. In this context the default value should be "awsdetailedlineitems"
func GetElasticSearchParams(accountList []string, durationBegin time.Time, durationEnd time.Time, params []string, client *elastic.Client, index string) *elastic.SearchService {
	query := elastic.NewBoolQuery()
	if len(accountList) > 0 {
		query = query.Filter(createQueryAccountFilter(accountList))
	}
	query = query.Filter(createQueryTimeRange(durationBegin, durationEnd))
	search := client.Search().Index(index).Size(0).Query(query)
	params = append(params, "cost")
	var allAggregationSlice []paramAggrAndName
	for _, paramName := range params {
		paramNameSplit := strings.Split(paramName, ":")
		// fmt.Printf("param = %v, paramNameSplit = %v\n", paramName, paramNameSplit)
		paramAggr := detailedLineItemFieldsName[paramNameSplit[0]](paramNameSplit)
		allAggregationSlice = append(allAggregationSlice, paramAggr...)
	}
	aggregationParamName := allAggregationSlice[0].paramName
	nestedAggregation := nestAggregation(allAggregationSlice)
	search.Aggregation(aggregationParamName, nestedAggregation)
	return search
}

// HandleRequest is a dummy request handler function. It does nothing except
// some logging and returns static data.
func HandleRequest(response http.ResponseWriter, request *http.Request, logger jsonlog.Logger) {
	logger.Debug("Request headers.", request.Header)
	response.WriteHeader(200)
	response.Write([]byte("Costs."))
}
<commit_msg>TRAC-539: added DateHistogramAggregation<commit_after>//   Copyright 2017 MSolution.IO
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package costs

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/signer/v4"
	"github.com/sha1sum/aws_signing_client"

	"github.com/trackit/jsonlog"
	"gopkg.in/olivere/elastic.v5"
)

var detailedLineItemFieldsName = map[string]func([]string) []paramAggrAndName{
	"product": createAggregationPerProduct,
	"region":  createAggregationPerRegion,
	"account": createAggregationPerAccount,
	"tag":     createAggregationPerTag,
	"cost":    createCostSumAggregation,
	"day":     createAggregationPerDay,
	"week":    createAggregationPerWeek,
	"month":   createAggregationPerMonth,
	"year":    createAggregationPerYear,
}

type paramAggrAndName struct {
	paramName string
	paramAggr elastic.Aggregation
}

const aggregationMaxSize = 0x7FFFFFFF

func createQueryAccountFilter(accountList []string) *elastic.TermsQuery {
	return elastic.NewTermsQuery("linked_account_id", accountList)
}

func createQueryTimeRange(durationBegin time.Time, durationEnd time.Time) *elastic.RangeQuery {
	return elastic.NewRangeQuery("usage_start_date").From(durationBegin).To(durationEnd)
}

func createAggregationPerProduct(paramSplit []string) []paramAggrAndName {
	res := make([]paramAggrAndName, 1)
	res[0] = paramAggrAndName{
		paramName: "product",
		paramAggr: elastic.NewTermsAggregation().Field("product_name").Size(aggregationMaxSize)}
	return res
}

func createAggregationPerRegion(paramSplit []string) []paramAggrAndName {
	res := make([]paramAggrAndName, 1)
	res[0] = paramAggrAndName{
		paramName: "region",
		paramAggr: elastic.NewTermsAggregation().Field("availability_zone").Size(aggregationMaxSize)}
	return res
}

func createAggregationPerAccount(paramSplit []string) []paramAggrAndName {
	res := make([]paramAggrAndName, 1)
	res[0] = paramAggrAndName{
		paramName: "account",
		paramAggr: elastic.NewTermsAggregation().Field("linked_account_id").Size(aggregationMaxSize)}
	return res
}

func createAggregationPerDay(paramList []string) []paramAggrAndName {
	res := make([]paramAggrAndName, 1)
	res[0] = paramAggrAndName{
		paramName: "day",
		paramAggr: elastic.NewDateHistogramAggregation().Field("usage_start_date").Interval("day")}
	return res
}

func createAggregationPerWeek(paramList []string) []paramAggrAndName {
	res := make([]paramAggrAndName, 1)
	res[0] = paramAggrAndName{
		paramName: "week",
		paramAggr: elastic.NewDateHistogramAggregation().Field("usage_start_date").Interval("week")}
	return res
}

func createAggregationPerMonth(paramList []string) []paramAggrAndName {
	res := make([]paramAggrAndName, 1)
	res[0] = paramAggrAndName{
		paramName: "month",
		paramAggr: elastic.NewDateHistogramAggregation().Field("usage_start_date").Interval("month")}
	return res
}

func createAggregationPerYear(paramList []string) []paramAggrAndName {
	res := make([]paramAggrAndName, 1)
	res[0] = paramAggrAndName{
		paramName: "year",
		paramAggr: elastic.NewDateHistogramAggregation().Field("usage_start_date").Interval("year")}
	return res
}

func createAggregationPerTag(paramSplit []string) []paramAggrAndName {
	res := make([]paramAggrAndName, 2)
	res[0] = paramAggrAndName{
		paramName: "tag_key",
		paramAggr: elastic.NewFilterAggregation().Filter(elastic.NewTermQuery("tag.key", fmt.Sprintf("user:%v", paramSplit[1])))}
	res[1] = paramAggrAndName{
		paramName: "tag_value",
		paramAggr: elastic.NewTermsAggregation().Field("tag.value").Size(aggregationMaxSize)}
	return res
}

func createCostSumAggregation(paramSplit []string) []paramAggrAndName {
	res := make([]paramAggrAndName, 1)
	res[0] = paramAggrAndName{
		paramName: "cost",
		paramAggr: elastic.NewSumAggregation().Field("cost")}
	return res
}

func reverseAggregationArray(aggregationArray []paramAggrAndName) []paramAggrAndName {
	for i := len(aggregationArray)/2 - 1; i >= 0; i-- {
		opp := len(aggregationArray) - 1 - i
		aggregationArray[i], aggregationArray[opp] = aggregationArray[opp], aggregationArray[i]
	}
	return aggregationArray
}

func nestAggregation(allAggrSlice []paramAggrAndName) elastic.Aggregation {
	allAggrSlice = reverseAggregationArray(allAggrSlice)
	aggrToNest := allAggrSlice[0]
	for _, baseAggr := range allAggrSlice[1:] {
		fmt.Printf("aggrToNest.paramName = %v; baseAggr.paramName = %v\n", aggrToNest.paramName, baseAggr.paramName)
		switch assertedBaseAggr := baseAggr.paramAggr.(type) {
		case *elastic.TermsAggregation:
			aggrBuff := assertedBaseAggr.SubAggregation(aggrToNest.paramName, aggrToNest.paramAggr)
			aggrToNest = paramAggrAndName{paramName: baseAggr.paramName, paramAggr: aggrBuff}
		case *elastic.FilterAggregation:
			aggrBuff := assertedBaseAggr.SubAggregation(aggrToNest.paramName, aggrToNest.paramAggr)
			aggrToNest = paramAggrAndName{paramName: baseAggr.paramName, paramAggr: aggrBuff}
		case *elastic.SumAggregation:
			aggrBuff := assertedBaseAggr.SubAggregation(aggrToNest.paramName, aggrToNest.paramAggr)
			aggrToNest = paramAggrAndName{paramName: baseAggr.paramName, paramAggr: aggrBuff}
		case *elastic.DateHistogramAggregation:
			aggrBuff := assertedBaseAggr.SubAggregation(aggrToNest.paramName, aggrToNest.paramAggr)
			aggrToNest = paramAggrAndName{paramName: baseAggr.paramName, paramAggr: aggrBuff}
		}
	}
	return aggrToNest.paramAggr
}

// GetElasticSearchParams is used to construct an ElasticSearch *elastic.SearchService used to perform a request on ES
// It takes as paramters :
// 	- accountList []string : A slice of strings representing aws account number, in the format of the field 'awsdetailedlineitem.linked_account_id'
//	- durationBeing time.TIme : A time.Time struct representing the begining of the time range in the query
//	- durationEnd time.Time : A time.Time struct representing the end of the time range in the query
//	- param []string : A slice of strings representing the different parameters, in the nesting order, that will create aggregations.
//	  Those can be :
//		- "product" : It will create a TermsAggregation on the field 'product_name'
//		- "region" : It will create a TermsAggregation on the field 'availability_zone'
//		- "account" : It will create a TermsAggregation on the field 'linked_account_id'
//		- "tag:<TAG_KEY>" : It will create a FilterAggregation on the field 'tag.key', filtering on the value 'user:<TAG_KEY>'. It will then create a TermsAggregation on the field 'tag.value'
//		- "[day|week|month|year]": It will create a DateHistogramAggregation on the specified duration on the field 'usage_start_date'
//	- client *elastic.Client : an instance of *elastic.Client that represent an Elastic Search client. It needs to be fully configured and ready to execute a client.Search()
//	- index string : The Elastic Search index on wich to execute the query. In this context the default value should be "awsdetailedlineitems"
func GetElasticSearchParams(accountList []string, durationBegin time.Time, durationEnd time.Time, params []string, client *elastic.Client, index string) *elastic.SearchService {
	query := elastic.NewBoolQuery()
	if len(accountList) > 0 {
		query = query.Filter(createQueryAccountFilter(accountList))
	}
	query = query.Filter(createQueryTimeRange(durationBegin, durationEnd))
	search := client.Search().Index(index).Size(0).Query(query)
	params = append(params, "cost")
	var allAggregationSlice []paramAggrAndName
	for _, paramName := range params {
		paramNameSplit := strings.Split(paramName, ":")
		// fmt.Printf("param = %v, paramNameSplit = %v\n", paramName, paramNameSplit)
		paramAggr := detailedLineItemFieldsName[paramNameSplit[0]](paramNameSplit)
		allAggregationSlice = append(allAggregationSlice, paramAggr...)
	}
	aggregationParamName := allAggregationSlice[0].paramName
	nestedAggregation := nestAggregation(allAggregationSlice)
	search.Aggregation(aggregationParamName, nestedAggregation)
	return search
}

func deserialize(searchResult *elastic.SearchResult) {
	if searchResult == nil || searchResult.TotalHits() == 0 {
		fmt.Println("Well that's not suppose to happen")
		return
	}
	fmt.Printf("Total hits : %v\n", searchResult.TotalHits())
	for i, hit := range searchResult.Hits.Hits {
		fmt.Printf("i = %v, hit = %v\n", i, hit)
	}
}

func PlaceholderFunc() {
	credentials := credentials.NewSharedCredentials("", "default")
	signer := v4.NewSigner(credentials)
	awsClient, err := aws_signing_client.New(signer, nil, "es", "us-west-2")
	if err != nil {
		fmt.Printf("error received : %v\n", err)
	}
	client, err := elastic.NewClient(
		elastic.SetURL("https://search-job-msol-prod-trackit-j6ofkgxgmxezkamcywpmqwfsn4.us-west-2.es.amazonaws.com"),
		elastic.SetScheme("https"),
		elastic.SetHttpClient(awsClient),
		elastic.SetSniff(false),
	)
	var accountList = []string{"298868543803", "135736586752"}
	// var accountList = []string{}
	var paramList = []string{"account", "product", "day"}
	// search := GetElasticSearchParams([]string{"394125495069"}, time.Date(2017, 10, 1, 0, 0, 0, 0, time.Local), time.Date(2017, 11, 1, 0, 0, 0, 0, time.Local), []string{"product"}, client, "awsdetailedlineitem")
	search := GetElasticSearchParams(accountList, time.Unix(1504572038, 0), time.Now(), paramList, client, "awsdetailedlineitem")
	a, err := search.Do(context.Background())
	if err != nil {
		panic(err)
	}
	res, err := json.MarshalIndent(*a, "", "  ")
	fmt.Printf("%v %v\n", string(res), err)
	deserialize(a)
	fmt.Println(err)

}

// HandleRequest is a dummy request handler function. It does nothing except
// some logging and returns static data.
func HandleRequest(response http.ResponseWriter, request *http.Request, logger jsonlog.Logger) {
	logger.Debug("Request headers.", request.Header)
	response.WriteHeader(200)
	response.Write([]byte("Costs."))
}
<|endoftext|>


<commit_before>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package configmapprovider // import "go.opentelemetry.io/collector/config/configmapprovider"

import (
	"bytes"
	"context"
	"strings"

	"github.com/knadh/koanf/maps"
	"github.com/magiconair/properties"

	"go.opentelemetry.io/collector/config"
)

// NewOverwritePropertiesConverter returns a service.ConfigMapConverterFunc, that overrides all the given properties into the
// input map.
//
// Properties must follow the Java properties format, key-value list separated by equal sign with a "."
// as key delimiter.
//  ["processors.batch.timeout=2s", "processors.batch/foo.timeout=3s"]
func NewOverwritePropertiesConverter(properties []string) config.MapConverterFunc {
	return func(_ context.Context, cfgMap *config.Map) error {
		return convert(properties, cfgMap)
	}
}

func convert(propsStr []string, cfgMap *config.Map) error {
	if len(propsStr) == 0 {
		return nil
	}

	b := &bytes.Buffer{}
	for _, property := range propsStr {
		property = strings.TrimSpace(property)
		b.WriteString(property)
		b.WriteString("\n")
	}

	var props *properties.Properties
	var err error
	if props, err = properties.Load(b.Bytes(), properties.UTF8); err != nil {
		return err
	}

	// Create a map manually instead of using properties.Map() to not expand the env vars.
	parsed := make(map[string]interface{}, props.Len())
	for _, key := range props.Keys() {
		value, _ := props.Get(key)
		parsed[key] = value
	}
	prop := maps.Unflatten(parsed, ".")

	return cfgMap.Merge(config.NewMapFromStringMap(prop))
}
<commit_msg>Fix nit in convertor commnent (#4999)<commit_after>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package configmapprovider // import "go.opentelemetry.io/collector/config/configmapprovider"

import (
	"bytes"
	"context"
	"strings"

	"github.com/knadh/koanf/maps"
	"github.com/magiconair/properties"

	"go.opentelemetry.io/collector/config"
)

// NewOverwritePropertiesConverter returns a config.MapConverterFunc, that overrides all the given properties into the
// input map.
//
// Properties must follow the Java properties format, key-value list separated by equal sign with a "."
// as key delimiter.
//  ["processors.batch.timeout=2s", "processors.batch/foo.timeout=3s"]
func NewOverwritePropertiesConverter(properties []string) config.MapConverterFunc {
	return func(_ context.Context, cfgMap *config.Map) error {
		return convert(properties, cfgMap)
	}
}

func convert(propsStr []string, cfgMap *config.Map) error {
	if len(propsStr) == 0 {
		return nil
	}

	b := &bytes.Buffer{}
	for _, property := range propsStr {
		property = strings.TrimSpace(property)
		b.WriteString(property)
		b.WriteString("\n")
	}

	var props *properties.Properties
	var err error
	if props, err = properties.Load(b.Bytes(), properties.UTF8); err != nil {
		return err
	}

	// Create a map manually instead of using properties.Map() to not expand the env vars.
	parsed := make(map[string]interface{}, props.Len())
	for _, key := range props.Keys() {
		value, _ := props.Get(key)
		parsed[key] = value
	}
	prop := maps.Unflatten(parsed, ".")

	return cfgMap.Merge(config.NewMapFromStringMap(prop))
}
<|endoftext|>


<commit_before>package tracker

import (
	"bufio"
	"bytes"
	"fmt"
	"github.com/jackpal/Taipei-Torrent/torrent"
	"io"
	"io/ioutil"
	"log"
	"math"
	"os"
	"os/exec"
	"path"
	"strconv"
	"testing"
	"time"
)

func TestScrapeURL(t *testing.T) {
	tests := []struct{ announce, scrape string }{
		{"", ""},
		{"foo", ""},
		{"x/announce", "x/scrape"},
		{"x/announce?ad#3", "x/scrape?ad#3"},
		{"announce/x", ""},
	}
	for _, test := range tests {
		scrape := ScrapePattern(test.announce)
		if scrape != test.scrape {
			t.Errorf("ScrapeURL(%#v) = %#v. Expected %#v", test.announce, scrape, test.scrape)
		}
	}
}

func TestSwarm1(t *testing.T) {
	testSwarm(t, 1)
}

/*
func TestSwarm10(t *testing.T) {
	testSwarm(t, 10)
}

/* Larger sizes don't work correctly.

func TestSwarm20(t *testing.T) {
	testSwarm(t, 20)
}

func TestSwarm50(t *testing.T) {
	testSwarm(t, 50)
}

func TestSwarm100(t *testing.T) {
	testSwarm(t, 100)
}

*/

func testSwarm(t *testing.T, leechCount int) {
	err := runSwarm(leechCount)
	if err != nil {
		t.Fatal("Error running testSwarm", err)
	}
}

type prog struct {
	instanceName string
	dirName      string
	cmd          *exec.Cmd
}

func (p *prog) start(doneCh chan *prog) (err error) {
	log.Println("starting", p.instanceName)
	out := logWriter(p.instanceName)
	p.cmd.Stdout = &out
	p.cmd.Stderr = &out
	err = p.cmd.Start()
	if err != nil {
		return
	}
	go func() {
		p.cmd.Wait()
		doneCh <- p
	}()
	return
}

func (p *prog) kill() (err error) {
	err = p.cmd.Process.Kill()
	return
}

func newProg(instanceName string, dir string, command string, arg ...string) (p *prog) {
	cmd := helperCommands(append([]string{command}, arg...)...)
	return &prog{instanceName: instanceName, dirName: dir, cmd: cmd}
}

func runSwarm(leechCount int) (err error) {
	var rootDir string
	rootDir, err = ioutil.TempDir("", "swarm")
	if err != nil {
		return
	}
	log.Printf("Temporary directory: %s", rootDir)
	seedDir := path.Join(rootDir, "seed")
	err = os.Mkdir(seedDir, 0700)
	if err != nil {
		return
	}
	seedData := path.Join(seedDir, "data")
	err = createDataFile(seedData, 1024*1024)
	if err != nil {
		return
	}
	torrentFile := path.Join(rootDir, "testSwarm.torrent")
	err = createTorrentFile(torrentFile, seedData, "127.0.0.1:8080/announce")
	if err != nil {
		return
	}

	doneCh := make(chan *prog, 1)

	tracker := newTracker("tracker", ":8080", rootDir, torrentFile)
	err = tracker.start(doneCh)
	if err != nil {
		return
	}
	defer tracker.kill()
	time.Sleep(100 * time.Microsecond)

	var seed, leech *prog
	seed = newTorrentClient("seed", 7000, torrentFile, seedDir, math.Inf(0))
	err = seed.start(doneCh)
	if err != nil {
		return
	}
	defer seed.kill()
	time.Sleep(50 * time.Microsecond)

	for l := 0; l < leechCount; l++ {
		leechDir := path.Join(rootDir, fmt.Sprintf("leech %d", l))
		err = os.Mkdir(leechDir, 0700)
		if err != nil {
			return
		}
		leech = newTorrentClient(fmt.Sprintf("leech %d", l), 7001+l, torrentFile, leechDir, 0)
		err = leech.start(doneCh)
		if err != nil {
			return
		}
		defer leech.kill()
	}

	timeout := make(chan bool, 1)
	go func() {
		// It takes about 3.5 seconds to complete the test on my computer.
		time.Sleep(50 * time.Second)
		timeout <- true
	}()

	for doneCount := 0; doneCount < leechCount; doneCount++ {
		select {
		case <-timeout:
			err = fmt.Errorf("Timout exceeded")
		case donePeer := <-doneCh:
			if donePeer == tracker || donePeer == seed {
				err = fmt.Errorf("%v finished before all leeches. Should not have.", donePeer)
			}
			err = compareData(seedData, donePeer.dirName)
		}
		if err != nil {
			return
		}
		log.Printf("Done: %d of %d", (doneCount + 1), leechCount)
	}
	if err != nil {
		return
	}
	// All is good. Clean up
	os.RemoveAll(rootDir)

	return
}

func newTracker(name string, addr string, fileDir string, torrentFile string) (p *prog) {
	return newProg(name, fileDir, "tracker", addr, torrentFile)
}

func newTorrentClient(name string, port int, torrentFile string, fileDir string, ratio float64) (p *prog) {
	return newProg(name, fileDir, "client",
		fmt.Sprintf("%v", port),
		fileDir,
		fmt.Sprintf("%v", ratio),
		torrentFile)
}

func createTorrentFile(torrentFileName, root, announcePath string) (err error) {
	var metaInfo *torrent.MetaInfo
	metaInfo, err = torrent.CreateMetaInfoFromFileSystem(nil, root, 0, false)
	if err != nil {
		return
	}
	metaInfo.Announce = "http://127.0.0.1:8080/announce"
	metaInfo.CreatedBy = "testSwarm"
	var torrentFile *os.File
	torrentFile, err = os.Create(torrentFileName)
	if err != nil {
		return
	}
	defer torrentFile.Close()
	err = metaInfo.Bencode(torrentFile)
	if err != nil {
		return
	}
	return
}

func createDataFile(name string, length int64) (err error) {
	if (length & 3) != 0 {
		return fmt.Errorf("createDataFile only supports length that is a multiple of 4. Not %d", length)
	}
	var file *os.File
	file, err = os.Create(name)
	if err != nil {
		return
	}
	defer file.Close()
	err = file.Truncate(length)
	if err != nil {
		return
	}
	w := bufio.NewWriter(file)
	b := make([]byte, 4)
	for i := int64(0); i < length; i += 4 {
		b[0] = byte(i >> 24)
		b[1] = byte(i >> 16)
		b[2] = byte(i >> 8)
		b[3] = byte(i)
		_, err = w.Write(b)
		if err != nil {
			return
		}
	}
	return
}

func compareData(sourceName, copyDirName string) (err error) {
	_, base := path.Split(sourceName)
	copyName := path.Join(copyDirName, base)
	err = compare(sourceName, copyName)
	return
}

// Compare two files (or directories) for equality.
func compare(aName, bName string) (err error) {
	var aFileInfo, bFileInfo os.FileInfo
	aFileInfo, err = os.Stat(aName)
	if err != nil {
		return
	}
	bFileInfo, err = os.Stat(bName)
	if err != nil {
		return
	}
	aIsDir, bIsDir := aFileInfo.IsDir(), bFileInfo.IsDir()
	if aIsDir != bIsDir {
		return fmt.Errorf("%s.IsDir() == %v != %s.IsDir() == %v",
			aName, aIsDir,
			bName, bIsDir)
	}
	var aFile, bFile *os.File
	aFile, err = os.Open(aName)
	if err != nil {
		return
	}
	defer aFile.Close()
	bFile, err = os.Open(bName)
	if err != nil {
		return
	}
	defer bFile.Close()
	if !aIsDir {
		aSize, bSize := aFileInfo.Size(), bFileInfo.Size()
		if aSize != bSize {
			return fmt.Errorf("%s.Size() == %v != %s.Size() == %v",
				aName, aSize,
				bName, bSize)
		}
		var aBuf, bBuf bytes.Buffer
		bufferSize := int64(128 * 1024)
		for i := int64(0); i < aSize; i += bufferSize {
			toRead := bufferSize
			remainder := aSize - i
			if toRead > remainder {
				toRead = remainder
			}
			_, err = io.CopyN(&aBuf, aFile, toRead)
			if err != nil {
				return
			}
			_, err = io.CopyN(&bBuf, bFile, toRead)
			if err != nil {
				return
			}
			aBytes, bBytes := aBuf.Bytes(), bBuf.Bytes()
			for j := int64(0); j < toRead; j++ {
				a, b := aBytes[j], bBytes[j]
				if a != b {
					err = fmt.Errorf("%s[%d] %d != %d", aName, i+j, a, b)
					return
				}
			}
			aBuf.Reset()
			bBuf.Reset()
		}
	} else {
		var aNames, bNames []string
		aNames, err = aFile.Readdirnames(0)
		if err != nil {
			return
		}
		bNames, err = bFile.Readdirnames(0)
		if err != nil {
			return
		}
		if len(aNames) != len(bName) {
			err = fmt.Errorf("Directories %v and %v don't contain same number of files %d != %d",
				aName, bName, len(aNames), len(bNames))
		}
		for _, name := range aNames {
			err = compare(path.Join(aName, name), path.Join(bName, name))
			if err != nil {
				return
			}
		}
	}
	return
}

// type logWriter

type logWriter string

func (l logWriter) Write(p []byte) (n int, err error) {
	log.Println(l, string(p))
	n = len(p)
	return
}

// A test that's used to run multiple processes. From http://golang.org/src/pkg/os/exec/exec_test.go

func helperCommands(s ...string) *exec.Cmd {
	cs := []string{"-test.run=TestHelperProcess", "--"}
	cs = append(cs, s...)
	cmd := exec.Command(os.Args[0], cs...)
	cmd.Env = []string{"GO_WANT_HELPER_PROCESS=1"}
	return cmd
}

func TestHelperProcess(*testing.T) {
	if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" {
		return
	}

	defer os.Exit(0)

	err := testHelperProcessImp(os.Args)
	if err != nil {
		fmt.Fprintf(os.Stderr, "error %v\n", err)
		os.Exit(3)
	}
}

func testHelperProcessImp(args []string) (err error) {
	for len(args) > 0 {
		if args[0] == "--" {
			args = args[1:]
			break
		}
		args = args[1:]
	}

	if len(args) == 0 {
		err = fmt.Errorf("No commands\n")
		return
	}

	cmd, args := args[0], args[1:]
	switch cmd {
	case "tracker":
		if len(args) < 2 {
			err = fmt.Errorf("tracker expected 2 or more args\n")
			return
		}
		addr, torrentFiles := args[0], args[1:]

		err = StartTracker(addr, torrentFiles)
		if err != nil {
			return
		}
	case "client":
		if len(args) < 4 {
			err = fmt.Errorf("client expected 4 or more args\n")
			return
		}
		portStr, fileDir, seedRatioStr, torrentFiles :=
			args[0], args[1], args[2], args[3:]
		var port uint64
		port, err = strconv.ParseUint(portStr, 10, 16)
		if err != nil {
			return
		}
		var seedRatio float64
		seedRatio, err = strconv.ParseFloat(seedRatioStr, 64)
		err = torrent.RunTorrents(int(port), fileDir, seedRatio, torrentFiles)
		if err != nil {
			return
		}
	default:
		err = fmt.Errorf("Unknown command %q\n", cmd)
		return
	}
	return
}
<commit_msg>Allow tracker and torrents to be run from tests.<commit_after>package tracker

import (
	"bufio"
	"bytes"
	"fmt"
	"github.com/jackpal/Taipei-Torrent/torrent"
	"io"
	"io/ioutil"
	"log"
	"math"
	"os"
	"os/exec"
	"path"
	"strconv"
	"testing"
	"time"
)

func TestScrapeURL(t *testing.T) {
	tests := []struct{ announce, scrape string }{
		{"", ""},
		{"foo", ""},
		{"x/announce", "x/scrape"},
		{"x/announce?ad#3", "x/scrape?ad#3"},
		{"announce/x", ""},
	}
	for _, test := range tests {
		scrape := ScrapePattern(test.announce)
		if scrape != test.scrape {
			t.Errorf("ScrapeURL(%#v) = %#v. Expected %#v", test.announce, scrape, test.scrape)
		}
	}
}

func TestSwarm1(t *testing.T) {
	testSwarm(t, 1)
}

func TestSwarm10(t *testing.T) {
	testSwarm(t, 10)
}

/* Larger sizes don't work correctly.

func TestSwarm20(t *testing.T) {
	testSwarm(t, 20)
}

func TestSwarm50(t *testing.T) {
	testSwarm(t, 50)
}

func TestSwarm100(t *testing.T) {
	testSwarm(t, 100)
}

*/

func testSwarm(t *testing.T, leechCount int) {
	err := runSwarm(leechCount)
	if err != nil {
		t.Fatal("Error running testSwarm", err)
	}
}

type prog struct {
	instanceName string
	dirName      string
	cmd          *exec.Cmd
}

func (p *prog) start(doneCh chan *prog) (err error) {
	log.Println("starting", p.instanceName)
	out := logWriter(p.instanceName)
	p.cmd.Stdout = &out
	p.cmd.Stderr = &out
	err = p.cmd.Start()
	if err != nil {
		return
	}
	go func() {
		p.cmd.Wait()
		doneCh <- p
	}()
	return
}

func (p *prog) kill() (err error) {
	err = p.cmd.Process.Kill()
	return
}

func newProg(instanceName string, dir string, command string, arg ...string) (p *prog) {
	cmd := helperCommands(append([]string{command}, arg...)...)
	return &prog{instanceName: instanceName, dirName: dir, cmd: cmd}
}

func runSwarm(leechCount int) (err error) {
	var rootDir string
	rootDir, err = ioutil.TempDir("", "swarm")
	if err != nil {
		return
	}
	log.Printf("Temporary directory: %s", rootDir)
	seedDir := path.Join(rootDir, "seed")
	err = os.Mkdir(seedDir, 0700)
	if err != nil {
		return
	}
	seedData := path.Join(seedDir, "data")
	err = createDataFile(seedData, 1024*1024)
	if err != nil {
		return
	}
	torrentFile := path.Join(rootDir, "testSwarm.torrent")
	err = createTorrentFile(torrentFile, seedData, "127.0.0.1:8080/announce")
	if err != nil {
		return
	}

	doneCh := make(chan *prog, 1)

	tracker := newTracker("tracker", ":8080", rootDir, torrentFile)
	err = tracker.start(doneCh)
	if err != nil {
		return
	}
	defer tracker.kill()
	time.Sleep(100 * time.Microsecond)

	var seed, leech *prog
	seed = newTorrentClient("seed", 7000, torrentFile, seedDir, math.Inf(0))
	err = seed.start(doneCh)
	if err != nil {
		return
	}
	defer seed.kill()
	time.Sleep(50 * time.Microsecond)

	for l := 0; l < leechCount; l++ {
		leechDir := path.Join(rootDir, fmt.Sprintf("leech %d", l))
		err = os.Mkdir(leechDir, 0700)
		if err != nil {
			return
		}
		leech = newTorrentClient(fmt.Sprintf("leech %d", l), 7001+l, torrentFile, leechDir, 0)
		err = leech.start(doneCh)
		if err != nil {
			return
		}
		defer leech.kill()
	}

	timeout := make(chan bool, 1)
	go func() {
		// It takes about 3.5 seconds to complete the test on my computer.
		time.Sleep(50 * time.Second)
		timeout <- true
	}()

	for doneCount := 0; doneCount < leechCount; doneCount++ {
		select {
		case <-timeout:
			err = fmt.Errorf("Timout exceeded")
		case donePeer := <-doneCh:
			if donePeer == tracker || donePeer == seed {
				err = fmt.Errorf("%v finished before all leeches. Should not have.", donePeer)
			}
			err = compareData(seedData, donePeer.dirName)
		}
		if err != nil {
			return
		}
		log.Printf("Done: %d of %d", (doneCount + 1), leechCount)
	}
	if err != nil {
		return
	}
	// All is good. Clean up
	os.RemoveAll(rootDir)

	return
}

func newTracker(name string, addr string, fileDir string, torrentFile string) (p *prog) {
	return newProg(name, fileDir, "tracker", addr, torrentFile)
}

func newTorrentClient(name string, port int, torrentFile string, fileDir string, ratio float64) (p *prog) {
	return newProg(name, fileDir, "client",
		fmt.Sprintf("%v", port),
		fileDir,
		fmt.Sprintf("%v", ratio),
		torrentFile)
}

func createTorrentFile(torrentFileName, root, announcePath string) (err error) {
	var metaInfo *torrent.MetaInfo
	metaInfo, err = torrent.CreateMetaInfoFromFileSystem(nil, root, 0, false)
	if err != nil {
		return
	}
	metaInfo.Announce = "http://127.0.0.1:8080/announce"
	metaInfo.CreatedBy = "testSwarm"
	var torrentFile *os.File
	torrentFile, err = os.Create(torrentFileName)
	if err != nil {
		return
	}
	defer torrentFile.Close()
	err = metaInfo.Bencode(torrentFile)
	if err != nil {
		return
	}
	return
}

func createDataFile(name string, length int64) (err error) {
	if (length & 3) != 0 {
		return fmt.Errorf("createDataFile only supports length that is a multiple of 4. Not %d", length)
	}
	var file *os.File
	file, err = os.Create(name)
	if err != nil {
		return
	}
	defer file.Close()
	err = file.Truncate(length)
	if err != nil {
		return
	}
	w := bufio.NewWriter(file)
	b := make([]byte, 4)
	for i := int64(0); i < length; i += 4 {
		b[0] = byte(i >> 24)
		b[1] = byte(i >> 16)
		b[2] = byte(i >> 8)
		b[3] = byte(i)
		_, err = w.Write(b)
		if err != nil {
			return
		}
	}
	return
}

func compareData(sourceName, copyDirName string) (err error) {
	_, base := path.Split(sourceName)
	copyName := path.Join(copyDirName, base)
	err = compare(sourceName, copyName)
	return
}

// Compare two files (or directories) for equality.
func compare(aName, bName string) (err error) {
	var aFileInfo, bFileInfo os.FileInfo
	aFileInfo, err = os.Stat(aName)
	if err != nil {
		return
	}
	bFileInfo, err = os.Stat(bName)
	if err != nil {
		return
	}
	aIsDir, bIsDir := aFileInfo.IsDir(), bFileInfo.IsDir()
	if aIsDir != bIsDir {
		return fmt.Errorf("%s.IsDir() == %v != %s.IsDir() == %v",
			aName, aIsDir,
			bName, bIsDir)
	}
	var aFile, bFile *os.File
	aFile, err = os.Open(aName)
	if err != nil {
		return
	}
	defer aFile.Close()
	bFile, err = os.Open(bName)
	if err != nil {
		return
	}
	defer bFile.Close()
	if !aIsDir {
		aSize, bSize := aFileInfo.Size(), bFileInfo.Size()
		if aSize != bSize {
			return fmt.Errorf("%s.Size() == %v != %s.Size() == %v",
				aName, aSize,
				bName, bSize)
		}
		var aBuf, bBuf bytes.Buffer
		bufferSize := int64(128 * 1024)
		for i := int64(0); i < aSize; i += bufferSize {
			toRead := bufferSize
			remainder := aSize - i
			if toRead > remainder {
				toRead = remainder
			}
			_, err = io.CopyN(&aBuf, aFile, toRead)
			if err != nil {
				return
			}
			_, err = io.CopyN(&bBuf, bFile, toRead)
			if err != nil {
				return
			}
			aBytes, bBytes := aBuf.Bytes(), bBuf.Bytes()
			for j := int64(0); j < toRead; j++ {
				a, b := aBytes[j], bBytes[j]
				if a != b {
					err = fmt.Errorf("%s[%d] %d != %d", aName, i+j, a, b)
					return
				}
			}
			aBuf.Reset()
			bBuf.Reset()
		}
	} else {
		var aNames, bNames []string
		aNames, err = aFile.Readdirnames(0)
		if err != nil {
			return
		}
		bNames, err = bFile.Readdirnames(0)
		if err != nil {
			return
		}
		if len(aNames) != len(bName) {
			err = fmt.Errorf("Directories %v and %v don't contain same number of files %d != %d",
				aName, bName, len(aNames), len(bNames))
		}
		for _, name := range aNames {
			err = compare(path.Join(aName, name), path.Join(bName, name))
			if err != nil {
				return
			}
		}
	}
	return
}

// type logWriter

type logWriter string

func (l logWriter) Write(p []byte) (n int, err error) {
	log.Println(l, string(p))
	n = len(p)
	return
}

// A test that's used to run multiple processes. From http://golang.org/src/pkg/os/exec/exec_test.go

func helperCommands(s ...string) *exec.Cmd {
	cs := []string{"-test.run=TestHelperProcess", "--"}
	cs = append(cs, s...)
	cmd := exec.Command(os.Args[0], cs...)
	cmd.Env = []string{"GO_WANT_HELPER_PROCESS=1"}
	return cmd
}

func TestHelperProcess(*testing.T) {
	if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" {
		return
	}

	defer os.Exit(0)

	err := testHelperProcessImp(os.Args)
	if err != nil {
		fmt.Fprintf(os.Stderr, "error %v\n", err)
		os.Exit(3)
	}
}

func testHelperProcessImp(args []string) (err error) {
	for len(args) > 0 {
		if args[0] == "--" {
			args = args[1:]
			break
		}
		args = args[1:]
	}

	if len(args) == 0 {
		err = fmt.Errorf("No commands\n")
		return
	}

	cmd, args := args[0], args[1:]
	switch cmd {
	case "tracker":
		if len(args) < 2 {
			err = fmt.Errorf("tracker expected 2 or more args\n")
			return
		}
		addr, torrentFiles := args[0], args[1:]

		err = StartTracker(addr, torrentFiles)
		if err != nil {
			return
		}
	case "client":
		if len(args) < 4 {
			err = fmt.Errorf("client expected 4 or more args\n")
			return
		}
		portStr, fileDir, seedRatioStr, torrentFiles :=
			args[0], args[1], args[2], args[3:]
		var port uint64
		port, err = strconv.ParseUint(portStr, 10, 16)
		if err != nil {
			return
		}
		var seedRatio float64
		seedRatio, err = strconv.ParseFloat(seedRatioStr, 64)
		err = torrent.RunTorrents(int(port), fileDir, seedRatio, torrentFiles)
		if err != nil {
			return
		}
	default:
		err = fmt.Errorf("Unknown command %q\n", cmd)
		return
	}
	return
}
<|endoftext|>


<commit_before>package core

import (
	"errors"
	"fmt"

	"github.com/connectordb/pipescript"
)

type imapTransform struct {
	script    *pipescript.Script // The uninitialized script to be used for splitting
	iter      *pipescript.SingleDatapointIterator
	scriptmap map[string]*pipescript.Script // Map of initialized scripts
	datamap   map[string]interface{}        // Map of data associated with scripts
}

func (t *imapTransform) Copy() (pipescript.TransformInstance, error) {
	var err error
	scriptmap := make(map[string]*pipescript.Script)
	for i, val := range t.scriptmap {
		scriptmap[i], err = val.Copy()
		if err != nil {
			return nil, err
		}
	}
	datamap := make(map[string]interface{})
	for i, val := range t.datamap {
		datamap[i] = val // No need to worry about copying datapoints
	}
	return &imapTransform{t.script, &pipescript.SingleDatapointIterator{}, scriptmap, datamap}, nil
}

func (t *imapTransform) Next(ti *pipescript.TransformIterator) (*pipescript.Datapoint, error) {
	te := ti.Next()
	if te.IsFinished() {

		// We need to take special care of finished sequences by clearing out the map, since the script may be reused
		t.scriptmap = make(map[string]*pipescript.Script)
		t.datamap = make(map[string]interface{})

		return te.Get()
	}

	// Convert the key value to string
	v, err := te.Args[0].DataString()
	if err != nil {
		return nil, err
	}

	//Check if the value exists
	s, ok := t.scriptmap[v]
	if !ok {
		if len(t.scriptmap) >= SplitMax {
			return nil, fmt.Errorf("Reached maximum map amount %d.", SplitMax)
		}

		// Initialize the new script, and add it to our map
		s, err = t.script.Copy()
		if err != nil {
			return nil, err
		}
		//Set the script input to be the internal iterator
		s.SetInput(t.iter)
		t.scriptmap[v] = s
	}

	//Send the current datapoint to the iterator
	t.iter.Set(te.Datapoint, nil)
	dp, err := s.Next()
	if err != nil {
		return nil, err
	}

	// Set the data in our map
	t.datamap[v] = dp.Data

	//Return the map
	return te.Set(t.datamap)
}

// Map splits the dtaapoints by its first argument
var IMap = pipescript.Transform{
	Name:        "imap",
	Description: `Same as map, but returns all intermediate values (is one to one)`,
	OneToOne:    true,
	Args: []pipescript.TransformArg{
		{
			Description: "The value to split on. This must be something that can be converted to string.",
		},
		{
			Description: "The script to instantiate for each different value of the first argument.",
			Hijacked:    true,
		},
	},
	Generator: func(name string, args []*pipescript.Script) (*pipescript.TransformInitializer, error) {
		if args[1].Peek {
			return nil, errors.New("Imap cannot be used with transforms that peek.")
		}
		scriptmap := make(map[string]*pipescript.Script)
		datamap := make(map[string]interface{})
		return &pipescript.TransformInitializer{
			Args:      []*pipescript.Script{args[0]},
			Transform: &imapTransform{args[1], &pipescript.SingleDatapointIterator{}, scriptmap, datamap},
		}, nil
	},
}
<commit_msg>Fixed copy map issue<commit_after>package core

import (
	"errors"
	"fmt"

	"github.com/connectordb/pipescript"
)

type imapTransform struct {
	script    *pipescript.Script // The uninitialized script to be used for splitting
	iter      *pipescript.SingleDatapointIterator
	scriptmap map[string]*pipescript.Script // Map of initialized scripts
	datamap   map[string]interface{}        // Map of data associated with scripts
}

func (t *imapTransform) Copy() (pipescript.TransformInstance, error) {
	var err error
	scriptmap := make(map[string]*pipescript.Script)
	for i, val := range t.scriptmap {
		scriptmap[i], err = val.Copy()
		if err != nil {
			return nil, err
		}
	}
	datamap := make(map[string]interface{})
	for i, val := range t.datamap {
		datamap[i] = val // No need to worry about copying datapoints
	}
	return &imapTransform{t.script, &pipescript.SingleDatapointIterator{}, scriptmap, datamap}, nil
}

func (t *imapTransform) Next(ti *pipescript.TransformIterator) (*pipescript.Datapoint, error) {
	te := ti.Next()
	if te.IsFinished() {

		// We need to take special care of finished sequences by clearing out the map, since the script may be reused
		t.scriptmap = make(map[string]*pipescript.Script)
		t.datamap = make(map[string]interface{})

		return te.Get()
	}

	// Convert the key value to string
	v, err := te.Args[0].DataString()
	if err != nil {
		return nil, err
	}

	//Check if the value exists
	s, ok := t.scriptmap[v]
	if !ok {
		if len(t.scriptmap) >= SplitMax {
			return nil, fmt.Errorf("Reached maximum map amount %d.", SplitMax)
		}

		// Initialize the new script, and add it to our map
		s, err = t.script.Copy()
		if err != nil {
			return nil, err
		}
		//Set the script input to be the internal iterator
		s.SetInput(t.iter)
		t.scriptmap[v] = s
	}

	//Send the current datapoint to the iterator
	t.iter.Set(te.Datapoint, nil)
	dp, err := s.Next()
	if err != nil {
		return nil, err
	}

	// Set the data in our map
	t.datamap[v] = dp.Data

	// We must copy the internal map, since we might be comparing previous values to now!
	datamap := make(map[string]interface{})
	for i, val := range t.datamap {
		datamap[i] = val // No need to worry about copying datapoints
	}

	//Return the map
	return te.Set(datamap)
}

// Map splits the datapoints by its first argument
var IMap = pipescript.Transform{
	Name:        "imap",
	Description: `Same as map, but returns all intermediate values (is one to one)`,
	OneToOne:    true,
	Args: []pipescript.TransformArg{
		{
			Description: "The value to split on. This must be something that can be converted to string.",
		},
		{
			Description: "The script to instantiate for each different value of the first argument.",
			Hijacked:    true,
		},
	},
	Generator: func(name string, args []*pipescript.Script) (*pipescript.TransformInitializer, error) {
		if args[1].Peek {
			return nil, errors.New("Imap cannot be used with transforms that peek.")
		}
		scriptmap := make(map[string]*pipescript.Script)
		datamap := make(map[string]interface{})
		return &pipescript.TransformInitializer{
			Args:      []*pipescript.Script{args[0]},
			Transform: &imapTransform{args[1], &pipescript.SingleDatapointIterator{}, scriptmap, datamap},
		}, nil
	},
}
<|endoftext|>


<commit_before><commit_msg>fix panic in rrd<commit_after><|endoftext|>


<commit_before><commit_msg>unused card gallery now includes thumbnail images<commit_after><|endoftext|>


<commit_before><commit_msg>show POST content when debugging<commit_after><|endoftext|>


<commit_before>package httpd

import (
	"encoding/base64"
	"fmt"
	"net"
	"net/http"
	"strconv"
	"strings"
	"time"
)

type loggingResponseWriter interface {
	http.ResponseWriter
	Status() int
	Size() int
}

// responseLogger is wrapper of http.ResponseWriter that keeps track of its HTTP status
// code and body size
type responseLogger struct {
	w      http.ResponseWriter
	status int
	size   int
}

func (l *responseLogger) Header() http.Header {
	return l.w.Header()
}

func (l *responseLogger) Write(b []byte) (int, error) {
	if l.status == 0 {
		// Set status if WriteHeader has not been called
		l.status = http.StatusOK
	}
	size, err := l.w.Write(b)
	l.size += size
	return size, err
}

func (l *responseLogger) WriteHeader(s int) {
	l.w.WriteHeader(s)
	l.status = s
}

func (l *responseLogger) Status() int {
	return l.status
}

func (l *responseLogger) Size() int {
	return l.size
}

// Common Log Format: http://en.wikipedia.org/wiki/Common_Log_Format

// buildLogLine creates a common log format
// in addittion to the common fields, we also append referrer, user agent and request ID
func buildLogLine(l *responseLogger, r *http.Request, start time.Time) string {
	username := parseUsername(r)

	host, _, err := net.SplitHostPort(r.RemoteAddr)

	if err != nil {
		host = r.RemoteAddr
	}

	uri := r.URL.RequestURI()

	referer := r.Referer()

	userAgent := r.UserAgent()

	fields := []string{
		host,
		"-",
		detect(username, "-"),
		fmt.Sprintf("[%s]", start.Format("02/Jan/2006:15:04:05 -0700")),
		r.Method,
		uri,
		r.Proto,
		detect(strconv.Itoa(l.Status()), "-"),
		strconv.Itoa(l.Size()),
		detect(referer, "-"),
		detect(userAgent, "-"),
		r.Header.Get("Request-Id"),
	}

	return strings.Join(fields, " ")
}

// detect detects the first presense of a non blank string and returns it
func detect(values ...string) string {
	for _, v := range values {
		if v != "" {
			return v
		}
	}
	return ""
}

// parses the uesrname either from the url or auth header
func parseUsername(r *http.Request) string {
	var (
		username = ""
		url      = r.URL
	)

	// get username from the url if passed there
	if url.User != nil {
		if name := url.User.Username(); name != "" {
			username = name
		}
	}

	// Try to get it from the authorization header if set there
	if username == "" {
		auth := r.Header.Get("Authorization")
		fields := strings.Split(auth, " ")
		if len(fields) == 2 {
			bs, err := base64.StdEncoding.DecodeString(fields[1])
			if err == nil {
				fields = strings.Split(string(bs), ":")
				if len(fields) >= 1 {
					username = fields[0]
				}
			}
		}
	}
	return username
}
<commit_msg>simpler basic username parsing for http logger<commit_after>package httpd

import (
	"fmt"
	"net"
	"net/http"
	"strconv"
	"strings"
	"time"
)

type loggingResponseWriter interface {
	http.ResponseWriter
	Status() int
	Size() int
}

// responseLogger is wrapper of http.ResponseWriter that keeps track of its HTTP status
// code and body size
type responseLogger struct {
	w      http.ResponseWriter
	status int
	size   int
}

func (l *responseLogger) Header() http.Header {
	return l.w.Header()
}

func (l *responseLogger) Write(b []byte) (int, error) {
	if l.status == 0 {
		// Set status if WriteHeader has not been called
		l.status = http.StatusOK
	}
	size, err := l.w.Write(b)
	l.size += size
	return size, err
}

func (l *responseLogger) WriteHeader(s int) {
	l.w.WriteHeader(s)
	l.status = s
}

func (l *responseLogger) Status() int {
	return l.status
}

func (l *responseLogger) Size() int {
	return l.size
}

// Common Log Format: http://en.wikipedia.org/wiki/Common_Log_Format

// buildLogLine creates a common log format
// in addittion to the common fields, we also append referrer, user agent and request ID
func buildLogLine(l *responseLogger, r *http.Request, start time.Time) string {
	username := parseUsername(r)

	host, _, err := net.SplitHostPort(r.RemoteAddr)

	if err != nil {
		host = r.RemoteAddr
	}

	uri := r.URL.RequestURI()

	referer := r.Referer()

	userAgent := r.UserAgent()

	fields := []string{
		host,
		"-",
		detect(username, "-"),
		fmt.Sprintf("[%s]", start.Format("02/Jan/2006:15:04:05 -0700")),
		r.Method,
		uri,
		r.Proto,
		detect(strconv.Itoa(l.Status()), "-"),
		strconv.Itoa(l.Size()),
		detect(referer, "-"),
		detect(userAgent, "-"),
		r.Header.Get("Request-Id"),
	}

	return strings.Join(fields, " ")
}

// detect detects the first presense of a non blank string and returns it
func detect(values ...string) string {
	for _, v := range values {
		if v != "" {
			return v
		}
	}
	return ""
}

// parses the uesrname either from the url or auth header
func parseUsername(r *http.Request) string {
	var (
		username = ""
		url      = r.URL
	)

	// get username from the url if passed there
	if url.User != nil {
		if name := url.User.Username(); name != "" {
			username = name
		}
	}

	// Try to get it from the authorization header if set there
	if username == "" {
		if u, _, ok := r.BasicAuth(); ok {
			username = u
		}
	}
	return username
}
<|endoftext|>