<commit_before>// dacloud.go

package dacloud

/*

#include <dacloud.h>
#include <stdlib.h>

typedef void * dago;
struct dago_cloud {
	struct da_cloud_config dcc;
	char cfile[PATH_MAX];
	int r;
};

typedef struct dago_cloud dago_cloud_t;

dago*
dago_cloud_init(char *cfile)
{
	dago_cloud_t *d = malloc(sizeof(*d));
	strncpy(d->cfile, cfile, sizeof(d->cfile) - 1);
	d->cfile[sizeof(d->cfile) - 1] = '\0';
	return ((dago)d);
}

int
dago_cloud_load(dago _d)
{
	if (_d) {
		dago_cloud_t *d = (dago_cloud_t *)_d;
		return (da_cloud_init(&d->dcc, d->cfile));
	}

	return (-1);
}

void
dago_cloud_free(dago _d)
{
	if (_d) {
		dago_cloud_t *d = (dago_cloud_t *)_d;
		da_cloud_fini(&d->dcc);
		free(d);
	}
}

struct dago_detect {
	struct da_cloud_config *dcc;
	struct da_cloud_header_head h;
	struct da_cloud_property_head p;
	struct da_cloud_property *pp;
	int hr;
};

typedef struct dago_detect dago_detect_t;

dago
dago_cloud_header_init(dago _d)
{
	if (_d) {
		dago_cloud_t *d = (dago_cloud_t *)_d;
		dago_detect_t *dd = malloc(sizeof(*dd));
		dd->dcc = &d->dcc;
		dd->hr = da_cloud_header_init(&dd->h);
		return (dd);
	}

	return (NULL);
}

int
dago_cloud_detect(dago _d)
{
	if (_d) {
		dago_detect_t *d = (dago_detect_t *)_d;
		return (da_cloud_detect(d->dcc, &d->h, &d->p));
	}

	return (-1);
}

void
dago_detect_fini(dago _d)
{
	if (_d) {
		dago_detect_t *d = (dago_detect_t *)_d;
		da_cloud_properties_free(&d->p);
		da_cloud_header_free(&d->h);
	}
}

dago
dago_prop_next(dago _d)
{
	struct da_cloud_property *d = (struct da_cloud_property *)_d;
	return (SLIST_NEXT(d, entries));
}

typedef enum da_cloud_property_type da_cloud_property_type;

da_cloud_property_type
dago_prop_type(dago _d)
{
	struct da_cloud_property *d = (struct da_cloud_property *)_d;
	return (d->type);
}

long
dago_prop_getinteger(dago _d)
{
	struct da_cloud_property *d = (struct da_cloud_property *)_d;
	return (d->value.l);
}

char *
dago_prop_getstring(dago _d)
{
	struct da_cloud_property *d = (struct da_cloud_property *)_d;
	return (d->value.s);
}

typedef struct da_cloud_property da_cloud_property_t;
#cgo LDFLAGS: -L. -L/usr/local/lib -ldacloud
#cgo CFLAGS: -I. -I/usr/local/include
*/
import "C"
import "unsafe"

type DaGo struct {
	dc C.dago;
	R int;
}

func Init(cfile string) DaGo {
	var ret DaGo;
	_cfile := C.CString(cfile);
	ret.dc = (C.dago)(C.dago_cloud_init(_cfile));
	ret.R = (int)(C.dago_cloud_load(ret.dc));
	defer C.free((unsafe.Pointer)(_cfile));
	return ret;
}

func Detect(f DaGo, hdrs map[string]string) map[string]interface{} {
	ret := make(map[string]interface{});
	det := (*C.dago_detect_t)(C.dago_cloud_header_init(f.dc));

	if (det.hr == 0) {
		for k, v := range hdrs {
			key := C.CString(k);
			val := C.CString(v);

			C.da_cloud_header_add(&det.h, key, val);
		}

		if (C.dago_cloud_detect(det) == 0) {
			for det.pp = det.p.list.slh_first;
			    det.pp != nil;
			    det.pp = (*C.da_cloud_property_t)(C.dago_prop_next(det.pp)) {
				    ptype := (C.da_cloud_property_type)(C.dago_prop_type(det.pp));
				    pkey := C.GoString(det.pp.name);

				    switch (ptype) {
				    case C.DA_CLOUD_LONG:
					    value := C.dago_prop_getinteger(det.pp);
					    ret[pkey] = (int)(value);
					    break;
				    case C.DA_CLOUD_BOOL:
					    value := C.dago_prop_getinteger(det.pp);
					    if value == 1 {
						    ret[pkey] = true;
					    } else {
						    ret[pkey] = false;
					    }
					    break;
				    case C.DA_CLOUD_STRING, C.DA_CLOUD_UNKNOWN:
					    value := C.dago_prop_getstring(det.pp);
					    ret[pkey] = C.GoString(value);
					    break;
				}
			}
		}

		C.dago_detect_fini(det);
	}

	return ret;
}

func Finalize(f DaGo) {
	C.dago_cloud_free(f.dc);
}
<commit_msg>update of go wrapper<commit_after>// dacloud.go

package dacloud

/*

#include <dacloud.h>
#include <stdlib.h>

typedef void * dago;
struct dago_cloud {
	struct da_cloud_config dcc;
	char cfile[PATH_MAX];
	int r;
};

typedef struct dago_cloud dago_cloud_t;

dago*
dago_cloud_init(char *cfile)
{
	dago_cloud_t *d = malloc(sizeof(*d));
	strncpy(d->cfile, cfile, sizeof(d->cfile) - 1);
	d->cfile[sizeof(d->cfile) - 1] = '\0';
	return ((dago)d);
}

int
dago_cloud_load(dago _d)
{
	if (_d) {
		dago_cloud_t *d = (dago_cloud_t *)_d;
		return (da_cloud_init(&d->dcc, d->cfile));
	}

	return (-1);
}

void
dago_cloud_free(dago _d)
{
	if (_d) {
		dago_cloud_t *d = (dago_cloud_t *)_d;
		da_cloud_fini(&d->dcc);
		free(d);
	}
}

struct dago_detect {
	struct da_cloud_config *dcc;
	struct da_cloud_header_head h;
	struct da_cloud_property_head p;
	struct da_cloud_property *pp;
	int hr;
};

typedef struct dago_detect dago_detect_t;

dago
dago_cloud_header_init(dago _d)
{
	if (_d) {
		dago_cloud_t *d = (dago_cloud_t *)_d;
		dago_detect_t *dd = malloc(sizeof(*dd));
		dd->dcc = &d->dcc;
		dd->hr = da_cloud_header_init(&dd->h);
		return (dd);
	}

	return (NULL);
}

int
dago_cloud_detect(dago _d)
{
	if (_d) {
		dago_detect_t *d = (dago_detect_t *)_d;
		return (da_cloud_detect(d->dcc, &d->h, &d->p));
	}

	return (-1);
}

void
dago_detect_fini(dago _d)
{
	if (_d) {
		dago_detect_t *d = (dago_detect_t *)_d;
		da_cloud_properties_free(&d->p);
		da_cloud_header_free(&d->h);
	}
}

dago
dago_prop_next(dago _d)
{
	struct da_cloud_property *d = (struct da_cloud_property *)_d;
	return (SLIST_NEXT(d, entries));
}

typedef enum da_cloud_property_type da_cloud_property_type;

da_cloud_property_type
dago_prop_type(dago _d)
{
	struct da_cloud_property *d = (struct da_cloud_property *)_d;
	return (d->type);
}

long
dago_prop_getinteger(dago _d)
{
	struct da_cloud_property *d = (struct da_cloud_property *)_d;
	return (d->value.l);
}

char *
dago_prop_getstring(dago _d)
{
	struct da_cloud_property *d = (struct da_cloud_property *)_d;
	return (d->value.s);
}

typedef struct da_cloud_property da_cloud_property_t;
#cgo LDFLAGS: -L. -L/usr/local/lib -ldacloud
#cgo CFLAGS: -I. -I/usr/local/include
*/
import "C"
import "unsafe"

type DaGo struct {
	dc C.dago;
	R int;
}

func Init(cfile string) DaGo {
	var ret DaGo;
	_cfile := C.CString(cfile);
	ret.dc = (C.dago)(C.dago_cloud_init(_cfile));
	ret.R = (int)(C.dago_cloud_load(ret.dc));
	defer C.free((unsafe.Pointer)(_cfile));
	return ret;
}

func Detect(f DaGo, hdrs map[string]string) map[string]interface{} {
	ret := make(map[string]interface{});
	det := (*C.dago_detect_t)(C.dago_cloud_header_init(f.dc));

	if (det.hr == 0) {
		for k, v := range hdrs {
			key := C.CString(k);
			val := C.CString(v);

			C.da_cloud_header_add(&det.h, key, val);

			defer C.free((unsafe.Pointer)(val));
			defer C.free((unsafe.Pointer)(key));
		}

		if (C.dago_cloud_detect(det) == 0) {
			for det.pp = det.p.list.slh_first;
			    det.pp != nil;
			    det.pp = (*C.da_cloud_property_t)(C.dago_prop_next(det.pp)) {
				    ptype := (C.da_cloud_property_type)(C.dago_prop_type(det.pp));
				    pkey := C.GoString(det.pp.name);

				    switch (ptype) {
				    case C.DA_CLOUD_LONG:
					    value := C.dago_prop_getinteger(det.pp);
					    ret[pkey] = (int)(value);
					    break;
				    case C.DA_CLOUD_BOOL:
					    value := C.dago_prop_getinteger(det.pp);
					    if value == 1 {
						    ret[pkey] = true;
					    } else {
						    ret[pkey] = false;
					    }
					    break;
				    case C.DA_CLOUD_STRING, C.DA_CLOUD_UNKNOWN:
					    value := C.dago_prop_getstring(det.pp);
					    ret[pkey] = C.GoString(value);
					    break;
				}
			}
		}

		C.dago_detect_fini(det);
	}

	return ret;
}

func Finalize(f DaGo) {
	C.dago_cloud_free(f.dc);
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"os"
	"strings"
	"time"

	"go.pedge.io/env"
	"go.pedge.io/proto/server"
	"google.golang.org/grpc"

	"github.com/fsouza/go-dockerclient"
	"github.com/pachyderm/pachyderm"
	"github.com/pachyderm/pachyderm/src/pfs"
	"github.com/pachyderm/pachyderm/src/pkg/container"
	"github.com/pachyderm/pachyderm/src/pps"
	"github.com/pachyderm/pachyderm/src/pps/jobserver"
	"github.com/pachyderm/pachyderm/src/pps/jobserver/run"
	"github.com/pachyderm/pachyderm/src/pps/persist"
	persistserver "github.com/pachyderm/pachyderm/src/pps/persist/server"
	"github.com/pachyderm/pachyderm/src/pps/pipelineserver"
)

var (
	defaultEnv = map[string]string{
		"PPS_ADDRESS":       "0.0.0.0",
		"PPS_PORT":          "651",
		"PPS_TRACE_PORT":    "1051",
		"PPS_DATABASE_NAME": "pachyderm",
	}
)

type appEnv struct {
	PachydermPfsd1Port string `env:"PACHYDERM_PFSD_1_PORT"`
	PfsAddress         string `env:"PFS_ADDRESS"`
	PfsMountDir        string `env:"PFS_MOUNT_DIR"`
	Address            string `env:"PPS_ADDRESS"`
	Port               int    `env:"PPS_PORT"`
	DatabaseAddress    string `env:"PPS_DATABASE_ADDRESS"`
	DatabaseName       string `env:"PPS_DATABASE_NAME"`
	DebugPort          int    `env:"PPS_TRACE_PORT"`
	RemoveContainers   bool   `env:"PPS_REMOVE_CONTAINERS"`
}

func main() {
	env.Main(do, &appEnv{}, defaultEnv)
}

func do(appEnvObj interface{}) error {
	appEnv := appEnvObj.(*appEnv)
	containerClient, err := getContainerClient()
	if err != nil {
		return err
	}
	rethinkAPIClient, err := getRethinkAPIClient(appEnv.DatabaseAddress, appEnv.DatabaseName)
	if err != nil {
		return err
	}
	pfsAddress := appEnv.PachydermPfsd1Port
	if pfsAddress == "" {
		pfsAddress = appEnv.PfsAddress
	} else {
		pfsAddress = strings.Replace(pfsAddress, "tcp://", "", -1)
	}
	clientConn, err := grpc.Dial(pfsAddress, grpc.WithInsecure())
	if err != nil {
		return err
	}
	pfsAPIClient := pfs.NewAPIClient(clientConn)
	jobAPIServer := jobserver.NewAPIServer(
		pfsAPIClient,
		rethinkAPIClient,
		containerClient,
		appEnv.PfsMountDir,
		jobserverrun.JobRunnerOptions{
			RemoveContainers: appEnv.RemoveContainers,
		},
	)
	jobAPIClient := pps.NewLocalJobAPIClient(jobAPIServer)
	pipelineAPIServer := pipelineserver.NewAPIServer(pfsAPIClient, jobAPIClient, rethinkAPIClient)
	errC := make(chan error)
	go func() {
		errC <- protoserver.Serve(
			uint16(appEnv.Port),
			func(s *grpc.Server) {
				pps.RegisterJobAPIServer(s, jobAPIServer)
				pps.RegisterPipelineAPIServer(s, pipelineAPIServer)
			},
			protoserver.ServeOptions{
				DebugPort: uint16(appEnv.DebugPort),
				Version:   pachyderm.Version,
			},
		)
	}()
	// TODO: pretty sure I had this problem before, this is bad, we would
	// prefer a callback for when the server is ready to accept requests
	time.Sleep(1 * time.Second)
	if err := pipelineAPIServer.Start(); err != nil {
		return err
	}
	return <-errC
}

func getContainerClient() (container.Client, error) {
	// TODO: this will just connect to the local instance of docker
	client, err := docker.NewClientFromEnv()
	if err != nil {
		return nil, err
	}
	return container.NewDockerClient(client), nil
}

func getRethinkAPIClient(address string, databaseName string) (persist.APIClient, error) {
	var err error
	if address == "" {
		address, err = getRethinkAddress()
		if err != nil {
			return nil, err
		}
	}
	if err := persistserver.InitDBs(address, databaseName); err != nil {
		return nil, err
	}
	rethinkAPIServer, err := persistserver.NewRethinkAPIServer(address, databaseName)
	if err != nil {
		return nil, err
	}
	return persist.NewLocalAPIClient(rethinkAPIServer), nil
}

func getRethinkAddress() (string, error) {
	rethinkAddr := os.Getenv("RETHINK_PORT_28015_TCP_ADDR")
	if rethinkAddr == "" {
		return "", errors.New("RETHINK_PORT_28015_TCP_ADDR not set")
	}
	return fmt.Sprintf("%s:28015", rethinkAddr), nil
}
<commit_msg>update todo<commit_after>package main

import (
	"errors"
	"fmt"
	"os"
	"strings"
	"time"

	"go.pedge.io/env"
	"go.pedge.io/proto/server"
	"google.golang.org/grpc"

	"github.com/fsouza/go-dockerclient"
	"github.com/pachyderm/pachyderm"
	"github.com/pachyderm/pachyderm/src/pfs"
	"github.com/pachyderm/pachyderm/src/pkg/container"
	"github.com/pachyderm/pachyderm/src/pps"
	"github.com/pachyderm/pachyderm/src/pps/jobserver"
	"github.com/pachyderm/pachyderm/src/pps/jobserver/run"
	"github.com/pachyderm/pachyderm/src/pps/persist"
	persistserver "github.com/pachyderm/pachyderm/src/pps/persist/server"
	"github.com/pachyderm/pachyderm/src/pps/pipelineserver"
)

var (
	defaultEnv = map[string]string{
		"PPS_ADDRESS":       "0.0.0.0",
		"PPS_PORT":          "651",
		"PPS_TRACE_PORT":    "1051",
		"PPS_DATABASE_NAME": "pachyderm",
	}
)

type appEnv struct {
	PachydermPfsd1Port string `env:"PACHYDERM_PFSD_1_PORT"`
	PfsAddress         string `env:"PFS_ADDRESS"`
	PfsMountDir        string `env:"PFS_MOUNT_DIR"`
	Address            string `env:"PPS_ADDRESS"`
	Port               int    `env:"PPS_PORT"`
	DatabaseAddress    string `env:"PPS_DATABASE_ADDRESS"`
	DatabaseName       string `env:"PPS_DATABASE_NAME"`
	DebugPort          int    `env:"PPS_TRACE_PORT"`
	RemoveContainers   bool   `env:"PPS_REMOVE_CONTAINERS"`
}

func main() {
	env.Main(do, &appEnv{}, defaultEnv)
}

func do(appEnvObj interface{}) error {
	appEnv := appEnvObj.(*appEnv)
	containerClient, err := getContainerClient()
	if err != nil {
		return err
	}
	rethinkAPIClient, err := getRethinkAPIClient(appEnv.DatabaseAddress, appEnv.DatabaseName)
	if err != nil {
		return err
	}
	pfsAddress := appEnv.PachydermPfsd1Port
	if pfsAddress == "" {
		pfsAddress = appEnv.PfsAddress
	} else {
		pfsAddress = strings.Replace(pfsAddress, "tcp://", "", -1)
	}
	clientConn, err := grpc.Dial(pfsAddress, grpc.WithInsecure())
	if err != nil {
		return err
	}
	pfsAPIClient := pfs.NewAPIClient(clientConn)
	jobAPIServer := jobserver.NewAPIServer(
		pfsAPIClient,
		rethinkAPIClient,
		containerClient,
		appEnv.PfsMountDir,
		jobserverrun.JobRunnerOptions{
			RemoveContainers: appEnv.RemoveContainers,
		},
	)
	jobAPIClient := pps.NewLocalJobAPIClient(jobAPIServer)
	pipelineAPIServer := pipelineserver.NewAPIServer(pfsAPIClient, jobAPIClient, rethinkAPIClient)
	errC := make(chan error)
	go func() {
		errC <- protoserver.Serve(
			uint16(appEnv.Port),
			func(s *grpc.Server) {
				pps.RegisterJobAPIServer(s, jobAPIServer)
				pps.RegisterPipelineAPIServer(s, pipelineAPIServer)
			},
			protoserver.ServeOptions{
				DebugPort: uint16(appEnv.DebugPort),
				Version:   pachyderm.Version,
			},
		)
	}()
	// TODO: pretty sure without this there is a problem, this is bad, we would
	// prefer a callback for when the server is ready to accept requests
	time.Sleep(1 * time.Second)
	if err := pipelineAPIServer.Start(); err != nil {
		return err
	}
	return <-errC
}

func getContainerClient() (container.Client, error) {
	// TODO: this will just connect to the local instance of docker
	client, err := docker.NewClientFromEnv()
	if err != nil {
		return nil, err
	}
	return container.NewDockerClient(client), nil
}

func getRethinkAPIClient(address string, databaseName string) (persist.APIClient, error) {
	var err error
	if address == "" {
		address, err = getRethinkAddress()
		if err != nil {
			return nil, err
		}
	}
	if err := persistserver.InitDBs(address, databaseName); err != nil {
		return nil, err
	}
	rethinkAPIServer, err := persistserver.NewRethinkAPIServer(address, databaseName)
	if err != nil {
		return nil, err
	}
	return persist.NewLocalAPIClient(rethinkAPIServer), nil
}

func getRethinkAddress() (string, error) {
	rethinkAddr := os.Getenv("RETHINK_PORT_28015_TCP_ADDR")
	if rethinkAddr == "" {
		return "", errors.New("RETHINK_PORT_28015_TCP_ADDR not set")
	}
	return fmt.Sprintf("%s:28015", rethinkAddr), nil
}
<|endoftext|>


<commit_before>package common

import (
	"regexp"
	"strings"
)

// Removes everything between (), [], {}
// FIXME Is it doable in one regexp ?
func ClearName(name string) string {
	st := " *\\([^)]*\\) *"
	reg, _ := regexp.Compile(st)
	res := reg.ReplaceAllString(name, "")

	st = " *\\[[^)]*\\] *"
	reg, _ = regexp.Compile(st)
	res = reg.ReplaceAllString(res, "")

	st = " *\\{[^)]*\\} *"
	reg, _ = regexp.Compile(st)
	res = reg.ReplaceAllString(res, "")

	res = RemoveSpecialChars(res)

	return res
}

// Remove the special chars from the given string
// and returns the result.
func RemoveSpecialChars(s string) string {
	// List of special characters
	specialChars := ".-_\"':[](){}!%#"

	// Copy the string
	result := s

	for i := 0; i < len(specialChars); i++ {
		result = strings.Replace(result, string(specialChars[i]), " ", -1)
	}

	return result
}

// Code re-used from https://github.com/remeh/go-subtitles
// to compute the percentage of matching words into the given strings.
func CompareFilename(first string, second string) float32 {

	// clears both filename
	first = ClearName(first)
	second = ClearName(second)

	// We now have two cleared file names.
	// The idea is to split by spaces and look how much of the "words"
	// of the subtitles filename we can find in the video filename
	havingPercentage := computeHavingPercentage(first, second)
	missingPercentage := computeMissingWordPercentage(first, second)

	return havingPercentage - missingPercentage
}

// Compute the percentage of word matching from subtitle to video filename
func computeHavingPercentage(filename string, subtitleName string) float32 {
	words := strings.Split(subtitleName, " ")
	found := 0
	for _, word := range words {
		if strings.Contains(filename, word) {
			found++
		}
	}
	havingPercentage := float32(found) / float32(len(words))
	return havingPercentage
}

// Compute the percentage of word missing from the subtitle filename.
func computeMissingWordPercentage(filename string, subtitleName string) float32 {
	words := strings.Split(filename, " ")
	found := 0
	for _, word := range words {
		if strings.Contains(subtitleName, word) {
			found++
		}
	}
	missingPercentage := 1.0 - (float32(found) / float32(len(words)))
	return missingPercentage
}
<commit_msg>Comment todo in matching algorithm<commit_after>package common

import (
	"regexp"
	"strings"
)

// Removes everything between (), [], {}
// FIXME Is it doable in one regexp ?
func ClearName(name string) string {
	st := " *\\([^)]*\\) *"
	reg, _ := regexp.Compile(st)
	res := reg.ReplaceAllString(name, "")

	st = " *\\[[^)]*\\] *"
	reg, _ = regexp.Compile(st)
	res = reg.ReplaceAllString(res, "")

	st = " *\\{[^)]*\\} *"
	reg, _ = regexp.Compile(st)
	res = reg.ReplaceAllString(res, "")

	res = RemoveSpecialChars(res)

	return res
}

// Remove the special chars from the given string
// and returns the result.
func RemoveSpecialChars(s string) string {
	// List of special characters
	specialChars := ".-_\"':[](){}!%#"

	// Copy the string
	result := s

	for i := 0; i < len(specialChars); i++ {
		result = strings.Replace(result, string(specialChars[i]), " ", -1)
	}

	return result
}

// Code re-used from https://github.com/remeh/go-subtitles
// to compute the percentage of matching words into the given strings.
func CompareFilename(first string, second string) float32 {

	// clears both filename
	first = ClearName(first)
	second = ClearName(second)

	// We now have two cleared file names.
	// The idea is to split by spaces and look how much of the "words"
	// of the subtitles filename we can find in the video filename
	havingPercentage := computeHavingPercentage(first, second)
	missingPercentage := computeMissingWordPercentage(first, second)

	// TODO FIXME This isn't correcly working as the one
	// not having enough words but the good ones are better match
	// than ones having enough words.
	// Example:
	// For: Castlevania Aria Of Sorrow
	// 'Castlevania' is a better match than
	// 'Castlevania Aria of'

	return havingPercentage - missingPercentage
}

// Compute the percentage of word matching from subtitle to video filename
func computeHavingPercentage(filename string, subtitleName string) float32 {
	words := strings.Split(subtitleName, " ")
	found := 0
	for _, word := range words {
		if strings.Contains(filename, word) {
			found++
		}
	}
	havingPercentage := float32(found) / float32(len(words))
	return havingPercentage
}

// Compute the percentage of word missing from the subtitle filename.
func computeMissingWordPercentage(filename string, subtitleName string) float32 {
	words := strings.Split(filename, " ")
	found := 0
	for _, word := range words {
		if strings.Contains(subtitleName, word) {
			found++
		}
	}
	missingPercentage := 1.0 - (float32(found) / float32(len(words)))
	return missingPercentage
}
<|endoftext|>


<commit_before>package goflat_test

import (
	"text/template"

	. "github.com/aminjam/goflat"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
)

var _ = Describe("Pipes", func() {
	var (
		pipes *Pipes
	)
	BeforeEach(func() {
		pipes = NewPipes()
	})

	Describe("when testing default pipes", func() {
		var (
			tmpl   *template.Template
			buffer *gbytes.Buffer
		)
		BeforeEach(func() {
			tmpl = template.New("tester").Funcs(pipes.Map)
			buffer = gbytes.NewBuffer()
		})
		It("should validate joins method", func() {
			const text = `{{ . | join "," }}`
			tmpl, err := tmpl.Parse(text)
			Expect(err).To(BeNil())
			err = tmpl.Execute(buffer, []string{"a", "b"})
			Eventually(buffer).Should(gbytes.Say(`a,b`))
		})
		Context("when validating a map method", func() {
			type Tester struct{ Name, Job string }
			t := []Tester{
				{"John", "Jabber"},
				{"Cherry", "Chatter"},
			}

			It("should accepet a single Field", func() {
				const text = `{{ . | map "Job" "|" }}`
				tmpl, err := tmpl.Parse(text)
				Expect(err).To(BeNil())
				err = tmpl.Execute(buffer, t)
				Expect(err).To(BeNil())
				Eventually(buffer).Should(gbytes.Say("Jabber Chatter"))
			})
			It("should accepet a comma seperated fields", func() {
				const text = `{{ . | map "Job,Name" "." }}`
				tmpl, err := tmpl.Parse(text)
				Expect(err).To(BeNil())
				err = tmpl.Execute(buffer, t)
				Expect(err).To(BeNil())
				Eventually(buffer).Should(gbytes.Say("Jabber.John Chatter.Cherry"))
			})
		})
		It("should validate replace method", func() {
			const text = `{{ . | replace "A" "D" }}`
			tmpl, err := tmpl.Parse(text)
			Expect(err).To(BeNil())
			err = tmpl.Execute(buffer, "AbCDAa")
			Eventually(buffer).Should(gbytes.Say(`DbCDDa`))
		})
		It("should validate split method", func() {
			const text = `{{ range (. | split " ") }}{{.}}-item {{end}}`
			tmpl, err := tmpl.Parse(text)
			Expect(err).To(BeNil())
			err = tmpl.Execute(buffer, "AB CD")
			Eventually(buffer).Should(gbytes.Say(`AB-item CD-item `))
		})
		It("should validate toUpper method", func() {
			const text = `{{ . | toUpper }}`
			tmpl, err := tmpl.Parse(text)
			Expect(err).To(BeNil())
			err = tmpl.Execute(buffer, "abA")
			Eventually(buffer).Should(gbytes.Say(`ABA`))
		})
		It("should validate toLower method", func() {
			const text = `{{ . | toLower }}`
			tmpl, err := tmpl.Parse(text)
			Expect(err).To(BeNil())
			err = tmpl.Execute(buffer, "BaBA")
			Eventually(buffer).Should(gbytes.Say(`baba`))
		})
	})
})
<commit_msg>Verify template.Execute returns nil for tests<commit_after>package goflat_test

import (
	"text/template"

	. "github.com/aminjam/goflat"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
)

var _ = Describe("Pipes", func() {
	var (
		pipes *Pipes
	)
	BeforeEach(func() {
		pipes = NewPipes()
	})

	Describe("when testing default pipes", func() {
		var (
			tmpl   *template.Template
			buffer *gbytes.Buffer
		)
		BeforeEach(func() {
			tmpl = template.New("tester").Funcs(pipes.Map)
			buffer = gbytes.NewBuffer()
		})
		It("should validate joins method", func() {
			const text = `{{ . | join "," }}`
			tmpl, err := tmpl.Parse(text)
			Expect(err).To(BeNil())
			err = tmpl.Execute(buffer, []string{"a", "b"})
			Expect(err).To(BeNil())
			Eventually(buffer).Should(gbytes.Say(`a,b`))
		})
		Context("when validating a map method", func() {
			type Tester struct{ Name, Job string }
			t := []Tester{
				{"John", "Jabber"},
				{"Cherry", "Chatter"},
			}

			It("should accepet a single Field", func() {
				const text = `{{ . | map "Job" "|" }}`
				tmpl, err := tmpl.Parse(text)
				Expect(err).To(BeNil())
				err = tmpl.Execute(buffer, t)
				Expect(err).To(BeNil())
				Eventually(buffer).Should(gbytes.Say("Jabber Chatter"))
			})
			It("should accepet a comma seperated fields", func() {
				const text = `{{ . | map "Job,Name" "." }}`
				tmpl, err := tmpl.Parse(text)
				Expect(err).To(BeNil())
				err = tmpl.Execute(buffer, t)
				Expect(err).To(BeNil())
				Eventually(buffer).Should(gbytes.Say("Jabber.John Chatter.Cherry"))
			})
		})
		It("should validate replace method", func() {
			const text = `{{ . | replace "A" "D" }}`
			tmpl, err := tmpl.Parse(text)
			Expect(err).To(BeNil())
			err = tmpl.Execute(buffer, "AbCDAa")
			Expect(err).To(BeNil())
			Eventually(buffer).Should(gbytes.Say(`DbCDDa`))
		})
		It("should validate split method", func() {
			const text = `{{ range (. | split " ") }}{{.}}-item {{end}}`
			tmpl, err := tmpl.Parse(text)
			Expect(err).To(BeNil())
			err = tmpl.Execute(buffer, "AB CD")
			Expect(err).To(BeNil())
			Eventually(buffer).Should(gbytes.Say(`AB-item CD-item `))
		})
		It("should validate toUpper method", func() {
			const text = `{{ . | toUpper }}`
			tmpl, err := tmpl.Parse(text)
			Expect(err).To(BeNil())
			err = tmpl.Execute(buffer, "abA")
			Expect(err).To(BeNil())
			Eventually(buffer).Should(gbytes.Say(`ABA`))
		})
		It("should validate toLower method", func() {
			const text = `{{ . | toLower }}`
			tmpl, err := tmpl.Parse(text)
			Expect(err).To(BeNil())
			err = tmpl.Execute(buffer, "BaBA")
			Expect(err).To(BeNil())
			Eventually(buffer).Should(gbytes.Say(`baba`))
		})
	})
})
<|endoftext|>


<commit_before>package datadog

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"time"

	backendTypes "github.com/atlassian/gostatsd/backend/types"
	"github.com/atlassian/gostatsd/types"

	log "github.com/Sirupsen/logrus"
	"github.com/cenkalti/backoff"
	"github.com/spf13/viper"
)

const (
	apiURL = "https://app.datadoghq.com"
	// BackendName is the name of this backend.
	BackendName                  = "datadog"
	dogstatsdVersion             = "5.6.3"
	dogstatsdUserAgent           = "python-requests/2.6.0 CPython/2.7.10"
	defaultMaxRequestElapsedTime = 15 * time.Second
	defaultClientTimeout         = 9 * time.Second
	// defaultMetricsPerBatch is the default number of metrics to send in a single batch.
	defaultMetricsPerBatch = 1000
	// maxResponseSize is the maximum response size we are willing to read.
	maxResponseSize = 10 * 1024
)

// client represents a Datadog client.
type client struct {
	apiKey                string
	apiEndpoint           string
	maxRequestElapsedTime time.Duration
	client                http.Client
	metricsPerBatch       uint
	now                   func() time.Time // Returns current time. Useful for testing.
}

const sampleConfig = `
[datadog]
	## Datadog API key
	api_key = "my-secret-key" # required.

	## Connection timeout.
	# timeout = "5s"
`

// event represents an event data structure for Datadog.
type event struct {
	Title          string   `json:"title"`
	Text           string   `json:"text"`
	DateHappened   int64    `json:"date_happened,omitempty"`
	Hostname       string   `json:"host,omitempty"`
	AggregationKey string   `json:"aggregation_key,omitempty"`
	SourceTypeName string   `json:"source_type_name,omitempty"`
	Tags           []string `json:"tags,omitempty"`
	Priority       string   `json:"priority,omitempty"`
	AlertType      string   `json:"alert_type,omitempty"`
}

// SendMetricsAsync flushes the metrics to Datadog, preparing payload synchronously but doing the send asynchronously.
func (d *client) SendMetricsAsync(ctx context.Context, metrics *types.MetricMap, cb backendTypes.SendCallback) {
	if metrics.NumStats == 0 {
		cb(nil)
		return
	}
	counter := 0
	results := make(chan error)
	d.processMetrics(metrics, func(ts *timeSeries) {
		go func() {
			err := d.postMetrics(ts)
			select {
			case <-ctx.Done():
			case results <- err:
			}
		}()
		counter++
	})
	go func() {
		errs := make([]error, 0, counter)
	loop:
		for c := 0; c < counter; c++ {
			select {
			case <-ctx.Done():
				errs = append(errs, ctx.Err())
				break loop
			case err := <-results:
				errs = append(errs, err)
			}
		}
		cb(errs)
	}()
}

func (d *client) processMetrics(metrics *types.MetricMap, cb func(*timeSeries)) {
	fl := flush{
		ts: &timeSeries{
			Series: make([]metric, 0, d.metricsPerBatch),
		},
		timestamp:        float64(d.now().Unix()),
		flushIntervalSec: metrics.FlushInterval.Seconds(),
		metricsPerBatch:  d.metricsPerBatch,
		cb:               cb,
	}

	metrics.Counters.Each(func(key, tagsKey string, counter types.Counter) {
		fl.addMetric(key, rate, counter.PerSecond, counter.Hostname, counter.Tags)
		fl.addMetric(fmt.Sprintf("%s.count", key), gauge, float64(counter.Value), counter.Hostname, counter.Tags)
		fl.maybeFlush()
	})

	metrics.Timers.Each(func(key, tagsKey string, timer types.Timer) {
		fl.addMetric(fmt.Sprintf("%s.lower", key), gauge, timer.Min, timer.Hostname, timer.Tags)
		fl.addMetric(fmt.Sprintf("%s.upper", key), gauge, timer.Max, timer.Hostname, timer.Tags)
		fl.addMetric(fmt.Sprintf("%s.count", key), gauge, float64(timer.Count), timer.Hostname, timer.Tags)
		fl.addMetric(fmt.Sprintf("%s.count_ps", key), rate, timer.PerSecond, timer.Hostname, timer.Tags)
		fl.addMetric(fmt.Sprintf("%s.mean", key), gauge, timer.Mean, timer.Hostname, timer.Tags)
		fl.addMetric(fmt.Sprintf("%s.median", key), gauge, timer.Median, timer.Hostname, timer.Tags)
		fl.addMetric(fmt.Sprintf("%s.std", key), gauge, timer.StdDev, timer.Hostname, timer.Tags)
		fl.addMetric(fmt.Sprintf("%s.sum", key), gauge, timer.Sum, timer.Hostname, timer.Tags)
		fl.addMetric(fmt.Sprintf("%s.sum_squares", key), gauge, timer.SumSquares, timer.Hostname, timer.Tags)
		for _, pct := range timer.Percentiles {
			fl.addMetric(fmt.Sprintf("%s.%s", key, pct.Str), gauge, pct.Float, timer.Hostname, timer.Tags)
		}
		fl.maybeFlush()
	})

	metrics.Gauges.Each(func(key, tagsKey string, g types.Gauge) {
		fl.addMetric(key, gauge, g.Value, g.Hostname, g.Tags)
		fl.maybeFlush()
	})

	metrics.Sets.Each(func(key, tagsKey string, set types.Set) {
		fl.addMetric(key, gauge, float64(len(set.Values)), set.Hostname, set.Tags)
		fl.maybeFlush()
	})

	fl.finish()
}

func (d *client) postMetrics(ts *timeSeries) error {
	return d.post("/api/v1/series", "metrics", ts)
}

// SendEvent sends an event to Datadog.
func (d *client) SendEvent(ctx context.Context, e *types.Event) error {
	return d.post("/api/v1/events", "events", &event{
		Title:          e.Title,
		Text:           e.Text,
		DateHappened:   e.DateHappened,
		Hostname:       e.Hostname,
		AggregationKey: e.AggregationKey,
		SourceTypeName: e.SourceTypeName,
		Tags:           e.Tags,
		Priority:       e.Priority.StringWithEmptyDefault(),
		AlertType:      e.AlertType.StringWithEmptyDefault(),
	})
}

// SampleConfig returns the sample config for the datadog backend.
func (d *client) SampleConfig() string {
	return sampleConfig
}

// BackendName returns the name of the backend.
func (d *client) BackendName() string {
	return BackendName
}

func (d *client) post(path, typeOfPost string, data interface{}) error {
	tsBytes, err := json.Marshal(data)
	if err != nil {
		return fmt.Errorf("[%s] unable to marshal %s: %v", BackendName, typeOfPost, err)
	}
	log.Debugf("[%s] %s json: %s", BackendName, typeOfPost, tsBytes)

	b := backoff.NewExponentialBackOff()
	b.MaxElapsedTime = d.maxRequestElapsedTime
	err = backoff.RetryNotify(d.doPost(path, tsBytes), b, func(err error, d time.Duration) {
		log.Warnf("[%s] failed to send %s, sleeping for %s: %v", BackendName, typeOfPost, d, err)
	})
	if err != nil {
		return fmt.Errorf("[%s] %v", BackendName, err)
	}

	return nil
}

func (d *client) doPost(path string, body []byte) backoff.Operation {
	authenticatedURL := d.authenticatedURL(path)
	return func() error {
		req, err := http.NewRequest("POST", authenticatedURL, bytes.NewReader(body))
		if err != nil {
			return fmt.Errorf("unable to create http.Request: %v", err)
		}
		req.Header.Set("Content-Type", "application/json")
		// Mimic dogstatsd code
		req.Header.Set("DD-Dogstatsd-Version", dogstatsdVersion)
		req.Header.Set("User-Agent", dogstatsdUserAgent)
		resp, err := d.client.Do(req)
		if err != nil {
			return fmt.Errorf("error POSTing: %s", strings.Replace(err.Error(), d.apiKey, "*****", -1))
		}
		defer resp.Body.Close()
		body := io.LimitReader(resp.Body, maxResponseSize)
		if resp.StatusCode < http.StatusOK || resp.StatusCode > http.StatusNoContent {
			b, _ := ioutil.ReadAll(body)
			log.Infof("[%s] failed request status: %d\n%s", BackendName, resp.StatusCode, b)
			return fmt.Errorf("received bad status code %d", resp.StatusCode)
		}
		_, _ = io.Copy(ioutil.Discard, body)
		return nil
	}
}

func (d *client) authenticatedURL(path string) string {
	q := url.Values{
		"api_key": []string{d.apiKey},
	}
	return fmt.Sprintf("%s%s?%s", d.apiEndpoint, path, q.Encode())
}

// NewClientFromViper returns a new Datadog API client.
func NewClientFromViper(v *viper.Viper) (backendTypes.Backend, error) {
	dd := getSubViper(v, "datadog")
	dd.SetDefault("api_endpoint", apiURL)
	dd.SetDefault("metrics_per_batch", defaultMetricsPerBatch)
	dd.SetDefault("timeout", defaultClientTimeout)
	dd.SetDefault("max_request_elapsed_time", defaultMaxRequestElapsedTime)
	return NewClient(
		dd.GetString("api_endpoint"),
		dd.GetString("api_key"),
		uint(dd.GetInt("metrics_per_batch")),
		dd.GetDuration("timeout"),
		dd.GetDuration("max_request_elapsed_time"),
	)
}

// NewClient returns a new Datadog API client.
func NewClient(apiEndpoint, apiKey string, metricsPerBatch uint, clientTimeout, maxRequestElapsedTime time.Duration) (backendTypes.Backend, error) {
	if apiEndpoint == "" {
		return nil, fmt.Errorf("[%s] apiEndpoint is required", BackendName)
	}
	if apiKey == "" {
		return nil, fmt.Errorf("[%s] apiKey is required", BackendName)
	}
	if metricsPerBatch <= 0 {
		return nil, fmt.Errorf("[%s] metricsPerBatch must be positive", BackendName)
	}
	if clientTimeout <= 0 {
		return nil, fmt.Errorf("[%s] clientTimeout must be positive", BackendName)
	}
	if maxRequestElapsedTime <= 0 {
		return nil, fmt.Errorf("[%s] maxRequestElapsedTime must be positive", BackendName)
	}
	log.Infof("[%s] maxRequestElapsedTime=%s clientTimeout=%s metricsPerBatch=%d", BackendName, maxRequestElapsedTime, clientTimeout, metricsPerBatch)
	return &client{
		apiKey:                apiKey,
		apiEndpoint:           apiEndpoint,
		maxRequestElapsedTime: maxRequestElapsedTime,
		client: http.Client{
			Timeout: clientTimeout,
		},
		metricsPerBatch: metricsPerBatch,
		now:             time.Now,
	}, nil
}

func getSubViper(v *viper.Viper, key string) *viper.Viper {
	n := v.Sub(key)
	if n == nil {
		n = viper.New()
	}
	return n
}
<commit_msg>Pass context to the http client<commit_after>package datadog

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"time"

	backendTypes "github.com/atlassian/gostatsd/backend/types"
	"github.com/atlassian/gostatsd/types"

	log "github.com/Sirupsen/logrus"
	"github.com/cenkalti/backoff"
	"github.com/spf13/viper"
)

const (
	apiURL = "https://app.datadoghq.com"
	// BackendName is the name of this backend.
	BackendName                  = "datadog"
	dogstatsdVersion             = "5.6.3"
	dogstatsdUserAgent           = "python-requests/2.6.0 CPython/2.7.10"
	defaultMaxRequestElapsedTime = 15 * time.Second
	defaultClientTimeout         = 9 * time.Second
	// defaultMetricsPerBatch is the default number of metrics to send in a single batch.
	defaultMetricsPerBatch = 1000
	// maxResponseSize is the maximum response size we are willing to read.
	maxResponseSize = 10 * 1024
)

// client represents a Datadog client.
type client struct {
	apiKey                string
	apiEndpoint           string
	maxRequestElapsedTime time.Duration
	client                http.Client
	metricsPerBatch       uint
	now                   func() time.Time // Returns current time. Useful for testing.
}

const sampleConfig = `
[datadog]
	## Datadog API key
	api_key = "my-secret-key" # required.

	## Connection timeout.
	# timeout = "5s"
`

// event represents an event data structure for Datadog.
type event struct {
	Title          string   `json:"title"`
	Text           string   `json:"text"`
	DateHappened   int64    `json:"date_happened,omitempty"`
	Hostname       string   `json:"host,omitempty"`
	AggregationKey string   `json:"aggregation_key,omitempty"`
	SourceTypeName string   `json:"source_type_name,omitempty"`
	Tags           []string `json:"tags,omitempty"`
	Priority       string   `json:"priority,omitempty"`
	AlertType      string   `json:"alert_type,omitempty"`
}

// SendMetricsAsync flushes the metrics to Datadog, preparing payload synchronously but doing the send asynchronously.
func (d *client) SendMetricsAsync(ctx context.Context, metrics *types.MetricMap, cb backendTypes.SendCallback) {
	if metrics.NumStats == 0 {
		cb(nil)
		return
	}
	counter := 0
	results := make(chan error)
	d.processMetrics(metrics, func(ts *timeSeries) {
		go func() {
			err := d.postMetrics(ctx, ts)
			select {
			case <-ctx.Done():
			case results <- err:
			}
		}()
		counter++
	})
	go func() {
		errs := make([]error, 0, counter)
	loop:
		for c := 0; c < counter; c++ {
			select {
			case <-ctx.Done():
				errs = append(errs, ctx.Err())
				break loop
			case err := <-results:
				errs = append(errs, err)
			}
		}
		cb(errs)
	}()
}

func (d *client) processMetrics(metrics *types.MetricMap, cb func(*timeSeries)) {
	fl := flush{
		ts: &timeSeries{
			Series: make([]metric, 0, d.metricsPerBatch),
		},
		timestamp:        float64(d.now().Unix()),
		flushIntervalSec: metrics.FlushInterval.Seconds(),
		metricsPerBatch:  d.metricsPerBatch,
		cb:               cb,
	}

	metrics.Counters.Each(func(key, tagsKey string, counter types.Counter) {
		fl.addMetric(key, rate, counter.PerSecond, counter.Hostname, counter.Tags)
		fl.addMetric(fmt.Sprintf("%s.count", key), gauge, float64(counter.Value), counter.Hostname, counter.Tags)
		fl.maybeFlush()
	})

	metrics.Timers.Each(func(key, tagsKey string, timer types.Timer) {
		fl.addMetric(fmt.Sprintf("%s.lower", key), gauge, timer.Min, timer.Hostname, timer.Tags)
		fl.addMetric(fmt.Sprintf("%s.upper", key), gauge, timer.Max, timer.Hostname, timer.Tags)
		fl.addMetric(fmt.Sprintf("%s.count", key), gauge, float64(timer.Count), timer.Hostname, timer.Tags)
		fl.addMetric(fmt.Sprintf("%s.count_ps", key), rate, timer.PerSecond, timer.Hostname, timer.Tags)
		fl.addMetric(fmt.Sprintf("%s.mean", key), gauge, timer.Mean, timer.Hostname, timer.Tags)
		fl.addMetric(fmt.Sprintf("%s.median", key), gauge, timer.Median, timer.Hostname, timer.Tags)
		fl.addMetric(fmt.Sprintf("%s.std", key), gauge, timer.StdDev, timer.Hostname, timer.Tags)
		fl.addMetric(fmt.Sprintf("%s.sum", key), gauge, timer.Sum, timer.Hostname, timer.Tags)
		fl.addMetric(fmt.Sprintf("%s.sum_squares", key), gauge, timer.SumSquares, timer.Hostname, timer.Tags)
		for _, pct := range timer.Percentiles {
			fl.addMetric(fmt.Sprintf("%s.%s", key, pct.Str), gauge, pct.Float, timer.Hostname, timer.Tags)
		}
		fl.maybeFlush()
	})

	metrics.Gauges.Each(func(key, tagsKey string, g types.Gauge) {
		fl.addMetric(key, gauge, g.Value, g.Hostname, g.Tags)
		fl.maybeFlush()
	})

	metrics.Sets.Each(func(key, tagsKey string, set types.Set) {
		fl.addMetric(key, gauge, float64(len(set.Values)), set.Hostname, set.Tags)
		fl.maybeFlush()
	})

	fl.finish()
}

func (d *client) postMetrics(ctx context.Context, ts *timeSeries) error {
	return d.post(ctx, "/api/v1/series", "metrics", ts)
}

// SendEvent sends an event to Datadog.
func (d *client) SendEvent(ctx context.Context, e *types.Event) error {
	return d.post(ctx, "/api/v1/events", "events", &event{
		Title:          e.Title,
		Text:           e.Text,
		DateHappened:   e.DateHappened,
		Hostname:       e.Hostname,
		AggregationKey: e.AggregationKey,
		SourceTypeName: e.SourceTypeName,
		Tags:           e.Tags,
		Priority:       e.Priority.StringWithEmptyDefault(),
		AlertType:      e.AlertType.StringWithEmptyDefault(),
	})
}

// SampleConfig returns the sample config for the datadog backend.
func (d *client) SampleConfig() string {
	return sampleConfig
}

// BackendName returns the name of the backend.
func (d *client) BackendName() string {
	return BackendName
}

func (d *client) post(ctx context.Context, path, typeOfPost string, data interface{}) error {
	tsBytes, err := json.Marshal(data)
	if err != nil {
		return fmt.Errorf("[%s] unable to marshal %s: %v", BackendName, typeOfPost, err)
	}
	log.Debugf("[%s] %s json: %s", BackendName, typeOfPost, tsBytes)

	b := backoff.NewExponentialBackOff()
	b.MaxElapsedTime = d.maxRequestElapsedTime
	err = backoff.RetryNotify(d.doPost(ctx, path, tsBytes), b, func(err error, d time.Duration) {
		log.Warnf("[%s] failed to send %s, sleeping for %s: %v", BackendName, typeOfPost, d, err)
	})
	if err != nil {
		return fmt.Errorf("[%s] %v", BackendName, err)
	}

	return nil
}

func (d *client) doPost(ctx context.Context, path string, body []byte) backoff.Operation {
	authenticatedURL := d.authenticatedURL(path)
	return func() error {
		req, err := http.NewRequest("POST", authenticatedURL, bytes.NewReader(body))
		if err != nil {
			return fmt.Errorf("unable to create http.Request: %v", err)
		}
		req = req.WithContext(ctx)
		req.Header.Set("Content-Type", "application/json")
		// Mimic dogstatsd code
		req.Header.Set("DD-Dogstatsd-Version", dogstatsdVersion)
		req.Header.Set("User-Agent", dogstatsdUserAgent)
		resp, err := d.client.Do(req)
		if err != nil {
			return fmt.Errorf("error POSTing: %s", strings.Replace(err.Error(), d.apiKey, "*****", -1))
		}
		defer resp.Body.Close()
		body := io.LimitReader(resp.Body, maxResponseSize)
		if resp.StatusCode < http.StatusOK || resp.StatusCode > http.StatusNoContent {
			b, _ := ioutil.ReadAll(body)
			log.Infof("[%s] failed request status: %d\n%s", BackendName, resp.StatusCode, b)
			return fmt.Errorf("received bad status code %d", resp.StatusCode)
		}
		_, _ = io.Copy(ioutil.Discard, body)
		return nil
	}
}

func (d *client) authenticatedURL(path string) string {
	q := url.Values{
		"api_key": []string{d.apiKey},
	}
	return fmt.Sprintf("%s%s?%s", d.apiEndpoint, path, q.Encode())
}

// NewClientFromViper returns a new Datadog API client.
func NewClientFromViper(v *viper.Viper) (backendTypes.Backend, error) {
	dd := getSubViper(v, "datadog")
	dd.SetDefault("api_endpoint", apiURL)
	dd.SetDefault("metrics_per_batch", defaultMetricsPerBatch)
	dd.SetDefault("timeout", defaultClientTimeout)
	dd.SetDefault("max_request_elapsed_time", defaultMaxRequestElapsedTime)
	return NewClient(
		dd.GetString("api_endpoint"),
		dd.GetString("api_key"),
		uint(dd.GetInt("metrics_per_batch")),
		dd.GetDuration("timeout"),
		dd.GetDuration("max_request_elapsed_time"),
	)
}

// NewClient returns a new Datadog API client.
func NewClient(apiEndpoint, apiKey string, metricsPerBatch uint, clientTimeout, maxRequestElapsedTime time.Duration) (backendTypes.Backend, error) {
	if apiEndpoint == "" {
		return nil, fmt.Errorf("[%s] apiEndpoint is required", BackendName)
	}
	if apiKey == "" {
		return nil, fmt.Errorf("[%s] apiKey is required", BackendName)
	}
	if metricsPerBatch <= 0 {
		return nil, fmt.Errorf("[%s] metricsPerBatch must be positive", BackendName)
	}
	if clientTimeout <= 0 {
		return nil, fmt.Errorf("[%s] clientTimeout must be positive", BackendName)
	}
	if maxRequestElapsedTime <= 0 {
		return nil, fmt.Errorf("[%s] maxRequestElapsedTime must be positive", BackendName)
	}
	log.Infof("[%s] maxRequestElapsedTime=%s clientTimeout=%s metricsPerBatch=%d", BackendName, maxRequestElapsedTime, clientTimeout, metricsPerBatch)
	return &client{
		apiKey:                apiKey,
		apiEndpoint:           apiEndpoint,
		maxRequestElapsedTime: maxRequestElapsedTime,
		client: http.Client{
			Timeout: clientTimeout,
		},
		metricsPerBatch: metricsPerBatch,
		now:             time.Now,
	}, nil
}

func getSubViper(v *viper.Viper, key string) *viper.Viper {
	n := v.Sub(key)
	if n == nil {
		n = viper.New()
	}
	return n
}
<|endoftext|>


<commit_before>package vaulted

import (
	"errors"
	"fmt"
	"os"
	"os/exec"
	"strings"
	"syscall"
)

var (
	ErrInvalidCommand = errors.New("Invalid command")
)

type Vault struct {
	Vars map[string]string `json:"vars"`
}

func (v *Vault) Spawn(cmd []string, env map[string]string) (*int, error) {
	if len(cmd) == 0 {
		return nil, ErrInvalidCommand
	}

	// lookup the path of the executable
	cmdpath, err := exec.LookPath(cmd[0])
	if err != nil {
		return nil, fmt.Errorf("Cannot find executable %s: %v", cmd[0], err)
	}

	// build the environ
	vars := make(map[string]string)
	for key, value := range v.Vars {
		vars[key] = value
	}
	for key, value := range env {
		vars[key] = value
	}

	// start the process
	var attr os.ProcAttr
	attr.Env = v.getEnviron(vars)
	attr.Files = []*os.File{os.Stdin, os.Stdout, os.Stderr}

	proc, err := os.StartProcess(cmdpath, cmd, &attr)
	if err != nil {
		return nil, fmt.Errorf("Failed to execute command: %v", err)
	}

	// wait for the process to exit
	state, _ := proc.Wait()

	var exitStatus int
	if !state.Success() {
		if status, ok := state.Sys().(syscall.WaitStatus); ok {
			exitStatus = status.ExitStatus()
		} else {
			exitStatus = 255
		}
	}

	// we only return an error if spawning the process failed, not if
	// the spawned command returned a failure status code.
	return &exitStatus, nil
}

func (v *Vault) getEnviron(vars map[string]string) []string {
	// load the current environ
	env := make(map[string]string)
	for _, envVar := range os.Environ() {
		parts := strings.SplitN(envVar, "=", 2)
		env[parts[0]] = parts[1]
	}

	// merge the provided vars
	for key, value := range vars {
		env[key] = value
	}

	// recombine into environ
	environ := make([]string, 0, len(env))
	for key, value := range env {
		environ = append(environ, fmt.Sprintf("%s=%s", key, value))
	}
	return environ
}
<commit_msg>Initial STS tokenization of AWS credentials<commit_after>package vaulted

import (
	"errors"
	"fmt"
	"os"
	"os/exec"
	"strings"
	"syscall"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/sts"
)

const (
	STS_DURATION = 3600
)

var (
	ErrInvalidCommand = errors.New("Invalid command")
)

type Vault struct {
	Vars   map[string]string `json:"vars"`
	AWSKey *AWSKey           `json:"aws_key,omitempty"`
}

type AWSKey struct {
	ID     string `json:"id"`
	Secret string `json:"secret"`
}

func (v *Vault) Spawn(cmd []string, env map[string]string) (*int, error) {
	if len(cmd) == 0 {
		return nil, ErrInvalidCommand
	}

	// lookup the path of the executable
	cmdpath, err := exec.LookPath(cmd[0])
	if err != nil {
		return nil, fmt.Errorf("Cannot find executable %s: %v", cmd[0], err)
	}

	// build the environ
	vars := make(map[string]string)
	for key, value := range v.Vars {
		vars[key] = value
	}
	for key, value := range env {
		vars[key] = value
	}

	if v.AWSKey != nil && v.AWSKey.ID != "" && v.AWSKey.Secret != "" {
		stsCreds, err := v.AWSKey.generateSTS()
		if err != nil {
			return nil, err
		}
		for k, v := range stsCreds {
			vars[k] = v
		}
	}

	// start the process
	var attr os.ProcAttr
	attr.Env = v.getEnviron(vars)
	attr.Files = []*os.File{os.Stdin, os.Stdout, os.Stderr}

	proc, err := os.StartProcess(cmdpath, cmd, &attr)
	if err != nil {
		return nil, fmt.Errorf("Failed to execute command: %v", err)
	}

	// wait for the process to exit
	state, _ := proc.Wait()

	var exitStatus int
	if !state.Success() {
		if status, ok := state.Sys().(syscall.WaitStatus); ok {
			exitStatus = status.ExitStatus()
		} else {
			exitStatus = 255
		}
	}

	// we only return an error if spawning the process failed, not if
	// the spawned command returned a failure status code.
	return &exitStatus, nil
}

func (v *Vault) getEnviron(vars map[string]string) []string {
	// load the current environ
	env := make(map[string]string)
	for _, envVar := range os.Environ() {
		parts := strings.SplitN(envVar, "=", 2)
		env[parts[0]] = parts[1]
	}

	// merge the provided vars
	for key, value := range vars {
		env[key] = value
	}

	// recombine into environ
	environ := make([]string, 0, len(env))
	for key, value := range env {
		environ = append(environ, fmt.Sprintf("%s=%s", key, value))
	}
	return environ
}

func (k *AWSKey) generateSTS() (map[string]string, error) {
	sess := session.New(&aws.Config{
		Credentials: credentials.NewStaticCredentials(
			k.ID,
			k.Secret,
			"", // Temporary session token
		),
	})

	params := &sts.GetSessionTokenInput{
		DurationSeconds: aws.Int64(STS_DURATION),
	}

	stsClient := sts.New(sess)
	resp, err := stsClient.GetSessionToken(params)
	if err != nil {
		return nil, err
	}

	return map[string]string{
		"AWS_ACCESS_KEY_ID":     *resp.Credentials.AccessKeyId,
		"AWS_SECRET_ACCESS_KEY": *resp.Credentials.SecretAccessKey,
		"AWS_SESSION_TOKEN":     *resp.Credentials.SessionToken,
	}, nil
}
<|endoftext|>


<commit_before>package god

import (
	"github.com/vanng822/gopid"
	"log"
	"os"
	"os/signal"
	"runtime"
	"syscall"
)

type Goz struct {
	gods []*God
	sigc chan os.Signal
}

func NewGoz() *Goz {
	z := new(Goz)
	z.gods = make([]*God, 0)
	return z
}

func (z *Goz) Add(d *God) {
	z.gods = append(z.gods, d)
}

func (z *Goz) Start() {
	args := Args{}

	if err := args.Parse(os.Args[1:]); err != nil || args.help || len(args.programs) == 0 {
		usage()
		return
	}

	if args.pidFile != "" {
		gopid.CheckPid(args.pidFile, args.force)
		gopid.CreatePid(args.pidFile)
		defer gopid.CleanPid(args.pidFile)
	}

	for i, p := range args.programs {
		pargs := args.programArgs[i]
		pargs = append(pargs, args.args...)
		z.Add(NewGod(p, pargs))
	}

	// need to handle panic and shut down others
	log.Printf("Number of goroutines %d before start", runtime.NumGoroutine())
	for _, d := range z.gods {
		go d.Start()
	}

	log.Printf("Number of goroutines %d after start", runtime.NumGoroutine())
	z.sigc = make(chan os.Signal, 1)
	signal.Notify(z.sigc, os.Kill, os.Interrupt, syscall.SIGTERM, syscall.SIGHUP)
	for {
		sig := <-z.sigc
		log.Printf("Got signal %v", sig)
		switch sig {
		case syscall.SIGHUP:
			z.Restart()
			break
		case syscall.SIGTERM:
			z.Stop()
			goto programExit
		case os.Kill:
			z.Stop()
			goto programExit
		case os.Interrupt:
			z.Stop()
			goto programExit
		default:
			log.Printf("Unhandled signal %v, stop program", sig)
			z.Stop()
			goto programExit

		}

	}
programExit:
	log.Println("Program exit")
}

func (z *Goz) Stop() {
	for _, d := range z.gods {
		d.stopping = true
		d.Stop()
	}
}

func (z *Goz) Restart() {
	log.Printf("Number of goroutines %d before restart", runtime.NumGoroutine())
	for _, d := range z.gods {
		go d.Restart()
	}
	log.Printf("Number of goroutines %d after restart", runtime.NumGoroutine())
}
<commit_msg>Skip goto statement, return should do the same job here<commit_after>package god

import (
	"github.com/vanng822/gopid"
	"log"
	"os"
	"os/signal"
	"runtime"
	"syscall"
)

type Goz struct {
	gods []*God
	sigc chan os.Signal
}

func NewGoz() *Goz {
	z := new(Goz)
	z.gods = make([]*God, 0)
	return z
}

func (z *Goz) Add(d *God) {
	z.gods = append(z.gods, d)
}

func (z *Goz) Start() {
	args := Args{}

	if err := args.Parse(os.Args[1:]); err != nil || args.help || len(args.programs) == 0 {
		usage()
		return
	}

	if args.pidFile != "" {
		gopid.CheckPid(args.pidFile, args.force)
		gopid.CreatePid(args.pidFile)
		defer gopid.CleanPid(args.pidFile)
	}

	for i, p := range args.programs {
		pargs := args.programArgs[i]
		pargs = append(pargs, args.args...)
		z.Add(NewGod(p, pargs))
	}

	// need to handle panic and shut down others
	log.Printf("Number of goroutines %d before start", runtime.NumGoroutine())
	for _, d := range z.gods {
		go d.Start()
	}

	log.Printf("Number of goroutines %d after start", runtime.NumGoroutine())
	z.sigc = make(chan os.Signal, 1)
	signal.Notify(z.sigc, os.Kill, os.Interrupt, syscall.SIGTERM, syscall.SIGHUP)
	for {
		sig := <-z.sigc
		log.Printf("Got signal %v", sig)
		switch sig {
		case syscall.SIGHUP:
			z.Restart()
			break
		case syscall.SIGTERM:
			z.Stop()
			return
		case os.Kill:
			z.Stop()
			return
		case os.Interrupt:
			z.Stop()
			return
		default:
			log.Printf("Unhandled signal %v, stop program", sig)
			z.Stop()
			return
		}
	}
}

func (z *Goz) Stop() {
	for _, d := range z.gods {
		d.stopping = true
		d.Stop()
	}
}

func (z *Goz) Restart() {
	log.Printf("Number of goroutines %d before restart", runtime.NumGoroutine())
	for _, d := range z.gods {
		go d.Restart()
	}
	log.Printf("Number of goroutines %d after restart", runtime.NumGoroutine())
}
<|endoftext|>


<commit_before>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package gaeemulation provides a server module that adds implementation of
// some https://godoc.org/go.chromium.org/luci/gae APIs to the global server context.
//
// The implementation is based on regular Cloud APIs and works from anywhere
// (not necessarily from Appengine).
//
// Usage:
//
//   func main() {
//     modules := []module.Module{
//       gaeemulation.NewModuleFromFlags(),
//     }
//     server.Main(nil, modules, func(srv *server.Server) error {
//       srv.Routes.GET("/", ..., func(c *router.Context) {
//         ent := Entity{ID: "..."}
//         err := datastore.Get(c.Context, &ent)
//         ...
//       })
//       return nil
//     })
//   }
//
// TODO(vadimsh): Currently provides datastore API only.
package gaeemulation

import (
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"os"

	"cloud.google.com/go/datastore"
	"google.golang.org/api/option"
	"google.golang.org/grpc"

	"go.chromium.org/luci/gae/filter/dscache"
	"go.chromium.org/luci/gae/filter/txndefer"
	"go.chromium.org/luci/gae/impl/cloud"

	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/server/auth"
	"go.chromium.org/luci/server/module"
	"go.chromium.org/luci/server/redisconn"
)

// ModuleName can be used to refer to this module when declaring dependencies.
var ModuleName = module.RegisterName("go.chromium.org/luci/server/gaeemulation")

// ModuleOptions contain configuration of the GAE Emulation server module
type ModuleOptions struct {
	DSCache string // currently either "disable" (default) or "redis"
}

// Register registers the command line flags.
func (o *ModuleOptions) Register(f *flag.FlagSet) {
	f.StringVar(
		&o.DSCache,
		"ds-cache",
		o.DSCache,
		`What datastore caching layer to use ("disable" or "redis").`,
	)
}

// NewModule returns a server module that adds implementation of
// some https://godoc.org/go.chromium.org/luci/gae APIs to the global server
// context.
func NewModule(opts *ModuleOptions) module.Module {
	if opts == nil {
		opts = &ModuleOptions{}
	}
	return &gaeModule{opts: opts}
}

// NewModuleFromFlags is a variant of NewModule that initializes options through
// command line flags.
//
// Calling this function registers flags in flag.CommandLine. They are usually
// parsed in server.Main(...).
func NewModuleFromFlags() module.Module {
	opts := &ModuleOptions{}
	opts.Register(flag.CommandLine)
	return NewModule(opts)
}

// gaeModule implements module.Module.
type gaeModule struct {
	opts *ModuleOptions
}

// Name is part of module.Module interface.
func (*gaeModule) Name() module.Name {
	return ModuleName
}

// Dependencies is part of module.Module interface.
func (*gaeModule) Dependencies() []module.Dependency {
	return []module.Dependency{
		module.OptionalDependency(redisconn.ModuleName), // for dscache, if enabled
	}
}

// Initialize is part of module.Module interface.
func (m *gaeModule) Initialize(ctx context.Context, host module.Host, opts module.HostOptions) (context.Context, error) {
	// Use zstd in luci/server's dscache. It will be default at some point.
	dscache.UseZstd = true

	var cacheImpl dscache.Cache
	switch m.opts.DSCache {
	case "", "disable":
		// don't use caching
	case "redis":
		pool := redisconn.GetPool(ctx)
		if pool == nil {
			return nil, errors.Reason("can't use `-ds-cache redis`: redisconn module is not configured").Err()
		}
		cacheImpl = redisCache{pool: pool}
	default:
		return nil, errors.Reason("unsupported -ds-cache %q", m.opts.DSCache).Err()
	}

	var client *datastore.Client
	if opts.CloudProject != "" {
		var err error
		if client, err = m.initDSClient(ctx, host, opts.CloudProject); err != nil {
			return nil, err
		}
	}
	cfg := &cloud.ConfigLite{
		IsDev:     !opts.Prod,
		ProjectID: opts.CloudProject,
		DS:        client, // if nil, datastore calls will fail gracefully(-ish)
	}

	ctx = cfg.Use(ctx)
	if cacheImpl != nil {
		ctx = dscache.FilterRDS(ctx, cacheImpl)
	}
	return txndefer.FilterRDS(ctx), nil
}

// initDSClient sets up Cloud Datastore client that uses AsSelf server token
// source.
func (m *gaeModule) initDSClient(ctx context.Context, host module.Host, cloudProject string) (*datastore.Client, error) {
	logging.Infof(ctx, "Setting up datastore client for project %q", cloudProject)

	// Enable auth only when using the real datastore.
	var clientOpts []option.ClientOption
	if addr := os.Getenv("DATASTORE_EMULATOR_HOST"); addr == "" {
		ts, err := auth.GetTokenSource(ctx, auth.AsSelf, auth.WithScopes(auth.CloudOAuthScopes...))
		if err != nil {
			return nil, errors.Annotate(err, "failed to initialize the token source").Err()
		}
		clientOpts = []option.ClientOption{option.WithTokenSource(ts)}
	}

	// A temporary interceptor to debug panics in "Commit" RPCs.
	clientOpts = append(clientOpts, option.WithGRPCDialOption(grpc.WithUnaryInterceptor(panicInterceptor)))

	client, err := datastore.NewClient(ctx, cloudProject, clientOpts...)
	if err != nil {
		return nil, errors.Annotate(err, "failed to instantiate the datastore client").Err()
	}

	host.RegisterCleanup(func(ctx context.Context) {
		if err := client.Close(); err != nil {
			logging.Warningf(ctx, "Failed to close the datastore client - %s", err)
		}
	})

	// TODO(vadimsh): "Ping" the datastore to verify the credentials are correct?

	return client, nil
}

func panicInterceptor(
	ctx context.Context,
	method string,
	request interface{},
	reply interface{},
	cc *grpc.ClientConn,
	invoker grpc.UnaryInvoker,
	opts ...grpc.CallOption,
) error {
	defer func() {
		if r := recover(); r != nil {
			blob, err := json.MarshalIndent(request, "", "  ")
			if err != nil {
				blob = []byte(fmt.Sprintf("<JSON marshaling failed: %s>", err))
			}
			logging.Errorf(ctx, "Caught panic in %q, request body is:\n%s", method, blob)
			panic(r)
		}
	}()
	return invoker(ctx, method, request, reply, cc, opts...)
}
<commit_msg>Revert "[server] Intercept panics when calling Datastore gRPC methods."<commit_after>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package gaeemulation provides a server module that adds implementation of
// some https://godoc.org/go.chromium.org/luci/gae APIs to the global server context.
//
// The implementation is based on regular Cloud APIs and works from anywhere
// (not necessarily from Appengine).
//
// Usage:
//
//   func main() {
//     modules := []module.Module{
//       gaeemulation.NewModuleFromFlags(),
//     }
//     server.Main(nil, modules, func(srv *server.Server) error {
//       srv.Routes.GET("/", ..., func(c *router.Context) {
//         ent := Entity{ID: "..."}
//         err := datastore.Get(c.Context, &ent)
//         ...
//       })
//       return nil
//     })
//   }
//
// TODO(vadimsh): Currently provides datastore API only.
package gaeemulation

import (
	"context"
	"flag"
	"os"

	"cloud.google.com/go/datastore"
	"google.golang.org/api/option"

	"go.chromium.org/luci/gae/filter/dscache"
	"go.chromium.org/luci/gae/filter/txndefer"
	"go.chromium.org/luci/gae/impl/cloud"

	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/server/auth"
	"go.chromium.org/luci/server/module"
	"go.chromium.org/luci/server/redisconn"
)

// ModuleName can be used to refer to this module when declaring dependencies.
var ModuleName = module.RegisterName("go.chromium.org/luci/server/gaeemulation")

// ModuleOptions contain configuration of the GAE Emulation server module
type ModuleOptions struct {
	DSCache string // currently either "disable" (default) or "redis"
}

// Register registers the command line flags.
func (o *ModuleOptions) Register(f *flag.FlagSet) {
	f.StringVar(
		&o.DSCache,
		"ds-cache",
		o.DSCache,
		`What datastore caching layer to use ("disable" or "redis").`,
	)
}

// NewModule returns a server module that adds implementation of
// some https://godoc.org/go.chromium.org/luci/gae APIs to the global server
// context.
func NewModule(opts *ModuleOptions) module.Module {
	if opts == nil {
		opts = &ModuleOptions{}
	}
	return &gaeModule{opts: opts}
}

// NewModuleFromFlags is a variant of NewModule that initializes options through
// command line flags.
//
// Calling this function registers flags in flag.CommandLine. They are usually
// parsed in server.Main(...).
func NewModuleFromFlags() module.Module {
	opts := &ModuleOptions{}
	opts.Register(flag.CommandLine)
	return NewModule(opts)
}

// gaeModule implements module.Module.
type gaeModule struct {
	opts *ModuleOptions
}

// Name is part of module.Module interface.
func (*gaeModule) Name() module.Name {
	return ModuleName
}

// Dependencies is part of module.Module interface.
func (*gaeModule) Dependencies() []module.Dependency {
	return []module.Dependency{
		module.OptionalDependency(redisconn.ModuleName), // for dscache, if enabled
	}
}

// Initialize is part of module.Module interface.
func (m *gaeModule) Initialize(ctx context.Context, host module.Host, opts module.HostOptions) (context.Context, error) {
	// Use zstd in luci/server's dscache. It will be default at some point.
	dscache.UseZstd = true

	var cacheImpl dscache.Cache
	switch m.opts.DSCache {
	case "", "disable":
		// don't use caching
	case "redis":
		pool := redisconn.GetPool(ctx)
		if pool == nil {
			return nil, errors.Reason("can't use `-ds-cache redis`: redisconn module is not configured").Err()
		}
		cacheImpl = redisCache{pool: pool}
	default:
		return nil, errors.Reason("unsupported -ds-cache %q", m.opts.DSCache).Err()
	}

	var client *datastore.Client
	if opts.CloudProject != "" {
		var err error
		if client, err = m.initDSClient(ctx, host, opts.CloudProject); err != nil {
			return nil, err
		}
	}
	cfg := &cloud.ConfigLite{
		IsDev:     !opts.Prod,
		ProjectID: opts.CloudProject,
		DS:        client, // if nil, datastore calls will fail gracefully(-ish)
	}

	ctx = cfg.Use(ctx)
	if cacheImpl != nil {
		ctx = dscache.FilterRDS(ctx, cacheImpl)
	}
	return txndefer.FilterRDS(ctx), nil
}

// initDSClient sets up Cloud Datastore client that uses AsSelf server token
// source.
func (m *gaeModule) initDSClient(ctx context.Context, host module.Host, cloudProject string) (*datastore.Client, error) {
	logging.Infof(ctx, "Setting up datastore client for project %q", cloudProject)

	// Enable auth only when using the real datastore.
	var clientOpts []option.ClientOption
	if addr := os.Getenv("DATASTORE_EMULATOR_HOST"); addr == "" {
		ts, err := auth.GetTokenSource(ctx, auth.AsSelf, auth.WithScopes(auth.CloudOAuthScopes...))
		if err != nil {
			return nil, errors.Annotate(err, "failed to initialize the token source").Err()
		}
		clientOpts = []option.ClientOption{option.WithTokenSource(ts)}
	}

	client, err := datastore.NewClient(ctx, cloudProject, clientOpts...)
	if err != nil {
		return nil, errors.Annotate(err, "failed to instantiate the datastore client").Err()
	}

	host.RegisterCleanup(func(ctx context.Context) {
		if err := client.Close(); err != nil {
			logging.Warningf(ctx, "Failed to close the datastore client - %s", err)
		}
	})

	// TODO(vadimsh): "Ping" the datastore to verify the credentials are correct?

	return client, nil
}
<|endoftext|>


<commit_before>package gitgo

import (
	"bufio"
	"bytes"
	"compress/zlib"
	"fmt"
	"io"
	"log"
	"os"
	"path"
	"path/filepath"
	"reflect"
)

type packObject struct {
	Name   SHA
	Offset int
	Data   []byte
	Type   packObjectType

	// only used for OBJ_OFS_DELTA
	negativeOffset int
	baseObjectName SHA
	baseOffset     int

	// the uncompressed size
	size int

	err error // was an error encountered while processing this object?
}

//go:generate stringer -type=packObjectType
type packObjectType uint8

const (
	_ packObjectType = iota
	OBJ_COMMIT
	OBJ_TREE
	OBJ_BLOB
	OBJ_TAG
	_
	OBJ_OFS_DELTA
	OBJ_REF_DELTA
)

type errReadSeeker struct {
	r   io.ReadSeeker
	err error
}

// Read, but only if no errors have been encountered
// in a previous read (including io.EOF)
func (er *errReadSeeker) read(buf []byte) int {
	var n int
	if er.err != nil {
		return 0
	}
	n, er.err = io.ReadFull(er.r, buf)
	return n
}

func (er *errReadSeeker) Seek(offset int64, whence int) (int64, error) {
	return er.r.Seek(offset, whence)
}

func GetIdxPath(dotGitRootPath string) (idxFilePath string, err error) {
	files, err := filepath.Glob(path.Join(dotGitRootPath, "objects/pack", "*.idx"))
	idxFilePath = files[0]
	return
}

func VerifyPack(pack io.ReadSeeker, idx io.Reader) ([]*packObject, error) {

	objects, err := parsePack(errReadSeeker{pack, nil}, idx)
	for _, object := range objects {
		if object.err != nil {
			continue
		}
		if object.Type == OBJ_OFS_DELTA {
			// TODO improve this
			// linear search to find the right offset
			var base *packObject
			for _, o := range objects {
				if o.Offset == object.Offset-object.negativeOffset {
					base = o
					break
				}
			}
			if base == nil {
				object.err = fmt.Errorf("could not find object with negative offset %d - %d for %s", object.Offset, object.negativeOffset, object.Name)
			} else {
				object.baseObjectName = base.Name
				log.Printf("%s SUCCESS", object.Name)
			}
		}
	}
	for _, o := range objects {
		log.Printf("%s: %d", o.Name, o.Offset)
	}
	return objects, err
}

func parsePack(pack errReadSeeker, idx io.Reader) (objects []*packObject, err error) {
	signature := make([]byte, 4)
	pack.read(signature)
	if string(signature) != "PACK" {
		return nil, fmt.Errorf("Received invalid signature: %s", string(signature))
	}
	if err != nil {
		return nil, err
	}
	version := make([]byte, 4)
	pack.read(version)

	// TODO use encoding/binary here
	v := version[3]
	switch v {
	case 2:
		// Parse version 2 packfile
		objects, err = parseIdx(idx, 2)
		if err != nil {
			return
		}
		objects, err = parsePackV2(pack, objects)
		return

	default:
		return nil, fmt.Errorf("cannot parse packfile with version %d", v)
	}
}

func Clone(r io.Reader) (*bufio.Reader, *bufio.Reader) {
	var b1 bytes.Buffer
	var b2 bytes.Buffer
	w := io.MultiWriter(&b1, &b2)
	io.Copy(w, r)
	return bufio.NewReader(&b1), bufio.NewReader(&b2)
}

func bytesToNum(b []byte) uint {
	var n uint
	for i := 0; i < len(b); i++ {
		n = n | (uint(b[len(b)-i-1]) << uint(i*8))
	}
	return n
}

// parsePackV2 parses a packfile that uses
// version 2 of the format
func parsePackV2(r errReadSeeker, objects []*packObject) ([]*packObject, error) {

	numObjectsBts := make([]byte, 4)
	r.read(numObjectsBts)
	if int(bytesToNum(numObjectsBts)) != len(objects) {
		return nil, fmt.Errorf("Expected %d objects and found %d", len(objects), numObjectsBts)
	}

	for _, object := range objects {

		var btsread int
		r.Seek(int64(object.Offset), os.SEEK_SET)
		_bytes := make([]byte, 1)
		btsread += r.read(_bytes)
		_byte := _bytes[0]

		// This will extract the last three bits of
		// the first nibble in the byte
		// which tells us the object type
		object.Type = packObjectType(((_byte >> 4) & 7))

		// determine the (decompressed) object size
		// and then deflate the following bytes

		// The most-significant byte (MSB)
		// tells us whether we need to read more bytes
		// to get the encoded object size
		MSB := (_byte & 128) // will be either 128 or 0

		// This will extract the last four bits of the byte
		var objectSize int = int((uint(_byte) & 15))

		// shift the first size by 0
		// and the rest by 4 + (i-1) * 7
		var shift uint = 4

		// If the most-significant bit is 0, this is the last byte
		// for the object size
		for MSB > 0 {
			// Keep reading the size until the MSB is 0
			_bytes := make([]byte, 1)
			btsread += r.read(_bytes)
			_byte := _bytes[0]

			MSB = (_byte & 128)

			objectSize += int((uint(_byte) & 127) << shift)
			shift += 7
		}

		object.size = objectSize
		switch {
		case object.Type < 5:
			// the object is a commit, tree, blob, or tag

			// (objectSize) is the size, in bytes, of this object *when expanded*
			// the IDX file tells us how many *compressed* bytes the object will take
			// (in other words, how much space to allocate for the result)
			object.Data = make([]byte, objectSize)

			zr, err := zlib.NewReader(r.r)
			if err != nil {
				return nil, err
			}

			n, err := zr.Read(object.Data)
			if err != nil {
				if err == io.EOF {
					err = nil
				} else {
					return nil, err
				}
			}
			zr.Close()

			if n != objectSize {
				return nil, fmt.Errorf("expected to read %d bytes, read %d", objectSize, n)
			}

		case object.Type == OBJ_OFS_DELTA:
			// read the n-byte offset
			// from the git docs:
			// "n bytes with MSB set in all but the last one.
			// The offset is then the number constructed by
			// concatenating the lower 7 bit of each byte, and
			// for n >= 2 adding 2^7 + 2^14 + ... + 2^(7*(n-1))
			// to the result."

			var offset int

			if object.Name == "b45377f6daf59a4cec9e8de64f5df1533a7994cd" {
				log.Print("\n\nasdf")
			}
			if object.Name == "b45377f6daf59a4cec9e8de64f5df1533a7994cd" {
				log.Printf("offset is %08b", offset)
			}

			// number of bytes read in variable length encoding
			var nbytes uint = 0

			MSB := 128
			for (MSB & 128) > 0 {
				nbytes++

				// Keep reading the size until the MSB is 0
				_bytes := make([]byte, 1)
				r.read(_bytes)
				_byte := _bytes[0]

				if object.Name == "b45377f6daf59a4cec9e8de64f5df1533a7994cd" {
					log.Printf("Read byte %08b", _byte)
				}
				sevenBytes := uint(_byte) & 127

				offset = (offset << 7) + int(sevenBytes)

				if object.Name == "b45377f6daf59a4cec9e8de64f5df1533a7994cd" {
					log.Printf("offset is %08b", offset)
				}

				MSB = int(_byte & 128)
				if MSB == 0 {
					break
				}
			}

			if nbytes >= 2 {
				offset += (1 << (7 * (nbytes - 1)))
			}

			if object.Name == "b45377f6daf59a4cec9e8de64f5df1533a7994cd" {
				tmp := offset + (1 << (7 * (nbytes - 1)))
				tmp2 := (1 << (7 * (nbytes - 1)))
				log.Printf("offset is %d", offset)
				log.Printf("tmp %d", tmp)
				log.Printf("tmp2 %d", tmp2)
				log.Printf("nbytes %d", nbytes)
				log.Printf("offset is %08b\n\n", offset)
			}
			object.negativeOffset = offset
			log.Printf("R %s %d %d", object.Name, object.Offset, object.negativeOffset)
			object.Data = make([]byte, objectSize)

			zr, err := zlib.NewReader(r.r)
			if err != nil {
				object.err = err
				continue
			}
			n, err := zr.Read(object.Data)
			if err != nil && err != io.EOF {
				object.err = err
				continue
			}
			object.Data = object.Data[:n]
			zr.Close()
			if len(object.Data) != objectSize {
				object.err = fmt.Errorf("received wrong object size: %d (expected %d)", object.Data, objectSize)
			}

		case object.Type == OBJ_REF_DELTA:
			r.Seek(int64(object.Offset), os.SEEK_SET)
			// Read the 20-byte base object name
			baseObjName := make([]byte, 20)

			r.read(baseObjName)
			object.Data = make([]byte, objectSize)

			zr, err := zlib.NewReader(r.r)
			if err != nil {
				object.err = err
				continue
			}
			n, err := zr.Read(object.Data)
			if err != nil && err != io.EOF {
				object.err = err
				continue
			}
			object.Data = object.Data[:n]
			zr.Close()
		}
	}

	return objects, r.err
}

func parseIdx(idx io.Reader, version int) (objects []*packObject, err error) {
	if version != 2 {
		return nil, fmt.Errorf("cannot parse IDX with version %d", version)
	}
	// parse version 2 idxfile

	// Version 2 starts with a 4-byte magic number
	header := make([]byte, 4)
	n, err := idx.Read(header)
	if err != nil {
		return nil, err
	}

	if !reflect.DeepEqual([]byte{255, 116, 79, 99}, header) {
		return nil, fmt.Errorf("invalid IDX header: %q", string(header))
	}

	// Then the version number in four bytes
	versionBts := make([]byte, 4)
	_, err = idx.Read(versionBts)
	if err != nil {
		return nil, err
	}
	// We already know the version, so we can ignore it

	// Then the fanout table
	// The fanout table has 256 entries, each 4 bytes long
	fanoutTableFlat := make([]byte, 256*4)
	n, err = idx.Read(fanoutTableFlat)
	if err == nil && n != len(fanoutTableFlat) {
		err = fmt.Errorf("read incomplete fanout table: %d", n)
	}
	if err != nil {
		return nil, err
	}

	// Initialize the flat fanout table
	fanoutTable := make([][]byte, 256)
	for i := 0; i < len(fanoutTableFlat); i += 4 {
		entry := fanoutTableFlat[i : i+4]
		fanoutTable[(i+1)/4] = entry
	}

	numObjects := int(bytesToNum(fanoutTable[len(fanoutTable)-1]))
	objects = make([]*packObject, numObjects)

	objectNames := make([]SHA, numObjects)

	for i := 0; i < numObjects; i++ {
		sha := make([]byte, 20)
		n, err = idx.Read(sha)
		if err != nil {
			return nil, err
		}

		objectNames[i] = SHA(fmt.Sprintf("%x", sha[:n]))
		objects[i] = &packObject{Name: SHA(fmt.Sprintf("%x", sha[:n]))}
	}

	// Then come 4-byte CRC32 values
	crc32Table := make([]byte, numObjects*4)
	_, err = idx.Read(crc32Table)
	if err != nil {
		return nil, err
	}

	// Next come 4-byte offset values
	// If the MSB is set, there is an index into the next table
	// otherwise, these are 31 bits each
	offsetsFlat := make([]byte, numObjects*4)
	_, err = idx.Read(offsetsFlat)
	if err != nil {
		return nil, err
	}

	offsets := make([]int, numObjects)
	for i := 0; i < len(offsets); i++ {
		offset := int(bytesToNum(offsetsFlat[i*4 : (i+1)*4]))
		// check if the MSB is 1
		if offset&2147483648 > 0 {
			return nil, fmt.Errorf("packfile is too large to parse")
		}
		offsets[i] = offset
		objects[i].Offset = offset
	}

	// If the pack file is more than 2 GB, there will be a table of 8-byte offset entries here
	// TODO implement this

	// This is the same as the checksum at the end of the corresponding packfile
	packfileChecksum := make([]byte, 20)
	_, err = idx.Read(packfileChecksum)
	if err != nil {
		return
	}

	// This is the checksum of all of the above data
	// We're not checking it now, but if we can't read it properly
	// that means an error has occurred earlier in parsing
	idxChecksum := make([]byte, 20)
	_, err = idx.Read(idxChecksum)
	if err != nil {
		return
	}

	// TODO check that there isn't any data left

	return objects, err
}
<commit_msg>Correctly calculate and store base offset for OFS_DELTA objects<commit_after>package gitgo

import (
	"bufio"
	"bytes"
	"compress/zlib"
	"fmt"
	"io"
	"log"
	"os"
	"path"
	"path/filepath"
	"reflect"
)

type packObject struct {
	Name   SHA
	Offset int
	Data   []byte
	Type   packObjectType

	// only used for OBJ_OFS_DELTA
	negativeOffset int
	baseObjectName SHA
	baseOffset     int

	// the uncompressed size
	size int

	err error // was an error encountered while processing this object?
}

//go:generate stringer -type=packObjectType
type packObjectType uint8

const (
	_ packObjectType = iota
	OBJ_COMMIT
	OBJ_TREE
	OBJ_BLOB
	OBJ_TAG
	_
	OBJ_OFS_DELTA
	OBJ_REF_DELTA
)

type errReadSeeker struct {
	r   io.ReadSeeker
	err error
}

// Read, but only if no errors have been encountered
// in a previous read (including io.EOF)
func (er *errReadSeeker) read(buf []byte) int {
	var n int
	if er.err != nil {
		return 0
	}
	n, er.err = io.ReadFull(er.r, buf)
	return n
}

func (er *errReadSeeker) Seek(offset int64, whence int) (int64, error) {
	return er.r.Seek(offset, whence)
}

func GetIdxPath(dotGitRootPath string) (idxFilePath string, err error) {
	files, err := filepath.Glob(path.Join(dotGitRootPath, "objects/pack", "*.idx"))
	idxFilePath = files[0]
	return
}

func VerifyPack(pack io.ReadSeeker, idx io.Reader) ([]*packObject, error) {

	objects, err := parsePack(errReadSeeker{pack, nil}, idx)
	for _, object := range objects {
		if object.err != nil {
			continue
		}
		if object.Type == OBJ_OFS_DELTA {
			// TODO improve this
			// linear search to find the right offset
			var base *packObject
			for _, o := range objects {
				if o.Offset == object.Offset-object.negativeOffset {
					base = o
					break
				}
			}
			if base == nil {
				object.err = fmt.Errorf("could not find object with negative offset %d - %d for %s", object.Offset, object.negativeOffset, object.Name)
			} else {
				object.baseObjectName = base.Name
				log.Printf("%s %d SUCCESS", object.Name, object.baseOffset)
			}
		}
	}
	for _, o := range objects {
		log.Printf("%s: %d", o.Name, o.Offset)
	}
	return objects, err
}

func parsePack(pack errReadSeeker, idx io.Reader) (objects []*packObject, err error) {
	signature := make([]byte, 4)
	pack.read(signature)
	if string(signature) != "PACK" {
		return nil, fmt.Errorf("Received invalid signature: %s", string(signature))
	}
	if err != nil {
		return nil, err
	}
	version := make([]byte, 4)
	pack.read(version)

	// TODO use encoding/binary here
	v := version[3]
	switch v {
	case 2:
		// Parse version 2 packfile
		objects, err = parseIdx(idx, 2)
		if err != nil {
			return
		}
		objects, err = parsePackV2(pack, objects)
		return

	default:
		return nil, fmt.Errorf("cannot parse packfile with version %d", v)
	}
}

func Clone(r io.Reader) (*bufio.Reader, *bufio.Reader) {
	var b1 bytes.Buffer
	var b2 bytes.Buffer
	w := io.MultiWriter(&b1, &b2)
	io.Copy(w, r)
	return bufio.NewReader(&b1), bufio.NewReader(&b2)
}

func bytesToNum(b []byte) uint {
	var n uint
	for i := 0; i < len(b); i++ {
		n = n | (uint(b[len(b)-i-1]) << uint(i*8))
	}
	return n
}

// parsePackV2 parses a packfile that uses
// version 2 of the format
func parsePackV2(r errReadSeeker, objects []*packObject) ([]*packObject, error) {

	numObjectsBts := make([]byte, 4)
	r.read(numObjectsBts)
	if int(bytesToNum(numObjectsBts)) != len(objects) {
		return nil, fmt.Errorf("Expected %d objects and found %d", len(objects), numObjectsBts)
	}

	for _, object := range objects {

		var btsread int
		r.Seek(int64(object.Offset), os.SEEK_SET)
		_bytes := make([]byte, 1)
		btsread += r.read(_bytes)
		_byte := _bytes[0]

		// This will extract the last three bits of
		// the first nibble in the byte
		// which tells us the object type
		object.Type = packObjectType(((_byte >> 4) & 7))

		// determine the (decompressed) object size
		// and then deflate the following bytes

		// The most-significant byte (MSB)
		// tells us whether we need to read more bytes
		// to get the encoded object size
		MSB := (_byte & 128) // will be either 128 or 0

		// This will extract the last four bits of the byte
		var objectSize int = int((uint(_byte) & 15))

		// shift the first size by 0
		// and the rest by 4 + (i-1) * 7
		var shift uint = 4

		// If the most-significant bit is 0, this is the last byte
		// for the object size
		for MSB > 0 {
			// Keep reading the size until the MSB is 0
			_bytes := make([]byte, 1)
			btsread += r.read(_bytes)
			_byte := _bytes[0]

			MSB = (_byte & 128)

			objectSize += int((uint(_byte) & 127) << shift)
			shift += 7
		}

		object.size = objectSize
		switch {
		case object.Type < 5:
			// the object is a commit, tree, blob, or tag

			// (objectSize) is the size, in bytes, of this object *when expanded*
			// the IDX file tells us how many *compressed* bytes the object will take
			// (in other words, how much space to allocate for the result)
			object.Data = make([]byte, objectSize)

			zr, err := zlib.NewReader(r.r)
			if err != nil {
				return nil, err
			}

			n, err := zr.Read(object.Data)
			if err != nil {
				if err == io.EOF {
					err = nil
				} else {
					return nil, err
				}
			}
			zr.Close()

			if n != objectSize {
				return nil, fmt.Errorf("expected to read %d bytes, read %d", objectSize, n)
			}

		case object.Type == OBJ_OFS_DELTA:
			// read the n-byte offset
			// from the git docs:
			// "n bytes with MSB set in all but the last one.
			// The offset is then the number constructed by
			// concatenating the lower 7 bit of each byte, and
			// for n >= 2 adding 2^7 + 2^14 + ... + 2^(7*(n-1))
			// to the result."

			var offset int

			// number of bytes read in variable length encoding
			var nbytes uint = 0

			MSB := 128
			for (MSB & 128) > 0 {
				nbytes++

				// Keep reading the size until the MSB is 0
				_bytes := make([]byte, 1)
				r.read(_bytes)
				_byte := _bytes[0]

				sevenBytes := uint(_byte) & 127

				offset = (offset << 7) + int(sevenBytes)

				MSB = int(_byte & 128)
				if MSB == 0 {
					break
				}
			}

			if nbytes >= 2 {
				offset += (1 << (7 * (nbytes - 1)))
			}

			object.negativeOffset = offset
			object.baseOffset = object.Offset - object.negativeOffset
			object.Data = make([]byte, objectSize)

			zr, err := zlib.NewReader(r.r)
			if err != nil {
				object.err = err
				continue
			}
			n, err := zr.Read(object.Data)
			if err != nil && err != io.EOF {
				object.err = err
				continue
			}
			object.Data = object.Data[:n]
			zr.Close()
			if len(object.Data) != objectSize {
				object.err = fmt.Errorf("received wrong object size: %d (expected %d)", object.Data, objectSize)
			}

		case object.Type == OBJ_REF_DELTA:
			r.Seek(int64(object.Offset), os.SEEK_SET)
			// Read the 20-byte base object name
			baseObjName := make([]byte, 20)

			r.read(baseObjName)
			object.Data = make([]byte, objectSize)

			zr, err := zlib.NewReader(r.r)
			if err != nil {
				object.err = err
				continue
			}
			n, err := zr.Read(object.Data)
			if err != nil && err != io.EOF {
				object.err = err
				continue
			}
			object.Data = object.Data[:n]
			zr.Close()
		}
	}

	return objects, r.err
}

func parseIdx(idx io.Reader, version int) (objects []*packObject, err error) {
	if version != 2 {
		return nil, fmt.Errorf("cannot parse IDX with version %d", version)
	}
	// parse version 2 idxfile

	// Version 2 starts with a 4-byte magic number
	header := make([]byte, 4)
	n, err := idx.Read(header)
	if err != nil {
		return nil, err
	}

	if !reflect.DeepEqual([]byte{255, 116, 79, 99}, header) {
		return nil, fmt.Errorf("invalid IDX header: %q", string(header))
	}

	// Then the version number in four bytes
	versionBts := make([]byte, 4)
	_, err = idx.Read(versionBts)
	if err != nil {
		return nil, err
	}
	// We already know the version, so we can ignore it

	// Then the fanout table
	// The fanout table has 256 entries, each 4 bytes long
	fanoutTableFlat := make([]byte, 256*4)
	n, err = idx.Read(fanoutTableFlat)
	if err == nil && n != len(fanoutTableFlat) {
		err = fmt.Errorf("read incomplete fanout table: %d", n)
	}
	if err != nil {
		return nil, err
	}

	// Initialize the flat fanout table
	fanoutTable := make([][]byte, 256)
	for i := 0; i < len(fanoutTableFlat); i += 4 {
		entry := fanoutTableFlat[i : i+4]
		fanoutTable[(i+1)/4] = entry
	}

	numObjects := int(bytesToNum(fanoutTable[len(fanoutTable)-1]))
	objects = make([]*packObject, numObjects)

	objectNames := make([]SHA, numObjects)

	for i := 0; i < numObjects; i++ {
		sha := make([]byte, 20)
		n, err = idx.Read(sha)
		if err != nil {
			return nil, err
		}

		objectNames[i] = SHA(fmt.Sprintf("%x", sha[:n]))
		objects[i] = &packObject{Name: SHA(fmt.Sprintf("%x", sha[:n]))}
	}

	// Then come 4-byte CRC32 values
	crc32Table := make([]byte, numObjects*4)
	_, err = idx.Read(crc32Table)
	if err != nil {
		return nil, err
	}

	// Next come 4-byte offset values
	// If the MSB is set, there is an index into the next table
	// otherwise, these are 31 bits each
	offsetsFlat := make([]byte, numObjects*4)
	_, err = idx.Read(offsetsFlat)
	if err != nil {
		return nil, err
	}

	offsets := make([]int, numObjects)
	for i := 0; i < len(offsets); i++ {
		offset := int(bytesToNum(offsetsFlat[i*4 : (i+1)*4]))
		// check if the MSB is 1
		if offset&2147483648 > 0 {
			return nil, fmt.Errorf("packfile is too large to parse")
		}
		offsets[i] = offset
		objects[i].Offset = offset
	}

	// If the pack file is more than 2 GB, there will be a table of 8-byte offset entries here
	// TODO implement this

	// This is the same as the checksum at the end of the corresponding packfile
	packfileChecksum := make([]byte, 20)
	_, err = idx.Read(packfileChecksum)
	if err != nil {
		return
	}

	// This is the checksum of all of the above data
	// We're not checking it now, but if we can't read it properly
	// that means an error has occurred earlier in parsing
	idxChecksum := make([]byte, 20)
	_, err = idx.Read(idxChecksum)
	if err != nil {
		return
	}

	// TODO check that there isn't any data left

	return objects, err
}
<|endoftext|>


<commit_before>package version

import "github.com/spf13/cobra"

var (
	// Cmd can be added to other commands to provide a version subcommand with
	// the correct version of swarm.
	Cmd = &cobra.Command{
		Use:   "version",
		Short: "Print version number of swarm",
		Run: func(cmd *cobra.Command, args []string) {
			PrintVersion()
		},
	}
)
<commit_msg>swarmctl: add a validation for 'swarmctl version'<commit_after>package version

import (
	"errors"

	"github.com/spf13/cobra"
)

var (
	// Cmd can be added to other commands to provide a version subcommand with
	// the correct version of swarm.
	Cmd = &cobra.Command{
		Use:   "version",
		Short: "Print version number of swarm",
		RunE: func(cmd *cobra.Command, args []string) error {
			if len(args) != 0 {
				return errors.New("version command takes no arguments")
			}

			PrintVersion()
			return nil
		},
	}
)
<|endoftext|>


<commit_before>//: ----------------------------------------------------------------------------
//: Copyright (C) 2017 Verizon.  All Rights Reserved.
//: All Rights Reserved
//:
//: file:    stats.go
//: details: TODO
//: author:  Mehrdad Arshad Rad
//: date:    02/01/2017
//:
//: Licensed under the Apache License, Version 2.0 (the "License");
//: you may not use this file except in compliance with the License.
//: You may obtain a copy of the License at
//:
//:     http://www.apache.org/licenses/LICENSE-2.0
//:
//: Unless required by applicable law or agreed to in writing, software
//: distributed under the License is distributed on an "AS IS" BASIS,
//: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//: See the License for the specific language governing permissions and
//: limitations under the License.
//: ----------------------------------------------------------------------------
package main

import (
	"encoding/json"
	"net"
	"net/http"
	"runtime"
	"time"
)

func StatsSysHandler(w http.ResponseWriter, r *http.Request) {
	var mem runtime.MemStats

	runtime.ReadMemStats(&mem)
	var data = &struct {
		MemAlloc        uint64
		MemTotalAlloc   uint64
		MemHeapAlloc    uint64
		MemHeapSys      uint64
		MemHeapReleased uint64
		MCacheInuse     uint64
		GCSys           uint64
		GCNext          uint64
		GCLast          string
		NumLogicalCPU   int
		NumGoroutine    int
		GoVersion       string
	}{
		mem.Alloc,
		mem.TotalAlloc,
		mem.HeapAlloc,
		mem.HeapSys,
		mem.HeapReleased,
		mem.MCacheInuse,
		mem.GCSys,
		mem.NextGC,
		time.Unix(0, int64(mem.LastGC)).String(),
		runtime.NumCPU(),
		runtime.NumGoroutine(),
		runtime.Version(),
	}

	j, _ := json.Marshal(data)

	w.Write(j)
}

func StatsFlowHandler(i *IPFIX, s *SFlow) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		var data = &struct {
			IPFIX *IPFIXStats
			SFlow *SFlowStats
		}{
			i.status(),
			s.status(),
		}
		j, _ := json.Marshal(data)

		w.Write(j)
	}
}

func statsHTTPServer(ipfix *IPFIX, sflow *SFlow) {
	if !opts.StatsEnabled {
		return
	}

	mux := http.NewServeMux()
	mux.HandleFunc("/sys", StatsSysHandler)
	mux.HandleFunc("/flow", StatsFlowHandler(ipfix, sflow))

	addr := net.JoinHostPort(opts.StatsHTTPAddr, opts.StatsHTTPPort)

	logger.Println("starting stats web server ...")
	err := http.ListenAndServe(addr, mux)
	if err != nil {
		logger.Fatal(err)
	}
}
<commit_msg>add start time to calc uptime<commit_after>//: ----------------------------------------------------------------------------
//: Copyright (C) 2017 Verizon.  All Rights Reserved.
//: All Rights Reserved
//:
//: file:    stats.go
//: details: TODO
//: author:  Mehrdad Arshad Rad
//: date:    02/01/2017
//:
//: Licensed under the Apache License, Version 2.0 (the "License");
//: you may not use this file except in compliance with the License.
//: You may obtain a copy of the License at
//:
//:     http://www.apache.org/licenses/LICENSE-2.0
//:
//: Unless required by applicable law or agreed to in writing, software
//: distributed under the License is distributed on an "AS IS" BASIS,
//: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//: See the License for the specific language governing permissions and
//: limitations under the License.
//: ----------------------------------------------------------------------------
package main

import (
	"encoding/json"
	"net"
	"net/http"
	"runtime"
	"time"
)

var startTime = time.Now().Unix()

func StatsSysHandler(w http.ResponseWriter, r *http.Request) {
	var mem runtime.MemStats

	runtime.ReadMemStats(&mem)
	var data = &struct {
		MemAlloc        uint64
		MemTotalAlloc   uint64
		MemHeapAlloc    uint64
		MemHeapSys      uint64
		MemHeapReleased uint64
		MCacheInuse     uint64
		GCSys           uint64
		GCNext          uint64
		GCLast          string
		NumLogicalCPU   int
		NumGoroutine    int
		GoVersion       string
		StartTime       int64
	}{
		mem.Alloc,
		mem.TotalAlloc,
		mem.HeapAlloc,
		mem.HeapSys,
		mem.HeapReleased,
		mem.MCacheInuse,
		mem.GCSys,
		mem.NextGC,
		time.Unix(0, int64(mem.LastGC)).String(),
		runtime.NumCPU(),
		runtime.NumGoroutine(),
		runtime.Version(),
		startTime,
	}

	j, _ := json.Marshal(data)

	w.Write(j)
}

func StatsFlowHandler(i *IPFIX, s *SFlow) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		var data = &struct {
			IPFIX *IPFIXStats
			SFlow *SFlowStats
		}{
			i.status(),
			s.status(),
		}
		j, _ := json.Marshal(data)

		w.Write(j)
	}
}

func statsHTTPServer(ipfix *IPFIX, sflow *SFlow) {
	if !opts.StatsEnabled {
		return
	}

	mux := http.NewServeMux()
	mux.HandleFunc("/sys", StatsSysHandler)
	mux.HandleFunc("/flow", StatsFlowHandler(ipfix, sflow))

	addr := net.JoinHostPort(opts.StatsHTTPAddr, opts.StatsHTTPPort)

	logger.Println("starting stats web server ...")
	err := http.ListenAndServe(addr, mux)
	if err != nil {
		logger.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package http

import (
	"fmt"
	"github.com/ProductHealth/gose4"
	"io"
	"net/http"
	"net/url"
	"strings"
	"github.com/ProductHealth/gommons/time"
)

type check struct {
	url             *url.URL
	method          string
	statusCode      int
	requestBody     *string
	requestHeaders  *map[string]string
	expectedHeaders *map[string]string
	expectedBody    *string
	config          gose4.Configuration
	requestFunc     func(*http.Request) (*http.Response, error)
}

func New(address, method string, statusCode int, config gose4.Configuration) (*check, error) {
	url, err := url.Parse(address)
	if err != nil {
		return nil, err
	}

	requestFunc := func(req *http.Request) (*http.Response, error) {
		return http.DefaultClient.Do(req)
	}

	return &check{url: url, method: method, statusCode: statusCode, config: config, requestFunc: requestFunc}, nil
}

func (hc *check) RequestBody(content string) {
	hc.requestBody = &content
}

func (hc *check) RequestHeaders(rh map[string]string) {
	hc.requestHeaders = &rh
}

func (hc *check) ExpectedHeaders(eh map[string]string) {
	hc.expectedHeaders = &eh
}

func (hc *check) ExpectedBody(content string) {
	hc.requestBody = &content
}

func (hc *check) Run() gose4.HealthCheckResult {
	sw := time.NewStopWatch()
	var bodyReader io.Reader = nil
	if hc.requestBody != nil {
		bodyReader = strings.NewReader(*(hc.requestBody))
	}
	request, err := http.NewRequest(hc.method, hc.url.String(), bodyReader)
	if err != nil {
		return gose4.Failed(sw.GetDuration(), err.Error())
	}
	response, err := hc.requestFunc(request)
	gose4.Infof("RESPONSE %#v ERR: %s", response, err)
	if err != nil {
		return gose4.Failed(sw.GetDuration(), fmt.Sprintf("Error while requesting %#v: %s", hc.url, err))
	}
	if response.Body != nil {
		defer response.Body.Close()
	}
	if hc.statusCode != response.StatusCode {
		return gose4.Failed(sw.GetDuration(), fmt.Sprintf("Returned response code %#v does not match required %#v ", response.StatusCode, hc.statusCode))
	}
	return gose4.Ok(sw.GetDuration())
}

func (hc *check) Configuration() gose4.Configuration {
	return hc.config
}
<commit_msg>Reduced logging level of http check<commit_after>package http

import (
	"fmt"
	"github.com/ProductHealth/gose4"
	"io"
	"net/http"
	"net/url"
	"strings"
	"github.com/ProductHealth/gommons/time"
)

type check struct {
	url             *url.URL
	method          string
	statusCode      int
	requestBody     *string
	requestHeaders  *map[string]string
	expectedHeaders *map[string]string
	expectedBody    *string
	config          gose4.Configuration
	requestFunc     func(*http.Request) (*http.Response, error)
}

func New(address, method string, statusCode int, config gose4.Configuration) (*check, error) {
	url, err := url.Parse(address)
	if err != nil {
		return nil, err
	}

	requestFunc := func(req *http.Request) (*http.Response, error) {
		return http.DefaultClient.Do(req)
	}

	return &check{url: url, method: method, statusCode: statusCode, config: config, requestFunc: requestFunc}, nil
}

func (hc *check) RequestBody(content string) {
	hc.requestBody = &content
}

func (hc *check) RequestHeaders(rh map[string]string) {
	hc.requestHeaders = &rh
}

func (hc *check) ExpectedHeaders(eh map[string]string) {
	hc.expectedHeaders = &eh
}

func (hc *check) ExpectedBody(content string) {
	hc.requestBody = &content
}

func (hc *check) Run() gose4.HealthCheckResult {
	sw := time.NewStopWatch()
	var bodyReader io.Reader = nil
	if hc.requestBody != nil {
		bodyReader = strings.NewReader(*(hc.requestBody))
	}
	request, err := http.NewRequest(hc.method, hc.url.String(), bodyReader)
	if err != nil {
		return gose4.Failed(sw.GetDuration(), err.Error())
	}
	response, err := hc.requestFunc(request)
	gose4.Debugf("RESPONSE %#v ERR: %s", response, err)
	if err != nil {
		return gose4.Failed(sw.GetDuration(), fmt.Sprintf("Error while requesting %#v: %s", hc.url, err))
	}
	if response.Body != nil {
		defer response.Body.Close()
	}
	if hc.statusCode != response.StatusCode {
		return gose4.Failed(sw.GetDuration(), fmt.Sprintf("Returned response code %#v does not match required %#v ", response.StatusCode, hc.statusCode))
	}
	return gose4.Ok(sw.GetDuration())
}

func (hc *check) Configuration() gose4.Configuration {
	return hc.config
}
<|endoftext|>