<commit_before>package apply

import (
	"fmt"
	"github.com/Aptomi/aptomi/pkg/slinga/engine/diff"
	"github.com/Aptomi/aptomi/pkg/slinga/engine/plugin"
	"github.com/Aptomi/aptomi/pkg/slinga/external"
	"github.com/Aptomi/aptomi/pkg/slinga/language"
	"github.com/Aptomi/aptomi/pkg/slinga/object"
	"github.com/Aptomi/aptomi/pkg/slinga/util"
	"math/rand"
	"strconv"
	"testing"
	"time"
)

func BenchmarkEngine(b *testing.B) {
	t := &testing.T{}
	gen := NewPolicyGenerator(
		239,
		30,
		50, // 1000
		6,
		6,
		4,
		2,
		100,
		500,
	)
	desiredPolicy, externalData := gen.makePolicyAndExternalData()
	for i := 0; i < b.N; i++ {
		RunEngine(t, desiredPolicy, externalData)
	}
}

type PolicyGenerator struct {
	random                    *rand.Rand
	labels                    int
	services                  int
	serviceCodeComponents     int
	codeParams                int
	serviceDependencyMaxChain int
	contextsPerService        int
	users                     int
	dependencies              int

	generatedLabels    map[string]string
	generatedLabelKeys []string

	generatedServices []*language.Service
	policy            *language.PolicyNamespace
	externalData      *external.Data
}

func NewPolicyGenerator(randSeed int64, labels, services, serviceCodeComponents, codeParams, serviceDependencyMaxChain, contextsPerService, users int, dependencies int) *PolicyGenerator {
	return &PolicyGenerator{
		random:                    rand.New(rand.NewSource(randSeed)),
		labels:                    labels,
		services:                  services,
		serviceCodeComponents:     serviceCodeComponents,
		codeParams:                codeParams,
		serviceDependencyMaxChain: serviceDependencyMaxChain,
		contextsPerService:        contextsPerService,
		users:                     users,
		dependencies:              dependencies,
		policy:                    language.NewPolicyNamespace(),
	}
}

func (gen *PolicyGenerator) makePolicyAndExternalData() (*language.PolicyNamespace, *external.Data) {
	// pre-generate the list of labels
	gen.makeLabels()
	fmt.Printf("Labels: %d\n", len(gen.generatedLabels))

	// generate services
	maxChainLen := gen.makeServices()
	fmt.Printf("Services: %d (max chain length %d)\n", len(gen.policy.Services), maxChainLen)

	// generate contexts
	gen.makeContexts()
	fmt.Printf("Contexts: %d\n", len(gen.policy.Contexts))

	// generate dependencies
	gen.makeDependencies()
	fmt.Printf("Dependencies: %d\n", len(gen.policy.Dependencies.DependenciesByID))

	// every user will have the same set of labels
	gen.externalData = external.NewData(
		NewUserLoaderImpl(gen.users, gen.generatedLabels),
		NewSecretLoaderImpl(),
	)

	// there will be one context matching for each service. it will re-define some of those labels
	// there will be other contexts, not matching
	return gen.policy, gen.externalData
}

func (gen *PolicyGenerator) randomString(length int) string {
	charset := "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
	b := make([]byte, length)
	for i := range b {
		if i == 0 {
			// first letter non-numeric
			b[i] = charset[gen.random.Intn(len(charset)-10)]
		} else {
			// other letters any
			b[i] = charset[gen.random.Intn(len(charset))]
		}
	}
	return string(b)
}

func (gen *PolicyGenerator) makeLabels() {
	gen.generatedLabels = make(map[string]string)
	for i := 0; i < gen.labels; i++ {
		name := gen.randomString(10)
		value := gen.randomString(25)
		gen.generatedLabels[name] = value
	}

	gen.generatedLabelKeys = []string{}
	for key, _ := range gen.generatedLabels {
		gen.generatedLabelKeys = append(gen.generatedLabelKeys, key)
	}
}

func (gen *PolicyGenerator) makeServices() int {
	gen.generatedServices = make([]*language.Service, gen.services)
	for i := 0; i < gen.services; i++ {
		gen.generatedServices[i] = gen.makeService()
	}

	// add some dependencies
	cnt := make([]int, gen.services)
	maxChainLen := 0
	for i := 0; i < gen.services; i++ {
		if maxChainLen < cnt[i] {
			maxChainLen = cnt[i]
		}

		// see if we have exceeded the max chain length
		if cnt[i]+1 > gen.serviceDependencyMaxChain {
			continue
		}

		// try to add at most one dependency from each service
		j := gen.random.Intn(gen.services)
		if j <= i {
			continue
		}

		if cnt[j] < cnt[i]+1 {
			cnt[j] = cnt[i] + 1
		}

		component := &language.ServiceComponent{
			Name:    "dep-" + strconv.Itoa(i),
			Service: "service-" + strconv.Itoa(j),
		}
		gen.generatedServices[i].Components = append(gen.generatedServices[i].Components, component)
	}
	return maxChainLen
}

func (gen *PolicyGenerator) makeService() *language.Service {
	id := len(gen.policy.Services)

	service := &language.Service{
		Metadata: object.Metadata{
			Kind:      language.ServiceObject.Kind,
			Namespace: "main",
			Name:      "service-" + strconv.Itoa(id),
		},
		Owner:      "user-" + strconv.Itoa(gen.random.Intn(gen.users)),
		Components: []*language.ServiceComponent{},
	}

	for i := 0; i < gen.serviceCodeComponents; i++ {
		labelName := gen.generatedLabelKeys[gen.random.Intn(len(gen.generatedLabelKeys))]

		params := util.NestedParameterMap{}
		for j := 0; j < gen.codeParams; j++ {
			name := "param-" + strconv.Itoa(j)
			value := "prefix-{{ .Labels." + labelName + " }}-suffix"
			params[name] = value
		}

		component := &language.ServiceComponent{
			Name: "component-" + strconv.Itoa(i),
			Code: &language.Code{
				Type:   "aptomi/code/unittests",
				Params: params,
			},
		}
		service.Components = append(service.Components, component)
	}

	gen.policy.AddObject(service)
	return service
}

func (gen *PolicyGenerator) makeContexts() {
	for i := 0; i < gen.services; i++ {
		// generate non-matching contexts
		for i := 0; i < gen.contextsPerService-1; i++ {
			context := &language.Context{
				Metadata: object.Metadata{
					Kind:      language.ContextObject.Kind,
					Namespace: "main",
					Name:      "context-" + gen.randomString(20),
				},
				Criteria: &language.Criteria{
					RequireAll: []string{"true"},
					RequireAny: []string{
						gen.randomString(20) + "=='" + gen.randomString(20) + "'",
						gen.randomString(20) + "=='" + gen.randomString(20) + "'",
						gen.randomString(20) + "=='" + gen.randomString(20) + "'",
					},
				},
			}
			gen.policy.AddObject(context)
		}

		// generate matching context
		context := &language.Context{
			Metadata: object.Metadata{
				Kind:      language.ContextObject.Kind,
				Namespace: "main",
				Name:      "context-" + gen.randomString(20),
			},
			Criteria: &language.Criteria{
				RequireAll: []string{"true"},
				RequireAny: []string{"service.Name == '" + ("service-" + strconv.Itoa(i)) + "'"},
			},
		}
		gen.policy.AddObject(context)
	}
}

func (gen *PolicyGenerator) makeDependencies() {
	for i := 0; i < gen.dependencies; i++ {
		dependency := &language.Dependency{
			Metadata: object.Metadata{
				Kind:      language.DependencyObject.Kind,
				Namespace: "main",
				Name:      "dependency-" + strconv.Itoa(i),
			},
			UserID:  "user-" + strconv.Itoa(gen.random.Intn(gen.users)),
			Service: "service-" + strconv.Itoa(gen.random.Intn(gen.services)),
		}
		gen.policy.AddObject(dependency)
	}
}

type UserLoaderImpl struct {
	users  int
	labels map[string]string

	cachedUsers *language.GlobalUsers
}

func NewUserLoaderImpl(users int, labels map[string]string) *UserLoaderImpl {
	return &UserLoaderImpl{
		users:  users,
		labels: labels,
	}
}

func (loader *UserLoaderImpl) LoadUsersAll() language.GlobalUsers {
	if loader.cachedUsers == nil {
		userMap := make(map[string]*language.User)
		for i := 0; i < loader.users; i++ {
			user := &language.User{
				ID:     "user-" + strconv.Itoa(i),
				Name:   "user-" + strconv.Itoa(i),
				Labels: loader.labels,
			}
			userMap[user.ID] = user
		}
		loader.cachedUsers = &language.GlobalUsers{Users: userMap}
	}
	return *loader.cachedUsers
}

func (loader *UserLoaderImpl) LoadUserByID(id string) *language.User {
	return loader.LoadUsersAll().Users[id]
}

func (loader *UserLoaderImpl) Summary() string {
	return "Synthetic user loader"
}

type SecretLoaderImpl struct {
}

func NewSecretLoaderImpl() *SecretLoaderImpl {
	return &SecretLoaderImpl{}
}

func (loader *SecretLoaderImpl) LoadSecretsByUserID(string) language.LabelSet {
	return language.NewLabelSetEmpty()
}

func RunEngine(t *testing.T, desiredPolicy *language.PolicyNamespace, externalData *external.Data) {
	timeStart := time.Now()

	actualPolicy := language.NewPolicyNamespace()
	actualState := resolvePolicy(t, actualPolicy, externalData)
	desiredState := resolvePolicy(t, desiredPolicy, externalData)

	// make plugin to successfully process all components, while failing all instances of component2
	pluginApplyFailComponent2 := NewEnginePluginImpl([]string{"fail-components-like-these"})

	// process all actions
	actions := diff.NewPolicyResolutionDiff(desiredState, actualState).Actions
	plugins := []plugin.EnginePlugin{pluginApplyFailComponent2}
	apply := NewEngineApply(
		desiredPolicy,
		desiredState,
		actualPolicy,
		actualState,
		externalData,
		actions,
		plugins,
	)

	actualState = applyAndCheck(t, apply, ResSuccess, 0, "")

	timeEnd := time.Now()
	timeDiff := timeEnd.Sub(timeStart)

	fmt.Printf("Resolved (dependencies): %d\n", len(desiredState.DependencyInstanceMap))
	fmt.Printf("Resolved (components): %d\n", len(actualState.ComponentInstanceMap))
	fmt.Printf("Time: %s\n", timeDiff.String())
}
<commit_msg>Fix engine/apply benchmark<commit_after>package apply

import (
	"fmt"
	"github.com/Aptomi/aptomi/pkg/slinga/engine/diff"
	"github.com/Aptomi/aptomi/pkg/slinga/external"
	"github.com/Aptomi/aptomi/pkg/slinga/language"
	"github.com/Aptomi/aptomi/pkg/slinga/object"
	"github.com/Aptomi/aptomi/pkg/slinga/util"
	"math/rand"
	"strconv"
	"testing"
	"time"
)

func BenchmarkEngine(b *testing.B) {
	t := &testing.T{}
	gen := NewPolicyGenerator(
		239,
		30,
		50, // 1000
		6,
		6,
		4,
		2,
		100,
		500,
	)
	desiredPolicy, externalData := gen.makePolicyAndExternalData()
	for i := 0; i < b.N; i++ {
		RunEngine(t, desiredPolicy, externalData)
	}
}

type PolicyGenerator struct {
	random                    *rand.Rand
	labels                    int
	services                  int
	serviceCodeComponents     int
	codeParams                int
	serviceDependencyMaxChain int
	contextsPerService        int
	users                     int
	dependencies              int

	generatedLabels    map[string]string
	generatedLabelKeys []string

	generatedServices []*language.Service
	policy            *language.PolicyNamespace
	externalData      *external.Data
}

func NewPolicyGenerator(randSeed int64, labels, services, serviceCodeComponents, codeParams, serviceDependencyMaxChain, contextsPerService, users int, dependencies int) *PolicyGenerator {
	return &PolicyGenerator{
		random:                    rand.New(rand.NewSource(randSeed)),
		labels:                    labels,
		services:                  services,
		serviceCodeComponents:     serviceCodeComponents,
		codeParams:                codeParams,
		serviceDependencyMaxChain: serviceDependencyMaxChain,
		contextsPerService:        contextsPerService,
		users:                     users,
		dependencies:              dependencies,
		policy:                    language.NewPolicyNamespace(),
	}
}

func (gen *PolicyGenerator) makePolicyAndExternalData() (*language.PolicyNamespace, *external.Data) {
	// pre-generate the list of labels
	gen.makeLabels()
	fmt.Printf("Labels: %d\n", len(gen.generatedLabels))

	// generate services
	maxChainLen := gen.makeServices()
	fmt.Printf("Services: %d (max chain length %d)\n", len(gen.policy.Services), maxChainLen)

	// generate contexts
	gen.makeContexts()
	fmt.Printf("Contexts: %d\n", len(gen.policy.Contexts))

	// generate dependencies
	gen.makeDependencies()
	fmt.Printf("Dependencies: %d\n", len(gen.policy.Dependencies.DependenciesByID))

	// generate cluster
	gen.makeCluster()

	// every user will have the same set of labels
	gen.externalData = external.NewData(
		NewUserLoaderImpl(gen.users, gen.generatedLabels),
		NewSecretLoaderImpl(),
	)

	// there will be one context matching for each service. it will re-define some of those labels
	// there will be other contexts, not matching
	return gen.policy, gen.externalData
}

func (gen *PolicyGenerator) randomString(length int) string {
	charset := "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
	b := make([]byte, length)
	for i := range b {
		if i == 0 {
			// first letter non-numeric
			b[i] = charset[gen.random.Intn(len(charset)-10)]
		} else {
			// other letters any
			b[i] = charset[gen.random.Intn(len(charset))]
		}
	}
	return string(b)
}

func (gen *PolicyGenerator) makeLabels() {
	gen.generatedLabels = make(map[string]string)
	for i := 0; i < gen.labels; i++ {
		name := gen.randomString(10)
		value := gen.randomString(25)
		gen.generatedLabels[name] = value
	}

	gen.generatedLabelKeys = []string{}
	for key, _ := range gen.generatedLabels {
		gen.generatedLabelKeys = append(gen.generatedLabelKeys, key)
	}
}

func (gen *PolicyGenerator) makeServices() int {
	gen.generatedServices = make([]*language.Service, gen.services)
	for i := 0; i < gen.services; i++ {
		gen.generatedServices[i] = gen.makeService()
	}

	// add some dependencies
	cnt := make([]int, gen.services)
	maxChainLen := 0
	for i := 0; i < gen.services; i++ {
		if maxChainLen < cnt[i] {
			maxChainLen = cnt[i]
		}

		// see if we have exceeded the max chain length
		if cnt[i]+1 > gen.serviceDependencyMaxChain {
			continue
		}

		// try to add at most one dependency from each service
		j := gen.random.Intn(gen.services)
		if j <= i {
			continue
		}

		if cnt[j] < cnt[i]+1 {
			cnt[j] = cnt[i] + 1
		}

		component := &language.ServiceComponent{
			Name:    "dep-" + strconv.Itoa(i),
			Service: "service-" + strconv.Itoa(j),
		}
		gen.generatedServices[i].Components = append(gen.generatedServices[i].Components, component)
	}
	return maxChainLen
}

func (gen *PolicyGenerator) makeService() *language.Service {
	id := len(gen.policy.Services)

	service := &language.Service{
		Metadata: object.Metadata{
			Kind:      language.ServiceObject.Kind,
			Namespace: "main",
			Name:      "service-" + strconv.Itoa(id),
		},
		Owner:      "user-" + strconv.Itoa(gen.random.Intn(gen.users)),
		Components: []*language.ServiceComponent{},
	}

	for i := 0; i < gen.serviceCodeComponents; i++ {
		labelName := gen.generatedLabelKeys[gen.random.Intn(len(gen.generatedLabelKeys))]

		params := util.NestedParameterMap{}
		params["cluster"] = "cluster-test"
		for j := 0; j < gen.codeParams; j++ {
			name := "param-" + strconv.Itoa(j)
			value := "prefix-{{ .Labels." + labelName + " }}-suffix"
			params[name] = value
		}

		component := &language.ServiceComponent{
			Name: "component-" + strconv.Itoa(i),
			Code: &language.Code{
				Type:   "aptomi/code/unittests",
				Params: params,
			},
		}
		service.Components = append(service.Components, component)
	}

	gen.policy.AddObject(service)
	return service
}

func (gen *PolicyGenerator) makeContexts() {
	for i := 0; i < gen.services; i++ {
		// generate non-matching contexts
		for i := 0; i < gen.contextsPerService-1; i++ {
			context := &language.Context{
				Metadata: object.Metadata{
					Kind:      language.ContextObject.Kind,
					Namespace: "main",
					Name:      "context-" + gen.randomString(20),
				},
				Criteria: &language.Criteria{
					RequireAll: []string{"true"},
					RequireAny: []string{
						gen.randomString(20) + "=='" + gen.randomString(20) + "'",
						gen.randomString(20) + "=='" + gen.randomString(20) + "'",
						gen.randomString(20) + "=='" + gen.randomString(20) + "'",
					},
				},
			}
			gen.policy.AddObject(context)
		}

		// generate matching context
		context := &language.Context{
			Metadata: object.Metadata{
				Kind:      language.ContextObject.Kind,
				Namespace: "main",
				Name:      "context-" + gen.randomString(20),
			},
			Criteria: &language.Criteria{
				RequireAll: []string{"true"},
				RequireAny: []string{"service.Name == '" + ("service-" + strconv.Itoa(i)) + "'"},
			},
		}
		gen.policy.AddObject(context)
	}
}

func (gen *PolicyGenerator) makeDependencies() {
	for i := 0; i < gen.dependencies; i++ {
		dependency := &language.Dependency{
			Metadata: object.Metadata{
				Kind:      language.DependencyObject.Kind,
				Namespace: "main",
				Name:      "dependency-" + strconv.Itoa(i),
			},
			UserID:  "user-" + strconv.Itoa(gen.random.Intn(gen.users)),
			Service: "service-" + strconv.Itoa(gen.random.Intn(gen.services)),
		}
		gen.policy.AddObject(dependency)
	}
}

func (gen *PolicyGenerator) makeCluster() {
	cluster := &language.Cluster{
		Metadata: object.Metadata{
			Kind:      language.ClusterObject.Kind,
			Namespace: "main",
			Name:      "cluster-test",
		},
	}
	gen.policy.AddObject(cluster)
}

type UserLoaderImpl struct {
	users  int
	labels map[string]string

	cachedUsers *language.GlobalUsers
}

func NewUserLoaderImpl(users int, labels map[string]string) *UserLoaderImpl {
	return &UserLoaderImpl{
		users:  users,
		labels: labels,
	}
}

func (loader *UserLoaderImpl) LoadUsersAll() language.GlobalUsers {
	if loader.cachedUsers == nil {
		userMap := make(map[string]*language.User)
		for i := 0; i < loader.users; i++ {
			user := &language.User{
				ID:     "user-" + strconv.Itoa(i),
				Name:   "user-" + strconv.Itoa(i),
				Labels: loader.labels,
			}
			userMap[user.ID] = user
		}
		loader.cachedUsers = &language.GlobalUsers{Users: userMap}
	}
	return *loader.cachedUsers
}

func (loader *UserLoaderImpl) LoadUserByID(id string) *language.User {
	return loader.LoadUsersAll().Users[id]
}

func (loader *UserLoaderImpl) Summary() string {
	return "Synthetic user loader"
}

type SecretLoaderImpl struct {
}

func NewSecretLoaderImpl() *SecretLoaderImpl {
	return &SecretLoaderImpl{}
}

func (loader *SecretLoaderImpl) LoadSecretsByUserID(string) language.LabelSet {
	return language.NewLabelSetEmpty()
}

func RunEngine(t *testing.T, desiredPolicy *language.PolicyNamespace, externalData *external.Data) {
	timeStart := time.Now()

	actualPolicy := language.NewPolicyNamespace()
	actualState := resolvePolicy(t, actualPolicy, externalData)
	desiredState := resolvePolicy(t, desiredPolicy, externalData)

	// process all actions
	actions := diff.NewPolicyResolutionDiff(desiredState, actualState).Actions

	apply := NewEngineApply(
		desiredPolicy,
		desiredState,
		actualPolicy,
		actualState,
		externalData,
		NewTestPluginRegistry("fail-components-like-these"),
		actions,
	)

	actualState = applyAndCheck(t, apply, ResSuccess, 0, "")

	timeEnd := time.Now()
	timeDiff := timeEnd.Sub(timeStart)

	fmt.Printf("Resolved (dependencies): %d\n", len(desiredState.DependencyInstanceMap))
	fmt.Printf("Resolved (components): %d\n", len(actualState.ComponentInstanceMap))
	fmt.Printf("Time: %s\n", timeDiff.String())
}
<|endoftext|>


<commit_before>package ams

import (
	"context"
	"fmt"
	"net/http"
	"net/http/httptest"
	"reflect"
	"testing"
)

func TestClient_GetAsset(t *testing.T) {
	expected := testAsset("sample-id", "Sample")
	m := http.NewServeMux()
	m.HandleFunc(fmt.Sprintf("/Assets('%v')", expected.ID),
		testJSONHandler(t, http.MethodGet, false, http.StatusOK, expected),
	)

	s := httptest.NewServer(m)
	defer s.Close()

	client, err := NewClient(s.URL, testTokenSource())
	if err != nil {
		t.Fatal(err)
	}

	actual, err := client.GetAsset(context.TODO(), expected.ID)
	if err != nil {
		t.Error(err)
	}
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("unexpected asset. expected: %#v, actual: %#v", expected, actual)
	}
}
<commit_msg>test: add GetAssets test<commit_after>package ams

import (
	"context"
	"fmt"
	"net/http"
	"net/http/httptest"
	"reflect"
	"testing"
)

func TestClient_GetAsset(t *testing.T) {
	expected := testAsset("sample-id", "Sample")
	m := http.NewServeMux()
	m.HandleFunc(fmt.Sprintf("/Assets('%v')", expected.ID),
		testJSONHandler(t, http.MethodGet, false, http.StatusOK, expected),
	)

	s := httptest.NewServer(m)
	defer s.Close()

	client, err := NewClient(s.URL, testTokenSource())
	if err != nil {
		t.Fatal(err)
	}

	actual, err := client.GetAsset(context.TODO(), expected.ID)
	if err != nil {
		t.Error(err)
	}
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("unexpected asset. expected: %#v, actual: %#v", expected, actual)
	}
}

func TestClient_GetAssets(t *testing.T) {
	expected := []Asset{
		testAsset("asset1", "Sample 1"),
		testAsset("asset2", "Sample 2"),
		testAsset("asset4", "Example 1"),
	}
	m := http.NewServeMux()
	m.HandleFunc("/Assets",
		testJSONHandler(t, http.MethodGet, false, http.StatusOK, testWrapValue(expected)),
	)

	s := httptest.NewServer(m)
	defer s.Close()

	client, err := NewClient(s.URL, testTokenSource())
	if err != nil {
		t.Fatal(err)
	}

	actual, err := client.GetAssets(context.TODO())
	if err != nil {
		t.Error(err)
	}
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("unexpected assets. expected: %#v, actual: %#v", expected, actual)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"net/http"
	_ "net/http/pprof"
	"os"
	"strings"

	"github.com/pachyderm/pachyderm/src/client"
	healthclient "github.com/pachyderm/pachyderm/src/client/health"
	pfsclient "github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pkg/discovery"
	"github.com/pachyderm/pachyderm/src/client/pkg/grpcutil"
	"github.com/pachyderm/pachyderm/src/client/pkg/shard"
	"github.com/pachyderm/pachyderm/src/client/pkg/uuid"
	ppsclient "github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/client/version"
	"github.com/pachyderm/pachyderm/src/server/health"
	pfs_persist "github.com/pachyderm/pachyderm/src/server/pfs/db"
	"github.com/pachyderm/pachyderm/src/server/pfs/drive"
	pfs_server "github.com/pachyderm/pachyderm/src/server/pfs/server"
	cache_pb "github.com/pachyderm/pachyderm/src/server/pkg/cache/groupcachepb"
	cache_server "github.com/pachyderm/pachyderm/src/server/pkg/cache/server"
	"github.com/pachyderm/pachyderm/src/server/pkg/cmdutil"
	"github.com/pachyderm/pachyderm/src/server/pkg/metrics"
	"github.com/pachyderm/pachyderm/src/server/pkg/netutil"
	ppsserver "github.com/pachyderm/pachyderm/src/server/pps"
	"github.com/pachyderm/pachyderm/src/server/pps/persist"
	persist_server "github.com/pachyderm/pachyderm/src/server/pps/persist/server"
	pps_server "github.com/pachyderm/pachyderm/src/server/pps/server"

	flag "github.com/spf13/pflag"
	"go.pedge.io/lion"
	"go.pedge.io/lion/proto"
	"google.golang.org/grpc"
	"k8s.io/kubernetes/pkg/api"
	kube_client "k8s.io/kubernetes/pkg/client/restclient"
	kube "k8s.io/kubernetes/pkg/client/unversioned"
)

var readinessCheck bool
var migrate string

func init() {
	flag.BoolVar(&readinessCheck, "readiness-check", false, "Set to true when checking if local pod is ready")
	flag.StringVar(&migrate, "migrate", "", "Use the format FROM_VERSION-TO_VERSION; e.g. 1.2.4-1.3.0")
	flag.Parse()
}

type appEnv struct {
	Port               uint16 `env:"PORT,default=650"`
	NumShards          uint64 `env:"NUM_SHARDS,default=32"`
	StorageRoot        string `env:"PACH_ROOT,default=/pach"`
	StorageBackend     string `env:"STORAGE_BACKEND,default="`
	DatabaseAddress    string `env:"RETHINK_PORT_28015_TCP_ADDR,required"`
	PPSDatabaseName    string `env:"DATABASE_NAME,default=pachyderm_pps"`
	PFSDatabaseName    string `env:"DATABASE_NAME,default=pachyderm_pfs"`
	KubeAddress        string `env:"KUBERNETES_PORT_443_TCP_ADDR,required"`
	EtcdAddress        string `env:"ETCD_PORT_2379_TCP_ADDR,required"`
	Namespace          string `env:"NAMESPACE,default=default"`
	Metrics            bool   `env:"METRICS,default=true"`
	Init               bool   `env:"INIT,default=false"`
	BlockCacheBytes    int64  `env:"BLOCK_CACHE_BYTES,default=1073741824"` //default = 1 gigabyte
	JobShimImage       string `env:"JOB_SHIM_IMAGE,default="`
	JobImagePullPolicy string `env:"JOB_IMAGE_PULL_POLICY,default="`
	LogLevel           string `env:"LOG_LEVEL,default=info"`
}

func main() {
	cmdutil.Main(do, &appEnv{})
}

func do(appEnvObj interface{}) error {
	go func() {
		lion.Println(http.ListenAndServe(":651", nil))
	}()
	appEnv := appEnvObj.(*appEnv)
	switch appEnv.LogLevel {
	case "debug":
		lion.SetLevel(lion.LevelDebug)
	case "info":
		lion.SetLevel(lion.LevelInfo)
	case "error":
		lion.SetLevel(lion.LevelError)
	default:
		lion.Errorf("Unrecognized log level %s, falling back to default of \"info\"", appEnv.LogLevel)
		lion.SetLevel(lion.LevelInfo)
	}
	etcdClient := getEtcdClient(appEnv)
	rethinkAddress := fmt.Sprintf("%s:28015", appEnv.DatabaseAddress)
	if appEnv.Init {
		if err := persist_server.InitDBs(rethinkAddress, appEnv.PPSDatabaseName); err != nil {
			return err
		}
		return pfs_persist.InitDB(rethinkAddress, appEnv.PFSDatabaseName)
	}
	if readinessCheck {
		c, err := client.NewFromAddress("127.0.0.1:650")
		if err != nil {
			return err
		}

		// We want to use a PPS API instead of a PFS API because PFS APIs
		// typically talk to every node, but the point of the readiness probe
		// is that it checks to see if this particular node is functioning,
		// and removing it from the service if it's not.  So if we use a PFS
		// API such as ListRepo for readiness probe, then the failure of any
		// node will result in the failures of all readiness probes, causing
		// all nodes to be removed from the pachd service.
		_, err = c.ListPipeline()
		if err != nil {
			return err
		}

		os.Exit(0)

		return nil
	}
	if migrate != "" {
		return persist_server.Migrate(rethinkAddress, appEnv.PPSDatabaseName, migrate)
	}

	clusterID, err := getClusterID(etcdClient)
	if err != nil {
		return err
	}
	kubeClient, err := getKubeClient(appEnv)
	if err != nil {
		return err
	}
	var reporter *metrics.Reporter
	if appEnv.Metrics {
		reporter = metrics.NewReporter(clusterID, kubeClient, rethinkAddress, appEnv.PFSDatabaseName, appEnv.PPSDatabaseName)
	}
	rethinkAPIServer, err := getRethinkAPIServer(appEnv)
	if err != nil {
		return err
	}
	address, err := netutil.ExternalIP()
	if err != nil {
		return err
	}
	address = fmt.Sprintf("%s:%d", address, appEnv.Port)
	sharder := shard.NewSharder(
		etcdClient,
		appEnv.NumShards,
		appEnv.Namespace,
	)
	go func() {
		if err := sharder.AssignRoles(address, nil); err != nil {
			protolion.Printf("error from sharder.AssignRoles: %s", sanitizeErr(err))
		}
	}()
	driver, err := getPFSDriver(address, appEnv)
	//	driver, err := drive.NewDriver(address)
	if err != nil {
		return err
	}
	router := shard.NewRouter(
		sharder,
		grpcutil.NewDialer(
			grpc.WithInsecure(),
		),
		address,
	)
	cacheServer := cache_server.NewCacheServer(router, appEnv.NumShards)
	go func() {
		if err := sharder.RegisterFrontends(nil, address, []shard.Frontend{cacheServer}); err != nil {
			protolion.Printf("error from sharder.RegisterFrontend %s", sanitizeErr(err))
		}
	}()
	apiServer := pfs_server.NewAPIServer(driver, reporter)
	ppsAPIServer := pps_server.NewAPIServer(
		ppsserver.NewHasher(appEnv.NumShards, appEnv.NumShards),
		address,
		kubeClient,
		getNamespace(),
		appEnv.JobShimImage,
		appEnv.JobImagePullPolicy,
		reporter,
	)
	go func() {
		if err := sharder.Register(nil, address, []shard.Server{ppsAPIServer, cacheServer}); err != nil {
			protolion.Printf("error from sharder.Register %s", sanitizeErr(err))
		}
	}()
	blockAPIServer, err := pfs_server.NewBlockAPIServer(appEnv.StorageRoot, appEnv.BlockCacheBytes, appEnv.StorageBackend)
	if err != nil {
		return err
	}
	healthServer := health.NewHealthServer()
	return grpcutil.Serve(
		func(s *grpc.Server) {
			pfsclient.RegisterAPIServer(s, apiServer)
			pfsclient.RegisterBlockAPIServer(s, blockAPIServer)
			ppsclient.RegisterAPIServer(s, ppsAPIServer)
			ppsserver.RegisterInternalPodAPIServer(s, ppsAPIServer)
			persist.RegisterAPIServer(s, rethinkAPIServer)
			cache_pb.RegisterGroupCacheServer(s, cacheServer)
			healthclient.RegisterHealthServer(s, healthServer)
		},
		grpcutil.ServeOptions{
			Version:    version.Version,
			MaxMsgSize: pfs_server.MaxMsgSize,
		},
		grpcutil.ServeEnv{
			GRPCPort: appEnv.Port,
		},
	)
}

func getEtcdClient(env *appEnv) discovery.Client {
	return discovery.NewEtcdClient(fmt.Sprintf("http://%s:2379", env.EtcdAddress))
}

const clusterIDKey = "cluster-id"

func getClusterID(client discovery.Client) (string, error) {
	id, err := client.Get(clusterIDKey)
	// if it's a key not found error then we create the key
	if err != nil && strings.HasPrefix(err.Error(), "100:") {
		// This might error if it races with another pachd trying to set the
		// cluster id so we ignore the error.
		client.Create(clusterIDKey, uuid.NewWithoutDashes(), 0)
	} else if err != nil {
		return "", err
	} else {
		return id, nil
	}
	return client.Get(clusterIDKey)
}

func getKubeClient(env *appEnv) (*kube.Client, error) {
	kubeClient, err := kube.NewInCluster()
	if err != nil {
		protolion.Errorf("falling back to insecure kube client due to error from NewInCluster: %s", sanitizeErr(err))
	} else {
		return kubeClient, err
	}
	config := &kube_client.Config{
		Host:     fmt.Sprintf("%s:443", env.KubeAddress),
		Insecure: true,
	}
	return kube.New(config)
}

func getPFSDriver(address string, env *appEnv) (drive.Driver, error) {
	rethinkAddress := fmt.Sprintf("%s:28015", env.DatabaseAddress)
	return pfs_persist.NewDriver(address, rethinkAddress, env.PFSDatabaseName)
}

func getRethinkAPIServer(env *appEnv) (persist.APIServer, error) {
	if err := persist_server.CheckDBs(fmt.Sprintf("%s:28015", env.DatabaseAddress), env.PPSDatabaseName); err != nil {
		return nil, err
	}
	return persist_server.NewRethinkAPIServer(fmt.Sprintf("%s:28015", env.DatabaseAddress), env.PPSDatabaseName)
}

// getNamespace returns the kubernetes namespace that this pachd pod runs in
func getNamespace() string {
	namespace := os.Getenv("PACHD_POD_NAMESPACE")
	if namespace != "" {
		return namespace
	}
	return api.NamespaceDefault
}

func sanitizeErr(err error) error {
	if err == nil {
		return nil
	}

	return errors.New(grpc.ErrorDesc(err))
}
<commit_msg>update pachd to initialize PFS with etcd<commit_after>package main

import (
	"errors"
	"fmt"
	"net/http"
	_ "net/http/pprof"
	"os"
	"strings"

	"github.com/pachyderm/pachyderm/src/client"
	healthclient "github.com/pachyderm/pachyderm/src/client/health"
	pfsclient "github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pkg/discovery"
	"github.com/pachyderm/pachyderm/src/client/pkg/grpcutil"
	"github.com/pachyderm/pachyderm/src/client/pkg/shard"
	"github.com/pachyderm/pachyderm/src/client/pkg/uuid"
	ppsclient "github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/client/version"
	"github.com/pachyderm/pachyderm/src/server/health"
	pfs_driver "github.com/pachyderm/pachyderm/src/server/pfs/drive"
	pfs_server "github.com/pachyderm/pachyderm/src/server/pfs/server"
	cache_pb "github.com/pachyderm/pachyderm/src/server/pkg/cache/groupcachepb"
	cache_server "github.com/pachyderm/pachyderm/src/server/pkg/cache/server"
	"github.com/pachyderm/pachyderm/src/server/pkg/cmdutil"
	"github.com/pachyderm/pachyderm/src/server/pkg/metrics"
	"github.com/pachyderm/pachyderm/src/server/pkg/netutil"
	ppsserver "github.com/pachyderm/pachyderm/src/server/pps"
	"github.com/pachyderm/pachyderm/src/server/pps/persist"
	persist_server "github.com/pachyderm/pachyderm/src/server/pps/persist/server"
	pps_server "github.com/pachyderm/pachyderm/src/server/pps/server"

	flag "github.com/spf13/pflag"
	"go.pedge.io/lion"
	"go.pedge.io/lion/proto"
	"google.golang.org/grpc"
	"k8s.io/kubernetes/pkg/api"
	kube_client "k8s.io/kubernetes/pkg/client/restclient"
	kube "k8s.io/kubernetes/pkg/client/unversioned"
)

var readinessCheck bool
var migrate string

func init() {
	flag.BoolVar(&readinessCheck, "readiness-check", false, "Set to true when checking if local pod is ready")
	flag.StringVar(&migrate, "migrate", "", "Use the format FROM_VERSION-TO_VERSION; e.g. 1.2.4-1.3.0")
	flag.Parse()
}

type appEnv struct {
	Port               uint16 `env:"PORT,default=650"`
	NumShards          uint64 `env:"NUM_SHARDS,default=32"`
	StorageRoot        string `env:"PACH_ROOT,default=/pach"`
	StorageBackend     string `env:"STORAGE_BACKEND,default="`
	DatabaseAddress    string `env:"RETHINK_PORT_28015_TCP_ADDR,required"`
	PPSDatabaseName    string `env:"DATABASE_NAME,default=pachyderm_pps"`
	PFSDatabaseName    string `env:"DATABASE_NAME,default=pachyderm_pfs"`
	KubeAddress        string `env:"KUBERNETES_PORT_443_TCP_ADDR,required"`
	EtcdAddress        string `env:"ETCD_PORT_2379_TCP_ADDR,required"`
	Namespace          string `env:"NAMESPACE,default=default"`
	Metrics            bool   `env:"METRICS,default=true"`
	Init               bool   `env:"INIT,default=false"`
	BlockCacheBytes    int64  `env:"BLOCK_CACHE_BYTES,default=1073741824"` //default = 1 gigabyte
	JobShimImage       string `env:"JOB_SHIM_IMAGE,default="`
	JobImagePullPolicy string `env:"JOB_IMAGE_PULL_POLICY,default="`
	LogLevel           string `env:"LOG_LEVEL,default=info"`
}

func main() {
	cmdutil.Main(do, &appEnv{})
}

func do(appEnvObj interface{}) error {
	go func() {
		lion.Println(http.ListenAndServe(":651", nil))
	}()
	appEnv := appEnvObj.(*appEnv)
	switch appEnv.LogLevel {
	case "debug":
		lion.SetLevel(lion.LevelDebug)
	case "info":
		lion.SetLevel(lion.LevelInfo)
	case "error":
		lion.SetLevel(lion.LevelError)
	default:
		lion.Errorf("Unrecognized log level %s, falling back to default of \"info\"", appEnv.LogLevel)
		lion.SetLevel(lion.LevelInfo)
	}
	etcdClient := getEtcdClient(appEnv)
	rethinkAddress := fmt.Sprintf("%s:28015", appEnv.DatabaseAddress)
	if appEnv.Init {
		if err := persist_server.InitDBs(rethinkAddress, appEnv.PPSDatabaseName); err != nil {
			return err
		}
	}
	if readinessCheck {
		c, err := client.NewFromAddress("127.0.0.1:650")
		if err != nil {
			return err
		}

		// We want to use a PPS API instead of a PFS API because PFS APIs
		// typically talk to every node, but the point of the readiness probe
		// is that it checks to see if this particular node is functioning,
		// and removing it from the service if it's not.  So if we use a PFS
		// API such as ListRepo for readiness probe, then the failure of any
		// node will result in the failures of all readiness probes, causing
		// all nodes to be removed from the pachd service.
		_, err = c.ListPipeline()
		if err != nil {
			return err
		}

		os.Exit(0)

		return nil
	}
	if migrate != "" {
		return persist_server.Migrate(rethinkAddress, appEnv.PPSDatabaseName, migrate)
	}

	clusterID, err := getClusterID(etcdClient)
	if err != nil {
		return err
	}
	kubeClient, err := getKubeClient(appEnv)
	if err != nil {
		return err
	}
	var reporter *metrics.Reporter
	if appEnv.Metrics {
		reporter = metrics.NewReporter(clusterID, kubeClient, rethinkAddress, appEnv.PFSDatabaseName, appEnv.PPSDatabaseName)
	}
	rethinkAPIServer, err := getRethinkAPIServer(appEnv)
	if err != nil {
		return err
	}
	address, err := netutil.ExternalIP()
	if err != nil {
		return err
	}
	address = fmt.Sprintf("%s:%d", address, appEnv.Port)
	sharder := shard.NewSharder(
		etcdClient,
		appEnv.NumShards,
		appEnv.Namespace,
	)
	go func() {
		if err := sharder.AssignRoles(address, nil); err != nil {
			protolion.Printf("error from sharder.AssignRoles: %s", sanitizeErr(err))
		}
	}()
	driver, err := pfs_driver.NewDriver(address, []string(appEnv.EtcdAddress), appEnv.PFSDatabaseName)
	//	driver, err := drive.NewDriver(address)
	if err != nil {
		return err
	}
	router := shard.NewRouter(
		sharder,
		grpcutil.NewDialer(
			grpc.WithInsecure(),
		),
		address,
	)
	cacheServer := cache_server.NewCacheServer(router, appEnv.NumShards)
	go func() {
		if err := sharder.RegisterFrontends(nil, address, []shard.Frontend{cacheServer}); err != nil {
			protolion.Printf("error from sharder.RegisterFrontend %s", sanitizeErr(err))
		}
	}()
	apiServer := pfs_server.NewAPIServer(driver, reporter)
	ppsAPIServer := pps_server.NewAPIServer(
		ppsserver.NewHasher(appEnv.NumShards, appEnv.NumShards),
		address,
		kubeClient,
		getNamespace(),
		appEnv.JobShimImage,
		appEnv.JobImagePullPolicy,
		reporter,
	)
	go func() {
		if err := sharder.Register(nil, address, []shard.Server{ppsAPIServer, cacheServer}); err != nil {
			protolion.Printf("error from sharder.Register %s", sanitizeErr(err))
		}
	}()
	blockAPIServer, err := pfs_server.NewBlockAPIServer(appEnv.StorageRoot, appEnv.BlockCacheBytes, appEnv.StorageBackend)
	if err != nil {
		return err
	}
	healthServer := health.NewHealthServer()
	return grpcutil.Serve(
		func(s *grpc.Server) {
			pfsclient.RegisterAPIServer(s, apiServer)
			pfsclient.RegisterBlockAPIServer(s, blockAPIServer)
			ppsclient.RegisterAPIServer(s, ppsAPIServer)
			ppsserver.RegisterInternalPodAPIServer(s, ppsAPIServer)
			persist.RegisterAPIServer(s, rethinkAPIServer)
			cache_pb.RegisterGroupCacheServer(s, cacheServer)
			healthclient.RegisterHealthServer(s, healthServer)
		},
		grpcutil.ServeOptions{
			Version:    version.Version,
			MaxMsgSize: pfs_server.MaxMsgSize,
		},
		grpcutil.ServeEnv{
			GRPCPort: appEnv.Port,
		},
	)
}

func getEtcdClient(env *appEnv) discovery.Client {
	return discovery.NewEtcdClient(fmt.Sprintf("http://%s:2379", env.EtcdAddress))
}

const clusterIDKey = "cluster-id"

func getClusterID(client discovery.Client) (string, error) {
	id, err := client.Get(clusterIDKey)
	// if it's a key not found error then we create the key
	if err != nil && strings.HasPrefix(err.Error(), "100:") {
		// This might error if it races with another pachd trying to set the
		// cluster id so we ignore the error.
		client.Create(clusterIDKey, uuid.NewWithoutDashes(), 0)
	} else if err != nil {
		return "", err
	} else {
		return id, nil
	}
	return client.Get(clusterIDKey)
}

func getKubeClient(env *appEnv) (*kube.Client, error) {
	kubeClient, err := kube.NewInCluster()
	if err != nil {
		protolion.Errorf("falling back to insecure kube client due to error from NewInCluster: %s", sanitizeErr(err))
	} else {
		return kubeClient, err
	}
	config := &kube_client.Config{
		Host:     fmt.Sprintf("%s:443", env.KubeAddress),
		Insecure: true,
	}
	return kube.New(config)
}

func getRethinkAPIServer(env *appEnv) (persist.APIServer, error) {
	if err := persist_server.CheckDBs(fmt.Sprintf("%s:28015", env.DatabaseAddress), env.PPSDatabaseName); err != nil {
		return nil, err
	}
	return persist_server.NewRethinkAPIServer(fmt.Sprintf("%s:28015", env.DatabaseAddress), env.PPSDatabaseName)
}

// getNamespace returns the kubernetes namespace that this pachd pod runs in
func getNamespace() string {
	namespace := os.Getenv("PACHD_POD_NAMESPACE")
	if namespace != "" {
		return namespace
	}
	return api.NamespaceDefault
}

func sanitizeErr(err error) error {
	if err == nil {
		return nil
	}

	return errors.New(grpc.ErrorDesc(err))
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
)

func dumpStart(d dumpTarget) result {
	t := time.Now()
	mongoHost := os.Getenv("MONGO_HOST")
	mongoPort := os.Getenv("MONGO_PORT")
	if mongoHost == "" {
		mongoHost = "mongo"
	}
	if mongoPort == "" {
		mongoPort = "27017"
	}
	var r result

	path, err := exec.LookPath("mongodump")
	if err != nil {
		log.Println("Could not find Mongodump", err)
		r.Result = "Could not find Mongodump " + err.Error()
		return r
	}
	fmt.Printf("mongodump is available at %s ", path)

	dumpCmd := exec.Command("mongodump", "--host", mongoHost, "--port", mongoPort, "--archive")
	body, err := dumpCmd.StdoutPipe()
	if err != nil {
		log.Println("failed executing mongodump", err)
		r.Result = "failed executing mongodump " + err.Error()
		return r
	}

	if err := dumpCmd.Start(); err != nil {
		log.Println("failed starting mongodump", err)
		r.Result = "failed starting mongodump " + err.Error()
		return r
	}

	// Wrap the pipe to hide the seek methods from the uploader
	var bodyWrap = struct {
		io.Reader
	}{body}

	uploader := s3manager.NewUploader(session.New(&aws.Config{Region: aws.String("us-east-1")}))
	result, err := uploader.Upload(&s3manager.UploadInput{
		Body:   bodyWrap,
		Bucket: aws.String(d.Bucket),
		Key:    aws.String(d.Path + t.Format("2006-01-02T15:04")),
	})
	if err != nil {
		log.Println("Failed to upload", err)
		r.Result = "Failed to upload " + err.Error()
		return r
	}

	if err := dumpCmd.Wait(); err != nil {
		log.Println("Failed to dump", err)
		r.Result = "Failed to dump " + err.Error()
		return r
	}

	log.Println("Successfully uploaded to", result.Location)
	r.Result = "success"
	return r
}
<commit_msg>make tests pass again<commit_after>package main

import (
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
)

func dumpStart(d dumpTarget) result {
	t := time.Now()
	mongoHost := os.Getenv("MONGOHOST")
	mongoPort := os.Getenv("MONGOPORT")
	if mongoHost == "" {
		mongoHost = "mongo"
	}
	if mongoPort == "" {
		mongoPort = "27017"
	}
	var r result

	path, err := exec.LookPath("mongodump")
	if err != nil {
		log.Println("Could not find Mongodump", err)
		r.Result = "Could not find Mongodump " + err.Error()
		return r
	}
	fmt.Printf("mongodump is available at %s ", path)

	dumpCmd := exec.Command("mongodump", "--host", mongoHost, "--port", mongoPort, "--archive")
	body, err := dumpCmd.StdoutPipe()
	if err != nil {
		log.Println("failed executing mongodump", err)
		r.Result = "failed executing mongodump " + err.Error()
		return r
	}

	if err := dumpCmd.Start(); err != nil {
		log.Println("failed starting mongodump", err)
		r.Result = "failed starting mongodump " + err.Error()
		return r
	}

	// Wrap the pipe to hide the seek methods from the uploader
	var bodyWrap = struct {
		io.Reader
	}{body}

	uploader := s3manager.NewUploader(session.New(&aws.Config{Region: aws.String("us-east-1")}))
	result, err := uploader.Upload(&s3manager.UploadInput{
		Body:   bodyWrap,
		Bucket: aws.String(d.Bucket),
		Key:    aws.String(d.Path + t.Format("2006-01-02T15:04")),
	})
	if err != nil {
		log.Println("Failed to upload", err)
		r.Result = "Failed to upload " + err.Error()
		return r
	}

	if err := dumpCmd.Wait(); err != nil {
		log.Println("Failed to dump", err)
		r.Result = "Failed to dump " + err.Error()
		return r
	}

	log.Println("Successfully uploaded to", result.Location)
	r.Result = "success"
	return r
}
<|endoftext|>


<commit_before>package storage

import "fmt"
import "time"
import "launchpad.net/gozk/zookeeper"

/**
 * ZooKeeper Storage Manager
 *
 * @author: Anant Bhardwaj
 * @date: 05/11/2014
 */

type StorageManager struct {
  conn *zookeeper.Conn
}

func MakeStorageManager() *StorageManager {
  sm := &StorageManager{}
  return sm
}

func (sm *StorageManager) Open(servers string) error {
  conn, session, err := zookeeper.Dial(servers, 5 * time.Second)
  if err != nil {
    fmt.Printf("Can't connect to zookeeper: %v\n", err)
    return err
  }

  // Wait for connection.
  event := <-session
  if event.State != zookeeper.STATE_CONNECTED {
    fmt.Printf("Can't connect to zookeeper: %v\n", event)
    return err
  }

  sm.conn = conn
  return nil
}

func (sm *StorageManager) Close() error {
  return sm.conn.Close()
}

func (sm *StorageManager) Create(path string, data string) error {
  stats, _ := sm.conn.Exists(path)
  var err error

  if stats == nil {
    _, err = sm.conn.Create(path, data, 0, zookeeper.WorldACL(zookeeper.PERM_ALL))
  }

  if err != nil {
    fmt.Printf("Error creating a node (%v): %v\n", path, err)
  }

  return err
}

func (sm *StorageManager) Delete(path string) error {

  err := sm.conn.Delete(path, -1)

  if err != nil {
    fmt.Printf("Error writing to node (%v): %v\n", path, err)
  }

  return err
}

func (sm *StorageManager) Write(path string, data string) error {

  _, err := sm.conn.Set(path, data, -1)

  if err != nil {
    err_create := sm.Create(path, data)
    if err_create != nil {
      fmt.Printf("Error writing to node (%v): %v\n", path, err)
      return err_create
    }
  }
  return err
}

func (sm *StorageManager) Read(path string) (string, error) {
  data, _, err := sm.conn.Get(path)

  if err != nil {
    fmt.Printf("Error reading from node (%v): %v\n", path, err)
  }

  return data, err
}
<commit_msg>persistant paxos<commit_after>package storage

import "fmt"
import "time"
import "launchpad.net/gozk/zookeeper"

/**
 * ZooKeeper Storage Manager
 *
 * @author: Anant Bhardwaj
 * @date: 05/11/2014
 */

type StorageManager struct {
  conn *zookeeper.Conn
}

func MakeStorageManager() *StorageManager {
  sm := &StorageManager{}
  return sm
}

func (sm *StorageManager) Open(servers string) error {
  conn, session, err := zookeeper.Dial(servers, 5 * time.Second)
  if err != nil {
    fmt.Printf("Can't connect to zookeeper: %v\n", err)
    return err
  }

  // Wait for connection.
  event := <-session
  if event.State != zookeeper.STATE_CONNECTED {
    fmt.Printf("Can't connect to zookeeper: %v\n", event)
    return err
  }

  sm.conn = conn
  return nil
}

func (sm *StorageManager) Close() error {
  return sm.conn.Close()
}

func (sm *StorageManager) Create(path string, data string) error {
  stats, _ := sm.conn.Exists(path)
  var err error

  if stats == nil {
    _, err = sm.conn.Create(path, data, 0, zookeeper.WorldACL(zookeeper.PERM_ALL))
  }

  if err != nil {
    fmt.Printf("Error creating a node (%v): %v\n", path, err)
  } else {
    fmt.Printf("Created node (%v): %v\n", path, data)
  }

  return err
}

func (sm *StorageManager) Delete(path string) error {

  err := sm.conn.Delete(path, -1)

  if err != nil {
    fmt.Printf("Error writing to node (%v): %v\n", path, err)
  } else {
    fmt.Printf("Deleted node (%v)", path)
  }

  return err
}

func (sm *StorageManager) Write(path string, data string) error {

  _, err := sm.conn.Set(path, data, -1)

  if err != nil {
    err_create := sm.Create(path, data)
    if err_create != nil {
      fmt.Printf("Error writing to node (%v): %v\n", path, err)
      return err_create
    }
  }

  fmt.Printf("Written node (%v): %v\n", path, data)
  return err
}

func (sm *StorageManager) Read(path string) (string, error) {
  data, _, err := sm.conn.Get(path)

  if err != nil {
    fmt.Printf("Error reading from node (%v): %v\n", path, err)
  }

  fmt.Printf("Read node (%v): %v\n", path, data)

  return data, err
}
<|endoftext|>


<commit_before>package runtime_test

import (
	"net/http"
	"reflect"
	"testing"

	"github.com/gengo/grpc-gateway/runtime"
	"golang.org/x/net/context"
	"google.golang.org/grpc/metadata"
)

func TestAnnotateContext(t *testing.T) {
	ctx := context.Background()

	request, _ := http.NewRequest("GET", "http://localhost", nil)
	request.Header = http.Header{}
	annotated := runtime.AnnotateContext(ctx, request)
	if annotated != ctx {
		t.Errorf("AnnotateContext(ctx, request) = %v; want %v", annotated, ctx)
	}
	request.Header.Add("Grpc-Metadata-FooBar", "Value1")
	request.Header.Add("Grpc-Metadata-Foo-BAZ", "Value2")
	annotated = runtime.AnnotateContext(ctx, request)
	md, ok := metadata.FromContext(annotated)
	if !ok || len(md) != 2 {
		t.Errorf("Expected 2 metadata items in context; got %v", md)
	}
	if got, want := md["Foobar"], []string{"Value1"}; !reflect.DeepEqual(got, want) {
		t.Errorf("md[\"Foobar\"] = %v; want %v", got, want)
	}
	if got, want := md["Foo-Baz"], []string{"Value2"}; !reflect.DeepEqual(got, want) {
		t.Errorf("md[\"Foo-Baz\"] = %v want %v", got, want)
	}
}
<commit_msg>Fix broken test<commit_after>package runtime_test

import (
	"net/http"
	"reflect"
	"testing"

	"github.com/gengo/grpc-gateway/runtime"
	"golang.org/x/net/context"
	"google.golang.org/grpc/metadata"
)

func TestAnnotateContext(t *testing.T) {
	ctx := context.Background()

	request, _ := http.NewRequest("GET", "http://localhost", nil)
	request.Header = http.Header{}
	annotated := runtime.AnnotateContext(ctx, request)
	if annotated != ctx {
		t.Errorf("AnnotateContext(ctx, request) = %v; want %v", annotated, ctx)
	}
	request.Header.Add("Grpc-Metadata-FooBar", "Value1")
	request.Header.Add("Grpc-Metadata-Foo-BAZ", "Value2")
	annotated = runtime.AnnotateContext(ctx, request)
	md, ok := metadata.FromContext(annotated)
	if !ok || len(md) != 2 {
		t.Errorf("Expected 2 metadata items in context; got %v", md)
	}
	if got, want := md["foobar"], []string{"Value1"}; !reflect.DeepEqual(got, want) {
		t.Errorf(`md["foobar"] = %q; want %q`, got, want)
	}
	if got, want := md["foo-baz"], []string{"Value2"}; !reflect.DeepEqual(got, want) {
		t.Errorf(`md["foo-baz"] = %q want %q`, got, want)
	}
}
<|endoftext|>


<commit_before>package runtime_test

import (
	"encoding/json"
	"fmt"
	"reflect"
	"testing"

	"github.com/golang/protobuf/proto"
	"github.com/golang/protobuf/ptypes/duration"
	"github.com/golang/protobuf/ptypes/timestamp"
	"github.com/grpc-ecosystem/grpc-gateway/runtime"
)

func TestConvertTimestamp(t *testing.T) {
	specs := []struct {
		name    string
		input   string
		output  *timestamp.Timestamp
		wanterr error
	}{
		{
			name:  "a valid RFC3339 timestamp",
			input: `"2016-05-10T10:19:13.123Z"`,
			output: &timestamp.Timestamp{
				Seconds: 1462875553,
				Nanos:   123000000,
			},
			wanterr: nil,
		},
		{
			name:    "invalid timestamp",
			input:   `"05-10-2016T10:19:13.123Z"`,
			output:  nil,
			wanterr: fmt.Errorf(`bad Timestamp: parsing time "05-10-2016T10:19:13.123Z" as "2006-01-02T15:04:05.999999999Z07:00": cannot parse "0-2016T10:19:13.123Z" as "2006"`),
		},
		{
			name:   "JSON number",
			input:  "123",
			output: nil,
			wanterr: &json.UnmarshalTypeError{
				Value:  "number",
				Type:   reflect.TypeOf("123"),
				Offset: 3,
			},
		},
		{
			name:   "JSON bool",
			input:  "true",
			output: nil,
			wanterr: &json.UnmarshalTypeError{
				Value:  "bool",
				Type:   reflect.TypeOf("123"),
				Offset: 4,
			},
		},
	}

	for _, spec := range specs {
		t.Run(spec.name, func(t *testing.T) {
			ts, err := runtime.Timestamp(spec.input)
			if spec.wanterr != nil {
				if !reflect.DeepEqual(err, spec.wanterr) {
					t.Errorf("got unexpected error\n%#v\nexpected\n%#v", err, spec.wanterr)
				}
				return
			}
			if !proto.Equal(ts, spec.output) {
				t.Errorf(
					"when testing %s; got\n%#v\nexpected\n%#v",
					spec.name,
					ts,
					spec.output,
				)
			}
		})
	}
}

func TestConvertDuration(t *testing.T) {
	specs := []struct {
		name    string
		input   string
		output  *duration.Duration
		wanterr error
	}{
		{
			name:  "a valid duration",
			input: `"123.456s"`,
			output: &duration.Duration{
				Seconds: 123,
				Nanos:   456000000,
			},
			wanterr: nil,
		},
		{
			name:    "invalid duration",
			input:   `"123years"`,
			output:  nil,
			wanterr: fmt.Errorf(`bad Duration: time: unknown unit years in duration 123years`),
		},
		{
			name:   "JSON number",
			input:  "123",
			output: nil,
			wanterr: &json.UnmarshalTypeError{
				Value:  "number",
				Type:   reflect.TypeOf("123"),
				Offset: 3,
			},
		},
		{
			name:   "JSON bool",
			input:  "true",
			output: nil,
			wanterr: &json.UnmarshalTypeError{
				Value:  "bool",
				Type:   reflect.TypeOf("123"),
				Offset: 4,
			},
		},
	}

	for _, spec := range specs {
		t.Run(spec.name, func(t *testing.T) {
			ts, err := runtime.Duration(spec.input)
			if spec.wanterr != nil {
				if !reflect.DeepEqual(err, spec.wanterr) {
					t.Errorf("got unexpected error\n%#v\nexpected\n%#v", err, spec.wanterr)
				}
				return
			}
			if !proto.Equal(ts, spec.output) {
				t.Errorf(
					"when testing %s; got\n%#v\nexpected\n%#v",
					spec.name,
					ts,
					spec.output,
				)
			}
		})
	}
}
<commit_msg>Rewrite conversion tests to ignore error format<commit_after>package runtime_test

import (
	"testing"

	"github.com/golang/protobuf/proto"
	"github.com/golang/protobuf/ptypes/duration"
	"github.com/golang/protobuf/ptypes/timestamp"
	"github.com/grpc-ecosystem/grpc-gateway/runtime"
)

func TestConvertTimestamp(t *testing.T) {
	specs := []struct {
		name    string
		input   string
		output  *timestamp.Timestamp
		wanterr bool
	}{
		{
			name:  "a valid RFC3339 timestamp",
			input: `"2016-05-10T10:19:13.123Z"`,
			output: &timestamp.Timestamp{
				Seconds: 1462875553,
				Nanos:   123000000,
			},
			wanterr: false,
		},
		{
			name:    "invalid timestamp",
			input:   `"05-10-2016T10:19:13.123Z"`,
			output:  nil,
			wanterr: true,
		},
		{
			name:    "JSON number",
			input:   "123",
			output:  nil,
			wanterr: true,
		},
		{
			name:    "JSON bool",
			input:   "true",
			output:  nil,
			wanterr: true,
		},
	}

	for _, spec := range specs {
		t.Run(spec.name, func(t *testing.T) {
			ts, err := runtime.Timestamp(spec.input)
			switch {
			case err != nil && !spec.wanterr:
				t.Errorf("got unexpected error\n%#v", err)
			case err == nil && spec.wanterr:
				t.Errorf("did not error when expecte")
			case !proto.Equal(ts, spec.output):
				t.Errorf(
					"when testing %s; got\n%#v\nexpected\n%#v",
					spec.name,
					ts,
					spec.output,
				)
			}
		})
	}
}

func TestConvertDuration(t *testing.T) {
	specs := []struct {
		name    string
		input   string
		output  *duration.Duration
		wanterr bool
	}{
		{
			name:  "a valid duration",
			input: `"123.456s"`,
			output: &duration.Duration{
				Seconds: 123,
				Nanos:   456000000,
			},
			wanterr: false,
		},
		{
			name:    "invalid duration",
			input:   `"123years"`,
			output:  nil,
			wanterr: true,
		},
		{
			name:    "JSON number",
			input:   "123",
			output:  nil,
			wanterr: true,
		},
		{
			name:    "JSON bool",
			input:   "true",
			output:  nil,
			wanterr: true,
		},
	}

	for _, spec := range specs {
		t.Run(spec.name, func(t *testing.T) {
			ts, err := runtime.Duration(spec.input)
			switch {
			case err != nil && !spec.wanterr:
				t.Errorf("got unexpected error\n%#v", err)
			case err == nil && spec.wanterr:
				t.Errorf("did not error when expecte")
			case !proto.Equal(ts, spec.output):
				t.Errorf(
					"when testing %s; got\n%#v\nexpected\n%#v",
					spec.name,
					ts,
					spec.output,
				)
			}
		})
	}
}
<|endoftext|>


<commit_before>package baudio

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"math"
	"runtime"
	"strconv"
	//"time"
)

const (
	FuncValueTypeFloat    = 0
	FuncValueTypeNotFloat = 1
)

type BChannel struct {
	funcValueType int
	funcs         []func(float64, int) float64
}

func newBChannel(fvt int) *BChannel {
	bc := &BChannel{
		funcValueType: fvt,
		funcs:         make([]func(float64, int) float64, 0),
	}
	return bc
}

func (bc *BChannel) push(fn func(float64, int) float64) {
	bc.funcs = append(bc.funcs, fn)
}

type BOptions struct {
	Size int
	Rate int
}

func NewBOptions() *BOptions {
	return &BOptions{
		Size: 2048,
		Rate: 44000,
	}
}

type B struct {
	readable   bool
	size       int
	rate       int
	t          float64
	i          int
	paused     bool
	ended      bool
	destroyed  bool
	channels   []*BChannel
	chEnd      chan bool
	chEndSox   chan bool
	chResume   chan func()
	chNextTick chan bool
	pipeReader *io.PipeReader
	pipeWriter *io.PipeWriter
	sox        *Sox
}

func New(opts *BOptions, fn func(float64, int) float64) *B {
	b := &B{
		readable:   true,
		size:       2048,
		rate:       44000,
		t:          0,
		i:          0,
		paused:     false,
		ended:      false,
		destroyed:  false,
		chEnd:      make(chan bool),
		chEndSox:   make(chan bool),
		chResume:   make(chan func()),
		chNextTick: make(chan bool),
	}
	b.pipeReader, b.pipeWriter = io.Pipe()
	if opts != nil {
		b.size = opts.Size
		b.rate = opts.Rate
	}
	if fn != nil {
		b.Push(fn)
	}
	go func() {
		if b.paused {
			b.chResume <- func() {
				go b.loop()
				b.main()
			}
		} else {
			go b.loop()
			b.main()
		}
	}()
	//go b.loop()
	return b
}

func (b *B) main() {
	for {
		// 2013-02-28 koyachi ここで何かしないとループまわらないのなぜ
		// => fmt.PrinfすることでnodeのnextTick的なものがつまれててそのうちPlay()のread待ちまで進めるのでは。
		//L1:
		//fmt.Println("main loop header")
		//fmt.Printf(".")
		//time.Sleep(1 * time.Millisecond)
		runtime.Gosched()
		select {
		case <-b.chEnd:
			fmt.Println("main chEnd")
			b.terminateMain()
			break
		case fn := <-b.chResume:
			//fmt.Println("main chResume")
			fn()
		case <-b.chNextTick:
			//fmt.Println("main chNextTick")
			go b.loop()
			//b.loop()
		default:
			//fmt.Println("main default")
			//go b.loop()
			//goto L1
		}
	}
}

func (b *B) terminateMain() {
	b.pipeWriter.Close()
	b.ended = true
	b.chEndSox <- true
}

// TODO: To Go Style (end,destroy,pause,resume are node.js's Stream interface.)
func (b *B) End() {
	b.ended = true
}

func (b *B) Destroy() {
	b.destroyed = true
	b.chEnd <- true
}

func (b *B) Pause() {
	b.paused = true
}

func (b *B) Resume() {
	if !b.paused {
		return
	}
	b.paused = false
	b.chResume <- func() {}
}

func (b *B) AddChannel(funcValueType int, fn func(float64, int) float64) {
	bc := newBChannel(funcValueType)
	bc.push(fn)
	b.channels = append(b.channels, bc)
}

func (b *B) Push(fn func(float64, int) float64) {
	index := len(b.channels)
	if len(b.channels) <= index {
		bc := newBChannel(FuncValueTypeFloat)
		b.channels = append(b.channels, bc)
	}
	b.channels[index].funcs = append(b.channels[index].funcs, fn)
}

func (b *B) loop() {
	buf := b.tick()
	if b.destroyed {
		// no more events
		//fmt.Println("loop destroyed")
	} else if b.paused {
		//fmt.Println("loop paused")
		b.chResume <- func() {
			b.pipeWriter.Write(buf.Bytes())
			b.chNextTick <- true
		}
	} else {
		//fmt.Println("loop !(destroyed || paused)")
		b.pipeWriter.Write(buf.Bytes())
		if b.ended {
			//fmt.Println("loop ended")
			b.chEnd <- true
		} else {
			//fmt.Println("loop !ended")
			b.chNextTick <- true
		}
	}
}

func (b *B) tick() *bytes.Buffer {
	bufSize := b.size * len(b.channels)
	byteBuffer := make([]byte, 0)
	buf := bytes.NewBuffer(byteBuffer)
	for i := 0; i < bufSize; i += 2 {
		lrIndex := int(i / 2)
		lenCh := len(b.channels)
		ch := b.channels[lrIndex%lenCh]
		t := float64(b.t) + math.Floor(float64(lrIndex))/float64(b.rate)/float64(lenCh)
		counter := b.i + int(math.Floor(float64(lrIndex)/float64(lenCh)))

		value := float64(0)
		n := float64(0)
		for j := 0; j < len(ch.funcs); j++ {
			x := ch.funcs[j](float64(t), counter)
			n += x
		}
		n /= float64(len(ch.funcs))

		if ch.funcValueType == FuncValueTypeFloat {
			value = signed(n)
		} else {
			b_ := math.Pow(2, float64(ch.funcValueType))
			x := math.Mod(math.Floor(n), b_) / b_ * math.Pow(2, 15)
			value = x
		}
		if err := binary.Write(buf, binary.LittleEndian, int16(clamp(value))); err != nil {
			panic(err)
		}
	}
	b.i += b.size / 2
	b.t += float64(b.size) / float64(2) / float64(b.rate)
	return buf
}

func clamp(x float64) float64 {
	return math.Max(math.Min(x, math.Pow(2, 15)-1), -math.Pow(2, 15))
}

func signed(n float64) float64 {
	b := math.Pow(2, 15)
	if n > 0 {
		return math.Min(b-1, math.Floor(b*n-1))
	}
	return math.Max(-b, math.Ceil(b*n-1))
}

func (b *B) Play(opts map[string]string) {
	go SoxPlay(mergeArgs(opts, map[string]string{
		"c": strconv.Itoa(len(b.channels)),
		"r": strconv.Itoa(b.rate),
		"t": "s16",
		"-": "DUMMY",
	}), b.waveReceiver())
	<-b.chEndSox
	b.pipeReader.Close()
}

func (b *B) Record(file string, opts map[string]string) {
	go SoxRecord(file, mergeArgs(opts, map[string]string{
		"c": strconv.Itoa(len(b.channels)),
		"r": strconv.Itoa(b.rate),
		"t": "s16",
		"-": "DUMMY",
	}), b.waveReceiver())
	<-b.chEndSox
	b.pipeReader.Close()
}

func (b *B) waveReceiver() func() []byte {
	readBuf := make([]byte, b.size*len(b.channels))
	return func() []byte {
		if _, err := b.pipeReader.Read(readBuf); err != nil {
			panic(err)
		}
		return readBuf
	}
}
<commit_msg>add types: GeneratorFunc, RuntimeOption<commit_after>package baudio

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"math"
	"runtime"
	"strconv"
	//"time"
)

const (
	FuncValueTypeFloat    = 0
	FuncValueTypeNotFloat = 1
)

type GeneratorFunc func(t float64, i int) float64
type RuntimeOption map[string]string

type BChannel struct {
	funcValueType int
	funcs         []GeneratorFunc
}

func newBChannel(fvt int) *BChannel {
	bc := &BChannel{
		funcValueType: fvt,
		funcs:         make([]GeneratorFunc, 0),
	}
	return bc
}

func (bc *BChannel) push(fn GeneratorFunc) {
	bc.funcs = append(bc.funcs, fn)
}

type BOptions struct {
	Size int
	Rate int
}

func NewBOptions() *BOptions {
	return &BOptions{
		Size: 2048,
		Rate: 44000,
	}
}

type B struct {
	readable   bool
	size       int
	rate       int
	t          float64
	i          int
	paused     bool
	ended      bool
	destroyed  bool
	channels   []*BChannel
	chEnd      chan bool
	chEndSox   chan bool
	chResume   chan func()
	chNextTick chan bool
	pipeReader *io.PipeReader
	pipeWriter *io.PipeWriter
	sox        *Sox
}

func New(opts *BOptions, fn GeneratorFunc) *B {
	b := &B{
		readable:   true,
		size:       2048,
		rate:       44000,
		t:          0,
		i:          0,
		paused:     false,
		ended:      false,
		destroyed:  false,
		chEnd:      make(chan bool),
		chEndSox:   make(chan bool),
		chResume:   make(chan func()),
		chNextTick: make(chan bool),
	}
	b.pipeReader, b.pipeWriter = io.Pipe()
	if opts != nil {
		b.size = opts.Size
		b.rate = opts.Rate
	}
	if fn != nil {
		b.Push(fn)
	}
	go func() {
		if b.paused {
			b.chResume <- func() {
				go b.loop()
				b.main()
			}
		} else {
			go b.loop()
			b.main()
		}
	}()
	//go b.loop()
	return b
}

func (b *B) main() {
	for {
		// 2013-02-28 koyachi ここで何かしないとループまわらないのなぜ
		// => fmt.PrinfすることでnodeのnextTick的なものがつまれててそのうちPlay()のread待ちまで進めるのでは。
		//L1:
		//fmt.Println("main loop header")
		//fmt.Printf(".")
		//time.Sleep(1 * time.Millisecond)
		runtime.Gosched()
		select {
		case <-b.chEnd:
			fmt.Println("main chEnd")
			b.terminateMain()
			break
		case fn := <-b.chResume:
			//fmt.Println("main chResume")
			fn()
		case <-b.chNextTick:
			//fmt.Println("main chNextTick")
			go b.loop()
			//b.loop()
		default:
			//fmt.Println("main default")
			//go b.loop()
			//goto L1
		}
	}
}

func (b *B) terminateMain() {
	b.pipeWriter.Close()
	b.ended = true
	b.chEndSox <- true
}

// TODO: To Go Style (end,destroy,pause,resume are node.js's Stream interface.)
func (b *B) End() {
	b.ended = true
}

func (b *B) Destroy() {
	b.destroyed = true
	b.chEnd <- true
}

func (b *B) Pause() {
	b.paused = true
}

func (b *B) Resume() {
	if !b.paused {
		return
	}
	b.paused = false
	b.chResume <- func() {}
}

func (b *B) AddChannel(funcValueType int, fn GeneratorFunc) {
	bc := newBChannel(funcValueType)
	bc.push(fn)
	b.channels = append(b.channels, bc)
}

func (b *B) Push(fn GeneratorFunc) {
	index := len(b.channels)
	if len(b.channels) <= index {
		bc := newBChannel(FuncValueTypeFloat)
		b.channels = append(b.channels, bc)
	}
	b.channels[index].funcs = append(b.channels[index].funcs, fn)
}

func (b *B) loop() {
	buf := b.tick()
	if b.destroyed {
		// no more events
		//fmt.Println("loop destroyed")
	} else if b.paused {
		//fmt.Println("loop paused")
		b.chResume <- func() {
			b.pipeWriter.Write(buf.Bytes())
			b.chNextTick <- true
		}
	} else {
		//fmt.Println("loop !(destroyed || paused)")
		b.pipeWriter.Write(buf.Bytes())
		if b.ended {
			//fmt.Println("loop ended")
			b.chEnd <- true
		} else {
			//fmt.Println("loop !ended")
			b.chNextTick <- true
		}
	}
}

func (b *B) tick() *bytes.Buffer {
	bufSize := b.size * len(b.channels)
	byteBuffer := make([]byte, 0)
	buf := bytes.NewBuffer(byteBuffer)
	for i := 0; i < bufSize; i += 2 {
		lrIndex := int(i / 2)
		lenCh := len(b.channels)
		ch := b.channels[lrIndex%lenCh]
		t := float64(b.t) + math.Floor(float64(lrIndex))/float64(b.rate)/float64(lenCh)
		counter := b.i + int(math.Floor(float64(lrIndex)/float64(lenCh)))

		value := float64(0)
		n := float64(0)
		for j := 0; j < len(ch.funcs); j++ {
			x := ch.funcs[j](float64(t), counter)
			n += x
		}
		n /= float64(len(ch.funcs))

		if ch.funcValueType == FuncValueTypeFloat {
			value = signed(n)
		} else {
			b_ := math.Pow(2, float64(ch.funcValueType))
			x := math.Mod(math.Floor(n), b_) / b_ * math.Pow(2, 15)
			value = x
		}
		if err := binary.Write(buf, binary.LittleEndian, int16(clamp(value))); err != nil {
			panic(err)
		}
	}
	b.i += b.size / 2
	b.t += float64(b.size) / float64(2) / float64(b.rate)
	return buf
}

func clamp(x float64) float64 {
	return math.Max(math.Min(x, math.Pow(2, 15)-1), -math.Pow(2, 15))
}

func signed(n float64) float64 {
	b := math.Pow(2, 15)
	if n > 0 {
		return math.Min(b-1, math.Floor(b*n-1))
	}
	return math.Max(-b, math.Ceil(b*n-1))
}

func (b *B) Play(opts RuntimeOption) {
	go SoxPlay(mergeArgs(opts, RuntimeOption{
		"c": strconv.Itoa(len(b.channels)),
		"r": strconv.Itoa(b.rate),
		"t": "s16",
		"-": "DUMMY",
	}), b.waveReceiver())
	<-b.chEndSox
	b.pipeReader.Close()
}

func (b *B) Record(file string, opts RuntimeOption) {
	go SoxRecord(file, mergeArgs(opts, RuntimeOption{
		"c": strconv.Itoa(len(b.channels)),
		"r": strconv.Itoa(b.rate),
		"t": "s16",
		"-": "DUMMY",
	}), b.waveReceiver())
	<-b.chEndSox
	b.pipeReader.Close()
}

func (b *B) waveReceiver() func() []byte {
	readBuf := make([]byte, b.size*len(b.channels))
	return func() []byte {
		if _, err := b.pipeReader.Read(readBuf); err != nil {
			panic(err)
		}
		return readBuf
	}
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/aes"
	"crypto/cipher"
	"crypto/hmac"
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"hash"
)

// aescbchmac implements plain AES-CBC+HMAC enctrypyion-decryption
type aescbchmac struct {
	c cipher.Block
	h hash.Hash
}

func newAesCbcHmac(key string) (PacketEncrypter, error) {

	if "" == key {
		return nil, errors.New("key is empty")
	}

	bkey, err := hex.DecodeString(key)
	if nil != err {
		return nil, errors.New("not valid hex string")
	}

	lbkey := len(bkey)
	if (lbkey != 48) && (lbkey != 56) && (lbkey != 64) {
		return nil, errors.New("Length of key must be 48, 56 or 64 bytes (96, 112 or 128 hex symbols) to select AES-128, AES-192 or AES-256 (+ 32 for HMAC sha256 key)")
	}

	a := aescbchmac{}
	a.c, err = aes.NewCipher(bkey[0 : lbkey-32])
	if nil != err {
		return nil, err
	}
	a.h = hmac.New(sha256.New, bkey[lbkey-32:])

	return &a, nil
}

func (a *aescbchmac) CheckSize(size int) bool {
	return size > (aes.BlockSize+32) && size%aes.BlockSize == 0
}

func (a *aescbchmac) AdjustInputSize(size int) int {
	if size%aes.BlockSize != 0 {
		return size + (aes.BlockSize - (size % aes.BlockSize))
	}
	return size
}

func (a *aescbchmac) Encrypt(input []byte, output []byte, iv []byte) int {
	copy(output[:aes.BlockSize], iv)
	cipher.NewCBCEncrypter(a.c, iv).CryptBlocks(output[aes.BlockSize:], input)

	inputLen := len(input)
	// whole len of output is len(input) + aes.BlockSize, so copy of last aes.BlockSize
	copy(iv, output[inputLen:])
	copy(output[inputLen+aes.BlockSize:], a.h.Sum(output[:inputLen+aes.BlockSize]))

	return inputLen + aes.BlockSize + 32
}

func (a *aescbchmac) Decrypt(input []byte, output []byte) int {
	inputLen := len(input)
	test := a.h.Sum(input[:inputLen-32])
	if !hmac.Equal(input[inputLen-32:], test) {
		return 0
	}

	resultLen := len(input) - aes.BlockSize - 32
	cipher.NewCBCDecrypter(a.c, input[:aes.BlockSize]).CryptBlocks(input[aes.BlockSize:resultLen], output[:resultLen])
	return resultLen
}

func (a *aescbchmac) OutputAdd() int {
	// adding IV and HMAC-SHA256 to each message
	return aes.BlockSize + 32
}

func (a *aescbchmac) IVLen() int {
	return aes.BlockSize
}

func init() {
	registredEncrypters["aescbchmac"] = newAesCbcHmac
}
<commit_msg>Use hmac::Size() instead of constant size<commit_after>package main

import (
	"crypto/aes"
	"crypto/cipher"
	"crypto/hmac"
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"hash"
)

// aescbchmac implements plain AES-CBC+HMAC enctrypyion-decryption
type aescbchmac struct {
	c        cipher.Block
	h        hash.Hash
	hashsize int
}

func newAesCbcHmac(key string) (PacketEncrypter, error) {

	if "" == key {
		return nil, errors.New("key is empty")
	}

	bkey, err := hex.DecodeString(key)
	if nil != err {
		return nil, errors.New("not valid hex string")
	}

	lbkey := len(bkey)
	if (lbkey != 48) && (lbkey != 56) && (lbkey != 64) {
		return nil, errors.New("Length of key must be 48, 56 or 64 bytes (96, 112 or 128 hex symbols) to select AES-128, AES-192 or AES-256 (+ 32 for HMAC sha256 key)")
	}

	a := aescbchmac{}
	a.c, err = aes.NewCipher(bkey[0 : lbkey-32])
	if nil != err {
		return nil, err
	}
	a.h = hmac.New(sha256.New, bkey[lbkey-32:])

	a.hashsize = a.h.Size()

	return &a, nil
}

func (a *aescbchmac) CheckSize(size int) bool {
	return size > (aes.BlockSize+a.hashsize) && size%aes.BlockSize == 0
}

func (a *aescbchmac) AdjustInputSize(size int) int {
	if size%aes.BlockSize != 0 {
		return size + (aes.BlockSize - (size % aes.BlockSize))
	}
	return size
}

func (a *aescbchmac) Encrypt(input []byte, output []byte, iv []byte) int {
	copy(output[:aes.BlockSize], iv)
	cipher.NewCBCEncrypter(a.c, iv).CryptBlocks(output[aes.BlockSize:], input)

	inputLen := len(input)
	// whole len of output is len(input) + aes.BlockSize, so copy of last aes.BlockSize
	copy(iv, output[inputLen:])
	copy(output[inputLen+aes.BlockSize:], a.h.Sum(output[:inputLen+aes.BlockSize]))

	return inputLen + aes.BlockSize + a.hashsize
}

func (a *aescbchmac) Decrypt(input []byte, output []byte) int {
	inputLen := len(input)
	test := a.h.Sum(input[:inputLen-a.hashsize])
	if !hmac.Equal(input[inputLen-a.hashsize:], test) {
		return 0
	}

	resultLen := len(input) - aes.BlockSize - a.hashsize
	cipher.NewCBCDecrypter(a.c, input[:aes.BlockSize]).CryptBlocks(input[aes.BlockSize:resultLen], output[:resultLen])
	return resultLen
}

func (a *aescbchmac) OutputAdd() int {
	// adding IV and HMAC-SHA256 to each message
	return aes.BlockSize + a.hashsize
}

func (a *aescbchmac) IVLen() int {
	return aes.BlockSize
}

func init() {
	registredEncrypters["aescbchmac"] = newAesCbcHmac
}
<|endoftext|>


<commit_before>package air

import (
	"encoding/json"
	"encoding/xml"
	"errors"
	"mime"
	"reflect"
)

// binder is a binder that binds request based on the MIME types.
type binder struct{}

// theBinder is the singleton of the `binder`.
var theBinder = &binder{}

// bind binds the r into the v.
func (b *binder) bind(v interface{}, r *Request) error {
	if r.Method == "GET" {
		return b.bindParams(v, r.Params())
	} else if r.Body == nil {
		return errors.New("request body cannot be empty")
	}

	mt, _, err := mime.ParseMediaType(r.Header("content-type").Value())
	if err != nil {
		return err
	}

	switch mt {
	case "application/json":
		err = json.NewDecoder(r.Body).Decode(v)
	case "application/xml":
		err = xml.NewDecoder(r.Body).Decode(v)
	case "application/x-www-form-urlencoded", "multipart/form-data":
		err = b.bindParams(v, r.Params())
	default:
		return errors.New("unsupported media type")
	}

	if err != nil {
		return err
	}

	return nil
}

// bindParams binds the params into the v.
func (b *binder) bindParams(v interface{}, params []*RequestParam) error {
	typ := reflect.TypeOf(v).Elem()
	if typ.Kind() != reflect.Struct {
		return errors.New("binding element must be a struct")
	}

	val := reflect.ValueOf(v).Elem()
	for i := 0; i < typ.NumField(); i++ {
		vf := val.Field(i)
		if !vf.CanSet() {
			continue
		}

		vfk := vf.Kind()
		if vfk == reflect.Struct {
			err := b.bindParams(vf.Addr().Interface(), params)
			if err != nil {
				return err
			}

			continue
		}

		tf := typ.Field(i)

		var pv *RequestParamValue
		for _, p := range params {
			if p.Name == tf.Name {
				pv = p.Value()
				break
			}
		}

		if pv == nil {
			continue
		}

		switch tf.Type.Kind() {
		case reflect.Bool:
			b, _ := pv.Bool()
			vf.SetBool(b)
		case reflect.Int,
			reflect.Int8,
			reflect.Int16,
			reflect.Int32,
			reflect.Int64:
			i64, _ := pv.Int64()
			vf.SetInt(i64)
		case reflect.Uint,
			reflect.Uint8,
			reflect.Uint16,
			reflect.Uint32,
			reflect.Uint64:
			ui64, _ := pv.Uint64()
			vf.SetUint(ui64)
		case reflect.Float32, reflect.Float64:
			f64, _ := pv.Float64()
			vf.SetFloat(f64)
		case reflect.String:
			vf.SetString(pv.String())
		default:
			return errors.New("unknown type")
		}
	}

	return nil
}
<commit_msg>add msgpack support<commit_after>package air

import (
	"encoding/json"
	"encoding/xml"
	"errors"
	"mime"
	"reflect"

	"github.com/vmihailenco/msgpack"
)

// binder is a binder that binds request based on the MIME types.
type binder struct{}

// theBinder is the singleton of the `binder`.
var theBinder = &binder{}

// bind binds the r into the v.
func (b *binder) bind(v interface{}, r *Request) error {
	if r.Method == "GET" {
		return b.bindParams(v, r.Params())
	} else if r.Body == nil {
		return errors.New("request body cannot be empty")
	}

	mt, _, err := mime.ParseMediaType(r.Header("content-type").Value())
	if err != nil {
		return err
	}

	switch mt {
	case "application/json":
		err = json.NewDecoder(r.Body).Decode(v)
	case "application/msgpack":
		err = msgpack.NewDecoder(r.Body).Decode(v)
	case "application/xml":
		err = xml.NewDecoder(r.Body).Decode(v)
	case "application/x-www-form-urlencoded", "multipart/form-data":
		err = b.bindParams(v, r.Params())
	default:
		return errors.New("unsupported media type")
	}

	if err != nil {
		return err
	}

	return nil
}

// bindParams binds the params into the v.
func (b *binder) bindParams(v interface{}, params []*RequestParam) error {
	typ := reflect.TypeOf(v).Elem()
	if typ.Kind() != reflect.Struct {
		return errors.New("binding element must be a struct")
	}

	val := reflect.ValueOf(v).Elem()
	for i := 0; i < typ.NumField(); i++ {
		vf := val.Field(i)
		if !vf.CanSet() {
			continue
		}

		vfk := vf.Kind()
		if vfk == reflect.Struct {
			err := b.bindParams(vf.Addr().Interface(), params)
			if err != nil {
				return err
			}

			continue
		}

		tf := typ.Field(i)

		var pv *RequestParamValue
		for _, p := range params {
			if p.Name == tf.Name {
				pv = p.Value()
				break
			}
		}

		if pv == nil {
			continue
		}

		switch tf.Type.Kind() {
		case reflect.Bool:
			b, _ := pv.Bool()
			vf.SetBool(b)
		case reflect.Int,
			reflect.Int8,
			reflect.Int16,
			reflect.Int32,
			reflect.Int64:
			i64, _ := pv.Int64()
			vf.SetInt(i64)
		case reflect.Uint,
			reflect.Uint8,
			reflect.Uint16,
			reflect.Uint32,
			reflect.Uint64:
			ui64, _ := pv.Uint64()
			vf.SetUint(ui64)
		case reflect.Float32, reflect.Float64:
			f64, _ := pv.Float64()
			vf.SetFloat(f64)
		case reflect.String:
			vf.SetString(pv.String())
		default:
			return errors.New("unknown type")
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
  "log"
)

func Registrar(new_state map[string]*FileState, input chan []*FileEvent) {
  state := new_state
  for events := range input {
    log.Printf("Registrar received %d events\n", len(events))
    // Take the last event found for each file source
    for _, event := range events {
      // skip stdin
      if *event.Source == "-" {
        continue
      }
      state[*event.Source] = &FileState{
        Source: event.Source,
        Offset: event.Offset,
      }
      file_ids(event.fileinfo, state[*event.Source])
      //log.Printf("State %s: %d\n", *event.Source, event.Offset)
    }

    if len(state) > 0 {
      WriteRegistry(state, ".logstash-forwarder")
    }
  }
}
<commit_msg>Tidy registrar<commit_after>package main

import (
  "log"
)

func Registrar(state map[string]*FileState, input chan []*FileEvent) {
  for events := range input {
    log.Printf("Registrar received %d events\n", len(events))
    // Take the last event found for each file source
    for _, event := range events {
      // skip stdin
      if *event.Source == "-" {
        continue
      }
      state[*event.Source] = &FileState{
        Source: event.Source,
        Offset: event.Offset,
      }
      file_ids(event.fileinfo, state[*event.Source])
      //log.Printf("State %s: %d\n", *event.Source, event.Offset)
    }

    if len(state) > 0 {
      WriteRegistry(state, ".logstash-forwarder")
    }
  }
}
<|endoftext|>