<commit_before>package credential

// TODO: properly comment all data structures and functions
import (
	"math/big"
)

type IdemixCredential struct {
	Signature  *CLSignature
	Pk         *PublicKey
	Attributes []*big.Int
}

type DisclosureProofBuilder struct {
	randomizedSignature   *CLSignature
	eCommit, vCommit      *big.Int
	attrRandomizers       map[int]*big.Int
	z                     *big.Int
	disclosedAttributes   []int
	undisclosedAttributes []int
	pk                    *PublicKey
	attributes            []*big.Int
}

func getUndisclosedAttributes(disclosedAttributes []int, numAttributes int) []int {
	check := make([]bool, numAttributes)
	for _, v := range disclosedAttributes {
		check[v] = true
	}
	r := make([]int, 0, numAttributes)
	for i, v := range check {
		if !v {
			r = append(r, i)
		}
	}
	return r
}

func (ic *IdemixCredential) CreateDisclosureProof(disclosedAttributes []int, context, nonce1 *big.Int) *ProofD {
	undisclosedAttributes := getUndisclosedAttributes(disclosedAttributes, len(ic.Attributes))

	randSig := ic.Signature.Randomize(ic.Pk)

	eCommit, _ := randomBigInt(ic.Pk.Params.LeCommit)
	vCommit, _ := randomBigInt(ic.Pk.Params.LvCommit)

	aCommits := make(map[int]*big.Int)
	for _, v := range undisclosedAttributes {
		aCommits[v], _ = randomBigInt(ic.Pk.Params.LmCommit)
	}

	// Z = A^{e_commit} * S^{v_commit}
	//     PROD_{i \in undisclosed} ( R_i^{a_commits{i}} )
	Ae := modPow(randSig.A, eCommit, &ic.Pk.N)
	Sv := modPow(&ic.Pk.S, vCommit, &ic.Pk.N)
	Z := new(big.Int).Mul(Ae, Sv)
	Z.Mod(Z, &ic.Pk.N)

	for _, v := range undisclosedAttributes {
		Z.Mul(Z, modPow(ic.Pk.R[v], aCommits[v], &ic.Pk.N))
		Z.Mod(Z, &ic.Pk.N)
	}

	c := hashCommit([]*big.Int{context, randSig.A, Z, nonce1})

	ePrime := new(big.Int).Sub(randSig.E, new(big.Int).Lsh(bigONE, ic.Pk.Params.Le-1))
	eResponse := new(big.Int).Mul(c, ePrime)
	eResponse.Add(eCommit, eResponse)
	vResponse := new(big.Int).Mul(c, randSig.V)
	vResponse.Add(vCommit, vResponse)

	aResponses := make(map[int]*big.Int)
	for _, v := range undisclosedAttributes {
		t := new(big.Int).Mul(c, ic.Attributes[v])
		aResponses[v] = t.Add(aCommits[v], t)
	}

	aDisclosed := make(map[int]*big.Int)
	for _, v := range disclosedAttributes {
		aDisclosed[v] = ic.Attributes[v]
	}

	return &ProofD{c: c, A: randSig.A, eResponse: eResponse, vResponse: vResponse, aResponses: aResponses, aDisclosed: aDisclosed}
}

func (ic *IdemixCredential) CreateDisclosureProofBuilder(disclosedAttributes []int) *DisclosureProofBuilder {
	d := &DisclosureProofBuilder{}
	d.pk = ic.Pk
	d.randomizedSignature = ic.Signature.Randomize(ic.Pk)
	d.eCommit, _ = randomBigInt(ic.Pk.Params.LeCommit)
	d.vCommit, _ = randomBigInt(ic.Pk.Params.LvCommit)

	d.attrRandomizers = make(map[int]*big.Int)
	d.disclosedAttributes = disclosedAttributes
	d.undisclosedAttributes = getUndisclosedAttributes(disclosedAttributes, len(ic.Attributes))
	d.attributes = ic.Attributes
	for _, v := range d.undisclosedAttributes {
		d.attrRandomizers[v], _ = randomBigInt(ic.Pk.Params.LmCommit)
	}

	return d
}

// TODO: Eventually replace skRandomizer with an array
func (d *DisclosureProofBuilder) Commit(skRandomizer *big.Int) []*big.Int {
	d.attrRandomizers[0] = skRandomizer

	// Z = A^{e_commit} * S^{v_commit}
	//     PROD_{i \in undisclosed} ( R_i^{a_commits{i}} )
	Ae := modPow(d.randomizedSignature.A, d.eCommit, &d.pk.N)
	Sv := modPow(&d.pk.S, d.vCommit, &d.pk.N)
	d.z = new(big.Int).Mul(Ae, Sv)
	d.z.Mod(d.z, &d.pk.N)

	for _, v := range d.undisclosedAttributes {
		d.z.Mul(d.z, modPow(d.pk.R[v], d.attrRandomizers[v], &d.pk.N))
		d.z.Mod(d.z, &d.pk.N)
	}

	return []*big.Int{d.randomizedSignature.A, d.z}
}

func (d *DisclosureProofBuilder) CreateProof(challenge *big.Int) Proof {
	ePrime := new(big.Int).Sub(d.randomizedSignature.E, new(big.Int).Lsh(bigONE, d.pk.Params.Le-1))
	eResponse := new(big.Int).Mul(challenge, ePrime)
	eResponse.Add(d.eCommit, eResponse)
	vResponse := new(big.Int).Mul(challenge, d.randomizedSignature.V)
	vResponse.Add(d.vCommit, vResponse)

	aResponses := make(map[int]*big.Int)
	for _, v := range d.undisclosedAttributes {
		t := new(big.Int).Mul(challenge, d.attributes[v])
		aResponses[v] = t.Add(d.attrRandomizers[v], t)
	}

	aDisclosed := make(map[int]*big.Int)
	for _, v := range d.disclosedAttributes {
		aDisclosed[v] = d.attributes[v]
	}

	return &ProofD{c: challenge, A: d.randomizedSignature.A, eResponse: eResponse, vResponse: vResponse, aResponses: aResponses, aDisclosed: aDisclosed}
}
<commit_msg>Cleanup of credential.go.<commit_after>package credential

import (
	"math/big"
)

// IdemixCredential represents an Idemix credential.
type IdemixCredential struct {
	Signature  *CLSignature
	Pk         *PublicKey
	Attributes []*big.Int
}

// DisclosureProofBuilder is an object that holds the state for the protocol to
// produce a disclosure proof.
type DisclosureProofBuilder struct {
	randomizedSignature   *CLSignature
	eCommit, vCommit      *big.Int
	attrRandomizers       map[int]*big.Int
	z                     *big.Int
	disclosedAttributes   []int
	undisclosedAttributes []int
	pk                    *PublicKey
	attributes            []*big.Int
}

// getUndisclosedAttributes computes, given a list of (indices of) disclosed
// attributes, a list of undisclosed attributes.
func getUndisclosedAttributes(disclosedAttributes []int, numAttributes int) []int {
	check := make([]bool, numAttributes)
	for _, v := range disclosedAttributes {
		check[v] = true
	}
	r := make([]int, 0, numAttributes)
	for i, v := range check {
		if !v {
			r = append(r, i)
		}
	}
	return r
}

// CreateDisclosureProof creates a disclosure proof (ProofD) voor the provided
// indices of disclosed attributes.
func (ic *IdemixCredential) CreateDisclosureProof(disclosedAttributes []int, context, nonce1 *big.Int) *ProofD {
	undisclosedAttributes := getUndisclosedAttributes(disclosedAttributes, len(ic.Attributes))

	randSig := ic.Signature.Randomize(ic.Pk)

	eCommit, _ := randomBigInt(ic.Pk.Params.LeCommit)
	vCommit, _ := randomBigInt(ic.Pk.Params.LvCommit)

	aCommits := make(map[int]*big.Int)
	for _, v := range undisclosedAttributes {
		aCommits[v], _ = randomBigInt(ic.Pk.Params.LmCommit)
	}

	// Z = A^{e_commit} * S^{v_commit}
	//     PROD_{i \in undisclosed} ( R_i^{a_commits{i}} )
	Ae := modPow(randSig.A, eCommit, &ic.Pk.N)
	Sv := modPow(&ic.Pk.S, vCommit, &ic.Pk.N)
	Z := new(big.Int).Mul(Ae, Sv)
	Z.Mod(Z, &ic.Pk.N)

	for _, v := range undisclosedAttributes {
		Z.Mul(Z, modPow(ic.Pk.R[v], aCommits[v], &ic.Pk.N))
		Z.Mod(Z, &ic.Pk.N)
	}

	c := hashCommit([]*big.Int{context, randSig.A, Z, nonce1})

	ePrime := new(big.Int).Sub(randSig.E, new(big.Int).Lsh(bigONE, ic.Pk.Params.Le-1))
	eResponse := new(big.Int).Mul(c, ePrime)
	eResponse.Add(eCommit, eResponse)
	vResponse := new(big.Int).Mul(c, randSig.V)
	vResponse.Add(vCommit, vResponse)

	aResponses := make(map[int]*big.Int)
	for _, v := range undisclosedAttributes {
		t := new(big.Int).Mul(c, ic.Attributes[v])
		aResponses[v] = t.Add(aCommits[v], t)
	}

	aDisclosed := make(map[int]*big.Int)
	for _, v := range disclosedAttributes {
		aDisclosed[v] = ic.Attributes[v]
	}

	return &ProofD{c: c, A: randSig.A, eResponse: eResponse, vResponse: vResponse, aResponses: aResponses, aDisclosed: aDisclosed}
}

// CreateDisclosureProofBuilder produces a DisclosureProofBuilder, an object to
// hold the state in the protocol for producing a disclosure proof that is
// linked to other proofs.
func (ic *IdemixCredential) CreateDisclosureProofBuilder(disclosedAttributes []int) *DisclosureProofBuilder {
	d := &DisclosureProofBuilder{}
	d.pk = ic.Pk
	d.randomizedSignature = ic.Signature.Randomize(ic.Pk)
	d.eCommit, _ = randomBigInt(ic.Pk.Params.LeCommit)
	d.vCommit, _ = randomBigInt(ic.Pk.Params.LvCommit)

	d.attrRandomizers = make(map[int]*big.Int)
	d.disclosedAttributes = disclosedAttributes
	d.undisclosedAttributes = getUndisclosedAttributes(disclosedAttributes, len(ic.Attributes))
	d.attributes = ic.Attributes
	for _, v := range d.undisclosedAttributes {
		d.attrRandomizers[v], _ = randomBigInt(ic.Pk.Params.LmCommit)
	}

	return d
}

// TODO: Eventually replace skRandomizer with an array

// Commit commits to the first attribute (the secret) using the provided
// randomizer.
func (d *DisclosureProofBuilder) Commit(skRandomizer *big.Int) []*big.Int {
	d.attrRandomizers[0] = skRandomizer

	// Z = A^{e_commit} * S^{v_commit}
	//     PROD_{i \in undisclosed} ( R_i^{a_commits{i}} )
	Ae := modPow(d.randomizedSignature.A, d.eCommit, &d.pk.N)
	Sv := modPow(&d.pk.S, d.vCommit, &d.pk.N)
	d.z = new(big.Int).Mul(Ae, Sv)
	d.z.Mod(d.z, &d.pk.N)

	for _, v := range d.undisclosedAttributes {
		d.z.Mul(d.z, modPow(d.pk.R[v], d.attrRandomizers[v], &d.pk.N))
		d.z.Mod(d.z, &d.pk.N)
	}

	return []*big.Int{d.randomizedSignature.A, d.z}
}

// CreateProof creates a (disclosure) proof with the provided challenge.
func (d *DisclosureProofBuilder) CreateProof(challenge *big.Int) Proof {
	ePrime := new(big.Int).Sub(d.randomizedSignature.E, new(big.Int).Lsh(bigONE, d.pk.Params.Le-1))
	eResponse := new(big.Int).Mul(challenge, ePrime)
	eResponse.Add(d.eCommit, eResponse)
	vResponse := new(big.Int).Mul(challenge, d.randomizedSignature.V)
	vResponse.Add(d.vCommit, vResponse)

	aResponses := make(map[int]*big.Int)
	for _, v := range d.undisclosedAttributes {
		t := new(big.Int).Mul(challenge, d.attributes[v])
		aResponses[v] = t.Add(d.attrRandomizers[v], t)
	}

	aDisclosed := make(map[int]*big.Int)
	for _, v := range d.disclosedAttributes {
		aDisclosed[v] = d.attributes[v]
	}

	return &ProofD{c: challenge, A: d.randomizedSignature.A, eResponse: eResponse, vResponse: vResponse, aResponses: aResponses, aDisclosed: aDisclosed}
}
<|endoftext|>


<commit_before>package metrics

import (
	"bytes"
	"os"
	"strings"
	"syscall"
	"testing"
	"time"
)

func TestInmemSignal(t *testing.T) {
	buf := bytes.NewBuffer(nil)
	inm := NewInmemSink(10*time.Millisecond, 50*time.Millisecond)
	sig := NewInmemSignal(inm, syscall.SIGUSR1, buf)
	defer sig.Stop()

	inm.SetGauge([]string{"foo"}, 42)
	inm.EmitKey([]string{"bar"}, 42)
	inm.IncrCounter([]string{"baz"}, 42)
	inm.AddSample([]string{"wow"}, 42)
	inm.SetGaugeWithLabels([]string{"asdf"}, 42, []Label{{"a", "b"}})
	inm.IncrCounterWithLabels([]string{"qwer"}, 42, []Label{{"a", "b"}})
	inm.AddSampleWithLabels([]string{"zxcv"}, 42, []Label{{"a", "b"}})

	// Wait for period to end
	time.Sleep(15 * time.Millisecond)

	// Send signal!
	syscall.Kill(os.Getpid(), syscall.SIGUSR1)

	// Wait for flush
	time.Sleep(10 * time.Millisecond)

	// Check the output
	out := string(buf.Bytes())
	if !strings.Contains(out, "[G] 'foo': 42") {
		t.Fatalf("bad: %v", out)
	}
	if !strings.Contains(out, "[P] 'bar': 42") {
		t.Fatalf("bad: %v", out)
	}
	if !strings.Contains(out, "[C] 'baz': Count: 1 Sum: 42") {
		t.Fatalf("bad: %v", out)
	}
	if !strings.Contains(out, "[S] 'wow': Count: 1 Sum: 42") {
		t.Fatalf("bad: %v", out)
	}
	if !strings.Contains(out, "[G] 'asdf.b': 42") {
		t.Fatalf("bad: %v", out)
	}
	if !strings.Contains(out, "[C] 'qwer.b': Count: 1 Sum: 42") {
		t.Fatalf("bad: %v", out)
	}
	if !strings.Contains(out, "[S] 'zxcv.b': Count: 1 Sum: 42") {
		t.Fatalf("bad: %v", out)
	}
}
<commit_msg>race condition in inmem_signal_test needs sync<commit_after>package metrics

import (
	"bytes"
	"os"
	"strings"
	"sync"
	"syscall"
	"testing"
	"time"
)

func TestInmemSignal(t *testing.T) {
	buf := newBuffer()
	inm := NewInmemSink(10*time.Millisecond, 50*time.Millisecond)
	sig := NewInmemSignal(inm, syscall.SIGUSR1, buf)
	defer sig.Stop()

	inm.SetGauge([]string{"foo"}, 42)
	inm.EmitKey([]string{"bar"}, 42)
	inm.IncrCounter([]string{"baz"}, 42)
	inm.AddSample([]string{"wow"}, 42)
	inm.SetGaugeWithLabels([]string{"asdf"}, 42, []Label{{"a", "b"}})
	inm.IncrCounterWithLabels([]string{"qwer"}, 42, []Label{{"a", "b"}})
	inm.AddSampleWithLabels([]string{"zxcv"}, 42, []Label{{"a", "b"}})

	// Wait for period to end
	time.Sleep(15 * time.Millisecond)

	// Send signal!
	syscall.Kill(os.Getpid(), syscall.SIGUSR1)

	// Wait for flush
	time.Sleep(10 * time.Millisecond)

	// Check the output
	out := buf.String()
	if !strings.Contains(out, "[G] 'foo': 42") {
		t.Fatalf("bad: %v", out)
	}
	if !strings.Contains(out, "[P] 'bar': 42") {
		t.Fatalf("bad: %v", out)
	}
	if !strings.Contains(out, "[C] 'baz': Count: 1 Sum: 42") {
		t.Fatalf("bad: %v", out)
	}
	if !strings.Contains(out, "[S] 'wow': Count: 1 Sum: 42") {
		t.Fatalf("bad: %v", out)
	}
	if !strings.Contains(out, "[G] 'asdf.b': 42") {
		t.Fatalf("bad: %v", out)
	}
	if !strings.Contains(out, "[C] 'qwer.b': Count: 1 Sum: 42") {
		t.Fatalf("bad: %v", out)
	}
	if !strings.Contains(out, "[S] 'zxcv.b': Count: 1 Sum: 42") {
		t.Fatalf("bad: %v", out)
	}
}

func newBuffer() *syncBuffer {
	return &syncBuffer{buf: bytes.NewBuffer(nil)}
}

type syncBuffer struct {
	buf  *bytes.Buffer
	lock sync.Mutex
}

func (s *syncBuffer) Write(p []byte) (int, error) {
	s.lock.Lock()
	defer s.lock.Unlock()

	return s.buf.Write(p)
}

func (s *syncBuffer) String() string {
	s.lock.Lock()
	defer s.lock.Unlock()

	return s.buf.String()
}
<|endoftext|>


<commit_before>// Copyright 2015 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package isolatedclient

import (
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"

	"github.com/luci/luci-go/client/internal/lhttp"
	"github.com/luci/luci-go/client/internal/retry"
	"github.com/luci/luci-go/client/internal/tracer"
	"github.com/luci/luci-go/common/isolated"
)

// IsolateServer is the low-level client interface to interact with an Isolate
// server.
type IsolateServer interface {
	ServerCapabilities() (*isolated.ServerCapabilities, error)
	// Contains looks up cache presence on the server of multiple items.
	//
	// The returned list is in the same order as 'items', with entries nil for
	// items that were present.
	Contains(items []*isolated.DigestItem) ([]*PushState, error)
	Push(state *PushState, src io.Reader) error
}

// PushState is per-item state passed from IsolateServer.Contains() to
// IsolateServer.Push().
//
// Its content is implementation specific.
type PushState struct {
	status    isolated.PreuploadStatus
	size      int64
	uploaded  bool
	finalized bool
}

// New returns a new IsolateServer client.
func New(host, namespace string) IsolateServer {
	i := &isolateServer{
		url:       strings.TrimRight(host, "/"),
		namespace: namespace,
	}
	tracer.NewTID(i, nil, i.url)
	return i
}

// Private details.

type isolateServer struct {
	url       string
	namespace string
}

func (i *isolateServer) postJSON(resource string, in, out interface{}) error {
	if len(resource) == 0 || resource[0] != '/' {
		return errors.New("resource must start with '/'")
	}
	_, err := lhttp.PostJSON(retry.Default, http.DefaultClient, i.url+resource, in, out)
	return err
}

func (i *isolateServer) ServerCapabilities() (*isolated.ServerCapabilities, error) {
	out := &isolated.ServerCapabilities{}
	if err := i.postJSON("/_ah/api/isolateservice/v1/server_details", map[string]string{}, out); err != nil {
		return nil, err
	}
	return out, nil
}

func (i *isolateServer) Contains(items []*isolated.DigestItem) (out []*PushState, err error) {
	end := tracer.Span(i, "contains", strconv.Itoa(len(items)), nil)
	defer func() { end(tracer.Args{"err": err}) }()
	in := isolated.DigestCollection{Items: items}
	in.Namespace.Namespace = i.namespace
	data := &isolated.UrlCollection{}
	if err = i.postJSON("/_ah/api/isolateservice/v1/preupload", in, data); err != nil {
		return nil, err
	}
	out = make([]*PushState, len(items))
	for _, e := range data.Items {
		index := int(e.Index)
		out[index] = &PushState{
			status: e,
			size:   items[index].Size,
		}
	}
	return out, nil
}

func (i *isolateServer) Push(state *PushState, src io.Reader) (err error) {
	// This push operation may be a retry after failed finalization call below,
	// no need to reupload contents in that case.
	if !state.uploaded {
		// PUT file to uploadURL.
		if err = i.doPush(state, src); err != nil {
			return
		}
		state.uploaded = true
	}

	// Optionally notify the server that it's done.
	if state.status.GSUploadURL != "" {
		end := tracer.Span(i, "finalize", "finalize", nil)
		defer func() { end(tracer.Args{"err": err}) }()
		// TODO(vadimsh): Calculate MD5 or CRC32C sum while uploading a file and
		// send it to isolated server. That way isolate server can verify that
		// the data safely reached Google Storage (GS provides MD5 and CRC32C of
		// stored files).
		in := isolated.FinalizeRequest{state.status.UploadTicket}
		if err = i.postJSON("/_ah/api/isolateservice/v1/finalize_gs_upload", in, nil); err != nil {
			return
		}
	}
	state.finalized = true
	return
}

func (i *isolateServer) doPush(state *PushState, src io.Reader) (err error) {
	end := tracer.Span(i, "push", strconv.FormatInt(state.size, 10), nil)
	defer func() { end(tracer.Args{"err": err}) }()
	reader, writer := io.Pipe()
	defer reader.Close()
	compressor := isolated.GetCompressor(writer)
	c := make(chan error)
	go func() {
		_, err2 := io.Copy(compressor, src)
		if err3 := compressor.Close(); err2 == nil {
			err2 = err3
		}
		_ = writer.Close()
		c <- err2
	}()
	defer func() {
		err4 := <-c
		if err == nil {
			err = err4
		}
	}()

	// DB upload.
	if state.status.GSUploadURL == "" {
		content, err2 := ioutil.ReadAll(reader)
		if err2 != nil {
			return err2
		}
		in := &isolated.StorageRequest{state.status.UploadTicket, content}
		return i.postJSON("/_ah/api/isolateservice/v1/store_inline", in, nil)
	}

	// Upload to GCS.
	client := &http.Client{}
	request, err5 := http.NewRequest("PUT", state.status.GSUploadURL, reader)
	if err5 != nil {
		return err5
	}
	request.Header.Set("Content-Type", "application/octet-stream")
	// TODO(maruel): For relatively small file, set request.ContentLength so the
	// TCP connection can be reused.
	resp, err6 := client.Do(request)
	if err6 != nil {
		return err6
	}
	_, err = io.Copy(ioutil.Discard, resp.Body)
	_ = resp.Body.Close()
	tracer.CounterAdd(i, "isolateserver", "upload", float64(state.size))
	return
}
<commit_msg>Remove incorrect comment and use DefaultClient.<commit_after>// Copyright 2015 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package isolatedclient

import (
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"

	"github.com/luci/luci-go/client/internal/lhttp"
	"github.com/luci/luci-go/client/internal/retry"
	"github.com/luci/luci-go/client/internal/tracer"
	"github.com/luci/luci-go/common/isolated"
)

// IsolateServer is the low-level client interface to interact with an Isolate
// server.
type IsolateServer interface {
	ServerCapabilities() (*isolated.ServerCapabilities, error)
	// Contains looks up cache presence on the server of multiple items.
	//
	// The returned list is in the same order as 'items', with entries nil for
	// items that were present.
	Contains(items []*isolated.DigestItem) ([]*PushState, error)
	Push(state *PushState, src io.Reader) error
}

// PushState is per-item state passed from IsolateServer.Contains() to
// IsolateServer.Push().
//
// Its content is implementation specific.
type PushState struct {
	status    isolated.PreuploadStatus
	size      int64
	uploaded  bool
	finalized bool
}

// New returns a new IsolateServer client.
func New(host, namespace string) IsolateServer {
	i := &isolateServer{
		url:       strings.TrimRight(host, "/"),
		namespace: namespace,
	}
	tracer.NewTID(i, nil, i.url)
	return i
}

// Private details.

type isolateServer struct {
	url       string
	namespace string
}

func (i *isolateServer) postJSON(resource string, in, out interface{}) error {
	if len(resource) == 0 || resource[0] != '/' {
		return errors.New("resource must start with '/'")
	}
	_, err := lhttp.PostJSON(retry.Default, http.DefaultClient, i.url+resource, in, out)
	return err
}

func (i *isolateServer) ServerCapabilities() (*isolated.ServerCapabilities, error) {
	out := &isolated.ServerCapabilities{}
	if err := i.postJSON("/_ah/api/isolateservice/v1/server_details", map[string]string{}, out); err != nil {
		return nil, err
	}
	return out, nil
}

func (i *isolateServer) Contains(items []*isolated.DigestItem) (out []*PushState, err error) {
	end := tracer.Span(i, "contains", strconv.Itoa(len(items)), nil)
	defer func() { end(tracer.Args{"err": err}) }()
	in := isolated.DigestCollection{Items: items}
	in.Namespace.Namespace = i.namespace
	data := &isolated.UrlCollection{}
	if err = i.postJSON("/_ah/api/isolateservice/v1/preupload", in, data); err != nil {
		return nil, err
	}
	out = make([]*PushState, len(items))
	for _, e := range data.Items {
		index := int(e.Index)
		out[index] = &PushState{
			status: e,
			size:   items[index].Size,
		}
	}
	return out, nil
}

func (i *isolateServer) Push(state *PushState, src io.Reader) (err error) {
	// This push operation may be a retry after failed finalization call below,
	// no need to reupload contents in that case.
	if !state.uploaded {
		// PUT file to uploadURL.
		if err = i.doPush(state, src); err != nil {
			return
		}
		state.uploaded = true
	}

	// Optionally notify the server that it's done.
	if state.status.GSUploadURL != "" {
		end := tracer.Span(i, "finalize", "finalize", nil)
		defer func() { end(tracer.Args{"err": err}) }()
		// TODO(vadimsh): Calculate MD5 or CRC32C sum while uploading a file and
		// send it to isolated server. That way isolate server can verify that
		// the data safely reached Google Storage (GS provides MD5 and CRC32C of
		// stored files).
		in := isolated.FinalizeRequest{state.status.UploadTicket}
		if err = i.postJSON("/_ah/api/isolateservice/v1/finalize_gs_upload", in, nil); err != nil {
			return
		}
	}
	state.finalized = true
	return
}

func (i *isolateServer) doPush(state *PushState, src io.Reader) (err error) {
	end := tracer.Span(i, "push", strconv.FormatInt(state.size, 10), nil)
	defer func() { end(tracer.Args{"err": err}) }()
	reader, writer := io.Pipe()
	defer reader.Close()
	compressor := isolated.GetCompressor(writer)
	c := make(chan error)
	go func() {
		_, err2 := io.Copy(compressor, src)
		if err3 := compressor.Close(); err2 == nil {
			err2 = err3
		}
		_ = writer.Close()
		c <- err2
	}()
	defer func() {
		err4 := <-c
		if err == nil {
			err = err4
		}
	}()

	// DB upload.
	if state.status.GSUploadURL == "" {
		content, err2 := ioutil.ReadAll(reader)
		if err2 != nil {
			return err2
		}
		in := &isolated.StorageRequest{state.status.UploadTicket, content}
		return i.postJSON("/_ah/api/isolateservice/v1/store_inline", in, nil)
	}

	// Upload to GCS.
	request, err5 := http.NewRequest("PUT", state.status.GSUploadURL, reader)
	if err5 != nil {
		return err5
	}
	request.Header.Set("Content-Type", "application/octet-stream")
	resp, err6 := http.DefaultClient.Do(request)
	if err6 != nil {
		return err6
	}
	_, err = io.Copy(ioutil.Discard, resp.Body)
	_ = resp.Body.Close()
	tracer.CounterAdd(i, "isolateserver", "upload", float64(state.size))
	return
}
<|endoftext|>


<commit_before>// TODO:
// Handle errors without quitting

package main

import (
	"bitmm/bitfinex"
	"fmt"
	// "github.com/davecgh/go-spew/spew"
	"log"
	"math"
	"os"
	"os/exec"
	"time"
)

// Trade inputs
const (
	SYMBOL    = "ltcusd" // Instrument to trade
	MINCHANGE = 0.00005  // Minumum change required to update prices
	TRADENUM  = 10       // Number of trades to use in calculations
	AMOUNT    = 1.00     // Size to trade
	BIDEDGE   = 0.01     // Required edge for a buy order
	ASKEDGE   = 0.01     // Required edge for a sell order
)

var api = bitfinex.New(os.Getenv("BITFINEX_KEY"), os.Getenv("BITFINEX_SECRET"))

func main() {
	fmt.Println("\nInitializing...")

	// Check for input to break loop
	inputChan := make(chan rune)
	go checkStdin(inputChan)

	// Run loop until user input is received
	runMainLoop(inputChan)
}

// Check for any user input
func checkStdin(inputChan chan<- rune) {
	var ch rune
	fmt.Scanf("%c", &ch)
	inputChan <- ch
}

// Infinite loop
func runMainLoop(inputChan <-chan rune) {
	// Exchange communication channels
	bookChan := make(chan bitfinex.Book)
	tradesChan := make(chan bitfinex.Trades)
	ordersChan := make(chan bitfinex.Orders)

	var (
		trades      bitfinex.Trades
		book        bitfinex.Book
		orders      bitfinex.Orders
		start       time.Time
		oldPosition float64
		newPosition float64
		oldTheo     float64
		newTheo     float64
		lastTrade   int
	)

	for {
		// Record time for each iteration
		start = time.Now()

		// Get data in separate goroutines
		go processTrades(tradesChan)
		go processBook(bookChan)

		// Possibly send orders when trades data returns
		trades = <-tradesChan
		if trades[0].Timestamp != lastTrade { // If new trades
			newTheo = calculateTheo(trades)
			newPosition = checkPosition()
		}
		go sendOrders(orders, oldTheo, newTheo, oldPosition, newPosition, ordersChan)

		lastTrade = trades[0].Timestamp
		oldTheo = newTheo
		oldPosition = newPosition

		// Print results when book and order data returns
		book = <-bookChan
		orders = <-ordersChan
		printResults(book, trades, orders, newTheo, newPosition, start)

		// Exit if anything entered by user
		select {
		case <-inputChan:
			exit()
			return
		default:
		}
	}
}

// Send orders to the exchange
func sendOrders(orders bitfinex.Orders, oldTheo, newTheo, oldPosition,
	newPosition float64, ordersChan chan<- bitfinex.Orders) {

	if math.Abs(oldTheo-newTheo) > MINCHANGE || math.Abs(oldPosition-
		newPosition) > 0.01 {
		// First cancel all orders
		cancelAll()

		var params []bitfinex.OrderParams

		if newPosition+AMOUNT < 0.01 { // Max short postion
			// One order at value to exit position
			params = []bitfinex.OrderParams{
				{SYMBOL, -newPosition, newTheo, "bitfinex", "buy", "limit"},
			}
		} else if newPosition-AMOUNT > -0.01 { // Max long postion
			// One order at value to exit position
			params = []bitfinex.OrderParams{
				{SYMBOL, newPosition, newTheo, "bitfinex", "sell", "limit"},
			}
		} else {
			// Two orders for edge
			params = []bitfinex.OrderParams{
				{SYMBOL, AMOUNT - newPosition, newTheo - BIDEDGE, "bitfinex", "buy", "limit"},
				{SYMBOL, AMOUNT + newPosition, newTheo + ASKEDGE, "bitfinex", "sell", "limit"},
			}
		}

		// Send new order request to the exchange
		orders, err := api.MultipleNewOrders(params)
		checkErr(err)
		ordersChan <- orders
	} else {
		ordersChan <- orders
	}
}

func checkPosition() float64 {
	position := 0.0
	posSlice, err := api.ActivePositions()
	checkErr(err)
	for _, pos := range posSlice {
		if pos.Symbol == SYMBOL {
			position = pos.Amount
		}
	}

	return position
}

// Get book data and send to channel
func processBook(bookChan chan<- bitfinex.Book) {
	book, err := api.Orderbook(SYMBOL, 5, 5)
	checkErr(err)

	bookChan <- book
}

// Get trade data and send to channel
func processTrades(tradesChan chan<- bitfinex.Trades) {
	trades, err := api.Trades(SYMBOL, TRADENUM)
	checkErr(err)

	tradesChan <- trades
}

// Calculate a volume-weighted moving average of trades
func calculateTheo(trades bitfinex.Trades) float64 {
	var sum1, sum2 float64
	for _, trade := range trades {
		sum1 += trade.Price * trade.Amount
		sum2 += trade.Amount
	}
	return sum1 / sum2
}

// Called on any error
func checkErr(err error) {
	if err != nil {
		exit()
		log.Fatal(err)
	}
}

// Call on exit
func exit() {
	cancelAll()
	fmt.Println("\nCancelled all orders.")
}

// Cancel all orders
func cancelAll() {
	cancelled := false
	for !cancelled {
		cancelled, _ = api.CancelAll()
	}
}

// Print results
func printResults(book bitfinex.Book, trades bitfinex.Trades,
	orders bitfinex.Orders, theo, position float64, start time.Time) {

	clearScreen()

	fmt.Println("----------------------------")
	fmt.Printf("%-10s%-10s%8s\n", " Bid", "  Ask", "Size ")
	fmt.Println("----------------------------")
	for i := range book.Asks {
		item := book.Asks[len(book.Asks)-1-i]
		fmt.Printf("%-10s%-10.4f%8.2f\n", "", item.Price, item.Amount)
	}
	for _, item := range book.Bids {
		fmt.Printf("%-10.4f%-10.2s%8.2f\n", item.Price, "", item.Amount)
	}
	fmt.Println("----------------------------")

	fmt.Println("\nLast Trades:")
	for _, trade := range trades {
		fmt.Printf("%-6.4f - size: %6.2f\n", trade.Price, trade.Amount)
	}

	fmt.Printf("\nPosition: %.2f\n", position)
	fmt.Printf("Theo:     %.4f\n", theo)

	fmt.Println("\nActive orders:")
	for _, order := range orders.Orders {
		fmt.Printf("%6.2f %s @ %6.4f\n", order.Amount, SYMBOL, order.Price)
	}

	fmt.Printf("\n%v processing time...", time.Since(start))
}

// Clear the terminal between prints
func clearScreen() {
	c := exec.Command("clear")
	c.Stdout = os.Stdout
	c.Run()
}
<commit_msg>handle errors without exit<commit_after>package main

import (
	"bitmm/bitfinex"
	"fmt"
	"math"
	"os"
	"os/exec"
	"time"
)

// Trade inputs
const (
	SYMBOL    = "ltcusd" // Instrument to trade
	MINCHANGE = 0.00005  // Minumum change required to update prices
	TRADENUM  = 10       // Number of trades to use in calculations
	AMOUNT    = 1.00     // Size to trade
	BIDEDGE   = 0.01     // Required edge for a buy order
	ASKEDGE   = 0.01     // Required edge for a sell order
)

var (
	api        = bitfinex.New(os.Getenv("BITFINEX_KEY"), os.Getenv("BITFINEX_SECRET"))
	apiErrors  = false
	liveOrders = false
)

func main() {
	fmt.Println("\nInitializing...")

	// Check for input to break loop
	inputChan := make(chan rune)
	go checkStdin(inputChan)

	// Run loop until user input is received
	runMainLoop(inputChan)
}

// Check for any user input
func checkStdin(inputChan chan<- rune) {
	var ch rune
	fmt.Scanf("%c", &ch)
	inputChan <- ch
}

// Infinite loop
func runMainLoop(inputChan <-chan rune) {
	// Exchange communication channels
	bookChan := make(chan bitfinex.Book)
	tradesChan := make(chan bitfinex.Trades)
	ordersChan := make(chan bitfinex.Orders)

	var (
		trades      bitfinex.Trades
		book        bitfinex.Book
		orders      bitfinex.Orders
		start       time.Time
		oldPosition float64
		newPosition float64
		oldTheo     float64
		newTheo     float64
		lastTrade   int
	)

	for {
		// Record time for each iteration
		start = time.Now()

		// Get data in separate goroutines
		go processTrades(tradesChan)
		go processBook(bookChan)

		// Possibly send orders when trades data returns
		trades = <-tradesChan
		if trades[0].Timestamp != lastTrade { // If new trades
			newTheo = calculateTheo(trades)
			newPosition = checkPosition()
		}
		go sendOrders(orders, oldTheo, newTheo, oldPosition, newPosition, ordersChan)

		lastTrade = trades[0].Timestamp
		oldTheo = newTheo
		oldPosition = newPosition

		// Print results when book and order data returns
		book = <-bookChan
		orders = <-ordersChan
		if !apiErrors {
			printResults(book, trades, orders, newTheo, newPosition, start)
		}
		apiErrors = false

		// Exit if anything entered by user
		select {
		case <-inputChan:
			exit()
			return
		default:
		}
	}
}

// Send orders to the exchange
func sendOrders(orders bitfinex.Orders, oldTheo, newTheo, oldPosition,
	newPosition float64, ordersChan chan<- bitfinex.Orders) {

	if (math.Abs(oldTheo-newTheo) > MINCHANGE || math.Abs(oldPosition-
		newPosition) > 0.01 || !liveOrders) && !apiErrors {
		// First cancel all orders
		if liveOrders {
			cancelAll()
		}

		var params []bitfinex.OrderParams

		if newPosition+AMOUNT < 0.01 { // Max short postion
			// One order at theo to exit position
			params = []bitfinex.OrderParams{
				{SYMBOL, -newPosition, newTheo, "bitfinex", "buy", "limit"},
			}
		} else if newPosition-AMOUNT > -0.01 { // Max long postion
			// One order at theo to exit position
			params = []bitfinex.OrderParams{
				{SYMBOL, newPosition, newTheo, "bitfinex", "sell", "limit"},
			}
		} else {
			// Two orders for edge
			params = []bitfinex.OrderParams{
				{SYMBOL, AMOUNT - newPosition, newTheo - BIDEDGE, "bitfinex", "buy", "limit"},
				{SYMBOL, AMOUNT + newPosition, newTheo + ASKEDGE, "bitfinex", "sell", "limit"},
			}
		}

		// Send new order request to the exchange
		orders, err := api.MultipleNewOrders(params)
		liveOrders = true
		checkErr(err)
		ordersChan <- orders
	} else {
		ordersChan <- orders
	}
}

func checkPosition() float64 {
	var position float64
	posSlice, err := api.ActivePositions()
	checkErr(err)
	for _, pos := range posSlice {
		if pos.Symbol == SYMBOL {
			position = pos.Amount
		}
	}

	return position
}

// Get book data and send to channel
func processBook(bookChan chan<- bitfinex.Book) {
	book, err := api.Orderbook(SYMBOL, 5, 5)
	checkErr(err)

	bookChan <- book
}

// Get trade data and send to channel
func processTrades(tradesChan chan<- bitfinex.Trades) {
	trades, err := api.Trades(SYMBOL, TRADENUM)
	checkErr(err)

	tradesChan <- trades
}

// Calculate a volume-weighted moving average of trades
func calculateTheo(trades bitfinex.Trades) float64 {
	var sum1, sum2 float64
	for _, trade := range trades {
		sum1 += trade.Price * trade.Amount
		sum2 += trade.Amount
	}
	return sum1 / sum2
}

// Called on any error
func checkErr(err error) {
	if err != nil {
		cancelAll()
		apiErrors = true
		fmt.Println(err)
	}
}

// Call on exit
func exit() {
	cancelAll()
	fmt.Println("\nCancelled all orders.")
}

// Cancel all orders
func cancelAll() {
	cancelled := false
	for !cancelled {
		cancelled, _ = api.CancelAll()
	}
	liveOrders = false
}

// Print results
func printResults(book bitfinex.Book, trades bitfinex.Trades,
	orders bitfinex.Orders, theo, position float64, start time.Time) {

	clearScreen()

	fmt.Println("----------------------------")
	fmt.Printf("%-10s%-10s%8s\n", " Bid", "  Ask", "Size ")
	fmt.Println("----------------------------")
	for i := range book.Asks {
		item := book.Asks[len(book.Asks)-1-i]
		fmt.Printf("%-10s%-10.4f%8.2f\n", "", item.Price, item.Amount)
	}
	for _, item := range book.Bids {
		fmt.Printf("%-10.4f%-10.2s%8.2f\n", item.Price, "", item.Amount)
	}
	fmt.Println("----------------------------")

	fmt.Println("\nLast Trades:")
	for _, trade := range trades {
		fmt.Printf("%-6.4f - size: %6.2f\n", trade.Price, trade.Amount)
	}

	fmt.Printf("\nPosition: %.2f\n", position)
	fmt.Printf("Theo:     %.4f\n", theo)

	fmt.Println("\nActive orders:")
	for _, order := range orders.Orders {
		fmt.Printf("%6.2f %s @ %6.4f\n", order.Amount, SYMBOL, order.Price)
	}

	fmt.Printf("\n%v processing time...", time.Since(start))
}

// Clear the terminal between prints
func clearScreen() {
	c := exec.Command("clear")
	c.Stdout = os.Stdout
	c.Run()
}
<|endoftext|>


<commit_before>package lz4

import (
	"encoding/binary"
	"errors"
)

var (
	// ErrInvalidSourceShortBuffer is returned by UncompressBlock or CompressBLock when a compressed
	// block is corrupted or the destination buffer is not large enough for the uncompressed data.
	ErrInvalidSourceShortBuffer = errors.New("lz4: invalid source or destination buffer too short")
	// ErrInvalid is returned when reading an invalid LZ4 archive.
	ErrInvalid = errors.New("lz4: bad magic number")
)

// blockHash hashes 4 bytes into a value < winSize.
func blockHash(x uint32) uint32 {
	const hasher uint32 = 2654435761 // Knuth multiplicative hash.
	return x * hasher >> hashShift
}

// CompressBlockBound returns the maximum size of a given buffer of size n, when not compressible.
func CompressBlockBound(n int) int {
	return n + n/255 + 16
}

// UncompressBlock uncompresses the source buffer into the destination one,
// and returns the uncompressed size.
//
// The destination buffer must be sized appropriately.
//
// An error is returned if the source data is invalid or the destination buffer is too small.
func UncompressBlock(src, dst []byte) (di int, err error) {
	sn := len(src)
	if sn == 0 {
		return 0, nil
	}

	di = decodeBlock(dst, src)
	switch di {
	case -1:
		return 0, errors.New("TODO: bad data")
	case -2:
		// this is short dst really
		return 0, ErrInvalidSourceShortBuffer
	}
	if di < 0 {
		return 0, ErrInvalidSourceShortBuffer
	}
	return di, nil
}

// CompressBlock compresses the source buffer into the destination one.
// This is the fast version of LZ4 compression and also the default one.
// The size of hashTable must be at least 64Kb.
//
// The size of the compressed data is returned. If it is 0 and no error, then the data is incompressible.
//
// An error is returned if the destination buffer is too small.
func CompressBlock(src, dst []byte, hashTable []int) (di int, err error) {
	defer func() {
		if recover() != nil {
			err = ErrInvalidSourceShortBuffer
		}
	}()

	sn, dn := len(src)-mfLimit, len(dst)
	if sn <= 0 || dn == 0 {
		return 0, nil
	}
	var si int

	// Fast scan strategy: the hash table only stores the last 4 bytes sequences.
	// const accInit = 1 << skipStrength

	anchor := si // Position of the current literals.
	// acc := accInit // Variable step: improves performance on non-compressible data.

	for si < sn {
		// Hash the next 4 bytes (sequence)...
		match := binary.LittleEndian.Uint32(src[si:])
		h := blockHash(match)

		ref := hashTable[h]
		hashTable[h] = si
		if ref >= sn { // Invalid reference (dirty hashtable).
			si++
			continue
		}
		offset := si - ref
		if offset <= 0 || offset >= winSize || // Out of window.
			match != binary.LittleEndian.Uint32(src[ref:]) { // Hash collision on different matches.
			// si += acc >> skipStrength
			// acc++
			si++
			continue
		}

		// Match found.
		// acc = accInit
		lLen := si - anchor // Literal length.

		// Encode match length part 1.
		si += minMatch
		mLen := si // Match length has minMatch already.
		// Find the longest match, first looking by batches of 8 bytes.
		for si < sn && binary.LittleEndian.Uint64(src[si:]) == binary.LittleEndian.Uint64(src[si-offset:]) {
			si += 8
		}
		// Then byte by byte.
		for si < sn && src[si] == src[si-offset] {
			si++
		}

		mLen = si - mLen
		if mLen < 0xF {
			dst[di] = byte(mLen)
		} else {
			dst[di] = 0xF
		}

		// Encode literals length.
		if lLen < 0xF {
			dst[di] |= byte(lLen << 4)
		} else {
			dst[di] |= 0xF0
			di++
			l := lLen - 0xF
			for ; l >= 0xFF; l -= 0xFF {
				dst[di] = 0xFF
				di++
			}
			dst[di] = byte(l)
		}
		di++

		// Literals.
		copy(dst[di:di+lLen], src[anchor:anchor+lLen])
		di += lLen + 2
		anchor = si

		// Encode offset.
		_ = dst[di] // Bound check elimination.
		dst[di-2], dst[di-1] = byte(offset), byte(offset>>8)

		// Encode match length part 2.
		if mLen >= 0xF {
			for mLen -= 0xF; mLen >= 0xFF; mLen -= 0xFF {
				dst[di] = 0xFF
				di++
			}
			dst[di] = byte(mLen)
			di++
		}
	}

	if anchor == 0 {
		// Incompressible.
		return 0, nil
	}

	// Last literals.
	lLen := len(src) - anchor
	if lLen < 0xF {
		dst[di] = byte(lLen << 4)
	} else {
		dst[di] = 0xF0
		di++
		for lLen -= 0xF; lLen >= 0xFF; lLen -= 0xFF {
			dst[di] = 0xFF
			di++
		}
		dst[di] = byte(lLen)
	}
	di++

	// Write the last literals.
	if di >= anchor {
		// Incompressible.
		return 0, nil
	}
	di += copy(dst[di:di+len(src)-anchor], src[anchor:])
	return di, nil
}

// CompressBlockHC compresses the source buffer src into the destination dst
// with max search depth (use 0 or negative value for no max).
//
// CompressBlockHC compression ratio is better than CompressBlock but it is also slower.
//
// The size of the compressed data is returned. If it is 0 and no error, then the data is not compressible.
//
// An error is returned if the destination buffer is too small.
func CompressBlockHC(src, dst []byte, depth int) (di int, err error) {
	defer func() {
		if recover() != nil {
			err = ErrInvalidSourceShortBuffer
		}
	}()

	sn, dn := len(src)-mfLimit, len(dst)
	if sn <= 0 || dn == 0 {
		return 0, nil
	}
	var si int

	// hashTable: stores the last position found for a given hash
	// chaingTable: stores previous positions for a given hash
	var hashTable, chainTable [winSize]int

	if depth <= 0 {
		depth = winSize
	}

	anchor := si
	for si < sn {
		// Hash the next 4 bytes (sequence).
		match := binary.LittleEndian.Uint32(src[si:])
		h := blockHash(match)

		// Follow the chain until out of window and give the longest match.
		mLen := 0
		offset := 0
		for next, try := hashTable[h], depth; try > 0 && next > 0 && si-next < winSize; next = chainTable[next&winMask] {
			// The first (mLen==0) or next byte (mLen>=minMatch) at current match length
			// must match to improve on the match length.
			if src[next+mLen] != src[si+mLen] {
				continue
			}
			ml := 0
			// Compare the current position with a previous with the same hash.
			for ml < sn-si && binary.LittleEndian.Uint64(src[next+ml:]) == binary.LittleEndian.Uint64(src[si+ml:]) {
				ml += 8
			}
			for ml < sn-si && src[next+ml] == src[si+ml] {
				ml++
			}
			if ml < minMatch || ml <= mLen {
				// Match too small (<minMath) or smaller than the current match.
				continue
			}
			// Found a longer match, keep its position and length.
			mLen = ml
			offset = si - next
			// Try another previous position with the same hash.
			try--
		}
		chainTable[si&winMask] = hashTable[h]
		hashTable[h] = si

		// No match found.
		if mLen == 0 {
			si++
			continue
		}

		// Match found.
		// Update hash/chain tables with overlapping bytes:
		// si already hashed, add everything from si+1 up to the match length.
		winStart := si + 1
		if ws := si + mLen - winSize; ws > winStart {
			winStart = ws
		}
		for si, ml := winStart, si+mLen; si < ml; {
			match >>= 8
			match |= uint32(src[si+3]) << 24
			h := blockHash(match)
			chainTable[si&winMask] = hashTable[h]
			hashTable[h] = si
			si++
		}

		lLen := si - anchor
		si += mLen
		mLen -= minMatch // Match length does not include minMatch.

		if mLen < 0xF {
			dst[di] = byte(mLen)
		} else {
			dst[di] = 0xF
		}

		// Encode literals length.
		if lLen < 0xF {
			dst[di] |= byte(lLen << 4)
		} else {
			dst[di] |= 0xF0
			di++
			l := lLen - 0xF
			for ; l >= 0xFF; l -= 0xFF {
				dst[di] = 0xFF
				di++
			}
			dst[di] = byte(l)
		}
		di++

		// Literals.
		copy(dst[di:di+lLen], src[anchor:anchor+lLen])
		di += lLen
		anchor = si

		// Encode offset.
		di += 2
		dst[di-2], dst[di-1] = byte(offset), byte(offset>>8)

		// Encode match length part 2.
		if mLen >= 0xF {
			for mLen -= 0xF; mLen >= 0xFF; mLen -= 0xFF {
				dst[di] = 0xFF
				di++
			}
			dst[di] = byte(mLen)
			di++
		}
	}

	if anchor == 0 {
		// Incompressible.
		return 0, nil
	}

	// Last literals.
	lLen := len(src) - anchor
	if lLen < 0xF {
		dst[di] = byte(lLen << 4)
	} else {
		dst[di] = 0xF0
		di++
		lLen -= 0xF
		for ; lLen >= 0xFF; lLen -= 0xFF {
			dst[di] = 0xFF
			di++
		}
		dst[di] = byte(lLen)
	}
	di++

	// Write the last literals.
	if di >= anchor {
		// Incompressible.
		return 0, nil
	}
	di += copy(dst[di:di+len(src)-anchor], src[anchor:])
	return di, nil
}
<commit_msg>dont distinguish error types<commit_after>package lz4

import (
	"encoding/binary"
	"errors"
)

var (
	// ErrInvalidSourceShortBuffer is returned by UncompressBlock or CompressBLock when a compressed
	// block is corrupted or the destination buffer is not large enough for the uncompressed data.
	ErrInvalidSourceShortBuffer = errors.New("lz4: invalid source or destination buffer too short")
	// ErrInvalid is returned when reading an invalid LZ4 archive.
	ErrInvalid = errors.New("lz4: bad magic number")
)

// blockHash hashes 4 bytes into a value < winSize.
func blockHash(x uint32) uint32 {
	const hasher uint32 = 2654435761 // Knuth multiplicative hash.
	return x * hasher >> hashShift
}

// CompressBlockBound returns the maximum size of a given buffer of size n, when not compressible.
func CompressBlockBound(n int) int {
	return n + n/255 + 16
}

// UncompressBlock uncompresses the source buffer into the destination one,
// and returns the uncompressed size.
//
// The destination buffer must be sized appropriately.
//
// An error is returned if the source data is invalid or the destination buffer is too small.
func UncompressBlock(src, dst []byte) (di int, err error) {
	sn := len(src)
	if sn == 0 {
		return 0, nil
	}

	di = decodeBlock(dst, src)
	if di < 0 {
		return 0, ErrInvalidSourceShortBuffer
	}
	return di, nil
}

// CompressBlock compresses the source buffer into the destination one.
// This is the fast version of LZ4 compression and also the default one.
// The size of hashTable must be at least 64Kb.
//
// The size of the compressed data is returned. If it is 0 and no error, then the data is incompressible.
//
// An error is returned if the destination buffer is too small.
func CompressBlock(src, dst []byte, hashTable []int) (di int, err error) {
	defer func() {
		if recover() != nil {
			err = ErrInvalidSourceShortBuffer
		}
	}()

	sn, dn := len(src)-mfLimit, len(dst)
	if sn <= 0 || dn == 0 {
		return 0, nil
	}
	var si int

	// Fast scan strategy: the hash table only stores the last 4 bytes sequences.
	// const accInit = 1 << skipStrength

	anchor := si // Position of the current literals.
	// acc := accInit // Variable step: improves performance on non-compressible data.

	for si < sn {
		// Hash the next 4 bytes (sequence)...
		match := binary.LittleEndian.Uint32(src[si:])
		h := blockHash(match)

		ref := hashTable[h]
		hashTable[h] = si
		if ref >= sn { // Invalid reference (dirty hashtable).
			si++
			continue
		}
		offset := si - ref
		if offset <= 0 || offset >= winSize || // Out of window.
			match != binary.LittleEndian.Uint32(src[ref:]) { // Hash collision on different matches.
			// si += acc >> skipStrength
			// acc++
			si++
			continue
		}

		// Match found.
		// acc = accInit
		lLen := si - anchor // Literal length.

		// Encode match length part 1.
		si += minMatch
		mLen := si // Match length has minMatch already.
		// Find the longest match, first looking by batches of 8 bytes.
		for si < sn && binary.LittleEndian.Uint64(src[si:]) == binary.LittleEndian.Uint64(src[si-offset:]) {
			si += 8
		}
		// Then byte by byte.
		for si < sn && src[si] == src[si-offset] {
			si++
		}

		mLen = si - mLen
		if mLen < 0xF {
			dst[di] = byte(mLen)
		} else {
			dst[di] = 0xF
		}

		// Encode literals length.
		if lLen < 0xF {
			dst[di] |= byte(lLen << 4)
		} else {
			dst[di] |= 0xF0
			di++
			l := lLen - 0xF
			for ; l >= 0xFF; l -= 0xFF {
				dst[di] = 0xFF
				di++
			}
			dst[di] = byte(l)
		}
		di++

		// Literals.
		copy(dst[di:di+lLen], src[anchor:anchor+lLen])
		di += lLen + 2
		anchor = si

		// Encode offset.
		_ = dst[di] // Bound check elimination.
		dst[di-2], dst[di-1] = byte(offset), byte(offset>>8)

		// Encode match length part 2.
		if mLen >= 0xF {
			for mLen -= 0xF; mLen >= 0xFF; mLen -= 0xFF {
				dst[di] = 0xFF
				di++
			}
			dst[di] = byte(mLen)
			di++
		}
	}

	if anchor == 0 {
		// Incompressible.
		return 0, nil
	}

	// Last literals.
	lLen := len(src) - anchor
	if lLen < 0xF {
		dst[di] = byte(lLen << 4)
	} else {
		dst[di] = 0xF0
		di++
		for lLen -= 0xF; lLen >= 0xFF; lLen -= 0xFF {
			dst[di] = 0xFF
			di++
		}
		dst[di] = byte(lLen)
	}
	di++

	// Write the last literals.
	if di >= anchor {
		// Incompressible.
		return 0, nil
	}
	di += copy(dst[di:di+len(src)-anchor], src[anchor:])
	return di, nil
}

// CompressBlockHC compresses the source buffer src into the destination dst
// with max search depth (use 0 or negative value for no max).
//
// CompressBlockHC compression ratio is better than CompressBlock but it is also slower.
//
// The size of the compressed data is returned. If it is 0 and no error, then the data is not compressible.
//
// An error is returned if the destination buffer is too small.
func CompressBlockHC(src, dst []byte, depth int) (di int, err error) {
	defer func() {
		if recover() != nil {
			err = ErrInvalidSourceShortBuffer
		}
	}()

	sn, dn := len(src)-mfLimit, len(dst)
	if sn <= 0 || dn == 0 {
		return 0, nil
	}
	var si int

	// hashTable: stores the last position found for a given hash
	// chaingTable: stores previous positions for a given hash
	var hashTable, chainTable [winSize]int

	if depth <= 0 {
		depth = winSize
	}

	anchor := si
	for si < sn {
		// Hash the next 4 bytes (sequence).
		match := binary.LittleEndian.Uint32(src[si:])
		h := blockHash(match)

		// Follow the chain until out of window and give the longest match.
		mLen := 0
		offset := 0
		for next, try := hashTable[h], depth; try > 0 && next > 0 && si-next < winSize; next = chainTable[next&winMask] {
			// The first (mLen==0) or next byte (mLen>=minMatch) at current match length
			// must match to improve on the match length.
			if src[next+mLen] != src[si+mLen] {
				continue
			}
			ml := 0
			// Compare the current position with a previous with the same hash.
			for ml < sn-si && binary.LittleEndian.Uint64(src[next+ml:]) == binary.LittleEndian.Uint64(src[si+ml:]) {
				ml += 8
			}
			for ml < sn-si && src[next+ml] == src[si+ml] {
				ml++
			}
			if ml < minMatch || ml <= mLen {
				// Match too small (<minMath) or smaller than the current match.
				continue
			}
			// Found a longer match, keep its position and length.
			mLen = ml
			offset = si - next
			// Try another previous position with the same hash.
			try--
		}
		chainTable[si&winMask] = hashTable[h]
		hashTable[h] = si

		// No match found.
		if mLen == 0 {
			si++
			continue
		}

		// Match found.
		// Update hash/chain tables with overlapping bytes:
		// si already hashed, add everything from si+1 up to the match length.
		winStart := si + 1
		if ws := si + mLen - winSize; ws > winStart {
			winStart = ws
		}
		for si, ml := winStart, si+mLen; si < ml; {
			match >>= 8
			match |= uint32(src[si+3]) << 24
			h := blockHash(match)
			chainTable[si&winMask] = hashTable[h]
			hashTable[h] = si
			si++
		}

		lLen := si - anchor
		si += mLen
		mLen -= minMatch // Match length does not include minMatch.

		if mLen < 0xF {
			dst[di] = byte(mLen)
		} else {
			dst[di] = 0xF
		}

		// Encode literals length.
		if lLen < 0xF {
			dst[di] |= byte(lLen << 4)
		} else {
			dst[di] |= 0xF0
			di++
			l := lLen - 0xF
			for ; l >= 0xFF; l -= 0xFF {
				dst[di] = 0xFF
				di++
			}
			dst[di] = byte(l)
		}
		di++

		// Literals.
		copy(dst[di:di+lLen], src[anchor:anchor+lLen])
		di += lLen
		anchor = si

		// Encode offset.
		di += 2
		dst[di-2], dst[di-1] = byte(offset), byte(offset>>8)

		// Encode match length part 2.
		if mLen >= 0xF {
			for mLen -= 0xF; mLen >= 0xFF; mLen -= 0xFF {
				dst[di] = 0xFF
				di++
			}
			dst[di] = byte(mLen)
			di++
		}
	}

	if anchor == 0 {
		// Incompressible.
		return 0, nil
	}

	// Last literals.
	lLen := len(src) - anchor
	if lLen < 0xF {
		dst[di] = byte(lLen << 4)
	} else {
		dst[di] = 0xF0
		di++
		lLen -= 0xF
		for ; lLen >= 0xFF; lLen -= 0xFF {
			dst[di] = 0xFF
			di++
		}
		dst[di] = byte(lLen)
	}
	di++

	// Write the last literals.
	if di >= anchor {
		// Incompressible.
		return 0, nil
	}
	di += copy(dst[di:di+len(src)-anchor], src[anchor:])
	return di, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package util

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"v.io/x/devtools/internal/envutil"
	"v.io/x/devtools/internal/tool"
)

const (
	rootEnv          = "V23_ROOT"
	metadataDirName  = ".v23"
	metadataFileName = "metadata.v2"
)

// LocalManifestFile returns the path to the local manifest.
func LocalManifestFile() (string, error) {
	root, err := V23Root()
	if err != nil {
		return "", err
	}
	return filepath.Join(root, ".local_manifest"), nil
}

// LocalSnapshotDir returns the path to the local snapshot directory.
func LocalSnapshotDir() (string, error) {
	root, err := V23Root()
	if err != nil {
		return "", err
	}
	return filepath.Join(root, ".snapshot"), nil
}

// ManifestDir returns the path to the manifest directory.
func ManifestDir() (string, error) {
	root, err := V23Root()
	if err != nil {
		return "", err
	}
	return filepath.Join(root, ".manifest", "v2"), nil
}

// ManifestFile returns the path to the manifest file with the given
// relative path.
func ManifestFile(name string) (string, error) {
	dir, err := ManifestDir()
	if err != nil {
		return "", err
	}
	return filepath.Join(dir, name), nil
}

// RemoteSnapshotDir returns the path to the remote snapshot directory.
func RemoteSnapshotDir() (string, error) {
	manifestDir, err := ManifestDir()
	if err != nil {
		return "", err
	}
	return filepath.Join(manifestDir, "snapshot"), nil
}

// ResolveManifestPath resolves the given manifest name to an absolute
// path in the local filesystem.
func ResolveManifestPath(name string) (string, error) {
	if name != "" {
		if filepath.IsAbs(name) {
			return name, nil
		}
		return ManifestFile(name)
	}
	path, err := LocalManifestFile()
	if err != nil {
		return "", err
	}
	if _, err := os.Stat(path); err != nil {
		if os.IsNotExist(err) {
			return ResolveManifestPath("default")
		}
		return "", fmt.Errorf("Stat(%v) failed: %v", err)
	}
	return path, nil
}

// DataDirPath returns the path to the data directory of the given tool.
func DataDirPath(ctx *tool.Context, toolName string) (string, error) {
	projects, tools, err := readManifest(ctx, false)
	if err != nil {
		return "", err
	}
	if toolName == "" {
		// If the tool name is not set, use "v23" as the default. As a
		// consequence, any manifest is assumed to specify a "v23" tool.
		toolName = "v23"
	}
	tool, ok := tools[toolName]
	if !ok {
		return "", fmt.Errorf("tool %q not found in the manifest", tool.Name)
	}
	projectName := tool.Project
	project, ok := projects[projectName]
	if !ok {
		return "", fmt.Errorf("project %q not found in the manifest", projectName)
	}
	return filepath.Join(project.Path, tool.Data), nil
}

// LoadConfig loads the tools configuration file into memory.
func LoadConfig(ctx *tool.Context) (*Config, error) {
	dataDir, err := DataDirPath(ctx, tool.Name)
	if err != nil {
		return nil, err
	}
	configPath := filepath.Join(dataDir, "conf.json")
	configBytes, err := ioutil.ReadFile(configPath)
	if err != nil {
		return nil, fmt.Errorf("ReadFile(%v) failed: %v", configPath, err)
	}
	var config Config
	if err := json.Unmarshal(configBytes, &config); err != nil {
		return nil, fmt.Errorf("Unmarshal(%v) failed: %v", string(configBytes), err)
	}
	return &config, nil
}

// VanadiumEnvironment returns the environment variables setting for
// vanadium. The util package captures the original state of the
// relevant environment variables when the tool is initialized, and
// every invocation of this function updates this original state
// according to the current config of the v23 tool.
func VanadiumEnvironment(ctx *tool.Context, platform Platform) (*envutil.Snapshot, error) {
	env := envutil.NewSnapshotFromOS()
	root, err := V23Root()
	if err != nil {
		return nil, err
	}
	config, err := LoadConfig(ctx)
	if err != nil {
		return nil, err
	}
	if err := setGoPath(ctx, env, root, config); err != nil {
		return nil, err
	}
	if err := setVdlPath(ctx, env, root, config); err != nil {
		return nil, err
	}
	if platform.OS == "darwin" || platform.OS == "linux" {
		if err := setSyncbaseCgoEnv(env, root, platform.OS); err != nil {
			return nil, err
		}
	}
	switch {
	case platform.Arch == runtime.GOARCH && platform.OS == runtime.GOOS:
		// If setting up the environment for the host, we are done.
	case platform.Arch == "arm" && platform.OS == "linux":
		// Set up cross-compilation for arm / linux.
		if err := setArmEnv(env, platform); err != nil {
			return nil, err
		}
	case platform.Arch == "arm" && platform.OS == "android":
		// Set up cross-compilation for arm / android.
		if err := setAndroidEnv(env, platform); err != nil {
			return nil, err
		}
	case (platform.Arch == "386" || platform.Arch == "amd64p32") && platform.OS == "nacl":
		// Set up cross-compilation nacl.
		if err := setNaclEnv(env, platform); err != nil {
			return nil, err
		}
	default:
		return nil, UnsupportedPlatformErr{platform}
	}
	return env, nil
}

// VanadiumGitRepoHost returns the URL that hosts vanadium git
// repositories.
func VanadiumGitRepoHost() string {
	return "https://vanadium.googlesource.com/"
}

// V23Root returns the root of the vanadium universe.
func V23Root() (string, error) {
	root := os.Getenv(rootEnv)
	if root == "" {
		return "", fmt.Errorf("%v is not set", rootEnv)
	}
	result, err := filepath.EvalSymlinks(root)
	if err != nil {
		return "", fmt.Errorf("EvalSymlinks(%v) failed: %v", root)
	}
	return result, nil
}

// setAndroidEnv sets the environment variables used for android
// cross-compilation.
func setAndroidEnv(env *envutil.Snapshot, platform Platform) error {
	root, err := V23Root()
	if err != nil {
		return err
	}
	// Set Go specific environment variables.
	env.Set("CGO_ENABLED", "1")
	env.Set("GOOS", platform.OS)
	env.Set("GOARCH", platform.Arch)
	env.Set("GOARM", strings.TrimPrefix(platform.SubArch, "v"))

	// Add the paths to vanadium cross-compilation tools to the PATH.
	path := env.GetTokens("PATH", ":")
	path = append([]string{
		filepath.Join(root, "third_party", "android", "go", "bin"),
	}, path...)
	env.SetTokens("PATH", path, ":")
	return nil
}

// setArmEnv sets the environment variables used for android
// cross-compilation.
func setArmEnv(env *envutil.Snapshot, platform Platform) error {
	root, err := V23Root()
	if err != nil {
		return err
	}
	// Set Go specific environment variables.
	env.Set("GOARCH", platform.Arch)
	env.Set("GOARM", strings.TrimPrefix(platform.SubArch, "v"))
	env.Set("GOOS", platform.OS)

	// Add the paths to vanadium cross-compilation tools to the PATH.
	path := env.GetTokens("PATH", ":")
	path = append([]string{
		filepath.Join(root, "third_party", "cout", "xgcc", "cross_arm"),
		filepath.Join(root, "third_party", "repos", "go_arm", "bin"),
	}, path...)
	env.SetTokens("PATH", path, ":")
	return nil
}

// setGoPath adds the paths to vanadium Go workspaces to the GOPATH
// variable.
func setGoPath(ctx *tool.Context, env *envutil.Snapshot, root string, config *Config) error {
	return setPathHelper(ctx, env, "GOPATH", root, config.GoWorkspaces())
}

// setVdlPath adds the paths to vanadium VDL workspaces to the VDLPATH
// variable.
func setVdlPath(ctx *tool.Context, env *envutil.Snapshot, root string, config *Config) error {
	return setPathHelper(ctx, env, "VDLPATH", root, config.VDLWorkspaces())
}

// setPathHelper is a utility function for setting path environment
// variables for different types of workspaces.
func setPathHelper(ctx *tool.Context, env *envutil.Snapshot, name, root string, workspaces []string) error {
	path := env.GetTokens(name, ":")
	projects, _, err := readManifest(ctx, false)
	if err != nil {
		return err
	}
	for _, workspace := range workspaces {
		absWorkspace := filepath.Join(root, workspace)
		// Only append an entry to the path if the workspace is rooted
		// under a v23 project that exists locally or vice versa.
		for _, project := range projects {
			// We check if <project.Path> is a prefix of <absWorkspace> to
			// account for Go workspaces nested under a single v23 project,
			// such as: $V23_ROOT/release/projects/chat/go.
			//
			// We check if <absWorkspace> is a prefix of <project.Path> to
			// account for Go workspaces that span multiple v23 projects,
			// such as: $V23_ROOT/release/go.
			if strings.HasPrefix(absWorkspace, project.Path) || strings.HasPrefix(project.Path, absWorkspace) {
				if _, err := os.Stat(filepath.Join(project.Path)); err == nil {
					path = append(path, absWorkspace)
					break
				}
			}
		}
	}
	env.SetTokens(name, path, ":")
	return nil
}

// setNaclEnv sets the environment variables used for nacl
// cross-compilation.
func setNaclEnv(env *envutil.Snapshot, platform Platform) error {
	env.Set("GOARCH", platform.Arch)
	env.Set("GOOS", platform.OS)
	return nil
}

// setSyncbaseCgoEnv sets the CGO_ENABLED variable and adds the LevelDB
// third-party C++ libraries vanadium Go code depends on to the CGO_CFLAGS and
// CGO_LDFLAGS variables.
func setSyncbaseCgoEnv(env *envutil.Snapshot, root, arch string) error {
	// Set the CGO_* variables for the vanadium syncbase component.
	env.Set("CGO_ENABLED", "1")
	cflags := env.GetTokens("CGO_CFLAGS", " ")
	ldflags := env.GetTokens("CGO_LDFLAGS", " ")
	dir := filepath.Join(root, "third_party", "cout", "leveldb")
	if _, err := os.Stat(dir); err != nil {
		if !os.IsNotExist(err) {
			return fmt.Errorf("Stat(%v) failed: %v", dir, err)
		}
	} else {
		cflags = append(cflags, filepath.Join("-I"+dir, "include"))
		ldflags = append(ldflags, filepath.Join("-L"+dir, "lib"))
		if arch == "linux" {
			ldflags = append(ldflags, "-Wl,-rpath", filepath.Join(dir, "lib"))
		}
	}
	env.SetTokens("CGO_CFLAGS", cflags, " ")
	env.SetTokens("CGO_LDFLAGS", ldflags, " ")
	return nil
}

// BuildCopRotationPath returns the path to the build cop rotation file.
func BuildCopRotationPath(ctx *tool.Context) (string, error) {
	dataDir, err := DataDirPath(ctx, tool.Name)
	if err != nil {
		return "", err
	}
	return filepath.Join(dataDir, "buildcop.xml"), nil
}
<commit_msg>devtools: Fix setPathHelper bug in checking for valid workspaces.<commit_after>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package util

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"v.io/x/devtools/internal/envutil"
	"v.io/x/devtools/internal/tool"
)

const (
	rootEnv          = "V23_ROOT"
	metadataDirName  = ".v23"
	metadataFileName = "metadata.v2"
)

// LocalManifestFile returns the path to the local manifest.
func LocalManifestFile() (string, error) {
	root, err := V23Root()
	if err != nil {
		return "", err
	}
	return filepath.Join(root, ".local_manifest"), nil
}

// LocalSnapshotDir returns the path to the local snapshot directory.
func LocalSnapshotDir() (string, error) {
	root, err := V23Root()
	if err != nil {
		return "", err
	}
	return filepath.Join(root, ".snapshot"), nil
}

// ManifestDir returns the path to the manifest directory.
func ManifestDir() (string, error) {
	root, err := V23Root()
	if err != nil {
		return "", err
	}
	return filepath.Join(root, ".manifest", "v2"), nil
}

// ManifestFile returns the path to the manifest file with the given
// relative path.
func ManifestFile(name string) (string, error) {
	dir, err := ManifestDir()
	if err != nil {
		return "", err
	}
	return filepath.Join(dir, name), nil
}

// RemoteSnapshotDir returns the path to the remote snapshot directory.
func RemoteSnapshotDir() (string, error) {
	manifestDir, err := ManifestDir()
	if err != nil {
		return "", err
	}
	return filepath.Join(manifestDir, "snapshot"), nil
}

// ResolveManifestPath resolves the given manifest name to an absolute
// path in the local filesystem.
func ResolveManifestPath(name string) (string, error) {
	if name != "" {
		if filepath.IsAbs(name) {
			return name, nil
		}
		return ManifestFile(name)
	}
	path, err := LocalManifestFile()
	if err != nil {
		return "", err
	}
	if _, err := os.Stat(path); err != nil {
		if os.IsNotExist(err) {
			return ResolveManifestPath("default")
		}
		return "", fmt.Errorf("Stat(%v) failed: %v", err)
	}
	return path, nil
}

// DataDirPath returns the path to the data directory of the given tool.
func DataDirPath(ctx *tool.Context, toolName string) (string, error) {
	projects, tools, err := readManifest(ctx, false)
	if err != nil {
		return "", err
	}
	if toolName == "" {
		// If the tool name is not set, use "v23" as the default. As a
		// consequence, any manifest is assumed to specify a "v23" tool.
		toolName = "v23"
	}
	tool, ok := tools[toolName]
	if !ok {
		return "", fmt.Errorf("tool %q not found in the manifest", tool.Name)
	}
	projectName := tool.Project
	project, ok := projects[projectName]
	if !ok {
		return "", fmt.Errorf("project %q not found in the manifest", projectName)
	}
	return filepath.Join(project.Path, tool.Data), nil
}

// LoadConfig loads the tools configuration file into memory.
func LoadConfig(ctx *tool.Context) (*Config, error) {
	dataDir, err := DataDirPath(ctx, tool.Name)
	if err != nil {
		return nil, err
	}
	configPath := filepath.Join(dataDir, "conf.json")
	configBytes, err := ioutil.ReadFile(configPath)
	if err != nil {
		return nil, fmt.Errorf("ReadFile(%v) failed: %v", configPath, err)
	}
	var config Config
	if err := json.Unmarshal(configBytes, &config); err != nil {
		return nil, fmt.Errorf("Unmarshal(%v) failed: %v", string(configBytes), err)
	}
	return &config, nil
}

// VanadiumEnvironment returns the environment variables setting for
// vanadium. The util package captures the original state of the
// relevant environment variables when the tool is initialized, and
// every invocation of this function updates this original state
// according to the current config of the v23 tool.
func VanadiumEnvironment(ctx *tool.Context, platform Platform) (*envutil.Snapshot, error) {
	env := envutil.NewSnapshotFromOS()
	root, err := V23Root()
	if err != nil {
		return nil, err
	}
	config, err := LoadConfig(ctx)
	if err != nil {
		return nil, err
	}
	if err := setGoPath(ctx, env, root, config); err != nil {
		return nil, err
	}
	if err := setVdlPath(ctx, env, root, config); err != nil {
		return nil, err
	}
	if platform.OS == "darwin" || platform.OS == "linux" {
		if err := setSyncbaseCgoEnv(env, root, platform.OS); err != nil {
			return nil, err
		}
	}
	switch {
	case platform.Arch == runtime.GOARCH && platform.OS == runtime.GOOS:
		// If setting up the environment for the host, we are done.
	case platform.Arch == "arm" && platform.OS == "linux":
		// Set up cross-compilation for arm / linux.
		if err := setArmEnv(env, platform); err != nil {
			return nil, err
		}
	case platform.Arch == "arm" && platform.OS == "android":
		// Set up cross-compilation for arm / android.
		if err := setAndroidEnv(env, platform); err != nil {
			return nil, err
		}
	case (platform.Arch == "386" || platform.Arch == "amd64p32") && platform.OS == "nacl":
		// Set up cross-compilation nacl.
		if err := setNaclEnv(env, platform); err != nil {
			return nil, err
		}
	default:
		return nil, UnsupportedPlatformErr{platform}
	}
	return env, nil
}

// VanadiumGitRepoHost returns the URL that hosts vanadium git
// repositories.
func VanadiumGitRepoHost() string {
	return "https://vanadium.googlesource.com/"
}

// V23Root returns the root of the vanadium universe.
func V23Root() (string, error) {
	root := os.Getenv(rootEnv)
	if root == "" {
		return "", fmt.Errorf("%v is not set", rootEnv)
	}
	result, err := filepath.EvalSymlinks(root)
	if err != nil {
		return "", fmt.Errorf("EvalSymlinks(%v) failed: %v", root)
	}
	return result, nil
}

// setAndroidEnv sets the environment variables used for android
// cross-compilation.
func setAndroidEnv(env *envutil.Snapshot, platform Platform) error {
	root, err := V23Root()
	if err != nil {
		return err
	}
	// Set Go specific environment variables.
	env.Set("CGO_ENABLED", "1")
	env.Set("GOOS", platform.OS)
	env.Set("GOARCH", platform.Arch)
	env.Set("GOARM", strings.TrimPrefix(platform.SubArch, "v"))

	// Add the paths to vanadium cross-compilation tools to the PATH.
	path := env.GetTokens("PATH", ":")
	path = append([]string{
		filepath.Join(root, "third_party", "android", "go", "bin"),
	}, path...)
	env.SetTokens("PATH", path, ":")
	return nil
}

// setArmEnv sets the environment variables used for android
// cross-compilation.
func setArmEnv(env *envutil.Snapshot, platform Platform) error {
	root, err := V23Root()
	if err != nil {
		return err
	}
	// Set Go specific environment variables.
	env.Set("GOARCH", platform.Arch)
	env.Set("GOARM", strings.TrimPrefix(platform.SubArch, "v"))
	env.Set("GOOS", platform.OS)

	// Add the paths to vanadium cross-compilation tools to the PATH.
	path := env.GetTokens("PATH", ":")
	path = append([]string{
		filepath.Join(root, "third_party", "cout", "xgcc", "cross_arm"),
		filepath.Join(root, "third_party", "repos", "go_arm", "bin"),
	}, path...)
	env.SetTokens("PATH", path, ":")
	return nil
}

// setGoPath adds the paths to vanadium Go workspaces to the GOPATH
// variable.
func setGoPath(ctx *tool.Context, env *envutil.Snapshot, root string, config *Config) error {
	return setPathHelper(ctx, env, "GOPATH", root, config.GoWorkspaces())
}

// setVdlPath adds the paths to vanadium VDL workspaces to the VDLPATH
// variable.
func setVdlPath(ctx *tool.Context, env *envutil.Snapshot, root string, config *Config) error {
	return setPathHelper(ctx, env, "VDLPATH", root, config.VDLWorkspaces())
}

// setPathHelper is a utility function for setting path environment
// variables for different types of workspaces.
func setPathHelper(ctx *tool.Context, env *envutil.Snapshot, name, root string, workspaces []string) error {
	path := env.GetTokens(name, ":")
	projects, _, err := readManifest(ctx, false)
	if err != nil {
		return err
	}
	for _, workspace := range workspaces {
		absWorkspace := filepath.Join(root, workspace)
		// Only append an entry to the path if the workspace is rooted
		// under a v23 project that exists locally or vice versa.
		for _, project := range projects {
			// We check if <project.Path> is a prefix of <absWorkspace> to
			// account for Go workspaces nested under a single v23 project,
			// such as: $V23_ROOT/release/projects/chat/go.
			//
			// We check if <absWorkspace> is a prefix of <project.Path> to
			// account for Go workspaces that span multiple v23 projects,
			// such as: $V23_ROOT/release/go.
			if strings.HasPrefix(absWorkspace, project.Path) || strings.HasPrefix(project.Path, absWorkspace) {
				if _, err := os.Stat(filepath.Join(absWorkspace)); err == nil {
					path = append(path, absWorkspace)
					break
				}
			}
		}
	}
	env.SetTokens(name, path, ":")
	return nil
}

// setNaclEnv sets the environment variables used for nacl
// cross-compilation.
func setNaclEnv(env *envutil.Snapshot, platform Platform) error {
	env.Set("GOARCH", platform.Arch)
	env.Set("GOOS", platform.OS)
	return nil
}

// setSyncbaseCgoEnv sets the CGO_ENABLED variable and adds the LevelDB
// third-party C++ libraries vanadium Go code depends on to the CGO_CFLAGS and
// CGO_LDFLAGS variables.
func setSyncbaseCgoEnv(env *envutil.Snapshot, root, arch string) error {
	// Set the CGO_* variables for the vanadium syncbase component.
	env.Set("CGO_ENABLED", "1")
	cflags := env.GetTokens("CGO_CFLAGS", " ")
	ldflags := env.GetTokens("CGO_LDFLAGS", " ")
	dir := filepath.Join(root, "third_party", "cout", "leveldb")
	if _, err := os.Stat(dir); err != nil {
		if !os.IsNotExist(err) {
			return fmt.Errorf("Stat(%v) failed: %v", dir, err)
		}
	} else {
		cflags = append(cflags, filepath.Join("-I"+dir, "include"))
		ldflags = append(ldflags, filepath.Join("-L"+dir, "lib"))
		if arch == "linux" {
			ldflags = append(ldflags, "-Wl,-rpath", filepath.Join(dir, "lib"))
		}
	}
	env.SetTokens("CGO_CFLAGS", cflags, " ")
	env.SetTokens("CGO_LDFLAGS", ldflags, " ")
	return nil
}

// BuildCopRotationPath returns the path to the build cop rotation file.
func BuildCopRotationPath(ctx *tool.Context) (string, error) {
	dataDir, err := DataDirPath(ctx, tool.Name)
	if err != nil {
		return "", err
	}
	return filepath.Join(dataDir, "buildcop.xml"), nil
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package simplepush

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"sync"
	"time"
)

type PropPinger interface {
	HasConfigStruct
	Register(uaid string, pingData []byte) error
	Send(uaid string, vers int64) (ok bool, err error)
	CanBypassWebsocket() bool
	Status() (bool, error)
}

var UnsupportedProtocolErr = errors.New("Unsupported Ping Request")
var ConfigurationErr = errors.New("Configuration Error")
var ProtocolErr = errors.New("A protocol error occurred. See logs for details.")

var AvailablePings = make(AvailableExtensions)

func init() {
	AvailablePings["noop"] = func() HasConfigStruct { return new(NoopPing) }
	AvailablePings["udp"] = func() HasConfigStruct { return new(UDPPing) }
	AvailablePings["gcm"] = func() HasConfigStruct { return new(GCMPing) }
	AvailablePings["default"] = AvailablePings["noop"]
}

// NoOp ping

type NoopPing struct {
	app    *Application
	config *NoopPingConfig
}

type NoopPingConfig struct{}

func (ml *NoopPing) ConfigStruct() interface{} {
	return &NoopPingConfig{}
}

// Generic configuration for an Ping
func (r *NoopPing) Init(app *Application, config interface{}) (err error) {
	conf := config.(*NoopPingConfig)
	r.config = conf
	return nil
}

// Register the ping to a user
func (r *NoopPing) Register(string, []byte) error {
	return nil
}

// Can the ping bypass telling the device on the websocket?
func (r *NoopPing) CanBypassWebsocket() bool {
	return false
}

// try to send the ping.
func (r *NoopPing) Send(string, int64) (bool, error) {
	return false, UnsupportedProtocolErr
}

func (r *NoopPing) Status() (bool, error) {
	return true, nil
}

//===
// "UDP" ping uses remote Carrier provided URL to establish a UDP
// based "ping" to the device. This UDP ping is contained within
// the carrier's network.
type UDPPing struct {
	config *UDPPingConfig
	app    *Application
}

type UDPPingConfig struct {
	URL string `toml:"url" env:"url"` //carrier UDP Proxy URL
	// Additional Carrier required elements here.
}

func (ml *UDPPing) ConfigStruct() interface{} {
	return &UDPPingConfig{
		URL: "https://example.com",
	}
}

func (r *UDPPing) Init(app *Application, config interface{}) error {
	r.app = app
	r.config = config.(*UDPPingConfig)
	return nil
}

func (r *UDPPing) Register(uaid string, pingData []byte) (err error) {
	if err = r.app.Store().PutPing(uaid, pingData); err != nil {
		r.app.Logger().Error("propping", "Could not store connect",
			LogFields{"error": err.Error()})
	}
	return nil
}

func (r *UDPPing) CanBypassWebsocket() bool {
	// If the Ping does not require communication to the client via
	// websocket, return true. If the ping should still attempt to
	// try using the client's websocket connection, return false.
	return false
}

// Send the version info to the Proprietary ping URL provided
// by the carrier.
func (r *UDPPing) Send(string, int64) (bool, error) {
	// Obviously, this needs to be filled out with the appropriate
	// setup and calls to communicate to the remote server.
	// Since UDP is not actually defined, we're returning this
	// error.
	return false, UnsupportedProtocolErr
}

func (r *UDPPing) Status() (bool, error) {
	return false, UnsupportedProtocolErr
}

// ===
// Google Cloud Messaging Proprietary Ping interface
// NOTE: This is still experimental.
type GCMPing struct {
	logger      *SimpleLogger
	metrics     *Metrics
	store       Store
	client      *http.Client
	url         string
	collapseKey string
	dryRun      bool
	apiKey      string
	ttl         uint64
	lastErr     error
	errLock     sync.RWMutex
}

type GCMPingConfig struct {
	APIKey      string `toml:"api_key" env:"api_key"` //GCM Dev API Key
	CollapseKey string `toml:"collapse_key" env:"collapse_key"`
	DryRun      bool   `toml:"dry_run" env:"dry_run"`
	TTL         string `toml:"ttl" env:"ttl"`
	URL         string `toml:"url" env:"url"` //GCM URL
}

type GCMRequest struct {
	Regs        [1]string `json:"registration_ids"`
	CollapseKey string    `json:"collapse_key"`
	TTL         uint64    `json:"time_to_live"`
	DryRun      bool      `json:"dry_run"`
}

type GCMPingData struct {
	RegID string `json:"regid"`
}

type GCMError int

func (err GCMError) Error() string {
	switch err {
	case 400:
		return "Invalid request payload"
	case 401:
		return "Error authenticating sender account"
	default:
		return fmt.Sprintf("Unexpected status code: %d", err)
	}
}

func (r *GCMPing) ConfigStruct() interface{} {
	return &GCMPingConfig{
		URL:         "https://android.googleapis.com/gcm/send",
		APIKey:      "YOUR_API_KEY",
		CollapseKey: "simplepush",
		DryRun:      false,
		TTL:         "72h",
	}
}

func (r *GCMPing) Init(app *Application, config interface{}) error {
	r.logger = app.Logger()
	r.metrics = app.Metrics()
	r.store = app.Store()
	conf := config.(*GCMPingConfig)

	r.url = conf.URL
	r.collapseKey = conf.CollapseKey
	r.dryRun = conf.DryRun

	if r.apiKey = conf.APIKey; len(r.apiKey) == 0 {
		r.logger.Alert("gcmping", "Missing GCM API key", nil)
		return ConfigurationErr
	}

	ttl, err := time.ParseDuration(conf.TTL)
	if err != nil {
		r.logger.Alert("gcmping", "Could not parse TTL",
			LogFields{"error": err.Error(), "ttl": conf.TTL})
		return err
	}
	r.ttl = uint64(ttl / time.Second)

	r.client = new(http.Client)
	return nil
}

func (r *GCMPing) CanBypassWebsocket() bool {
	// GCM can work even if the client's websocket connection
	// has timed out or closed. We do not need to try to send the
	// message on both channels.
	return true
}

func (r *GCMPing) Register(uaid string, pingData []byte) (err error) {
	ping := new(GCMPingData)
	if err = json.Unmarshal(pingData, ping); err != nil {
		return err
	}
	if len(ping.RegID) == 0 {
		r.logger.Error("gcmping",
			"No user registration ID present. Cannot send message",
			nil)
		return ConfigurationErr
	}
	request := &GCMRequest{
		// google docs lie. You MUST send the regid as an array, even if it's one
		// element.
		Regs:        [1]string{ping.RegID},
		CollapseKey: r.collapseKey,
		TTL:         r.ttl,
		DryRun:      r.dryRun,
	}
	requestData, err := json.Marshal(request)
	if err != nil {
		if r.logger.ShouldLog(ERROR) {
			r.logger.Error("gcmping", "Could not marshal connection string for storage",
				LogFields{"error": err.Error()})
		}
		return err
	}
	if err = r.store.PutPing(uaid, requestData); err != nil {
		if r.logger.ShouldLog(WARNING) {
			r.logger.Warn("gcmping", "Could not store connect",
				LogFields{"error": err.Error()})
		}
		return err
	}
	return nil
}

func (r *GCMPing) Send(uaid string, vers int64) (ok bool, err error) {
	pingData, err := r.store.FetchPing(uaid)
	if err != nil {
		return false, err
	}
	if len(pingData) == 0 {
		return false, nil
	}
	req, err := http.NewRequest("POST", r.url, bytes.NewBuffer(pingData))
	if err != nil {
		if r.logger.ShouldLog(ERROR) {
			r.logger.Error("propping",
				"Could not create request for GCM Post",
				LogFields{"error": err.Error()})
		}
		return false, err
	}
	req.Header.Add("Authorization", "key="+r.apiKey)
	req.Header.Add("Content-Type", "application/json")
	r.metrics.Increment("propretary.ping.gcm")
	resp, err := r.client.Do(req)
	defer resp.Body.Close()
	if err != nil {
		if r.logger.ShouldLog(ERROR) {
			r.logger.Error("propping",
				"Failed to send GCM message",
				LogFields{"error": err.Error()})
		}
		return false, err
	}
	if resp.StatusCode < 200 || resp.StatusCode > 299 {
		if r.logger.ShouldLog(WARNING) {
			r.logger.Warn("propping",
				"GCM returned non success message",
				LogFields{"error": resp.Status})
		}
		r.setLastErr(GCMError(resp.StatusCode))
		return false, ProtocolErr
	}
	return true, nil
}

func (r *GCMPing) Status() (ok bool, err error) {
	if err = r.getLastErr(); err != nil {
		return false, err
	}
	return true, nil
}

func (r *GCMPing) setLastErr(err error) {
	r.errLock.Lock()
	r.lastErr = err
	r.errLock.Unlock()
}

func (r *GCMPing) getLastErr() (err error) {
	r.errLock.RLock()
	err = r.lastErr
	r.errLock.RUnlock()
	return
}
<commit_msg>`NoopPing`: Remove `UnsupportedProtocolErr`.<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package simplepush

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"sync"
	"time"
)

type PropPinger interface {
	HasConfigStruct
	Register(uaid string, pingData []byte) error
	Send(uaid string, vers int64) (ok bool, err error)
	CanBypassWebsocket() bool
	Status() (bool, error)
}

var UnsupportedProtocolErr = errors.New("Unsupported Ping Request")
var ConfigurationErr = errors.New("Configuration Error")
var ProtocolErr = errors.New("A protocol error occurred. See logs for details.")

var AvailablePings = make(AvailableExtensions)

func init() {
	AvailablePings["noop"] = func() HasConfigStruct { return new(NoopPing) }
	AvailablePings["udp"] = func() HasConfigStruct { return new(UDPPing) }
	AvailablePings["gcm"] = func() HasConfigStruct { return new(GCMPing) }
	AvailablePings["default"] = AvailablePings["noop"]
}

// NoOp ping

type NoopPing struct {
	app    *Application
	config *NoopPingConfig
}

type NoopPingConfig struct{}

func (ml *NoopPing) ConfigStruct() interface{} {
	return &NoopPingConfig{}
}

// Generic configuration for an Ping
func (r *NoopPing) Init(app *Application, config interface{}) (err error) {
	conf := config.(*NoopPingConfig)
	r.config = conf
	return nil
}

// Register the ping to a user
func (r *NoopPing) Register(string, []byte) error {
	return nil
}

// Can the ping bypass telling the device on the websocket?
func (r *NoopPing) CanBypassWebsocket() bool {
	return false
}

// try to send the ping.
func (r *NoopPing) Send(string, int64) (bool, error) {
	return false, nil
}

func (r *NoopPing) Status() (bool, error) {
	return true, nil
}

//===
// "UDP" ping uses remote Carrier provided URL to establish a UDP
// based "ping" to the device. This UDP ping is contained within
// the carrier's network.
type UDPPing struct {
	config *UDPPingConfig
	app    *Application
}

type UDPPingConfig struct {
	URL string `toml:"url" env:"url"` //carrier UDP Proxy URL
	// Additional Carrier required elements here.
}

func (ml *UDPPing) ConfigStruct() interface{} {
	return &UDPPingConfig{
		URL: "https://example.com",
	}
}

func (r *UDPPing) Init(app *Application, config interface{}) error {
	r.app = app
	r.config = config.(*UDPPingConfig)
	return nil
}

func (r *UDPPing) Register(uaid string, pingData []byte) (err error) {
	if err = r.app.Store().PutPing(uaid, pingData); err != nil {
		r.app.Logger().Error("propping", "Could not store connect",
			LogFields{"error": err.Error()})
	}
	return nil
}

func (r *UDPPing) CanBypassWebsocket() bool {
	// If the Ping does not require communication to the client via
	// websocket, return true. If the ping should still attempt to
	// try using the client's websocket connection, return false.
	return false
}

// Send the version info to the Proprietary ping URL provided
// by the carrier.
func (r *UDPPing) Send(string, int64) (bool, error) {
	// Obviously, this needs to be filled out with the appropriate
	// setup and calls to communicate to the remote server.
	// Since UDP is not actually defined, we're returning this
	// error.
	return false, UnsupportedProtocolErr
}

func (r *UDPPing) Status() (bool, error) {
	return false, UnsupportedProtocolErr
}

// ===
// Google Cloud Messaging Proprietary Ping interface
// NOTE: This is still experimental.
type GCMPing struct {
	logger      *SimpleLogger
	metrics     *Metrics
	store       Store
	client      *http.Client
	url         string
	collapseKey string
	dryRun      bool
	apiKey      string
	ttl         uint64
	lastErr     error
	errLock     sync.RWMutex
}

type GCMPingConfig struct {
	APIKey      string `toml:"api_key" env:"api_key"` //GCM Dev API Key
	CollapseKey string `toml:"collapse_key" env:"collapse_key"`
	DryRun      bool   `toml:"dry_run" env:"dry_run"`
	TTL         string `toml:"ttl" env:"ttl"`
	URL         string `toml:"url" env:"url"` //GCM URL
}

type GCMRequest struct {
	Regs        [1]string `json:"registration_ids"`
	CollapseKey string    `json:"collapse_key"`
	TTL         uint64    `json:"time_to_live"`
	DryRun      bool      `json:"dry_run"`
}

type GCMPingData struct {
	RegID string `json:"regid"`
}

type GCMError int

func (err GCMError) Error() string {
	switch err {
	case 400:
		return "Invalid request payload"
	case 401:
		return "Error authenticating sender account"
	default:
		return fmt.Sprintf("Unexpected status code: %d", err)
	}
}

func (r *GCMPing) ConfigStruct() interface{} {
	return &GCMPingConfig{
		URL:         "https://android.googleapis.com/gcm/send",
		APIKey:      "YOUR_API_KEY",
		CollapseKey: "simplepush",
		DryRun:      false,
		TTL:         "72h",
	}
}

func (r *GCMPing) Init(app *Application, config interface{}) error {
	r.logger = app.Logger()
	r.metrics = app.Metrics()
	r.store = app.Store()
	conf := config.(*GCMPingConfig)

	r.url = conf.URL
	r.collapseKey = conf.CollapseKey
	r.dryRun = conf.DryRun

	if r.apiKey = conf.APIKey; len(r.apiKey) == 0 {
		r.logger.Alert("gcmping", "Missing GCM API key", nil)
		return ConfigurationErr
	}

	ttl, err := time.ParseDuration(conf.TTL)
	if err != nil {
		r.logger.Alert("gcmping", "Could not parse TTL",
			LogFields{"error": err.Error(), "ttl": conf.TTL})
		return err
	}
	r.ttl = uint64(ttl / time.Second)

	r.client = new(http.Client)
	return nil
}

func (r *GCMPing) CanBypassWebsocket() bool {
	// GCM can work even if the client's websocket connection
	// has timed out or closed. We do not need to try to send the
	// message on both channels.
	return true
}

func (r *GCMPing) Register(uaid string, pingData []byte) (err error) {
	ping := new(GCMPingData)
	if err = json.Unmarshal(pingData, ping); err != nil {
		return err
	}
	if len(ping.RegID) == 0 {
		r.logger.Error("gcmping",
			"No user registration ID present. Cannot send message",
			nil)
		return ConfigurationErr
	}
	request := &GCMRequest{
		// google docs lie. You MUST send the regid as an array, even if it's one
		// element.
		Regs:        [1]string{ping.RegID},
		CollapseKey: r.collapseKey,
		TTL:         r.ttl,
		DryRun:      r.dryRun,
	}
	requestData, err := json.Marshal(request)
	if err != nil {
		if r.logger.ShouldLog(ERROR) {
			r.logger.Error("gcmping", "Could not marshal connection string for storage",
				LogFields{"error": err.Error()})
		}
		return err
	}
	if err = r.store.PutPing(uaid, requestData); err != nil {
		if r.logger.ShouldLog(WARNING) {
			r.logger.Warn("gcmping", "Could not store connect",
				LogFields{"error": err.Error()})
		}
		return err
	}
	return nil
}

func (r *GCMPing) Send(uaid string, vers int64) (ok bool, err error) {
	pingData, err := r.store.FetchPing(uaid)
	if err != nil {
		return false, err
	}
	if len(pingData) == 0 {
		return false, nil
	}
	req, err := http.NewRequest("POST", r.url, bytes.NewBuffer(pingData))
	if err != nil {
		if r.logger.ShouldLog(ERROR) {
			r.logger.Error("propping",
				"Could not create request for GCM Post",
				LogFields{"error": err.Error()})
		}
		return false, err
	}
	req.Header.Add("Authorization", "key="+r.apiKey)
	req.Header.Add("Content-Type", "application/json")
	r.metrics.Increment("propretary.ping.gcm")
	resp, err := r.client.Do(req)
	defer resp.Body.Close()
	if err != nil {
		if r.logger.ShouldLog(ERROR) {
			r.logger.Error("propping",
				"Failed to send GCM message",
				LogFields{"error": err.Error()})
		}
		return false, err
	}
	if resp.StatusCode < 200 || resp.StatusCode > 299 {
		if r.logger.ShouldLog(WARNING) {
			r.logger.Warn("propping",
				"GCM returned non success message",
				LogFields{"error": resp.Status})
		}
		r.setLastErr(GCMError(resp.StatusCode))
		return false, ProtocolErr
	}
	return true, nil
}

func (r *GCMPing) Status() (ok bool, err error) {
	if err = r.getLastErr(); err != nil {
		return false, err
	}
	return true, nil
}

func (r *GCMPing) setLastErr(err error) {
	r.errLock.Lock()
	r.lastErr = err
	r.errLock.Unlock()
}

func (r *GCMPing) getLastErr() (err error) {
	r.errLock.RLock()
	err = r.lastErr
	r.errLock.RUnlock()
	return
}
<|endoftext|>


<commit_before>package auth

import (
	"fmt"
	"os"

	"github.com/micro/cli/v2"
	"github.com/micro/go-micro/v2"
	"github.com/micro/go-micro/v2/auth"
	srvAuth "github.com/micro/go-micro/v2/auth/service"
	pb "github.com/micro/go-micro/v2/auth/service/proto"
	"github.com/micro/go-micro/v2/config/cmd"
	log "github.com/micro/go-micro/v2/logger"
	"github.com/micro/go-micro/v2/util/config"
	"github.com/micro/micro/v2/auth/api"
	"github.com/micro/micro/v2/auth/handler"
	"github.com/micro/micro/v2/auth/web"
)

var (
	// Name of the service
	Name = "go.micro.auth"
	// Address of the service
	Address = ":8010"
)

// run the auth service
func run(ctx *cli.Context, srvOpts ...micro.Option) {
	log.Init(log.WithFields(map[string]interface{}{"service": "auth"}))

	// Init plugins
	for _, p := range Plugins() {
		p.Init(ctx)
	}

	if len(ctx.String("address")) > 0 {
		Address = ctx.String("address")
	}
	if len(Address) > 0 {
		srvOpts = append(srvOpts, micro.Address(Address))
	}

	// Init plugins
	for _, p := range Plugins() {
		p.Init(ctx)
	}

	// setup service
	srvOpts = append(srvOpts, micro.Name(Name))
	service := micro.NewService(srvOpts...)

	// run service
	pb.RegisterAuthHandler(service.Server(), handler.New())
	if err := service.Run(); err != nil {
		log.Fatal(err)
	}
}

func authFromContext(ctx *cli.Context) auth.Auth {
	if ctx.Bool("platform") {
		os.Setenv("MICRO_PROXY", "service")
		os.Setenv("MICRO_PROXY_ADDRESS", "proxy.micro.mu:443")
		return srvAuth.NewAuth()
	}

	return *cmd.DefaultCmd.Options().Auth
}

// login using a token
func login(ctx *cli.Context) {
	if ctx.Args().Len() != 1 {
		fmt.Println("Usage: `micro login [token]`")
		os.Exit(1)
	}
	token := ctx.Args().First()

	// Execute the request
	acc, err := authFromContext(ctx).Inspect(token)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	} else if acc == nil {
		fmt.Printf("[%v] did not generate an account\n", authFromContext(ctx).String())
		os.Exit(1)
	}

	// Store the token in micro config
	if err := config.Set("token", token); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	// Inform the user
	fmt.Println("You have been logged in")
}

func Commands(srvOpts ...micro.Option) []*cli.Command {
	commands := []*cli.Command{
		&cli.Command{
			Name:  "auth",
			Usage: "Run the auth service",
			Action: func(ctx *cli.Context) error {
				run(ctx)
				return nil
			},
			Subcommands: append([]*cli.Command{
				{
					Name:        "api",
					Usage:       "Run the auth api",
					Description: "Run the auth api",
					Action: func(ctx *cli.Context) error {
						api.Run(ctx, srvOpts...)
						return nil
					},
				},
				{
					Name:        "web",
					Usage:       "Run the auth web",
					Description: "Run the auth web",
					Action: func(ctx *cli.Context) error {
						web.Run(ctx, srvOpts...)
						return nil
					},
				},
			}),
			Flags: []cli.Flag{
				&cli.StringFlag{
					Name:    "address",
					Usage:   "Set the auth http address e.g 0.0.0.0:8010",
					EnvVars: []string{"MICRO_SERVER_ADDRESS"},
				},
				&cli.StringFlag{
					Name:    "auth_provider",
					EnvVars: []string{"MICRO_AUTH_PROVIDER"},
					Usage:   "Auth provider enables account generation",
				},
			},
		},
		&cli.Command{
			Name:  "login",
			Usage: "Login using a token",
			Action: func(ctx *cli.Context) error {
				login(ctx)
				return nil
			},
			Flags: []cli.Flag{
				&cli.BoolFlag{
					Name:  "platform",
					Usage: "Connect to the platform",
					Value: false,
				},
			},
		},
	}

	for _, c := range commands {
		for _, p := range Plugins() {
			if cmds := p.Commands(); len(cmds) > 0 {
				c.Subcommands = append(c.Subcommands, cmds...)
			}

			if flags := p.Flags(); len(flags) > 0 {
				c.Flags = append(c.Flags, flags...)
			}
		}
	}

	return commands
}
<commit_msg>Auth fixes (#680)<commit_after>package auth

import (
	"fmt"
	"os"

	"github.com/micro/cli/v2"
	"github.com/micro/go-micro/v2"
	"github.com/micro/go-micro/v2/auth"
	srvAuth "github.com/micro/go-micro/v2/auth/service"
	pb "github.com/micro/go-micro/v2/auth/service/proto"
	"github.com/micro/go-micro/v2/config/cmd"
	log "github.com/micro/go-micro/v2/logger"
	"github.com/micro/go-micro/v2/store"
	"github.com/micro/go-micro/v2/util/config"
	"github.com/micro/micro/v2/auth/api"
	"github.com/micro/micro/v2/auth/handler"
	"github.com/micro/micro/v2/auth/web"
)

var (
	// Name of the service
	Name = "go.micro.auth"
	// Address of the service
	Address = ":8010"
)

// run the auth service
func run(ctx *cli.Context, srvOpts ...micro.Option) {
	log.Init(log.WithFields(map[string]interface{}{"service": "auth"}))

	// Init plugins
	for _, p := range Plugins() {
		p.Init(ctx)
	}

	if len(ctx.String("address")) > 0 {
		Address = ctx.String("address")
	}
	if len(Address) > 0 {
		srvOpts = append(srvOpts, micro.Address(Address))
	}

	// Init plugins
	for _, p := range Plugins() {
		p.Init(ctx)
	}

	// set store namespace
	store.DefaultStore.Init(store.Namespace(Name))

	// setup service
	srvOpts = append(srvOpts, micro.Name(Name))
	service := micro.NewService(srvOpts...)

	// run service
	pb.RegisterAuthHandler(service.Server(), handler.New())
	if err := service.Run(); err != nil {
		log.Fatal(err)
	}
}

func authFromContext(ctx *cli.Context) auth.Auth {
	if ctx.Bool("platform") {
		os.Setenv("MICRO_PROXY", "service")
		os.Setenv("MICRO_PROXY_ADDRESS", "proxy.micro.mu:443")
		return srvAuth.NewAuth()
	}

	return *cmd.DefaultCmd.Options().Auth
}

// login using a token
func login(ctx *cli.Context) {
	if ctx.Args().Len() != 1 {
		fmt.Println("Usage: `micro login [token]`")
		os.Exit(1)
	}
	token := ctx.Args().First()

	// Execute the request
	acc, err := authFromContext(ctx).Inspect(token)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	} else if acc == nil {
		fmt.Printf("[%v] did not generate an account\n", authFromContext(ctx).String())
		os.Exit(1)
	}

	// Store the token in micro config
	if err := config.Set("token", token); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	// Inform the user
	fmt.Println("You have been logged in")
}

func Commands(srvOpts ...micro.Option) []*cli.Command {
	commands := []*cli.Command{
		&cli.Command{
			Name:  "auth",
			Usage: "Run the auth service",
			Action: func(ctx *cli.Context) error {
				run(ctx)
				return nil
			},
			Subcommands: append([]*cli.Command{
				{
					Name:        "api",
					Usage:       "Run the auth api",
					Description: "Run the auth api",
					Action: func(ctx *cli.Context) error {
						api.Run(ctx, srvOpts...)
						return nil
					},
				},
				{
					Name:        "web",
					Usage:       "Run the auth web",
					Description: "Run the auth web",
					Action: func(ctx *cli.Context) error {
						web.Run(ctx, srvOpts...)
						return nil
					},
				},
			}),
			Flags: []cli.Flag{
				&cli.StringFlag{
					Name:    "address",
					Usage:   "Set the auth http address e.g 0.0.0.0:8010",
					EnvVars: []string{"MICRO_SERVER_ADDRESS"},
				},
				&cli.StringFlag{
					Name:    "auth_provider",
					EnvVars: []string{"MICRO_AUTH_PROVIDER"},
					Usage:   "Auth provider enables account generation",
				},
			},
		},
		&cli.Command{
			Name:  "login",
			Usage: "Login using a token",
			Action: func(ctx *cli.Context) error {
				login(ctx)
				return nil
			},
			Flags: []cli.Flag{
				&cli.BoolFlag{
					Name:  "platform",
					Usage: "Connect to the platform",
					Value: false,
				},
			},
		},
	}

	for _, c := range commands {
		for _, p := range Plugins() {
			if cmds := p.Commands(); len(cmds) > 0 {
				c.Subcommands = append(c.Subcommands, cmds...)
			}

			if flags := p.Flags(); len(flags) > 0 {
				c.Flags = append(c.Flags, flags...)
			}
		}
	}

	return commands
}
<|endoftext|>


<commit_before>package carbon

import (
	"bytes"
	"errors"
	"fmt"
	"net"
	"sync"
	"time"

	"github.com/grafana/metrictank/cmd/mt-fakemetrics/out"
	"github.com/grafana/metrictank/schema"
	"github.com/raintank/met"
)

var errClosed = errors.New("output is closed")

type Carbon struct {
	sync.Mutex
	out.OutStats
	addr   string
	conn   net.Conn
	closed bool
}

func New(addr string, stats met.Backend) (*Carbon, error) {
	conn, err := net.Dial("tcp", addr)
	if err != nil {
		return nil, err
	}

	return &Carbon{
		sync.Mutex{},
		out.NewStats(stats, "carbon"),
		addr,
		conn,
		false,
	}, nil
}

func (n *Carbon) Close() error {
	n.Lock()
	if n.closed {
		n.Unlock()
		return nil
	}
	err := n.conn.Close()
	n.closed = true
	n.Unlock()
	return err
}

func (n *Carbon) Flush(metrics []*schema.MetricData) error {
	if len(metrics) == 0 {
		n.FlushDuration.Value(0)
		return nil
	}
	n.Lock()
	if n.closed {
		return errClosed
	}
	preFlush := time.Now()
	buf := bytes.NewBufferString("")
	for _, m := range metrics {
		buf.WriteString(fmt.Sprintf("%s %f %d\n", m.Name, m.Value, m.Time))
	}
	prePub := time.Now()
	_, err := n.conn.Write(buf.Bytes())
	n.Unlock()
	if err != nil {
		n.PublishErrors.Inc(1)
		return err
	}
	n.MessageBytes.Value(int64(buf.Len()))
	n.MessageMetrics.Value(int64(len(metrics)))
	n.PublishedMetrics.Inc(int64(len(metrics)))
	n.PublishedMessages.Inc(1)
	n.PublishDuration.Value(time.Since(prePub))
	n.FlushDuration.Value(time.Since(preFlush))
	return nil
}
<commit_msg>Tags support for Carbon output<commit_after>package carbon

import (
	"bytes"
	"errors"
	"fmt"
	"net"
	"sync"
	"time"

	"github.com/grafana/metrictank/cmd/mt-fakemetrics/out"
	"github.com/grafana/metrictank/schema"
	"github.com/raintank/met"
)

var errClosed = errors.New("output is closed")

type Carbon struct {
	sync.Mutex
	out.OutStats
	addr   string
	conn   net.Conn
	closed bool
}

func New(addr string, stats met.Backend) (*Carbon, error) {
	conn, err := net.Dial("tcp", addr)
	if err != nil {
		return nil, err
	}

	return &Carbon{
		sync.Mutex{},
		out.NewStats(stats, "carbon"),
		addr,
		conn,
		false,
	}, nil
}

func (n *Carbon) Close() error {
	n.Lock()
	if n.closed {
		n.Unlock()
		return nil
	}
	err := n.conn.Close()
	n.closed = true
	n.Unlock()
	return err
}

func (n *Carbon) Flush(metrics []*schema.MetricData) error {
	if len(metrics) == 0 {
		n.FlushDuration.Value(0)
		return nil
	}
	n.Lock()
	if n.closed {
		return errClosed
	}
	preFlush := time.Now()
	buf := bytes.NewBufferString("")
	for _, m := range metrics {
		tagsString := ""
		for i, tag := range m.Tags {
			if i == 0 {
				tagsString += ";"
			} else {
				tagsString += ","

			}
			tagsString += tag
		}
		buf.WriteString(fmt.Sprintf("%s%s %f %d\n", m.Name, tagsString, m.Value, m.Time))
	}
	prePub := time.Now()
	_, err := n.conn.Write(buf.Bytes())
	n.Unlock()
	if err != nil {
		n.PublishErrors.Inc(1)
		return err
	}
	n.MessageBytes.Value(int64(buf.Len()))
	n.MessageMetrics.Value(int64(len(metrics)))
	n.PublishedMetrics.Inc(int64(len(metrics)))
	n.PublishedMessages.Inc(1)
	n.PublishDuration.Value(time.Since(prePub))
	n.FlushDuration.Value(time.Since(preFlush))
	return nil
}
<|endoftext|>