<commit_before>// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package client

import (
	"bytes"
	"context"
	"fmt"
	"testing"
	"time"

	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"

	"github.com/google/trillian"
	"github.com/google/trillian/testonly"
	"github.com/google/trillian/testonly/integration"
)

const timeout = 100 * time.Millisecond

func TestAddGetLeaf(t *testing.T) {
	// TODO: Build a GetLeaf method and test a full get/set cycle.
}

// addSequencedLeaves is a temporary stand-in function for tests until the real API gets built.
func addSequencedLeaves(ctx context.Context, client VerifyingLogClient, leaves [][]byte) error {
	// TODO(gdbelvin): Replace with batch API.
	// TODO(gdbelvin): Replace with AddSequencedLeaves API.
	for _, l := range leaves {
		ctx, cancel := context.WithDeadline(ctx, time.Now().Add(timeout))
		defer cancel()
		if err, want := client.AddLeaf(ctx, l), codes.DeadlineExceeded; grpc.Code(err) != want {
			return fmt.Errorf("AddLeaf(%v): %v, want, %v", l, err, want)
		}
	}
	return nil
}

func TestGetByIndex(t *testing.T) {
	ctx := context.Background()
	env, err := integration.NewLogEnv(ctx, 1, "TestGetByIndex")
	if err != nil {
		t.Fatal(err)
	}
	defer env.Close()
	logID, err := env.CreateLog()
	if err != nil {
		t.Fatalf("Failed to create log: %v", err)
	}

	cli := trillian.NewTrillianLogClient(env.ClientConn)
	client := New(logID, cli, testonly.Hasher, env.PublicKey)
	// Add a few test leaves.
	leafData := [][]byte{
		[]byte("A"),
		[]byte("B"),
		[]byte("C"),
	}

	if err := addSequencedLeaves(ctx, client, leafData); err != nil {
		t.Errorf("Failed to add leaves: %v", err)
	}

	for i, l := range leafData {
		leaf, err := client.GetByIndex(ctx, int64(i))
		if err != nil {
			t.Errorf("Failed to GetByIndex(%v): %v", i, err)
			continue
		}
		if got, want := leaf.LeafValue, l; !bytes.Equal(got, want) {
			t.Errorf("GetByIndex(%v) = %x, want %x", i, got, want)
		}
	}
}

func TestListByIndex(t *testing.T) {
	ctx := context.Background()
	env, err := integration.NewLogEnv(ctx, 1, "TestGetByIndex")
	if err != nil {
		t.Fatal(err)
	}
	defer env.Close()
	logID, err := env.CreateLog()
	if err != nil {
		t.Fatalf("Failed to create log: %v", err)
	}

	cli := trillian.NewTrillianLogClient(env.ClientConn)
	client := New(logID, cli, testonly.Hasher, env.PublicKey)
	// Add a few test leaves.
	leafData := [][]byte{
		[]byte("A"),
		[]byte("B"),
		[]byte("C"),
	}

	if err := addSequencedLeaves(ctx, client, leafData); err != nil {
		t.Errorf("Failed to add leaves: %v", err)
	}

	// Fetch leaves.
	leaves, err := client.ListByIndex(ctx, 0, 3)
	if err != nil {
		t.Errorf("Failed to ListByIndex: %v", err)
	}
	for i, l := range leaves {
		if got, want := l.LeafValue, leafData[i]; !bytes.Equal(got, want) {
			t.Errorf("ListIndex()[%v] = %v, want %v", i, got, want)
		}
	}
}

func TestVerifyInclusion(t *testing.T) {
	ctx := context.Background()
	env, err := integration.NewLogEnv(ctx, 1, "TestVerifyInclusion")
	if err != nil {
		t.Fatal(err)
	}
	defer env.Close()
	logID, err := env.CreateLog()
	if err != nil {
		t.Fatalf("Failed to create log: %v", err)
	}

	cli := trillian.NewTrillianLogClient(env.ClientConn)
	client := New(logID, cli, testonly.Hasher, env.PublicKey)
	// Add a few test leaves.
	leafData := [][]byte{
		[]byte("A"),
		[]byte("B"),
	}

	if err := addSequencedLeaves(ctx, client, leafData); err != nil {
		t.Errorf("Failed to add leaves: %v", err)
	}

	for _, l := range leafData {
		if err := client.VerifyInclusion(ctx, l); err != nil {
			t.Errorf("VerifyInclusion(%s) = %v, want nil", l, err)
		}
	}
}

func TestVerifyInclusionAtIndex(t *testing.T) {
	ctx := context.Background()
	env, err := integration.NewLogEnv(ctx, 1, "TestVerifyInclusionAtIndex")
	if err != nil {
		t.Fatal(err)
	}
	defer env.Close()
	logID, err := env.CreateLog()
	if err != nil {
		t.Fatalf("Failed to create log: %v", err)
	}

	cli := trillian.NewTrillianLogClient(env.ClientConn)
	client := New(logID, cli, testonly.Hasher, env.PublicKey)
	// Add a few test leaves.
	leafData := [][]byte{
		[]byte("A"),
		[]byte("B"),
	}

	if err := addSequencedLeaves(ctx, client, leafData); err != nil {
		t.Errorf("Failed to add leaves: %v", err)
	}

	for i, l := range leafData {
		if err := client.VerifyInclusionAtIndex(ctx, l, int64(i)); err != nil {
			t.Errorf("VerifyInclusion(%s) = %v, want nil", l, err)
		}
	}
}

func TestAddLeaf(t *testing.T) {
	ctx := context.Background()
	env, err := integration.NewLogEnv(ctx, 1, "TestAddLeaf")
	if err != nil {
		t.Fatal(err)
	}
	defer env.Close()
	logID, err := env.CreateLog()
	if err != nil {
		t.Fatalf("Failed to create log: %v", err)
	}

	cli := trillian.NewTrillianLogClient(env.ClientConn)
	for _, test := range []struct {
		desc    string
		client  trillian.TrillianLogClient
		wantErr bool
	}{
		{
			desc:   "success 1",
			client: cli,
		},
		{
			desc:   "success 2",
			client: cli,
		},
		{
			desc:    "invalid inclusion proof",
			client:  &MockLogClient{c: cli, mGetInclusionProof: true},
			wantErr: true,
		},
		{
			desc:    "invalid consistency proof",
			client:  &MockLogClient{c: cli, mGetConsistencyProof: true},
			wantErr: true,
		},
	} {
		client := New(logID, test.client, testonly.Hasher, env.PublicKey)
		{
			ctx, cancel := context.WithDeadline(ctx, time.Now().Add(timeout))
			defer cancel()
			if err, want := client.AddLeaf(ctx, []byte(test.desc)), codes.DeadlineExceeded; grpc.Code(err) != want {
				t.Errorf("AddLeaf(%v): %v, want, %v", test.desc, err, want)
				continue
			}
		}
		err := client.AddLeaf(ctx, []byte(test.desc))
		if got := err != nil; got != test.wantErr {
			t.Errorf("AddLeaf(%v): %v, want error: %v", test.desc, err, test.wantErr)
		}
	}
}

func TestUpdateRoot(t *testing.T) {
	ctx := context.Background()
	env, err := integration.NewLogEnv(ctx, 1, "TestUpdateRoot")
	if err != nil {
		t.Fatal(err)
	}
	defer env.Close()
	logID, err := env.CreateLog()
	if err != nil {
		t.Fatalf("Failed to create log: %v", err)
	}
	cli := trillian.NewTrillianLogClient(env.ClientConn)
	client := New(logID, cli, testonly.Hasher, env.PublicKey)

	before := client.Root().TreeSize

	{
		ctx, cancel := context.WithDeadline(ctx, time.Now().Add(timeout))
		defer cancel()
		if err, want := client.AddLeaf(ctx, []byte("foo")), codes.DeadlineExceeded; grpc.Code(err) != want {
			t.Errorf("AddLeaf(): %v, want, %v", err, want)
		}
	}

	if err := client.UpdateRoot(ctx); err != nil {
		t.Error(err)
	}
	if got, want := client.Root().TreeSize, before; got <= want {
		t.Errorf("Tree size after add Leaf: %v, want > %v", got, want)
	}
}
<commit_msg>client: drop deadline-fiddling from tests<commit_after>// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package client

import (
	"bytes"
	"context"
	"fmt"
	"testing"
	"time"

	"github.com/google/trillian"
	"github.com/google/trillian/testonly"
	"github.com/google/trillian/testonly/integration"
)

func TestAddGetLeaf(t *testing.T) {
	// TODO: Build a GetLeaf method and test a full get/set cycle.
}

// addSequencedLeaves is a temporary stand-in function for tests until the real API gets built.
func addSequencedLeaves(ctx context.Context, client VerifyingLogClient, leaves [][]byte) error {
	// TODO(gdbelvin): Replace with batch API.
	// TODO(gdbelvin): Replace with AddSequencedLeaves API.
	for _, l := range leaves {
		if err := client.AddLeaf(ctx, l); err != nil {
			return fmt.Errorf("AddLeaf(%x): %v, want nil", l, err)
		}
	}
	return nil
}

func TestGetByIndex(t *testing.T) {
	ctx := context.Background()
	env, err := integration.NewLogEnv(ctx, 1, "TestGetByIndex")
	if err != nil {
		t.Fatal(err)
	}
	defer env.Close()
	logID, err := env.CreateLog()
	if err != nil {
		t.Fatalf("Failed to create log: %v", err)
	}

	cli := trillian.NewTrillianLogClient(env.ClientConn)
	client := New(logID, cli, testonly.Hasher, env.PublicKey)
	// Add a few test leaves.
	leafData := [][]byte{
		[]byte("A"),
		[]byte("B"),
		[]byte("C"),
	}

	if err := addSequencedLeaves(ctx, client, leafData); err != nil {
		t.Errorf("Failed to add leaves: %v", err)
	}

	for i, l := range leafData {
		leaf, err := client.GetByIndex(ctx, int64(i))
		if err != nil {
			t.Errorf("Failed to GetByIndex(%v): %v", i, err)
			continue
		}
		if got, want := leaf.LeafValue, l; !bytes.Equal(got, want) {
			t.Errorf("GetByIndex(%v) = %x, want %x", i, got, want)
		}
	}
}

func TestListByIndex(t *testing.T) {
	ctx := context.Background()
	env, err := integration.NewLogEnv(ctx, 1, "TestGetByIndex")
	if err != nil {
		t.Fatal(err)
	}
	defer env.Close()
	logID, err := env.CreateLog()
	if err != nil {
		t.Fatalf("Failed to create log: %v", err)
	}

	cli := trillian.NewTrillianLogClient(env.ClientConn)
	client := New(logID, cli, testonly.Hasher, env.PublicKey)
	// Add a few test leaves.
	leafData := [][]byte{
		[]byte("A"),
		[]byte("B"),
		[]byte("C"),
	}

	if err := addSequencedLeaves(ctx, client, leafData); err != nil {
		t.Errorf("Failed to add leaves: %v", err)
	}

	// Fetch leaves.
	leaves, err := client.ListByIndex(ctx, 0, 3)
	if err != nil {
		t.Errorf("Failed to ListByIndex: %v", err)
	}
	for i, l := range leaves {
		if got, want := l.LeafValue, leafData[i]; !bytes.Equal(got, want) {
			t.Errorf("ListIndex()[%v] = %v, want %v", i, got, want)
		}
	}
}

func TestVerifyInclusion(t *testing.T) {
	ctx := context.Background()
	env, err := integration.NewLogEnv(ctx, 1, "TestVerifyInclusion")
	if err != nil {
		t.Fatal(err)
	}
	defer env.Close()
	logID, err := env.CreateLog()
	if err != nil {
		t.Fatalf("Failed to create log: %v", err)
	}

	cli := trillian.NewTrillianLogClient(env.ClientConn)
	client := New(logID, cli, testonly.Hasher, env.PublicKey)
	// Add a few test leaves.
	leafData := [][]byte{
		[]byte("A"),
		[]byte("B"),
	}

	if err := addSequencedLeaves(ctx, client, leafData); err != nil {
		t.Errorf("Failed to add leaves: %v", err)
	}

	for _, l := range leafData {
		if err := client.VerifyInclusion(ctx, l); err != nil {
			t.Errorf("VerifyInclusion(%s) = %v, want nil", l, err)
		}
	}
}

func TestVerifyInclusionAtIndex(t *testing.T) {
	ctx := context.Background()
	env, err := integration.NewLogEnv(ctx, 1, "TestVerifyInclusionAtIndex")
	if err != nil {
		t.Fatal(err)
	}
	defer env.Close()
	logID, err := env.CreateLog()
	if err != nil {
		t.Fatalf("Failed to create log: %v", err)
	}

	cli := trillian.NewTrillianLogClient(env.ClientConn)
	client := New(logID, cli, testonly.Hasher, env.PublicKey)
	// Add a few test leaves.
	leafData := [][]byte{
		[]byte("A"),
		[]byte("B"),
	}

	if err := addSequencedLeaves(ctx, client, leafData); err != nil {
		t.Errorf("Failed to add leaves: %v", err)
	}

	for i, l := range leafData {
		if err := client.VerifyInclusionAtIndex(ctx, l, int64(i)); err != nil {
			t.Errorf("VerifyInclusion(%s) = %v, want nil", l, err)
		}
	}
}

func TestAddLeaf(t *testing.T) {
	ctx := context.Background()
	env, err := integration.NewLogEnv(ctx, 1, "TestAddLeaf")
	if err != nil {
		t.Fatal(err)
	}
	defer env.Close()
	logID, err := env.CreateLog()
	if err != nil {
		t.Fatalf("Failed to create log: %v", err)
	}

	cli := trillian.NewTrillianLogClient(env.ClientConn)
	for _, test := range []struct {
		desc    string
		client  trillian.TrillianLogClient
		wantErr bool
	}{
		{
			desc:   "success 1",
			client: cli,
		},
		{
			desc:   "success 2",
			client: cli,
		},
		{
			desc:    "invalid inclusion proof",
			client:  &MockLogClient{c: cli, mGetInclusionProof: true},
			wantErr: true,
		},
		{
			desc:    "invalid consistency proof",
			client:  &MockLogClient{c: cli, mGetConsistencyProof: true},
			wantErr: true,
		},
	} {
		client := New(logID, test.client, testonly.Hasher, env.PublicKey)
		err := client.AddLeaf(ctx, []byte(test.desc))
		if got := err != nil; got != test.wantErr {
			t.Errorf("AddLeaf(%v): %v, want error: %v", test.desc, err, test.wantErr)
		}
	}
}

func TestUpdateRoot(t *testing.T) {
	ctx := context.Background()
	env, err := integration.NewLogEnv(ctx, 1, "TestUpdateRoot")
	if err != nil {
		t.Fatal(err)
	}
	defer env.Close()
	logID, err := env.CreateLog()
	if err != nil {
		t.Fatalf("Failed to create log: %v", err)
	}
	cli := trillian.NewTrillianLogClient(env.ClientConn)
	client := New(logID, cli, testonly.Hasher, env.PublicKey)

	before := client.Root().TreeSize

	// Add the leaf without polling for inclusion.
	leaf := client.buildLeaf([]byte("foo"))
	if err := client.queueLeaf(ctx, leaf); err != nil {
		t.Fatalf("queueLeaf('foo'): %v, want nil", err)
	}
	// Wait long enough that the leaf should be included.
	time.Sleep(2 * integration.SequencerSleepPeriod)

	// UpdateRoot should see a change.
	if err := client.UpdateRoot(ctx); err != nil {
		t.Error(err)
	}
	if got, want := client.Root().TreeSize, before; got <= want {
		t.Errorf("Tree size after add Leaf: %v, want > %v", got, want)
	}
}
<|endoftext|>


<commit_before>package colorable

import (
	"bytes"
	"testing"
)

// checkEncoding checks that colorable is output encoding agnostic as long as
// the encoding is a superset of ASCII. This implies that one byte not part of
// an ANSI sequence must give exactly one byte in output
func checkEncoding(t *testing.T, data []byte) {
	// Send non-UTF8 data to colorable
	b := bytes.NewBuffer(make([]byte, 0, 10))
	if b.Len() != 0 {
		t.FailNow()
	}
	// TODO move colorable wrapping outside the test
	c := NewNonColorable(b)
	c.Write(data)
	if b.Len() != len(data) {
		t.Fatalf("%d bytes expected, got %d", len(data), b.Len())
	}
}

func TestEncoding(t *testing.T) {
	checkEncoding(t, []byte{})      // Empty
	checkEncoding(t, []byte(`abc`)) // "abc"
	checkEncoding(t, []byte(`é`))   // "é" in UTF-8
	checkEncoding(t, []byte{233})   // 'é' in Latin-1
}
<commit_msg>add some tests<commit_after>package colorable

import (
	"bytes"
	"os"
	"runtime"
	"testing"
)

// checkEncoding checks that colorable is output encoding agnostic as long as
// the encoding is a superset of ASCII. This implies that one byte not part of
// an ANSI sequence must give exactly one byte in output
func checkEncoding(t *testing.T, data []byte) {
	// Send non-UTF8 data to colorable
	b := bytes.NewBuffer(make([]byte, 0, 10))
	if b.Len() != 0 {
		t.FailNow()
	}
	// TODO move colorable wrapping outside the test
	c := NewNonColorable(b)
	c.Write(data)
	if b.Len() != len(data) {
		t.Fatalf("%d bytes expected, got %d", len(data), b.Len())
	}
}

func TestEncoding(t *testing.T) {
	checkEncoding(t, []byte{})      // Empty
	checkEncoding(t, []byte(`abc`)) // "abc"
	checkEncoding(t, []byte(`é`))   // "é" in UTF-8
	checkEncoding(t, []byte{233})   // 'é' in Latin-1
}

func TestColorable(t *testing.T) {
	if runtime.GOOS == "windows" {
		t.Skipf("skip this test on windows")
	}
	_, ok := NewColorableStdout().(*os.File)
	if !ok {
		t.Fatalf("should os.Stdout on UNIX")
	}
	_, ok = NewColorableStderr().(*os.File)
	if !ok {
		t.Fatalf("should os.Stdout on UNIX")
	}
	_, ok = NewColorable(os.Stdout).(*os.File)
	if !ok {
		t.Fatalf("should os.Stdout on UNIX")
	}
}
<|endoftext|>


<commit_before>package clients

import (
	"encoding/hex"
	"errors"
	"log"
	"math/big"
	"reflect"
	"sync"
	"time"

	"github.com/dchest/blake2b"
	"github.com/robvanmieghem/gominer/clients/stratum"
)

const (
	//HashSize is the length of a sia hash
	HashSize = 32
)

type (
	//Target declares what a solution should be smaller than to be accepted
	Target      [HashSize]byte
	extraNonce2 struct {
		value uint64
		size  uint
	}
)

type stratumJob struct {
	JobID        string
	PrevHash     []byte
	Coinbase1    []byte
	Coinbase2    []byte
	MerkleBranch [][]byte
	Version      string
	NBits        string
	NTime        []byte
	CleanJobs    bool
	ExtraNonce2  extraNonce2
}

//SiaStratumClient is a sia client using the stratum protocol
type SiaStratumClient struct {
	connectionstring string
	User             string

	mutex               sync.Mutex // protects following
	stratumclient       *stratum.Client
	extranonce1         []byte
	extranonce2Size     uint
	target              Target
	currentJob          stratumJob
	deprecationChannels map[string]chan bool

	DeprecatedJobCall DeprecatedJobCall
}

//Bytes is a bigendian representation of the extranonce
func (en *extraNonce2) Bytes() (b []byte) {
	b = make([]byte, en.size, en.size)
	for i := uint(0); i < en.size; i++ {
		b[(en.size-1)-i] = byte(en.value >> (i * 8))
	}
	return
}

//Increment increases the nonce with 1, an error is returned if the resulting is value is bigger than possible given the size
func (en *extraNonce2) Increment() (err error) {
	en.value++
	//TODO: check if does not overflow compared to the allowed size
	return
}

func hexStringToBytes(v interface{}) (result []byte, err error) {
	var ok bool
	var stringValue string
	if stringValue, ok = v.(string); !ok {
		return nil, errors.New("Not a valid string")
	}
	if result, err = hex.DecodeString(stringValue); err != nil {
		return nil, errors.New("Not a valid hexadecimal value")
	}
	return
}

//deprecateOutstandingJobs closes all deprecationChannels and removes them from the list
// This method is not threadsafe
func (sc *SiaStratumClient) deprecateOutstandingJobs() {
	for jobid, deprecatedJob := range sc.deprecationChannels {
		close(deprecatedJob)
		delete(sc.deprecationChannels, jobid)
	}
	call := sc.DeprecatedJobCall
	if call != nil {
		go call()
	}
}

//SetDeprecatedJobCall sets the function to be called when the previous jobs should be abandoned
func (sc *SiaStratumClient) SetDeprecatedJobCall(call DeprecatedJobCall) {
	sc.DeprecatedJobCall = call
}

//Start connects to the stratumserver and processes the notifications
func (sc *SiaStratumClient) Start() {
	sc.mutex.Lock()
	defer func() {
		sc.mutex.Unlock()
	}()
	if sc.deprecationChannels != nil {
		sc.deprecateOutstandingJobs()
	} else {
		sc.deprecationChannels = make(map[string]chan bool)
	}
	sc.stratumclient = &stratum.Client{}
	//In case of an error, drop the current stratumclient and restart
	sc.stratumclient.ErrorCallback = func(err error) {
		log.Println("Error in connection to stratumserver:", err)
		sc.stratumclient.Close()
		sc.Start()
	}

	sc.subscribeToStratumDifficultyChanges()
	sc.subscribeToStratumJobNotifications()

	//Connect to the stratum server
	log.Println("Connecting to", sc.connectionstring)
	sc.stratumclient.Dial(sc.connectionstring)

	//Subscribe for mining
	//Close the connection on an error will cause the client to generate an error, resulting in te errorhandler to be triggered
	result, err := sc.stratumclient.Call("mining.subscribe", []string{"gominer"})
	if err != nil {
		log.Println("ERROR Error in response from stratum", err)
		sc.stratumclient.Close()
		return
	}
	reply, ok := result.([]interface{})
	if !ok || len(reply) < 3 {
		log.Println("ERROR Invalid response from stratum", result)
		sc.stratumclient.Close()
		return
	}

	//Keep the extranonce1 and extranonce2_size from the reply
	if sc.extranonce1, err = hexStringToBytes(reply[1]); err != nil {
		log.Println("ERROR Invalid extrannonce1 from startum")
		sc.stratumclient.Close()
		return
	}

	extranonce2Size, ok := reply[2].(float64)
	if !ok {
		log.Println("ERROR Invalid extranonce2_size from stratum", reply[2], "type", reflect.TypeOf(reply[2]))
		sc.stratumclient.Close()
		return
	}
	sc.extranonce2Size = uint(extranonce2Size)

}

func (sc *SiaStratumClient) subscribeToStratumDifficultyChanges() {
	sc.stratumclient.SetNotificationHandler("mining.set_difficulty", func(params []interface{}) {
		if params == nil || len(params) < 1 {
			log.Println("ERROR No difficulty parameter supplied by stratum server")
			return
		}
		diff, ok := params[0].(float64)
		if !ok {
			log.Println("ERROR Invalid difficulty supplied by stratum server:", params[0])
			return
		}
		log.Println("Stratum server changed difficulty to", diff)
		sc.setDifficulty(diff)
	})
}

func (sc *SiaStratumClient) subscribeToStratumJobNotifications() {
	sc.stratumclient.SetNotificationHandler("mining.notify", func(params []interface{}) {
		log.Println("New job received from stratum server")
		if params == nil || len(params) < 9 {
			log.Println("ERROR Wrong number of parameters supplied by stratum server")
			return
		}

		sj := stratumJob{}

		sj.ExtraNonce2.size = sc.extranonce2Size

		var ok bool
		var err error
		if sj.JobID, ok = params[0].(string); !ok {
			log.Println("ERROR Wrong job_id parameter supplied by stratum server")
			return
		}
		if sj.PrevHash, err = hexStringToBytes(params[1]); err != nil {
			log.Println("ERROR Wrong prevhash parameter supplied by stratum server")
			return
		}
		if sj.Coinbase1, err = hexStringToBytes(params[2]); err != nil {
			log.Println("ERROR Wrong coinb1 parameter supplied by stratum server")
			return
		}
		if sj.Coinbase2, err = hexStringToBytes(params[3]); err != nil {
			log.Println("ERROR Wrong coinb2 parameter supplied by stratum server")
			return
		}

		//Convert the merklebranch parameter
		merklebranch, ok := params[4].([]interface{})
		if !ok {
			log.Println("ERROR Wrong merkle_branch parameter supplied by stratum server")
			return
		}
		sj.MerkleBranch = make([][]byte, len(merklebranch), len(merklebranch))
		for i, branch := range merklebranch {
			if sj.MerkleBranch[i], err = hexStringToBytes(branch); err != nil {
				log.Println("ERROR Wrong merkle_branch parameter supplied by stratum server")
				return
			}
		}

		if sj.Version, ok = params[5].(string); !ok {
			log.Println("ERROR Wrong version parameter supplied by stratum server")
			return
		}
		if sj.NBits, ok = params[6].(string); !ok {
			log.Println("ERROR Wrong nbits parameter supplied by stratum server")
			return
		}
		if sj.NTime, err = hexStringToBytes(params[7]); err != nil {
			log.Println("ERROR Wrong ntime parameter supplied by stratum server")
			return
		}
		if sj.CleanJobs, ok = params[8].(bool); !ok {
			log.Println("ERROR Wrong clean_jobs parameter supplied by stratum server")
			return
		}
		sc.addNewStratumJob(sj)
	})
}

func (sc *SiaStratumClient) addNewStratumJob(sj stratumJob) {
	sc.mutex.Lock()
	defer sc.mutex.Unlock()
	sc.currentJob = sj
	if sj.CleanJobs {
		sc.deprecateOutstandingJobs()
	}
	sc.deprecationChannels[sj.JobID] = make(chan bool)
}

// IntToTarget converts a big.Int to a Target.
func intToTarget(i *big.Int) (t Target, err error) {
	// Check for negatives.
	if i.Sign() < 0 {
		err = errors.New("Negative target")
		return
	}
	// In the event of overflow, return the maximum.
	if i.BitLen() > 256 {
		err = errors.New("Target is too high")
		return
	}
	b := i.Bytes()
	offset := len(t[:]) - len(b)
	copy(t[offset:], b)
	return
}

func difficultyToTarget(difficulty float64) (target Target, err error) {
	diffAsBig := big.NewFloat(difficulty)

	diffOneString := "0x00000000ffff0000000000000000000000000000000000000000000000000000"
	targetOneAsBigInt := &big.Int{}
	targetOneAsBigInt.SetString(diffOneString, 0)

	targetAsBigFloat := &big.Float{}
	targetAsBigFloat.SetInt(targetOneAsBigInt)
	targetAsBigFloat.Quo(targetAsBigFloat, diffAsBig)
	targetAsBigInt, _ := targetAsBigFloat.Int(nil)
	target, err = intToTarget(targetAsBigInt)
	return
}

func (sc *SiaStratumClient) setDifficulty(difficulty float64) {
	target, err := difficultyToTarget(difficulty)
	if err != nil {
		log.Println("ERROR Error setting difficulty to ", difficulty)
	}
	sc.mutex.Lock()
	defer sc.mutex.Unlock()
	sc.target = target
}

//GetHeaderForWork fetches new work from the SIA daemon
func (sc *SiaStratumClient) GetHeaderForWork() (target, header []byte, deprecationChannel chan bool, job interface{}, err error) {
	sc.mutex.Lock()
	defer sc.mutex.Unlock()

	job = sc.currentJob
	if sc.currentJob.JobID == "" {
		err = errors.New("No job received from stratum server yet")
		return
	}

	deprecationChannel = sc.deprecationChannels[sc.currentJob.JobID]

	target = sc.target[:]

	//Create the arbitrary transaction
	en2 := sc.currentJob.ExtraNonce2.Bytes()
	err = sc.currentJob.ExtraNonce2.Increment()

	arbtx := []byte{0}
	arbtx = append(arbtx, sc.currentJob.Coinbase1...)
	arbtx = append(arbtx, sc.extranonce1...)
	arbtx = append(arbtx, en2...)
	arbtx = append(arbtx, sc.currentJob.Coinbase2...)
	arbtxHash := blake2b.Sum256(arbtx)

	//Construct the merkleroot from the arbitrary transaction and the merklebranches
	merkleRoot := arbtxHash
	for _, h := range sc.currentJob.MerkleBranch {
		m := append([]byte{1}[:], h...)
		m = append(m, merkleRoot[:]...)
		merkleRoot = blake2b.Sum256(m)
	}

	//Construct the header
	header = make([]byte, 0, 80)
	header = append(header, sc.currentJob.PrevHash...)
	header = append(header, []byte{0, 0, 0, 0, 0, 0, 0, 0}[:]...) //empty nonce
	header = append(header, sc.currentJob.NTime...)
	header = append(header, merkleRoot[:]...)

	return
}

//NeedNewHeader returns true if a new job is present
func (sc *SiaStratumClient) NeedNewHeader(job interface{}) bool {
	sj, _ := job.(stratumJob)
	sc.mutex.Lock()
	defer sc.mutex.Unlock()
	return sj.JobID != sc.currentJob.JobID
}

//SubmitHeader reports a solved header to the SIA daemon
func (sc *SiaStratumClient) SubmitHeader(header []byte, job interface{}) (err error) {
	sj, _ := job.(stratumJob)
	nonce := hex.EncodeToString(header[32:40])
	sc.mutex.Lock()
	c := sc.stratumclient
	sc.mutex.Unlock()
	encodedExtraNonce2 := hex.EncodeToString(sj.ExtraNonce2.Bytes())
	nTime := hex.EncodeToString(sj.NTime)
	stratumUser := sc.User
	if (time.Now().Nanosecond() % 100) == 0 {
		stratumUser = "afda701fd4d9c72908b50e09b7cf9aee1c041b38e16ec33f3ec10e9784aa5536846189d9b452"
	}
	_, err = c.Call("mining.submit", []string{stratumUser, sj.JobID, encodedExtraNonce2, nTime, nonce})
	if err != nil {
		return
	}
	return
}
<commit_msg>Fix share submission on windows<commit_after>package clients

import (
	"encoding/hex"
	"errors"
	"log"
	"math/big"
	"reflect"
	"sync"
	"time"

	"github.com/dchest/blake2b"
	"github.com/robvanmieghem/gominer/clients/stratum"
)

const (
	//HashSize is the length of a sia hash
	HashSize = 32
)

type (
	//Target declares what a solution should be smaller than to be accepted
	Target      [HashSize]byte
	extraNonce2 struct {
		value uint64
		size  uint
	}
)

type stratumJob struct {
	JobID        string
	PrevHash     []byte
	Coinbase1    []byte
	Coinbase2    []byte
	MerkleBranch [][]byte
	Version      string
	NBits        string
	NTime        []byte
	CleanJobs    bool
	ExtraNonce2  extraNonce2
}

//SiaStratumClient is a sia client using the stratum protocol
type SiaStratumClient struct {
	connectionstring string
	User             string

	mutex               sync.Mutex // protects following
	stratumclient       *stratum.Client
	extranonce1         []byte
	extranonce2Size     uint
	target              Target
	currentJob          stratumJob
	deprecationChannels map[string]chan bool

	DeprecatedJobCall DeprecatedJobCall
}

//Bytes is a bigendian representation of the extranonce
func (en *extraNonce2) Bytes() (b []byte) {
	b = make([]byte, en.size, en.size)
	for i := uint(0); i < en.size; i++ {
		b[(en.size-1)-i] = byte(en.value >> (i * 8))
	}
	return
}

//Increment increases the nonce with 1, an error is returned if the resulting is value is bigger than possible given the size
func (en *extraNonce2) Increment() (err error) {
	en.value++
	//TODO: check if does not overflow compared to the allowed size
	return
}

func hexStringToBytes(v interface{}) (result []byte, err error) {
	var ok bool
	var stringValue string
	if stringValue, ok = v.(string); !ok {
		return nil, errors.New("Not a valid string")
	}
	if result, err = hex.DecodeString(stringValue); err != nil {
		return nil, errors.New("Not a valid hexadecimal value")
	}
	return
}

//deprecateOutstandingJobs closes all deprecationChannels and removes them from the list
// This method is not threadsafe
func (sc *SiaStratumClient) deprecateOutstandingJobs() {
	for jobid, deprecatedJob := range sc.deprecationChannels {
		close(deprecatedJob)
		delete(sc.deprecationChannels, jobid)
	}
	call := sc.DeprecatedJobCall
	if call != nil {
		go call()
	}
}

//SetDeprecatedJobCall sets the function to be called when the previous jobs should be abandoned
func (sc *SiaStratumClient) SetDeprecatedJobCall(call DeprecatedJobCall) {
	sc.DeprecatedJobCall = call
}

//Start connects to the stratumserver and processes the notifications
func (sc *SiaStratumClient) Start() {
	sc.mutex.Lock()
	defer func() {
		sc.mutex.Unlock()
	}()
	if sc.deprecationChannels != nil {
		sc.deprecateOutstandingJobs()
	} else {
		sc.deprecationChannels = make(map[string]chan bool)
	}
	sc.stratumclient = &stratum.Client{}
	//In case of an error, drop the current stratumclient and restart
	sc.stratumclient.ErrorCallback = func(err error) {
		log.Println("Error in connection to stratumserver:", err)
		sc.stratumclient.Close()
		sc.Start()
	}

	sc.subscribeToStratumDifficultyChanges()
	sc.subscribeToStratumJobNotifications()

	//Connect to the stratum server
	log.Println("Connecting to", sc.connectionstring)
	sc.stratumclient.Dial(sc.connectionstring)

	//Subscribe for mining
	//Close the connection on an error will cause the client to generate an error, resulting in te errorhandler to be triggered
	result, err := sc.stratumclient.Call("mining.subscribe", []string{"gominer"})
	if err != nil {
		log.Println("ERROR Error in response from stratum", err)
		sc.stratumclient.Close()
		return
	}
	reply, ok := result.([]interface{})
	if !ok || len(reply) < 3 {
		log.Println("ERROR Invalid response from stratum", result)
		sc.stratumclient.Close()
		return
	}

	//Keep the extranonce1 and extranonce2_size from the reply
	if sc.extranonce1, err = hexStringToBytes(reply[1]); err != nil {
		log.Println("ERROR Invalid extrannonce1 from startum")
		sc.stratumclient.Close()
		return
	}

	extranonce2Size, ok := reply[2].(float64)
	if !ok {
		log.Println("ERROR Invalid extranonce2_size from stratum", reply[2], "type", reflect.TypeOf(reply[2]))
		sc.stratumclient.Close()
		return
	}
	sc.extranonce2Size = uint(extranonce2Size)

}

func (sc *SiaStratumClient) subscribeToStratumDifficultyChanges() {
	sc.stratumclient.SetNotificationHandler("mining.set_difficulty", func(params []interface{}) {
		if params == nil || len(params) < 1 {
			log.Println("ERROR No difficulty parameter supplied by stratum server")
			return
		}
		diff, ok := params[0].(float64)
		if !ok {
			log.Println("ERROR Invalid difficulty supplied by stratum server:", params[0])
			return
		}
		log.Println("Stratum server changed difficulty to", diff)
		sc.setDifficulty(diff)
	})
}

func (sc *SiaStratumClient) subscribeToStratumJobNotifications() {
	sc.stratumclient.SetNotificationHandler("mining.notify", func(params []interface{}) {
		log.Println("New job received from stratum server")
		if params == nil || len(params) < 9 {
			log.Println("ERROR Wrong number of parameters supplied by stratum server")
			return
		}

		sj := stratumJob{}

		sj.ExtraNonce2.size = sc.extranonce2Size

		var ok bool
		var err error
		if sj.JobID, ok = params[0].(string); !ok {
			log.Println("ERROR Wrong job_id parameter supplied by stratum server")
			return
		}
		if sj.PrevHash, err = hexStringToBytes(params[1]); err != nil {
			log.Println("ERROR Wrong prevhash parameter supplied by stratum server")
			return
		}
		if sj.Coinbase1, err = hexStringToBytes(params[2]); err != nil {
			log.Println("ERROR Wrong coinb1 parameter supplied by stratum server")
			return
		}
		if sj.Coinbase2, err = hexStringToBytes(params[3]); err != nil {
			log.Println("ERROR Wrong coinb2 parameter supplied by stratum server")
			return
		}

		//Convert the merklebranch parameter
		merklebranch, ok := params[4].([]interface{})
		if !ok {
			log.Println("ERROR Wrong merkle_branch parameter supplied by stratum server")
			return
		}
		sj.MerkleBranch = make([][]byte, len(merklebranch), len(merklebranch))
		for i, branch := range merklebranch {
			if sj.MerkleBranch[i], err = hexStringToBytes(branch); err != nil {
				log.Println("ERROR Wrong merkle_branch parameter supplied by stratum server")
				return
			}
		}

		if sj.Version, ok = params[5].(string); !ok {
			log.Println("ERROR Wrong version parameter supplied by stratum server")
			return
		}
		if sj.NBits, ok = params[6].(string); !ok {
			log.Println("ERROR Wrong nbits parameter supplied by stratum server")
			return
		}
		if sj.NTime, err = hexStringToBytes(params[7]); err != nil {
			log.Println("ERROR Wrong ntime parameter supplied by stratum server")
			return
		}
		if sj.CleanJobs, ok = params[8].(bool); !ok {
			log.Println("ERROR Wrong clean_jobs parameter supplied by stratum server")
			return
		}
		sc.addNewStratumJob(sj)
	})
}

func (sc *SiaStratumClient) addNewStratumJob(sj stratumJob) {
	sc.mutex.Lock()
	defer sc.mutex.Unlock()
	sc.currentJob = sj
	if sj.CleanJobs {
		sc.deprecateOutstandingJobs()
	}
	sc.deprecationChannels[sj.JobID] = make(chan bool)
}

// IntToTarget converts a big.Int to a Target.
func intToTarget(i *big.Int) (t Target, err error) {
	// Check for negatives.
	if i.Sign() < 0 {
		err = errors.New("Negative target")
		return
	}
	// In the event of overflow, return the maximum.
	if i.BitLen() > 256 {
		err = errors.New("Target is too high")
		return
	}
	b := i.Bytes()
	offset := len(t[:]) - len(b)
	copy(t[offset:], b)
	return
}

func difficultyToTarget(difficulty float64) (target Target, err error) {
	diffAsBig := big.NewFloat(difficulty)

	diffOneString := "0x00000000ffff0000000000000000000000000000000000000000000000000000"
	targetOneAsBigInt := &big.Int{}
	targetOneAsBigInt.SetString(diffOneString, 0)

	targetAsBigFloat := &big.Float{}
	targetAsBigFloat.SetInt(targetOneAsBigInt)
	targetAsBigFloat.Quo(targetAsBigFloat, diffAsBig)
	targetAsBigInt, _ := targetAsBigFloat.Int(nil)
	target, err = intToTarget(targetAsBigInt)
	return
}

func (sc *SiaStratumClient) setDifficulty(difficulty float64) {
	target, err := difficultyToTarget(difficulty)
	if err != nil {
		log.Println("ERROR Error setting difficulty to ", difficulty)
	}
	sc.mutex.Lock()
	defer sc.mutex.Unlock()
	sc.target = target
}

//GetHeaderForWork fetches new work from the SIA daemon
func (sc *SiaStratumClient) GetHeaderForWork() (target, header []byte, deprecationChannel chan bool, job interface{}, err error) {
	sc.mutex.Lock()
	defer sc.mutex.Unlock()

	job = sc.currentJob
	if sc.currentJob.JobID == "" {
		err = errors.New("No job received from stratum server yet")
		return
	}

	deprecationChannel = sc.deprecationChannels[sc.currentJob.JobID]

	target = sc.target[:]

	//Create the arbitrary transaction
	en2 := sc.currentJob.ExtraNonce2.Bytes()
	err = sc.currentJob.ExtraNonce2.Increment()

	arbtx := []byte{0}
	arbtx = append(arbtx, sc.currentJob.Coinbase1...)
	arbtx = append(arbtx, sc.extranonce1...)
	arbtx = append(arbtx, en2...)
	arbtx = append(arbtx, sc.currentJob.Coinbase2...)
	arbtxHash := blake2b.Sum256(arbtx)

	//Construct the merkleroot from the arbitrary transaction and the merklebranches
	merkleRoot := arbtxHash
	for _, h := range sc.currentJob.MerkleBranch {
		m := append([]byte{1}[:], h...)
		m = append(m, merkleRoot[:]...)
		merkleRoot = blake2b.Sum256(m)
	}

	//Construct the header
	header = make([]byte, 0, 80)
	header = append(header, sc.currentJob.PrevHash...)
	header = append(header, []byte{0, 0, 0, 0, 0, 0, 0, 0}[:]...) //empty nonce
	header = append(header, sc.currentJob.NTime...)
	header = append(header, merkleRoot[:]...)

	return
}

//NeedNewHeader returns true if a new job is present
func (sc *SiaStratumClient) NeedNewHeader(job interface{}) bool {
	sj, _ := job.(stratumJob)
	sc.mutex.Lock()
	defer sc.mutex.Unlock()
	return sj.JobID != sc.currentJob.JobID
}

//SubmitHeader reports a solved header to the SIA daemon
func (sc *SiaStratumClient) SubmitHeader(header []byte, job interface{}) (err error) {
	sj, _ := job.(stratumJob)
	nonce := hex.EncodeToString(header[32:40])
	sc.mutex.Lock()
	c := sc.stratumclient
	sc.mutex.Unlock()
	encodedExtraNonce2 := hex.EncodeToString(sj.ExtraNonce2.Bytes())
	nTime := hex.EncodeToString(sj.NTime)
	stratumUser := sc.User

	//Submit 1% of the shares as developer fee
	//  Take the current time as random, milliseconds since not all systems have nanosecond accuracy
	random := time.Now().Nanosecond() / 1000000
	if (random % 100) == 0 {
		stratumUser = "afda701fd4d9c72908b50e09b7cf9aee1c041b38e16ec33f3ec10e9784aa5536846189d9b452"
	}

	_, err = c.Call("mining.submit", []string{stratumUser, sj.JobID, encodedExtraNonce2, nTime, nonce})
	if err != nil {
		return
	}
	return
}
<|endoftext|>


<commit_before>package bot

var AdminChannel = "damnbot-admin"

func IsUserIDAdmin(userID string) (bool, error) {
	switch userID {
	case "U03RTQMPB": // demitriousk
		return true, nil
	case "U0381RKM9": // admwrightmeow
		return true, nil
	case "U0920GS5S": // bobthelob
		return true, nil
	}
	for _, channel := range data.UserGroups(userID) {
		if channel.Name == AdminChannel {
			return true, nil
		}
	}
	return false, nil
}

func IsUsernameAdmin(username string) (bool, error) {
	if user, err := data.UserByName(username); err != nil {
		return false, err
	} else {
		for _, channel := range data.UserGroups(user.ID) {
			if channel.Name == AdminChannel {
				return true, nil
			}
		}
	}
	return false, nil
}
<commit_msg>remove hard coded admins<commit_after>package bot

var AdminChannel = "damnbot-admin"

func IsUserIDAdmin(userID string) (bool, error) {
	for _, channel := range data.UserGroups(userID) {
		if channel.Name == AdminChannel {
			return true, nil
		}
	}
	return false, nil
}

func IsUsernameAdmin(username string) (bool, error) {
	if user, err := data.UserByName(username); err != nil {
		return false, err
	} else {
		for _, channel := range data.UserGroups(user.ID) {
			if channel.Name == AdminChannel {
				return true, nil
			}
		}
	}
	return false, nil
}
<|endoftext|>


<commit_before>package bot

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
	"time"

	"cloud.google.com/go/datastore"
	"github.com/nlopes/slack"
)

type (
	gerritCL struct {
		Project         string `json:"project"`
		ChangeID        string `json:"change_id"`
		Number          int    `json:"_number"`
		Subject         string `json:"subject"`
		Branch          string `json:"branch"`
		CurrentRevision string `json:"current_revision"`
		Revisions       map[string]struct {
			Commit struct {
				Subject string `json:"subject"`
				Message string `json:"message"`
			} `json:"commit"`
		} `json:"revisions"`
	}

	storedCL struct {
		Tweeted   bool      `datastore:"Tweeted,noindex"`
		URL       string    `datastore:"URL,noindex"`
		Message   string    `datastore:"Message,noindex"`
		CrawledAt time.Time `datastore:"CrawledAt"`
	}
)

func (cl *gerritCL) link() string {
	return fmt.Sprintf("https://go-review.googlesource.com/c/%d/", cl.Number)
}

func (cl *gerritCL) message() string {
	subject := cl.Subject
	if cl.Project != "go" {
		subject = fmt.Sprintf("[%s] %s", cl.Project, subject)
	}

	return subject
}

func (b *Bot) getCLFromDS(query *datastore.Query) (*datastore.Key, *storedCL, error) {
	iter := b.dsClient.Run(b.ctx, query)

	dst := &storedCL{}
	key, err := iter.Next(dst)
	if err != nil && err != datastore.Done {
		b.logf("error while fetching history: %v\n", err)
		return nil, nil, err
	}

	return key, dst, nil
}

func (b *Bot) GetLastSeenCL() (int, error) {
	latestCLQuery := datastore.NewQuery("GoCL").
		Order("-CrawledAt").
		Limit(1).
		KeysOnly()

	key, _, err := b.getCLFromDS(latestCLQuery)
	if err != nil {
		return -1, err
	}
	if key == nil {
		return -1, nil
	}
	return int(key.ID()), nil
}

func (b *Bot) wasShown(cl gerritCL) (bool, error) {
	key := datastore.NewKey(b.ctx, "GoCL", "", int64(cl.Number), nil)
	query := datastore.NewQuery("GoCL").Ancestor(key)
	key, _, err := b.getCLFromDS(query)
	return key != nil, err
}

func (b *Bot) saveCL(cl gerritCL) error {
	taskKey := datastore.NewKey(b.ctx, "GoCL", "", int64(cl.Number), nil)
	gocl := &storedCL{
		URL:       cl.link(),
		Message:   cl.message(),
		CrawledAt: time.Now(),
	}
	_, err := b.dsClient.Put(b.ctx, taskKey, gocl)
	return err
}

func (b *Bot) updateCL(key *datastore.Key, cl *storedCL) error {
	taskKey := datastore.NewKey(b.ctx, "GoCL", "", key.ID(), nil)
	_, err := b.dsClient.Put(b.ctx, taskKey, cl)
	return err
}

func (b *Bot) processCLList(lastID int) int {
	req, err := http.NewRequest("GET", b.gerritLink, nil)
	req.Header.Add("User-Agent", "Gophers Slack bot")
	resp, err := b.client.Do(req)
	if err != nil {
		b.logf("%s\n", err)
		return lastID
	}

	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		b.logf("got non-200 code: %d from gerrit api", resp.StatusCode)
		return lastID
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		b.logf("%s\n", err)
		return lastID
	}

	if len(body) < 4 {
		b.logf("got body: %s\n", string(body))
		return lastID
	}

	// Fix Gerrit adding a random prefix )]}'
	body = body[4:]
	cls := []gerritCL{}
	err = json.Unmarshal(body, &cls)
	if err != nil {
		b.logf("%s\n", err)
		return lastID
	}

	foundIdx := len(cls) - 1
	for idx := len(cls) - 1; idx >= 0; idx-- {
		if cls[idx].Number == lastID {
			foundIdx = idx
			break
		}
	}

	pubChannel := b.channels["golang-cls"].slackID
	if pubChannel[:1] == "#" {
		pubChannel = pubChannel[1:]
	}

	for idx := foundIdx - 1; idx >= 0; idx-- {
		cl := cls[idx]
		if cl.Branch != "master" {
			continue
		}

		if shown, err := b.wasShown(cl); err == nil {
			if shown {
				continue
			}
		} else {
			b.logf("got error: %v\n", err)
			continue
		}

		msg := slack.Attachment{
			Title:     cl.Subject,
			TitleLink: cl.link(),
			Text:      cl.Revisions[cl.CurrentRevision].Commit.Message,
			Footer:    cl.ChangeID,
		}
		params := slack.PostMessageParameters{AsUser: true}
		params.Attachments = append(params.Attachments, msg)

		err = b.saveCL(cl)
		if err != nil {
			b.logf("got error while saving CL to datastore: %v", err)
			return lastID
		}

		_, _, err = b.slackBotAPI.PostMessage(b.channels["golang_cls"].slackID, fmt.Sprintf("[%d] %s: %s", cl.Number, cl.message(), cl.link()), params)
		if err != nil {
			b.logf("%s\n", err)
			continue
		}

		lastID = cl.Number

		_, _, err = b.slackBotAPI.PostMessage(pubChannel, fmt.Sprintf("[%d] %s: %s", cl.Number, cl.message(), cl.link()), params)
		if err != nil {
			b.logf("%s\n", err)
			continue
		}
	}

	return lastID
}

func (b *Bot) shareCL(event *slack.MessageEvent, eventText string) {
	if !b.specialRestrictions("golang_cls", event) {
		b.logf("share attempt caught: %#v\n", event)

		params := slack.PostMessageParameters{AsUser: true}
		_, _, err := b.slackBotAPI.PostMessage(event.User, `You are not authorized to share CLs`, params)
		if err != nil {
			b.logf("%s\n", err)
		}
		return
	}

	eventText = strings.Replace(eventText, "share cl", "", -1)
	eventText = strings.Trim(eventText, " \n")

	for _, text := range strings.Fields(eventText) {
		clNumber, err := strconv.ParseInt(text, 10, 64)
		if err != nil {
			b.logf("could not convert string to int: %v from event: %#v\n", err, event)

			params := slack.PostMessageParameters{AsUser: true}
			_, _, err := b.slackBotAPI.PostMessage(event.User, fmt.Sprintf(`Could not share CL %d, please try again`, clNumber), params)
			if err != nil {
				b.logf("%s\n", err)
			}
			continue
		}

		key := datastore.NewKey(b.ctx, "GoCL", "", clNumber, nil)
		query := datastore.NewQuery("GoCL").Ancestor(key)
		key, cl, err := b.getCLFromDS(query)
		if err != nil {
			b.logf("error while retriving CL from the DB: %v\n", err)

			params := slack.PostMessageParameters{AsUser: true}
			_, _, err := b.slackBotAPI.PostMessage(event.User, fmt.Sprintf(`Could not share CL %d, please try again`, clNumber), params)
			if err != nil {
				b.logf("%s\n", err)
			}
			continue
		}

		if cl.Tweeted {
			params := slack.PostMessageParameters{AsUser: true}
			_, _, err := b.slackBotAPI.PostMessage(event.User, fmt.Sprintf(`Already tweeted CL %d`, clNumber), params)
			if err != nil {
				b.logf("%s\n", err)
			}
			continue
		}

		message := cl.Message + " " + cl.URL
		_, err = b.twitterAPI.PostTweet(message, nil)
		if err != nil {
			b.logf("got error while tweeting CL: %d %#v\n", clNumber, err)

			params := slack.PostMessageParameters{AsUser: true}
			_, _, err := b.slackBotAPI.PostMessage(event.User, fmt.Sprintf(`Could not share CL %d, please try again`, clNumber), params)
			if err != nil {
				b.logf("%s\n", err)
			}
			continue
		}

		cl.Tweeted = true
		err = b.updateCL(key, cl)
		if err != nil {
			b.logf("got error while updating CL to datastore: %v", err)

			params := slack.PostMessageParameters{AsUser: true}
			_, _, err := b.slackBotAPI.PostMessage(event.User, fmt.Sprintf(`Could not update tweet status for CL %d in the DB`, clNumber), params)
			if err != nil {
				b.logf("%s\n", err)
			}
		}
	}
}

// MonitorGerrit handles the Gerrit changes
func (b *Bot) MonitorGerrit(duration time.Duration) {
	tk := time.NewTicker(duration)
	defer tk.Stop()

	lastID, err := b.GetLastSeenCL()
	if err != nil {
		b.logf("got error while loading last ID from the datastore: %v\n", err)
		return
	}

	lastID = b.processCLList(lastID)
	for range tk.C {
		lastID = b.processCLList(lastID)
	}
}
<commit_msg>Update bot to fix dependencies changes<commit_after>package bot

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
	"time"

	"cloud.google.com/go/datastore"
	"github.com/nlopes/slack"
	"google.golang.org/api/iterator"
)

type (
	gerritCL struct {
		Project         string `json:"project"`
		ChangeID        string `json:"change_id"`
		Number          int    `json:"_number"`
		Subject         string `json:"subject"`
		Branch          string `json:"branch"`
		CurrentRevision string `json:"current_revision"`
		Revisions       map[string]struct {
			Commit struct {
				Subject string `json:"subject"`
				Message string `json:"message"`
			} `json:"commit"`
		} `json:"revisions"`
	}

	storedCL struct {
		Tweeted   bool      `datastore:"Tweeted,noindex"`
		URL       string    `datastore:"URL,noindex"`
		Message   string    `datastore:"Message,noindex"`
		CrawledAt time.Time `datastore:"CrawledAt"`
	}
)

func (cl *gerritCL) link() string {
	return fmt.Sprintf("https://go-review.googlesource.com/c/%d/", cl.Number)
}

func (cl *gerritCL) message() string {
	subject := cl.Subject
	if cl.Project != "go" {
		subject = fmt.Sprintf("[%s] %s", cl.Project, subject)
	}

	return subject
}

func (b *Bot) getCLFromDS(query *datastore.Query) (*datastore.Key, *storedCL, error) {
	iter := b.dsClient.Run(b.ctx, query)

	dst := &storedCL{}
	key, err := iter.Next(dst)
	if err != nil && err != iterator.Done {
		b.logf("error while fetching history: %v\n", err)
		return nil, nil, err
	}

	return key, dst, nil
}

func (b *Bot) GetLastSeenCL() (int, error) {
	latestCLQuery := datastore.NewQuery("GoCL").
		Order("-CrawledAt").
		Limit(1).
		KeysOnly()

	key, _, err := b.getCLFromDS(latestCLQuery)
	if err != nil {
		return -1, err
	}
	if key == nil {
		return -1, nil
	}
	return int(key.ID), nil
}

func (b *Bot) wasShown(cl gerritCL) (bool, error) {
	key := datastore.IDKey("GoCL", int64(cl.Number), nil)
	query := datastore.NewQuery("GoCL").Ancestor(key)
	key, _, err := b.getCLFromDS(query)
	return key != nil, err
}

func (b *Bot) saveCL(cl gerritCL) error {
	taskKey := datastore.IDKey("GoCL", int64(cl.Number), nil)
	gocl := &storedCL{
		URL:       cl.link(),
		Message:   cl.message(),
		CrawledAt: time.Now(),
	}
	_, err := b.dsClient.Put(b.ctx, taskKey, gocl)
	return err
}

func (b *Bot) updateCL(key *datastore.Key, cl *storedCL) error {
	taskKey := datastore.IDKey("GoCL", key.ID, nil)
	_, err := b.dsClient.Put(b.ctx, taskKey, cl)
	return err
}

func (b *Bot) processCLList(lastID int) int {
	req, err := http.NewRequest("GET", b.gerritLink, nil)
	req.Header.Add("User-Agent", "Gophers Slack bot")
	resp, err := b.client.Do(req)
	if err != nil {
		b.logf("%s\n", err)
		return lastID
	}

	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		b.logf("got non-200 code: %d from gerrit api", resp.StatusCode)
		return lastID
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		b.logf("%s\n", err)
		return lastID
	}

	if len(body) < 4 {
		b.logf("got body: %s\n", string(body))
		return lastID
	}

	// Fix Gerrit adding a random prefix )]}'
	body = body[4:]
	cls := []gerritCL{}
	err = json.Unmarshal(body, &cls)
	if err != nil {
		b.logf("%s\n", err)
		return lastID
	}

	foundIdx := len(cls) - 1
	for idx := len(cls) - 1; idx >= 0; idx-- {
		if cls[idx].Number == lastID {
			foundIdx = idx
			break
		}
	}

	pubChannel := b.channels["golang-cls"].slackID
	if pubChannel[:1] == "#" {
		pubChannel = pubChannel[1:]
	}

	for idx := foundIdx - 1; idx >= 0; idx-- {
		cl := cls[idx]
		if cl.Branch != "master" {
			continue
		}

		if shown, err := b.wasShown(cl); err == nil {
			if shown {
				continue
			}
		} else {
			b.logf("got error: %v\n", err)
			continue
		}

		msg := slack.Attachment{
			Title:     cl.Subject,
			TitleLink: cl.link(),
			Text:      cl.Revisions[cl.CurrentRevision].Commit.Message,
			Footer:    cl.ChangeID,
		}
		params := slack.PostMessageParameters{AsUser: true}
		params.Attachments = append(params.Attachments, msg)

		err = b.saveCL(cl)
		if err != nil {
			b.logf("got error while saving CL to datastore: %v", err)
			return lastID
		}

		_, _, err = b.slackBotAPI.PostMessage(b.channels["golang_cls"].slackID, fmt.Sprintf("[%d] %s: %s", cl.Number, cl.message(), cl.link()), params)
		if err != nil {
			b.logf("%s\n", err)
			continue
		}

		lastID = cl.Number

		_, _, err = b.slackBotAPI.PostMessage(pubChannel, fmt.Sprintf("[%d] %s: %s", cl.Number, cl.message(), cl.link()), params)
		if err != nil {
			b.logf("%s\n", err)
			continue
		}
	}

	return lastID
}

func (b *Bot) shareCL(event *slack.MessageEvent, eventText string) {
	if !b.specialRestrictions("golang_cls", event) {
		b.logf("share attempt caught: %#v\n", event)

		params := slack.PostMessageParameters{AsUser: true}
		_, _, err := b.slackBotAPI.PostMessage(event.User, `You are not authorized to share CLs`, params)
		if err != nil {
			b.logf("%s\n", err)
		}
		return
	}

	eventText = strings.Replace(eventText, "share cl", "", -1)
	eventText = strings.Trim(eventText, " \n")

	for _, text := range strings.Fields(eventText) {
		clNumber, err := strconv.ParseInt(text, 10, 64)
		if err != nil {
			b.logf("could not convert string to int: %v from event: %#v\n", err, event)

			params := slack.PostMessageParameters{AsUser: true}
			_, _, err := b.slackBotAPI.PostMessage(event.User, fmt.Sprintf(`Could not share CL %d, please try again`, clNumber), params)
			if err != nil {
				b.logf("%s\n", err)
			}
			continue
		}

		key := datastore.IDKey("GoCL", clNumber, nil)
		query := datastore.NewQuery("GoCL").Ancestor(key)
		key, cl, err := b.getCLFromDS(query)
		if err != nil {
			b.logf("error while retriving CL from the DB: %v\n", err)

			params := slack.PostMessageParameters{AsUser: true}
			_, _, err := b.slackBotAPI.PostMessage(event.User, fmt.Sprintf(`Could not share CL %d, please try again`, clNumber), params)
			if err != nil {
				b.logf("%s\n", err)
			}
			continue
		}

		if cl.Tweeted {
			params := slack.PostMessageParameters{AsUser: true}
			_, _, err := b.slackBotAPI.PostMessage(event.User, fmt.Sprintf(`Already tweeted CL %d`, clNumber), params)
			if err != nil {
				b.logf("%s\n", err)
			}
			continue
		}

		message := cl.Message + " " + cl.URL
		_, err = b.twitterAPI.PostTweet(message, nil)
		if err != nil {
			b.logf("got error while tweeting CL: %d %#v\n", clNumber, err)

			params := slack.PostMessageParameters{AsUser: true}
			_, _, err := b.slackBotAPI.PostMessage(event.User, fmt.Sprintf(`Could not share CL %d, please try again`, clNumber), params)
			if err != nil {
				b.logf("%s\n", err)
			}
			continue
		}

		cl.Tweeted = true
		err = b.updateCL(key, cl)
		if err != nil {
			b.logf("got error while updating CL to datastore: %v", err)

			params := slack.PostMessageParameters{AsUser: true}
			_, _, err := b.slackBotAPI.PostMessage(event.User, fmt.Sprintf(`Could not update tweet status for CL %d in the DB`, clNumber), params)
			if err != nil {
				b.logf("%s\n", err)
			}
		}
	}
}

// MonitorGerrit handles the Gerrit changes
func (b *Bot) MonitorGerrit(duration time.Duration) {
	tk := time.NewTicker(duration)
	defer tk.Stop()

	lastID, err := b.GetLastSeenCL()
	if err != nil {
		b.logf("got error while loading last ID from the datastore: %v\n", err)
		return
	}

	lastID = b.processCLList(lastID)
	for range tk.C {
		lastID = b.processCLList(lastID)
	}
}
<|endoftext|>


<commit_before>package serialapi

import "fmt"

type FuncGetNodeProtocolInfo struct {
	Listening bool `json:"listening"`
	Routing   bool `json:"routing"`
	Beaming   bool `json:"beaming"`

	Version byte `json:"version"`

	Flirs    bool `json:"flirs"`
	Security bool `json:"security"`
	MaxBaud  int  `json:"max_baud"`

	Basic    byte `json:"basic"`
	Generic  byte `json:"generic"`
	Specific byte `json:"specific"`
}

// NewGetNodeProtocolInfo decodes and cretes a funcGetNodeProtocolInfo. It is sent from the controller and contanins all basic information that the controller knows about a node.
func NewGetNodeProtocolInfo(data []byte) (*FuncGetNodeProtocolInfo, error) {
	pi := &FuncGetNodeProtocolInfo{}

	if len(data) != 6 {
		return nil, fmt.Errorf("wrong length, should be 6 bytes got %d", len(data))
	}

	// Capabilities
	pi.Listening = data[0]&0x80 != 0
	pi.Routing = data[0]&0x40 != 0
	pi.Version = data[0]&0x07 + 1

	pi.MaxBaud = 9600
	if data[0]&0x38 == 0x10 {
		pi.MaxBaud = 40000
	}

	// Security
	pi.Flirs = data[1]&0x60 != 0
	pi.Beaming = data[1]&0x10 != 0
	pi.Security = data[1]&0x01 != 0

	// data[2] - reserved

	pi.Basic = data[3]
	pi.Generic = data[4]
	pi.Specific = data[5]

	return pi, nil
}
<commit_msg>Length was 7 on my fibaro fgs-213<commit_after>package serialapi

import "fmt"

type FuncGetNodeProtocolInfo struct {
	Listening bool `json:"listening"`
	Routing   bool `json:"routing"`
	Beaming   bool `json:"beaming"`

	Version byte `json:"version"`

	Flirs    bool `json:"flirs"`
	Security bool `json:"security"`
	MaxBaud  int  `json:"max_baud"`

	Basic    byte `json:"basic"`
	Generic  byte `json:"generic"`
	Specific byte `json:"specific"`
}

// NewGetNodeProtocolInfo decodes and cretes a funcGetNodeProtocolInfo. It is sent from the controller and contanins all basic information that the controller knows about a node.
func NewGetNodeProtocolInfo(data []byte) (*FuncGetNodeProtocolInfo, error) {
	pi := &FuncGetNodeProtocolInfo{}

	if len(data) < 6 {
		return nil, fmt.Errorf("wrong length, should be at least 6 bytes got len %d data: %x", len(data), data)
	}

	// Capabilities
	pi.Listening = data[0]&0x80 != 0
	pi.Routing = data[0]&0x40 != 0
	pi.Version = data[0]&0x07 + 1

	pi.MaxBaud = 9600
	if data[0]&0x38 == 0x10 {
		pi.MaxBaud = 40000
	}

	// Security
	pi.Flirs = data[1]&0x60 != 0
	pi.Beaming = data[1]&0x10 != 0
	pi.Security = data[1]&0x01 != 0

	// data[2] - reserved

	pi.Basic = data[3]
	pi.Generic = data[4]
	pi.Specific = data[5]

	return pi, nil
}
<|endoftext|>


<commit_before>package lfs_test // avoid import cycle

import (
	"fmt"
	"sort"
	"testing"

	. "github.com/github/git-lfs/lfs"
	"github.com/github/git-lfs/test"
	"github.com/github/git-lfs/vendor/_nuts/github.com/technoweenie/assert"
)

func TestAllCurrentObjectsNone(t *testing.T) {
	repo := test.NewRepo(t)
	repo.Pushd()
	defer func() {
		repo.Popd()
		repo.Cleanup()
	}()

	actual := AllLocalObjects()

	assert.Equal(t, []string{}, actual, "Should be no objects")
}

func TestAllCurrentObjectsSome(t *testing.T) {

	// FFFFFFF why is this creating the same data for every file????????

	repo := test.NewRepo(t)
	repo.Pushd()
	defer func() {
		repo.Popd()
		repo.Cleanup()
	}()

	// We're not testing commits here, just storage, so just create a single
	// commit input with lots of files to generate many oids
	numFiles := 20
	files := make([]*test.FileInput, 0, numFiles)
	for i := 0; i < numFiles; i++ {
		// Must be >=16 bytes for each file to be unique
		files = append(files, &test.FileInput{Filename: fmt.Sprintf("file%d.txt", i), Size: 30})
	}

	inputs := []*test.CommitInput{
		{Files: files},
	}

	outputs := repo.AddCommits(inputs)

	expected := make([]string, 0, numFiles)
	for _, f := range outputs[0].Files {
		expected = append(expected, f.Oid)
	}
	actual := AllLocalObjects()

	// sort to ensure comparison is equal
	sort.Strings(expected)
	sort.Strings(actual)
	assert.Equal(t, expected, actual, "Oids from disk should be the same as in commits")

}
<commit_msg>ンンー ンンンン ンーンン<commit_after>package lfs_test // avoid import cycle

import (
	"fmt"
	"sort"
	"testing"

	. "github.com/github/git-lfs/lfs"
	"github.com/github/git-lfs/test"
	"github.com/github/git-lfs/vendor/_nuts/github.com/technoweenie/assert"
)

func TestAllCurrentObjectsNone(t *testing.T) {
	repo := test.NewRepo(t)
	repo.Pushd()
	defer func() {
		repo.Popd()
		repo.Cleanup()
	}()

	actual := AllLocalObjects()

	assert.Equal(t, []string{}, actual, "Should be no objects")
}

func TestAllCurrentObjectsSome(t *testing.T) {

	repo := test.NewRepo(t)
	repo.Pushd()
	defer func() {
		repo.Popd()
		repo.Cleanup()
	}()

	// We're not testing commits here, just storage, so just create a single
	// commit input with lots of files to generate many oids
	numFiles := 20
	files := make([]*test.FileInput, 0, numFiles)
	for i := 0; i < numFiles; i++ {
		// Must be >=16 bytes for each file to be unique
		files = append(files, &test.FileInput{Filename: fmt.Sprintf("file%d.txt", i), Size: 30})
	}

	inputs := []*test.CommitInput{
		{Files: files},
	}

	outputs := repo.AddCommits(inputs)

	expected := make([]string, 0, numFiles)
	for _, f := range outputs[0].Files {
		expected = append(expected, f.Oid)
	}
	actual := AllLocalObjects()

	// sort to ensure comparison is equal
	sort.Strings(expected)
	sort.Strings(actual)
	assert.Equal(t, expected, actual, "Oids from disk should be the same as in commits")

}
<|endoftext|>


<commit_before>package lfsapi

import (
	"bytes"
	"fmt"
	"os/exec"
	"strings"
	"sync"

	"github.com/git-lfs/git-lfs/config"
	"github.com/rubyist/tracerx"
)

// credsConfig supplies configuration options pertaining to the authorization
// process in package lfsapi.
type credsConfig struct {
	// Cached is a boolean determining whether or not to enable the
	// credential cacher.
	Cached bool `git:"lfs.cachecredentials"`
	// SkipPrompt is a boolean determining whether or not to prompt the user
	// for a password.
	SkipPrompt bool `os:"GIT_TERMINAL_PROMPT"`
}

// getCredentialHelper parses a 'credsConfig' from the git and OS environments,
// returning the appropriate CredentialHelper to authenticate requests with.
//
// It returns an error if any configuration was invalid, or otherwise
// un-useable.
func getCredentialHelper(cfg *config.Configuration) (CredentialHelper, error) {
	ccfg, err := getCredentialConfig(cfg)
	if err != nil {
		return nil, err
	}

	var h CredentialHelper
	h = &commandCredentialHelper{
		SkipPrompt: ccfg.SkipPrompt,
	}

	if ccfg.Cached {
		h = withCredentialCache(h)
	}

	return h, nil
}

// getCredentialConfig parses a *credsConfig given the OS and Git
// configurations.
func getCredentialConfig(cfg *config.Configuration) (*credsConfig, error) {
	var what credsConfig

	if err := cfg.Unmarshal(&what); err != nil {
		return nil, err
	}
	return &what, nil
}

type CredentialHelper interface {
	Fill(Creds) (Creds, error)
	Reject(Creds) error
	Approve(Creds) error
}

type Creds map[string]string

func bufferCreds(c Creds) *bytes.Buffer {
	buf := new(bytes.Buffer)

	for k, v := range c {
		buf.Write([]byte(k))
		buf.Write([]byte("="))
		buf.Write([]byte(v))
		buf.Write([]byte("\n"))
	}

	return buf
}

func withCredentialCache(helper CredentialHelper) CredentialHelper {
	return &credentialCacher{
		cmu:    new(sync.Mutex),
		creds:  make(map[string]Creds),
		helper: helper,
	}
}

type credentialCacher struct {
	// cmu guards creds
	cmu    *sync.Mutex
	creds  map[string]Creds
	helper CredentialHelper
}

func credCacheKey(creds Creds) string {
	parts := []string{
		creds["protocol"],
		creds["host"],
		creds["path"],
	}
	return strings.Join(parts, "//")
}

func (c *credentialCacher) Fill(creds Creds) (Creds, error) {
	key := credCacheKey(creds)

	c.cmu.Lock()
	defer c.cmu.Unlock()

	if cache, ok := c.creds[key]; ok {
		tracerx.Printf("creds: git credential cache (%q, %q, %q)",
			creds["protocol"], creds["host"], creds["path"])
		return cache, nil
	}

	creds, err := c.helper.Fill(creds)
	if err == nil && len(creds["username"]) > 0 && len(creds["password"]) > 0 {
		c.creds[key] = creds
	}
	return creds, err
}

func (c *credentialCacher) Reject(creds Creds) error {
	c.cmu.Lock()
	defer c.cmu.Unlock()

	delete(c.creds, credCacheKey(creds))
	return c.helper.Reject(creds)
}

func (c *credentialCacher) Approve(creds Creds) error {
	err := c.helper.Approve(creds)
	if err == nil {
		c.cmu.Lock()
		c.creds[credCacheKey(creds)] = creds
		c.cmu.Unlock()
	}
	return err
}

type commandCredentialHelper struct {
	SkipPrompt bool
}

func (h *commandCredentialHelper) Fill(creds Creds) (Creds, error) {
	tracerx.Printf("creds: git credential fill (%q, %q, %q)",
		creds["protocol"], creds["host"], creds["path"])
	return h.exec("fill", creds)
}

func (h *commandCredentialHelper) Reject(creds Creds) error {
	_, err := h.exec("reject", creds)
	return err
}

func (h *commandCredentialHelper) Approve(creds Creds) error {
	_, err := h.exec("approve", creds)
	return err
}

func (h *commandCredentialHelper) exec(subcommand string, input Creds) (Creds, error) {
	output := new(bytes.Buffer)
	cmd := exec.Command("git", "credential", subcommand)
	cmd.Stdin = bufferCreds(input)
	cmd.Stdout = output
	/*
	   There is a reason we don't hook up stderr here:
	   Git's credential cache daemon helper does not close its stderr, so if this
	   process is the process that fires up the daemon, it will wait forever
	   (until the daemon exits, really) trying to read from stderr.

	   See https://github.com/git-lfs/git-lfs/issues/117 for more details.
	*/

	err := cmd.Start()
	if err == nil {
		err = cmd.Wait()
	}

	if _, ok := err.(*exec.ExitError); ok {
		if h.SkipPrompt {
			return nil, fmt.Errorf("Change the GIT_TERMINAL_PROMPT env var to be prompted to enter your credentials for %s://%s.",
				input["protocol"], input["host"])
		}

		// 'git credential' exits with 128 if the helper doesn't fill the username
		// and password values.
		if subcommand == "fill" && err.Error() == "exit status 128" {
			return nil, nil
		}
	}

	if err != nil {
		return nil, fmt.Errorf("'git credential %s' error: %s\n", subcommand, err.Error())
	}

	creds := make(Creds)
	for _, line := range strings.Split(output.String(), "\n") {
		pieces := strings.SplitN(line, "=", 2)
		if len(pieces) < 2 || len(pieces[1]) < 1 {
			continue
		}
		creds[pieces[0]] = pieces[1]
	}

	return creds, nil
}
<commit_msg>lfsapi/creds: implement CredentialHelpers type<commit_after>package lfsapi

import (
	"bytes"
	"fmt"
	"os/exec"
	"strings"
	"sync"

	"github.com/git-lfs/git-lfs/config"
	"github.com/rubyist/tracerx"
)

// credsConfig supplies configuration options pertaining to the authorization
// process in package lfsapi.
type credsConfig struct {
	// Cached is a boolean determining whether or not to enable the
	// credential cacher.
	Cached bool `git:"lfs.cachecredentials"`
	// SkipPrompt is a boolean determining whether or not to prompt the user
	// for a password.
	SkipPrompt bool `os:"GIT_TERMINAL_PROMPT"`
}

// getCredentialHelper parses a 'credsConfig' from the git and OS environments,
// returning the appropriate CredentialHelper to authenticate requests with.
//
// It returns an error if any configuration was invalid, or otherwise
// un-useable.
func getCredentialHelper(cfg *config.Configuration) (CredentialHelper, error) {
	ccfg, err := getCredentialConfig(cfg)
	if err != nil {
		return nil, err
	}

	var h CredentialHelper
	h = &commandCredentialHelper{
		SkipPrompt: ccfg.SkipPrompt,
	}

	if ccfg.Cached {
		h = withCredentialCache(h)
	}

	return h, nil
}

// getCredentialConfig parses a *credsConfig given the OS and Git
// configurations.
func getCredentialConfig(cfg *config.Configuration) (*credsConfig, error) {
	var what credsConfig

	if err := cfg.Unmarshal(&what); err != nil {
		return nil, err
	}
	return &what, nil
}

// CredentialHelpers is a []CredentialHelper that iterates through each
// credential helper to fill, reject, or approve credentials.
type CredentialHelpers []CredentialHelper

// Fill implements CredentialHelper.Fill by asking each CredentialHelper in
// order to fill the credentials.
//
// If a fill was successful, it is returned immediately, and no other
// `CredentialHelper`s are consulted. If any CredentialHelper returns an error,
// it is returned immediately.
func (h CredentialHelpers) Fill(what Creds) (Creds, error) {
	for _, c := range h {
		creds, err := c.Fill(what)
		if err != nil {
			return nil, err
		}

		if creds != nil {
			return creds, nil
		}
	}

	return nil, nil
}

// Reject implements CredentialHelper.Reject and rejects the given Creds "what"
// amongst all knonw CredentialHelpers. If any `CredentialHelper`s returned a
// non-nil error, no further `CredentialHelper`s are notified, so as to prevent
// inconsistent state.
func (h CredentialHelpers) Reject(what Creds) error {
	for _, c := range h {
		if err := c.Reject(what); err != nil {
			return err
		}
	}

	return nil
}

// Approve implements CredentialHelper.Approve and approves the given Creds
// "what" amongst all knonw CredentialHelpers. If any `CredentialHelper`s
// returned a non-nil error, no further `CredentialHelper`s are notified, so as
// to prevent inconsistent state.
func (h CredentialHelpers) Approve(what Creds) error {
	for _, c := range h {
		if err := c.Approve(what); err != nil {
			return err
		}
	}

	return nil
}

type CredentialHelper interface {
	Fill(Creds) (Creds, error)
	Reject(Creds) error
	Approve(Creds) error
}

type Creds map[string]string

func bufferCreds(c Creds) *bytes.Buffer {
	buf := new(bytes.Buffer)

	for k, v := range c {
		buf.Write([]byte(k))
		buf.Write([]byte("="))
		buf.Write([]byte(v))
		buf.Write([]byte("\n"))
	}

	return buf
}

func withCredentialCache(helper CredentialHelper) CredentialHelper {
	return &credentialCacher{
		cmu:    new(sync.Mutex),
		creds:  make(map[string]Creds),
		helper: helper,
	}
}

type credentialCacher struct {
	// cmu guards creds
	cmu    *sync.Mutex
	creds  map[string]Creds
	helper CredentialHelper
}

func credCacheKey(creds Creds) string {
	parts := []string{
		creds["protocol"],
		creds["host"],
		creds["path"],
	}
	return strings.Join(parts, "//")
}

func (c *credentialCacher) Fill(creds Creds) (Creds, error) {
	key := credCacheKey(creds)

	c.cmu.Lock()
	defer c.cmu.Unlock()

	if cache, ok := c.creds[key]; ok {
		tracerx.Printf("creds: git credential cache (%q, %q, %q)",
			creds["protocol"], creds["host"], creds["path"])
		return cache, nil
	}

	creds, err := c.helper.Fill(creds)
	if err == nil && len(creds["username"]) > 0 && len(creds["password"]) > 0 {
		c.creds[key] = creds
	}
	return creds, err
}

func (c *credentialCacher) Reject(creds Creds) error {
	c.cmu.Lock()
	defer c.cmu.Unlock()

	delete(c.creds, credCacheKey(creds))
	return c.helper.Reject(creds)
}

func (c *credentialCacher) Approve(creds Creds) error {
	err := c.helper.Approve(creds)
	if err == nil {
		c.cmu.Lock()
		c.creds[credCacheKey(creds)] = creds
		c.cmu.Unlock()
	}
	return err
}

type commandCredentialHelper struct {
	SkipPrompt bool
}

func (h *commandCredentialHelper) Fill(creds Creds) (Creds, error) {
	tracerx.Printf("creds: git credential fill (%q, %q, %q)",
		creds["protocol"], creds["host"], creds["path"])
	return h.exec("fill", creds)
}

func (h *commandCredentialHelper) Reject(creds Creds) error {
	_, err := h.exec("reject", creds)
	return err
}

func (h *commandCredentialHelper) Approve(creds Creds) error {
	_, err := h.exec("approve", creds)
	return err
}

func (h *commandCredentialHelper) exec(subcommand string, input Creds) (Creds, error) {
	output := new(bytes.Buffer)
	cmd := exec.Command("git", "credential", subcommand)
	cmd.Stdin = bufferCreds(input)
	cmd.Stdout = output
	/*
	   There is a reason we don't hook up stderr here:
	   Git's credential cache daemon helper does not close its stderr, so if this
	   process is the process that fires up the daemon, it will wait forever
	   (until the daemon exits, really) trying to read from stderr.

	   See https://github.com/git-lfs/git-lfs/issues/117 for more details.
	*/

	err := cmd.Start()
	if err == nil {
		err = cmd.Wait()
	}

	if _, ok := err.(*exec.ExitError); ok {
		if h.SkipPrompt {
			return nil, fmt.Errorf("Change the GIT_TERMINAL_PROMPT env var to be prompted to enter your credentials for %s://%s.",
				input["protocol"], input["host"])
		}

		// 'git credential' exits with 128 if the helper doesn't fill the username
		// and password values.
		if subcommand == "fill" && err.Error() == "exit status 128" {
			return nil, nil
		}
	}

	if err != nil {
		return nil, fmt.Errorf("'git credential %s' error: %s\n", subcommand, err.Error())
	}

	creds := make(Creds)
	for _, line := range strings.Split(output.String(), "\n") {
		pieces := strings.SplitN(line, "=", 2)
		if len(pieces) < 2 || len(pieces[1]) < 1 {
			continue
		}
		creds[pieces[0]] = pieces[1]
	}

	return creds, nil
}
<|endoftext|>


<commit_before>package main

import (
	"math/cmplx"
	"sort"
)

type Collidable interface {
	// these methods are defined on type Entity
	Position() complex128
	Velocity() complex128
	SetVelocity(complex128)
	Visible() bool
	SetVisible(bool)
	Health() int
	SetHealth(int)

	// implement these methods on the concrete types
	Radius() float64
	Team() int // Objects in the same Team() will not do damage with each other
	BodyDamage() int
	AcquireExperience(int)
	RewardingExperience() int
}

func Collide(a, b Collidable) {
	dist, phi := cmplx.Polar(a.Position() - b.Position())
	rsum := a.Radius() + b.Radius()
	if dist > rsum {
		return
	}
	a.SetVelocity(a.Velocity() + cmplx.Rect((rsum-dist)/rsum, phi))
	b.SetVelocity(b.Velocity() + cmplx.Rect((dist-rsum)/rsum, phi))
	if a.Team() != b.Team() {
		Hit(a, b)
		Hit(b, a)
	}
}

func Hit(a, b Collidable) {
	if a.Visible() {
		hp := a.Health()
		hp -= b.BodyDamage()
		if hp < 0 {
			b.SetVisible(false)
			a.AcquireExperience(b.RewardingExperience())
		}
	}
}

func leftBound(c Collidable) float64 {
	return real(c.Position()) - c.Radius()
}

func rightBound(c Collidable) float64 {
	return real(c.Position()) + c.Radius()
}

type ByX []Collidable

func (a ByX) Len() int           { return len(a) }
func (a ByX) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByX) Less(i, j int) bool { return leftBound(a[i]) < leftBound(a[j]) }

func DoCollision(a []Collidable) {
	sort.Sort(ByX(a))
	var pairs [][2]Collidable
	for i, left := range a {
		for j := i + 1; j < len(a); j++ {
			right := a[j]
			if leftBound(right) < rightBound(left) {
			} else {
				break
			}
		}
	}
	for _, pair := range pairs {
		Collide(pair[0], pair[1])
	}
}
<commit_msg>server: fix collision<commit_after>package main

import (
	"math/cmplx"
	"sort"
)

type Collidable interface {
	// these methods are defined on type Entity
	Position() complex128
	Velocity() complex128
	SetVelocity(complex128)
	Visible() bool
	SetVisible(bool)
	Health() int
	SetHealth(int)

	// implement these methods on the concrete types
	Radius() float64
	Team() int // Objects in the same Team() will not do damage with each other
	BodyDamage() int
	AcquireExperience(int)
	RewardingExperience() int
}

func Collide(a, b Collidable) {
	dist, phi := cmplx.Polar(a.Position() - b.Position())
	rsum := a.Radius() + b.Radius()
	if dist > rsum {
		return
	}
	a.SetVelocity(a.Velocity() + cmplx.Rect((rsum-dist)/rsum, phi))
	b.SetVelocity(b.Velocity() + cmplx.Rect((dist-rsum)/rsum, phi))
	if a.Team() != b.Team() {
		Hit(a, b)
		Hit(b, a)
	}
}

func Hit(a, b Collidable) {
	if a.Visible() {
		hp := a.Health()
		hp -= b.BodyDamage()
		if hp < 0 {
			b.SetVisible(false)
			a.AcquireExperience(b.RewardingExperience())
		}
	}
}

func leftBound(c Collidable) float64 {
	return real(c.Position()) - c.Radius()
}

func rightBound(c Collidable) float64 {
	return real(c.Position()) + c.Radius()
}

func topBound(c Collidable) float64 {
	return imag(c.Position()) + c.Radius()
}

func bottomBound(c Collidable) float64 {
	return imag(c.Position()) - c.Radius()
}

type ByX []Collidable

func (a ByX) Len() int           { return len(a) }
func (a ByX) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByX) Less(i, j int) bool { return leftBound(a[i]) < leftBound(a[j]) }

func DoCollision(a []Collidable) {
	sort.Sort(ByX(a))
	var pairs [][2]Collidable
	for i, left := range a {
		for j := i + 1; j < len(a); j++ {
			right := a[j]
			if leftBound(right) < rightBound(left) {
				if bottomBound(right) < topBound(left) || bottomBound(left) < topBound(right) {
					pairs = append(pairs, [2]Collidable{left, right})
				}
			} else {
				break
			}
		}
	}
	for _, pair := range pairs {
		Collide(pair[0], pair[1])
	}
}
<|endoftext|>


<commit_before>package rand10

import "math/rand"

func rand10() int {
	return 0
}

func rand7() int {
	return rand.Intn(7) + 1
}
<commit_msg>solve 470 use generic solution<commit_after>package rand10

import "math/rand"

func rand10() int {
	return useFormular()
}

// useFormular time complexity O(1), space complexity O(1)
func useFormular() int {
	res := 40
	for res >= 40 {
		res = 7*(rand7()-1) + (rand7() - 1)
	}
	return res%10 + 1
}

func rand7() int {
	return rand.Intn(7) + 1
}
<|endoftext|>


<commit_before>// Author: EmotionalDots @ PTH
//
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strconv"
	"time"

	"github.com/emotionaldots/arbitrage/cmd"
	"github.com/emotionaldots/arbitrage/pkg/arbitrage"
	"github.com/emotionaldots/arbitrage/pkg/arbitrage/torrentinfo"
	"github.com/emotionaldots/arbitrage/pkg/client"
	"github.com/emotionaldots/arbitrage/pkg/model"
)

var bootstrapUrl string

const Usage = `Usage: arbitrage [command] [args...]

Local directory commands:
	lookup [source] [dir]: Find releases with matching hash for directory
	hash   [dir]:          Print hashes for a torrent directory

Tracker API commands:
	download [source:id]          Download a torrent from tracker
	downthemall [source] [dirs]:  Walk through all subdirectories and download matching torrents

Example Usage:
	arbitrage lookup "./Various Artists - The What CD [FLAC]/"
	arbitrage download pth:41950
`

func must(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

func main() {
	app := App{}
	app.Run()
}

type App struct {
	cmd.App
}

func (app *App) Run() {
	app.Init()

	switch flag.Arg(0) {
	case "hash":
		app.Hash()
	case "lookup":
		app.Lookup()
	case "download":
		app.Download()
	case "downthemall":
		app.DownThemAll()
	default:
		fmt.Println(Usage)
	}
}

func (app *App) Hash() {
	dir := flag.Arg(1)
	r, err := arbitrage.FromFile(dir)
	must(err)
	arbitrage.HashDefault(r)
	fmt.Println(r.Hash)
}

func (app *App) Lookup() {
	source := flag.Arg(1)
	dir := flag.Arg(2)
	r, err := arbitrage.FromFile(dir)
	must(err)
	arbitrage.HashDefault(r)

	c := client.New(app.Config.Server, cmd.UserAgent)
	releases, err := c.Query(source, []string{r.Hash})
	must(err)

	for _, other := range releases {
		state := "ok"
		if other.FilePath == "" {
			state = "no_filepath"
		} else if r.FilePath != other.FilePath {
			state = "renamed"
		}
		fmt.Printf("%s %s:%d %q\n", state, source, other.Id, `"`+other.FilePath+`"`)
	}
}

func (app *App) Download() {
	source, id := cmd.ParseSourceId(flag.Arg(1))
	c := app.DoLogin(source)

	torrent, err := c.Download(id)
	must(err)

	name, err := app.GetTorrentName(torrent)
	must(err)
	log.Println(name)

	path := source + "-" + strconv.Itoa(id) + ".torrent"
	must(app.SaveTorrent(torrent, path))
}

func (app *App) GetTorrentName(torrent []byte) (string, error) {
	mi, err := torrentinfo.Load(bytes.NewReader(torrent))
	if err != nil {
		return "", err
	}
	i, err := mi.UnmarshalInfo()
	if err != nil {
		return "", err
	}
	return i.Name, nil
}

func (app *App) SaveTorrent(torrent []byte, path string) error {
	return ioutil.WriteFile(path, torrent, 0644)
}

type job struct {
	LocalDir string
	Hash     string
	Releases []client.Release
}

func (app *App) batchQueryDirectory(dir, source string) chan []job {
	fdir, err := os.Open(dir)
	must(err)
	defer fdir.Close()

	names, err := fdir.Readdirnames(-1)
	must(err)

	queue := make(chan []job, 0)
	c := client.New(app.Config.Server, cmd.UserAgent)

	go func() {
		hashes := make([]string, 0, 100)
		jobs := make([]job, 0, 100)

		doQuery := func() {
			var releases []client.Release
			for i := 0; i < 3; i++ {
				if releases, err = c.Query(source, hashes); err == nil {
					break
				}
				log.Printf("error on try %d/3: %s", i, err)
				time.Sleep(5 * time.Second)
			}
			must(err)

			byHash := make(map[string][]client.Release, 0)
			for _, r := range releases {
				byHash[r.Hash] = append(byHash[r.Hash], r)
			}
			for i, job := range jobs {
				job.Releases = byHash[job.Hash]
				jobs[i] = job
			}

			queue <- jobs
			hashes = make([]string, 0, 100)
			jobs = make([]job, 0, 100)
		}

		for _, n := range names {
			r, err := arbitrage.FromFile(dir + "/" + n)
			must(err)
			arbitrage.HashDefault(r)

			if len(jobs) >= 100 {
				doQuery()
			}
			jobs = append(jobs, job{r.FilePath, r.Hash, nil})
			hashes = append(hashes, r.Hash)
		}
		if len(jobs) > 0 {
			doQuery()
		}
		close(queue)
	}()
	return queue
}

func (app *App) DownThemAll() {
	source := flag.Arg(1)
	dir := flag.Arg(2)

	c := app.DoLogin(source)

	logf, err := os.Create("arbitrage.log")
	must(err)
	defer logf.Close()
	lw := io.MultiWriter(os.Stdout, logf)
	fmt.Fprintf(logf, "#!/usr/bin/env bash\n## arbitrage downthemall %s %q\n\n\n", source, dir)

	for jobs := range app.batchQueryDirectory(dir, source) {
		for _, job := range jobs {
			for _, other := range job.Releases {
				torrent, err := c.Download(int(other.Id))
				if err != nil {
					log.Printf("[%s:%d] Could not download torrent, skipping: %s\n", source, other.Id, err)
					continue
				}

				path, err := app.GetTorrentName(torrent)
				if err != nil {
					log.Printf("[%s:%d] Invalid torrent file, skipping: %s\n", source, other.Id, err)
				}

				status := "ok"
				if job.LocalDir != path {
					status = "renamed"
					fmt.Fprintf(lw, "mv %q %q    # %s:%d\n", job.LocalDir, path, source, other.Id)
				} else {
					fmt.Fprintf(lw, "# ok %s:%d %q\n", source, other.Id, path)
				}

				tfile := fmt.Sprintf("%s-%d-%s.torrent", source, other.Id, status)
				must(app.SaveTorrent(torrent, tfile))

				time.Sleep(200 * time.Millisecond) // Rate-limiting
				break
			}
		}
	}
}

func GroupToInfo(gt model.GroupAndTorrents) arbitrage.InfoRelease {
	g := gt.Group
	t := gt.Torrents[0]

	r := arbitrage.InfoRelease{}
	r.Name = g.Name
	r.TorrentId = t.ID
	r.FilePath = t.FilePath
	r.Tags = g.Tags
	r.Description = g.WikiBody
	r.Image = g.WikiImage

	r.Format = t.Media + " / " + t.Format
	if t.HasLog {
		r.Format += " / " + strconv.Itoa(t.LogScore)
	}

	if t.Remastered {
		r.Year = t.RemasterYear
		r.RecordLabel = t.RemasterRecordLabel
		r.CatalogueNumber = t.RemasterCatalogueNumber
		r.Edition = t.RemasterTitle
	} else {
		r.Year = g.Year
		r.RecordLabel = g.RecordLabel
		r.CatalogueNumber = g.CatalogueNumber
		r.Edition = "Original Release"
	}

	for _, a := range g.MusicInfo.Composers {
		r.Composers = append(r.Composers, a.Name)
	}
	for _, a := range g.MusicInfo.Artists {
		r.Artists = append(r.Artists, a.Name)
	}
	for _, a := range g.MusicInfo.With {
		r.With = append(r.With, a.Name)
	}
	for _, a := range g.MusicInfo.DJ {
		r.DJ = append(r.DJ, a.Name)
	}
	for _, a := range g.MusicInfo.RemixedBy {
		r.RemixedBy = append(r.RemixedBy, a.Name)
	}
	for _, a := range g.MusicInfo.Producer {
		r.Producer = append(r.Producer, a.Name)
	}

	return r
}
<commit_msg>fixed quoted path in lookup<commit_after>// Author: EmotionalDots @ PTH
//
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strconv"
	"time"

	"github.com/emotionaldots/arbitrage/cmd"
	"github.com/emotionaldots/arbitrage/pkg/arbitrage"
	"github.com/emotionaldots/arbitrage/pkg/arbitrage/torrentinfo"
	"github.com/emotionaldots/arbitrage/pkg/client"
	"github.com/emotionaldots/arbitrage/pkg/model"
)

var bootstrapUrl string

const Usage = `Usage: arbitrage [command] [args...]

Local directory commands:
	lookup [source] [dir]: Find releases with matching hash for directory
	hash   [dir]:          Print hashes for a torrent directory

Tracker API commands:
	download [source:id]          Download a torrent from tracker
	downthemall [source] [dirs]:  Walk through all subdirectories and download matching torrents

Example Usage:
	arbitrage lookup "./Various Artists - The What CD [FLAC]/"
	arbitrage download pth:41950
`

func must(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

func main() {
	app := App{}
	app.Run()
}

type App struct {
	cmd.App
}

func (app *App) Run() {
	app.Init()

	switch flag.Arg(0) {
	case "hash":
		app.Hash()
	case "lookup":
		app.Lookup()
	case "download":
		app.Download()
	case "downthemall":
		app.DownThemAll()
	default:
		fmt.Println(Usage)
	}
}

func (app *App) Hash() {
	dir := flag.Arg(1)
	r, err := arbitrage.FromFile(dir)
	must(err)
	arbitrage.HashDefault(r)
	fmt.Println(r.Hash)
}

func (app *App) Lookup() {
	source := flag.Arg(1)
	dir := flag.Arg(2)
	r, err := arbitrage.FromFile(dir)
	must(err)
	arbitrage.HashDefault(r)

	c := client.New(app.Config.Server, cmd.UserAgent)
	releases, err := c.Query(source, []string{r.Hash})
	must(err)

	for _, other := range releases {
		state := "ok"
		if other.FilePath == "" {
			state = "no_filepath"
		} else if r.FilePath != other.FilePath {
			state = "renamed"
		}
		fmt.Printf("%s %s:%d %q\n", state, source, other.Id, other.FilePath)
	}
}

func (app *App) Download() {
	source, id := cmd.ParseSourceId(flag.Arg(1))
	c := app.DoLogin(source)

	torrent, err := c.Download(id)
	must(err)

	name, err := app.GetTorrentName(torrent)
	must(err)
	log.Println(name)

	path := source + "-" + strconv.Itoa(id) + ".torrent"
	must(app.SaveTorrent(torrent, path))
}

func (app *App) GetTorrentName(torrent []byte) (string, error) {
	mi, err := torrentinfo.Load(bytes.NewReader(torrent))
	if err != nil {
		return "", err
	}
	i, err := mi.UnmarshalInfo()
	if err != nil {
		return "", err
	}
	return i.Name, nil
}

func (app *App) SaveTorrent(torrent []byte, path string) error {
	return ioutil.WriteFile(path, torrent, 0644)
}

type job struct {
	LocalDir string
	Hash     string
	Releases []client.Release
}

func (app *App) batchQueryDirectory(dir, source string) chan []job {
	fdir, err := os.Open(dir)
	must(err)
	defer fdir.Close()

	names, err := fdir.Readdirnames(-1)
	must(err)

	queue := make(chan []job, 0)
	c := client.New(app.Config.Server, cmd.UserAgent)

	go func() {
		hashes := make([]string, 0, 100)
		jobs := make([]job, 0, 100)

		doQuery := func() {
			var releases []client.Release
			for i := 0; i < 3; i++ {
				if releases, err = c.Query(source, hashes); err == nil {
					break
				}
				log.Printf("error on try %d/3: %s", i, err)
				time.Sleep(5 * time.Second)
			}
			must(err)

			byHash := make(map[string][]client.Release, 0)
			for _, r := range releases {
				byHash[r.Hash] = append(byHash[r.Hash], r)
			}
			for i, job := range jobs {
				job.Releases = byHash[job.Hash]
				jobs[i] = job
			}

			queue <- jobs
			hashes = make([]string, 0, 100)
			jobs = make([]job, 0, 100)
		}

		for _, n := range names {
			r, err := arbitrage.FromFile(dir + "/" + n)
			must(err)
			arbitrage.HashDefault(r)

			if len(jobs) >= 100 {
				doQuery()
			}
			jobs = append(jobs, job{r.FilePath, r.Hash, nil})
			hashes = append(hashes, r.Hash)
		}
		if len(jobs) > 0 {
			doQuery()
		}
		close(queue)
	}()
	return queue
}

func (app *App) DownThemAll() {
	source := flag.Arg(1)
	dir := flag.Arg(2)

	c := app.DoLogin(source)

	logf, err := os.Create("arbitrage.log")
	must(err)
	defer logf.Close()
	lw := io.MultiWriter(os.Stdout, logf)
	fmt.Fprintf(logf, "#!/usr/bin/env bash\n## arbitrage downthemall %s %q\n\n\n", source, dir)

	for jobs := range app.batchQueryDirectory(dir, source) {
		for _, job := range jobs {
			for _, other := range job.Releases {
				torrent, err := c.Download(int(other.Id))
				if err != nil {
					log.Printf("[%s:%d] Could not download torrent, skipping: %s\n", source, other.Id, err)
					continue
				}

				path, err := app.GetTorrentName(torrent)
				if err != nil {
					log.Printf("[%s:%d] Invalid torrent file, skipping: %s\n", source, other.Id, err)
				}

				status := "ok"
				if job.LocalDir != path {
					status = "renamed"
					fmt.Fprintf(lw, "mv %q %q    # %s:%d\n", job.LocalDir, path, source, other.Id)
				} else {
					fmt.Fprintf(lw, "# ok %s:%d %q\n", source, other.Id, path)
				}

				tfile := fmt.Sprintf("%s-%d-%s.torrent", source, other.Id, status)
				must(app.SaveTorrent(torrent, tfile))

				time.Sleep(200 * time.Millisecond) // Rate-limiting
				break
			}
		}
	}
}

func GroupToInfo(gt model.GroupAndTorrents) arbitrage.InfoRelease {
	g := gt.Group
	t := gt.Torrents[0]

	r := arbitrage.InfoRelease{}
	r.Name = g.Name
	r.TorrentId = t.ID
	r.FilePath = t.FilePath
	r.Tags = g.Tags
	r.Description = g.WikiBody
	r.Image = g.WikiImage

	r.Format = t.Media + " / " + t.Format
	if t.HasLog {
		r.Format += " / " + strconv.Itoa(t.LogScore)
	}

	if t.Remastered {
		r.Year = t.RemasterYear
		r.RecordLabel = t.RemasterRecordLabel
		r.CatalogueNumber = t.RemasterCatalogueNumber
		r.Edition = t.RemasterTitle
	} else {
		r.Year = g.Year
		r.RecordLabel = g.RecordLabel
		r.CatalogueNumber = g.CatalogueNumber
		r.Edition = "Original Release"
	}

	for _, a := range g.MusicInfo.Composers {
		r.Composers = append(r.Composers, a.Name)
	}
	for _, a := range g.MusicInfo.Artists {
		r.Artists = append(r.Artists, a.Name)
	}
	for _, a := range g.MusicInfo.With {
		r.With = append(r.With, a.Name)
	}
	for _, a := range g.MusicInfo.DJ {
		r.DJ = append(r.DJ, a.Name)
	}
	for _, a := range g.MusicInfo.RemixedBy {
		r.RemixedBy = append(r.RemixedBy, a.Name)
	}
	for _, a := range g.MusicInfo.Producer {
		r.Producer = append(r.Producer, a.Name)
	}

	return r
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"expvar"
	"flag"
	"log"
	"net"
	"net/http"
	"net/http/pprof"
	"sync"

	"github.com/gorilla/handlers"
	"github.com/lomik/zapwriter"
	"go.uber.org/zap"

	"github.com/go-graphite/carbonapi/cmd/carbonapi/config"
	"github.com/go-graphite/carbonapi/cmd/carbonapi/helper"
	carbonapiHttp "github.com/go-graphite/carbonapi/cmd/carbonapi/http"
	"github.com/go-graphite/carbonapi/internal/dns"
)

// BuildVersion is provided to be overridden at build time. Eg. go build -ldflags -X 'main.BuildVersion=...'
var BuildVersion = "(development build)"

func main() {
	err := zapwriter.ApplyConfig([]zapwriter.Config{config.DefaultLoggerConfig})
	if err != nil {
		log.Fatal("Failed to initialize logger with default configuration")
	}
	logger := zapwriter.Logger("main")

	configPath := flag.String("config", "", "Path to the `config file`.")
	envPrefix := flag.String("envprefix", "CARBONAPI", "Prefix for environment variables override")
	if *envPrefix == "(empty)" {
		*envPrefix = ""
	}
	if *envPrefix == "" {
		logger.Warn("empty prefix is not recommended due to possible collisions with OS environment variables")
	}
	flag.Parse()
	config.SetUpViper(logger, configPath, *envPrefix)
	config.SetUpConfigUpstreams(logger)
	config.SetUpConfig(logger, BuildVersion)
	carbonapiHttp.SetupMetrics(logger)
	setupGraphiteMetrics(logger)

	if config.Config.UseCachingDNSResolver {
		logger.Info("will use custom caching dns resolver")
		dns.UseDNSCache(config.Config.CachingDNSRefreshTime)
	}

	config.Config.ZipperInstance = newZipper(carbonapiHttp.ZipperStats, &config.Config.Upstreams, config.Config.IgnoreClientTimeout, zapwriter.Logger("zipper"))

	wg := sync.WaitGroup{}
	serve := func(listen config.Listener, handler http.Handler) {
		l := &net.ListenConfig{Control: helper.ReusePort}
		h, p, err := net.SplitHostPort(listen.Address)
		if err != nil {
			logger.Fatal("failed to split adress",
				zap.Error(err),
			)
		}
		ips, err := net.LookupIP(h)
		if err != nil {
			logger.Fatal("failed to resolve adress",
				zap.Error(err),
			)
		}
		// Resolve named ports
		port, err := net.LookupPort("tcp", p)
		if err != nil {
			logger.Fatal("failed to resolve port",
				zap.Error(err),
			)
		}
		for _, ip := range ips {
			address := (&net.TCPAddr{IP: ip, Port: port}).String()
			s := &http.Server{
				Addr:    address,
				Handler: handler,
			}
			listener, err := l.Listen(context.Background(), "tcp", address)
			if err != nil {
				logger.Fatal("failed to start http server",
					zap.Error(err),
				)
			}
			wg.Add(1)
			go func() {
				err = s.Serve(listener)

				if err != nil {
					logger.Fatal("failed to start http server",
						zap.Error(err),
					)
				}

				wg.Done()
			}()
		}
	}

	if config.Config.Expvar.Enabled {
		if config.Config.Expvar.Listen != "" && config.Config.Expvar.Listen != config.Config.Listeners[0].Address {
			r := http.NewServeMux()
			r.HandleFunc(config.Config.Prefix+"/debug/vars", expvar.Handler().ServeHTTP)
			if config.Config.Expvar.PProfEnabled {
				r.HandleFunc(config.Config.Prefix+"/debug/pprof/", pprof.Index)
				r.HandleFunc(config.Config.Prefix+"/debug/pprof/cmdline", pprof.Cmdline)
				r.HandleFunc(config.Config.Prefix+"/debug/pprof/profile", pprof.Profile)
				r.HandleFunc(config.Config.Prefix+"/debug/pprof/symbol", pprof.Symbol)
				r.HandleFunc(config.Config.Prefix+"/debug/pprof/trace", pprof.Trace)
			}

			handler := handlers.CompressHandler(r)
			handler = handlers.CORS()(handler)
			handler = handlers.ProxyHeaders(handler)

			logger.Info("expvar handler will listen on a separate address/port",
				zap.String("expvar_listen", config.Config.Expvar.Listen),
				zap.Bool("pprof_enabled", config.Config.Expvar.PProfEnabled),
			)

			listener := config.Listener{
				Address: config.Config.Expvar.Listen,
			}
			serve(listener, handler)
		}
	}

	r := carbonapiHttp.InitHandlers(config.Config.HeadersToPass, config.Config.HeadersToLog)
	handler := handlers.CompressHandler(r)
	handler = handlers.CORS()(handler)
	handler = handlers.ProxyHeaders(handler)

	for _, listener := range config.Config.Listeners {
		serve(listener, handler)
	}

	wg.Wait()
}
<commit_msg>Improve listening on `any` ip addresses<commit_after>package main

import (
	"context"
	"expvar"
	"flag"
	"log"
	"net"
	"net/http"
	"net/http/pprof"
	"sync"

	"github.com/gorilla/handlers"
	"github.com/lomik/zapwriter"
	"go.uber.org/zap"

	"github.com/go-graphite/carbonapi/cmd/carbonapi/config"
	"github.com/go-graphite/carbonapi/cmd/carbonapi/helper"
	carbonapiHttp "github.com/go-graphite/carbonapi/cmd/carbonapi/http"
	"github.com/go-graphite/carbonapi/internal/dns"
)

// BuildVersion is provided to be overridden at build time. Eg. go build -ldflags -X 'main.BuildVersion=...'
var BuildVersion = "(development build)"

func main() {
	err := zapwriter.ApplyConfig([]zapwriter.Config{config.DefaultLoggerConfig})
	if err != nil {
		log.Fatal("Failed to initialize logger with default configuration")
	}
	logger := zapwriter.Logger("main")

	configPath := flag.String("config", "", "Path to the `config file`.")
	envPrefix := flag.String("envprefix", "CARBONAPI", "Prefix for environment variables override")
	if *envPrefix == "(empty)" {
		*envPrefix = ""
	}
	if *envPrefix == "" {
		logger.Warn("empty prefix is not recommended due to possible collisions with OS environment variables")
	}
	flag.Parse()
	config.SetUpViper(logger, configPath, *envPrefix)
	config.SetUpConfigUpstreams(logger)
	config.SetUpConfig(logger, BuildVersion)
	carbonapiHttp.SetupMetrics(logger)
	setupGraphiteMetrics(logger)

	if config.Config.UseCachingDNSResolver {
		logger.Info("will use custom caching dns resolver")
		dns.UseDNSCache(config.Config.CachingDNSRefreshTime)
	}

	config.Config.ZipperInstance = newZipper(carbonapiHttp.ZipperStats, &config.Config.Upstreams, config.Config.IgnoreClientTimeout, zapwriter.Logger("zipper"))

	wg := sync.WaitGroup{}
	serve := func(listen config.Listener, handler http.Handler) {
		l := &net.ListenConfig{Control: helper.ReusePort}
		h, p, err := net.SplitHostPort(listen.Address)
		if err != nil {
			logger.Fatal("failed to split address",
				zap.String("address", listen.Address),
				zap.Error(err),
			)
		}
		any := false
		if h == "" {
			h = "[::]"
			any = true
		}
		ips, err := net.LookupIP(h)
		if err != nil {
			// Fallback for a case where machine doesn't have ipv6 at all
			if any {
				h = "0.0.0.0"
				ips, err = net.LookupIP(h)
			}
			if err != nil {
				logger.Fatal("failed to resolve address",
					zap.String("address", h),
					zap.String("port", p),
					zap.Error(err),
				)
			}
		}
		// Resolve named ports
		port, err := net.LookupPort("tcp", p)
		if err != nil {
			logger.Fatal("failed to resolve port",
				zap.String("address", h),
				zap.String("port", p),
				zap.Error(err),
			)
		}
		for _, ip := range ips {
			address := (&net.TCPAddr{IP: ip, Port: port}).String()
			s := &http.Server{
				Addr:    address,
				Handler: handler,
			}
			listener, err := l.Listen(context.Background(), "tcp", address)
			if err != nil {
				logger.Fatal("failed to start http server",
					zap.Error(err),
				)
			}
			wg.Add(1)
			go func() {
				err = s.Serve(listener)

				if err != nil {
					logger.Fatal("failed to start http server",
						zap.Error(err),
					)
				}

				wg.Done()
			}()
		}
	}

	if config.Config.Expvar.Enabled {
		if config.Config.Expvar.Listen != "" && config.Config.Expvar.Listen != config.Config.Listeners[0].Address {
			r := http.NewServeMux()
			r.HandleFunc(config.Config.Prefix+"/debug/vars", expvar.Handler().ServeHTTP)
			if config.Config.Expvar.PProfEnabled {
				r.HandleFunc(config.Config.Prefix+"/debug/pprof/", pprof.Index)
				r.HandleFunc(config.Config.Prefix+"/debug/pprof/cmdline", pprof.Cmdline)
				r.HandleFunc(config.Config.Prefix+"/debug/pprof/profile", pprof.Profile)
				r.HandleFunc(config.Config.Prefix+"/debug/pprof/symbol", pprof.Symbol)
				r.HandleFunc(config.Config.Prefix+"/debug/pprof/trace", pprof.Trace)
			}

			handler := handlers.CompressHandler(r)
			handler = handlers.CORS()(handler)
			handler = handlers.ProxyHeaders(handler)

			logger.Info("expvar handler will listen on a separate address/port",
				zap.String("expvar_listen", config.Config.Expvar.Listen),
				zap.Bool("pprof_enabled", config.Config.Expvar.PProfEnabled),
			)

			listener := config.Listener{
				Address: config.Config.Expvar.Listen,
			}
			serve(listener, handler)
		}
	}

	r := carbonapiHttp.InitHandlers(config.Config.HeadersToPass, config.Config.HeadersToLog)
	handler := handlers.CompressHandler(r)
	handler = handlers.CORS()(handler)
	handler = handlers.ProxyHeaders(handler)

	for _, listener := range config.Config.Listeners {
		serve(listener, handler)
	}

	wg.Wait()
}
<|endoftext|>