<commit_before>//  Copyright (c) 2018 Rackspace
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
//  implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package objectserver

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/httptest"
	"net/url"
	"os"
	"strconv"
	"testing"
	"time"

	"github.com/stretchr/testify/require"
	"github.com/troubling/hummingbird/common"
	"github.com/troubling/hummingbird/common/conf"
	"github.com/troubling/hummingbird/common/ring"
	"github.com/troubling/hummingbird/common/srv"
	"github.com/troubling/hummingbird/common/test"
	"go.uber.org/zap"
)

func TestEcEngineConstructor(t *testing.T) {
	configString := "[app:object-server]\nmount_check=false\n"
	pol := conf.Policy{Index: 0, Type: "hec", Name: "gold",
		Aliases: []string{}, Default: true, Deprecated: false,
		Config: map[string]string{"policy_type": "hec",
			"default": "yes", "name": "gold", "data_shards": "2",
			"parity_shards": "1"}}
	config, _ := conf.StringConfig(configString)
	_, err := ecEngineConstructor(
		config, &pol, &flag.FlagSet{})
	require.Nil(t, err)
}

func getTestEce() (*ecEngine, string, error) {
	driveRoot, err := ioutil.TempDir("", "")
	if err != nil {
		return nil, "", err
	}
	defer os.RemoveAll(driveRoot)
	policy := conf.Policy{Index: 0, Type: "hec", Name: "gold",
		Aliases: []string{}, Default: true, Deprecated: false,
		Config: map[string]string{"policy_type": "hec",
			"default": "yes", "name": "gold", "data_shards": "2",
			"parity_shards": "1"}}
	testRing := &test.FakeRing{}
	transport := &http.Transport{
		MaxIdleConnsPerHost: 256,
		MaxIdleConns:        0,
		IdleConnTimeout:     5 * time.Second,
		DisableCompression:  true,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 5 * time.Second,
		}).Dial,
		ExpectContinueTimeout: 10 * time.Minute,
	}
	logger, _ := zap.NewProduction()
	engine := &ecEngine{
		driveRoot:      driveRoot,
		hashPathPrefix: "a",
		hashPathSuffix: "b",
		reserve:        0,
		policy:         0,
		logger:         logger,
		ring:           testRing,
		idbs:           map[string]*IndexDB{},
		dbPartPower:    1,
		numSubDirs:     32,
		client: &http.Client{
			Timeout:   time.Second,
			Transport: transport,
		},
	}
	if engine.dataShards, err = strconv.Atoi(policy.Config["data_shards"]); err != nil {
		return nil, driveRoot, err
	}
	if engine.parityShards, err = strconv.Atoi(policy.Config["parity_shards"]); err != nil {
		return nil, driveRoot, err
	}
	if engine.chunkSize, err = strconv.Atoi(policy.Config["chunk_size"]); err != nil {
		engine.chunkSize = 1 << 20
	}
	return engine, driveRoot, nil
}

func TestGetObjectsToReplicate(t *testing.T) {
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		remoteItems := []*IndexDBItem{}
		d, err := json.Marshal(remoteItems)
		require.Nil(t, err)
		w.WriteHeader(200)
		w.Write(d)
	}))
	u, err := url.Parse(ts.URL)
	require.Nil(t, err)
	host, ports, err := net.SplitHostPort(u.Host)
	require.Nil(t, err)
	port, err := strconv.Atoi(ports)
	require.Nil(t, err)

	ece, dr, err := getTestEce()
	if dr != "" {
		defer os.RemoveAll(dr)
	}
	require.Nil(t, err)
	idb, err := ece.getDB("sdb1")
	require.Nil(t, err)

	timestamp := time.Now().UnixNano()
	body := "just testing"
	hsh0 := md5hash("object0")
	partition, _ := strconv.ParseInt(hsh0[:2], 16, 64)
	partition >>= 2
	f, err := idb.TempFile(hsh0, 0, timestamp, int64(len(body)), true)
	require.Nil(t, err)
	f.Write([]byte(body))
	metadata, err := json.Marshal(map[string]string{"moo": "cow"})
	require.Nil(t, err)
	require.Nil(t, idb.Commit(f, hsh0, 0, timestamp, "PUT", "", metadata, false, ""))
	osc := make(chan ObjectStabilizer)
	cancel := make(chan struct{})
	defer close(cancel)
	go ece.GetObjectsToReplicate(
		PriorityRepJob{
			Partition:  uint64(partition),
			FromDevice: &ring.Device{Device: "sdb1"},
			ToDevice:   &ring.Device{Device: "sdb2", Scheme: "http", Port: port, Ip: host},
		}, osc, cancel)
	os := <-osc
	require.Equal(t, "cow", os.Metadata()["moo"])
	os = <-osc
	require.Nil(t, os)
}

func TestGetObjectsToReplicateRemoteHasAll(t *testing.T) {
	ece, dr, err := getTestEce()
	if dr != "" {
		defer os.RemoveAll(dr)
	}
	require.Nil(t, err)
	idb, err := ece.getDB("sdb1")
	require.Nil(t, err)

	timestamp := time.Now().UnixNano()
	body := "just testing"
	hsh0 := "00000000000000000000000000000001"
	f, err := idb.TempFile(hsh0, 0, timestamp, int64(len(body)), true)
	require.Nil(t, err)
	f.Write([]byte(body))
	metadata, err := json.Marshal(map[string]string{"name": "o1"})
	require.Nil(t, err)
	require.Nil(t, idb.Commit(f, hsh0, 0, timestamp, "PUT", "", metadata, false, ""))
	hsh1 := "00000000000000000000000000000002"
	metadata, err = json.Marshal(map[string]string{"name": "o2"})
	require.Nil(t, err)
	f, err = idb.TempFile(hsh1, 0, timestamp, int64(len(body)), true)
	require.Nil(t, err)
	f.Write([]byte(body))
	require.Nil(t, idb.Commit(f, hsh1, 0, timestamp, "PUT", "", metadata, false, ""))
	remoteItems, err := idb.List("", "", "", 0)
	require.Nil(t, err)
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		d, err := json.Marshal(remoteItems)
		require.Nil(t, err)
		w.WriteHeader(200)
		w.Write(d)
	}))
	u, err := url.Parse(ts.URL)
	require.Nil(t, err)
	host, ports, err := net.SplitHostPort(u.Host)
	require.Nil(t, err)
	port, err := strconv.Atoi(ports)
	require.Nil(t, err)

	osc := make(chan ObjectStabilizer)
	cancel := make(chan struct{})
	defer close(cancel)
	go ece.GetObjectsToReplicate(
		PriorityRepJob{FromDevice: &ring.Device{Device: "sdb1"},
			ToDevice: &ring.Device{Device: "sdb2", Scheme: "http", Port: port, Ip: host}}, osc, cancel)
	os := <-osc
	require.Nil(t, os)
}

func TestGetObjectsToReplicateRemoteHasSome(t *testing.T) {
	ece, dr, err := getTestEce()
	if dr != "" {
		defer os.RemoveAll(dr)
	}
	require.Nil(t, err)
	idb, err := ece.getDB("sdb1")
	require.Nil(t, err)

	timestamp := time.Now().UnixNano()
	body := "just testing"
	hsh0 := "00000000000000000000000000000001"
	f, err := idb.TempFile(hsh0, 0, timestamp, int64(len(body)), true)
	require.Nil(t, err)
	f.Write([]byte(body))
	metadata, err := json.Marshal(map[string]string{"name": "o1"})
	require.Nil(t, err)
	require.Nil(t, idb.Commit(f, hsh0, 0, timestamp, "PUT", "", metadata, false, ""))
	hsh1 := "00000000000000000000000000000002"
	metadata, err = json.Marshal(map[string]string{"name": "o2"})
	require.Nil(t, err)
	f, err = idb.TempFile(hsh1, 0, timestamp, int64(len(body)), true)
	require.Nil(t, err)
	f.Write([]byte(body))
	require.Nil(t, idb.Commit(f, hsh1, 0, timestamp, "PUT", "", metadata, false, ""))
	remoteItems, err := idb.List("", "", "", 0)
	require.Nil(t, err)
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		newRemoteItems := []*IndexDBItem{remoteItems[0]}
		d, err := json.Marshal(newRemoteItems)
		require.Nil(t, err)
		w.WriteHeader(200)
		w.Write(d)
	}))
	u, err := url.Parse(ts.URL)
	require.Nil(t, err)
	host, ports, err := net.SplitHostPort(u.Host)
	require.Nil(t, err)
	port, err := strconv.Atoi(ports)
	require.Nil(t, err)

	osc := make(chan ObjectStabilizer)
	cancel := make(chan struct{})
	defer close(cancel)
	go ece.GetObjectsToReplicate(
		PriorityRepJob{FromDevice: &ring.Device{Device: "sdb1"},
			ToDevice: &ring.Device{Device: "sdb2", Scheme: "http", Port: port, Ip: host}}, osc, cancel)
	os := <-osc
	require.Equal(t, "o2", os.Metadata()["name"])
	os = <-osc
	require.Nil(t, os)
}

func TestEcShardDelete(t *testing.T) {
	ece, err := getTestEce()
	require.Nil(t, err)
	idb, err := ece.getDB("sdb1")
	require.Nil(t, err)

	timestamp := time.Now().UnixNano()
	body := "just testing"
	hsh0 := "00000000000000000000000000000001"
	f, err := idb.TempFile(hsh0, 0, timestamp, int64(len(body)), true)
	require.Nil(t, err)
	f.Write([]byte(body))
	metadata, err := json.Marshal(map[string]string{"name": "o1"})
	require.Nil(t, err)
	require.Nil(t, idb.Commit(f, hsh0, 0, timestamp, "PUT", "", metadata, false, ""))
	req, _ := http.NewRequest("DELETE", fmt.Sprintf("/ec-shard/sdb1/%s/0", hsh0), nil)
	req.Header.Set("X-Timestamp",
		common.CanonicalTimestampFromTime(time.Now().AddDate(0, 0, -1)))
	req = srv.SetVars(req, map[string]string{"index": "0", "device": "sdb1", "hash": hsh0})
	w := &httptest.ResponseRecorder{}
	ece.ecShardDeleteHandler(w, req)
	resp := w.Result()
	require.Equal(t, 409, resp.StatusCode)
	req.Header.Set("X-Timestamp",
		common.CanonicalTimestampFromTime(time.Now().AddDate(0, 0, 1)))
	w = &httptest.ResponseRecorder{}
	ece.ecShardDeleteHandler(w, req)
	resp = w.Result()
	require.Equal(t, 204, resp.StatusCode)
}
<commit_msg>HUM-805 Fix test due to getTestEce change<commit_after>//  Copyright (c) 2018 Rackspace
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
//  implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package objectserver

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/httptest"
	"net/url"
	"os"
	"strconv"
	"testing"
	"time"

	"github.com/stretchr/testify/require"
	"github.com/troubling/hummingbird/common"
	"github.com/troubling/hummingbird/common/conf"
	"github.com/troubling/hummingbird/common/ring"
	"github.com/troubling/hummingbird/common/srv"
	"github.com/troubling/hummingbird/common/test"
	"go.uber.org/zap"
)

func TestEcEngineConstructor(t *testing.T) {
	configString := "[app:object-server]\nmount_check=false\n"
	pol := conf.Policy{Index: 0, Type: "hec", Name: "gold",
		Aliases: []string{}, Default: true, Deprecated: false,
		Config: map[string]string{"policy_type": "hec",
			"default": "yes", "name": "gold", "data_shards": "2",
			"parity_shards": "1"}}
	config, _ := conf.StringConfig(configString)
	_, err := ecEngineConstructor(
		config, &pol, &flag.FlagSet{})
	require.Nil(t, err)
}

func getTestEce() (*ecEngine, string, error) {
	driveRoot, err := ioutil.TempDir("", "")
	if err != nil {
		return nil, "", err
	}
	defer os.RemoveAll(driveRoot)
	policy := conf.Policy{Index: 0, Type: "hec", Name: "gold",
		Aliases: []string{}, Default: true, Deprecated: false,
		Config: map[string]string{"policy_type": "hec",
			"default": "yes", "name": "gold", "data_shards": "2",
			"parity_shards": "1"}}
	testRing := &test.FakeRing{}
	transport := &http.Transport{
		MaxIdleConnsPerHost: 256,
		MaxIdleConns:        0,
		IdleConnTimeout:     5 * time.Second,
		DisableCompression:  true,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 5 * time.Second,
		}).Dial,
		ExpectContinueTimeout: 10 * time.Minute,
	}
	logger, _ := zap.NewProduction()
	engine := &ecEngine{
		driveRoot:      driveRoot,
		hashPathPrefix: "a",
		hashPathSuffix: "b",
		reserve:        0,
		policy:         0,
		logger:         logger,
		ring:           testRing,
		idbs:           map[string]*IndexDB{},
		dbPartPower:    1,
		numSubDirs:     32,
		client: &http.Client{
			Timeout:   time.Second,
			Transport: transport,
		},
	}
	if engine.dataShards, err = strconv.Atoi(policy.Config["data_shards"]); err != nil {
		return nil, driveRoot, err
	}
	if engine.parityShards, err = strconv.Atoi(policy.Config["parity_shards"]); err != nil {
		return nil, driveRoot, err
	}
	if engine.chunkSize, err = strconv.Atoi(policy.Config["chunk_size"]); err != nil {
		engine.chunkSize = 1 << 20
	}
	return engine, driveRoot, nil
}

func TestGetObjectsToReplicate(t *testing.T) {
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		remoteItems := []*IndexDBItem{}
		d, err := json.Marshal(remoteItems)
		require.Nil(t, err)
		w.WriteHeader(200)
		w.Write(d)
	}))
	u, err := url.Parse(ts.URL)
	require.Nil(t, err)
	host, ports, err := net.SplitHostPort(u.Host)
	require.Nil(t, err)
	port, err := strconv.Atoi(ports)
	require.Nil(t, err)

	ece, dr, err := getTestEce()
	if dr != "" {
		defer os.RemoveAll(dr)
	}
	require.Nil(t, err)
	idb, err := ece.getDB("sdb1")
	require.Nil(t, err)

	timestamp := time.Now().UnixNano()
	body := "just testing"
	hsh0 := md5hash("object0")
	partition, _ := strconv.ParseInt(hsh0[:2], 16, 64)
	partition >>= 2
	f, err := idb.TempFile(hsh0, 0, timestamp, int64(len(body)), true)
	require.Nil(t, err)
	f.Write([]byte(body))
	metadata, err := json.Marshal(map[string]string{"moo": "cow"})
	require.Nil(t, err)
	require.Nil(t, idb.Commit(f, hsh0, 0, timestamp, "PUT", "", metadata, false, ""))
	osc := make(chan ObjectStabilizer)
	cancel := make(chan struct{})
	defer close(cancel)
	go ece.GetObjectsToReplicate(
		PriorityRepJob{
			Partition:  uint64(partition),
			FromDevice: &ring.Device{Device: "sdb1"},
			ToDevice:   &ring.Device{Device: "sdb2", Scheme: "http", Port: port, Ip: host},
		}, osc, cancel)
	os := <-osc
	require.Equal(t, "cow", os.Metadata()["moo"])
	os = <-osc
	require.Nil(t, os)
}

func TestGetObjectsToReplicateRemoteHasAll(t *testing.T) {
	ece, dr, err := getTestEce()
	if dr != "" {
		defer os.RemoveAll(dr)
	}
	require.Nil(t, err)
	idb, err := ece.getDB("sdb1")
	require.Nil(t, err)

	timestamp := time.Now().UnixNano()
	body := "just testing"
	hsh0 := "00000000000000000000000000000001"
	f, err := idb.TempFile(hsh0, 0, timestamp, int64(len(body)), true)
	require.Nil(t, err)
	f.Write([]byte(body))
	metadata, err := json.Marshal(map[string]string{"name": "o1"})
	require.Nil(t, err)
	require.Nil(t, idb.Commit(f, hsh0, 0, timestamp, "PUT", "", metadata, false, ""))
	hsh1 := "00000000000000000000000000000002"
	metadata, err = json.Marshal(map[string]string{"name": "o2"})
	require.Nil(t, err)
	f, err = idb.TempFile(hsh1, 0, timestamp, int64(len(body)), true)
	require.Nil(t, err)
	f.Write([]byte(body))
	require.Nil(t, idb.Commit(f, hsh1, 0, timestamp, "PUT", "", metadata, false, ""))
	remoteItems, err := idb.List("", "", "", 0)
	require.Nil(t, err)
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		d, err := json.Marshal(remoteItems)
		require.Nil(t, err)
		w.WriteHeader(200)
		w.Write(d)
	}))
	u, err := url.Parse(ts.URL)
	require.Nil(t, err)
	host, ports, err := net.SplitHostPort(u.Host)
	require.Nil(t, err)
	port, err := strconv.Atoi(ports)
	require.Nil(t, err)

	osc := make(chan ObjectStabilizer)
	cancel := make(chan struct{})
	defer close(cancel)
	go ece.GetObjectsToReplicate(
		PriorityRepJob{FromDevice: &ring.Device{Device: "sdb1"},
			ToDevice: &ring.Device{Device: "sdb2", Scheme: "http", Port: port, Ip: host}}, osc, cancel)
	os := <-osc
	require.Nil(t, os)
}

func TestGetObjectsToReplicateRemoteHasSome(t *testing.T) {
	ece, dr, err := getTestEce()
	if dr != "" {
		defer os.RemoveAll(dr)
	}
	require.Nil(t, err)
	idb, err := ece.getDB("sdb1")
	require.Nil(t, err)

	timestamp := time.Now().UnixNano()
	body := "just testing"
	hsh0 := "00000000000000000000000000000001"
	f, err := idb.TempFile(hsh0, 0, timestamp, int64(len(body)), true)
	require.Nil(t, err)
	f.Write([]byte(body))
	metadata, err := json.Marshal(map[string]string{"name": "o1"})
	require.Nil(t, err)
	require.Nil(t, idb.Commit(f, hsh0, 0, timestamp, "PUT", "", metadata, false, ""))
	hsh1 := "00000000000000000000000000000002"
	metadata, err = json.Marshal(map[string]string{"name": "o2"})
	require.Nil(t, err)
	f, err = idb.TempFile(hsh1, 0, timestamp, int64(len(body)), true)
	require.Nil(t, err)
	f.Write([]byte(body))
	require.Nil(t, idb.Commit(f, hsh1, 0, timestamp, "PUT", "", metadata, false, ""))
	remoteItems, err := idb.List("", "", "", 0)
	require.Nil(t, err)
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		newRemoteItems := []*IndexDBItem{remoteItems[0]}
		d, err := json.Marshal(newRemoteItems)
		require.Nil(t, err)
		w.WriteHeader(200)
		w.Write(d)
	}))
	u, err := url.Parse(ts.URL)
	require.Nil(t, err)
	host, ports, err := net.SplitHostPort(u.Host)
	require.Nil(t, err)
	port, err := strconv.Atoi(ports)
	require.Nil(t, err)

	osc := make(chan ObjectStabilizer)
	cancel := make(chan struct{})
	defer close(cancel)
	go ece.GetObjectsToReplicate(
		PriorityRepJob{FromDevice: &ring.Device{Device: "sdb1"},
			ToDevice: &ring.Device{Device: "sdb2", Scheme: "http", Port: port, Ip: host}}, osc, cancel)
	os := <-osc
	require.Equal(t, "o2", os.Metadata()["name"])
	os = <-osc
	require.Nil(t, os)
}

func TestEcShardDelete(t *testing.T) {
	ece, dr, err := getTestEce()
	if dr != "" {
		defer os.RemoveAll(dr)
	}
	require.Nil(t, err)
	idb, err := ece.getDB("sdb1")
	require.Nil(t, err)

	timestamp := time.Now().UnixNano()
	body := "just testing"
	hsh0 := "00000000000000000000000000000001"
	f, err := idb.TempFile(hsh0, 0, timestamp, int64(len(body)), true)
	require.Nil(t, err)
	f.Write([]byte(body))
	metadata, err := json.Marshal(map[string]string{"name": "o1"})
	require.Nil(t, err)
	require.Nil(t, idb.Commit(f, hsh0, 0, timestamp, "PUT", "", metadata, false, ""))
	req, _ := http.NewRequest("DELETE", fmt.Sprintf("/ec-shard/sdb1/%s/0", hsh0), nil)
	req.Header.Set("X-Timestamp",
		common.CanonicalTimestampFromTime(time.Now().AddDate(0, 0, -1)))
	req = srv.SetVars(req, map[string]string{"index": "0", "device": "sdb1", "hash": hsh0})
	w := &httptest.ResponseRecorder{}
	ece.ecShardDeleteHandler(w, req)
	resp := w.Result()
	require.Equal(t, 409, resp.StatusCode)
	req.Header.Set("X-Timestamp",
		common.CanonicalTimestampFromTime(time.Now().AddDate(0, 0, 1)))
	w = &httptest.ResponseRecorder{}
	ece.ecShardDeleteHandler(w, req)
	resp = w.Result()
	require.Equal(t, 204, resp.StatusCode)
}
<|endoftext|>


<commit_before>package plugins

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/docker/go-connections/sockets"
	"github.com/docker/go-connections/tlsconfig"
)

const (
	versionMimetype = "application/vnd.docker.plugins.v1.1+json"
	defaultTimeOut  = 30
)

// NewClient creates a new plugin client (http).
func NewClient(addr string, tlsConfig tlsconfig.Options) (*Client, error) {
	tr := &http.Transport{}

	c, err := tlsconfig.Client(tlsConfig)
	if err != nil {
		return nil, err
	}
	tr.TLSClientConfig = c

	protoAndAddr := strings.Split(addr, "://")
	sockets.ConfigureTCPTransport(tr, protoAndAddr[0], protoAndAddr[1])

	scheme := protoAndAddr[0]
	if scheme != "https" {
		scheme = "http"
	}
	return &Client{&http.Client{Transport: tr}, scheme, protoAndAddr[1]}, nil
}

// Client represents a plugin client.
type Client struct {
	http   *http.Client // http client to use
	scheme string       // scheme protocol of the plugin
	addr   string       // http address of the plugin
}

// Call calls the specified method with the specified arguments for the plugin.
// It will retry for 30 seconds if a failure occurs when calling.
func (c *Client) Call(serviceMethod string, args interface{}, ret interface{}) error {
	var buf bytes.Buffer
	if args != nil {
		if err := json.NewEncoder(&buf).Encode(args); err != nil {
			return err
		}
	}
	body, err := c.callWithRetry(serviceMethod, &buf, true)
	if err != nil {
		return err
	}
	defer body.Close()
	if ret != nil {
		if err := json.NewDecoder(body).Decode(&ret); err != nil {
			logrus.Errorf("%s: error reading plugin resp: %v", serviceMethod, err)
			return err
		}
	}
	return nil
}

// Stream calls the specified method with the specified arguments for the plugin and returns the response body
func (c *Client) Stream(serviceMethod string, args interface{}) (io.ReadCloser, error) {
	var buf bytes.Buffer
	if err := json.NewEncoder(&buf).Encode(args); err != nil {
		return nil, err
	}
	return c.callWithRetry(serviceMethod, &buf, true)
}

// SendFile calls the specified method, and passes through the IO stream
func (c *Client) SendFile(serviceMethod string, data io.Reader, ret interface{}) error {
	body, err := c.callWithRetry(serviceMethod, data, true)
	if err != nil {
		return err
	}
	if err := json.NewDecoder(body).Decode(&ret); err != nil {
		logrus.Errorf("%s: error reading plugin resp: %v", serviceMethod, err)
		return err
	}
	return nil
}

func (c *Client) callWithRetry(serviceMethod string, data io.Reader, retry bool) (io.ReadCloser, error) {
	req, err := http.NewRequest("POST", "/"+serviceMethod, data)
	if err != nil {
		return nil, err
	}
	req.Header.Add("Accept", versionMimetype)
	req.URL.Scheme = c.scheme
	req.URL.Host = c.addr

	var retries int
	start := time.Now()

	for {
		resp, err := c.http.Do(req)
		if err != nil {
			if !retry {
				return nil, err
			}

			timeOff := backoff(retries)
			if abort(start, timeOff) {
				return nil, err
			}
			retries++
			logrus.Warnf("Unable to connect to plugin: %s, retrying in %v", c.addr, timeOff)
			time.Sleep(timeOff)
			continue
		}

		if resp.StatusCode != http.StatusOK {
			b, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				return nil, fmt.Errorf("%s: %s", serviceMethod, err)
			}

			// Plugins' Response(s) should have an Err field indicating what went
			// wrong. Try to unmarshal into ResponseErr. Otherwise fallback to just
			// return the string(body)
			type responseErr struct {
				Err string
			}
			remoteErr := responseErr{}
			if err := json.Unmarshal(b, &remoteErr); err == nil {
				if remoteErr.Err != "" {
					return nil, fmt.Errorf("%s: %s", serviceMethod, remoteErr.Err)
				}
			}
			// old way...
			return nil, fmt.Errorf("%s: %s", serviceMethod, string(b))
		}
		return resp.Body, nil
	}
}

func backoff(retries int) time.Duration {
	b, max := 1, defaultTimeOut
	for b < max && retries > 0 {
		b *= 2
		retries--
	}
	if b > max {
		b = max
	}
	return time.Duration(b) * time.Second
}

func abort(start time.Time, timeOff time.Duration) bool {
	return timeOff+time.Since(start) >= time.Duration(defaultTimeOut)*time.Second
}
<commit_msg>Bump plugin API version<commit_after>package plugins

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/docker/go-connections/sockets"
	"github.com/docker/go-connections/tlsconfig"
)

const (
	versionMimetype = "application/vnd.docker.plugins.v1.2+json"
	defaultTimeOut  = 30
)

// NewClient creates a new plugin client (http).
func NewClient(addr string, tlsConfig tlsconfig.Options) (*Client, error) {
	tr := &http.Transport{}

	c, err := tlsconfig.Client(tlsConfig)
	if err != nil {
		return nil, err
	}
	tr.TLSClientConfig = c

	protoAndAddr := strings.Split(addr, "://")
	sockets.ConfigureTCPTransport(tr, protoAndAddr[0], protoAndAddr[1])

	scheme := protoAndAddr[0]
	if scheme != "https" {
		scheme = "http"
	}
	return &Client{&http.Client{Transport: tr}, scheme, protoAndAddr[1]}, nil
}

// Client represents a plugin client.
type Client struct {
	http   *http.Client // http client to use
	scheme string       // scheme protocol of the plugin
	addr   string       // http address of the plugin
}

// Call calls the specified method with the specified arguments for the plugin.
// It will retry for 30 seconds if a failure occurs when calling.
func (c *Client) Call(serviceMethod string, args interface{}, ret interface{}) error {
	var buf bytes.Buffer
	if args != nil {
		if err := json.NewEncoder(&buf).Encode(args); err != nil {
			return err
		}
	}
	body, err := c.callWithRetry(serviceMethod, &buf, true)
	if err != nil {
		return err
	}
	defer body.Close()
	if ret != nil {
		if err := json.NewDecoder(body).Decode(&ret); err != nil {
			logrus.Errorf("%s: error reading plugin resp: %v", serviceMethod, err)
			return err
		}
	}
	return nil
}

// Stream calls the specified method with the specified arguments for the plugin and returns the response body
func (c *Client) Stream(serviceMethod string, args interface{}) (io.ReadCloser, error) {
	var buf bytes.Buffer
	if err := json.NewEncoder(&buf).Encode(args); err != nil {
		return nil, err
	}
	return c.callWithRetry(serviceMethod, &buf, true)
}

// SendFile calls the specified method, and passes through the IO stream
func (c *Client) SendFile(serviceMethod string, data io.Reader, ret interface{}) error {
	body, err := c.callWithRetry(serviceMethod, data, true)
	if err != nil {
		return err
	}
	if err := json.NewDecoder(body).Decode(&ret); err != nil {
		logrus.Errorf("%s: error reading plugin resp: %v", serviceMethod, err)
		return err
	}
	return nil
}

func (c *Client) callWithRetry(serviceMethod string, data io.Reader, retry bool) (io.ReadCloser, error) {
	req, err := http.NewRequest("POST", "/"+serviceMethod, data)
	if err != nil {
		return nil, err
	}
	req.Header.Add("Accept", versionMimetype)
	req.URL.Scheme = c.scheme
	req.URL.Host = c.addr

	var retries int
	start := time.Now()

	for {
		resp, err := c.http.Do(req)
		if err != nil {
			if !retry {
				return nil, err
			}

			timeOff := backoff(retries)
			if abort(start, timeOff) {
				return nil, err
			}
			retries++
			logrus.Warnf("Unable to connect to plugin: %s, retrying in %v", c.addr, timeOff)
			time.Sleep(timeOff)
			continue
		}

		if resp.StatusCode != http.StatusOK {
			b, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				return nil, fmt.Errorf("%s: %s", serviceMethod, err)
			}

			// Plugins' Response(s) should have an Err field indicating what went
			// wrong. Try to unmarshal into ResponseErr. Otherwise fallback to just
			// return the string(body)
			type responseErr struct {
				Err string
			}
			remoteErr := responseErr{}
			if err := json.Unmarshal(b, &remoteErr); err == nil {
				if remoteErr.Err != "" {
					return nil, fmt.Errorf("%s: %s", serviceMethod, remoteErr.Err)
				}
			}
			// old way...
			return nil, fmt.Errorf("%s: %s", serviceMethod, string(b))
		}
		return resp.Body, nil
	}
}

func backoff(retries int) time.Duration {
	b, max := 1, defaultTimeOut
	for b < max && retries > 0 {
		b *= 2
		retries--
	}
	if b > max {
		b = max
	}
	return time.Duration(b) * time.Second
}

func abort(start time.Time, timeOff time.Duration) bool {
	return timeOff+time.Since(start) >= time.Duration(defaultTimeOut)*time.Second
}
<|endoftext|>


<commit_before>package registry

import (
	"context"

	"github.com/jmoiron/sqlx"

	// mysql
	_ "github.com/go-sql-driver/mysql"
)

type registrar struct {
	DB *sqlx.DB
}

const listUnitResidentsQuery = `select * from residents
inner join units_residents
on residents.id = units_residents.resident
where residents.id is ?;`

// ListUnitResidents implements registrar
func (reg *registrar) ListUnitResidents(ctx context.Context, unitID int64) (residents []*Resident, err error) {
	err = reg.DB.SelectContext(ctx, residents, listUnitResidentsQuery, unitID)
	return
}

const getUnitByNameQuery = `select * from units
where units.name = ?;`

// GetUnitByName implmements registrar
func (reg *registrar) GetUnitByName(ctx context.Context, name string) (unit *Unit, err error) {
	err = reg.DB.SelectContext(ctx, unit, getUnitByNameQuery, name)
	return
}

const registerResidentQuery = `insert into residents (firstname, middlename, lastname)
values (:firstname, :middlename, :lastname);`

// RegisterResident implements registrar
func (reg *registrar) RegisterResident(ctx context.Context, resident *Resident) (err error) {
	result, err := reg.DB.NamedExecContext(ctx, registerResidentQuery, resident)
	if err != nil {
		return
	}
	resident.ID, err = result.LastInsertId()
	return
}

const (
	moveOutResidentQuery = `delete from units_residents ur
where ur.resident = ?;`
	moveInResidentQuery = `insert into units_residents (unit, resident)
values (?, ?);`
)

// MoveResident implements registrar
func (reg *registrar) MoveResident(ctx context.Context, residentID, newUnitID int64) (err error) {
	_, err = reg.DB.ExecContext(ctx, moveOutResidentQuery, residentID)
	if err != nil {
		return
	}
	_, err = reg.DB.ExecContext(ctx, moveInResidentQuery, newUnitID, residentID)
	return
}

// DeregisterResident implements registrar
func (reg *registrar) DeregisterResident(ctx context.Context, residentID int64) (err error) {
	return
}
<commit_msg>implements DeregisterResident<commit_after>package registry

import (
	"context"

	"github.com/jmoiron/sqlx"

	// mysql
	_ "github.com/go-sql-driver/mysql"
)

type registrar struct {
	DB *sqlx.DB
}

const listUnitResidentsQuery = `select * from residents
inner join units_residents
on residents.id = units_residents.resident
where residents.id is ?;`

// ListUnitResidents implements registrar
func (reg *registrar) ListUnitResidents(ctx context.Context, unitID int64) (residents []*Resident, err error) {
	err = reg.DB.SelectContext(ctx, residents, listUnitResidentsQuery, unitID)
	return
}

const getUnitByNameQuery = `select * from units
where units.name = ?;`

// GetUnitByName implmements registrar
func (reg *registrar) GetUnitByName(ctx context.Context, name string) (unit *Unit, err error) {
	err = reg.DB.SelectContext(ctx, unit, getUnitByNameQuery, name)
	return
}

const registerResidentQuery = `insert into residents (firstname, middlename, lastname)
values (:firstname, :middlename, :lastname);`

// RegisterResident implements registrar
func (reg *registrar) RegisterResident(ctx context.Context, resident *Resident) (err error) {
	result, err := reg.DB.NamedExecContext(ctx, registerResidentQuery, resident)
	if err != nil {
		return
	}
	resident.ID, err = result.LastInsertId()
	return
}

const (
	moveOutResidentQuery = `delete from units_residents ur
where ur.resident = ?;`
	moveInResidentQuery = `insert into units_residents (unit, resident)
values (?, ?);`
)

// MoveResident implements registrar
func (reg *registrar) MoveResident(ctx context.Context, residentID, newUnitID int64) (err error) {
	_, err = reg.DB.ExecContext(ctx, moveOutResidentQuery, residentID)
	if err != nil {
		return
	}
	_, err = reg.DB.ExecContext(ctx, moveInResidentQuery, newUnitID, residentID)
	return
}

const (
	deregisterResidentQuery = `delete from residents where residents.id = ?;`
)

// DeregisterResident implements registrar
func (reg *registrar) DeregisterResident(ctx context.Context, residentID int64) (err error) {
	_, err = reg.DB.ExecContext(ctx, moveOutResidentQuery, residentID)
	return
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/http"
	"os"
	"reflect"
	"strings"

	"github.com/spf13/cobra"
)

const (
	VERSION  = "0.2.0"
	hostname = "http://localhost:9980"
)

// get wraps a GET request with a status code check, such that if the GET does
// not return 200, the error will be read and returned. The response body is
// not closed.
func get(call string) (resp *http.Response, err error) {
	resp, err = http.Get(hostname + call)
	if err != nil {
		return
	}
	// check error code
	if resp.StatusCode != 200 {
		errResp, _ := ioutil.ReadAll(resp.Body)
		err = errors.New(strings.TrimSpace(string(errResp)))
	}
	return
}

// getAPI makes an API call and decodes the response.
func getAPI(call string, obj interface{}) (err error) {
	resp, err := get(call)
	if err != nil {
		return
	}
	defer resp.Body.Close()
	err = json.NewDecoder(resp.Body).Decode(obj)
	if err != nil {
		return
	}
	return
}

// callAPI makes an API call and discards the response.
func callAPI(call string) (err error) {
	resp, err := get(call)
	if err != nil {
		return
	}
	resp.Body.Close()
	return
}

// wrap wraps a generic command with a check that the command has been
// passed the correct number of arguments. The command must take only strings
// as arguments.
func wrap(fn interface{}) func(*cobra.Command, []string) {
	fnVal, fnType := reflect.ValueOf(fn), reflect.TypeOf(fn)
	if fnType.Kind() != reflect.Func {
		panic("wrapped function has wrong type signature")
	}
	for i := 0; i < fnType.NumIn(); i++ {
		if fnType.In(i).Kind() != reflect.String {
			panic("wrapped function has wrong type signature")
		}
	}

	return func(cmd *cobra.Command, args []string) {
		if len(args) != fnType.NumIn() {
			cmd.Usage()
			return
		}
		argVals := make([]reflect.Value, fnType.NumIn())
		for i := range args {
			argVals[i] = reflect.ValueOf(args[i])
		}
		fnVal.Call(argVals)
	}
}

func version(*cobra.Command, []string) {
	println("Sia Client v" + VERSION)
}

func main() {
	root := &cobra.Command{
		Use:   os.Args[0],
		Short: "Sia Client v" + VERSION,
		Long:  "Sia Client v" + VERSION,
		Run:   version,
	}

	// create command tree
	root.AddCommand(&cobra.Command{
		Use:   "version",
		Short: "Print version information",
		Long:  "Print version information.",
		Run:   version,
	})

	root.AddCommand(hostCmd)
	hostCmd.AddCommand(hostSetCmd, hostAnnounceCmd, hostStatusCmd)

	root.AddCommand(minerCmd)
	minerCmd.AddCommand(minerStartCmd, minerStopCmd, minerStatusCmd)

	root.AddCommand(walletCmd)
	walletCmd.AddCommand(walletAddressCmd, walletSendCmd, walletStatusCmd)

	root.AddCommand(fileCmd)
	fileCmd.AddCommand(fileUploadCmd, fileDownloadCmd, fileStatusCmd)

	root.AddCommand(peerCmd)
	peerCmd.AddCommand(peerAddCmd, peerRemoveCmd, peerStatusCmd)

	root.AddCommand(updateCmd)
	updateCmd.AddCommand(updateCheckCmd, updateApplyCmd)

	root.AddCommand(statusCmd, stopCmd, syncCmd)

	// run
	root.Execute()
}
<commit_msg>handle API 404 error specially<commit_after>package main

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/http"
	"os"
	"reflect"
	"strings"

	"github.com/spf13/cobra"
)

const (
	VERSION  = "0.2.0"
	hostname = "http://localhost:9980"
)

// get wraps a GET request with a status code check, such that if the GET does
// not return 200, the error will be read and returned. The response body is
// not closed.
func get(call string) (resp *http.Response, err error) {
	resp, err = http.Get(hostname + call)
	if err != nil {
		return
	}
	// check error code
	if resp.StatusCode == http.StatusNotFound {
		err = errors.New("API call not recognized: " + call)
	} else if resp.StatusCode != http.StatusOK {
		errResp, _ := ioutil.ReadAll(resp.Body)
		err = errors.New(strings.TrimSpace(string(errResp)))
	}
	return
}

// getAPI makes an API call and decodes the response.
func getAPI(call string, obj interface{}) (err error) {
	resp, err := get(call)
	if err != nil {
		return
	}
	defer resp.Body.Close()
	err = json.NewDecoder(resp.Body).Decode(obj)
	if err != nil {
		return
	}
	return
}

// callAPI makes an API call and discards the response.
func callAPI(call string) (err error) {
	resp, err := get(call)
	if err != nil {
		return
	}
	resp.Body.Close()
	return
}

// wrap wraps a generic command with a check that the command has been
// passed the correct number of arguments. The command must take only strings
// as arguments.
func wrap(fn interface{}) func(*cobra.Command, []string) {
	fnVal, fnType := reflect.ValueOf(fn), reflect.TypeOf(fn)
	if fnType.Kind() != reflect.Func {
		panic("wrapped function has wrong type signature")
	}
	for i := 0; i < fnType.NumIn(); i++ {
		if fnType.In(i).Kind() != reflect.String {
			panic("wrapped function has wrong type signature")
		}
	}

	return func(cmd *cobra.Command, args []string) {
		if len(args) != fnType.NumIn() {
			cmd.Usage()
			return
		}
		argVals := make([]reflect.Value, fnType.NumIn())
		for i := range args {
			argVals[i] = reflect.ValueOf(args[i])
		}
		fnVal.Call(argVals)
	}
}

func version(*cobra.Command, []string) {
	println("Sia Client v" + VERSION)
}

func main() {
	root := &cobra.Command{
		Use:   os.Args[0],
		Short: "Sia Client v" + VERSION,
		Long:  "Sia Client v" + VERSION,
		Run:   version,
	}

	// create command tree
	root.AddCommand(&cobra.Command{
		Use:   "version",
		Short: "Print version information",
		Long:  "Print version information.",
		Run:   version,
	})

	root.AddCommand(hostCmd)
	hostCmd.AddCommand(hostSetCmd, hostAnnounceCmd, hostStatusCmd)

	root.AddCommand(minerCmd)
	minerCmd.AddCommand(minerStartCmd, minerStopCmd, minerStatusCmd)

	root.AddCommand(walletCmd)
	walletCmd.AddCommand(walletAddressCmd, walletSendCmd, walletStatusCmd)

	root.AddCommand(fileCmd)
	fileCmd.AddCommand(fileUploadCmd, fileDownloadCmd, fileStatusCmd)

	root.AddCommand(peerCmd)
	peerCmd.AddCommand(peerAddCmd, peerRemoveCmd, peerStatusCmd)

	root.AddCommand(updateCmd)
	updateCmd.AddCommand(updateCheckCmd, updateApplyCmd)

	root.AddCommand(statusCmd, stopCmd, syncCmd)

	// run
	root.Execute()
}
<|endoftext|>


<commit_before>package siad

import (
	"errors"
	"io"
	"net"
	"os"
	"strconv"
	"sync"

	"github.com/NebulousLabs/Andromeda/encoding"
	"github.com/NebulousLabs/Andromeda/hash"
	"github.com/NebulousLabs/Andromeda/siacore"
)

const (
	HostAnnouncementPrefix = uint64(1)
	AcceptContractResponse = "accept"
)

type ContractEntry struct {
	ID       siacore.ContractID
	Contract *siacore.FileContract
}

type Host struct {
	Settings HostAnnouncement

	SpaceRemaining uint64

	Files map[hash.Hash]string
	index int

	ForwardContracts  map[siacore.BlockHeight][]ContractEntry
	BackwardContracts map[siacore.BlockHeight][]ContractEntry

	sync.Mutex
}

func CreateHost() *Host {
	return new(Host)
}

// SetHostSettings changes the settings according to the input. Need a setter
// because Environment.host is not exported.
func (e *Environment) SetHostSettings(ha HostAnnouncement) {
	e.host.Settings = ha
}

// Wallet.HostAnnounceSelf() creates a host announcement transaction, adding
// information to the arbitrary data and then signing the transaction.
func (e *Environment) HostAnnounceSelf(freezeVolume siacore.Currency, freezeUnlockHeight siacore.BlockHeight, minerFee siacore.Currency) (t siacore.Transaction, err error) {
	info := e.host.Settings

	// Fund the transaction.
	err = e.wallet.FundTransaction(freezeVolume+minerFee, &t)
	if err != nil {
		return
	}

	// Add the miner fee.
	t.MinerFees = append(t.MinerFees, minerFee)

	// Add the output with the freeze volume.
	freezeConditions := e.wallet.SpendConditions
	freezeConditions.TimeLock = freezeUnlockHeight
	info.FreezeIndex = uint64(len(t.Outputs))
	info.SpendConditions = freezeConditions
	t.Outputs = append(t.Outputs, siacore.Output{Value: freezeVolume, SpendHash: freezeConditions.CoinAddress()})

	// Frozen money can't currently be recovered.
	/*
		num, exists := w.OpenFreezeConditions[freezeUnlockHeight]
		if exists {
			w.OpenFreezeConditions[freezeUnlockHeight] = num + 1
		} else {
			w.OpenFreezeConditions[freezeUnlockHeight] = 1
		}
	*/

	// Add the announcement as arbitrary data.
	prefixBytes := encoding.Marshal(HostAnnouncementPrefix)
	announcementBytes := encoding.Marshal(info)
	t.ArbitraryData = append(prefixBytes, announcementBytes...)

	// Sign the transaction.
	e.wallet.SignTransaction(&t)

	e.AcceptTransaction(t)

	// TODO: Have a different method for setting max filesize.
	e.host.SpaceRemaining = e.host.Settings.MaxFilesize

	return
}

// considerContract takes a contract and verifies that the negotiations, such
// as price, tolerance, etc. are all valid within the host settings. If so,
// inputs are added to fund the burn part of the contract fund, then the
// updated contract is signed and returned.
func (e *Environment) considerContract(t siacore.Transaction) (nt siacore.Transaction, err error) {
	// Set the new transaction equal to the old transaction. Pretty sure that
	// go does not allow you to return the same variable that was used as
	// input. We could use a pointer, but that might be a bad idea. This call
	// is happening over the network anyway.
	nt = t

	contractDuration := nt.FileContracts[0].End - nt.FileContracts[0].Start // Duration according to the contract.
	fullDuration := nt.FileContracts[0].End - e.Height()                    // Duration that the host will actually be storing the file.
	fileSize := nt.FileContracts[0].FileSize

	// Check that there is only one file contract.
	if len(nt.FileContracts) != 1 {
		err = errors.New("will not accept a transaction with more than one file contract")
		return
	}

	// Check that the file size listed in the contract is in bounds.
	if fileSize < e.host.Settings.MinFilesize || fileSize > e.host.Settings.MaxFilesize {
		err = errors.New("file is of incorrect size")
		return
	}

	// Check that the duration of the contract is in bounds.
	if fullDuration < e.host.Settings.MinDuration || fullDuration < e.host.Settings.MinDuration {
		err = errors.New("contract duration is out of bounds")
		return
	}

	// Check that challenges will not be happening too frequently or infrequently.
	if nt.FileContracts[0].ChallengeFrequency < e.host.Settings.MaxChallengeFrequency || nt.FileContracts[0].ChallengeFrequency > e.host.Settings.MinChallengeFrequency {
		err = errors.New("challenges frequency is too often")
		return
	}

	// Check that tolerance is acceptible.
	if nt.FileContracts[0].Tolerance < e.host.Settings.MinTolerance {
		err = errors.New("tolerance is too low")
		return
	}

	// Outputs for successful proofs need to go to the correct address.
	if nt.FileContracts[0].ValidProofAddress != e.CoinAddress() {
		err = errors.New("coins are not paying out to correct address")
		return
	}

	// Output for failed proofs needs to be the 0 address.
	emptyAddress := siacore.CoinAddress{}
	if nt.FileContracts[0].MissedProofAddress != emptyAddress {
		err = errors.New("burn payout needs to go to the empty address")
		return
	}

	// Verify that output for failed proofs matches burn.
	requiredBurn := e.host.Settings.Burn * siacore.Currency(fileSize) * siacore.Currency(nt.FileContracts[0].ChallengeFrequency)
	if nt.FileContracts[0].MissedProofPayout > requiredBurn {
		err = errors.New("burn payout is too high for a missed proof.")
		return
	}

	// Verify that the outputs for successful proofs are high enough.
	requiredSize := e.host.Settings.Price * siacore.Currency(fileSize) * siacore.Currency(nt.FileContracts[0].ChallengeFrequency)
	if nt.FileContracts[0].ValidProofPayout < requiredSize {
		err = errors.New("valid proof payout is too low")
		return
	}

	// Verify that the contract fund covers the payout and burn for the whole
	// duration.
	requiredFund := (e.host.Settings.Burn + e.host.Settings.Price) * siacore.Currency(fileSize) * siacore.Currency(contractDuration)
	if nt.FileContracts[0].ContractFund < requiredFund {
		err = errors.New("ContractFund does not cover the entire duration of the contract.")
		return
	}

	// Add some inputs and outputs to the transaction to fund the burn half.
	e.wallet.FundTransaction(e.host.Settings.Burn*siacore.Currency(fileSize)*siacore.Currency(contractDuration), &nt)
	e.wallet.SignTransaction(&nt)

	// Check that the transaction is valid after the host signature.
	e.state.Lock()
	err = e.state.ValidTransaction(nt)
	e.state.Unlock()
	if err != nil {
		err = errors.New("post-verified transaction not valid - most likely a client error, but could be a host error too")
		return
	}

	return
}

// NegotiateContract is an RPC that negotiates a file contract. If the
// negotiation is successful, the file is downloaded and the host begins
// submitting proofs of storage.
func (e *Environment) NegotiateContract(conn net.Conn, data []byte) (err error) {
	// read transaction
	var t siacore.Transaction
	if err = encoding.Unmarshal(data, &t); err != nil {
		return
	}
	// consider contract
	if t, err = e.considerContract(t); err != nil {
		_, err = encoding.WriteObject(conn, err.Error())
		return
	} else if _, err = encoding.WriteObject(conn, AcceptContractResponse); err != nil {
		return
	}
	// read file data
	file, err := os.Create(strconv.Itoa(e.host.index))
	if err != nil {
		return
	}
	defer file.Close()
	_, err = io.Copy(file, conn)
	if err != nil {
		return
	}

	// Check that the file matches the merkle root in the contract.
	_, err = file.Seek(0, 0)
	if err != nil {
		return
	}
	merkleRoot, err := hash.ReaderMerkleRoot(file, hash.CalculateSegments(uint64(t.FileContracts[0].FileSize)))
	if err != nil {
		return
	}
	if merkleRoot != t.FileContracts[0].FileMerkleRoot {
		err = errors.New("uploaded file has wrong merkle root")
		return
	}

	// Check that the file arrived in time.
	if e.Height() >= t.FileContracts[0].Start-2 {
		err = errors.New("file not uploaded in time, refusing to go forward with contract.")
		return
	}

	// record filename for later retrieval
	e.host.Files[t.FileContracts[0].FileMerkleRoot] = strconv.Itoa(e.host.index)
	e.host.index++

	// Submit the transaction.
	e.AcceptTransaction(t)

	// Put the contract in a list where the host will be performing proofs of
	// storage.
	e.host.ForwardContracts[t.FileContracts[0].Start+1] = append(e.host.ForwardContracts[t.FileContracts[0].Start+1], ContractEntry{ID: t.FileContractID(0), Contract: &t.FileContracts[0]})

	return
}

// RetrieveFile is an RPC that uploads a specified file to a client.
func (e *Environment) RetrieveFile(conn net.Conn, data []byte) (err error) {
	var merkle hash.Hash
	if err = encoding.Unmarshal(data, &merkle); err != nil {
		return
	}
	filename, ok := e.host.Files[merkle]
	if !ok {
		return errors.New("no record of that file")
	}
	f, err := os.Open(filename)
	if err != nil {
		return
	}

	// transmit file
	_, err = io.Copy(conn, f)
	if err != nil {
		return
	}
	return
}

func (e *Environment) CreateProof(contract siacore.FileContract, contractID siacore.ContractID, stateHeight siacore.BlockHeight) (sp siacore.StorageProof, err error) {
	filename, ok := e.host.Files[contract.FileMerkleRoot]
	if !ok {
		err = errors.New("no record of that file")
	}
	file, err := os.Open(filename)
	if err != nil {
		return
	}
	numSegments := hash.CalculateSegments(contract.FileSize)
	triggerBlock, err := e.BlockAtHeight(stateHeight)
	if err != nil {
		return
	}
	proofIndex := siacore.ContractProofIndex(contractID, stateHeight, contract, triggerBlock.ID())
	base, hashSet, err := hash.BuildReaderProof(file, numSegments, proofIndex)
	if err != nil {
		return
	}
	sp = siacore.StorageProof{contractID, base, hashSet}
	return
}

// storageProofMaintenance tracks when storage proofs need to be submitted as
// transactions, then creates the proof and submits the transaction.
func (e *Environment) storageProofMaintenance(stateHeight siacore.BlockHeight, rewoundBlocks []siacore.BlockID, appliedBlocks []siacore.BlockID) {
	height := stateHeight - siacore.BlockHeight(len(appliedBlocks))
	var proofs []siacore.StorageProof
	for _ = range rewoundBlocks {
		needActionContracts := e.host.BackwardContracts[height]
		for _, contractEntry := range needActionContracts {
			// Create a proof for this contract.
			proof, err := e.CreateProof(*contractEntry.Contract, contractEntry.ID, height)
			if err != nil {
				panic(err)
			}
			proofs = append(proofs, proof)
		}
		height++
	}

	height = stateHeight - siacore.BlockHeight(len(appliedBlocks))
	for _ = range appliedBlocks {
		needActionContracts := e.host.ForwardContracts[height]
		for _, contractEntry := range needActionContracts {
			// Create a proof for this contract.
			proof, err := e.CreateProof(*contractEntry.Contract, contractEntry.ID, height)
			if err != nil {
				panic(err)
			}
			proofs = append(proofs, proof)

			// Add this contract proof to the backwards contracts list.
			e.host.BackwardContracts[height-2] = append(e.host.BackwardContracts[height-2], contractEntry)

			// Add this contract entry to ForwardContracts windowsize blocks into the future.
		}
		delete(e.host.ForwardContracts, height)
		height++

	}
	// bundle proofs into a txn
	// e.AcceptTransaction(t)
}
<commit_msg>technically everything should work now wink wink lol<commit_after>package siad

import (
	"errors"
	"io"
	"net"
	"os"
	"strconv"
	"sync"

	"github.com/NebulousLabs/Andromeda/encoding"
	"github.com/NebulousLabs/Andromeda/hash"
	"github.com/NebulousLabs/Andromeda/siacore"
)

const (
	HostAnnouncementPrefix = uint64(1)
	AcceptContractResponse = "accept"
)

type ContractEntry struct {
	ID       siacore.ContractID
	Contract *siacore.FileContract
}

type Host struct {
	Settings HostAnnouncement

	SpaceRemaining uint64

	Files map[hash.Hash]string
	index int

	ForwardContracts  map[siacore.BlockHeight][]ContractEntry
	BackwardContracts map[siacore.BlockHeight][]ContractEntry

	sync.Mutex
}

func CreateHost() *Host {
	return new(Host)
}

// SetHostSettings changes the settings according to the input. Need a setter
// because Environment.host is not exported.
func (e *Environment) SetHostSettings(ha HostAnnouncement) {
	e.host.Settings = ha
}

// Wallet.HostAnnounceSelf() creates a host announcement transaction, adding
// information to the arbitrary data and then signing the transaction.
func (e *Environment) HostAnnounceSelf(freezeVolume siacore.Currency, freezeUnlockHeight siacore.BlockHeight, minerFee siacore.Currency) (t siacore.Transaction, err error) {
	info := e.host.Settings

	// Fund the transaction.
	err = e.wallet.FundTransaction(freezeVolume+minerFee, &t)
	if err != nil {
		return
	}

	// Add the miner fee.
	t.MinerFees = append(t.MinerFees, minerFee)

	// Add the output with the freeze volume.
	freezeConditions := e.wallet.SpendConditions
	freezeConditions.TimeLock = freezeUnlockHeight
	info.FreezeIndex = uint64(len(t.Outputs))
	info.SpendConditions = freezeConditions
	t.Outputs = append(t.Outputs, siacore.Output{Value: freezeVolume, SpendHash: freezeConditions.CoinAddress()})

	// Frozen money can't currently be recovered.
	/*
		num, exists := w.OpenFreezeConditions[freezeUnlockHeight]
		if exists {
			w.OpenFreezeConditions[freezeUnlockHeight] = num + 1
		} else {
			w.OpenFreezeConditions[freezeUnlockHeight] = 1
		}
	*/

	// Add the announcement as arbitrary data.
	prefixBytes := encoding.Marshal(HostAnnouncementPrefix)
	announcementBytes := encoding.Marshal(info)
	t.ArbitraryData = append(prefixBytes, announcementBytes...)

	// Sign the transaction.
	e.wallet.SignTransaction(&t)

	e.AcceptTransaction(t)

	// TODO: Have a different method for setting max filesize.
	e.host.SpaceRemaining = e.host.Settings.MaxFilesize

	return
}

// considerContract takes a contract and verifies that the negotiations, such
// as price, tolerance, etc. are all valid within the host settings. If so,
// inputs are added to fund the burn part of the contract fund, then the
// updated contract is signed and returned.
func (e *Environment) considerContract(t siacore.Transaction) (nt siacore.Transaction, err error) {
	// Set the new transaction equal to the old transaction. Pretty sure that
	// go does not allow you to return the same variable that was used as
	// input. We could use a pointer, but that might be a bad idea. This call
	// is happening over the network anyway.
	nt = t

	contractDuration := nt.FileContracts[0].End - nt.FileContracts[0].Start // Duration according to the contract.
	fullDuration := nt.FileContracts[0].End - e.Height()                    // Duration that the host will actually be storing the file.
	fileSize := nt.FileContracts[0].FileSize

	// Check that there is only one file contract.
	if len(nt.FileContracts) != 1 {
		err = errors.New("will not accept a transaction with more than one file contract")
		return
	}

	// Check that the file size listed in the contract is in bounds.
	if fileSize < e.host.Settings.MinFilesize || fileSize > e.host.Settings.MaxFilesize {
		err = errors.New("file is of incorrect size")
		return
	}

	// Check that the duration of the contract is in bounds.
	if fullDuration < e.host.Settings.MinDuration || fullDuration < e.host.Settings.MinDuration {
		err = errors.New("contract duration is out of bounds")
		return
	}

	// Check that challenges will not be happening too frequently or infrequently.
	if nt.FileContracts[0].ChallengeFrequency < e.host.Settings.MaxChallengeFrequency || nt.FileContracts[0].ChallengeFrequency > e.host.Settings.MinChallengeFrequency {
		err = errors.New("challenges frequency is too often")
		return
	}

	// Check that tolerance is acceptible.
	if nt.FileContracts[0].Tolerance < e.host.Settings.MinTolerance {
		err = errors.New("tolerance is too low")
		return
	}

	// Outputs for successful proofs need to go to the correct address.
	if nt.FileContracts[0].ValidProofAddress != e.CoinAddress() {
		err = errors.New("coins are not paying out to correct address")
		return
	}

	// Output for failed proofs needs to be the 0 address.
	emptyAddress := siacore.CoinAddress{}
	if nt.FileContracts[0].MissedProofAddress != emptyAddress {
		err = errors.New("burn payout needs to go to the empty address")
		return
	}

	// Verify that output for failed proofs matches burn.
	requiredBurn := e.host.Settings.Burn * siacore.Currency(fileSize) * siacore.Currency(nt.FileContracts[0].ChallengeFrequency)
	if nt.FileContracts[0].MissedProofPayout > requiredBurn {
		err = errors.New("burn payout is too high for a missed proof.")
		return
	}

	// Verify that the outputs for successful proofs are high enough.
	requiredSize := e.host.Settings.Price * siacore.Currency(fileSize) * siacore.Currency(nt.FileContracts[0].ChallengeFrequency)
	if nt.FileContracts[0].ValidProofPayout < requiredSize {
		err = errors.New("valid proof payout is too low")
		return
	}

	// Verify that the contract fund covers the payout and burn for the whole
	// duration.
	requiredFund := (e.host.Settings.Burn + e.host.Settings.Price) * siacore.Currency(fileSize) * siacore.Currency(contractDuration)
	if nt.FileContracts[0].ContractFund < requiredFund {
		err = errors.New("ContractFund does not cover the entire duration of the contract.")
		return
	}

	// Add some inputs and outputs to the transaction to fund the burn half.
	e.wallet.FundTransaction(e.host.Settings.Burn*siacore.Currency(fileSize)*siacore.Currency(contractDuration), &nt)
	e.wallet.SignTransaction(&nt)

	// Check that the transaction is valid after the host signature.
	e.state.Lock()
	err = e.state.ValidTransaction(nt)
	e.state.Unlock()
	if err != nil {
		err = errors.New("post-verified transaction not valid - most likely a client error, but could be a host error too")
		return
	}

	return
}

// NegotiateContract is an RPC that negotiates a file contract. If the
// negotiation is successful, the file is downloaded and the host begins
// submitting proofs of storage.
func (e *Environment) NegotiateContract(conn net.Conn, data []byte) (err error) {
	// read transaction
	var t siacore.Transaction
	if err = encoding.Unmarshal(data, &t); err != nil {
		return
	}
	// consider contract
	if t, err = e.considerContract(t); err != nil {
		_, err = encoding.WriteObject(conn, err.Error())
		return
	} else if _, err = encoding.WriteObject(conn, AcceptContractResponse); err != nil {
		return
	}
	// read file data
	file, err := os.Create(strconv.Itoa(e.host.index))
	if err != nil {
		return
	}
	defer file.Close()
	_, err = io.Copy(file, conn)
	if err != nil {
		return
	}

	// Check that the file matches the merkle root in the contract.
	_, err = file.Seek(0, 0)
	if err != nil {
		return
	}
	merkleRoot, err := hash.ReaderMerkleRoot(file, hash.CalculateSegments(uint64(t.FileContracts[0].FileSize)))
	if err != nil {
		return
	}
	if merkleRoot != t.FileContracts[0].FileMerkleRoot {
		err = errors.New("uploaded file has wrong merkle root")
		return
	}

	// Check that the file arrived in time.
	if e.Height() >= t.FileContracts[0].Start-2 {
		err = errors.New("file not uploaded in time, refusing to go forward with contract.")
		return
	}

	// record filename for later retrieval
	e.host.Files[t.FileContracts[0].FileMerkleRoot] = strconv.Itoa(e.host.index)
	e.host.index++

	// Submit the transaction.
	e.AcceptTransaction(t)

	// Put the contract in a list where the host will be performing proofs of
	// storage.
	e.host.ForwardContracts[t.FileContracts[0].Start+1] = append(e.host.ForwardContracts[t.FileContracts[0].Start+1], ContractEntry{ID: t.FileContractID(0), Contract: &t.FileContracts[0]})

	return
}

// RetrieveFile is an RPC that uploads a specified file to a client.
func (e *Environment) RetrieveFile(conn net.Conn, data []byte) (err error) {
	var merkle hash.Hash
	if err = encoding.Unmarshal(data, &merkle); err != nil {
		return
	}
	filename, ok := e.host.Files[merkle]
	if !ok {
		return errors.New("no record of that file")
	}
	f, err := os.Open(filename)
	if err != nil {
		return
	}

	// transmit file
	_, err = io.Copy(conn, f)
	if err != nil {
		return
	}
	return
}

func (e *Environment) CreateProof(contract siacore.FileContract, contractID siacore.ContractID, stateHeight siacore.BlockHeight) (sp siacore.StorageProof, err error) {
	filename, ok := e.host.Files[contract.FileMerkleRoot]
	if !ok {
		err = errors.New("no record of that file")
	}
	file, err := os.Open(filename)
	if err != nil {
		return
	}
	numSegments := hash.CalculateSegments(contract.FileSize)
	triggerBlock, err := e.BlockAtHeight(stateHeight)
	if err != nil {
		return
	}
	proofIndex := siacore.ContractProofIndex(contractID, stateHeight, contract, triggerBlock.ID())
	base, hashSet, err := hash.BuildReaderProof(file, numSegments, proofIndex)
	if err != nil {
		return
	}
	sp = siacore.StorageProof{contractID, base, hashSet}
	return
}

// storageProofMaintenance tracks when storage proofs need to be submitted as
// transactions, then creates the proof and submits the transaction.
func (e *Environment) storageProofMaintenance(stateHeight siacore.BlockHeight, rewoundBlocks []siacore.BlockID, appliedBlocks []siacore.BlockID) {
	height := stateHeight - siacore.BlockHeight(len(appliedBlocks))
	var proofs []siacore.StorageProof
	for _ = range rewoundBlocks {
		needActionContracts := e.host.BackwardContracts[height]
		for _, contractEntry := range needActionContracts {
			// Create a proof for this contract.
			proof, err := e.CreateProof(*contractEntry.Contract, contractEntry.ID, height)
			if err != nil {
				panic(err)
			}
			proofs = append(proofs, proof)
		}
		height++
	}

	height = stateHeight - siacore.BlockHeight(len(appliedBlocks))
	for _ = range appliedBlocks {
		needActionContracts := e.host.ForwardContracts[height]
		for _, contractEntry := range needActionContracts {
			// Create a proof for this contract.
			proof, err := e.CreateProof(*contractEntry.Contract, contractEntry.ID, height)
			if err != nil {
				panic(err)
			}
			proofs = append(proofs, proof)

			// Add this contract proof to the backwards contracts list.
			e.host.BackwardContracts[height-2] = append(e.host.BackwardContracts[height-2], contractEntry)

			// Add this contract entry to ForwardContracts windowsize blocks into the future.
			e.host.ForwardContracts[height+contractEntry.Contract.ChallengeFrequency] = append(e.host.ForwardContracts[height+contractEntry.Contract.ChallengeFrequency], contractEntry)
		}
		delete(e.host.ForwardContracts, height)
		height++

	}

	txn := siacore.Transaction{
		StorageProofs: proofs,
	}
	e.AcceptTransaction(txn)
}
<|endoftext|>


<commit_before>// +build linux
// +build !mips,!mipsle,!mips64,!mips64le

package main

import (
	"syscall"

	"golang.org/x/sys/unix"
)

var signalMap = map[string]syscall.Signal{
	"ABRT":   unix.SIGABRT,
	"ALRM":   unix.SIGALRM,
	"BUS":    unix.SIGBUS,
	"CHLD":   unix.SIGCHLD,
	"CLD":    unix.SIGCLD,
	"CONT":   unix.SIGCONT,
	"FPE":    unix.SIGFPE,
	"HUP":    unix.SIGHUP,
	"ILL":    unix.SIGILL,
	"INT":    unix.SIGINT,
	"IO":     unix.SIGIO,
	"IOT":    unix.SIGIOT,
	"KILL":   unix.SIGKILL,
	"PIPE":   unix.SIGPIPE,
	"POLL":   unix.SIGPOLL,
	"PROF":   unix.SIGPROF,
	"PWR":    unix.SIGPWR,
	"QUIT":   unix.SIGQUIT,
	"SEGV":   unix.SIGSEGV,
	"STKFLT": unix.SIGSTKFLT,
	"STOP":   unix.SIGSTOP,
	"SYS":    unix.SIGSYS,
	"TERM":   unix.SIGTERM,
	"TRAP":   unix.SIGTRAP,
	"TSTP":   unix.SIGTSTP,
	"TTIN":   unix.SIGTTIN,
	"TTOU":   unix.SIGTTOU,
	"UNUSED": unix.SIGUNUSED,
	"URG":    unix.SIGURG,
	"USR1":   unix.SIGUSR1,
	"USR2":   unix.SIGUSR2,
	"VTALRM": unix.SIGVTALRM,
	"WINCH":  unix.SIGWINCH,
	"XCPU":   unix.SIGXCPU,
	"XFSZ":   unix.SIGXFSZ,
}
<commit_msg>Stop using unix.SIGUNUSED which has been removed from golang.org/x/sys<commit_after>// +build linux
// +build !mips,!mipsle,!mips64,!mips64le

package main

import (
	"syscall"

	"golang.org/x/sys/unix"
)

var signalMap = map[string]syscall.Signal{
	"ABRT":   unix.SIGABRT,
	"ALRM":   unix.SIGALRM,
	"BUS":    unix.SIGBUS,
	"CHLD":   unix.SIGCHLD,
	"CLD":    unix.SIGCLD,
	"CONT":   unix.SIGCONT,
	"FPE":    unix.SIGFPE,
	"HUP":    unix.SIGHUP,
	"ILL":    unix.SIGILL,
	"INT":    unix.SIGINT,
	"IO":     unix.SIGIO,
	"IOT":    unix.SIGIOT,
	"KILL":   unix.SIGKILL,
	"PIPE":   unix.SIGPIPE,
	"POLL":   unix.SIGPOLL,
	"PROF":   unix.SIGPROF,
	"PWR":    unix.SIGPWR,
	"QUIT":   unix.SIGQUIT,
	"SEGV":   unix.SIGSEGV,
	"STKFLT": unix.SIGSTKFLT,
	"STOP":   unix.SIGSTOP,
	"SYS":    unix.SIGSYS,
	"TERM":   unix.SIGTERM,
	"TRAP":   unix.SIGTRAP,
	"TSTP":   unix.SIGTSTP,
	"TTIN":   unix.SIGTTIN,
	"TTOU":   unix.SIGTTOU,
	"URG":    unix.SIGURG,
	"USR1":   unix.SIGUSR1,
	"USR2":   unix.SIGUSR2,
	"VTALRM": unix.SIGVTALRM,
	"WINCH":  unix.SIGWINCH,
	"XCPU":   unix.SIGXCPU,
	"XFSZ":   unix.SIGXFSZ,
}
<|endoftext|>


<commit_before>package fusetest

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"testing"

	. "github.com/smartystreets/goconvey/convey"
)

func testGitClone(t *testing.T, mountDir string) {
	Convey("Git", t, createDir(mountDir, "GitClone", func(dirPath string) {
		Convey("It should clone repo", func() {
			clone := func(dir string) error {
				cmd := exec.Command("git", "clone", "-q", "https://github.com/sent-hil/bitesized", dir)
				_, err := cmd.Output()
				return err
			}

			md5 := func(dir string) (string, error) {
				cmd := exec.Command("bash", "-c", fmt.Sprintf("ls %s | md5sum", dir))
				output, err := cmd.Output()

				return string(output), err
			}

			So(clone(dirPath), ShouldBeNil)

			Convey("It should compare contents of mount & outside repos", func() {
				// clone git repo to outside mount
				temp, err := ioutil.TempDir("", "")
				So(err, ShouldBeNil)

				So(clone(temp), ShouldBeNil)

				// compare if md5 of inside mount repo is same as outside mount repo
				ta, err := md5(filepath.Join(dirPath, "bitesized"))
				So(err, ShouldBeNil)

				tb, err := md5(filepath.Join(temp, "bitesized"))
				So(err, ShouldBeNil)

				So(ta, ShouldEqual, tb)

				Reset(func() { os.RemoveAll(temp) })
			})
		})
	}))
}
<commit_msg>fusetest: switch to md5 instead of md5sum<commit_after>package fusetest

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"testing"

	. "github.com/smartystreets/goconvey/convey"
)

func testGitClone(t *testing.T, mountDir string) {
	Convey("Git", t, createDir(mountDir, "GitClone", func(dirPath string) {
		Convey("It should clone repo", func() {
			clone := func(dir string) error {
				cmd := exec.Command("git", "clone", "-q", "https://github.com/sent-hil/bitesized", dir)
				_, err := cmd.Output()
				return err
			}

			md5 := func(dir string) (string, error) {
				cmd := exec.Command("bash", "-c", fmt.Sprintf("ls %s | md5", dir))
				output, err := cmd.Output()

				return string(output), err
			}

			So(clone(dirPath), ShouldBeNil)

			Convey("It should compare contents of mount & outside repos", func() {
				// clone git repo to outside mount
				temp, err := ioutil.TempDir("", "")
				So(err, ShouldBeNil)

				So(clone(temp), ShouldBeNil)

				// compare if md5 of inside mount repo is same as outside mount repo
				ta, err := md5(filepath.Join(dirPath, "bitesized"))
				So(err, ShouldBeNil)

				tb, err := md5(filepath.Join(temp, "bitesized"))
				So(err, ShouldBeNil)

				So(ta, ShouldEqual, tb)

				Reset(func() { os.RemoveAll(temp) })
			})
		})
	}))
}
<|endoftext|>


<commit_before>// Copyright 2013, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package zktopo

import (
	"encoding/json"
	"fmt"
	"path"
	"sort"
	"time"

	log "github.com/golang/glog"
	"github.com/youtube/vitess/go/jscfg"
	"github.com/youtube/vitess/go/vt/topo"
	"github.com/youtube/vitess/go/zk"
	"launchpad.net/gozk/zookeeper"
)

// WatchSleepDuration is how many seconds interval to poll for in case
// the directory that contains a file to watch doesn't exist, or a watch
// is broken. It is exported so individual test and main programs
// can change it.
var WatchSleepDuration = 30 * time.Second

/*
This file contains the serving graph management code of zktopo.Server
*/
func zkPathForCell(cell string) string {
	return fmt.Sprintf("/zk/%v/vt/ns", cell)
}

func zkPathForVtKeyspace(cell, keyspace string) string {
	return path.Join(zkPathForCell(cell), keyspace)
}

func zkPathForVtShard(cell, keyspace, shard string) string {
	return path.Join(zkPathForVtKeyspace(cell, keyspace), shard)
}

func zkPathForVtName(cell, keyspace, shard string, tabletType topo.TabletType) string {
	return path.Join(zkPathForVtShard(cell, keyspace, shard), string(tabletType))
}

// GetSrvTabletTypesPerShard is part of the topo.Server interface
func (zkts *Server) GetSrvTabletTypesPerShard(cell, keyspace, shard string) ([]topo.TabletType, error) {
	zkSgShardPath := zkPathForVtShard(cell, keyspace, shard)
	children, _, err := zkts.zconn.Children(zkSgShardPath)
	if err != nil {
		if zookeeper.IsError(err, zookeeper.ZNONODE) {
			err = topo.ErrNoNode
		}
		return nil, err
	}
	result := make([]topo.TabletType, 0, len(children))
	for _, tt := range children {
		// these two are used for locking
		if tt == "action" || tt == "actionlog" {
			continue
		}
		result = append(result, topo.TabletType(tt))
	}
	return result, nil
}

// UpdateEndPoints is part of the topo.Server interface
func (zkts *Server) UpdateEndPoints(cell, keyspace, shard string, tabletType topo.TabletType, addrs *topo.EndPoints) error {
	path := zkPathForVtName(cell, keyspace, shard, tabletType)
	data := jscfg.ToJson(addrs)
	_, err := zk.CreateRecursive(zkts.zconn, path, data, 0, zookeeper.WorldACL(zookeeper.PERM_ALL))
	if err != nil {
		if zookeeper.IsError(err, zookeeper.ZNODEEXISTS) {
			// Node already exists - just stomp away. Multiple writers shouldn't be here.
			// We use RetryChange here because it won't update the node unnecessarily.
			f := func(oldValue string, oldStat zk.Stat) (string, error) {
				return data, nil
			}
			err = zkts.zconn.RetryChange(path, 0, zookeeper.WorldACL(zookeeper.PERM_ALL), f)
		}
	}
	return err
}

// GetEndPoints is part of the topo.Server interface
func (zkts *Server) GetEndPoints(cell, keyspace, shard string, tabletType topo.TabletType) (*topo.EndPoints, error) {
	path := zkPathForVtName(cell, keyspace, shard, tabletType)
	data, _, err := zkts.zconn.Get(path)
	if err != nil {
		if zookeeper.IsError(err, zookeeper.ZNONODE) {
			err = topo.ErrNoNode
		}
		return nil, err
	}
	result := &topo.EndPoints{}
	if len(data) > 0 {
		if err := json.Unmarshal([]byte(data), result); err != nil {
			return nil, fmt.Errorf("EndPoints unmarshal failed: %v %v", data, err)
		}
	}
	return result, nil
}

// DeleteEndPoints is part of the topo.Server interface
func (zkts *Server) DeleteEndPoints(cell, keyspace, shard string, tabletType topo.TabletType) error {
	path := zkPathForVtName(cell, keyspace, shard, tabletType)
	err := zkts.zconn.Delete(path, -1)
	if err != nil {
		if zookeeper.IsError(err, zookeeper.ZNONODE) {
			err = topo.ErrNoNode
		}
		return err
	}
	return nil
}

// UpdateSrvShard is part of the topo.Server interface
func (zkts *Server) UpdateSrvShard(cell, keyspace, shard string, srvShard *topo.SrvShard) error {
	path := zkPathForVtShard(cell, keyspace, shard)
	data := jscfg.ToJson(srvShard)
	_, err := zkts.zconn.Set(path, data, -1)
	return err
}

// GetSrvShard is part of the topo.Server interface
func (zkts *Server) GetSrvShard(cell, keyspace, shard string) (*topo.SrvShard, error) {
	path := zkPathForVtShard(cell, keyspace, shard)
	data, stat, err := zkts.zconn.Get(path)
	if err != nil {
		if zookeeper.IsError(err, zookeeper.ZNONODE) {
			err = topo.ErrNoNode
		}
		return nil, err
	}
	srvShard := topo.NewSrvShard(int64(stat.Version()))
	if len(data) > 0 {
		if err := json.Unmarshal([]byte(data), srvShard); err != nil {
			return nil, fmt.Errorf("SrvShard unmarshal failed: %v %v", data, err)
		}
	}
	return srvShard, nil
}

// DeleteSrvShard is part of the topo.Server interface
func (zkts *Server) DeleteSrvShard(cell, keyspace, shard string) error {
	path := zkPathForVtShard(cell, keyspace, shard)
	err := zkts.zconn.Delete(path, -1)
	if err != nil {
		if zookeeper.IsError(err, zookeeper.ZNONODE) {
			err = topo.ErrNoNode
		}
		return err
	}
	return nil
}

// UpdateSrvKeyspace is part of the topo.Server interface
func (zkts *Server) UpdateSrvKeyspace(cell, keyspace string, srvKeyspace *topo.SrvKeyspace) error {
	path := zkPathForVtKeyspace(cell, keyspace)
	data := jscfg.ToJson(srvKeyspace)
	_, err := zkts.zconn.Set(path, data, -1)
	if zookeeper.IsError(err, zookeeper.ZNONODE) {
		_, err = zk.CreateRecursive(zkts.zconn, path, data, 0, zookeeper.WorldACL(zookeeper.PERM_ALL))
	}
	return err
}

// GetSrvKeyspace is part of the topo.Server interface
func (zkts *Server) GetSrvKeyspace(cell, keyspace string) (*topo.SrvKeyspace, error) {
	path := zkPathForVtKeyspace(cell, keyspace)
	data, stat, err := zkts.zconn.Get(path)
	if err != nil {
		if zookeeper.IsError(err, zookeeper.ZNONODE) {
			err = topo.ErrNoNode
		}
		return nil, err
	}
	srvKeyspace := topo.NewSrvKeyspace(int64(stat.Version()))
	if len(data) > 0 {
		if err := json.Unmarshal([]byte(data), srvKeyspace); err != nil {
			return nil, fmt.Errorf("SrvKeyspace unmarshal failed: %v %v", data, err)
		}
	}
	return srvKeyspace, nil
}

// GetSrvKeyspaceNames is part of the topo.Server interface
func (zkts *Server) GetSrvKeyspaceNames(cell string) ([]string, error) {
	children, _, err := zkts.zconn.Children(zkPathForCell(cell))
	if err != nil {
		if zookeeper.IsError(err, zookeeper.ZNONODE) {
			return nil, nil
		}
		return nil, err
	}

	sort.Strings(children)
	return children, nil
}

var errSkipUpdate = fmt.Errorf("skip update")

func (zkts *Server) updateTabletEndpoint(oldValue string, oldStat zk.Stat, addr *topo.EndPoint) (newValue string, err error) {
	if oldStat == nil {
		// The incoming object doesn't exist - we haven't been placed in the serving
		// graph yet, so don't update. Assume the next process that rebuilds the graph
		// will get the updated tablet location.
		return "", errSkipUpdate
	}

	var addrs *topo.EndPoints
	if oldValue != "" {
		addrs = &topo.EndPoints{}
		if len(oldValue) > 0 {
			if err := json.Unmarshal([]byte(oldValue), addrs); err != nil {
				return "", fmt.Errorf("EndPoints unmarshal failed: %v %v", oldValue, err)
			}
		}

		foundTablet := false
		for i, entry := range addrs.Entries {
			if entry.Uid == addr.Uid {
				foundTablet = true
				if !topo.EndPointEquality(&entry, addr) {
					addrs.Entries[i] = *addr
				}
				break
			}
		}

		if !foundTablet {
			addrs.Entries = append(addrs.Entries, *addr)
		}
	} else {
		addrs = topo.NewEndPoints()
		addrs.Entries = append(addrs.Entries, *addr)
	}
	return jscfg.ToJson(addrs), nil
}

// UpdateTabletEndpoint is part of the topo.Server interface
func (zkts *Server) UpdateTabletEndpoint(cell, keyspace, shard string, tabletType topo.TabletType, addr *topo.EndPoint) error {
	path := zkPathForVtName(cell, keyspace, shard, tabletType)
	f := func(oldValue string, oldStat zk.Stat) (string, error) {
		return zkts.updateTabletEndpoint(oldValue, oldStat, addr)
	}
	err := zkts.zconn.RetryChange(path, 0, zookeeper.WorldACL(zookeeper.PERM_ALL), f)
	if err == errSkipUpdate || zookeeper.IsError(err, zookeeper.ZNONODE) {
		err = nil
	}
	return err
}

// WatchEndPoints is part of the topo.Server interface
func (zkts *Server) WatchEndPoints(cell, keyspace, shard string, tabletType topo.TabletType) (<-chan *topo.EndPoints, chan<- struct{}, error) {
	path := zkPathForVtName(cell, keyspace, shard, tabletType)

	notifications := make(chan *topo.EndPoints, 10)
	stopWatching := make(chan struct{})

	go func() {
		for {
			// set the watch
			data, _, watch, err := zkts.zconn.GetW(path)
			if err != nil {
				if zookeeper.IsError(err, zookeeper.ZNONODE) {
					// the parent directory doesn't exist
					notifications <- nil
				}

				log.Errorf("Cannot set watch on %v, waiting for %s to retry: %v", path, WatchSleepDuration, err)
				time.Sleep(WatchSleepDuration)
				continue
			}

			// get the initial value, send it, or send nil if no
			// data
			var ep *topo.EndPoints
			sendIt := true
			if len(data) > 0 {
				ep = &topo.EndPoints{}
				if err := json.Unmarshal([]byte(data), ep); err != nil {
					log.Errorf("EndPoints unmarshal failed: %v %v", data, err)
					sendIt = false
				}
			}
			if sendIt {
				notifications <- ep
			}

			// now act on the watch
			select {
			case event, ok := <-watch:
				if !ok {
					log.Warningf("watch on %v was closed, waiting for %s to retry", path, WatchSleepDuration)
					time.Sleep(WatchSleepDuration)
					continue
				}

				if !event.Ok() {
					log.Warningf("received a non-OK event, waiting for %s to retry", path, WatchSleepDuration)
					time.Sleep(WatchSleepDuration)
					continue
				}
			case <-stopWatching:
				// user is not interested any more
				close(notifications)
				return
			}
		}
	}()

	return notifications, stopWatching, nil

}
<commit_msg>When sleeping waiting for time, also monitor abort channel.<commit_after>// Copyright 2013, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package zktopo

import (
	"encoding/json"
	"fmt"
	"path"
	"sort"
	"time"

	log "github.com/golang/glog"
	"github.com/youtube/vitess/go/jscfg"
	"github.com/youtube/vitess/go/vt/topo"
	"github.com/youtube/vitess/go/zk"
	"launchpad.net/gozk/zookeeper"
)

// WatchSleepDuration is how many seconds interval to poll for in case
// the directory that contains a file to watch doesn't exist, or a watch
// is broken. It is exported so individual test and main programs
// can change it.
var WatchSleepDuration = 30 * time.Second

/*
This file contains the serving graph management code of zktopo.Server
*/
func zkPathForCell(cell string) string {
	return fmt.Sprintf("/zk/%v/vt/ns", cell)
}

func zkPathForVtKeyspace(cell, keyspace string) string {
	return path.Join(zkPathForCell(cell), keyspace)
}

func zkPathForVtShard(cell, keyspace, shard string) string {
	return path.Join(zkPathForVtKeyspace(cell, keyspace), shard)
}

func zkPathForVtName(cell, keyspace, shard string, tabletType topo.TabletType) string {
	return path.Join(zkPathForVtShard(cell, keyspace, shard), string(tabletType))
}

// GetSrvTabletTypesPerShard is part of the topo.Server interface
func (zkts *Server) GetSrvTabletTypesPerShard(cell, keyspace, shard string) ([]topo.TabletType, error) {
	zkSgShardPath := zkPathForVtShard(cell, keyspace, shard)
	children, _, err := zkts.zconn.Children(zkSgShardPath)
	if err != nil {
		if zookeeper.IsError(err, zookeeper.ZNONODE) {
			err = topo.ErrNoNode
		}
		return nil, err
	}
	result := make([]topo.TabletType, 0, len(children))
	for _, tt := range children {
		// these two are used for locking
		if tt == "action" || tt == "actionlog" {
			continue
		}
		result = append(result, topo.TabletType(tt))
	}
	return result, nil
}

// UpdateEndPoints is part of the topo.Server interface
func (zkts *Server) UpdateEndPoints(cell, keyspace, shard string, tabletType topo.TabletType, addrs *topo.EndPoints) error {
	path := zkPathForVtName(cell, keyspace, shard, tabletType)
	data := jscfg.ToJson(addrs)
	_, err := zk.CreateRecursive(zkts.zconn, path, data, 0, zookeeper.WorldACL(zookeeper.PERM_ALL))
	if err != nil {
		if zookeeper.IsError(err, zookeeper.ZNODEEXISTS) {
			// Node already exists - just stomp away. Multiple writers shouldn't be here.
			// We use RetryChange here because it won't update the node unnecessarily.
			f := func(oldValue string, oldStat zk.Stat) (string, error) {
				return data, nil
			}
			err = zkts.zconn.RetryChange(path, 0, zookeeper.WorldACL(zookeeper.PERM_ALL), f)
		}
	}
	return err
}

// GetEndPoints is part of the topo.Server interface
func (zkts *Server) GetEndPoints(cell, keyspace, shard string, tabletType topo.TabletType) (*topo.EndPoints, error) {
	path := zkPathForVtName(cell, keyspace, shard, tabletType)
	data, _, err := zkts.zconn.Get(path)
	if err != nil {
		if zookeeper.IsError(err, zookeeper.ZNONODE) {
			err = topo.ErrNoNode
		}
		return nil, err
	}
	result := &topo.EndPoints{}
	if len(data) > 0 {
		if err := json.Unmarshal([]byte(data), result); err != nil {
			return nil, fmt.Errorf("EndPoints unmarshal failed: %v %v", data, err)
		}
	}
	return result, nil
}

// DeleteEndPoints is part of the topo.Server interface
func (zkts *Server) DeleteEndPoints(cell, keyspace, shard string, tabletType topo.TabletType) error {
	path := zkPathForVtName(cell, keyspace, shard, tabletType)
	err := zkts.zconn.Delete(path, -1)
	if err != nil {
		if zookeeper.IsError(err, zookeeper.ZNONODE) {
			err = topo.ErrNoNode
		}
		return err
	}
	return nil
}

// UpdateSrvShard is part of the topo.Server interface
func (zkts *Server) UpdateSrvShard(cell, keyspace, shard string, srvShard *topo.SrvShard) error {
	path := zkPathForVtShard(cell, keyspace, shard)
	data := jscfg.ToJson(srvShard)
	_, err := zkts.zconn.Set(path, data, -1)
	return err
}

// GetSrvShard is part of the topo.Server interface
func (zkts *Server) GetSrvShard(cell, keyspace, shard string) (*topo.SrvShard, error) {
	path := zkPathForVtShard(cell, keyspace, shard)
	data, stat, err := zkts.zconn.Get(path)
	if err != nil {
		if zookeeper.IsError(err, zookeeper.ZNONODE) {
			err = topo.ErrNoNode
		}
		return nil, err
	}
	srvShard := topo.NewSrvShard(int64(stat.Version()))
	if len(data) > 0 {
		if err := json.Unmarshal([]byte(data), srvShard); err != nil {
			return nil, fmt.Errorf("SrvShard unmarshal failed: %v %v", data, err)
		}
	}
	return srvShard, nil
}

// DeleteSrvShard is part of the topo.Server interface
func (zkts *Server) DeleteSrvShard(cell, keyspace, shard string) error {
	path := zkPathForVtShard(cell, keyspace, shard)
	err := zkts.zconn.Delete(path, -1)
	if err != nil {
		if zookeeper.IsError(err, zookeeper.ZNONODE) {
			err = topo.ErrNoNode
		}
		return err
	}
	return nil
}

// UpdateSrvKeyspace is part of the topo.Server interface
func (zkts *Server) UpdateSrvKeyspace(cell, keyspace string, srvKeyspace *topo.SrvKeyspace) error {
	path := zkPathForVtKeyspace(cell, keyspace)
	data := jscfg.ToJson(srvKeyspace)
	_, err := zkts.zconn.Set(path, data, -1)
	if zookeeper.IsError(err, zookeeper.ZNONODE) {
		_, err = zk.CreateRecursive(zkts.zconn, path, data, 0, zookeeper.WorldACL(zookeeper.PERM_ALL))
	}
	return err
}

// GetSrvKeyspace is part of the topo.Server interface
func (zkts *Server) GetSrvKeyspace(cell, keyspace string) (*topo.SrvKeyspace, error) {
	path := zkPathForVtKeyspace(cell, keyspace)
	data, stat, err := zkts.zconn.Get(path)
	if err != nil {
		if zookeeper.IsError(err, zookeeper.ZNONODE) {
			err = topo.ErrNoNode
		}
		return nil, err
	}
	srvKeyspace := topo.NewSrvKeyspace(int64(stat.Version()))
	if len(data) > 0 {
		if err := json.Unmarshal([]byte(data), srvKeyspace); err != nil {
			return nil, fmt.Errorf("SrvKeyspace unmarshal failed: %v %v", data, err)
		}
	}
	return srvKeyspace, nil
}

// GetSrvKeyspaceNames is part of the topo.Server interface
func (zkts *Server) GetSrvKeyspaceNames(cell string) ([]string, error) {
	children, _, err := zkts.zconn.Children(zkPathForCell(cell))
	if err != nil {
		if zookeeper.IsError(err, zookeeper.ZNONODE) {
			return nil, nil
		}
		return nil, err
	}

	sort.Strings(children)
	return children, nil
}

var errSkipUpdate = fmt.Errorf("skip update")

func (zkts *Server) updateTabletEndpoint(oldValue string, oldStat zk.Stat, addr *topo.EndPoint) (newValue string, err error) {
	if oldStat == nil {
		// The incoming object doesn't exist - we haven't been placed in the serving
		// graph yet, so don't update. Assume the next process that rebuilds the graph
		// will get the updated tablet location.
		return "", errSkipUpdate
	}

	var addrs *topo.EndPoints
	if oldValue != "" {
		addrs = &topo.EndPoints{}
		if len(oldValue) > 0 {
			if err := json.Unmarshal([]byte(oldValue), addrs); err != nil {
				return "", fmt.Errorf("EndPoints unmarshal failed: %v %v", oldValue, err)
			}
		}

		foundTablet := false
		for i, entry := range addrs.Entries {
			if entry.Uid == addr.Uid {
				foundTablet = true
				if !topo.EndPointEquality(&entry, addr) {
					addrs.Entries[i] = *addr
				}
				break
			}
		}

		if !foundTablet {
			addrs.Entries = append(addrs.Entries, *addr)
		}
	} else {
		addrs = topo.NewEndPoints()
		addrs.Entries = append(addrs.Entries, *addr)
	}
	return jscfg.ToJson(addrs), nil
}

// UpdateTabletEndpoint is part of the topo.Server interface
func (zkts *Server) UpdateTabletEndpoint(cell, keyspace, shard string, tabletType topo.TabletType, addr *topo.EndPoint) error {
	path := zkPathForVtName(cell, keyspace, shard, tabletType)
	f := func(oldValue string, oldStat zk.Stat) (string, error) {
		return zkts.updateTabletEndpoint(oldValue, oldStat, addr)
	}
	err := zkts.zconn.RetryChange(path, 0, zookeeper.WorldACL(zookeeper.PERM_ALL), f)
	if err == errSkipUpdate || zookeeper.IsError(err, zookeeper.ZNONODE) {
		err = nil
	}
	return err
}

// WatchEndPoints is part of the topo.Server interface
func (zkts *Server) WatchEndPoints(cell, keyspace, shard string, tabletType topo.TabletType) (<-chan *topo.EndPoints, chan<- struct{}, error) {
	path := zkPathForVtName(cell, keyspace, shard, tabletType)

	notifications := make(chan *topo.EndPoints, 10)
	stopWatching := make(chan struct{})

	// waitOrInterrupted will return true if stopWatching is triggered
	waitOrInterrupted := func() bool {
		timer := time.After(WatchSleepDuration)
		select {
		case <-stopWatching:
			close(notifications)
			return true
		case <-timer:
		}
		return false
	}

	go func() {
		for {
			// set the watch
			data, _, watch, err := zkts.zconn.GetW(path)
			if err != nil {
				if zookeeper.IsError(err, zookeeper.ZNONODE) {
					// the parent directory doesn't exist
					notifications <- nil
				}

				log.Errorf("Cannot set watch on %v, waiting for %s to retry: %v", path, WatchSleepDuration, err)
				if waitOrInterrupted() {
					return
				}
				continue
			}

			// get the initial value, send it, or send nil if no
			// data
			var ep *topo.EndPoints
			sendIt := true
			if len(data) > 0 {
				ep = &topo.EndPoints{}
				if err := json.Unmarshal([]byte(data), ep); err != nil {
					log.Errorf("EndPoints unmarshal failed: %v %v", data, err)
					sendIt = false
				}
			}
			if sendIt {
				notifications <- ep
			}

			// now act on the watch
			select {
			case event, ok := <-watch:
				if !ok {
					log.Warningf("watch on %v was closed, waiting for %s to retry", path, WatchSleepDuration)
					if waitOrInterrupted() {
						return
					}
					continue
				}

				if !event.Ok() {
					log.Warningf("received a non-OK event, waiting for %s to retry", path, WatchSleepDuration)
					if waitOrInterrupted() {
						return
					}
				}
			case <-stopWatching:
				// user is not interested any more
				close(notifications)
				return
			}
		}
	}()

	return notifications, stopWatching, nil

}
<|endoftext|>