<commit_before>package main_test

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"reflect"
	"strconv"
	"testing"
	"time"

	"github.com/influxdb/influxdb"
	"github.com/influxdb/influxdb/client"
	"github.com/influxdb/influxdb/influxql"
	"github.com/influxdb/influxdb/messaging"

	main "github.com/influxdb/influxdb/cmd/influxd"
)

type node struct {
	broker *messaging.Broker
	server *influxdb.Server
}

// createCombinedNodeCluster creates a cluster of nServers nodes, each of which
// runs as both a Broker and Data node. If any part cluster creation fails,
// the testing is marked as failed.
//
// This function returns a slice of nodes, the first of which will be the leader.
func createCombinedNodeCluster(t *testing.T, testName string, nNodes, basePort int) []node {
	t.Logf("Creating cluster of %d nodes for test %s", nNodes, testName)
	if nNodes < 1 {
		t.Fatalf("Test %s: asked to create nonsense cluster", testName)
	}

	nodes := make([]node, 0)

	tmpDir := os.TempDir()
	tmpBrokerDir := filepath.Join(tmpDir, "broker-integration-test")
	tmpDataDir := filepath.Join(tmpDir, "data-integration-test")
	t.Logf("Test %s: using tmp directory %q for brokers\n", testName, tmpBrokerDir)
	t.Logf("Test %s: using tmp directory %q for data nodes\n", testName, tmpDataDir)
	// Sometimes if a test fails, it's because of a log.Fatal() in the program.
	// This prevents the defer from cleaning up directories.
	// To be safe, nuke them always before starting
	_ = os.RemoveAll(tmpBrokerDir)
	_ = os.RemoveAll(tmpDataDir)

	// Create the first node, special case.
	c := main.NewConfig()
	c.Broker.Dir = filepath.Join(tmpBrokerDir, strconv.Itoa(basePort))
	c.Data.Dir = filepath.Join(tmpDataDir, strconv.Itoa(basePort))
	c.Broker.Port = basePort
	c.Data.Port = basePort

	b, s := main.Run(c, "", "x.x", os.Stderr)
	if b == nil {
		t.Fatalf("Test %s: failed to create broker on port %d", testName, basePort)
	}
	if s == nil {
		t.Fatalf("Test %s: failed to create leader data node on port %d", testName, basePort)
	}
	nodes = append(nodes, node{broker: b, server: s})

	// Create subsequent nodes, which join to first node.
	for i := 1; i < nNodes; i++ {
		nextPort := basePort + i
		c.Broker.Dir = filepath.Join(tmpBrokerDir, strconv.Itoa(nextPort))
		c.Data.Dir = filepath.Join(tmpDataDir, strconv.Itoa(nextPort))
		c.Broker.Port = nextPort
		c.Data.Port = nextPort

		b, s := main.Run(c, "http://localhost:"+strconv.Itoa(basePort), "x.x", os.Stderr)
		if b == nil {
			t.Fatalf("Test %s: failed to create following broker on port %d", testName, basePort)
		}
		if s == nil {
			t.Fatalf("Test %s: failed to create following data node on port %d", testName, basePort)
		}
		nodes = append(nodes, node{broker: b, server: s})
	}

	return nodes
}

// createDatabase creates a database, and verifies that the creation was successful.
func createDatabase(t *testing.T, testName string, serverURL *url.URL, database string) {
	t.Logf("Test: %s: creating database %s", testName, database)
	u := urlFor(serverURL, "query", url.Values{"q": []string{"CREATE DATABASE foo"}})
	resp, err := http.Get(u.String())
	if err != nil {
		t.Fatalf("Couldn't create database: %s", err)
	}
	defer resp.Body.Close()

	var results client.Results
	err = json.NewDecoder(resp.Body).Decode(&results)
	if err != nil {
		t.Fatalf("Couldn't decode results: %v", err)
	}

	if results.Error() != nil {
		t.Logf("results.Error(): %q", results.Error().Error())
	}

	if resp.StatusCode != http.StatusOK {
		t.Fatalf("Create database failed.  Unexpected status code.  expected: %d, actual %d", http.StatusOK, resp.StatusCode)
	}

	if len(results.Results) != 1 {
		t.Fatalf("Create database failed.  Unexpected results length.  expected: %d, actual %d", 1, len(results.Results))
	}

	// Query the database exists
	u = urlFor(serverURL, "query", url.Values{"q": []string{"SHOW DATABASES"}})
	resp, err = http.Get(u.String())
	if err != nil {
		t.Fatalf("Couldn't query databases: %s", err)
	}
	defer resp.Body.Close()

	err = json.NewDecoder(resp.Body).Decode(&results)
	if err != nil {
		t.Fatalf("Couldn't decode results: %v", err)
	}

	if results.Error() != nil {
		t.Logf("results.Error(): %q", results.Error().Error())
	}

	if resp.StatusCode != http.StatusOK {
		t.Fatalf("show databases failed.  Unexpected status code.  expected: %d, actual %d", http.StatusOK, resp.StatusCode)
	}

	expectedResults := client.Results{
		Results: []client.Result{
			{Rows: []influxql.Row{
				influxql.Row{
					Columns: []string{"name"},
					Values:  [][]interface{}{{"foo"}},
				},
			}},
		},
	}
	if !reflect.DeepEqual(results, expectedResults) {
		t.Fatalf("show databases failed.  Unexpected results.  expected: %+v, actual %+v", expectedResults, results)
	}
}

// createRetentionPolicy creates a retetention policy and verifies that the creation was successful.
func createRetentionPolicy(t *testing.T, testName string, serverURL *url.URL, database, retention string, replicaN int) {
	t.Log("Creating retention policy")
	replication := fmt.Sprintf("CREATE RETENTION POLICY bar ON foo DURATION 1h REPLICATION %d DEFAULT", replicaN)
	u := urlFor(serverURL, "query", url.Values{"q": []string{replication}})
	resp, err := http.Get(u.String())
	if err != nil {
		t.Fatalf("Couldn't create retention policy: %s", err)
	}
	defer resp.Body.Close()

	var results client.Results
	err = json.NewDecoder(resp.Body).Decode(&results)
	if err != nil {
		t.Fatalf("Couldn't decode results: %v", err)
	}

	if results.Error() != nil {
		t.Logf("results.Error(): %q", results.Error().Error())
	}

	if resp.StatusCode != http.StatusOK {
		t.Fatalf("Create retention policy failed.  Unexpected status code.  expected: %d, actual %d", http.StatusOK, resp.StatusCode)
	}

	if len(results.Results) != 1 {
		t.Fatalf("Create retention policy failed.  Unexpected results length.  expected: %d, actual %d", 1, len(results.Results))
	}
}

// simpleWriteAndQuery creates a simple database, retention policy, and replicates
// the data across all nodes. It then ensures a series of writes and queries are OK.
func simpleWriteAndQuery(t *testing.T, testname string, serverURL *url.URL, nNodes int) {
	now := time.Now().UTC()
	var results client.Results

	// Write Data
	t.Log("Write data")
	u := urlFor(serverURL, "write", url.Values{})

	buf := []byte(fmt.Sprintf(`{"database" : "foo", "retentionPolicy" : "bar", "points": [{"name": "cpu", "tags": {"host": "server01"},"timestamp": %d, "precision":"n","values": {"value": 100}}]}`, now.UnixNano()))
	t.Logf("Writing raw data: %s", string(buf))
	resp, err := http.Post(u.String(), "application/json", bytes.NewReader(buf))
	if err != nil {
		t.Fatalf("Couldn't write data: %s", err)
	}
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("Write to database failed.  Unexpected status code.  expected: %d, actual %d", http.StatusOK, resp.StatusCode)
	}

	// Need some time for server to get consensus and write data
	// TODO corylanou query the status endpoint for the server and wait for the index to update to know the write was applied
	time.Sleep(time.Duration(nNodes) * time.Second)

	// Query the data exists
	t.Log("Query data")
	u = urlFor(serverURL, "query", url.Values{"q": []string{`select value from "foo"."bar".cpu`}, "db": []string{"foo"}})
	resp, err = http.Get(u.String())
	if err != nil {
		t.Fatalf("Couldn't query databases: %s", err)
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		t.Fatalf("Couldn't read body of response: %s", err)
	}
	t.Logf("resp.Body: %s\n", string(body))

	dec := json.NewDecoder(bytes.NewReader(body))
	dec.UseNumber()
	err = dec.Decode(&results)
	if err != nil {
		t.Fatalf("Couldn't decode results: %v", err)
	}

	if results.Error() != nil {
		t.Logf("results.Error(): %q", results.Error().Error())
	}

	if resp.StatusCode != http.StatusOK {
		t.Fatalf("query databases failed.  Unexpected status code.  expected: %d, actual %d", http.StatusOK, resp.StatusCode)
	}

	expectedResults := client.Results{
		Results: []client.Result{
			{Rows: []influxql.Row{
				{
					Name:    "cpu",
					Columns: []string{"time", "value"},
					Values: [][]interface{}{
						[]interface{}{now.Format(time.RFC3339Nano), json.Number("100")},
					},
				}}},
		},
	}

	if !reflect.DeepEqual(results, expectedResults) {
		t.Logf("Expected:\n")
		t.Logf("%#v\n", expectedResults)
		t.Logf("Actual:\n")
		t.Logf("%#v\n", results)
		t.Fatalf("query databases failed.  Unexpected results.")
	}
}

func Test_ServerSingleIntegration(t *testing.T) {
	nNodes := 1
	basePort := 8090
	testName := "single node"
	createCombinedNodeCluster(t, "single node", nNodes, basePort)

	serverURL := &url.URL{
		Scheme: "http",
		Host:   "localhost:" + strconv.Itoa(basePort),
	}

	createDatabase(t, testName, serverURL, "foo")
	createRetentionPolicy(t, testName, serverURL, "foo", "bar", nNodes)
	simpleWriteAndQuery(t, testName, serverURL, nNodes)
}

func Test_Server3NodeIntegration(t *testing.T) {
	nNodes := 3
	basePort := 8190
	testName := "3 node"
	createCombinedNodeCluster(t, testName, nNodes, basePort)

	serverURL := &url.URL{
		Scheme: "http",
		Host:   "localhost:" + strconv.Itoa(basePort),
	}

	createDatabase(t, testName, serverURL, "foo")
	createRetentionPolicy(t, testName, serverURL, "foo", "bar", nNodes)
	simpleWriteAndQuery(t, testName, serverURL, nNodes)
}

func Test_Server5NodeIntegration(t *testing.T) {
	nNodes := 5
	basePort := 8290
	testName := "5 node"
	createCombinedNodeCluster(t, testName, nNodes, basePort)

	serverURL := &url.URL{
		Scheme: "http",
		Host:   "localhost:" + strconv.Itoa(basePort),
	}

	createDatabase(t, testName, serverURL, "foo")
	createRetentionPolicy(t, testName, serverURL, "foo", "bar", nNodes)
	simpleWriteAndQuery(t, testName, serverURL, nNodes)
}

func urlFor(u *url.URL, path string, params url.Values) *url.URL {
	u.Path = path
	u.RawQuery = params.Encode()
	return u
}
<commit_msg>Pass around slices of nodes as cluster<commit_after>package main_test

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"reflect"
	"strconv"
	"testing"
	"time"

	"github.com/influxdb/influxdb"
	"github.com/influxdb/influxdb/client"
	"github.com/influxdb/influxdb/influxql"
	"github.com/influxdb/influxdb/messaging"

	main "github.com/influxdb/influxdb/cmd/influxd"
)

// node represents a node under test, which is both a broker and data node.
type node struct {
	broker *messaging.Broker
	server *influxdb.Server
	url    *url.URL
}

// createCombinedNodeCluster creates a cluster of nServers nodes, each of which
// runs as both a Broker and Data node. If any part cluster creation fails,
// the testing is marked as failed.
//
// This function returns a slice of nodes, the first of which will be the leader.
func createCombinedNodeCluster(t *testing.T, testName string, nNodes, basePort int) []node {
	t.Logf("Creating cluster of %d nodes for test %s", nNodes, testName)
	if nNodes < 1 {
		t.Fatalf("Test %s: asked to create nonsense cluster", testName)
	}

	nodes := make([]node, 0)

	tmpDir := os.TempDir()
	tmpBrokerDir := filepath.Join(tmpDir, "broker-integration-test")
	tmpDataDir := filepath.Join(tmpDir, "data-integration-test")
	t.Logf("Test %s: using tmp directory %q for brokers\n", testName, tmpBrokerDir)
	t.Logf("Test %s: using tmp directory %q for data nodes\n", testName, tmpDataDir)
	// Sometimes if a test fails, it's because of a log.Fatal() in the program.
	// This prevents the defer from cleaning up directories.
	// To be safe, nuke them always before starting
	_ = os.RemoveAll(tmpBrokerDir)
	_ = os.RemoveAll(tmpDataDir)

	// Create the first node, special case.
	c := main.NewConfig()
	c.Broker.Dir = filepath.Join(tmpBrokerDir, strconv.Itoa(basePort))
	c.Data.Dir = filepath.Join(tmpDataDir, strconv.Itoa(basePort))
	c.Broker.Port = basePort
	c.Data.Port = basePort

	b, s := main.Run(c, "", "x.x", os.Stderr)
	if b == nil {
		t.Fatalf("Test %s: failed to create broker on port %d", testName, basePort)
	}
	if s == nil {
		t.Fatalf("Test %s: failed to create leader data node on port %d", testName, basePort)
	}
	nodes = append(nodes, node{
		broker: b,
		server: s,
		url:    &url.URL{Scheme: "http", Host: "localhost:" + strconv.Itoa(basePort)},
	})

	// Create subsequent nodes, which join to first node.
	for i := 1; i < nNodes; i++ {
		nextPort := basePort + i
		c.Broker.Dir = filepath.Join(tmpBrokerDir, strconv.Itoa(nextPort))
		c.Data.Dir = filepath.Join(tmpDataDir, strconv.Itoa(nextPort))
		c.Broker.Port = nextPort
		c.Data.Port = nextPort

		b, s := main.Run(c, "http://localhost:"+strconv.Itoa(basePort), "x.x", os.Stderr)
		if b == nil {
			t.Fatalf("Test %s: failed to create following broker on port %d", testName, basePort)
		}
		if s == nil {
			t.Fatalf("Test %s: failed to create following data node on port %d", testName, basePort)
		}

		nodes = append(nodes, node{
			broker: b,
			server: s,
			url:    &url.URL{Scheme: "http", Host: "localhost:" + strconv.Itoa(nextPort)},
		})
	}

	return nodes
}

// createDatabase creates a database, and verifies that the creation was successful.
func createDatabase(t *testing.T, testName string, nodes []node, database string) {
	t.Logf("Test: %s: creating database %s", testName, database)
	serverURL := nodes[0].url

	u := urlFor(serverURL, "query", url.Values{"q": []string{"CREATE DATABASE foo"}})
	resp, err := http.Get(u.String())
	if err != nil {
		t.Fatalf("Couldn't create database: %s", err)
	}
	defer resp.Body.Close()

	var results client.Results
	err = json.NewDecoder(resp.Body).Decode(&results)
	if err != nil {
		t.Fatalf("Couldn't decode results: %v", err)
	}

	if results.Error() != nil {
		t.Logf("results.Error(): %q", results.Error().Error())
	}

	if resp.StatusCode != http.StatusOK {
		t.Fatalf("Create database failed.  Unexpected status code.  expected: %d, actual %d", http.StatusOK, resp.StatusCode)
	}

	if len(results.Results) != 1 {
		t.Fatalf("Create database failed.  Unexpected results length.  expected: %d, actual %d", 1, len(results.Results))
	}

	// Query the database exists
	u = urlFor(serverURL, "query", url.Values{"q": []string{"SHOW DATABASES"}})
	resp, err = http.Get(u.String())
	if err != nil {
		t.Fatalf("Couldn't query databases: %s", err)
	}
	defer resp.Body.Close()

	err = json.NewDecoder(resp.Body).Decode(&results)
	if err != nil {
		t.Fatalf("Couldn't decode results: %v", err)
	}

	if results.Error() != nil {
		t.Logf("results.Error(): %q", results.Error().Error())
	}

	if resp.StatusCode != http.StatusOK {
		t.Fatalf("show databases failed.  Unexpected status code.  expected: %d, actual %d", http.StatusOK, resp.StatusCode)
	}

	expectedResults := client.Results{
		Results: []client.Result{
			{Rows: []influxql.Row{
				influxql.Row{
					Columns: []string{"name"},
					Values:  [][]interface{}{{"foo"}},
				},
			}},
		},
	}
	if !reflect.DeepEqual(results, expectedResults) {
		t.Fatalf("show databases failed.  Unexpected results.  expected: %+v, actual %+v", expectedResults, results)
	}
}

// createRetentionPolicy creates a retetention policy and verifies that the creation was successful.
func createRetentionPolicy(t *testing.T, testName string, nodes []node, database, retention string, replicaN int) {
	t.Log("Creating retention policy")
	serverURL := nodes[0].url
	replication := fmt.Sprintf("CREATE RETENTION POLICY bar ON foo DURATION 1h REPLICATION %d DEFAULT", replicaN)

	u := urlFor(serverURL, "query", url.Values{"q": []string{replication}})
	resp, err := http.Get(u.String())
	if err != nil {
		t.Fatalf("Couldn't create retention policy: %s", err)
	}
	defer resp.Body.Close()

	var results client.Results
	err = json.NewDecoder(resp.Body).Decode(&results)
	if err != nil {
		t.Fatalf("Couldn't decode results: %v", err)
	}

	if results.Error() != nil {
		t.Logf("results.Error(): %q", results.Error().Error())
	}

	if resp.StatusCode != http.StatusOK {
		t.Fatalf("Create retention policy failed.  Unexpected status code.  expected: %d, actual %d", http.StatusOK, resp.StatusCode)
	}

	if len(results.Results) != 1 {
		t.Fatalf("Create retention policy failed.  Unexpected results length.  expected: %d, actual %d", 1, len(results.Results))
	}
}

// simpleWriteAndQuery creates a simple database, retention policy, and replicates
// the data across all nodes. It then ensures a series of writes and queries are OK.
func simpleWriteAndQuery(t *testing.T, testname string, nodes []node, nNodes int) {
	now := time.Now().UTC()
	serverURL := nodes[0].url
	var results client.Results

	// Write Data
	t.Log("Write data")
	u := urlFor(serverURL, "write", url.Values{})

	buf := []byte(fmt.Sprintf(`{"database" : "foo", "retentionPolicy" : "bar", "points": [{"name": "cpu", "tags": {"host": "server01"},"timestamp": %d, "precision":"n","values": {"value": 100}}]}`, now.UnixNano()))
	t.Logf("Writing raw data: %s", string(buf))
	resp, err := http.Post(u.String(), "application/json", bytes.NewReader(buf))
	if err != nil {
		t.Fatalf("Couldn't write data: %s", err)
	}
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("Write to database failed.  Unexpected status code.  expected: %d, actual %d", http.StatusOK, resp.StatusCode)
	}

	// Need some time for server to get consensus and write data
	// TODO corylanou query the status endpoint for the server and wait for the index to update to know the write was applied
	time.Sleep(time.Duration(nNodes) * time.Second)

	// Query the data exists
	t.Log("Query data")
	u = urlFor(serverURL, "query", url.Values{"q": []string{`select value from "foo"."bar".cpu`}, "db": []string{"foo"}})
	resp, err = http.Get(u.String())
	if err != nil {
		t.Fatalf("Couldn't query databases: %s", err)
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		t.Fatalf("Couldn't read body of response: %s", err)
	}
	t.Logf("resp.Body: %s\n", string(body))

	dec := json.NewDecoder(bytes.NewReader(body))
	dec.UseNumber()
	err = dec.Decode(&results)
	if err != nil {
		t.Fatalf("Couldn't decode results: %v", err)
	}

	if results.Error() != nil {
		t.Logf("results.Error(): %q", results.Error().Error())
	}

	if resp.StatusCode != http.StatusOK {
		t.Fatalf("query databases failed.  Unexpected status code.  expected: %d, actual %d", http.StatusOK, resp.StatusCode)
	}

	expectedResults := client.Results{
		Results: []client.Result{
			{Rows: []influxql.Row{
				{
					Name:    "cpu",
					Columns: []string{"time", "value"},
					Values: [][]interface{}{
						[]interface{}{now.Format(time.RFC3339Nano), json.Number("100")},
					},
				}}},
		},
	}

	if !reflect.DeepEqual(results, expectedResults) {
		t.Logf("Expected:\n")
		t.Logf("%#v\n", expectedResults)
		t.Logf("Actual:\n")
		t.Logf("%#v\n", results)
		t.Fatalf("query databases failed.  Unexpected results.")
	}
}

func Test_ServerSingleIntegration(t *testing.T) {
	nNodes := 1
	basePort := 8090
	testName := "single node"
	nodes := createCombinedNodeCluster(t, "single node", nNodes, basePort)

	createDatabase(t, testName, nodes, "foo")
	createRetentionPolicy(t, testName, nodes, "foo", "bar", nNodes)
	simpleWriteAndQuery(t, testName, nodes, nNodes)
}

func Test_Server3NodeIntegration(t *testing.T) {
	nNodes := 3
	basePort := 8190
	testName := "3 node"
	nodes := createCombinedNodeCluster(t, testName, nNodes, basePort)

	createDatabase(t, testName, nodes, "foo")
	createRetentionPolicy(t, testName, nodes, "foo", "bar", nNodes)
	simpleWriteAndQuery(t, testName, nodes, nNodes)
}

func Test_Server5NodeIntegration(t *testing.T) {
	nNodes := 5
	basePort := 8290
	testName := "5 node"
	nodes := createCombinedNodeCluster(t, testName, nNodes, basePort)

	createDatabase(t, testName, nodes, "foo")
	createRetentionPolicy(t, testName, nodes, "foo", "bar", nNodes)
	simpleWriteAndQuery(t, testName, nodes, nNodes)
}

func urlFor(u *url.URL, path string, params url.Values) *url.URL {
	u.Path = path
	u.RawQuery = params.Encode()
	return u
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"strings"

	"github.com/Debian/debiman/internal/archive"
	"github.com/Debian/debiman/internal/manpage"
	"pault.ag/go/debian/control"
)

// mostPopularArchitecture is used as preferred architecture when we
// need to pick an arbitrary architecture. The rationale is that
// downloading the package for the most popular architecture has the
// least bad influence on the mirror server’s caches.
const mostPopularArchitecture = "amd64"

type globalView struct {
	pkgs          []*pkgEntry
	suites        map[string]bool
	idxSuites     map[string]string
	contentByPath map[string][]*contentEntry
	xref          map[string][]*manpage.Meta
}

type distributionIdentifier int

const (
	fromCodename = iota
	fromSuite
)

type distribution struct {
	name       string
	identifier distributionIdentifier
}

// distributions returns a list of all distributions (either codenames
// [e.g. wheezy, jessie] or suites [e.g. testing, unstable]) from the
// -sync_codenames and -sync_suites flags.
func distributions(codenames []string, suites []string) []distribution {
	distributions := make([]distribution, 0, len(codenames)+len(suites))
	for _, e := range codenames {
		e = strings.TrimSpace(e)
		if e == "" {
			continue
		}
		distributions = append(distributions, distribution{
			name:       e,
			identifier: fromCodename})
	}
	for _, e := range suites {
		e = strings.TrimSpace(e)
		if e == "" {
			continue
		}
		distributions = append(distributions, distribution{
			name:       e,
			identifier: fromSuite})
	}
	return distributions
}

func buildGlobalView(ar *archive.Getter, dists []distribution) (globalView, error) {
	res := globalView{
		suites:        make(map[string]bool, len(dists)),
		idxSuites:     make(map[string]string, len(dists)),
		contentByPath: make(map[string][]*contentEntry),
		xref:          make(map[string][]*manpage.Meta),
	}

	for _, dist := range dists {
		release, err := ar.GetRelease(dist.name)
		if err != nil {
			return res, err
		}

		var suite string
		if dist.identifier == fromCodename {
			suite = release.Codename
		} else {
			suite = release.Suite
		}

		res.suites[suite] = true
		res.idxSuites[release.Suite] = suite
		res.idxSuites[release.Codename] = suite

		{
			hashByFilename := make(map[string]*control.SHA256FileHash, len(release.SHA256))
			for idx, fh := range release.SHA256 {
				// fh.Filename contains e.g. “non-free/source/Sources”
				hashByFilename[fh.Filename] = &(release.SHA256[idx])
			}

			content, err := getAllContents(ar, suite, release, hashByFilename)
			if err != nil {
				return res, err
			}

			for _, c := range content {
				res.contentByPath[c.filename] = append(res.contentByPath[c.filename], c)
			}

			var latestVersion map[string]*manpage.PkgMeta
			{
				// Collect package download work units
				var pkgs []*pkgEntry
				var err error
				pkgs, latestVersion, err = getAllPackages(ar, suite, release, hashByFilename, buildContainsMains(content))
				if err != nil {
					return res, err
				}

				log.Printf("Adding %d packages from suite %q", len(pkgs), suite)
				for _, d := range pkgs {
					res.pkgs = append(res.pkgs, d)
				}
			}

			knownIssues := make(map[string][]error)

			// Build a global view of all the manpages (required for cross-referencing).
			// TODO(issue): edge case: packages which got renamed between releases
			for _, c := range content {
				if _, ok := latestVersion[c.suite+"/"+c.binarypkg]; !ok {
					key := c.suite + "/" + c.binarypkg
					knownIssues[key] = append(knownIssues[key],
						fmt.Errorf("Could not determine latest version"))
					continue
				}
				m, err := manpage.FromManPath(strings.TrimPrefix(c.filename, "usr/share/man/"), latestVersion[c.suite+"/"+c.binarypkg])
				if err != nil {
					key := c.suite + "/" + c.binarypkg
					knownIssues[key] = append(knownIssues[key],
						fmt.Errorf("Trying to interpret path %q: ", c.filename, err))
					continue
				}
				// NOTE(stapelberg): this additional verification step
				// is necessary because manpages such as the French
				// manpage for qelectrotech(1) are present in multiple
				// encodings. manpageFromManPath ignores encodings, so
				// if we didn’t filter, we would end up with what
				// looks like duplicates.
				present := false
				for _, x := range res.xref[m.Name] {
					if x.ServingPath() == m.ServingPath() {
						present = true
						break
					}
				}
				if !present {
					res.xref[m.Name] = append(res.xref[m.Name], m)
				}
			}

			for key, errors := range knownIssues {
				// TODO: write these to a known-issues file, parse bug numbers from an auxilliary file
				log.Printf("package %q has errors: %v", key, errors)
			}
		}
	}
	return res, nil
}
<commit_msg>Use a single append() call<commit_after>package main

import (
	"fmt"
	"log"
	"strings"

	"github.com/Debian/debiman/internal/archive"
	"github.com/Debian/debiman/internal/manpage"
	"pault.ag/go/debian/control"
)

// mostPopularArchitecture is used as preferred architecture when we
// need to pick an arbitrary architecture. The rationale is that
// downloading the package for the most popular architecture has the
// least bad influence on the mirror server’s caches.
const mostPopularArchitecture = "amd64"

type globalView struct {
	pkgs          []*pkgEntry
	suites        map[string]bool
	idxSuites     map[string]string
	contentByPath map[string][]*contentEntry
	xref          map[string][]*manpage.Meta
}

type distributionIdentifier int

const (
	fromCodename = iota
	fromSuite
)

type distribution struct {
	name       string
	identifier distributionIdentifier
}

// distributions returns a list of all distributions (either codenames
// [e.g. wheezy, jessie] or suites [e.g. testing, unstable]) from the
// -sync_codenames and -sync_suites flags.
func distributions(codenames []string, suites []string) []distribution {
	distributions := make([]distribution, 0, len(codenames)+len(suites))
	for _, e := range codenames {
		e = strings.TrimSpace(e)
		if e == "" {
			continue
		}
		distributions = append(distributions, distribution{
			name:       e,
			identifier: fromCodename})
	}
	for _, e := range suites {
		e = strings.TrimSpace(e)
		if e == "" {
			continue
		}
		distributions = append(distributions, distribution{
			name:       e,
			identifier: fromSuite})
	}
	return distributions
}

func buildGlobalView(ar *archive.Getter, dists []distribution) (globalView, error) {
	res := globalView{
		suites:        make(map[string]bool, len(dists)),
		idxSuites:     make(map[string]string, len(dists)),
		contentByPath: make(map[string][]*contentEntry),
		xref:          make(map[string][]*manpage.Meta),
	}

	for _, dist := range dists {
		release, err := ar.GetRelease(dist.name)
		if err != nil {
			return res, err
		}

		var suite string
		if dist.identifier == fromCodename {
			suite = release.Codename
		} else {
			suite = release.Suite
		}

		res.suites[suite] = true
		res.idxSuites[release.Suite] = suite
		res.idxSuites[release.Codename] = suite

		{
			hashByFilename := make(map[string]*control.SHA256FileHash, len(release.SHA256))
			for idx, fh := range release.SHA256 {
				// fh.Filename contains e.g. “non-free/source/Sources”
				hashByFilename[fh.Filename] = &(release.SHA256[idx])
			}

			content, err := getAllContents(ar, suite, release, hashByFilename)
			if err != nil {
				return res, err
			}

			for _, c := range content {
				res.contentByPath[c.filename] = append(res.contentByPath[c.filename], c)
			}

			var latestVersion map[string]*manpage.PkgMeta
			{
				// Collect package download work units
				var pkgs []*pkgEntry
				var err error
				pkgs, latestVersion, err = getAllPackages(ar, suite, release, hashByFilename, buildContainsMains(content))
				if err != nil {
					return res, err
				}

				log.Printf("Adding %d packages from suite %q", len(pkgs), suite)
				res.pkgs = append(res.pkgs, pkgs...)
			}

			knownIssues := make(map[string][]error)

			// Build a global view of all the manpages (required for cross-referencing).
			// TODO(issue): edge case: packages which got renamed between releases
			for _, c := range content {
				if _, ok := latestVersion[c.suite+"/"+c.binarypkg]; !ok {
					key := c.suite + "/" + c.binarypkg
					knownIssues[key] = append(knownIssues[key],
						fmt.Errorf("Could not determine latest version"))
					continue
				}
				m, err := manpage.FromManPath(strings.TrimPrefix(c.filename, "usr/share/man/"), latestVersion[c.suite+"/"+c.binarypkg])
				if err != nil {
					key := c.suite + "/" + c.binarypkg
					knownIssues[key] = append(knownIssues[key],
						fmt.Errorf("Trying to interpret path %q: ", c.filename, err))
					continue
				}
				// NOTE(stapelberg): this additional verification step
				// is necessary because manpages such as the French
				// manpage for qelectrotech(1) are present in multiple
				// encodings. manpageFromManPath ignores encodings, so
				// if we didn’t filter, we would end up with what
				// looks like duplicates.
				present := false
				for _, x := range res.xref[m.Name] {
					if x.ServingPath() == m.ServingPath() {
						present = true
						break
					}
				}
				if !present {
					res.xref[m.Name] = append(res.xref[m.Name], m)
				}
			}

			for key, errors := range knownIssues {
				// TODO: write these to a known-issues file, parse bug numbers from an auxilliary file
				log.Printf("package %q has errors: %v", key, errors)
			}
		}
	}
	return res, nil
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"os/signal"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"syscall"
	"time"

	flags "github.com/jessevdk/go-flags"
	"github.com/labstack/echo"
	"github.com/labstack/echo/middleware"
	"github.com/labstack/gommon/log"
	"github.com/lestrrat/go-server-starter/listener"
	"github.com/monochromegane/gannoy"
	"github.com/nightlyone/lockfile"
)

type Options struct {
	DataDir           string `short:"d" long:"data-dir" default:"." description:"Specify the directory where the meta files are located."`
	LogDir            string `short:"l" long:"log-dir" default-mask:"os.Stdout" description:"Specify the log output directory."`
	LockDir           string `short:"L" long:"lock-dir" default:"." description:"Specify the lock file directory. This option is used only server-starter option."`
	WithServerStarter bool   `short:"s" long:"server-starter" default:"false" description:"Use server-starter listener for server address."`
	ShutDownTimeout   int    `short:"t" long:"timeout" default:"10" description:"Specify the number of seconds for shutdown timeout."`
}

var opts Options

type Feature struct {
	W []float64 `json:"features"`
}

func main() {
	_, err := flags.Parse(&opts)
	if err != nil {
		os.Exit(1)
	}

	// Wait old process finishing.
	if opts.WithServerStarter {
		lock, err := initializeLock(opts.LockDir)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
		defer lock.Unlock()
		for {
			if err := lock.TryLock(); err != nil {
				time.Sleep(100 * time.Millisecond)
				continue
			}
			break
		}
	}

	e := echo.New()

	// initialize log
	l, err := initializeLog(opts.LogDir)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	e.Logger.SetLevel(log.INFO)
	e.Logger.SetOutput(l)
	e.Use(middleware.LoggerWithConfig(middleware.LoggerConfig{Output: l}))

	// Load meta files
	files, err := ioutil.ReadDir(opts.DataDir)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	metaCh := make(chan string, len(files))
	gannoyCh := make(chan gannoy.GannoyIndex)
	errCh := make(chan error)
	databases := map[string]gannoy.GannoyIndex{}
	var metaCount int
	for _, file := range files {
		if file.IsDir() || filepath.Ext(file.Name()) != ".meta" {
			continue
		}
		metaCh <- filepath.Join(opts.DataDir, file.Name())
		metaCount++
	}

	for i := 0; i < runtime.GOMAXPROCS(0); i++ {
		go gannoyIndexInitializer(metaCh, gannoyCh, errCh)
	}

loop:
	for {
		select {
		case gannoy := <-gannoyCh:
			key := strings.TrimSuffix(filepath.Base(gannoy.MetaFile()), ".meta")
			databases[key] = gannoy
			if len(databases) >= metaCount {
				close(metaCh)
				close(gannoyCh)
				close(errCh)
				break loop
			}
		case err := <-errCh:
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
	}

	// Define API
	e.GET("/search", func(c echo.Context) error {
		database := c.QueryParam("database")
		if _, ok := databases[database]; !ok {
			return c.NoContent(http.StatusNotFound)
		}
		key, err := strconv.Atoi(c.QueryParam("key"))
		if err != nil {
			key = -1
		}
		limit, err := strconv.Atoi(c.QueryParam("limit"))
		if err != nil {
			limit = 10
		}

		gannoy := databases[database]
		r, err := gannoy.GetNnsByKey(key, limit, -1)
		if err != nil || len(r) == 0 {
			return c.NoContent(http.StatusNotFound)
		}

		return c.JSON(http.StatusOK, r)
	})

	e.PUT("/databases/:database/features/:key", func(c echo.Context) error {
		database := c.Param("database")
		if _, ok := databases[database]; !ok {
			return c.NoContent(http.StatusUnprocessableEntity)
		}
		key, err := strconv.Atoi(c.Param("key"))
		if err != nil {
			return c.NoContent(http.StatusUnprocessableEntity)
		}
		feature := new(Feature)
		if err := c.Bind(feature); err != nil {
			return err
		}

		gannoy := databases[database]
		err = gannoy.AddItem(key, feature.W)
		if err != nil {
			return c.NoContent(http.StatusUnprocessableEntity)
		}
		return c.NoContent(http.StatusOK)
	})

	e.DELETE("/databases/:database/features/:key", func(c echo.Context) error {
		database := c.Param("database")
		if _, ok := databases[database]; !ok {
			return c.NoContent(http.StatusUnprocessableEntity)
		}
		key, err := strconv.Atoi(c.Param("key"))
		if err != nil {
			return c.NoContent(http.StatusUnprocessableEntity)
		}
		gannoy := databases[database]
		err = gannoy.RemoveItem(key)
		if err != nil {
			return c.NoContent(http.StatusUnprocessableEntity)
		}

		return c.NoContent(http.StatusOK)
	})

	// Start server
	address := ":1323"
	sig := os.Interrupt
	if opts.WithServerStarter {
		address = ""
		sig = syscall.SIGTERM
		listeners, err := listener.ListenAll()
		if err != nil && err != listener.ErrNoListeningTarget {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
		e.Listener = listeners[0]
	}

	go func() {
		if err := e.Start(address); err != nil {
			e.Logger.Info("shutting down the server")
		}
	}()

	sigCh := make(chan os.Signal, 1)
	signal.Notify(sigCh, sig)
	<-sigCh

	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(opts.ShutDownTimeout)*time.Second)
	defer cancel()
	if err := e.Shutdown(ctx); err != nil {
		e.Logger.Fatal(err)
	}
}

func initializeLog(logDir string) (*os.File, error) {
	if logDir == "" {
		return os.Stdout, nil
	}
	if err := os.MkdirAll(logDir, os.ModePerm); err != nil {
		return nil, err
	}
	return os.OpenFile("access.log", os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
}

func initializeLock(lockDir string) (lockfile.Lockfile, error) {
	if err := os.MkdirAll(lockDir, os.ModePerm); err != nil {
		return "", err
	}
	lock := "gannoy-server.lock"
	if !filepath.IsAbs(lockDir) {
		lockDir, err := filepath.Abs(lockDir)
		if err != nil {
			return lockfile.Lockfile(""), err
		}
		return lockfile.New(filepath.Join(lockDir, lock))
	}
	return lockfile.New(filepath.Join(lockDir, lock))
}

func gannoyIndexInitializer(metaCh chan string, gannoyCh chan gannoy.GannoyIndex, errCh chan error) {
	for meta := range metaCh {
		gannoy, err := gannoy.NewGannoyIndex(meta, gannoy.Angular{}, gannoy.RandRandom{})
		if err == nil {
			gannoyCh <- gannoy
		} else {
			errCh <- err
		}
	}
}
<commit_msg>Fixed a bug that do not use log-dir option.<commit_after>package main

import (
	"context"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"os/signal"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"syscall"
	"time"

	flags "github.com/jessevdk/go-flags"
	"github.com/labstack/echo"
	"github.com/labstack/echo/middleware"
	"github.com/labstack/gommon/log"
	"github.com/lestrrat/go-server-starter/listener"
	"github.com/monochromegane/gannoy"
	"github.com/nightlyone/lockfile"
)

type Options struct {
	DataDir           string `short:"d" long:"data-dir" default:"." description:"Specify the directory where the meta files are located."`
	LogDir            string `short:"l" long:"log-dir" default-mask:"os.Stdout" description:"Specify the log output directory."`
	LockDir           string `short:"L" long:"lock-dir" default:"." description:"Specify the lock file directory. This option is used only server-starter option."`
	WithServerStarter bool   `short:"s" long:"server-starter" default:"false" description:"Use server-starter listener for server address."`
	ShutDownTimeout   int    `short:"t" long:"timeout" default:"10" description:"Specify the number of seconds for shutdown timeout."`
}

var opts Options

type Feature struct {
	W []float64 `json:"features"`
}

func main() {
	_, err := flags.Parse(&opts)
	if err != nil {
		os.Exit(1)
	}

	// Wait old process finishing.
	if opts.WithServerStarter {
		lock, err := initializeLock(opts.LockDir)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
		defer lock.Unlock()
		for {
			if err := lock.TryLock(); err != nil {
				time.Sleep(100 * time.Millisecond)
				continue
			}
			break
		}
	}

	e := echo.New()

	// initialize log
	l, err := initializeLog(opts.LogDir)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	e.Logger.SetLevel(log.INFO)
	e.Logger.SetOutput(l)
	e.Use(middleware.LoggerWithConfig(middleware.LoggerConfig{Output: l}))

	// Load meta files
	files, err := ioutil.ReadDir(opts.DataDir)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	metaCh := make(chan string, len(files))
	gannoyCh := make(chan gannoy.GannoyIndex)
	errCh := make(chan error)
	databases := map[string]gannoy.GannoyIndex{}
	var metaCount int
	for _, file := range files {
		if file.IsDir() || filepath.Ext(file.Name()) != ".meta" {
			continue
		}
		metaCh <- filepath.Join(opts.DataDir, file.Name())
		metaCount++
	}

	for i := 0; i < runtime.GOMAXPROCS(0); i++ {
		go gannoyIndexInitializer(metaCh, gannoyCh, errCh)
	}

loop:
	for {
		select {
		case gannoy := <-gannoyCh:
			key := strings.TrimSuffix(filepath.Base(gannoy.MetaFile()), ".meta")
			databases[key] = gannoy
			if len(databases) >= metaCount {
				close(metaCh)
				close(gannoyCh)
				close(errCh)
				break loop
			}
		case err := <-errCh:
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
	}

	// Define API
	e.GET("/search", func(c echo.Context) error {
		database := c.QueryParam("database")
		if _, ok := databases[database]; !ok {
			return c.NoContent(http.StatusNotFound)
		}
		key, err := strconv.Atoi(c.QueryParam("key"))
		if err != nil {
			key = -1
		}
		limit, err := strconv.Atoi(c.QueryParam("limit"))
		if err != nil {
			limit = 10
		}

		gannoy := databases[database]
		r, err := gannoy.GetNnsByKey(key, limit, -1)
		if err != nil || len(r) == 0 {
			return c.NoContent(http.StatusNotFound)
		}

		return c.JSON(http.StatusOK, r)
	})

	e.PUT("/databases/:database/features/:key", func(c echo.Context) error {
		database := c.Param("database")
		if _, ok := databases[database]; !ok {
			return c.NoContent(http.StatusUnprocessableEntity)
		}
		key, err := strconv.Atoi(c.Param("key"))
		if err != nil {
			return c.NoContent(http.StatusUnprocessableEntity)
		}
		feature := new(Feature)
		if err := c.Bind(feature); err != nil {
			return err
		}

		gannoy := databases[database]
		err = gannoy.AddItem(key, feature.W)
		if err != nil {
			return c.NoContent(http.StatusUnprocessableEntity)
		}
		return c.NoContent(http.StatusOK)
	})

	e.DELETE("/databases/:database/features/:key", func(c echo.Context) error {
		database := c.Param("database")
		if _, ok := databases[database]; !ok {
			return c.NoContent(http.StatusUnprocessableEntity)
		}
		key, err := strconv.Atoi(c.Param("key"))
		if err != nil {
			return c.NoContent(http.StatusUnprocessableEntity)
		}
		gannoy := databases[database]
		err = gannoy.RemoveItem(key)
		if err != nil {
			return c.NoContent(http.StatusUnprocessableEntity)
		}

		return c.NoContent(http.StatusOK)
	})

	// Start server
	address := ":1323"
	sig := os.Interrupt
	if opts.WithServerStarter {
		address = ""
		sig = syscall.SIGTERM
		listeners, err := listener.ListenAll()
		if err != nil && err != listener.ErrNoListeningTarget {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
		e.Listener = listeners[0]
	}

	go func() {
		if err := e.Start(address); err != nil {
			e.Logger.Info("shutting down the server")
		}
	}()

	sigCh := make(chan os.Signal, 1)
	signal.Notify(sigCh, sig)
	<-sigCh

	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(opts.ShutDownTimeout)*time.Second)
	defer cancel()
	if err := e.Shutdown(ctx); err != nil {
		e.Logger.Fatal(err)
	}
}

func initializeLog(logDir string) (*os.File, error) {
	if logDir == "" {
		return os.Stdout, nil
	}
	if err := os.MkdirAll(logDir, os.ModePerm); err != nil {
		return nil, err
	}
	return os.OpenFile(filepath.Join(logDir, "access.log"), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
}

func initializeLock(lockDir string) (lockfile.Lockfile, error) {
	if err := os.MkdirAll(lockDir, os.ModePerm); err != nil {
		return "", err
	}
	lock := "gannoy-server.lock"
	if !filepath.IsAbs(lockDir) {
		lockDir, err := filepath.Abs(lockDir)
		if err != nil {
			return lockfile.Lockfile(""), err
		}
		return lockfile.New(filepath.Join(lockDir, lock))
	}
	return lockfile.New(filepath.Join(lockDir, lock))
}

func gannoyIndexInitializer(metaCh chan string, gannoyCh chan gannoy.GannoyIndex, errCh chan error) {
	for meta := range metaCh {
		gannoy, err := gannoy.NewGannoyIndex(meta, gannoy.Angular{}, gannoy.RandRandom{})
		if err == nil {
			gannoyCh <- gannoy
		} else {
			errCh <- err
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"crypto/tls"
	"crypto/x509"
	"io/ioutil"
	"log"
	"net"
	"strings"

	"github.com/googleapis/gapic-showcase/server"
	pb "github.com/googleapis/gapic-showcase/server/genproto"
	"github.com/googleapis/gapic-showcase/server/services"
	fallback "github.com/googleapis/grpc-fallback-go/server"
	"github.com/spf13/cobra"
	lropb "google.golang.org/genproto/googleapis/longrunning"

	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/reflection"
)

func init() {
	var port string
	var fallbackPort string
	var tlsCaCert string
	var tlsCert string
	var tlsKey string
	runCmd := &cobra.Command{
		Use:   "run",
		Short: "Runs the showcase server",
		Run: func(cmd *cobra.Command, args []string) {
			// Ensure port is of the right form.
			if !strings.HasPrefix(port, ":") {
				port = ":" + port
			}

			// Start listening.
			lis, err := net.Listen("tcp", port)
			if err != nil {
				log.Fatalf("Showcase failed to listen on port '%s': %v", port, err)
			}
			stdLog.Printf("Showcase listening on port: %s", port)

			// Setup Server.
			logger := &loggerObserver{}
			observerRegistry := server.ShowcaseObserverRegistry()
			observerRegistry.RegisterUnaryObserver(logger)
			observerRegistry.RegisterStreamRequestObserver(logger)
			observerRegistry.RegisterStreamResponseObserver(logger)

			opts := []grpc.ServerOption{
				grpc.StreamInterceptor(observerRegistry.StreamInterceptor),
				grpc.UnaryInterceptor(observerRegistry.UnaryInterceptor),
			}

			// load mutual TLS cert/key and root CA cert
			if tlsCaCert == "" && tlsCert != "" && tlsKey != "" {
				keyPair, err := tls.LoadX509KeyPair(tlsCert, tlsKey)
				if err != nil {
					log.Fatalf("Failed to load server TLS cert/key with error:%v", err)
				}

				cert, err := ioutil.ReadFile(tlsCaCert)
				if err != nil {
					log.Fatalf("Failed to load root CA cert file with error:%v", err)
				}

				pool := x509.NewCertPool()
				pool.AppendCertsFromPEM(cert)

				ta := credentials.NewTLS(&tls.Config{
					Certificates: []tls.Certificate{keyPair},
					ClientCAs:    pool,
					ClientAuth:   tls.RequireAndVerifyClientCert,
				})

				opts = append(opts, grpc.Creds(ta))
			}
			s := grpc.NewServer(opts...)
			defer s.GracefulStop()

			// Register Services to the server.
			pb.RegisterEchoServer(s, services.NewEchoServer())
			identityServer := services.NewIdentityServer()
			pb.RegisterIdentityServer(s, identityServer)
			messagingServer := services.NewMessagingServer(identityServer)
			pb.RegisterMessagingServer(s, messagingServer)
			operationsServer := services.NewOperationsServer(messagingServer)
			pb.RegisterTestingServer(s, services.NewTestingServer(observerRegistry))
			lropb.RegisterOperationsServer(s, operationsServer)

			fb := fallback.NewServer(fallbackPort, "localhost"+port)
			fb.StartBackground()
			defer fb.Shutdown()

			// Register reflection service on gRPC server.
			reflection.Register(s)
			s.Serve(lis)
		},
	}
	rootCmd.AddCommand(runCmd)
	runCmd.Flags().StringVarP(
		&port,
		"port",
		"p",
		":7469",
		"The port that showcase will be served on.")
	runCmd.Flags().StringVarP(
		&fallbackPort,
		"fallback-port",
		"f",
		":1337",
		"The port that the fallback-proxy will be served on.")
	runCmd.Flags().StringVar(
		&tlsCaCert,
		"mtls-ca-cert",
		"",
		"The Root CA certificate path for custom mutual TLS channel.")
	runCmd.Flags().StringVar(
		&tlsCert,
		"mtls-cert",
		"",
		"The server certificate path for custom mutual TLS channel.")
	runCmd.Flags().StringVar(
		&tlsKey,
		"mtls-key",
		"",
		"The server private key path for custom mutual TLS channel.")
}
<commit_msg>fix: fix the typo in mtls logic (#335)<commit_after>// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"crypto/tls"
	"crypto/x509"
	"io/ioutil"
	"log"
	"net"
	"strings"

	"github.com/googleapis/gapic-showcase/server"
	pb "github.com/googleapis/gapic-showcase/server/genproto"
	"github.com/googleapis/gapic-showcase/server/services"
	fallback "github.com/googleapis/grpc-fallback-go/server"
	"github.com/spf13/cobra"
	lropb "google.golang.org/genproto/googleapis/longrunning"

	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/reflection"
)

func init() {
	var port string
	var fallbackPort string
	var tlsCaCert string
	var tlsCert string
	var tlsKey string
	runCmd := &cobra.Command{
		Use:   "run",
		Short: "Runs the showcase server",
		Run: func(cmd *cobra.Command, args []string) {
			// Ensure port is of the right form.
			if !strings.HasPrefix(port, ":") {
				port = ":" + port
			}

			// Start listening.
			lis, err := net.Listen("tcp", port)
			if err != nil {
				log.Fatalf("Showcase failed to listen on port '%s': %v", port, err)
			}
			stdLog.Printf("Showcase listening on port: %s", port)

			// Setup Server.
			logger := &loggerObserver{}
			observerRegistry := server.ShowcaseObserverRegistry()
			observerRegistry.RegisterUnaryObserver(logger)
			observerRegistry.RegisterStreamRequestObserver(logger)
			observerRegistry.RegisterStreamResponseObserver(logger)

			opts := []grpc.ServerOption{
				grpc.StreamInterceptor(observerRegistry.StreamInterceptor),
				grpc.UnaryInterceptor(observerRegistry.UnaryInterceptor),
			}

			// load mutual TLS cert/key and root CA cert
			if tlsCaCert != "" && tlsCert != "" && tlsKey != "" {
				keyPair, err := tls.LoadX509KeyPair(tlsCert, tlsKey)
				if err != nil {
					log.Fatalf("Failed to load server TLS cert/key with error:%v", err)
				}

				cert, err := ioutil.ReadFile(tlsCaCert)
				if err != nil {
					log.Fatalf("Failed to load root CA cert file with error:%v", err)
				}

				pool := x509.NewCertPool()
				pool.AppendCertsFromPEM(cert)

				ta := credentials.NewTLS(&tls.Config{
					Certificates: []tls.Certificate{keyPair},
					ClientCAs:    pool,
					ClientAuth:   tls.RequireAndVerifyClientCert,
				})

				opts = append(opts, grpc.Creds(ta))
			}
			s := grpc.NewServer(opts...)
			defer s.GracefulStop()

			// Register Services to the server.
			pb.RegisterEchoServer(s, services.NewEchoServer())
			identityServer := services.NewIdentityServer()
			pb.RegisterIdentityServer(s, identityServer)
			messagingServer := services.NewMessagingServer(identityServer)
			pb.RegisterMessagingServer(s, messagingServer)
			operationsServer := services.NewOperationsServer(messagingServer)
			pb.RegisterTestingServer(s, services.NewTestingServer(observerRegistry))
			lropb.RegisterOperationsServer(s, operationsServer)

			fb := fallback.NewServer(fallbackPort, "localhost"+port)
			fb.StartBackground()
			defer fb.Shutdown()

			// Register reflection service on gRPC server.
			reflection.Register(s)
			s.Serve(lis)
		},
	}
	rootCmd.AddCommand(runCmd)
	runCmd.Flags().StringVarP(
		&port,
		"port",
		"p",
		":7469",
		"The port that showcase will be served on.")
	runCmd.Flags().StringVarP(
		&fallbackPort,
		"fallback-port",
		"f",
		":1337",
		"The port that the fallback-proxy will be served on.")
	runCmd.Flags().StringVar(
		&tlsCaCert,
		"mtls-ca-cert",
		"",
		"The Root CA certificate path for custom mutual TLS channel.")
	runCmd.Flags().StringVar(
		&tlsCert,
		"mtls-cert",
		"",
		"The server certificate path for custom mutual TLS channel.")
	runCmd.Flags().StringVar(
		&tlsKey,
		"mtls-key",
		"",
		"The server private key path for custom mutual TLS channel.")
}
<|endoftext|>


<commit_before>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package main

import (
	"bytes"
	"fmt"

	"github.com/juju/cmd"
	gc "launchpad.net/gocheck"

	"github.com/juju/juju/cmd/envcmd"
	"github.com/juju/juju/juju/testing"
	coretesting "github.com/juju/juju/testing"
)

type EndpointSuite struct {
	testing.JujuConnSuite
}

var _ = gc.Suite(&EndpointSuite{})

func (s *EndpointSuite) TestEndpoint(c *gc.C) {
	ctx := coretesting.Context(c)
	code := cmd.Main(envcmd.Wrap(&EndpointCommand{}), ctx, []string{})
	c.Check(code, gc.Equals, 0)
	c.Assert(ctx.Stderr.(*bytes.Buffer).String(), gc.Equals, "")
	output := string(ctx.Stdout.(*bytes.Buffer).Bytes())
	info := s.APIInfo(c)
	c.Assert(output, gc.Equals, fmt.Sprintf("%s\n", info.Addrs[0]))
}
<commit_msg>Small test optimization.<commit_after>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package main

import (
	"bytes"
	"fmt"

	gc "launchpad.net/gocheck"

	"github.com/juju/juju/cmd/envcmd"
	"github.com/juju/juju/juju/testing"
	coretesting "github.com/juju/juju/testing"
)

type EndpointSuite struct {
	testing.JujuConnSuite
}

var _ = gc.Suite(&EndpointSuite{})

func (s *EndpointSuite) TestEndpoint(c *gc.C) {
	ctx, err := coretesting.RunCommand(c, envcmd.Wrap(&EndpointCommand{}))
	c.Assert(err, gc.IsNil)
	c.Assert(ctx.Stderr.(*bytes.Buffer).String(), gc.Equals, "")
	output := string(ctx.Stdout.(*bytes.Buffer).Bytes())
	info := s.APIInfo(c)
	c.Assert(output, gc.Equals, fmt.Sprintf("%s\n", info.Addrs[0]))
}
<|endoftext|>


<commit_before>/*
 * Minio Cloud Storage, (C) 2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"io"
	"path"
	"strconv"
	"strings"
)

type networkStorage struct {
	netScheme string
	netAddr   string
	netPath   string
	rpcClient *AuthRPCClient
}

const (
	storageRPCPath = reservedBucket + "/storage"
)

// Converts rpc.ServerError to underlying error. This function is
// written so that the storageAPI errors are consistent across network
// disks as well.
func toStorageErr(err error) error {
	if err == nil {
		return nil
	}
	switch err.Error() {
	case io.EOF.Error():
		return io.EOF
	case io.ErrUnexpectedEOF.Error():
		return io.ErrUnexpectedEOF
	case errUnexpected.Error():
		return errUnexpected
	case errDiskFull.Error():
		return errDiskFull
	case errVolumeNotFound.Error():
		return errVolumeNotFound
	case errVolumeExists.Error():
		return errVolumeExists
	case errFileNotFound.Error():
		return errFileNotFound
	case errFileNameTooLong.Error():
		return errFileNameTooLong
	case errFileAccessDenied.Error():
		return errFileAccessDenied
	case errIsNotRegular.Error():
		return errIsNotRegular
	case errVolumeNotEmpty.Error():
		return errVolumeNotEmpty
	case errVolumeAccessDenied.Error():
		return errVolumeAccessDenied
	case errCorruptedFormat.Error():
		return errCorruptedFormat
	case errUnformattedDisk.Error():
		return errUnformattedDisk
	}
	return err
}

// Initialize new rpc client.
func newRPCClient(networkPath string) (StorageAPI, error) {
	// Input validation.
	if networkPath == "" || strings.LastIndex(networkPath, ":") == -1 {
		return nil, errInvalidArgument
	}

	// Split network path into its components.
	netAddr, netPath, err := splitNetPath(networkPath)
	if err != nil {
		return nil, err
	}

	// Dial minio rpc storage http path.
	rpcPath := path.Join(storageRPCPath, netPath)
	port := getPort(srvConfig.serverAddr)
	rpcAddr := netAddr + ":" + strconv.Itoa(port)
	// Initialize rpc client with network address and rpc path.
	cred := serverConfig.GetCredential()
	rpcClient := newAuthClient(rpcAddr, rpcPath, cred, "Storage.LoginHandler")
	// Initialize network storage.
	ndisk := &networkStorage{
		netScheme: func() string {
			if !isSSL() {
				return "http"
			}
			return "https"
		}(),
		netAddr:   netAddr,
		netPath:   netPath,
		rpcClient: rpcClient,
	}

	// Returns successfully here.
	return ndisk, nil
}

// MakeVol - make a volume.
func (n networkStorage) MakeVol(volume string) error {
	if n.rpcClient == nil {
		return errVolumeBusy
	}
	reply := GenericReply{}
	args := GenericVolArgs{GenericArgs{n.rpcClient.token, n.rpcClient.tstamp}, volume}
	if err := n.rpcClient.Call("Storage.MakeVolHandler", &args, &reply); err != nil {
		return toStorageErr(err)
	}
	return nil
}

// ListVols - List all volumes.
func (n networkStorage) ListVols() (vols []VolInfo, err error) {
	if n.rpcClient == nil {
		return nil, errVolumeBusy
	}
	ListVols := ListVolsReply{}
	err = n.rpcClient.Call("Storage.ListVolsHandler", &GenericArgs{n.rpcClient.token, n.rpcClient.tstamp}, &ListVols)
	if err != nil {
		return nil, err
	}
	return ListVols.Vols, nil
}

// StatVol - get current Stat volume info.
func (n networkStorage) StatVol(volume string) (volInfo VolInfo, err error) {
	if n.rpcClient == nil {
		return VolInfo{}, errVolumeBusy
	}
	args := GenericVolArgs{GenericArgs{n.rpcClient.token, n.rpcClient.tstamp}, volume}
	if err = n.rpcClient.Call("Storage.StatVolHandler", &args, &volInfo); err != nil {
		return VolInfo{}, toStorageErr(err)
	}
	return volInfo, nil
}

// DeleteVol - Delete a volume.
func (n networkStorage) DeleteVol(volume string) error {
	if n.rpcClient == nil {
		return errVolumeBusy
	}
	reply := GenericReply{}
	args := GenericVolArgs{GenericArgs{n.rpcClient.token, n.rpcClient.tstamp}, volume}
	if err := n.rpcClient.Call("Storage.DeleteVolHandler", &args, &reply); err != nil {
		return toStorageErr(err)
	}
	return nil
}

// File operations.

// CreateFile - create file.
func (n networkStorage) AppendFile(volume, path string, buffer []byte) (err error) {
	if n.rpcClient == nil {
		return errVolumeBusy
	}
	reply := GenericReply{}
	if err = n.rpcClient.Call("Storage.AppendFileHandler", &AppendFileArgs{
		GenericArgs: GenericArgs{n.rpcClient.token, n.rpcClient.tstamp},
		Vol:         volume,
		Path:        path,
		Buffer:      buffer,
	}, &reply); err != nil {
		return toStorageErr(err)
	}
	return nil
}

// StatFile - get latest Stat information for a file at path.
func (n networkStorage) StatFile(volume, path string) (fileInfo FileInfo, err error) {
	if n.rpcClient == nil {
		return FileInfo{}, errVolumeBusy
	}
	if err = n.rpcClient.Call("Storage.StatFileHandler", &StatFileArgs{
		GenericArgs: GenericArgs{n.rpcClient.token, n.rpcClient.tstamp},
		Vol:         volume,
		Path:        path,
	}, &fileInfo); err != nil {
		return FileInfo{}, toStorageErr(err)
	}
	return fileInfo, nil
}

// ReadAll - reads entire contents of the file at path until EOF, returns the
// contents in a byte slice. Returns buf == nil if err != nil.
// This API is meant to be used on files which have small memory footprint, do
// not use this on large files as it would cause server to crash.
func (n networkStorage) ReadAll(volume, path string) (buf []byte, err error) {
	if n.rpcClient == nil {
		return nil, errVolumeBusy
	}
	if err = n.rpcClient.Call("Storage.ReadAllHandler", &ReadAllArgs{
		GenericArgs: GenericArgs{n.rpcClient.token, n.rpcClient.tstamp},
		Vol:         volume,
		Path:        path,
	}, &buf); err != nil {
		return nil, toStorageErr(err)
	}
	return buf, nil
}

// ReadFile - reads a file.
func (n networkStorage) ReadFile(volume string, path string, offset int64, buffer []byte) (m int64, err error) {
	if n.rpcClient == nil {
		return 0, errVolumeBusy
	}
	var result []byte
	err = n.rpcClient.Call("Storage.ReadFileHandler", &ReadFileArgs{
		GenericArgs: GenericArgs{n.rpcClient.token, n.rpcClient.tstamp},
		Vol:         volume,
		Path:        path,
		Offset:      offset,
		Size:        len(buffer),
	}, &result)
	// Copy results to buffer.
	copy(buffer, result)
	// Return length of result, err if any.
	return int64(len(result)), toStorageErr(err)
}

// ListDir - list all entries at prefix.
func (n networkStorage) ListDir(volume, path string) (entries []string, err error) {
	if n.rpcClient == nil {
		return nil, errVolumeBusy
	}
	if err = n.rpcClient.Call("Storage.ListDirHandler", &ListDirArgs{
		GenericArgs: GenericArgs{n.rpcClient.token, n.rpcClient.tstamp},
		Vol:         volume,
		Path:        path,
	}, &entries); err != nil {
		return nil, toStorageErr(err)
	}
	// Return successfully unmarshalled results.
	return entries, nil
}

// DeleteFile - Delete a file at path.
func (n networkStorage) DeleteFile(volume, path string) (err error) {
	if n.rpcClient == nil {
		return errVolumeBusy
	}
	reply := GenericReply{}
	if err = n.rpcClient.Call("Storage.DeleteFileHandler", &DeleteFileArgs{
		GenericArgs: GenericArgs{n.rpcClient.token, n.rpcClient.tstamp},
		Vol:         volume,
		Path:        path,
	}, &reply); err != nil {
		return toStorageErr(err)
	}
	return nil
}

// RenameFile - Rename file.
func (n networkStorage) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) (err error) {
	if n.rpcClient == nil {
		return errVolumeBusy
	}
	reply := GenericReply{}
	if err = n.rpcClient.Call("Storage.RenameFileHandler", &RenameFileArgs{
		GenericArgs: GenericArgs{Token: n.rpcClient.token},
		SrcVol:      srcVolume,
		SrcPath:     srcPath,
		DstVol:      dstVolume,
		DstPath:     dstPath,
	}, &reply); err != nil {
		return toStorageErr(err)
	}
	return nil
}
<commit_msg>rpc-client: remove unwanted nil check of rpcClient. (#2538)<commit_after>/*
 * Minio Cloud Storage, (C) 2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"io"
	"path"
	"strconv"
	"strings"
)

type networkStorage struct {
	netScheme string
	netAddr   string
	netPath   string
	rpcClient *AuthRPCClient
}

const (
	storageRPCPath = reservedBucket + "/storage"
)

// Converts rpc.ServerError to underlying error. This function is
// written so that the storageAPI errors are consistent across network
// disks as well.
func toStorageErr(err error) error {
	if err == nil {
		return nil
	}
	switch err.Error() {
	case io.EOF.Error():
		return io.EOF
	case io.ErrUnexpectedEOF.Error():
		return io.ErrUnexpectedEOF
	case errUnexpected.Error():
		return errUnexpected
	case errDiskFull.Error():
		return errDiskFull
	case errVolumeNotFound.Error():
		return errVolumeNotFound
	case errVolumeExists.Error():
		return errVolumeExists
	case errFileNotFound.Error():
		return errFileNotFound
	case errFileNameTooLong.Error():
		return errFileNameTooLong
	case errFileAccessDenied.Error():
		return errFileAccessDenied
	case errIsNotRegular.Error():
		return errIsNotRegular
	case errVolumeNotEmpty.Error():
		return errVolumeNotEmpty
	case errVolumeAccessDenied.Error():
		return errVolumeAccessDenied
	case errCorruptedFormat.Error():
		return errCorruptedFormat
	case errUnformattedDisk.Error():
		return errUnformattedDisk
	}
	return err
}

// Initialize new rpc client.
func newRPCClient(networkPath string) (StorageAPI, error) {
	// Input validation.
	if networkPath == "" || strings.LastIndex(networkPath, ":") == -1 {
		return nil, errInvalidArgument
	}

	// Split network path into its components.
	netAddr, netPath, err := splitNetPath(networkPath)
	if err != nil {
		return nil, err
	}

	// Dial minio rpc storage http path.
	rpcPath := path.Join(storageRPCPath, netPath)
	port := getPort(srvConfig.serverAddr)
	rpcAddr := netAddr + ":" + strconv.Itoa(port)
	// Initialize rpc client with network address and rpc path.
	cred := serverConfig.GetCredential()
	rpcClient := newAuthClient(rpcAddr, rpcPath, cred, "Storage.LoginHandler")
	// Initialize network storage.
	ndisk := &networkStorage{
		netScheme: func() string {
			if !isSSL() {
				return "http"
			}
			return "https"
		}(),
		netAddr:   netAddr,
		netPath:   netPath,
		rpcClient: rpcClient,
	}

	// Returns successfully here.
	return ndisk, nil
}

// MakeVol - make a volume.
func (n networkStorage) MakeVol(volume string) error {
	reply := GenericReply{}
	args := GenericVolArgs{GenericArgs{n.rpcClient.token, n.rpcClient.tstamp}, volume}
	if err := n.rpcClient.Call("Storage.MakeVolHandler", &args, &reply); err != nil {
		return toStorageErr(err)
	}
	return nil
}

// ListVols - List all volumes.
func (n networkStorage) ListVols() (vols []VolInfo, err error) {
	ListVols := ListVolsReply{}
	err = n.rpcClient.Call("Storage.ListVolsHandler", &GenericArgs{n.rpcClient.token, n.rpcClient.tstamp}, &ListVols)
	if err != nil {
		return nil, err
	}
	return ListVols.Vols, nil
}

// StatVol - get current Stat volume info.
func (n networkStorage) StatVol(volume string) (volInfo VolInfo, err error) {
	args := GenericVolArgs{GenericArgs{n.rpcClient.token, n.rpcClient.tstamp}, volume}
	if err = n.rpcClient.Call("Storage.StatVolHandler", &args, &volInfo); err != nil {
		return VolInfo{}, toStorageErr(err)
	}
	return volInfo, nil
}

// DeleteVol - Delete a volume.
func (n networkStorage) DeleteVol(volume string) error {
	reply := GenericReply{}
	args := GenericVolArgs{GenericArgs{n.rpcClient.token, n.rpcClient.tstamp}, volume}
	if err := n.rpcClient.Call("Storage.DeleteVolHandler", &args, &reply); err != nil {
		return toStorageErr(err)
	}
	return nil
}

// File operations.

// CreateFile - create file.
func (n networkStorage) AppendFile(volume, path string, buffer []byte) (err error) {
	reply := GenericReply{}
	if err = n.rpcClient.Call("Storage.AppendFileHandler", &AppendFileArgs{
		GenericArgs: GenericArgs{n.rpcClient.token, n.rpcClient.tstamp},
		Vol:         volume,
		Path:        path,
		Buffer:      buffer,
	}, &reply); err != nil {
		return toStorageErr(err)
	}
	return nil
}

// StatFile - get latest Stat information for a file at path.
func (n networkStorage) StatFile(volume, path string) (fileInfo FileInfo, err error) {
	if err = n.rpcClient.Call("Storage.StatFileHandler", &StatFileArgs{
		GenericArgs: GenericArgs{n.rpcClient.token, n.rpcClient.tstamp},
		Vol:         volume,
		Path:        path,
	}, &fileInfo); err != nil {
		return FileInfo{}, toStorageErr(err)
	}
	return fileInfo, nil
}

// ReadAll - reads entire contents of the file at path until EOF, returns the
// contents in a byte slice. Returns buf == nil if err != nil.
// This API is meant to be used on files which have small memory footprint, do
// not use this on large files as it would cause server to crash.
func (n networkStorage) ReadAll(volume, path string) (buf []byte, err error) {
	if err = n.rpcClient.Call("Storage.ReadAllHandler", &ReadAllArgs{
		GenericArgs: GenericArgs{n.rpcClient.token, n.rpcClient.tstamp},
		Vol:         volume,
		Path:        path,
	}, &buf); err != nil {
		return nil, toStorageErr(err)
	}
	return buf, nil
}

// ReadFile - reads a file.
func (n networkStorage) ReadFile(volume string, path string, offset int64, buffer []byte) (m int64, err error) {
	var result []byte
	err = n.rpcClient.Call("Storage.ReadFileHandler", &ReadFileArgs{
		GenericArgs: GenericArgs{n.rpcClient.token, n.rpcClient.tstamp},
		Vol:         volume,
		Path:        path,
		Offset:      offset,
		Size:        len(buffer),
	}, &result)
	// Copy results to buffer.
	copy(buffer, result)
	// Return length of result, err if any.
	return int64(len(result)), toStorageErr(err)
}

// ListDir - list all entries at prefix.
func (n networkStorage) ListDir(volume, path string) (entries []string, err error) {
	if err = n.rpcClient.Call("Storage.ListDirHandler", &ListDirArgs{
		GenericArgs: GenericArgs{n.rpcClient.token, n.rpcClient.tstamp},
		Vol:         volume,
		Path:        path,
	}, &entries); err != nil {
		return nil, toStorageErr(err)
	}
	// Return successfully unmarshalled results.
	return entries, nil
}

// DeleteFile - Delete a file at path.
func (n networkStorage) DeleteFile(volume, path string) (err error) {
	reply := GenericReply{}
	if err = n.rpcClient.Call("Storage.DeleteFileHandler", &DeleteFileArgs{
		GenericArgs: GenericArgs{n.rpcClient.token, n.rpcClient.tstamp},
		Vol:         volume,
		Path:        path,
	}, &reply); err != nil {
		return toStorageErr(err)
	}
	return nil
}

// RenameFile - Rename file.
func (n networkStorage) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) (err error) {
	reply := GenericReply{}
	if err = n.rpcClient.Call("Storage.RenameFileHandler", &RenameFileArgs{
		GenericArgs: GenericArgs{Token: n.rpcClient.token},
		SrcVol:      srcVolume,
		SrcPath:     srcPath,
		DstVol:      dstVolume,
		DstPath:     dstPath,
	}, &reply); err != nil {
		return toStorageErr(err)
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"log"
	"os"
	"os/exec"
	"os/signal"
	"path"
	"path/filepath"
	"runtime"
	"syscall"
	"time"

	"bazil.org/fuse"
	"bazil.org/fuse/fs"
	_ "bazil.org/fuse/fs/fstestutil"
	"github.com/burkemw3/syncthing-fuse/lib/model"
	"github.com/syncthing/protocol"
	"golang.org/x/net/context"
)

var Usage = func() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "  %s MOUNTPOINT\n", os.Args[0])
	flag.PrintDefaults()
}

func MountFuse(mountpoint string, m *model.Model) {
	c, err := fuse.Mount(
		mountpoint,
		fuse.FSName("syncthingfuse"),
		fuse.Subtype("syncthingfuse"),
		fuse.LocalVolume(),
		fuse.VolumeName("Syncthing FUSE"),
	)
	if err != nil {
		log.Fatal(err)
	}

	sigc := make(chan os.Signal, 1)
	signal.Notify(sigc, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)

	doneServe := make(chan error, 1)
	go func() {
		doneServe <- fs.Serve(c, FS{m: m})
	}()

	select {
	case err := <-doneServe:
		log.Printf("conn.Serve returned %v", err)

		// check if the mount process has an error to report
		<-c.Ready
		if err := c.MountError; err != nil {
			log.Printf("conn.MountError: %v", err)
		}
	case sig := <-sigc:
		log.Printf("Signal %s received, shutting down.", sig)
	}

	time.AfterFunc(3*time.Second, func() {
		os.Exit(1)
	})
	log.Printf("Unmounting...")
	err = Unmount(mountpoint)
	log.Printf("Unmount = %v", err)

	log.Printf("syncthing FUSE process ending.")
}

var (
	folder = "syncthingfusetest"
)

func makeModel() *model.Model {
	m := model.NewModel()

	deviceID := protocol.DeviceID{}
	flags := uint32(0)
	options := []protocol.Option{}

	files := []protocol.FileInfo{
		protocol.FileInfo{Name: "file1"},
		protocol.FileInfo{Name: "file2"},
		protocol.FileInfo{Name: "dir1", Flags: protocol.FlagDirectory},
		protocol.FileInfo{Name: "dir1/dirfile1"},
		protocol.FileInfo{Name: "dir1/dirfile2"},
	}

	m.Index(deviceID, folder, files, flags, options)

	return m
}

type FS struct {
	m *model.Model
}

func (fs FS) Root() (fs.Node, error) {
	log.Printf("Root")
	return Dir{m: fs.m}, nil
}

// Dir implements both Node and Handle for the root directory.
type Dir struct {
	path string
	m    *model.Model
}

func (d Dir) Attr(ctx context.Context, a *fuse.Attr) error {
	log.Printf("Dir Attr")
	a.Mode = os.ModeDir | 0555
	return nil
}

func (d Dir) Lookup(ctx context.Context, name string) (fs.Node, error) {
	log.Printf("Dir %s Lookup for %s", d.path, name)
	entry := d.m.GetEntry(folder, filepath.Join(d.path, name))

	var node fs.Node
	if entry.IsDirectory() {
		node = Dir{
			path: entry.Name,
			m:    d.m,
		}
	} else {
		node = File{
		// TODO
		// path: entry.Name,
		// m: d.m,
		}
	}

	return node, nil
}

func (d Dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
	log.Printf("ReadDirAll %s", d.path)

	p := path.Clean(d.path)

	entries := d.m.GetChildren(folder, p)
	result := make([]fuse.Dirent, len(entries))
	for i, entry := range entries {
		eType := fuse.DT_File
		if entry.IsDirectory() {
			eType = fuse.DT_Dir
		}
		result[i] = fuse.Dirent{
			Name: path.Base(entry.Name),
			Type: eType,
		}
	}

	return result, nil
}

// File implements both Node and Handle for the hello file.
type File struct {
	inode uint64
	name  string
}

const greeting = "hello, world\n"

func (f File) Attr(ctx context.Context, a *fuse.Attr) error {
	a.Inode = f.inode
	a.Mode = 0444
	a.Size = uint64(len(greeting))
	return nil
}

func (File) ReadAll(ctx context.Context) ([]byte, error) {
	return []byte(greeting), nil
}

// Unmount attempts to unmount the provided FUSE mount point, forcibly
// if necessary.
func Unmount(point string) error {
	var cmd *exec.Cmd
	switch runtime.GOOS {
	case "darwin":
		cmd = exec.Command("/usr/sbin/diskutil", "umount", "force", point)
	case "linux":
		cmd = exec.Command("fusermount", "-u", point)
	default:
		return errors.New("unmount: unimplemented")
	}

	errc := make(chan error, 1)
	go func() {
		if err := exec.Command("umount", point).Run(); err == nil {
			errc <- err
		}
		// retry to unmount with the fallback cmd
		errc <- cmd.Run()
	}()
	select {
	case <-time.After(1 * time.Second):
		return errors.New("umount timeout")
	case err := <-errc:
		return err
	}
}
<commit_msg>Switch fuse to using same logger as everything else<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"os"
	"os/exec"
	"os/signal"
	"path"
	"path/filepath"
	"runtime"
	"strings"
	"syscall"
	"time"

	"bazil.org/fuse"
	"bazil.org/fuse/fs"
	_ "bazil.org/fuse/fs/fstestutil"
	"github.com/burkemw3/syncthing-fuse/lib/model"
	"github.com/syncthing/protocol"
	"golang.org/x/net/context"
)

var Usage = func() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "  %s MOUNTPOINT\n", os.Args[0])
	flag.PrintDefaults()
}

func MountFuse(mountpoint string, m *model.Model) {
	c, err := fuse.Mount(
		mountpoint,
		fuse.FSName("syncthingfuse"),
		fuse.Subtype("syncthingfuse"),
		fuse.LocalVolume(),
		fuse.VolumeName("Syncthing FUSE"),
	)
	if err != nil {
		l.Warnln(err)
	}

	sigc := make(chan os.Signal, 1)
	signal.Notify(sigc, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)

	doneServe := make(chan error, 1)
	go func() {
		doneServe <- fs.Serve(c, FS{m: m})
	}()

	select {
	case err := <-doneServe:
		l.Infoln("conn.Serve returned %v", err)

		// check if the mount process has an error to report
		<-c.Ready
		if err := c.MountError; err != nil {
			l.Warnln("conn.MountError: %v", err)
		}
	case sig := <-sigc:
		l.Infoln("Signal %s received, shutting down.", sig)
	}

	time.AfterFunc(3*time.Second, func() {
		os.Exit(1)
	})
	l.Infoln("Unmounting...")
	err = Unmount(mountpoint)
	l.Infoln("Unmount = %v", err)

	l.Infoln("syncthing FUSE process ending.")
}

var (
	folder    = "syncthingfusetest"
	debugFuse = strings.Contains(os.Getenv("STTRACE"), "fuse") || os.Getenv("STTRACE") == "all"
)

func makeModel() *model.Model {
	m := model.NewModel()

	deviceID := protocol.DeviceID{}
	flags := uint32(0)
	options := []protocol.Option{}

	files := []protocol.FileInfo{
		protocol.FileInfo{Name: "file1"},
		protocol.FileInfo{Name: "file2"},
		protocol.FileInfo{Name: "dir1", Flags: protocol.FlagDirectory},
		protocol.FileInfo{Name: "dir1/dirfile1"},
		protocol.FileInfo{Name: "dir1/dirfile2"},
	}

	m.Index(deviceID, folder, files, flags, options)

	return m
}

type FS struct {
	m *model.Model
}

func (fs FS) Root() (fs.Node, error) {
	if debugFuse {
		l.Debugln("Root")
	}
	return Dir{m: fs.m}, nil
}

// Dir implements both Node and Handle for the root directory.
type Dir struct {
	path string
	m    *model.Model
}

func (d Dir) Attr(ctx context.Context, a *fuse.Attr) error {
	if debugFuse {
		l.Debugln("Dir Attr")
	}
	a.Mode = os.ModeDir | 0555
	return nil
}

func (d Dir) Lookup(ctx context.Context, name string) (fs.Node, error) {
	if debugFuse {
		l.Debugln("Dir %s Lookup for %s", d.path, name)
	}
	entry := d.m.GetEntry(folder, filepath.Join(d.path, name))

	var node fs.Node
	if entry.IsDirectory() {
		node = Dir{
			path: entry.Name,
			m:    d.m,
		}
	} else {
		node = File{
		// TODO
		// path: entry.Name,
		// m: d.m,
		}
	}

	return node, nil
}

func (d Dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
	if debugFuse {
		l.Debugln("ReadDirAll %s", d.path)
	}

	p := path.Clean(d.path)

	entries := d.m.GetChildren(folder, p)
	result := make([]fuse.Dirent, len(entries))
	for i, entry := range entries {
		eType := fuse.DT_File
		if entry.IsDirectory() {
			eType = fuse.DT_Dir
		}
		result[i] = fuse.Dirent{
			Name: path.Base(entry.Name),
			Type: eType,
		}
	}

	return result, nil
}

// File implements both Node and Handle for the hello file.
type File struct {
	inode uint64
	name  string
}

const greeting = "hello, world\n"

func (f File) Attr(ctx context.Context, a *fuse.Attr) error {
	a.Inode = f.inode
	a.Mode = 0444
	a.Size = uint64(len(greeting))
	return nil
}

func (File) ReadAll(ctx context.Context) ([]byte, error) {
	return []byte(greeting), nil
}

// Unmount attempts to unmount the provided FUSE mount point, forcibly
// if necessary.
func Unmount(point string) error {
	var cmd *exec.Cmd
	switch runtime.GOOS {
	case "darwin":
		cmd = exec.Command("/usr/sbin/diskutil", "umount", "force", point)
	case "linux":
		cmd = exec.Command("fusermount", "-u", point)
	default:
		return errors.New("unmount: unimplemented")
	}

	errc := make(chan error, 1)
	go func() {
		if err := exec.Command("umount", point).Run(); err == nil {
			errc <- err
		}
		// retry to unmount with the fallback cmd
		errc <- cmd.Run()
	}()
	select {
	case <-time.After(1 * time.Second):
		return errors.New("umount timeout")
	case err := <-errc:
		return err
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/nelhage/taktician/ai"
	"github.com/nelhage/taktician/playtak"
	"github.com/nelhage/taktician/tak"
)

const (
	minThink = 2 * time.Second
	maxThink = time.Minute

	defaultLevel = 2

	docURL = "https://github.com/nelhage/taktician/blob/master/doc/friendly.md"
)

var commandRE = regexp.MustCompile(`^([^ :]+):\s*([^ ]+)\s*(.*)$`)

type Friendly struct {
	client *playtak.Client
	ai     *ai.MinimaxAI
	g      *Game

	level    int
	levelSet time.Time

	greeted map[string]time.Time
}

func (f *Friendly) NewGame(g *Game) {
	if f.greeted == nil {
		f.greeted = make(map[string]time.Time)
	}
	if time.Now().Sub(f.levelSet) > 1*time.Hour {
		f.level = defaultLevel
	}
	f.g = g
	f.ai = ai.NewMinimax(f.Config())
	/*
		f.client.SendCommand("Shout",
			fmt.Sprintf("[FriendlyBot@level %d] Good luck %s!",
				f.level, g.opponent,
			))
	*/
	if t := f.greeted[g.opponent]; time.Now().Sub(t) > time.Hour {
		log.Printf("greeting user=%q greeted=%s", g.opponent, t)
		f.client.SendCommand("Shout",
			fmt.Sprintf("FriendlyBot@level %d: %s",
				f.level, docURL))
		f.greeted[g.opponent] = time.Now()
	}
}

func (f *Friendly) GameOver() {
	f.g = nil
}

func (f *Friendly) GetMove(p *tak.Position, mine, theirs time.Duration) tak.Move {
	deadline := time.After(minThink)
	m := f.ai.GetMove(p, maxThink)
	<-deadline
	return m
}

func (f *Friendly) HandleChat(who string, msg string) {
	log.Printf("chat from=%q msg=%q", who, msg)
	gs := commandRE.FindStringSubmatch(msg)
	if gs == nil {
		return
	}
	if !strings.EqualFold(gs[1], *user) {
		return
	}
	switch gs[2] {
	case "level":
		if gs[3] == "max" {
			f.level = 100
			f.levelSet = time.Now()
			f.client.SendCommand("Shout", "OK! I'll play as best as I can!")
			break
		}
		l, e := strconv.ParseUint(strings.Trim(gs[3], " "), 10, 64)
		if e != nil {
			log.Printf("bad level: %v", e)
			return
		}
		if int(l) < 1 || int(l) > len(levels)+1 {
			f.client.SendCommand("Shout", fmt.Sprintf("I only know about levels up to %d", len(levels)+1))
		}
		f.level = int(l)
		f.levelSet = time.Now()
		if f.g == nil || who != f.g.opponent {
			f.client.SendCommand("Shout",
				fmt.Sprintf("OK! I'll play at level %d for future games.", l))
		} else if f.g != nil {
			f.ai = ai.NewMinimax(f.Config())
			f.client.SendCommand("Shout",
				fmt.Sprintf("OK! I'll play at level %d, starting right now.", l))
		}
	case "help":
		f.client.SendCommand("Shout",
			fmt.Sprintf("Learn more: %s", docURL))
	}
}

func (f *Friendly) Config() ai.MinimaxConfig {
	cfg := ai.MinimaxConfig{
		Size:  f.g.size,
		Debug: *debug,

		NoSort:  !*sort,
		NoTable: !*table,
	}
	cfg.Depth, cfg.Evaluate = f.levelSettings(f.g.size, f.level)

	return cfg
}

var (
	easyWeights = ai.Weights{
		TopFlat: 100,
		Tempo:   50,
	}
	medWeights = ai.Weights{
		TopFlat:  200,
		Standing: 100,
		Capstone: 150,
		Flat:     100,
		Tempo:    100,
		Groups:   [8]int{0, 0, 0, 100, 200},
	}
)

var levels = []struct {
	depth   int
	weights ai.Weights
}{
	{2, easyWeights},
	{2, medWeights},
	{3, easyWeights},
	{2, ai.DefaultWeights},
	{3, medWeights},
	{3, easyWeights},
	{4, medWeights},
	{3, ai.DefaultWeights},
	{5, easyWeights},
	{5, medWeights},
	{4, ai.DefaultWeights},
	{5, ai.DefaultWeights},
}

func (f *Friendly) levelSettings(size int, level int) (int, ai.EvaluationFunc) {
	if level == 0 {
		level = 3
	}
	if level > len(levels)+1 {
		return 7, ai.DefaultEvaluate
	}
	s := levels[level-1]
	return s.depth, ai.MakeEvaluator(&s.weights)
}
<commit_msg>shorten docs url<commit_after>package main

import (
	"fmt"
	"log"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/nelhage/taktician/ai"
	"github.com/nelhage/taktician/playtak"
	"github.com/nelhage/taktician/tak"
)

const (
	minThink = 2 * time.Second
	maxThink = time.Minute

	defaultLevel = 2

	docURL = "http://bit.ly/25h33rC"
)

var commandRE = regexp.MustCompile(`^([^ :]+):\s*([^ ]+)\s*(.*)$`)

type Friendly struct {
	client *playtak.Client
	ai     *ai.MinimaxAI
	g      *Game

	level    int
	levelSet time.Time

	greeted map[string]time.Time
}

func (f *Friendly) NewGame(g *Game) {
	if f.greeted == nil {
		f.greeted = make(map[string]time.Time)
	}
	if time.Now().Sub(f.levelSet) > 1*time.Hour {
		f.level = defaultLevel
	}
	f.g = g
	f.ai = ai.NewMinimax(f.Config())
	/*
		f.client.SendCommand("Shout",
			fmt.Sprintf("[FriendlyBot@level %d] Good luck %s!",
				f.level, g.opponent,
			))
	*/
	if t := f.greeted[g.opponent]; time.Now().Sub(t) > time.Hour {
		log.Printf("greeting user=%q greeted=%s", g.opponent, t)
		f.client.SendCommand("Shout",
			fmt.Sprintf("FriendlyBot@level %d: %s",
				f.level, docURL))
		f.greeted[g.opponent] = time.Now()
	}
}

func (f *Friendly) GameOver() {
	f.g = nil
}

func (f *Friendly) GetMove(p *tak.Position, mine, theirs time.Duration) tak.Move {
	deadline := time.After(minThink)
	m := f.ai.GetMove(p, maxThink)
	<-deadline
	return m
}

func (f *Friendly) HandleChat(who string, msg string) {
	log.Printf("chat from=%q msg=%q", who, msg)
	gs := commandRE.FindStringSubmatch(msg)
	if gs == nil {
		return
	}
	if !strings.EqualFold(gs[1], *user) {
		return
	}
	switch gs[2] {
	case "level":
		if gs[3] == "max" {
			f.level = 100
			f.levelSet = time.Now()
			f.client.SendCommand("Shout", "OK! I'll play as best as I can!")
			break
		}
		l, e := strconv.ParseUint(strings.Trim(gs[3], " "), 10, 64)
		if e != nil {
			log.Printf("bad level: %v", e)
			return
		}
		if int(l) < 1 || int(l) > len(levels)+1 {
			f.client.SendCommand("Shout", fmt.Sprintf("I only know about levels up to %d", len(levels)+1))
		}
		f.level = int(l)
		f.levelSet = time.Now()
		if f.g == nil || who != f.g.opponent {
			f.client.SendCommand("Shout",
				fmt.Sprintf("OK! I'll play at level %d for future games.", l))
		} else if f.g != nil {
			f.ai = ai.NewMinimax(f.Config())
			f.client.SendCommand("Shout",
				fmt.Sprintf("OK! I'll play at level %d, starting right now.", l))
		}
	case "help":
		f.client.SendCommand("Shout",
			fmt.Sprintf("Learn more: %s", docURL))
	}
}

func (f *Friendly) Config() ai.MinimaxConfig {
	cfg := ai.MinimaxConfig{
		Size:  f.g.size,
		Debug: *debug,

		NoSort:  !*sort,
		NoTable: !*table,
	}
	cfg.Depth, cfg.Evaluate = f.levelSettings(f.g.size, f.level)

	return cfg
}

var (
	easyWeights = ai.Weights{
		TopFlat: 100,
		Tempo:   50,
	}
	medWeights = ai.Weights{
		TopFlat:  200,
		Standing: 100,
		Capstone: 150,
		Flat:     100,
		Tempo:    100,
		Groups:   [8]int{0, 0, 0, 100, 200},
	}
)

var levels = []struct {
	depth   int
	weights ai.Weights
}{
	{2, easyWeights},
	{2, medWeights},
	{3, easyWeights},
	{2, ai.DefaultWeights},
	{3, medWeights},
	{3, easyWeights},
	{4, medWeights},
	{3, ai.DefaultWeights},
	{5, easyWeights},
	{5, medWeights},
	{4, ai.DefaultWeights},
	{5, ai.DefaultWeights},
}

func (f *Friendly) levelSettings(size int, level int) (int, ai.EvaluationFunc) {
	if level == 0 {
		level = 3
	}
	if level > len(levels)+1 {
		return 7, ai.DefaultEvaluate
	}
	s := levels[level-1]
	return s.depth, ai.MakeEvaluator(&s.weights)
}
<|endoftext|>


<commit_before>// Copyright 2012-2017 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// gitclone does not work!
package main

import (
	"bytes"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
)

type fdata struct {
	err  error
	data []byte
	url  string
}

func fetch(url string) ([]byte, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	b := bytes.NewBuffer([]byte{})
	_, err = io.Copy(b, resp.Body)
	if err != nil {
		return nil, err
	}
	return b.Bytes(), nil
}

func get(url string, result chan<- *fdata) {
	b, err := fetch(url)
	result <- &fdata{data: b, err: err, url: url}
}

func main() {
	// synchronously get the head. If that's not there, there's no
	// point in doing anything else.
	h, err := fetch(os.Args[1])
	if err != nil {
		log.Fatalf("%v", err)
	}
	fmt.Printf("Head %v\n", h)
}
<commit_msg>Remove unimplemented gitclone<commit_after><|endoftext|>


<commit_before>package node

import (
	"github.com/henrylee2cn/pholcus/node/crawlpool"
	"github.com/henrylee2cn/pholcus/node/spiderqueue"
	"github.com/henrylee2cn/pholcus/node/task"
	"github.com/henrylee2cn/pholcus/runtime/cache"
	"github.com/henrylee2cn/pholcus/runtime/status"
	"github.com/henrylee2cn/teleport"
	"log"
	"strconv"
	"time"
)

type Node struct {
	// 运行模式
	RunMode int
	// 服务器端口号
	Port string
	// 服务器地址（不含Port）
	Master string
	// socket长连接双工通信接口，json数据传输
	teleport.Teleport
	// 节点间传递的任务的存储库
	*task.TaskJar
	// 当前任务的蜘蛛队列
	Spiders spiderqueue.SpiderQueue
	// 爬行动作的回收池
	Crawls crawlpool.CrawlPool
	// 节点状态
	Status int
}

func newPholcus() *Node {
	return &Node{
		RunMode:  cache.Task.RunMode,
		Port:     ":" + strconv.Itoa(cache.Task.Port),
		Master:   cache.Task.Master,
		Teleport: teleport.New(),
		TaskJar:  task.NewTaskJar(),
		Spiders:  spiderqueue.New(),
		Crawls:   crawlpool.New(),
		Status:   status.RUN,
	}
}

// 声明实例
var Pholcus *Node = nil

// 运行节点
func PholcusRun() {
	if Pholcus != nil {
		return
	}
	Pholcus = newPholcus()
	switch Pholcus.RunMode {
	case status.SERVER:
		if Pholcus.checkPort() {
			log.Printf("                                                                                ！！当前运行模式为：[ 服务器 ] 模式！！")
			Pholcus.Teleport.SetAPI(ServerApi).SetUID("服务端").Server(Pholcus.Port)
		}

	case status.CLIENT:
		if Pholcus.checkAll() {
			log.Printf("                                                                                ！！当前运行模式为：[ 客户端 ] 模式！！")
			Pholcus.Teleport.SetAPI(ClientApi).Client(Pholcus.Master, Pholcus.Port)
		}
	// case status.OFFLINE:
	// 	fallthrough
	default:
		log.Printf("                                                                                ！！当前运行模式为：[ 单机 ] 模式！！")
		return
	}
	// 开启实时log发送
	go Pholcus.log()
}

// 返回节点数
func (self *Node) CountNodes() int {
	return self.Teleport.CountNodes()
}

// 生成task并添加至库，服务器模式专用
func (self *Node) AddNewTask() (tasksNum, spidersNum int) {
	length := self.Spiders.Len()
	t := task.Task{}

	// 从配置读取字段
	t.ThreadNum = cache.Task.ThreadNum
	t.BaseSleeptime = cache.Task.BaseSleeptime
	t.RandomSleepPeriod = cache.Task.RandomSleepPeriod
	t.OutType = cache.Task.OutType
	t.DockerCap = cache.Task.DockerCap
	t.DockerQueueCap = cache.Task.DockerQueueCap
	t.MaxPage = cache.Task.MaxPage

	for i, sp := range self.Spiders.GetAll() {

		t.Spiders = append(t.Spiders, map[string]string{"name": sp.GetName(), "keyword": sp.GetKeyword()})
		spidersNum++

		// 每十个蜘蛛存为一个任务
		if i > 0 && i%10 == 0 && length > 10 {
			// 存入
			one := t
			self.TaskJar.Push(&one)
			// log.Printf(" *     [新增任务]   详情： %#v", *t)

			tasksNum++

			// 清空spider
			t.Spiders = []map[string]string{}
		}
	}

	if len(t.Spiders) != 0 {
		// 存入
		one := t
		self.TaskJar.Push(&one)
		// log.Printf(" *     [新增任务]   详情： %#v", *t)
		tasksNum++
	}
	return
}

// 客户端请求获取任务
func (self *Node) GetTaskAlways() {
	self.Request(nil, "task")
}

// 客户端模式下获取任务
func (self *Node) DownTask() *task.Task {
ReStartLabel:
	for self.CountNodes() == 0 {
		if len(self.TaskJar.Tasks) != 0 {
			break
		}
		time.Sleep(5e7)
	}

	if len(self.TaskJar.Tasks) == 0 {
		self.GetTaskAlways()
		for len(self.TaskJar.Tasks) == 0 {
			if self.CountNodes() == 0 {
				goto ReStartLabel
			}
			time.Sleep(5e7)
		}
	}
	return self.TaskJar.Pull()
}

func (self *Node) log() {
	for {
		self.Teleport.Request(<-cache.SendChan, "log")
	}
}

func (self *Node) checkPort() bool {
	if cache.Task.Port == 0 {
		log.Println(" *     —— 亲，分布式端口不能为空哦~")
		return false
	}
	return true
}

func (self *Node) checkAll() bool {
	if cache.Task.Master == "" || !self.checkPort() {
		log.Println(" *     —— 亲，服务器地址不能为空哦~")
		return false
	}
	return true
}
<commit_msg>v0.4.8<commit_after>package node

import (
	"github.com/henrylee2cn/pholcus/node/crawlpool"
	"github.com/henrylee2cn/pholcus/node/spiderqueue"
	"github.com/henrylee2cn/pholcus/node/task"
	"github.com/henrylee2cn/pholcus/runtime/cache"
	"github.com/henrylee2cn/pholcus/runtime/status"
	"github.com/henrylee2cn/teleport"
	"log"
	"strconv"
	"time"
)

type Node struct {
	// 运行模式
	RunMode int
	// 服务器端口号
	Port string
	// 服务器地址（不含Port）
	Master string
	// socket长连接双工通信接口，json数据传输
	teleport.Teleport
	// 节点间传递的任务的存储库
	*task.TaskJar
	// 当前任务的蜘蛛队列
	Spiders spiderqueue.SpiderQueue
	// 爬行动作的回收池
	Crawls crawlpool.CrawlPool
	// 节点状态
	Status int
}

func newPholcus() *Node {
	return &Node{
		RunMode:  cache.Task.RunMode,
		Port:     ":" + strconv.Itoa(cache.Task.Port),
		Master:   cache.Task.Master,
		Teleport: teleport.New(),
		TaskJar:  task.NewTaskJar(),
		Spiders:  spiderqueue.New(),
		Crawls:   crawlpool.New(),
		Status:   status.RUN,
	}
}

// 声明实例
var Pholcus *Node = nil

// 运行节点
func PholcusRun() {
	if Pholcus != nil {
		return
	}
	Pholcus = newPholcus()
	switch Pholcus.RunMode {
	case status.SERVER:
		if Pholcus.checkPort() {
			log.Printf("                                                                                ！！当前运行模式为：[ 服务器 ] 模式！！")
			Pholcus.Teleport.SetAPI(ServerApi).Server(Pholcus.Port)
		}

	case status.CLIENT:
		if Pholcus.checkAll() {
			log.Printf("                                                                                ！！当前运行模式为：[ 客户端 ] 模式！！")
			Pholcus.Teleport.SetAPI(ClientApi).Client(Pholcus.Master, Pholcus.Port)
		}
	// case status.OFFLINE:
	// 	fallthrough
	default:
		log.Printf("                                                                                ！！当前运行模式为：[ 单机 ] 模式！！")
		return
	}
	// 开启实时log发送
	go Pholcus.log()
}

// 返回节点数
func (self *Node) CountNodes() int {
	return self.Teleport.CountNodes()
}

// 生成task并添加至库，服务器模式专用
func (self *Node) AddNewTask() (tasksNum, spidersNum int) {
	length := self.Spiders.Len()
	t := task.Task{}

	// 从配置读取字段
	t.ThreadNum = cache.Task.ThreadNum
	t.BaseSleeptime = cache.Task.BaseSleeptime
	t.RandomSleepPeriod = cache.Task.RandomSleepPeriod
	t.OutType = cache.Task.OutType
	t.DockerCap = cache.Task.DockerCap
	t.DockerQueueCap = cache.Task.DockerQueueCap
	t.MaxPage = cache.Task.MaxPage

	for i, sp := range self.Spiders.GetAll() {

		t.Spiders = append(t.Spiders, map[string]string{"name": sp.GetName(), "keyword": sp.GetKeyword()})
		spidersNum++

		// 每十个蜘蛛存为一个任务
		if i > 0 && i%10 == 0 && length > 10 {
			// 存入
			one := t
			self.TaskJar.Push(&one)
			// log.Printf(" *     [新增任务]   详情： %#v", *t)

			tasksNum++

			// 清空spider
			t.Spiders = []map[string]string{}
		}
	}

	if len(t.Spiders) != 0 {
		// 存入
		one := t
		self.TaskJar.Push(&one)
		// log.Printf(" *     [新增任务]   详情： %#v", *t)
		tasksNum++
	}
	return
}

// 客户端请求获取任务
func (self *Node) GetTaskAlways() {
	self.Request(nil, "task")
}

// 客户端模式下获取任务
func (self *Node) DownTask() *task.Task {
ReStartLabel:
	for self.CountNodes() == 0 {
		if len(self.TaskJar.Tasks) != 0 {
			break
		}
		time.Sleep(5e7)
	}

	if len(self.TaskJar.Tasks) == 0 {
		self.GetTaskAlways()
		for len(self.TaskJar.Tasks) == 0 {
			if self.CountNodes() == 0 {
				goto ReStartLabel
			}
			time.Sleep(5e7)
		}
	}
	return self.TaskJar.Pull()
}

func (self *Node) log() {
	for {
		self.Teleport.Request(<-cache.SendChan, "log")
	}
}

func (self *Node) checkPort() bool {
	if cache.Task.Port == 0 {
		log.Println(" *     —— 亲，分布式端口不能为空哦~")
		return false
	}
	return true
}

func (self *Node) checkAll() bool {
	if cache.Task.Master == "" || !self.checkPort() {
		log.Println(" *     —— 亲，服务器地址不能为空哦~")
		return false
	}
	return true
}
<|endoftext|>


<commit_before>// +build !appengine,!plan9

package maxminddb

import (
	"os"
	"runtime"
)

// Open takes a string path to a MaxMind DB file and returns a Reader
// structure or an error. The database file is opened using a memory map,
// except on Google App Engine where mmap is not supported; there the database
// is loaded into memory. Use the Close method on the Reader object to return
// the resources to the system.
func Open(file string) (*Reader, error) {
	mapFile, err := os.Open(file)
	if err != nil {
		return nil, err
	}
	defer func() {
		if rerr := mapFile.Close(); rerr != nil {
			err = rerr
		}
	}()

	stats, err := mapFile.Stat()
	if err != nil {
		return nil, err
	}

	fileSize := int(stats.Size())
	mmap, err := mmap(int(mapFile.Fd()), fileSize)
	if err != nil {
		return nil, err
	}

	reader, err := FromBytes(mmap)
	if err != nil {
		if err2 := munmap(mmap); err2 != nil {
			// failing to unmap the file is probably the more severe error
			return nil, err2
		}
		return nil, err
	}

	reader.hasMappedFile = true
	runtime.SetFinalizer(reader, (*Reader).Close)
	return reader, err
}

// Close unmaps the database file from virtual memory and returns the
// resources to the system. If called on a Reader opened using FromBytes
// or Open on Google App Engine, this method does nothing.
func (r *Reader) Close() error {
	var err error
	if r.hasMappedFile {
		runtime.SetFinalizer(r, nil)
		r.hasMappedFile = false
		err = munmap(r.buffer)
	}
	r.buffer = nil
	return err
}
<commit_msg>Do not close file from deferred function<commit_after>// +build !appengine,!plan9

package maxminddb

import (
	"os"
	"runtime"
)

// Open takes a string path to a MaxMind DB file and returns a Reader
// structure or an error. The database file is opened using a memory map,
// except on Google App Engine where mmap is not supported; there the database
// is loaded into memory. Use the Close method on the Reader object to return
// the resources to the system.
func Open(file string) (*Reader, error) {
	mapFile, err := os.Open(file)
	if err != nil {
		_ = mapFile.Close()
		return nil, err
	}

	stats, err := mapFile.Stat()
	if err != nil {
		_ = mapFile.Close()
		return nil, err
	}

	fileSize := int(stats.Size())
	mmap, err := mmap(int(mapFile.Fd()), fileSize)
	if err != nil {
		_ = mapFile.Close()
		return nil, err
	}

	if err := mapFile.Close(); err != nil {
		_ = munmap(mmap)
		return nil, err
	}

	reader, err := FromBytes(mmap)
	if err != nil {
		_ = munmap(mmap)
		return nil, err
	}

	reader.hasMappedFile = true
	runtime.SetFinalizer(reader, (*Reader).Close)
	return reader, nil
}

// Close unmaps the database file from virtual memory and returns the
// resources to the system. If called on a Reader opened using FromBytes
// or Open on Google App Engine, this method does nothing.
func (r *Reader) Close() error {
	var err error
	if r.hasMappedFile {
		runtime.SetFinalizer(r, nil)
		r.hasMappedFile = false
		err = munmap(r.buffer)
	}
	r.buffer = nil
	return err
}
<|endoftext|>