<commit_before>package pcf_pipelines_test

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	yaml "gopkg.in/yaml.v2"

	"github.com/concourse/atc"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var placeholderRegexp = regexp.MustCompile("{{([a-zA-Z0-9-_]+)}}")

var _ = Describe("pcf-pipelines", func() {
	cwd, err := os.Getwd()
	if err != nil {
		log.Fatalf("failed to get working dir: %s", err)
	}

	root := filepath.Dir(cwd)
	baseDir := filepath.Base(cwd)

	var pipelinePaths []string
	err = filepath.Walk(cwd, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if filepath.Base(path) == "pipeline.yml" {
			relPipelinePath, err := filepath.Rel(cwd, path)
			if err != nil {
				return err
			}
			pipelinePaths = append(pipelinePaths, relPipelinePath)
		}

		return nil
	})
	if err != nil {
		log.Fatalf("failed to walk: %s", err)
	}

	for _, path := range pipelinePaths {
		pipelinePath := path

		Context(fmt.Sprintf("pipeline at %s", pipelinePath), func() {
			var configBytes []byte

			BeforeEach(func() {
				var err error
				configBytes, err = ioutil.ReadFile(pipelinePath)
				Expect(err).NotTo(HaveOccurred())
			})

			It("specifies only valid job names in any `passed` definitions in the buildplan", func() {
				var config atc.Config
				cleanConfigBytes := placeholderRegexp.ReplaceAll(configBytes, []byte("true"))
				err := yaml.Unmarshal(cleanConfigBytes, &config)
				Expect(err).NotTo(HaveOccurred())

				for _, job := range config.Jobs {
					for _, plan := range job.Plans() {
						checkValidJobsList(config.Jobs, plan.Passed, job.Name)
					}
				}
			})

			It("specifies all and only the params that the pipeline's tasks expect", func() {
				var config atc.Config
				cleanConfigBytes := placeholderRegexp.ReplaceAll(configBytes, []byte("true"))
				err := yaml.Unmarshal(cleanConfigBytes, &config)
				Expect(err).NotTo(HaveOccurred())

				for _, job := range config.Jobs {
					for _, task := range allTasksInPlan(&job.Plan) {
						failMessage := fmt.Sprintf("Found error in the following pipeline:\n    %s\n\nin the following task's params:\n    %s/%s\n", pipelinePath, job.Name, task.Name())

						var configParams []string
						for k := range task.Params {
							configParams = append(configParams, k)
						}

						if strings.HasPrefix(task.TaskConfigPath, baseDir) {
							taskPath := strings.TrimPrefix(task.TaskConfigPath, baseDir+"/")
							relpath, err := filepath.Rel(cwd, filepath.Join(cwd, taskPath))
							Expect(err).NotTo(HaveOccurred())

							bs, err := ioutil.ReadFile(relpath)
							Expect(err).NotTo(HaveOccurred())

							taskConfig := atc.TaskConfig{}
							err = yaml.Unmarshal(bs, &taskConfig)
							Expect(err).NotTo(HaveOccurred())

							var taskParams []string
							for k := range taskConfig.Params {
								taskParams = append(taskParams, k)
							}

							assertUnorderedEqual(taskParams, configParams, failMessage)
						}
					}
				}
			})

			It("has a params file with all and only the params that the pipeline specifies", func() {
				paramsPath := filepath.Join(filepath.Dir(pipelinePath), "params.yml")
				_, err := os.Lstat(paramsPath)
				Expect(err).NotTo(HaveOccurred())

				bs, err := ioutil.ReadFile(paramsPath)
				Expect(err).NotTo(HaveOccurred())

				paramsMap := map[string]interface{}{}
				err = yaml.Unmarshal(bs, paramsMap)
				Expect(err).NotTo(HaveOccurred())

				var params []string
				for k := range paramsMap {
					params = append(params, k)
				}

				matches := placeholderRegexp.FindAllStringSubmatch(string(configBytes), -1)

				uniqueMatches := map[string]struct{}{}
				for _, match := range matches {
					uniqueMatches[match[1]] = struct{}{}
				}

				var placeholders []string
				for match := range uniqueMatches {
					placeholders = append(placeholders, match)
				}

				failMessage := fmt.Sprintf(`
Found error with the following pipeline:
%s

in the following params template:
%s
`, pipelinePath, paramsPath)

				assertUnorderedEqual(placeholders, params, failMessage)
			})

			It("provides all of the resources that the tasks it defines require", func() {
				var config atc.Config
				cleanConfigBytes := placeholderRegexp.ReplaceAll(configBytes, []byte("true"))
				err := yaml.Unmarshal(cleanConfigBytes, &config)
				Expect(err).NotTo(HaveOccurred())

				for _, job := range config.Jobs {
					tasks := allTasksInPlan(&job.Plan)
					resources := availableResources(&job.Plan)

					for i, task := range tasks {
						if !strings.HasPrefix(task.TaskConfigPath, "pcf-pipelines") {
							continue
						}

						var inputs []atc.TaskInputConfig
						if task.TaskConfig != nil {
							inputs = task.TaskConfig.Inputs
						} else {
							inputs = taskInputConfigs(filepath.Join(root, task.TaskConfigPath))
						}

						for k := range task.InputMapping {
							var validInputMapping bool
							for _, input := range inputs {
								if input.Name == k {
									validInputMapping = true
									break
								}
							}
							if !validInputMapping {
								Fail(fmt.Sprintf("could not find input mapping for '%s' in '%s'\n", k, inputs))
							}
						}

						for j := 0; j < i; j++ {
							upstreamTask := tasks[j]

							if upstreamTask.TaskConfig != nil {
								for _, output := range upstreamTask.TaskConfig.Outputs {
									resources = append(resources, output.Name)
								}
							}

							if upstreamTask.TaskConfigPath != "" {
								var upstreamTaskConfig atc.TaskConfig
								bs, err := ioutil.ReadFile(filepath.Join(root, upstreamTask.TaskConfigPath))
								Expect(err).NotTo(HaveOccurred())

								err = yaml.Unmarshal(bs, &upstreamTaskConfig)
								Expect(err).NotTo(HaveOccurred())

								for _, output := range upstreamTaskConfig.Outputs {
									resources = append(resources, output.Name)
								}
							}

							for _, v := range upstreamTask.OutputMapping {
								resources = append(resources, v)
							}
						}

					OUTER:
						for _, input := range inputs {
							for _, actual := range resources {
								if input.Name == actual {
									continue OUTER
								}

								for k, v := range task.InputMapping {
									if k == input.Name && v == actual {
										continue OUTER
									}
								}
							}

							Fail(fmt.Sprintf("did not find matching get, put, output or input_mapping of '%s', which is required by task '%s'", input.Name, task.Name()))
						}
					}
				}
			})
		})
	}
})

func checkValidJobsList(jobs atc.JobConfigs, jobNames []string, location string) {
	for _, jobName := range jobNames {
		_, exists := jobs.Lookup(jobName)
		Expect(exists).Should(BeTrue(), fmt.Sprintf("%s is not a valid job defined in %s", jobName, location))
	}
}

func allTasksInPlan(seq *atc.PlanSequence) []atc.PlanConfig {
	var tasks []atc.PlanConfig

	for _, planConfig := range *seq {
		if planConfig.Aggregate != nil {
			tasks = append(tasks, allTasksInPlan(planConfig.Aggregate)...)
		}
		if planConfig.Do != nil {
			tasks = append(tasks, allTasksInPlan(planConfig.Do)...)
		}
		if planConfig.Task != "" {
			tasks = append(tasks, planConfig)
		}
	}

	return tasks
}

func availableResources(seq *atc.PlanSequence) []string {
	var resources []string

	for _, planConfig := range *seq {
		if planConfig.Aggregate != nil {
			resources = append(resources, availableResources(planConfig.Aggregate)...)
		}

		if planConfig.Do != nil {
			resources = append(resources, availableResources(planConfig.Do)...)
		}

		if planConfig.Get != "" {
			resources = append(resources, planConfig.Get)
		}

		if planConfig.Put != "" {
			resources = append(resources, planConfig.Put)
		}
	}

	return resources
}

func assertUnorderedEqual(left, right []string, failMessage string) {
	for _, l := range left {
		Expect(right).To(ContainElement(l), failMessage)
	}

	for _, r := range right {
		var found bool

		for _, l := range left {
			if r == l {
				found = true
				break
			}
		}

		if !found {
			Expect(right).NotTo(ContainElement(r), failMessage)
		}
	}
}

var taskConfigs = map[string]*atc.TaskConfig{}

func taskInputConfigs(path string) []atc.TaskInputConfig {
	taskConfig, ok := taskConfigs[path]

	if !ok {
		bs, err := ioutil.ReadFile(path)
		Expect(err).NotTo(HaveOccurred())

		err = yaml.Unmarshal(bs, &taskConfig)
		Expect(err).NotTo(HaveOccurred())

		taskConfigs[path] = taskConfig
	}

	return taskConfig.Inputs
}
<commit_msg>Update placeholderRegexp to support parens<commit_after>package pcf_pipelines_test

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	yaml "gopkg.in/yaml.v2"

	"github.com/concourse/atc"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var oldPlaceholderRegexp = regexp.MustCompile("{{([a-zA-Z0-9-_]+)}}")
var placeholderRegexp = regexp.MustCompile("[({]{2}([a-zA-Z0-9-_]+)(?:)[)}]{2}")

var _ = Describe("pcf-pipelines", func() {
	cwd, err := os.Getwd()
	if err != nil {
		log.Fatalf("failed to get working dir: %s", err)
	}

	root := filepath.Dir(cwd)
	baseDir := filepath.Base(cwd)

	var pipelinePaths []string
	err = filepath.Walk(cwd, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if filepath.Base(path) == "pipeline.yml" {
			relPipelinePath, err := filepath.Rel(cwd, path)
			if err != nil {
				return err
			}
			pipelinePaths = append(pipelinePaths, relPipelinePath)
		}

		return nil
	})
	if err != nil {
		log.Fatalf("failed to walk: %s", err)
	}

	for _, path := range pipelinePaths {
		pipelinePath := path

		Context(fmt.Sprintf("pipeline at %s", pipelinePath), func() {
			var configBytes []byte

			BeforeEach(func() {
				var err error
				configBytes, err = ioutil.ReadFile(pipelinePath)
				Expect(err).NotTo(HaveOccurred())
			})

			It("specifies only valid job names in any `passed` definitions in the buildplan", func() {
				var config atc.Config
				cleanConfigBytes := oldPlaceholderRegexp.ReplaceAll(configBytes, []byte("true"))
				err := yaml.Unmarshal(cleanConfigBytes, &config)
				Expect(err).NotTo(HaveOccurred())

				for _, job := range config.Jobs {
					for _, plan := range job.Plans() {
						checkValidJobsList(config.Jobs, plan.Passed, job.Name)
					}
				}
			})

			It("specifies all and only the params that the pipeline's tasks expect", func() {
				var config atc.Config
				cleanConfigBytes := oldPlaceholderRegexp.ReplaceAll(configBytes, []byte("true"))
				err := yaml.Unmarshal(cleanConfigBytes, &config)
				Expect(err).NotTo(HaveOccurred())

				for _, job := range config.Jobs {
					for _, task := range allTasksInPlan(&job.Plan) {
						failMessage := fmt.Sprintf("Found error in the following pipeline:\n    %s\n\nin the following task's params:\n    %s/%s\n", pipelinePath, job.Name, task.Name())

						var configParams []string
						for k := range task.Params {
							configParams = append(configParams, k)
						}

						if strings.HasPrefix(task.TaskConfigPath, baseDir) {
							taskPath := strings.TrimPrefix(task.TaskConfigPath, baseDir+"/")
							relpath, err := filepath.Rel(cwd, filepath.Join(cwd, taskPath))
							Expect(err).NotTo(HaveOccurred())

							bs, err := ioutil.ReadFile(relpath)
							Expect(err).NotTo(HaveOccurred())

							taskConfig := atc.TaskConfig{}
							err = yaml.Unmarshal(bs, &taskConfig)
							Expect(err).NotTo(HaveOccurred())

							var taskParams []string
							for k := range taskConfig.Params {
								taskParams = append(taskParams, k)
							}

							assertUnorderedEqual(taskParams, configParams, failMessage)
						}
					}
				}
			})

			It("has a params file with all and only the params that the pipeline specifies", func() {
				paramsPath := filepath.Join(filepath.Dir(pipelinePath), "params.yml")
				_, err := os.Lstat(paramsPath)
				Expect(err).NotTo(HaveOccurred())

				bs, err := ioutil.ReadFile(paramsPath)
				Expect(err).NotTo(HaveOccurred())

				paramsMap := map[string]interface{}{}
				err = yaml.Unmarshal(bs, paramsMap)
				Expect(err).NotTo(HaveOccurred())

				var params []string
				for k := range paramsMap {
					params = append(params, k)
				}

				matches := placeholderRegexp.FindAllStringSubmatch(string(configBytes), -1)

				uniqueMatches := map[string]struct{}{}
				for _, match := range matches {
					uniqueMatches[match[1]] = struct{}{}
				}

				var placeholders []string
				for match := range uniqueMatches {
					placeholders = append(placeholders, match)
				}

				failMessage := fmt.Sprintf(`
Found error with the following pipeline:
%s

in the following params template:
%s
`, pipelinePath, paramsPath)

				assertUnorderedEqual(placeholders, params, failMessage)
			})

			It("provides all of the resources that the tasks it defines require", func() {
				var config atc.Config
				cleanConfigBytes := placeholderRegexp.ReplaceAll(configBytes, []byte("true"))
				err := yaml.Unmarshal(cleanConfigBytes, &config)
				Expect(err).NotTo(HaveOccurred())

				for _, job := range config.Jobs {
					tasks := allTasksInPlan(&job.Plan)
					resources := availableResources(&job.Plan)

					for i, task := range tasks {
						if !strings.HasPrefix(task.TaskConfigPath, "pcf-pipelines") {
							continue
						}

						var inputs []atc.TaskInputConfig
						if task.TaskConfig != nil {
							inputs = task.TaskConfig.Inputs
						} else {
							inputs = taskInputConfigs(filepath.Join(root, task.TaskConfigPath))
						}

						for k := range task.InputMapping {
							var validInputMapping bool
							for _, input := range inputs {
								if input.Name == k {
									validInputMapping = true
									break
								}
							}
							if !validInputMapping {
								Fail(fmt.Sprintf("could not find input mapping for '%s' in '%s'\n", k, inputs))
							}
						}

						for j := 0; j < i; j++ {
							upstreamTask := tasks[j]

							if upstreamTask.TaskConfig != nil {
								for _, output := range upstreamTask.TaskConfig.Outputs {
									resources = append(resources, output.Name)
								}
							}

							if upstreamTask.TaskConfigPath != "" {
								var upstreamTaskConfig atc.TaskConfig
								bs, err := ioutil.ReadFile(filepath.Join(root, upstreamTask.TaskConfigPath))
								Expect(err).NotTo(HaveOccurred())

								err = yaml.Unmarshal(bs, &upstreamTaskConfig)
								Expect(err).NotTo(HaveOccurred())

								for _, output := range upstreamTaskConfig.Outputs {
									resources = append(resources, output.Name)
								}
							}

							for _, v := range upstreamTask.OutputMapping {
								resources = append(resources, v)
							}
						}

					OUTER:
						for _, input := range inputs {
							for _, actual := range resources {
								if input.Name == actual {
									continue OUTER
								}

								for k, v := range task.InputMapping {
									if k == input.Name && v == actual {
										continue OUTER
									}
								}
							}

							Fail(fmt.Sprintf("did not find matching get, put, output or input_mapping of '%s', which is required by task '%s'", input.Name, task.Name()))
						}
					}
				}
			})
		})
	}
})

func checkValidJobsList(jobs atc.JobConfigs, jobNames []string, location string) {
	for _, jobName := range jobNames {
		_, exists := jobs.Lookup(jobName)
		Expect(exists).Should(BeTrue(), fmt.Sprintf("%s is not a valid job defined in %s", jobName, location))
	}
}

func allTasksInPlan(seq *atc.PlanSequence) []atc.PlanConfig {
	var tasks []atc.PlanConfig

	for _, planConfig := range *seq {
		if planConfig.Aggregate != nil {
			tasks = append(tasks, allTasksInPlan(planConfig.Aggregate)...)
		}
		if planConfig.Do != nil {
			tasks = append(tasks, allTasksInPlan(planConfig.Do)...)
		}
		if planConfig.Task != "" {
			tasks = append(tasks, planConfig)
		}
	}

	return tasks
}

func availableResources(seq *atc.PlanSequence) []string {
	var resources []string

	for _, planConfig := range *seq {
		if planConfig.Aggregate != nil {
			resources = append(resources, availableResources(planConfig.Aggregate)...)
		}

		if planConfig.Do != nil {
			resources = append(resources, availableResources(planConfig.Do)...)
		}

		if planConfig.Get != "" {
			resources = append(resources, planConfig.Get)
		}

		if planConfig.Put != "" {
			resources = append(resources, planConfig.Put)
		}
	}

	return resources
}

func assertUnorderedEqual(left, right []string, failMessage string) {
	for _, l := range left {
		Expect(right).To(ContainElement(l), failMessage)
	}

	for _, r := range right {
		var found bool

		for _, l := range left {
			if r == l {
				found = true
				break
			}
		}

		if !found {
			Expect(right).NotTo(ContainElement(r), failMessage)
		}
	}
}

var taskConfigs = map[string]*atc.TaskConfig{}

func taskInputConfigs(path string) []atc.TaskInputConfig {
	taskConfig, ok := taskConfigs[path]

	if !ok {
		bs, err := ioutil.ReadFile(path)
		Expect(err).NotTo(HaveOccurred())

		err = yaml.Unmarshal(bs, &taskConfig)
		Expect(err).NotTo(HaveOccurred())

		taskConfigs[path] = taskConfig
	}

	return taskConfig.Inputs
}
<|endoftext|>


<commit_before>package main

import (
	. "github.com/frezadev/hdc/hive"
	// . "github.com/eaciit/hdc/hive"
	"github.com/pkg/profile"
	"log"
)

type Students struct {
	Name    string
	Age     int
	Phone   string
	Address string
}

func fatalCheck(what string, e error) {
	if e != nil {
		log.Fatalf("%s: %s", what, e.Error())
	}
}

func main() {
	defer profile.Start(profile.CPUProfile, profile.MemProfile, profile.BlockProfile).Stop()

	h := HiveConfig("192.168.0.223:10000", "default", "hdfs", "", "")
	err := h.Conn.Open()
	fatalCheck("Populate", err)

	var student Students

	totalWorker := 10
	retVal, err := h.LoadFileWithWorker("/home/developer/contoh.txt", "students", "csv", "dd/MM/yyyy", &student, totalWorker)

	if err != nil {
		fatalCheck("Populate", err)
	}

	h.Conn.Close()
	log.Printf("retVal: \n%v\n", retVal)
}
<commit_msg>change import part<commit_after>package main

import (
	// . "github.com/frezadev/hdc/hive"
	. "github.com/eaciit/hdc/hive"
	"github.com/pkg/profile"
	"log"
)

type Students struct {
	Name    string
	Age     int
	Phone   string
	Address string
}

func fatalCheck(what string, e error) {
	if e != nil {
		log.Fatalf("%s: %s", what, e.Error())
	}
}

func main() {
	defer profile.Start(profile.CPUProfile, profile.MemProfile, profile.BlockProfile).Stop()

	h := HiveConfig("192.168.0.223:10000", "default", "hdfs", "", "")
	err := h.Conn.Open()
	fatalCheck("Populate", err)

	var student Students

	totalWorker := 10
	retVal, err := h.LoadFileWithWorker("/home/developer/contoh.txt", "students", "csv", "dd/MM/yyyy", &student, totalWorker)

	if err != nil {
		fatalCheck("Populate", err)
	}

	h.Conn.Close()
	log.Printf("retVal: \n%v\n", retVal)
}
<|endoftext|>


<commit_before>// Copyright 2016-2021 The Libsacloud Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package certificateauthority

import (
	"testing"
	"time"

	"github.com/sacloud/libsacloud/v2/sacloud"
	"github.com/sacloud/libsacloud/v2/sacloud/testutil"
	"github.com/sacloud/libsacloud/v2/sacloud/types"
)

func TestCertificateAuthorityService_CRUD(t *testing.T) {
	if !testutil.IsAccTest() {
		t.Skip("TestCertificateAuthorityService_CRUD only exec when running an Acceptance Test")
	}

	svc := New(testutil.SingletonAPICaller())
	name := testutil.ResourceName("ca")

	testutil.RunCRUD(t, &testutil.CRUDTestCase{
		Parallel:           true,
		PreCheck:           nil,
		SetupAPICallerFunc: testutil.SingletonAPICaller,
		Setup:              nil,
		Create: &testutil.CRUDTestFunc{
			Func: func(ctx *testutil.CRUDTestContext, _ sacloud.APICaller) (interface{}, error) {
				return svc.Create(&CreateRequest{
					Name:             name,
					Description:      "test",
					Tags:             types.Tags{"tag1", "tag2"},
					Country:          "JP",
					Organization:     "usacloud",
					OrganizationUnit: []string{"ou1","ou2"},
					CommonName:       "www.usacloud.jp",
					NotAfter:         time.Now().Add(365 * 24 * time.Hour),
					Clients:          []*ClientCert{
						{
							Country:                   "JP",
							Organization:              "usacloud",
							CommonName:                "client.usacloud.jp",
							NotAfter:                  time.Now().Add(365 * 24 * time.Hour),
							IssuanceMethod:            "url",
						},
					},
					WaitDuration:     0,
				})
			},
		},
		Read: &testutil.CRUDTestFunc{
			Func: func(ctx *testutil.CRUDTestContext, _ sacloud.APICaller) (interface{}, error) {
				return svc.Read(&ReadRequest{ID: ctx.ID})
			},
		},
		Delete: &testutil.CRUDTestDeleteFunc{
			Func: func(ctx *testutil.CRUDTestContext, _ sacloud.APICaller) error {
				return svc.Delete(&DeleteRequest{ID: ctx.ID})
			},
		},
	})
}
<commit_msg>go fmt<commit_after>// Copyright 2016-2021 The Libsacloud Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package certificateauthority

import (
	"testing"
	"time"

	"github.com/sacloud/libsacloud/v2/sacloud"
	"github.com/sacloud/libsacloud/v2/sacloud/testutil"
	"github.com/sacloud/libsacloud/v2/sacloud/types"
)

func TestCertificateAuthorityService_CRUD(t *testing.T) {
	if !testutil.IsAccTest() {
		t.Skip("TestCertificateAuthorityService_CRUD only exec when running an Acceptance Test")
	}

	svc := New(testutil.SingletonAPICaller())
	name := testutil.ResourceName("ca")

	testutil.RunCRUD(t, &testutil.CRUDTestCase{
		Parallel:           true,
		PreCheck:           nil,
		SetupAPICallerFunc: testutil.SingletonAPICaller,
		Setup:              nil,
		Create: &testutil.CRUDTestFunc{
			Func: func(ctx *testutil.CRUDTestContext, _ sacloud.APICaller) (interface{}, error) {
				return svc.Create(&CreateRequest{
					Name:             name,
					Description:      "test",
					Tags:             types.Tags{"tag1", "tag2"},
					Country:          "JP",
					Organization:     "usacloud",
					OrganizationUnit: []string{"ou1", "ou2"},
					CommonName:       "www.usacloud.jp",
					NotAfter:         time.Now().Add(365 * 24 * time.Hour),
					Clients: []*ClientCert{
						{
							Country:        "JP",
							Organization:   "usacloud",
							CommonName:     "client.usacloud.jp",
							NotAfter:       time.Now().Add(365 * 24 * time.Hour),
							IssuanceMethod: "url",
						},
					},
					WaitDuration: 0,
				})
			},
		},
		Read: &testutil.CRUDTestFunc{
			Func: func(ctx *testutil.CRUDTestContext, _ sacloud.APICaller) (interface{}, error) {
				return svc.Read(&ReadRequest{ID: ctx.ID})
			},
		},
		Delete: &testutil.CRUDTestDeleteFunc{
			Func: func(ctx *testutil.CRUDTestContext, _ sacloud.APICaller) error {
				return svc.Delete(&DeleteRequest{ID: ctx.ID})
			},
		},
	})
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"sort"
	"strings"
	"syscall"

	"github.com/olekukonko/tablewriter"
	"gopkg.in/yaml.v2"

	"github.com/lxc/lxd"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/i18n"
	"github.com/lxc/lxd/shared/termios"
)

type networkCmd struct {
}

func (c *networkCmd) showByDefault() bool {
	return true
}

func (c *networkCmd) networkEditHelp() string {
	return i18n.G(
		`### This is a yaml representation of the network.
### Any line starting with a '# will be ignored.
###
### A network consists of a set of configuration items.
###
### An example would look like:
### name: lxdbr0
### config:
###   ipv4.address: 10.62.42.1/24
###   ipv4.nat: true
###   ipv6.address: fd00:56ad:9f7a:9800::1/64
###   ipv6.nat: true
### managed: true
### type: bridge
###
### Note that only the configuration can be changed.`)
}

func (c *networkCmd) usage() string {
	return i18n.G(
		`Manage networks.

lxc network list [<remote>:]                              List available networks.
lxc network show [<remote>:]<network>                     Show details of a network.
lxc network create [<remote>:]<network> [key=value...]    Create a network.
lxc network get [<remote>:]<network> <key>                Get network configuration.
lxc network set [<remote>:]<network> <key> <value>        Set network configuration.
lxc network unset [<remote>:]<network> <key>              Unset network configuration.
lxc network delete [<remote>:]<network>                   Delete a network.
lxc network edit [<remote>:]<network>
    Edit network, either by launching external editor or reading STDIN.
    Example: lxc network edit <network> # launch editor
             cat network.yaml | lxc network edit <network> # read from network.yaml

lxc network attach [<remote>:]<network> <container> [device name]
lxc network attach-profile [<remote>:]<network> <profile> [device name]

lxc network detach [<remote>:]<network> <container> [device name]
lxc network detach-profile [<remote>:]<network> <container> [device name]`)
}

func (c *networkCmd) flags() {}

func (c *networkCmd) run(config *lxd.Config, args []string) error {
	if len(args) < 1 {
		return errArgs
	}

	if args[0] == "list" {
		return c.doNetworkList(config, args)
	}

	if len(args) < 2 {
		return errArgs
	}

	remote, network := config.ParseRemoteAndContainer(args[1])
	client, err := lxd.NewClient(config, remote)
	if err != nil {
		return err
	}

	switch args[0] {
	case "attach":
		return c.doNetworkAttach(client, network, args[2:])
	case "attach-profile":
		return c.doNetworkAttachProfile(client, network, args[2:])
	case "create":
		return c.doNetworkCreate(client, network, args[2:])
	case "delete":
		return c.doNetworkDelete(client, network)
	case "detach":
		return c.doNetworkDetach(client, network, args[2:])
	case "detach-profile":
		return c.doNetworkDetachProfile(client, network, args[2:])
	case "edit":
		return c.doNetworkEdit(client, network)
	case "get":
		return c.doNetworkGet(client, network, args[2:])
	case "set":
		return c.doNetworkSet(client, network, args[2:])
	case "unset":
		return c.doNetworkSet(client, network, args[2:])
	case "show":
		return c.doNetworkShow(client, network)
	default:
		return errArgs
	}
}

func (c *networkCmd) doNetworkAttach(client *lxd.Client, name string, args []string) error {
	if len(args) < 1 || len(args) > 2 {
		return errArgs
	}

	container := args[0]
	devName := name
	if len(args) > 1 {
		devName = args[1]
	}

	network, err := client.NetworkGet(name)
	if err != nil {
		return err
	}

	nicType := "macvlan"
	if network.Type == "bridge" {
		nicType = "bridged"
	}

	props := []string{fmt.Sprintf("nictype=%s", nicType), fmt.Sprintf("parent=%s", name)}
	resp, err := client.ContainerDeviceAdd(container, devName, "nic", props)
	if err != nil {
		return err
	}

	return client.WaitForSuccess(resp.Operation)
}

func (c *networkCmd) doNetworkAttachProfile(client *lxd.Client, name string, args []string) error {
	if len(args) < 1 || len(args) > 2 {
		return errArgs
	}

	profile := args[0]
	devName := name
	if len(args) > 1 {
		devName = args[1]
	}

	network, err := client.NetworkGet(name)
	if err != nil {
		return err
	}

	nicType := "macvlan"
	if network.Type == "bridge" {
		nicType = "bridged"
	}

	props := []string{fmt.Sprintf("nictype=%s", nicType), fmt.Sprintf("parent=%s", name)}
	_, err = client.ProfileDeviceAdd(profile, devName, "nic", props)
	return err
}

func (c *networkCmd) doNetworkCreate(client *lxd.Client, name string, args []string) error {
	config := map[string]string{}

	for i := 0; i < len(args); i++ {
		entry := strings.SplitN(args[i], "=", 2)
		if len(entry) < 2 {
			return errArgs
		}

		config[entry[0]] = entry[1]
	}

	err := client.NetworkCreate(name, config)
	if err == nil {
		fmt.Printf(i18n.G("Network %s created")+"\n", name)
	}

	return err
}

func (c *networkCmd) doNetworkDetach(client *lxd.Client, name string, args []string) error {
	if len(args) < 1 || len(args) > 2 {
		return errArgs
	}

	containerName := args[0]
	devName := ""
	if len(args) > 1 {
		devName = args[1]
	}

	container, err := client.ContainerInfo(containerName)
	if err != nil {
		return err
	}

	if devName == "" {
		for n, d := range container.Devices {
			if d["type"] == "nic" && d["parent"] == name {
				if devName != "" {
					return fmt.Errorf(i18n.G("More than one device matches, specify the device name."))
				}

				devName = n
			}
		}
	}

	if devName == "" {
		return fmt.Errorf(i18n.G("No device found for this network"))
	}

	device, ok := container.Devices[devName]
	if !ok {
		return fmt.Errorf(i18n.G("The specified device doesn't exist"))
	}

	if device["type"] != "nic" || device["parent"] != name {
		return fmt.Errorf(i18n.G("The specified device doesn't match the network"))
	}

	resp, err := client.ContainerDeviceDelete(containerName, devName)
	if err != nil {
		return err
	}

	return client.WaitForSuccess(resp.Operation)
}

func (c *networkCmd) doNetworkDetachProfile(client *lxd.Client, name string, args []string) error {
	if len(args) < 1 || len(args) > 2 {
		return errArgs
	}

	profileName := args[0]
	devName := ""
	if len(args) > 1 {
		devName = args[1]
	}

	profile, err := client.ProfileConfig(profileName)
	if err != nil {
		return err
	}

	if devName == "" {
		for n, d := range profile.Devices {
			if d["type"] == "nic" && d["parent"] == name {
				if devName != "" {
					return fmt.Errorf(i18n.G("More than one device matches, specify the device name."))
				}

				devName = n
			}
		}
	}

	if devName == "" {
		return fmt.Errorf(i18n.G("No device found for this network"))
	}

	device, ok := profile.Devices[devName]
	if !ok {
		return fmt.Errorf(i18n.G("The specified device doesn't exist"))
	}

	if device["type"] != "nic" || device["parent"] != name {
		return fmt.Errorf(i18n.G("The specified device doesn't match the network"))
	}

	_, err = client.ProfileDeviceDelete(profileName, devName)
	return err
}

func (c *networkCmd) doNetworkDelete(client *lxd.Client, name string) error {
	err := client.NetworkDelete(name)
	if err == nil {
		fmt.Printf(i18n.G("Network %s deleted")+"\n", name)
	}

	return err
}

func (c *networkCmd) doNetworkEdit(client *lxd.Client, name string) error {
	// If stdin isn't a terminal, read text from it
	if !termios.IsTerminal(int(syscall.Stdin)) {
		contents, err := ioutil.ReadAll(os.Stdin)
		if err != nil {
			return err
		}

		newdata := api.NetworkPut{}
		err = yaml.Unmarshal(contents, &newdata)
		if err != nil {
			return err
		}
		return client.NetworkPut(name, newdata)
	}

	// Extract the current value
	network, err := client.NetworkGet(name)
	if err != nil {
		return err
	}

	data, err := yaml.Marshal(&network)
	if err != nil {
		return err
	}

	// Spawn the editor
	content, err := shared.TextEditor("", []byte(c.networkEditHelp()+"\n\n"+string(data)))
	if err != nil {
		return err
	}

	for {
		// Parse the text received from the editor
		newdata := api.NetworkPut{}
		err = yaml.Unmarshal(content, &newdata)
		if err == nil {
			err = client.NetworkPut(name, newdata)
		}

		// Respawn the editor
		if err != nil {
			fmt.Fprintf(os.Stderr, i18n.G("Config parsing error: %s")+"\n", err)
			fmt.Println(i18n.G("Press enter to open the editor again"))

			_, err := os.Stdin.Read(make([]byte, 1))
			if err != nil {
				return err
			}

			content, err = shared.TextEditor("", content)
			if err != nil {
				return err
			}
			continue
		}
		break
	}
	return nil
}

func (c *networkCmd) doNetworkGet(client *lxd.Client, name string, args []string) error {
	// we shifted @args so so it should read "<key>"
	if len(args) != 1 {
		return errArgs
	}

	resp, err := client.NetworkGet(name)
	if err != nil {
		return err
	}

	for k, v := range resp.Config {
		if k == args[0] {
			fmt.Printf("%s\n", v)
		}
	}
	return nil
}

func (c *networkCmd) doNetworkList(config *lxd.Config, args []string) error {
	var remote string
	if len(args) > 1 {
		var name string
		remote, name = config.ParseRemoteAndContainer(args[1])
		if name != "" {
			return fmt.Errorf(i18n.G("Cannot provide container name to list"))
		}
	} else {
		remote = config.DefaultRemote
	}

	client, err := lxd.NewClient(config, remote)
	if err != nil {
		return err
	}

	networks, err := client.ListNetworks()
	if err != nil {
		return err
	}

	data := [][]string{}
	for _, network := range networks {
		if shared.StringInSlice(network.Type, []string{"loopback", "unknown"}) {
			continue
		}

		strManaged := i18n.G("NO")
		if network.Managed {
			strManaged = i18n.G("YES")
		}

		strUsedBy := fmt.Sprintf("%d", len(network.UsedBy))
		data = append(data, []string{network.Name, network.Type, strManaged, strUsedBy})
	}

	table := tablewriter.NewWriter(os.Stdout)
	table.SetAutoWrapText(false)
	table.SetAlignment(tablewriter.ALIGN_LEFT)
	table.SetRowLine(true)
	table.SetHeader([]string{
		i18n.G("NAME"),
		i18n.G("TYPE"),
		i18n.G("MANAGED"),
		i18n.G("USED BY")})
	sort.Sort(byName(data))
	table.AppendBulk(data)
	table.Render()

	return nil
}

func (c *networkCmd) doNetworkSet(client *lxd.Client, name string, args []string) error {
	// we shifted @args so so it should read "<key> [<value>]"
	if len(args) < 1 {
		return errArgs
	}

	network, err := client.NetworkGet(name)
	if err != nil {
		return err
	}

	key := args[0]
	var value string
	if len(args) < 2 {
		value = ""
	} else {
		value = args[1]
	}

	if !termios.IsTerminal(int(syscall.Stdin)) && value == "-" {
		buf, err := ioutil.ReadAll(os.Stdin)
		if err != nil {
			return fmt.Errorf("Can't read from stdin: %s", err)
		}
		value = string(buf[:])
	}

	network.Config[key] = value

	return client.NetworkPut(name, network.Writable())
}

func (c *networkCmd) doNetworkShow(client *lxd.Client, name string) error {
	network, err := client.NetworkGet(name)
	if err != nil {
		return err
	}

	data, err := yaml.Marshal(&network)
	fmt.Printf("%s", data)

	return nil
}
<commit_msg>lxc: Better handle network modifications<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"sort"
	"strings"
	"syscall"

	"github.com/olekukonko/tablewriter"
	"gopkg.in/yaml.v2"

	"github.com/lxc/lxd"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/i18n"
	"github.com/lxc/lxd/shared/termios"
)

type networkCmd struct {
}

func (c *networkCmd) showByDefault() bool {
	return true
}

func (c *networkCmd) networkEditHelp() string {
	return i18n.G(
		`### This is a yaml representation of the network.
### Any line starting with a '# will be ignored.
###
### A network consists of a set of configuration items.
###
### An example would look like:
### name: lxdbr0
### config:
###   ipv4.address: 10.62.42.1/24
###   ipv4.nat: true
###   ipv6.address: fd00:56ad:9f7a:9800::1/64
###   ipv6.nat: true
### managed: true
### type: bridge
###
### Note that only the configuration can be changed.`)
}

func (c *networkCmd) usage() string {
	return i18n.G(
		`Manage networks.

lxc network list [<remote>:]                              List available networks.
lxc network show [<remote>:]<network>                     Show details of a network.
lxc network create [<remote>:]<network> [key=value...]    Create a network.
lxc network get [<remote>:]<network> <key>                Get network configuration.
lxc network set [<remote>:]<network> <key> <value>        Set network configuration.
lxc network unset [<remote>:]<network> <key>              Unset network configuration.
lxc network delete [<remote>:]<network>                   Delete a network.
lxc network edit [<remote>:]<network>
    Edit network, either by launching external editor or reading STDIN.
    Example: lxc network edit <network> # launch editor
             cat network.yaml | lxc network edit <network> # read from network.yaml

lxc network attach [<remote>:]<network> <container> [device name]
lxc network attach-profile [<remote>:]<network> <profile> [device name]

lxc network detach [<remote>:]<network> <container> [device name]
lxc network detach-profile [<remote>:]<network> <container> [device name]`)
}

func (c *networkCmd) flags() {}

func (c *networkCmd) run(config *lxd.Config, args []string) error {
	if len(args) < 1 {
		return errArgs
	}

	if args[0] == "list" {
		return c.doNetworkList(config, args)
	}

	if len(args) < 2 {
		return errArgs
	}

	remote, network := config.ParseRemoteAndContainer(args[1])
	client, err := lxd.NewClient(config, remote)
	if err != nil {
		return err
	}

	switch args[0] {
	case "attach":
		return c.doNetworkAttach(client, network, args[2:])
	case "attach-profile":
		return c.doNetworkAttachProfile(client, network, args[2:])
	case "create":
		return c.doNetworkCreate(client, network, args[2:])
	case "delete":
		return c.doNetworkDelete(client, network)
	case "detach":
		return c.doNetworkDetach(client, network, args[2:])
	case "detach-profile":
		return c.doNetworkDetachProfile(client, network, args[2:])
	case "edit":
		return c.doNetworkEdit(client, network)
	case "get":
		return c.doNetworkGet(client, network, args[2:])
	case "set":
		return c.doNetworkSet(client, network, args[2:])
	case "unset":
		return c.doNetworkSet(client, network, args[2:])
	case "show":
		return c.doNetworkShow(client, network)
	default:
		return errArgs
	}
}

func (c *networkCmd) doNetworkAttach(client *lxd.Client, name string, args []string) error {
	if len(args) < 1 || len(args) > 2 {
		return errArgs
	}

	container := args[0]
	devName := name
	if len(args) > 1 {
		devName = args[1]
	}

	network, err := client.NetworkGet(name)
	if err != nil {
		return err
	}

	nicType := "macvlan"
	if network.Type == "bridge" {
		nicType = "bridged"
	}

	props := []string{fmt.Sprintf("nictype=%s", nicType), fmt.Sprintf("parent=%s", name)}
	resp, err := client.ContainerDeviceAdd(container, devName, "nic", props)
	if err != nil {
		return err
	}

	return client.WaitForSuccess(resp.Operation)
}

func (c *networkCmd) doNetworkAttachProfile(client *lxd.Client, name string, args []string) error {
	if len(args) < 1 || len(args) > 2 {
		return errArgs
	}

	profile := args[0]
	devName := name
	if len(args) > 1 {
		devName = args[1]
	}

	network, err := client.NetworkGet(name)
	if err != nil {
		return err
	}

	nicType := "macvlan"
	if network.Type == "bridge" {
		nicType = "bridged"
	}

	props := []string{fmt.Sprintf("nictype=%s", nicType), fmt.Sprintf("parent=%s", name)}
	_, err = client.ProfileDeviceAdd(profile, devName, "nic", props)
	return err
}

func (c *networkCmd) doNetworkCreate(client *lxd.Client, name string, args []string) error {
	config := map[string]string{}

	for i := 0; i < len(args); i++ {
		entry := strings.SplitN(args[i], "=", 2)
		if len(entry) < 2 {
			return errArgs
		}

		config[entry[0]] = entry[1]
	}

	err := client.NetworkCreate(name, config)
	if err == nil {
		fmt.Printf(i18n.G("Network %s created")+"\n", name)
	}

	return err
}

func (c *networkCmd) doNetworkDetach(client *lxd.Client, name string, args []string) error {
	if len(args) < 1 || len(args) > 2 {
		return errArgs
	}

	containerName := args[0]
	devName := ""
	if len(args) > 1 {
		devName = args[1]
	}

	container, err := client.ContainerInfo(containerName)
	if err != nil {
		return err
	}

	if devName == "" {
		for n, d := range container.Devices {
			if d["type"] == "nic" && d["parent"] == name {
				if devName != "" {
					return fmt.Errorf(i18n.G("More than one device matches, specify the device name."))
				}

				devName = n
			}
		}
	}

	if devName == "" {
		return fmt.Errorf(i18n.G("No device found for this network"))
	}

	device, ok := container.Devices[devName]
	if !ok {
		return fmt.Errorf(i18n.G("The specified device doesn't exist"))
	}

	if device["type"] != "nic" || device["parent"] != name {
		return fmt.Errorf(i18n.G("The specified device doesn't match the network"))
	}

	resp, err := client.ContainerDeviceDelete(containerName, devName)
	if err != nil {
		return err
	}

	return client.WaitForSuccess(resp.Operation)
}

func (c *networkCmd) doNetworkDetachProfile(client *lxd.Client, name string, args []string) error {
	if len(args) < 1 || len(args) > 2 {
		return errArgs
	}

	profileName := args[0]
	devName := ""
	if len(args) > 1 {
		devName = args[1]
	}

	profile, err := client.ProfileConfig(profileName)
	if err != nil {
		return err
	}

	if devName == "" {
		for n, d := range profile.Devices {
			if d["type"] == "nic" && d["parent"] == name {
				if devName != "" {
					return fmt.Errorf(i18n.G("More than one device matches, specify the device name."))
				}

				devName = n
			}
		}
	}

	if devName == "" {
		return fmt.Errorf(i18n.G("No device found for this network"))
	}

	device, ok := profile.Devices[devName]
	if !ok {
		return fmt.Errorf(i18n.G("The specified device doesn't exist"))
	}

	if device["type"] != "nic" || device["parent"] != name {
		return fmt.Errorf(i18n.G("The specified device doesn't match the network"))
	}

	_, err = client.ProfileDeviceDelete(profileName, devName)
	return err
}

func (c *networkCmd) doNetworkDelete(client *lxd.Client, name string) error {
	err := client.NetworkDelete(name)
	if err == nil {
		fmt.Printf(i18n.G("Network %s deleted")+"\n", name)
	}

	return err
}

func (c *networkCmd) doNetworkEdit(client *lxd.Client, name string) error {
	// If stdin isn't a terminal, read text from it
	if !termios.IsTerminal(int(syscall.Stdin)) {
		contents, err := ioutil.ReadAll(os.Stdin)
		if err != nil {
			return err
		}

		newdata := api.NetworkPut{}
		err = yaml.Unmarshal(contents, &newdata)
		if err != nil {
			return err
		}
		return client.NetworkPut(name, newdata)
	}

	// Extract the current value
	network, err := client.NetworkGet(name)
	if err != nil {
		return err
	}

	if !network.Managed {
		return fmt.Errorf(i18n.G("Only managed networks can be modified."))
	}

	data, err := yaml.Marshal(&network)
	if err != nil {
		return err
	}

	// Spawn the editor
	content, err := shared.TextEditor("", []byte(c.networkEditHelp()+"\n\n"+string(data)))
	if err != nil {
		return err
	}

	for {
		// Parse the text received from the editor
		newdata := api.NetworkPut{}
		err = yaml.Unmarshal(content, &newdata)
		if err == nil {
			err = client.NetworkPut(name, newdata)
		}

		// Respawn the editor
		if err != nil {
			fmt.Fprintf(os.Stderr, i18n.G("Config parsing error: %s")+"\n", err)
			fmt.Println(i18n.G("Press enter to open the editor again"))

			_, err := os.Stdin.Read(make([]byte, 1))
			if err != nil {
				return err
			}

			content, err = shared.TextEditor("", content)
			if err != nil {
				return err
			}
			continue
		}
		break
	}
	return nil
}

func (c *networkCmd) doNetworkGet(client *lxd.Client, name string, args []string) error {
	// we shifted @args so so it should read "<key>"
	if len(args) != 1 {
		return errArgs
	}

	resp, err := client.NetworkGet(name)
	if err != nil {
		return err
	}

	for k, v := range resp.Config {
		if k == args[0] {
			fmt.Printf("%s\n", v)
		}
	}
	return nil
}

func (c *networkCmd) doNetworkList(config *lxd.Config, args []string) error {
	var remote string
	if len(args) > 1 {
		var name string
		remote, name = config.ParseRemoteAndContainer(args[1])
		if name != "" {
			return fmt.Errorf(i18n.G("Cannot provide container name to list"))
		}
	} else {
		remote = config.DefaultRemote
	}

	client, err := lxd.NewClient(config, remote)
	if err != nil {
		return err
	}

	networks, err := client.ListNetworks()
	if err != nil {
		return err
	}

	data := [][]string{}
	for _, network := range networks {
		if shared.StringInSlice(network.Type, []string{"loopback", "unknown"}) {
			continue
		}

		strManaged := i18n.G("NO")
		if network.Managed {
			strManaged = i18n.G("YES")
		}

		strUsedBy := fmt.Sprintf("%d", len(network.UsedBy))
		data = append(data, []string{network.Name, network.Type, strManaged, strUsedBy})
	}

	table := tablewriter.NewWriter(os.Stdout)
	table.SetAutoWrapText(false)
	table.SetAlignment(tablewriter.ALIGN_LEFT)
	table.SetRowLine(true)
	table.SetHeader([]string{
		i18n.G("NAME"),
		i18n.G("TYPE"),
		i18n.G("MANAGED"),
		i18n.G("USED BY")})
	sort.Sort(byName(data))
	table.AppendBulk(data)
	table.Render()

	return nil
}

func (c *networkCmd) doNetworkSet(client *lxd.Client, name string, args []string) error {
	// we shifted @args so so it should read "<key> [<value>]"
	if len(args) < 1 {
		return errArgs
	}

	network, err := client.NetworkGet(name)
	if err != nil {
		return err
	}

	if !network.Managed {
		return fmt.Errorf(i18n.G("Only managed networks can be modified."))
	}

	key := args[0]
	var value string
	if len(args) < 2 {
		value = ""
	} else {
		value = args[1]
	}

	if !termios.IsTerminal(int(syscall.Stdin)) && value == "-" {
		buf, err := ioutil.ReadAll(os.Stdin)
		if err != nil {
			return fmt.Errorf(i18n.G("Can't read from stdin: %s"), err)
		}
		value = string(buf[:])
	}

	network.Config[key] = value

	return client.NetworkPut(name, network.Writable())
}

func (c *networkCmd) doNetworkShow(client *lxd.Client, name string) error {
	network, err := client.NetworkGet(name)
	if err != nil {
		return err
	}

	data, err := yaml.Marshal(&network)
	fmt.Printf("%s", data)

	return nil
}
<|endoftext|>


<commit_before>package drain

import (
	"bytes"
	"encoding/json"
	"fmt"
	"logyard/util/pubsub"
	"net/url"
	"strconv"
	"strings"
	"text/template"
)

type DrainConfig struct {
	Name    string
	Type    string
	Scheme  string
	Host    string // host+port part of the uri (optional in some drains)
	Path    string
	Filters []string           // Filter messages by these keys.
	Format  *template.Template // Format message json using Go's
	// template library; if
	// format==raw, send the raw
	// stream: "<key> <msg>"
	Params    map[string]string // Params specific to that drain type.
	rawFormat bool
}

// GetParam returns the corresponding param; else the default value (def)
func (c *DrainConfig) GetParam(key string, def string) string {
	if val, ok := c.Params[key]; ok {
		return val
	}
	return def
}

func (c *DrainConfig) GetParamInt(key string, def int) (int, error) {
	data := c.GetParam(key, "")
	if data == "" {
		return def, nil
	}
	var val int
	var err error
	if val, err = strconv.Atoi(data); err != nil {
		return 0, err
	}
	return val, nil
}

func (c *DrainConfig) GetParamBool(key string, def bool) (bool, error) {
	data := c.GetParam(key, "")
	if data == "" {
		return def, nil
	}
	var val bool
	var err error
	if val, err = strconv.ParseBool(data); err != nil {
		return false, err
	}
	return val, nil
}

// FormatJSON formats the given message and returns it with a newline
func (c *DrainConfig) FormatJSON(msg pubsub.Message) ([]byte, error) {
	if c.Format == nil {
		if c.rawFormat {
			// <key> <json>
			return []byte(fmt.Sprintf("%s %s\n", msg.Key, msg.Value)), nil
		} else {
			// <json>
			return []byte(msg.Value + "\n"), nil
		}
	}
	record := make(map[string]interface{})
	err := json.Unmarshal([]byte(msg.Value), &record)
	if err != nil {
		return nil, err
	}
	var buf bytes.Buffer
	err = c.Format.Execute(&buf, record)
	if err != nil {
		return nil, err
	}
	return append(buf.Bytes(), byte('\n')), nil
}

// ParseDrainUri creates a DrainConfig from the drain URI.
func ParseDrainUri(name string, uri string, namedFormats map[string]string) (*DrainConfig, error) {
	url, err := url.Parse(uri)
	if err != nil {
		return nil, err
	}
	config := DrainConfig{Name: name, Type: url.Scheme}
	if _, ok := DRAINS[config.Type]; !ok {
		return nil, fmt.Errorf("unknown drain type: %s", uri)
	}

	config.Scheme = url.Scheme
	config.Host = url.Host
	config.Path = url.Path

	// Go doesn't correctly parse file:// uris with empty <host>.
	// http://tools.ietf.org/html/rfc1738
	if url.Scheme == "file" {
		if strings.HasPrefix(url.Path, "//") {
			config.Path = url.Path[2:]
		}
	}

	params := url.Query()

	// parse filters
	if filters, ok := params["filter"]; ok {
		params.Del("filter")
		config.Filters = filters
	} else {
		// default filter: all
		config.Filters = []string{""}
	}

	if len(config.Filters) == 0 {
		panic("filters can't be empty")
	}

	// parse format
	if format, ok := params["format"]; ok {
		params.Del("format")

		config.Format, config.rawFormat, err = parseFormat(name, format[0], namedFormats)
		if err != nil {
			return nil, err
		}
	}

	// assign the rest of the params
	config.Params = make(map[string]string)
	for k, v := range params {
		// NOTE: multi value params are not supported.
		config.Params[k] = v[0]
	}

	return &config, nil
}

func parseFormat(
	name, format string, aliases map[string]string) (*template.Template, bool, error) {
	if format == "raw" {
		return nil, true, nil
	}
	if value, ok := aliases[format]; ok {
		format = value
	}
	tmpl, err := template.New(name).Parse(format)
	return tmpl, false, err
}

// ConstructDrainURI constructs the drain URI from given parameters.
func ConstructDrainURI(
	name, uri string, filters []string, params map[string]string) (string, error) {
	if uri == "" {
		return "", fmt.Errorf("URI cannot be empty")
	}

	if !strings.Contains(uri, "://") {
		return "", fmt.Errorf("Not an URI: %s", uri)
	}

	// Build the query string
	query := url.Values{}
	for _, filter := range filters {
		query.Add("filter", filter)
	}
	for key, value := range params {
		if key == "filter" {
			return "", fmt.Errorf("params cannot have a key called 'filter'")
		}
		query.Set(key, value)
	}

	uri += "?" + query.Encode()
	return uri, nil
}
<commit_msg>recognize format=json in drain URIs<commit_after>package drain

import (
	"bytes"
	"encoding/json"
	"fmt"
	"logyard/util/pubsub"
	"net/url"
	"strconv"
	"strings"
	"text/template"
)

type DrainConfig struct {
	Name    string
	Type    string
	Scheme  string
	Host    string // host+port part of the uri (optional in some drains)
	Path    string
	Filters []string           // Filter messages by these keys.
	Format  *template.Template // Format message json using Go's
	// template library; if
	// format==raw, send the raw
	// stream: "<key> <msg>"
	Params    map[string]string // Params specific to that drain type.
	rawFormat bool
}

// GetParam returns the corresponding param; else the default value (def)
func (c *DrainConfig) GetParam(key string, def string) string {
	if val, ok := c.Params[key]; ok {
		return val
	}
	return def
}

func (c *DrainConfig) GetParamInt(key string, def int) (int, error) {
	data := c.GetParam(key, "")
	if data == "" {
		return def, nil
	}
	var val int
	var err error
	if val, err = strconv.Atoi(data); err != nil {
		return 0, err
	}
	return val, nil
}

func (c *DrainConfig) GetParamBool(key string, def bool) (bool, error) {
	data := c.GetParam(key, "")
	if data == "" {
		return def, nil
	}
	var val bool
	var err error
	if val, err = strconv.ParseBool(data); err != nil {
		return false, err
	}
	return val, nil
}

// FormatJSON formats the given message and returns it with a newline
func (c *DrainConfig) FormatJSON(msg pubsub.Message) ([]byte, error) {
	if c.Format == nil {
		if c.rawFormat {
			// <key> <json>
			return []byte(fmt.Sprintf("%s %s\n", msg.Key, msg.Value)), nil
		} else {
			// <json>
			return []byte(msg.Value + "\n"), nil
		}
	}
	record := make(map[string]interface{})
	err := json.Unmarshal([]byte(msg.Value), &record)
	if err != nil {
		return nil, err
	}
	var buf bytes.Buffer
	err = c.Format.Execute(&buf, record)
	if err != nil {
		return nil, err
	}
	return append(buf.Bytes(), byte('\n')), nil
}

// ParseDrainUri creates a DrainConfig from the drain URI.
func ParseDrainUri(name string, uri string, namedFormats map[string]string) (*DrainConfig, error) {
	url, err := url.Parse(uri)
	if err != nil {
		return nil, err
	}
	config := DrainConfig{Name: name, Type: url.Scheme}
	if _, ok := DRAINS[config.Type]; !ok {
		return nil, fmt.Errorf("unknown drain type: %s", uri)
	}

	config.Scheme = url.Scheme
	config.Host = url.Host
	config.Path = url.Path

	// Go doesn't correctly parse file:// uris with empty <host>.
	// http://tools.ietf.org/html/rfc1738
	if url.Scheme == "file" {
		if strings.HasPrefix(url.Path, "//") {
			config.Path = url.Path[2:]
		}
	}

	params := url.Query()

	// parse filters
	if filters, ok := params["filter"]; ok {
		params.Del("filter")
		config.Filters = filters
	} else {
		// default filter: all
		config.Filters = []string{""}
	}

	if len(config.Filters) == 0 {
		panic("filters can't be empty")
	}

	// parse format
	if format, ok := params["format"]; ok {
		params.Del("format")

		if format[0] != "json" {
			config.Format, config.rawFormat, err = parseFormat(name, format[0], namedFormats)
			if err != nil {
				return nil, err
			}
		}
	}

	// assign the rest of the params
	config.Params = make(map[string]string)
	for k, v := range params {
		// NOTE: multi value params are not supported.
		config.Params[k] = v[0]
	}

	return &config, nil
}

func parseFormat(
	name, format string, aliases map[string]string) (*template.Template, bool, error) {
	if format == "raw" {
		return nil, true, nil
	}
	if value, ok := aliases[format]; ok {
		format = value
	}
	tmpl, err := template.New(name).Parse(format)
	return tmpl, false, err
}

// ConstructDrainURI constructs the drain URI from given parameters.
func ConstructDrainURI(
	name, uri string, filters []string, params map[string]string) (string, error) {
	if uri == "" {
		return "", fmt.Errorf("URI cannot be empty")
	}

	if !strings.Contains(uri, "://") {
		return "", fmt.Errorf("Not an URI: %s", uri)
	}

	// Build the query string
	query := url.Values{}
	for _, filter := range filters {
		query.Add("filter", filter)
	}
	for key, value := range params {
		if key == "filter" {
			return "", fmt.Errorf("params cannot have a key called 'filter'")
		}
		query.Set(key, value)
	}

	uri += "?" + query.Encode()
	return uri, nil
}
<|endoftext|>


<commit_before>// Copyright 2017 Jeff Foley. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package amass

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/PuerkitoBio/gocrawl"
	"github.com/PuerkitoBio/goquery"
)

type ArchiveService struct {
	BaseAmassService

	responses chan *AmassRequest
	archives  []Archiver
	filter    map[string]struct{}
}

func NewArchiveService(in, out chan *AmassRequest, config *AmassConfig) *ArchiveService {
	as := &ArchiveService{
		responses: make(chan *AmassRequest, 50),
		filter:    make(map[string]struct{}),
	}
	// Modify the crawler's http client to use our DialContext
	gocrawl.HttpClient.Transport = &http.Transport{
		DialContext:           config.DialContext,
		MaxIdleConns:          200,
		IdleConnTimeout:       90 * time.Second,
		TLSHandshakeTimeout:   10 * time.Second,
		ExpectContinueTimeout: 5 * time.Second,
	}
	// Setup the service
	as.BaseAmassService = *NewBaseAmassService("Web Archive Service", config, as)
	as.archives = []Archiver{
		WaybackMachineArchive(as.responses),
		LibraryCongressArchive(as.responses),
		ArchiveIsArchive(as.responses),
		ArchiveItArchive(as.responses),
		ArquivoArchive(as.responses),
		UKWebArchive(as.responses),
		UKGovArchive(as.responses),
	}

	as.input = in
	as.output = out
	return as
}

func (as *ArchiveService) OnStart() error {
	as.BaseAmassService.OnStart()

	go as.processRequests()
	go as.processOutput()
	return nil
}

func (as *ArchiveService) OnStop() error {
	as.BaseAmassService.OnStop()
	return nil
}

func (as *ArchiveService) processRequests() {
loop:
	for {
		select {
		case req := <-as.Input():
			go as.executeAllArchives(req)
		case <-as.Quit():
			break loop
		}
	}
}

func (as *ArchiveService) processOutput() {
	t := time.NewTicker(10 * time.Second)
	defer t.Stop()
loop:
	for {
		select {
		case out := <-as.responses:
			as.SetActive(true)
			if !as.duplicate(out.Name) {
				as.SendOut(out)
			}
		case <-t.C:
			as.SetActive(false)
		case <-as.Quit():
			break loop
		}
	}
}

// Returns true if the subdomain name is a duplicate entry in the filter.
// If not, the subdomain name is added to the filter
func (as *ArchiveService) duplicate(sub string) bool {
	if _, found := as.filter[sub]; found {
		return true
	}
	as.filter[sub] = struct{}{}
	return false
}

func (as *ArchiveService) executeAllArchives(req *AmassRequest) {
	as.SetActive(true)

	for _, archive := range as.archives {
		go archive.Search(req)
	}
}

// Archiver - represents all objects that perform Memento web archive searches for domain names
type Archiver interface {
	Search(req *AmassRequest)
}

type memento struct {
	Name     string
	URL      string
	Output   chan<- *AmassRequest
	Requests chan *AmassRequest
}

func (m *memento) Search(req *AmassRequest) {
	m.Requests <- req
}

func MementoWebArchive(u, name string, out chan<- *AmassRequest) Archiver {
	m := &memento{
		Name:     name,
		URL:      u,
		Output:   out,
		Requests: make(chan *AmassRequest, 100),
	}
	go m.processRequests()
	return m
}

func ArchiveItArchive(out chan<- *AmassRequest) Archiver {
	return MementoWebArchive("https://wayback.archive-it.org/all", "Archive-It", out)
}

func ArchiveIsArchive(out chan<- *AmassRequest) Archiver {
	return MementoWebArchive("http://archive.is", "Archive Today", out)
}

func ArquivoArchive(out chan<- *AmassRequest) Archiver {
	return MementoWebArchive("http://arquivo.pt/wayback", "Arquivo Arc", out)
}

func LibraryCongressArchive(out chan<- *AmassRequest) Archiver {
	return MementoWebArchive("http://webarchive.loc.gov/all", "LoC Archive", out)
}

func UKWebArchive(out chan<- *AmassRequest) Archiver {
	return MementoWebArchive("http://www.webarchive.org.uk/wayback/archive", "Open UK Arc", out)
}

func UKGovArchive(out chan<- *AmassRequest) Archiver {
	return MementoWebArchive("http://webarchive.nationalarchives.gov.uk", "UK Gov Arch", out)
}

func WaybackMachineArchive(out chan<- *AmassRequest) Archiver {
	return MementoWebArchive("http://web.archive.org/web", "Wayback Arc", out)
}

/* Private functions */

func (m *memento) processRequests() {
	var running int
	var queue []*AmassRequest
	done := make(chan int, 10)

	t := time.NewTicker(1 * time.Second)
	defer t.Stop()

	year := time.Now().Year()
	// Only have up to 10 crawlers running at the same time
	for {
		select {
		case sd := <-m.Requests:
			queue = append(queue, sd)
		case <-t.C:
			if running >= 10 || len(queue) <= 0 {
				break
			}

			s := queue[0]
			if len(queue) == 1 {
				queue = []*AmassRequest{}
			} else {
				queue = queue[1:]
			}

			go m.crawl(strconv.Itoa(year), s, done, 10*time.Second)
			running++
		case <-done:
			running--
		}
	}
}

func (m *memento) crawl(year string, req *AmassRequest, done chan int, timeout time.Duration) {
	domain := req.Domain
	if domain == "" {
		done <- 1
		return
	}

	ext := &ext{
		DefaultExtender: &gocrawl.DefaultExtender{},
		domainRE:        SubdomainRegex(domain),
		mementoRE:       regexp.MustCompile(m.URL + "/[0-9]+/"),
		filter:          make(map[string]bool), // Filter for not double-checking URLs
		base:            m.URL,
		year:            year,
		sub:             req.Name,
		domain:          domain,
		names:           m.Output,
		source:          m.Name,
	}

	// Set custom options
	opts := gocrawl.NewOptions(ext)
	opts.CrawlDelay = 500 * time.Millisecond
	opts.LogFlags = gocrawl.LogError
	opts.SameHostOnly = true
	opts.MaxVisits = 20

	c := gocrawl.NewCrawlerWithOptions(opts)
	go c.Run(fmt.Sprintf("%s/%s/%s", m.URL, year, req.Name))

	<-time.After(timeout)
	c.Stop()
	done <- 1
}

type ext struct {
	*gocrawl.DefaultExtender
	domainRE                *regexp.Regexp
	mementoRE               *regexp.Regexp
	filter                  map[string]bool
	flock                   sync.RWMutex
	base, year, sub, domain string
	names                   chan<- *AmassRequest
	source                  string
}

func (e *ext) reducedURL(u *url.URL) string {
	orig := u.String()

	idx := e.mementoRE.FindStringIndex(orig)
	if idx == nil {
		return ""
	}

	i := idx[1]
	return fmt.Sprintf("%s/%s/%s", e.base, e.year, orig[i:])
}

func (e *ext) Log(logFlags gocrawl.LogFlags, msgLevel gocrawl.LogFlags, msg string) {
	return
}

func (e *ext) RequestRobots(ctx *gocrawl.URLContext, robotAgent string) (data []byte, doRequest bool) {
	return nil, false
}

func (e *ext) Filter(ctx *gocrawl.URLContext, isVisited bool) bool {
	if isVisited {
		return false
	}

	u := ctx.URL().String()
	r := e.reducedURL(ctx.URL())

	if !strings.Contains(ctx.URL().Path, e.sub) {
		return false
	}

	e.flock.RLock()
	_, ok := e.filter[r]
	e.flock.RUnlock()

	if ok {
		return false
	}

	if u != r {
		// The more refined version has been requested
		// and will cause the reduced version to be filtered
		e.flock.Lock()
		e.filter[r] = true
		e.flock.Unlock()
	}
	return true
}

func (e *ext) Visit(ctx *gocrawl.URLContext, res *http.Response, doc *goquery.Document) (interface{}, bool) {
	in, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, true
	}

	for _, f := range e.domainRE.FindAllString(string(in), -1) {
		e.names <- &AmassRequest{
			Name:   f,
			Domain: e.domain,
			Tag:    ARCHIVE,
			Source: e.source,
		}
	}
	return nil, true
}
<commit_msg>small improvement to web crawling for archives due to issue #34<commit_after>// Copyright 2017 Jeff Foley. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package amass

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/PuerkitoBio/gocrawl"
	"github.com/PuerkitoBio/goquery"
)

type ArchiveService struct {
	BaseAmassService

	responses chan *AmassRequest
	archives  []Archiver
	filter    map[string]struct{}
}

func NewArchiveService(in, out chan *AmassRequest, config *AmassConfig) *ArchiveService {
	as := &ArchiveService{
		responses: make(chan *AmassRequest, 50),
		filter:    make(map[string]struct{}),
	}
	// Modify the crawler's http client to use our DialContext
	gocrawl.HttpClient.Transport = &http.Transport{
		DialContext:           config.DialContext,
		MaxIdleConns:          200,
		IdleConnTimeout:       90 * time.Second,
		TLSHandshakeTimeout:   10 * time.Second,
		ExpectContinueTimeout: 5 * time.Second,
	}
	// Setup the service
	as.BaseAmassService = *NewBaseAmassService("Web Archive Service", config, as)
	as.archives = []Archiver{
		WaybackMachineArchive(as.responses),
		LibraryCongressArchive(as.responses),
		ArchiveIsArchive(as.responses),
		ArchiveItArchive(as.responses),
		ArquivoArchive(as.responses),
		UKWebArchive(as.responses),
		UKGovArchive(as.responses),
	}

	as.input = in
	as.output = out
	return as
}

func (as *ArchiveService) OnStart() error {
	as.BaseAmassService.OnStart()

	go as.processRequests()
	go as.processOutput()
	return nil
}

func (as *ArchiveService) OnStop() error {
	as.BaseAmassService.OnStop()
	return nil
}

func (as *ArchiveService) processRequests() {
loop:
	for {
		select {
		case req := <-as.Input():
			go as.executeAllArchives(req)
		case <-as.Quit():
			break loop
		}
	}
}

func (as *ArchiveService) processOutput() {
	t := time.NewTicker(10 * time.Second)
	defer t.Stop()
loop:
	for {
		select {
		case out := <-as.responses:
			as.SetActive(true)
			if !as.duplicate(out.Name) {
				as.SendOut(out)
			}
		case <-t.C:
			as.SetActive(false)
		case <-as.Quit():
			break loop
		}
	}
}

// Returns true if the subdomain name is a duplicate entry in the filter.
// If not, the subdomain name is added to the filter
func (as *ArchiveService) duplicate(sub string) bool {
	if _, found := as.filter[sub]; found {
		return true
	}
	as.filter[sub] = struct{}{}
	return false
}

func (as *ArchiveService) executeAllArchives(req *AmassRequest) {
	as.SetActive(true)

	for _, archive := range as.archives {
		go archive.Search(req)
	}
}

// Archiver - represents all objects that perform Memento web archive searches for domain names
type Archiver interface {
	Search(req *AmassRequest)
}

type memento struct {
	Name     string
	URL      string
	Output   chan<- *AmassRequest
	Requests chan *AmassRequest
}

func (m *memento) Search(req *AmassRequest) {
	m.Requests <- req
}

func MementoWebArchive(u, name string, out chan<- *AmassRequest) Archiver {
	m := &memento{
		Name:     name,
		URL:      u,
		Output:   out,
		Requests: make(chan *AmassRequest, 100),
	}
	go m.processRequests()
	return m
}

func ArchiveItArchive(out chan<- *AmassRequest) Archiver {
	return MementoWebArchive("https://wayback.archive-it.org/all", "Archive-It", out)
}

func ArchiveIsArchive(out chan<- *AmassRequest) Archiver {
	return MementoWebArchive("http://archive.is", "Archive Today", out)
}

func ArquivoArchive(out chan<- *AmassRequest) Archiver {
	return MementoWebArchive("http://arquivo.pt/wayback", "Arquivo Arc", out)
}

func LibraryCongressArchive(out chan<- *AmassRequest) Archiver {
	return MementoWebArchive("http://webarchive.loc.gov/all", "LoC Archive", out)
}

func UKWebArchive(out chan<- *AmassRequest) Archiver {
	return MementoWebArchive("http://www.webarchive.org.uk/wayback/archive", "Open UK Arc", out)
}

func UKGovArchive(out chan<- *AmassRequest) Archiver {
	return MementoWebArchive("http://webarchive.nationalarchives.gov.uk", "UK Gov Arch", out)
}

func WaybackMachineArchive(out chan<- *AmassRequest) Archiver {
	return MementoWebArchive("http://web.archive.org/web", "Wayback Arc", out)
}

/* Private functions */

func (m *memento) processRequests() {
	var running int
	var queue []*AmassRequest
	done := make(chan struct{}, 10)

	t := time.NewTicker(1 * time.Second)
	defer t.Stop()

	year := time.Now().Year()
	// Only have up to 10 crawlers running at the same time
	for {
		select {
		case sd := <-m.Requests:
			queue = append(queue, sd)
		case <-t.C:
			if running >= 10 || len(queue) <= 0 {
				break
			}

			s := queue[0]
			if len(queue) == 1 {
				queue = []*AmassRequest{}
			} else {
				queue = queue[1:]
			}

			go m.crawl(strconv.Itoa(year), s, done)
			running++
		case <-done:
			running--
		}
	}
}

func (m *memento) crawl(year string, req *AmassRequest, done chan struct{}) {
	domain := req.Domain
	if domain == "" {
		done <- struct{}{}
		return
	}

	ext := &ext{
		DefaultExtender: &gocrawl.DefaultExtender{},
		domainRE:        SubdomainRegex(domain),
		mementoRE:       regexp.MustCompile(m.URL + "/[0-9]+/"),
		filter:          make(map[string]bool), // Filter for not double-checking URLs
		base:            m.URL,
		year:            year,
		sub:             req.Name,
		domain:          domain,
		names:           m.Output,
		source:          m.Name,
	}

	// Set custom options
	opts := gocrawl.NewOptions(ext)
	opts.CrawlDelay = 500 * time.Millisecond
	opts.LogFlags = gocrawl.LogError
	opts.SameHostOnly = true
	opts.MaxVisits = 20

	c := gocrawl.NewCrawlerWithOptions(opts)
	c.Run(fmt.Sprintf("%s/%s/%s", m.URL, year, req.Name))
	done <- struct{}{}
}

type ext struct {
	*gocrawl.DefaultExtender
	domainRE                *regexp.Regexp
	mementoRE               *regexp.Regexp
	filter                  map[string]bool
	flock                   sync.RWMutex
	base, year, sub, domain string
	names                   chan<- *AmassRequest
	source                  string
}

func (e *ext) reducedURL(u *url.URL) string {
	orig := u.String()

	idx := e.mementoRE.FindStringIndex(orig)
	if idx == nil {
		return ""
	}

	i := idx[1]
	return fmt.Sprintf("%s/%s/%s", e.base, e.year, orig[i:])
}

func (e *ext) Log(logFlags gocrawl.LogFlags, msgLevel gocrawl.LogFlags, msg string) {
	return
}

func (e *ext) RequestRobots(ctx *gocrawl.URLContext, robotAgent string) (data []byte, doRequest bool) {
	return nil, false
}

func (e *ext) Filter(ctx *gocrawl.URLContext, isVisited bool) bool {
	if isVisited {
		return false
	}

	u := ctx.URL().String()
	r := e.reducedURL(ctx.URL())

	if !strings.Contains(ctx.URL().Path, e.sub) {
		return false
	}

	e.flock.RLock()
	_, ok := e.filter[r]
	e.flock.RUnlock()

	if ok {
		return false
	}

	if u != r {
		// The more refined version has been requested
		// and will cause the reduced version to be filtered
		e.flock.Lock()
		e.filter[r] = true
		e.flock.Unlock()
	}
	return true
}

func (e *ext) Visit(ctx *gocrawl.URLContext, res *http.Response, doc *goquery.Document) (interface{}, bool) {
	in, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, true
	}

	for _, f := range e.domainRE.FindAllString(string(in), -1) {
		e.names <- &AmassRequest{
			Name:   f,
			Domain: e.domain,
			Tag:    ARCHIVE,
			Source: e.source,
		}
	}
	return nil, true
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package instrument

import (
	"sync/atomic"
	"time"

	"github.com/uber-go/tally"
)

var (
	nullStopWatchStart tally.Stopwatch
)

// sampledTimer is a sampled timer that implements the tally timer interface.
// NB(xichen): the sampling logic should eventually be implemented in tally.
type sampledTimer struct {
	tally.Timer

	cnt  uint64
	rate uint64
}

// NB(xichen): return an error instead of panicing.
func newSampledTimer(base tally.Timer, rate float64) tally.Timer {
	if rate <= 0.0 || rate > 1.0 {
		panic("sampled timer must have a sampling rate between 0.0 and 1.0")
	}
	return &sampledTimer{
		Timer: base,
		rate:  uint64(1.0 / rate),
	}
}

func (t *sampledTimer) shouldSample() bool {
	return atomic.AddUint64(&t.cnt, 1)%t.rate == 0
}

func (t *sampledTimer) Start() tally.Stopwatch {
	if !t.shouldSample() {
		return nullStopWatchStart
	}
	return t.Timer.Start()
}

func (t *sampledTimer) Stop(startTime tally.Stopwatch) {
	if startTime == nullStopWatchStart { // nolint: badtime
		// If startTime is nullStopWatchStart, do nothing.
		return
	}
	startTime.Stop()
}

func (t *sampledTimer) Record(d time.Duration) {
	if !t.shouldSample() {
		return
	}
	t.Timer.Record(d)
}

// MethodMetrics is a bundle of common metrics with a uniform naming scheme.
type MethodMetrics struct {
	Errors         tally.Counter
	Success        tally.Counter
	ErrorsLatency  tally.Timer
	SuccessLatency tally.Timer
}

// ReportSuccess reports a success.
func (m *MethodMetrics) ReportSuccess(d time.Duration) {
	m.Success.Inc(1)
	m.SuccessLatency.Record(d)
}

// ReportError reports an error.
func (m *MethodMetrics) ReportError(d time.Duration) {
	m.Errors.Inc(1)
	m.ErrorsLatency.Record(d)
}

// ReportSuccessOrError increments Error/Success counter dependending on the error.
func (m *MethodMetrics) ReportSuccessOrError(e error, d time.Duration) {
	if e != nil {
		m.ReportError(d)
	} else {
		m.ReportSuccess(d)
	}
}

// NewMethodMetrics returns a new Method metrics for the given method name.
func NewMethodMetrics(scope tally.Scope, methodName string, samplingRate float64) MethodMetrics {
	return MethodMetrics{
		Errors:         scope.Counter(methodName + ".errors"),
		Success:        scope.Counter(methodName + ".success"),
		ErrorsLatency:  newSampledTimer(scope.Timer(methodName+".errors-latency"), samplingRate),
		SuccessLatency: newSampledTimer(scope.Timer(methodName+".success-latency"), samplingRate),
	}
}

// BatchMethodMetrics is a bundle of common metrics for methods with batch semantics.
type BatchMethodMetrics struct {
	RetryableErrors    tally.Counter
	NonRetryableErrors tally.Counter
	Errors             tally.Counter
	Success            tally.Counter
	Latency            tally.Timer
}

// NewBatchMethodMetrics creates new batch method metrics.
func NewBatchMethodMetrics(
	scope tally.Scope,
	methodName string,
	samplingRate float64,
) BatchMethodMetrics {
	return BatchMethodMetrics{
		RetryableErrors:    scope.Counter(methodName + ".retryable-errors"),
		NonRetryableErrors: scope.Counter(methodName + ".non-retryable-errors"),
		Errors:             scope.Counter(methodName + ".errors"),
		Success:            scope.Counter(methodName + ".success"),
		Latency:            newSampledTimer(scope.Timer(methodName+".latency"), samplingRate),
	}
}

// ReportSuccess reports successess.
func (m *BatchMethodMetrics) ReportSuccess(n int) {
	m.Success.Inc(int64(n))
}

// ReportRetryableErrors reports retryable errors.
func (m *BatchMethodMetrics) ReportRetryableErrors(n int) {
	m.RetryableErrors.Inc(int64(n))
	m.Errors.Inc(int64(n))
}

// ReportNonRetryableErrors reports non-retryable errors.
func (m *BatchMethodMetrics) ReportNonRetryableErrors(n int) {
	m.NonRetryableErrors.Inc(int64(n))
	m.Errors.Inc(int64(n))
}

// ReportLatency reports latency.
func (m *BatchMethodMetrics) ReportLatency(d time.Duration) {
	m.Latency.Record(d)
}
<commit_msg>Export constructors for creating sampled timers (#117)<commit_after>// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package instrument

import (
	"fmt"
	"sync/atomic"
	"time"

	"github.com/uber-go/tally"
)

var (
	nullStopWatchStart tally.Stopwatch
)

// sampledTimer is a sampled timer that implements the tally timer interface.
// NB(xichen): the sampling logic should eventually be implemented in tally.
type sampledTimer struct {
	tally.Timer

	cnt  uint64
	rate uint64
}

// NewSampledTimer creates a new sampled timer.
func NewSampledTimer(base tally.Timer, rate float64) (tally.Timer, error) {
	if rate <= 0.0 || rate > 1.0 {
		return nil, fmt.Errorf("sampling rate %f must be between 0.0 and 1.0", rate)
	}
	return &sampledTimer{
		Timer: base,
		rate:  uint64(1.0 / rate),
	}, nil
}

// MustCreateSampledTimer creates a new sampled timer, and panics if an error
// is encountered.
func MustCreateSampledTimer(base tally.Timer, rate float64) tally.Timer {
	t, err := NewSampledTimer(base, rate)
	if err != nil {
		panic(err)
	}
	return t
}

func (t *sampledTimer) shouldSample() bool {
	return atomic.AddUint64(&t.cnt, 1)%t.rate == 0
}

func (t *sampledTimer) Start() tally.Stopwatch {
	if !t.shouldSample() {
		return nullStopWatchStart
	}
	return t.Timer.Start()
}

func (t *sampledTimer) Stop(startTime tally.Stopwatch) {
	if startTime == nullStopWatchStart { // nolint: badtime
		// If startTime is nullStopWatchStart, do nothing.
		return
	}
	startTime.Stop()
}

func (t *sampledTimer) Record(d time.Duration) {
	if !t.shouldSample() {
		return
	}
	t.Timer.Record(d)
}

// MethodMetrics is a bundle of common metrics with a uniform naming scheme.
type MethodMetrics struct {
	Errors         tally.Counter
	Success        tally.Counter
	ErrorsLatency  tally.Timer
	SuccessLatency tally.Timer
}

// ReportSuccess reports a success.
func (m *MethodMetrics) ReportSuccess(d time.Duration) {
	m.Success.Inc(1)
	m.SuccessLatency.Record(d)
}

// ReportError reports an error.
func (m *MethodMetrics) ReportError(d time.Duration) {
	m.Errors.Inc(1)
	m.ErrorsLatency.Record(d)
}

// ReportSuccessOrError increments Error/Success counter dependending on the error.
func (m *MethodMetrics) ReportSuccessOrError(e error, d time.Duration) {
	if e != nil {
		m.ReportError(d)
	} else {
		m.ReportSuccess(d)
	}
}

// NewMethodMetrics returns a new Method metrics for the given method name.
func NewMethodMetrics(scope tally.Scope, methodName string, samplingRate float64) MethodMetrics {
	return MethodMetrics{
		Errors:         scope.Counter(methodName + ".errors"),
		Success:        scope.Counter(methodName + ".success"),
		ErrorsLatency:  MustCreateSampledTimer(scope.Timer(methodName+".errors-latency"), samplingRate),
		SuccessLatency: MustCreateSampledTimer(scope.Timer(methodName+".success-latency"), samplingRate),
	}
}

// BatchMethodMetrics is a bundle of common metrics for methods with batch semantics.
type BatchMethodMetrics struct {
	RetryableErrors    tally.Counter
	NonRetryableErrors tally.Counter
	Errors             tally.Counter
	Success            tally.Counter
	Latency            tally.Timer
}

// NewBatchMethodMetrics creates new batch method metrics.
func NewBatchMethodMetrics(
	scope tally.Scope,
	methodName string,
	samplingRate float64,
) BatchMethodMetrics {
	return BatchMethodMetrics{
		RetryableErrors:    scope.Counter(methodName + ".retryable-errors"),
		NonRetryableErrors: scope.Counter(methodName + ".non-retryable-errors"),
		Errors:             scope.Counter(methodName + ".errors"),
		Success:            scope.Counter(methodName + ".success"),
		Latency:            MustCreateSampledTimer(scope.Timer(methodName+".latency"), samplingRate),
	}
}

// ReportSuccess reports successess.
func (m *BatchMethodMetrics) ReportSuccess(n int) {
	m.Success.Inc(int64(n))
}

// ReportRetryableErrors reports retryable errors.
func (m *BatchMethodMetrics) ReportRetryableErrors(n int) {
	m.RetryableErrors.Inc(int64(n))
	m.Errors.Inc(int64(n))
}

// ReportNonRetryableErrors reports non-retryable errors.
func (m *BatchMethodMetrics) ReportNonRetryableErrors(n int) {
	m.NonRetryableErrors.Inc(int64(n))
	m.Errors.Inc(int64(n))
}

// ReportLatency reports latency.
func (m *BatchMethodMetrics) ReportLatency(d time.Duration) {
	m.Latency.Record(d)
}
<|endoftext|>


<commit_before>// Copyright ©2011-2012 The bíogo Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package align

import (
	"code.google.com/p/biogo/alphabet"
	"code.google.com/p/biogo/io/seqio/fasta"
	"code.google.com/p/biogo/seq/linear"
	check "launchpad.net/gocheck"
	"strings"
	"testing"
)

// Tests
func Test(t *testing.T) { check.TestingT(t) }

type S struct{}

var _ = check.Suite(&S{})

func (s *S) TestWarning(c *check.C) { c.Log("\nFIXME: Tests only in example tests.\n") }

func BenchmarkSWAlign(b *testing.B) {
	b.StopTimer()
	t := &linear.Seq{}
	t.Alpha = alphabet.DNA
	r := fasta.NewReader(strings.NewReader(crspFa), t)
	swsa, _ := r.Read()
	swsb, _ := r.Read()

	smith := SW{
		{2, -1, -1, -1, -1},
		{-1, 2, -1, -1, -1},
		{-1, -1, 2, -1, -1},
		{-1, -1, -1, 2, -1},
		{-1, -1, -1, -1, 0},
	}
	b.StartTimer()
	for i := 0; i < b.N; i++ {
		smith.Align(swsa, swsb)
	}
}

func BenchmarkNWAlign(b *testing.B) {
	b.StopTimer()
	t := &linear.Seq{}
	t.Alpha = alphabet.DNA
	r := fasta.NewReader(strings.NewReader(crspFa), t)
	nwsa, _ := r.Read()
	nwsb, _ := r.Read()

	needle := NW{
		{10, -3, -1, -4, -5},
		{-3, 9, -5, 0, -5},
		{-1, -5, 7, -3, -5},
		{-4, 0, -3, 8, -5},
		{-4, -4, -4, -4, 0},
	}

	b.StartTimer()
	for i := 0; i < b.N; i++ {
		needle.Align(nwsa, nwsb)
	}
}

func BenchmarkSWAffineAlign(b *testing.B) {
	b.StopTimer()
	t := &linear.Seq{}
	t.Alpha = alphabet.DNA
	r := fasta.NewReader(strings.NewReader(crspFa), t)
	swsa, _ := r.Read()
	swsb, _ := r.Read()

	smith := SWAffine{
		Matrix: Linear{
			{2, -1, -1, -1, -1},
			{-1, 2, -1, -1, -1},
			{-1, -1, 2, -1, -1},
			{-1, -1, -1, 2, -1},
			{-1, -1, -1, -1, 0},
		},
		GapOpen: -5,
	}
	b.StartTimer()
	for i := 0; i < b.N; i++ {
		smith.Align(swsa, swsb)
	}
}

func BenchmarkNWAffineAlign(b *testing.B) {
	b.StopTimer()
	t := &linear.Seq{}
	t.Alpha = alphabet.DNA
	r := fasta.NewReader(strings.NewReader(crspFa), t)
	nwsa, _ := r.Read()
	nwsb, _ := r.Read()

	needle := NWAffine{
		Matrix: Linear{
			{10, -3, -1, -4, -5},
			{-3, 9, -5, 0, -5},
			{-1, -5, 7, -3, -5},
			{-4, 0, -3, 8, -5},
			{-4, -4, -4, -4, 0},
		},
		GapOpen: -10,
	}

	b.StartTimer()
	for i := 0; i < b.N; i++ {
		needle.Align(nwsa, nwsb)
	}
}
<commit_msg>Update benchmarks to use gapped alphabet<commit_after>// Copyright ©2011-2012 The bíogo Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package align

import (
	"code.google.com/p/biogo/alphabet"
	"code.google.com/p/biogo/io/seqio/fasta"
	"code.google.com/p/biogo/seq/linear"
	check "launchpad.net/gocheck"
	"strings"
	"testing"
)

// Tests
func Test(t *testing.T) { check.TestingT(t) }

type S struct{}

var _ = check.Suite(&S{})

func (s *S) TestWarning(c *check.C) { c.Log("\nFIXME: Tests only in example tests.\n") }

func BenchmarkSWAlign(b *testing.B) {
	b.StopTimer()
	t := &linear.Seq{}
	t.Alpha = alphabet.DNAgapped
	r := fasta.NewReader(strings.NewReader(crspFa), t)
	swsa, _ := r.Read()
	swsb, _ := r.Read()

	smith := SW{
		{2, -1, -1, -1, -1},
		{-1, 2, -1, -1, -1},
		{-1, -1, 2, -1, -1},
		{-1, -1, -1, 2, -1},
		{-1, -1, -1, -1, 0},
	}
	b.StartTimer()
	for i := 0; i < b.N; i++ {
		smith.Align(swsa, swsb)
	}
}

func BenchmarkNWAlign(b *testing.B) {
	b.StopTimer()
	t := &linear.Seq{}
	t.Alpha = alphabet.DNAgapped
	r := fasta.NewReader(strings.NewReader(crspFa), t)
	nwsa, _ := r.Read()
	nwsb, _ := r.Read()

	needle := NW{
		{10, -3, -1, -4, -5},
		{-3, 9, -5, 0, -5},
		{-1, -5, 7, -3, -5},
		{-4, 0, -3, 8, -5},
		{-4, -4, -4, -4, 0},
	}

	b.StartTimer()
	for i := 0; i < b.N; i++ {
		needle.Align(nwsa, nwsb)
	}
}

func BenchmarkSWAffineAlign(b *testing.B) {
	b.StopTimer()
	t := &linear.Seq{}
	t.Alpha = alphabet.DNAgapped
	r := fasta.NewReader(strings.NewReader(crspFa), t)
	swsa, _ := r.Read()
	swsb, _ := r.Read()

	smith := SWAffine{
		Matrix: Linear{
			{2, -1, -1, -1, -1},
			{-1, 2, -1, -1, -1},
			{-1, -1, 2, -1, -1},
			{-1, -1, -1, 2, -1},
			{-1, -1, -1, -1, 0},
		},
		GapOpen: -5,
	}
	b.StartTimer()
	for i := 0; i < b.N; i++ {
		smith.Align(swsa, swsb)
	}
}

func BenchmarkNWAffineAlign(b *testing.B) {
	b.StopTimer()
	t := &linear.Seq{}
	t.Alpha = alphabet.DNAgapped
	r := fasta.NewReader(strings.NewReader(crspFa), t)
	nwsa, _ := r.Read()
	nwsb, _ := r.Read()

	needle := NWAffine{
		Matrix: Linear{
			{10, -3, -1, -4, -5},
			{-3, 9, -5, 0, -5},
			{-1, -5, 7, -3, -5},
			{-4, 0, -3, 8, -5},
			{-4, -4, -4, -4, 0},
		},
		GapOpen: -10,
	}

	b.StartTimer()
	for i := 0; i < b.N; i++ {
		needle.Align(nwsa, nwsb)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2021 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"context"
	"fmt"
	"io"
	"log"
	"time"
	vtgatepb "vitess.io/vitess/go/vt/proto/vtgate"

	binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata"
	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
	_ "vitess.io/vitess/go/vt/vtctl/grpcvtctlclient"
	_ "vitess.io/vitess/go/vt/vtgate/grpcvtgateconn"
	"vitess.io/vitess/go/vt/vtgate/vtgateconn"
)

/*
	This is a sample client for streaming using the vstream API. It is setup to work with the local example and you can
    either stream from the unsharded commerce keyspace or the customer keyspace after the sharding step.
*/
func main() {
	ctx := context.Background()
	streamCustomer := true
	var vgtid *binlogdatapb.VGtid
	if streamCustomer {
		vgtid = &binlogdatapb.VGtid{
			ShardGtids: []*binlogdatapb.ShardGtid{{
				Keyspace: "customer",
				Shard:    "-80",
				// Gtid "" is to stream from the start, "current" is to stream from the current gtid
				// you can also specify a gtid to start with.
				Gtid:     "", //"current"  // "MySQL56/36a89abd-978f-11eb-b312-04ed332e05c2:1-265"
			}, {
				Keyspace: "customer",
				Shard:    "80-",
				Gtid:     "",
			}}}
	} else {
		vgtid = &binlogdatapb.VGtid{
			ShardGtids: []*binlogdatapb.ShardGtid{{
				Keyspace: "commerce",
				Shard:    "0",
				Gtid:     "",
			}}}
	}
	filter := &binlogdatapb.Filter{
		Rules: []*binlogdatapb.Rule{{
			Match:  "customer",
			Filter: "select * from customer",
		}},
	}
	conn, err := vtgateconn.Dial(ctx, "localhost:15991")
	if err != nil {
		log.Fatal(err)
	}
	defer conn.Close()
	flags := &vtgatepb.VStreamFlags{
		//MinimizeSkew:      false,
		//HeartbeatInterval: 60, //seconds
	}
	reader, err := conn.VStream(ctx, topodatapb.TabletType_MASTER, vgtid, filter, flags)
	for {
		e, err := reader.Recv()
		switch err {
		case nil:
			_ = e
			fmt.Printf("%v\n", e)
		case io.EOF:
			fmt.Printf("stream ended\n")
			return
		default:
			fmt.Printf("%s:: remote error: %v\n", time.Now(), err)
			return
		}
	}
}
<commit_msg>go-fmted<commit_after>/*
Copyright 2021 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"context"
	"fmt"
	"io"
	"log"
	"time"
	vtgatepb "vitess.io/vitess/go/vt/proto/vtgate"

	binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata"
	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
	_ "vitess.io/vitess/go/vt/vtctl/grpcvtctlclient"
	_ "vitess.io/vitess/go/vt/vtgate/grpcvtgateconn"
	"vitess.io/vitess/go/vt/vtgate/vtgateconn"
)

/*
	This is a sample client for streaming using the vstream API. It is setup to work with the local example and you can
    either stream from the unsharded commerce keyspace or the customer keyspace after the sharding step.
*/
func main() {
	ctx := context.Background()
	streamCustomer := true
	var vgtid *binlogdatapb.VGtid
	if streamCustomer {
		vgtid = &binlogdatapb.VGtid{
			ShardGtids: []*binlogdatapb.ShardGtid{{
				Keyspace: "customer",
				Shard:    "-80",
				// Gtid "" is to stream from the start, "current" is to stream from the current gtid
				// you can also specify a gtid to start with.
				Gtid: "", //"current"  // "MySQL56/36a89abd-978f-11eb-b312-04ed332e05c2:1-265"
			}, {
				Keyspace: "customer",
				Shard:    "80-",
				Gtid:     "",
			}}}
	} else {
		vgtid = &binlogdatapb.VGtid{
			ShardGtids: []*binlogdatapb.ShardGtid{{
				Keyspace: "commerce",
				Shard:    "0",
				Gtid:     "",
			}}}
	}
	filter := &binlogdatapb.Filter{
		Rules: []*binlogdatapb.Rule{{
			Match:  "customer",
			Filter: "select * from customer",
		}},
	}
	conn, err := vtgateconn.Dial(ctx, "localhost:15991")
	if err != nil {
		log.Fatal(err)
	}
	defer conn.Close()
	flags := &vtgatepb.VStreamFlags{
		//MinimizeSkew:      false,
		//HeartbeatInterval: 60, //seconds
	}
	reader, err := conn.VStream(ctx, topodatapb.TabletType_MASTER, vgtid, filter, flags)
	for {
		e, err := reader.Recv()
		switch err {
		case nil:
			_ = e
			fmt.Printf("%v\n", e)
		case io.EOF:
			fmt.Printf("stream ended\n")
			return
		default:
			fmt.Printf("%s:: remote error: %v\n", time.Now(), err)
			return
		}
	}
}
<|endoftext|>