<commit_before>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package framework

import (
	"bytes"
	"flag"
	"fmt"
	"os/exec"
	"strings"
	"testing"
	"time"

	"github.com/google/cadvisor/client"
	"github.com/google/cadvisor/client/v2"
	"github.com/google/cadvisor/integration/common"
)

var host = flag.String("host", "localhost", "Address of the host being tested")
var port = flag.Int("port", 8080, "Port of the application on the host being tested")

// Integration test framework.
type Framework interface {
	// Clean the framework state.
	Cleanup()

	// The testing.T used by the framework and the current test.
	T() *testing.T

	// Returns the hostname being tested.
	Hostname() HostnameInfo

	// Returns the Docker actions for the test framework.
	Docker() DockerActions

	// Returns the shell actions for the test framework.
	Shell() ShellActions

	// Returns the cAdvisor actions for the test framework.
	Cadvisor() CadvisorActions
}

// Instantiates a Framework. Cleanup *must* be called. Class is thread-compatible.
// All framework actions report fatal errors on the t specified at creation time.
//
// Typical use:
//
// func TestFoo(t *testing.T) {
// 	fm := framework.New(t)
// 	defer fm.Cleanup()
//      ... actual test ...
// }
func New(t *testing.T) Framework {
	// All integration tests are large.
	if testing.Short() {
		t.Skip("Skipping framework test in short mode")
	}

	// Try to see if non-localhost hosts are GCE instances.
	var gceInstanceName string
	hostname := *host
	if hostname != "localhost" {
		gceInstanceName = hostname
		gceIp, err := common.GetGceIp(hostname)
		if err == nil {
			hostname = gceIp
		}
	}

	fm := &realFramework{
		hostname: HostnameInfo{
			Host:            hostname,
			Port:            *port,
			GceInstanceName: gceInstanceName,
		},
		t:        t,
		cleanups: make([]func(), 0),
	}
	fm.shellActions = shellActions{
		fm: fm,
	}
	fm.dockerActions = dockerActions{
		fm: fm,
	}

	return fm
}

const (
	Aufs         string = "aufs"
	Overlay      string = "overlay"
	DeviceMapper string = "devicemapper"
	Unknown      string = ""
)

type DockerActions interface {
	// Run the no-op pause Docker container and return its ID.
	RunPause() string

	// Run the specified command in a Docker busybox container and return its ID.
	RunBusybox(cmd ...string) string

	// Runs a Docker container in the background. Uses the specified DockerRunArgs and command.
	// Returns the ID of the new container.
	//
	// e.g.:
	// Run(DockerRunArgs{Image: "busybox"}, "ping", "www.google.com")
	//   -> docker run busybox ping www.google.com
	Run(args DockerRunArgs, cmd ...string) string
	RunStress(args DockerRunArgs, cmd ...string) string

	Version() []string
	StorageDriver() string
}

type ShellActions interface {
	// Runs a specified command and arguments. Returns the stdout and stderr.
	Run(cmd string, args ...string) (string, string)
	RunStress(cmd string, args ...string) (string, string)
}

type CadvisorActions interface {
	// Returns a cAdvisor client to the machine being tested.
	Client() *client.Client
	ClientV2() *v2.Client
}

type realFramework struct {
	hostname         HostnameInfo
	t                *testing.T
	cadvisorClient   *client.Client
	cadvisorClientV2 *v2.Client

	shellActions  shellActions
	dockerActions dockerActions

	// Cleanup functions to call on Cleanup()
	cleanups []func()
}

type shellActions struct {
	fm *realFramework
}

type dockerActions struct {
	fm *realFramework
}

type HostnameInfo struct {
	Host            string
	Port            int
	GceInstanceName string
}

// Returns: http://<host>:<port>/
func (self HostnameInfo) FullHostname() string {
	return fmt.Sprintf("http://%s:%d/", self.Host, self.Port)
}

func (self *realFramework) T() *testing.T {
	return self.t
}

func (self *realFramework) Hostname() HostnameInfo {
	return self.hostname
}

func (self *realFramework) Shell() ShellActions {
	return self.shellActions
}

func (self *realFramework) Docker() DockerActions {
	return self.dockerActions
}

func (self *realFramework) Cadvisor() CadvisorActions {
	return self
}

// Call all cleanup functions.
func (self *realFramework) Cleanup() {
	for _, cleanupFunc := range self.cleanups {
		cleanupFunc()
	}
}

// Gets a client to the cAdvisor being tested.
func (self *realFramework) Client() *client.Client {
	if self.cadvisorClient == nil {
		cadvisorClient, err := client.NewClient(self.Hostname().FullHostname())
		if err != nil {
			self.t.Fatalf("Failed to instantiate the cAdvisor client: %v", err)
		}
		self.cadvisorClient = cadvisorClient
	}
	return self.cadvisorClient
}

// Gets a v2 client to the cAdvisor being tested.
func (self *realFramework) ClientV2() *v2.Client {
	if self.cadvisorClientV2 == nil {
		cadvisorClientV2, err := v2.NewClient(self.Hostname().FullHostname())
		if err != nil {
			self.t.Fatalf("Failed to instantiate the cAdvisor client: %v", err)
		}
		self.cadvisorClientV2 = cadvisorClientV2
	}
	return self.cadvisorClientV2
}

func (self dockerActions) RunPause() string {
	return self.Run(DockerRunArgs{
		Image: "kubernetes/pause",
	})
}

// Run the specified command in a Docker busybox container.
func (self dockerActions) RunBusybox(cmd ...string) string {
	return self.Run(DockerRunArgs{
		Image: "busybox",
	}, cmd...)
}

type DockerRunArgs struct {
	// Image to use.
	Image string

	// Arguments to the Docker CLI.
	Args []string

	InnerArgs []string
}

// TODO(vmarmol): Use the Docker remote API.
// TODO(vmarmol): Refactor a set of "RunCommand" actions.
// Runs a Docker container in the background. Uses the specified DockerRunArgs and command.
//
// e.g.:
// RunDockerContainer(DockerRunArgs{Image: "busybox"}, "ping", "www.google.com")
//   -> docker run busybox ping www.google.com
func (self dockerActions) Run(args DockerRunArgs, cmd ...string) string {
	dockerCommand := append(append(append([]string{"docker", "run", "-d"}, args.Args...), args.Image), cmd...)

	output, _ := self.fm.Shell().Run("sudo", dockerCommand...)

	// The last line is the container ID.
	elements := strings.Fields(output)
	containerId := elements[len(elements)-1]

	self.fm.cleanups = append(self.fm.cleanups, func() {
		self.fm.Shell().Run("sudo", "docker", "rm", "-f", containerId)
	})
	return containerId
}

func (self dockerActions) Version() []string {
	dockerCommand := []string{"docker", "version", "-f", "'{{.Server.Version}}'"}
	output, _ := self.fm.Shell().Run("sudo", dockerCommand...)
	if len(output) != 1 {
		self.fm.T().Fatalf("need 1 arguments in output %v to get the version but have %v", output, len(output))
	}
	return strings.Split(output, ".")
}

func (self dockerActions) StorageDriver() string {
	dockerCommand := []string{"docker", "info"}
	output, _ := self.fm.Shell().Run("sudo", dockerCommand...)
	if len(output) < 1 {
		self.fm.T().Fatalf("failed to find docker storage driver - %v", output)
	}
	for _, line := range strings.Split(output, "\n") {
		line = strings.TrimSpace(line)
		if strings.HasPrefix(line, "Storage Driver: ") {
			idx := strings.LastIndex(line, ": ") + 2
			driver := line[idx:]
			switch driver {
			case Aufs:
				return Aufs
			case Overlay:
				return Overlay
			case DeviceMapper:
				return DeviceMapper
			default:
				return Unknown
			}
		}
	}
	self.fm.T().Fatalf("failed to find docker storage driver from info - %v", output)
	return Unknown
}

func (self dockerActions) RunStress(args DockerRunArgs, cmd ...string) string {
	dockerCommand := append(append(append(append([]string{"docker", "run", "-m=4M", "-d", "-t", "-i"}, args.Args...), args.Image), args.InnerArgs...), cmd...)

	output, _ := self.fm.Shell().RunStress("sudo", dockerCommand...)

	// The last line is the container ID.
	if len(output) < 1 {
		self.fm.T().Fatalf("need 1 arguments in output %v to get the name but have %v", output, len(output))
	}
	elements := strings.Fields(output)
	containerId := elements[len(elements)-1]

	self.fm.cleanups = append(self.fm.cleanups, func() {
		self.fm.Shell().Run("sudo", "docker", "rm", "-f", containerId)
	})
	return containerId
}

func (self shellActions) Run(command string, args ...string) (string, string) {
	var cmd *exec.Cmd
	if self.fm.Hostname().Host == "localhost" {
		// Just run locally.
		cmd = exec.Command(command, args...)
	} else {
		// We must SSH to the remote machine and run the command.
		args = append(common.GetGCComputeArgs("ssh", self.fm.Hostname().GceInstanceName, "--", command), args...)
		cmd = exec.Command("gcloud", args...)
	}
	var stdout bytes.Buffer
	var stderr bytes.Buffer
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	err := cmd.Run()
	if err != nil {
		self.fm.T().Fatalf("Failed to run %q %v in %q with error: %q. Stdout: %q, Stderr: %s", command, args, self.fm.Hostname().Host, err, stdout.String(), stderr.String())
		return "", ""
	}
	return stdout.String(), stderr.String()
}

func (self shellActions) RunStress(command string, args ...string) (string, string) {
	var cmd *exec.Cmd
	if self.fm.Hostname().Host == "localhost" {
		// Just run locally.
		cmd = exec.Command(command, args...)
	} else {
		// We must SSH to the remote machine and run the command.
		args = append(common.GetGCComputeArgs("ssh", self.fm.Hostname().GceInstanceName, "--", command), args...)
		cmd = exec.Command("gcloud", args...)
	}
	var stdout bytes.Buffer
	var stderr bytes.Buffer
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	err := cmd.Run()
	if err != nil {
		self.fm.T().Logf("Ran %q %v in %q and received error: %q. Stdout: %q, Stderr: %s", command, args, self.fm.Hostname().Host, err, stdout.String(), stderr.String())
		return stdout.String(), stderr.String()
	}
	return stdout.String(), stderr.String()
}

// Runs retryFunc until no error is returned. After dur time the last error is returned.
// Note that the function does not timeout the execution of retryFunc when the limit is reached.
func RetryForDuration(retryFunc func() error, dur time.Duration) error {
	waitUntil := time.Now().Add(dur)
	var err error
	for time.Now().Before(waitUntil) {
		err = retryFunc()
		if err == nil {
			return nil
		}
	}
	return err
}
<commit_msg>fix version handling in e2e tests<commit_after>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package framework

import (
	"bytes"
	"flag"
	"fmt"
	"os/exec"
	"strings"
	"testing"
	"time"

	"github.com/google/cadvisor/client"
	"github.com/google/cadvisor/client/v2"
	"github.com/google/cadvisor/integration/common"
)

var host = flag.String("host", "localhost", "Address of the host being tested")
var port = flag.Int("port", 8080, "Port of the application on the host being tested")

// Integration test framework.
type Framework interface {
	// Clean the framework state.
	Cleanup()

	// The testing.T used by the framework and the current test.
	T() *testing.T

	// Returns the hostname being tested.
	Hostname() HostnameInfo

	// Returns the Docker actions for the test framework.
	Docker() DockerActions

	// Returns the shell actions for the test framework.
	Shell() ShellActions

	// Returns the cAdvisor actions for the test framework.
	Cadvisor() CadvisorActions
}

// Instantiates a Framework. Cleanup *must* be called. Class is thread-compatible.
// All framework actions report fatal errors on the t specified at creation time.
//
// Typical use:
//
// func TestFoo(t *testing.T) {
// 	fm := framework.New(t)
// 	defer fm.Cleanup()
//      ... actual test ...
// }
func New(t *testing.T) Framework {
	// All integration tests are large.
	if testing.Short() {
		t.Skip("Skipping framework test in short mode")
	}

	// Try to see if non-localhost hosts are GCE instances.
	var gceInstanceName string
	hostname := *host
	if hostname != "localhost" {
		gceInstanceName = hostname
		gceIp, err := common.GetGceIp(hostname)
		if err == nil {
			hostname = gceIp
		}
	}

	fm := &realFramework{
		hostname: HostnameInfo{
			Host:            hostname,
			Port:            *port,
			GceInstanceName: gceInstanceName,
		},
		t:        t,
		cleanups: make([]func(), 0),
	}
	fm.shellActions = shellActions{
		fm: fm,
	}
	fm.dockerActions = dockerActions{
		fm: fm,
	}

	return fm
}

const (
	Aufs         string = "aufs"
	Overlay      string = "overlay"
	DeviceMapper string = "devicemapper"
	Unknown      string = ""
)

type DockerActions interface {
	// Run the no-op pause Docker container and return its ID.
	RunPause() string

	// Run the specified command in a Docker busybox container and return its ID.
	RunBusybox(cmd ...string) string

	// Runs a Docker container in the background. Uses the specified DockerRunArgs and command.
	// Returns the ID of the new container.
	//
	// e.g.:
	// Run(DockerRunArgs{Image: "busybox"}, "ping", "www.google.com")
	//   -> docker run busybox ping www.google.com
	Run(args DockerRunArgs, cmd ...string) string
	RunStress(args DockerRunArgs, cmd ...string) string

	Version() []string
	StorageDriver() string
}

type ShellActions interface {
	// Runs a specified command and arguments. Returns the stdout and stderr.
	Run(cmd string, args ...string) (string, string)
	RunStress(cmd string, args ...string) (string, string)
}

type CadvisorActions interface {
	// Returns a cAdvisor client to the machine being tested.
	Client() *client.Client
	ClientV2() *v2.Client
}

type realFramework struct {
	hostname         HostnameInfo
	t                *testing.T
	cadvisorClient   *client.Client
	cadvisorClientV2 *v2.Client

	shellActions  shellActions
	dockerActions dockerActions

	// Cleanup functions to call on Cleanup()
	cleanups []func()
}

type shellActions struct {
	fm *realFramework
}

type dockerActions struct {
	fm *realFramework
}

type HostnameInfo struct {
	Host            string
	Port            int
	GceInstanceName string
}

// Returns: http://<host>:<port>/
func (self HostnameInfo) FullHostname() string {
	return fmt.Sprintf("http://%s:%d/", self.Host, self.Port)
}

func (self *realFramework) T() *testing.T {
	return self.t
}

func (self *realFramework) Hostname() HostnameInfo {
	return self.hostname
}

func (self *realFramework) Shell() ShellActions {
	return self.shellActions
}

func (self *realFramework) Docker() DockerActions {
	return self.dockerActions
}

func (self *realFramework) Cadvisor() CadvisorActions {
	return self
}

// Call all cleanup functions.
func (self *realFramework) Cleanup() {
	for _, cleanupFunc := range self.cleanups {
		cleanupFunc()
	}
}

// Gets a client to the cAdvisor being tested.
func (self *realFramework) Client() *client.Client {
	if self.cadvisorClient == nil {
		cadvisorClient, err := client.NewClient(self.Hostname().FullHostname())
		if err != nil {
			self.t.Fatalf("Failed to instantiate the cAdvisor client: %v", err)
		}
		self.cadvisorClient = cadvisorClient
	}
	return self.cadvisorClient
}

// Gets a v2 client to the cAdvisor being tested.
func (self *realFramework) ClientV2() *v2.Client {
	if self.cadvisorClientV2 == nil {
		cadvisorClientV2, err := v2.NewClient(self.Hostname().FullHostname())
		if err != nil {
			self.t.Fatalf("Failed to instantiate the cAdvisor client: %v", err)
		}
		self.cadvisorClientV2 = cadvisorClientV2
	}
	return self.cadvisorClientV2
}

func (self dockerActions) RunPause() string {
	return self.Run(DockerRunArgs{
		Image: "kubernetes/pause",
	})
}

// Run the specified command in a Docker busybox container.
func (self dockerActions) RunBusybox(cmd ...string) string {
	return self.Run(DockerRunArgs{
		Image: "busybox",
	}, cmd...)
}

type DockerRunArgs struct {
	// Image to use.
	Image string

	// Arguments to the Docker CLI.
	Args []string

	InnerArgs []string
}

// TODO(vmarmol): Use the Docker remote API.
// TODO(vmarmol): Refactor a set of "RunCommand" actions.
// Runs a Docker container in the background. Uses the specified DockerRunArgs and command.
//
// e.g.:
// RunDockerContainer(DockerRunArgs{Image: "busybox"}, "ping", "www.google.com")
//   -> docker run busybox ping www.google.com
func (self dockerActions) Run(args DockerRunArgs, cmd ...string) string {
	dockerCommand := append(append(append([]string{"docker", "run", "-d"}, args.Args...), args.Image), cmd...)

	output, _ := self.fm.Shell().Run("sudo", dockerCommand...)

	// The last line is the container ID.
	elements := strings.Fields(output)
	containerId := elements[len(elements)-1]

	self.fm.cleanups = append(self.fm.cleanups, func() {
		self.fm.Shell().Run("sudo", "docker", "rm", "-f", containerId)
	})
	return containerId
}

func (self dockerActions) Version() []string {
	dockerCommand := []string{"docker", "version", "-f", "'{{.Server.Version}}'"}
	output, _ := self.fm.Shell().Run("sudo", dockerCommand...)
	output = strings.TrimSpace(output)
	ret := strings.Split(output, ".")
	if len(ret) != 3 {
		self.fm.T().Fatalf("invalid version %v", output)
	}
	return ret
}

func (self dockerActions) StorageDriver() string {
	dockerCommand := []string{"docker", "info"}
	output, _ := self.fm.Shell().Run("sudo", dockerCommand...)
	if len(output) < 1 {
		self.fm.T().Fatalf("failed to find docker storage driver - %v", output)
	}
	for _, line := range strings.Split(output, "\n") {
		line = strings.TrimSpace(line)
		if strings.HasPrefix(line, "Storage Driver: ") {
			idx := strings.LastIndex(line, ": ") + 2
			driver := line[idx:]
			switch driver {
			case Aufs:
				return Aufs
			case Overlay:
				return Overlay
			case DeviceMapper:
				return DeviceMapper
			default:
				return Unknown
			}
		}
	}
	self.fm.T().Fatalf("failed to find docker storage driver from info - %v", output)
	return Unknown
}

func (self dockerActions) RunStress(args DockerRunArgs, cmd ...string) string {
	dockerCommand := append(append(append(append([]string{"docker", "run", "-m=4M", "-d", "-t", "-i"}, args.Args...), args.Image), args.InnerArgs...), cmd...)

	output, _ := self.fm.Shell().RunStress("sudo", dockerCommand...)

	// The last line is the container ID.
	if len(output) < 1 {
		self.fm.T().Fatalf("need 1 arguments in output %v to get the name but have %v", output, len(output))
	}
	elements := strings.Fields(output)
	containerId := elements[len(elements)-1]

	self.fm.cleanups = append(self.fm.cleanups, func() {
		self.fm.Shell().Run("sudo", "docker", "rm", "-f", containerId)
	})
	return containerId
}

func (self shellActions) Run(command string, args ...string) (string, string) {
	var cmd *exec.Cmd
	if self.fm.Hostname().Host == "localhost" {
		// Just run locally.
		cmd = exec.Command(command, args...)
	} else {
		// We must SSH to the remote machine and run the command.
		args = append(common.GetGCComputeArgs("ssh", self.fm.Hostname().GceInstanceName, "--", command), args...)
		cmd = exec.Command("gcloud", args...)
	}
	var stdout bytes.Buffer
	var stderr bytes.Buffer
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	err := cmd.Run()
	if err != nil {
		self.fm.T().Fatalf("Failed to run %q %v in %q with error: %q. Stdout: %q, Stderr: %s", command, args, self.fm.Hostname().Host, err, stdout.String(), stderr.String())
		return "", ""
	}
	return stdout.String(), stderr.String()
}

func (self shellActions) RunStress(command string, args ...string) (string, string) {
	var cmd *exec.Cmd
	if self.fm.Hostname().Host == "localhost" {
		// Just run locally.
		cmd = exec.Command(command, args...)
	} else {
		// We must SSH to the remote machine and run the command.
		args = append(common.GetGCComputeArgs("ssh", self.fm.Hostname().GceInstanceName, "--", command), args...)
		cmd = exec.Command("gcloud", args...)
	}
	var stdout bytes.Buffer
	var stderr bytes.Buffer
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	err := cmd.Run()
	if err != nil {
		self.fm.T().Logf("Ran %q %v in %q and received error: %q. Stdout: %q, Stderr: %s", command, args, self.fm.Hostname().Host, err, stdout.String(), stderr.String())
		return stdout.String(), stderr.String()
	}
	return stdout.String(), stderr.String()
}

// Runs retryFunc until no error is returned. After dur time the last error is returned.
// Note that the function does not timeout the execution of retryFunc when the limit is reached.
func RetryForDuration(retryFunc func() error, dur time.Duration) error {
	waitUntil := time.Now().Add(dur)
	var err error
	for time.Now().Before(waitUntil) {
		err = retryFunc()
		if err == nil {
			return nil
		}
	}
	return err
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// The shenzhen-go binary serves a visual Go environment.
package main

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"os/exec"
	"runtime"
	"time"

	"github.com/improbable-eng/grpc-web/go/grpcweb"
	"github.com/zserge/webview"
	"google.golang.org/grpc"

	pb "github.com/google/shenzhen-go/dev/proto/go"
	"github.com/google/shenzhen-go/dev/server"
	"github.com/google/shenzhen-go/dev/server/view"
)

const pingMsg = "Pong!"

var (
	uiAddr            = flag.String("ui_addr", "localhost:0", "`address` to bind UI server to")
	useDefaultBrowser = flag.Bool("use_browser", true, "Load in the system's default web browser instead of the inbuilt webview")
)

func systemOpen(url string) error {
	switch runtime.GOOS {
	case "darwin":
		return exec.Command("open", url).Run()
	case "linux":
		// Seems to work on Linux Mint. YMMV.
		return exec.Command("xdg-open", url).Run()
	case "windows":
		return exec.Command("cmd.exe", "/C", "start", url).Run()
	default:
		fmt.Printf("Ready to open %s\n", url)
		return nil
	}
}

func webviewOpen(url string) error {
	return webview.Open("Shenzhen Go", url, 1152, 720, true)
}

func isUp(url string) bool {
	resp, err := http.Get(url)
	if err != nil {
		return false
	}
	defer resp.Body.Close()
	msg, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return false
	}
	return string(msg) == pingMsg
}

func waitForUp(addr net.Addr) error {
	url := fmt.Sprintf(`http://%s/ping`, addr)
	try := time.NewTicker(100 * time.Millisecond)
	defer try.Stop()
	timeout := time.NewTimer(5 * time.Second)
	defer timeout.Stop()
	for {
		select {
		case <-try.C:
			if isUp(url) {
				return nil
			}
		case <-timeout.C:
			return errors.New("timed out waiting 5s for server")
		}
	}
}

func open(addr net.Addr, path string, useBrowser bool) {
	url := fmt.Sprintf(`http://%s/%s`, addr, path)
	opener := systemOpen
	if !useBrowser {
		opener = webviewOpen
	}
	if err := opener(url); err != nil {
		fmt.Fprintf(os.Stderr, "Couldn't automatically open: %v\n", err)
		fmt.Printf("Ready to open %s\n", url)
	}
}

func serve(addr chan<- net.Addr) error {
	ln, err := net.Listen("tcp", *uiAddr)
	if err != nil {
		return fmt.Errorf("net.Listen: %v", err)
	}
	defer ln.Close()

	if addr != nil {
		addr <- ln.Addr()
	}

	http.HandleFunc("/ping", func(w http.ResponseWriter, _ *http.Request) { w.Write([]byte(pingMsg)) })
	http.Handle("/favicon.ico", view.Favicon)
	http.Handle("/.static/", http.StripPrefix("/.static/", view.Static))

	gs := grpc.NewServer()
	pb.RegisterShenzhenGoServer(gs, server.S)
	ws := grpcweb.WrapServer(gs)
	http.Handle("/.api/", http.StripPrefix("/.api/", ws))

	// Finally, all unknown paths are assumed to be files.
	http.Handle("/", server.S)
	if err := http.Serve(ln, nil); err != nil {
		return fmt.Errorf("http.Serve: %v", err)
	}
	return nil
}

func usage() {
	fmt.Fprintf(flag.CommandLine.Output(), `Shenzhen Go is a tool for managing Shenzhen Go source code.

Usage:

  %s [command] [files]

The (optional) commands are:

  build     generate and build Go packages
  edit      launch a Shenzhen Go server and open the editor interface
  generate  generate Go packages
  install   generate and install Go packages
  run       generate Go package and run binaries
  serve     launch a Shenzhen Go server

"edit" is the default command.

Flags:`, os.Args[0])
	// TODO: Add per-command help. `shenzhen-go help [command]`

	flag.PrintDefaults()
}

func main() {
	flag.Usage = usage
	flag.Parse()

	openUI := true
	args := flag.Args()
	if len(args) > 0 {
		switch args[0] {
		case "build":
			log.Fatalf("TODO: build is not yet implemented")
		case "edit":
			args = args[1:]
		case "generate":
			log.Fatalf("TODO: generate is not yet implemented")
		case "help":
			usage()
			return
		case "install":
			log.Fatalf("TODO: install is not yet implemented")
		case "run":
			log.Fatalf("TODO: run is not yet implemented")
		case "serve":
			if len(args) > 1 {
				log.Print(`Note: extra arguments to "serve" command are ignored`)
			}
			openUI = false
		default:
			// Edit, but every arg is a file.
		}
	} else {
		// Opens the browser at the root.
		args = []string{""}
	}

	adch := make(chan net.Addr)
	wait := make(chan struct{})
	go func() {
		serve(adch)
		close(wait)
	}()

	// Wait until properly serving.
	addr := <-adch
	if err := waitForUp(addr); err != nil {
		log.Fatalf("Couldn't reach server: %v", err)
	}
	log.Printf("Serving HTTP on %v", addr)

	if openUI {
		for _, a := range args {
			// Launch "open" which should launch a browser,
			// or ask the user to do so.
			// This must be called from the main thread to avoid
			// https://github.com/zserge/webview/issues/29.
			open(addr, a, *useDefaultBrowser)
		}
	}

	// Job done.
	<-wait
}
<commit_msg>Fix build on pre 1.10 - flagset didn't have Output<commit_after>// Copyright 2016 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// The shenzhen-go binary serves a visual Go environment.
package main

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"os/exec"
	"runtime"
	"time"

	"github.com/improbable-eng/grpc-web/go/grpcweb"
	"github.com/zserge/webview"
	"google.golang.org/grpc"

	pb "github.com/google/shenzhen-go/dev/proto/go"
	"github.com/google/shenzhen-go/dev/server"
	"github.com/google/shenzhen-go/dev/server/view"
)

const pingMsg = "Pong!"

var (
	uiAddr            = flag.String("ui_addr", "localhost:0", "`address` to bind UI server to")
	useDefaultBrowser = flag.Bool("use_browser", true, "Load in the system's default web browser instead of the inbuilt webview")
)

func systemOpen(url string) error {
	switch runtime.GOOS {
	case "darwin":
		return exec.Command("open", url).Run()
	case "linux":
		// Seems to work on Linux Mint. YMMV.
		return exec.Command("xdg-open", url).Run()
	case "windows":
		return exec.Command("cmd.exe", "/C", "start", url).Run()
	default:
		fmt.Printf("Ready to open %s\n", url)
		return nil
	}
}

func webviewOpen(url string) error {
	return webview.Open("Shenzhen Go", url, 1152, 720, true)
}

func isUp(url string) bool {
	resp, err := http.Get(url)
	if err != nil {
		return false
	}
	defer resp.Body.Close()
	msg, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return false
	}
	return string(msg) == pingMsg
}

func waitForUp(addr net.Addr) error {
	url := fmt.Sprintf(`http://%s/ping`, addr)
	try := time.NewTicker(100 * time.Millisecond)
	defer try.Stop()
	timeout := time.NewTimer(5 * time.Second)
	defer timeout.Stop()
	for {
		select {
		case <-try.C:
			if isUp(url) {
				return nil
			}
		case <-timeout.C:
			return errors.New("timed out waiting 5s for server")
		}
	}
}

func open(addr net.Addr, path string, useBrowser bool) {
	url := fmt.Sprintf(`http://%s/%s`, addr, path)
	opener := systemOpen
	if !useBrowser {
		opener = webviewOpen
	}
	if err := opener(url); err != nil {
		fmt.Fprintf(os.Stderr, "Couldn't automatically open: %v\n", err)
		fmt.Printf("Ready to open %s\n", url)
	}
}

func serve(addr chan<- net.Addr) error {
	ln, err := net.Listen("tcp", *uiAddr)
	if err != nil {
		return fmt.Errorf("net.Listen: %v", err)
	}
	defer ln.Close()

	if addr != nil {
		addr <- ln.Addr()
	}

	http.HandleFunc("/ping", func(w http.ResponseWriter, _ *http.Request) { w.Write([]byte(pingMsg)) })
	http.Handle("/favicon.ico", view.Favicon)
	http.Handle("/.static/", http.StripPrefix("/.static/", view.Static))

	gs := grpc.NewServer()
	pb.RegisterShenzhenGoServer(gs, server.S)
	ws := grpcweb.WrapServer(gs)
	http.Handle("/.api/", http.StripPrefix("/.api/", ws))

	// Finally, all unknown paths are assumed to be files.
	http.Handle("/", server.S)
	if err := http.Serve(ln, nil); err != nil {
		return fmt.Errorf("http.Serve: %v", err)
	}
	return nil
}

func usage() {
	fmt.Fprintf(os.Stderr, `Shenzhen Go is a tool for managing Shenzhen Go source code.

Usage:

  %s [command] [files]

The (optional) commands are:

  build     generate and build Go packages
  edit      launch a Shenzhen Go server and open the editor interface
  generate  generate Go packages
  install   generate and install Go packages
  run       generate Go package and run binaries
  serve     launch a Shenzhen Go server

"edit" is the default command.

Flags:`, os.Args[0])
	// TODO: Add per-command help. `shenzhen-go help [command]`

	flag.PrintDefaults()
}

func main() {
	flag.Usage = usage
	flag.Parse()

	openUI := true
	args := flag.Args()
	if len(args) > 0 {
		switch args[0] {
		case "build":
			log.Fatalf("TODO: build is not yet implemented")
		case "edit":
			args = args[1:]
		case "generate":
			log.Fatalf("TODO: generate is not yet implemented")
		case "help":
			usage()
			return
		case "install":
			log.Fatalf("TODO: install is not yet implemented")
		case "run":
			log.Fatalf("TODO: run is not yet implemented")
		case "serve":
			if len(args) > 1 {
				log.Print(`Note: extra arguments to "serve" command are ignored`)
			}
			openUI = false
		default:
			// Edit, but every arg is a file.
		}
	} else {
		// Opens the browser at the root.
		args = []string{""}
	}

	adch := make(chan net.Addr)
	wait := make(chan struct{})
	go func() {
		serve(adch)
		close(wait)
	}()

	// Wait until properly serving.
	addr := <-adch
	if err := waitForUp(addr); err != nil {
		log.Fatalf("Couldn't reach server: %v", err)
	}
	log.Printf("Serving HTTP on %v", addr)

	if openUI {
		for _, a := range args {
			// Launch "open" which should launch a browser,
			// or ask the user to do so.
			// This must be called from the main thread to avoid
			// https://github.com/zserge/webview/issues/29.
			open(addr, a, *useDefaultBrowser)
		}
	}

	// Job done.
	<-wait
}
<|endoftext|>


<commit_before>package smpp34

import (
	"bytes"
	"encoding/binary"
	"io"
	"reflect"
)

type PduReadErr string

type Pdu interface {
	Fields() map[string]Field
	MandatoryFieldsList() []string
	GetField(string) Field
	GetHeader() *Header
	TLVFields() map[uint16]*TLVField
	Writer() []byte
	SetField(f string, v interface{}) error
	SetTLVField(t, l int, v []byte) error
	SetSeqNum(uint32)
	Ok() bool
}

func (p PduReadErr) Error() string {
	return string(p)
}

func ParsePdu(data []byte) (Pdu, error) {
	if len(data) < 16 {
		return nil, PduReadErr("Invalid PDU. Length under 16 bytes")
	}

	header := ParsePduHeader(data[:16])

	switch header.Id {
	case SUBMIT_SM:
		n, err := NewSubmitSm(header, data[16:])
		return Pdu(n), err
	case SUBMIT_SM_RESP:
		n, err := NewSubmitSmResp(header, data[16:])
		return Pdu(n), err
	case DELIVER_SM:
		n, err := NewDeliverSm(header, data[16:])
		return Pdu(n), err
	case DELIVER_SM_RESP:
		n, err := NewDeliverSmResp(header, data[16:])
		return Pdu(n), err
	case BIND_TRANSCEIVER, BIND_RECEIVER, BIND_TRANSMITTER:
		n, err := NewBind(header, data[16:])
		return Pdu(n), err
	case BIND_TRANSCEIVER_RESP, BIND_RECEIVER_RESP, BIND_TRANSMITTER_RESP:
		n, err := NewBindResp(header, data[16:])
		return Pdu(n), err
	case ENQUIRE_LINK:
		n, err := NewEnquireLink(header)
		return Pdu(n), err
	case ENQUIRE_LINK_RESP:
		n, err := NewEnquireLinkResp(header)
		return Pdu(n), err
	case UNBIND:
		n, err := NewUnbind(header)
		return Pdu(n), err
	case UNBIND_RESP:
		n, err := NewUnbindResp(header)
		return Pdu(n), err
	default:
		return nil, PduReadErr(header.Id.Error())
	}
}

func ParsePduHeader(data []byte) *Header {
	return NewPduHeader(
		unpackUi32(data[:4]),
		CMDId(unpackUi32(data[4:8])),
		CMDStatus(unpackUi32(data[8:12])),
		unpackUi32(data[12:16]),
	)
}

func create_pdu_fields(fieldNames []string, r *bytes.Buffer) (map[string]Field, map[uint16]*TLVField, error) {

	fields := make(map[string]Field)
	eof := false
	for _, k := range fieldNames {
		switch k {
		case SERVICE_TYPE, SOURCE_ADDR, DESTINATION_ADDR, SCHEDULE_DELIVERY_TIME, VALIDITY_PERIOD, SYSTEM_ID, PASSWORD, SYSTEM_TYPE, ADDRESS_RANGE, MESSAGE_ID:
			t, err := r.ReadBytes(0x00)

			if err == io.EOF {
				eof = true
			} else if err != nil {
				return nil, nil, err
			}

			fields[k] = NewVariableField(t)
		case SOURCE_ADDR_TON, SOURCE_ADDR_NPI, DEST_ADDR_TON, DEST_ADDR_NPI, ESM_CLASS, PROTOCOL_ID, PRIORITY_FLAG, REGISTERED_DELIVERY, REPLACE_IF_PRESENT_FLAG, DATA_CODING, SM_DEFAULT_MSG_ID, INTERFACE_VERSION, ADDR_TON, ADDR_NPI:
			t, err := r.ReadByte()

			if err == io.EOF {
				eof = true
			} else if err != nil {
				return nil, nil, err
			}

			fields[k] = NewFixedField(t)
		case SM_LENGTH:
			// Short Message Length
			t, err := r.ReadByte()

			if err == io.EOF {
				eof = true
			} else if err != nil {
				return nil, nil, err
			}

			fields[k] = NewFixedField(t)

			// Short Message
			p := make([]byte, t)

			_, err = r.Read(p)
			if err == io.EOF {
				eof = true
			} else if err != nil {
				return nil, nil, err
			}

			fields[SHORT_MESSAGE] = NewVariableField(p)
		case SHORT_MESSAGE:
			continue
		}
	}

	// Optional Fields
	tlvs := map[uint16]*TLVField{}
	var err error

	if !eof {
		tlvs, err = parse_tlv_fields(r)

		if err != nil {
			return nil, nil, err
		}
	}

	return fields, tlvs, nil
}

func parse_tlv_fields(r *bytes.Buffer) (map[uint16]*TLVField, error) {
	tlvs := map[uint16]*TLVField{}

	for {
		p := make([]byte, 4)
		_, err := r.Read(p)

		if err == io.EOF {
			break
		} else if err != nil {
			return nil, err
		}

		// length
		l := unpackUi16(p[2:4])

		// Get Value
		v := make([]byte, l)

		_, err = r.Read(v)
		if err != nil {
			return nil, err
		}

		tlvs[unpackUi16(p[0:2])] = &TLVField{
			unpackUi16(p[0:2]),
			unpackUi16(p[2:4]),
			v,
		}
	}

	return tlvs, nil
}

func validate_pdu_field(f string, v interface{}) bool {
	switch f {
	case SOURCE_ADDR_TON, SOURCE_ADDR_NPI, DEST_ADDR_TON, DEST_ADDR_NPI, ESM_CLASS, PROTOCOL_ID, PRIORITY_FLAG, REGISTERED_DELIVERY, REPLACE_IF_PRESENT_FLAG, DATA_CODING, SM_DEFAULT_MSG_ID, INTERFACE_VERSION, ADDR_TON, ADDR_NPI, SM_LENGTH:
		if validate_pdu_field_type(0x00, v) {
			return true
		}
	case SERVICE_TYPE, SOURCE_ADDR, DESTINATION_ADDR, SCHEDULE_DELIVERY_TIME, VALIDITY_PERIOD, SYSTEM_ID, PASSWORD, SYSTEM_TYPE, ADDRESS_RANGE, MESSAGE_ID, SHORT_MESSAGE:
		if validate_pdu_field_type("string", v) {
			return true
		}
	}
	return false
}

func validate_pdu_field_type(t interface{}, v interface{}) bool {
	if reflect.TypeOf(t) == reflect.TypeOf(v) {
		return true
	}

	return false
}

func included_check(a []string, v string) bool {
	for _, k := range a {
		if k == v {
			return true
		}
	}
	return false
}

func unpackUi32(b []byte) (n uint32) {
	n = binary.BigEndian.Uint32(b)
	return
}

func packUi32(n uint32) (b []byte) {
	b = make([]byte, 4)
	binary.BigEndian.PutUint32(b, n)
	return
}

func unpackUi16(b []byte) (n uint16) {
	n = binary.BigEndian.Uint16(b)
	return
}

func packUi16(n uint16) (b []byte) {
	b = make([]byte, 2)
	binary.BigEndian.PutUint16(b, n)
	return
}

func packUi8(n uint8) (b []byte) {
	b = make([]byte, 2)
	binary.BigEndian.PutUint16(b, uint16(n))
	return b[1:]
}
<commit_msg>Updated PDU errors<commit_after>package smpp34

import (
	"bytes"
	"encoding/binary"
	"io"
	"reflect"
)

type PduReadErr string
type PduCmdIdErr string

type Pdu interface {
	Fields() map[string]Field
	MandatoryFieldsList() []string
	GetField(string) Field
	GetHeader() *Header
	TLVFields() map[uint16]*TLVField
	Writer() []byte
	SetField(f string, v interface{}) error
	SetTLVField(t, l int, v []byte) error
	SetSeqNum(uint32)
	Ok() bool
}

func (p PduReadErr) Error() string {
	return string(p)
}

func (p PduCmdIdErr) Error() string {
	return string(p)
}

func ParsePdu(data []byte) (Pdu, error) {
	if len(data) < 16 {
		return nil, PduReadErr("Invalid PDU. Length under 16 bytes")
	}

	header := ParsePduHeader(data[:16])

	switch header.Id {
	case SUBMIT_SM:
		n, err := NewSubmitSm(header, data[16:])
		return Pdu(n), err
	case SUBMIT_SM_RESP:
		n, err := NewSubmitSmResp(header, data[16:])
		return Pdu(n), err
	case DELIVER_SM:
		n, err := NewDeliverSm(header, data[16:])
		return Pdu(n), err
	case DELIVER_SM_RESP:
		n, err := NewDeliverSmResp(header, data[16:])
		return Pdu(n), err
	case BIND_TRANSCEIVER, BIND_RECEIVER, BIND_TRANSMITTER:
		n, err := NewBind(header, data[16:])
		return Pdu(n), err
	case BIND_TRANSCEIVER_RESP, BIND_RECEIVER_RESP, BIND_TRANSMITTER_RESP:
		n, err := NewBindResp(header, data[16:])
		return Pdu(n), err
	case ENQUIRE_LINK:
		n, err := NewEnquireLink(header)
		return Pdu(n), err
	case ENQUIRE_LINK_RESP:
		n, err := NewEnquireLinkResp(header)
		return Pdu(n), err
	case UNBIND:
		n, err := NewUnbind(header)
		return Pdu(n), err
	case UNBIND_RESP:
		n, err := NewUnbindResp(header)
		return Pdu(n), err
	default:
		return nil, PduCmdIdErr(header.Id.Error())
	}
}

func ParsePduHeader(data []byte) *Header {
	return NewPduHeader(
		unpackUi32(data[:4]),
		CMDId(unpackUi32(data[4:8])),
		CMDStatus(unpackUi32(data[8:12])),
		unpackUi32(data[12:16]),
	)
}

func create_pdu_fields(fieldNames []string, r *bytes.Buffer) (map[string]Field, map[uint16]*TLVField, error) {

	fields := make(map[string]Field)
	eof := false
	for _, k := range fieldNames {
		switch k {
		case SERVICE_TYPE, SOURCE_ADDR, DESTINATION_ADDR, SCHEDULE_DELIVERY_TIME, VALIDITY_PERIOD, SYSTEM_ID, PASSWORD, SYSTEM_TYPE, ADDRESS_RANGE, MESSAGE_ID:
			t, err := r.ReadBytes(0x00)

			if err == io.EOF {
				eof = true
			} else if err != nil {
				return nil, nil, err
			}

			fields[k] = NewVariableField(t)
		case SOURCE_ADDR_TON, SOURCE_ADDR_NPI, DEST_ADDR_TON, DEST_ADDR_NPI, ESM_CLASS, PROTOCOL_ID, PRIORITY_FLAG, REGISTERED_DELIVERY, REPLACE_IF_PRESENT_FLAG, DATA_CODING, SM_DEFAULT_MSG_ID, INTERFACE_VERSION, ADDR_TON, ADDR_NPI:
			t, err := r.ReadByte()

			if err == io.EOF {
				eof = true
			} else if err != nil {
				return nil, nil, err
			}

			fields[k] = NewFixedField(t)
		case SM_LENGTH:
			// Short Message Length
			t, err := r.ReadByte()

			if err == io.EOF {
				eof = true
			} else if err != nil {
				return nil, nil, err
			}

			fields[k] = NewFixedField(t)

			// Short Message
			p := make([]byte, t)

			_, err = r.Read(p)
			if err == io.EOF {
				eof = true
			} else if err != nil {
				return nil, nil, err
			}

			fields[SHORT_MESSAGE] = NewVariableField(p)
		case SHORT_MESSAGE:
			continue
		}
	}

	// Optional Fields
	tlvs := map[uint16]*TLVField{}
	var err error

	if !eof {
		tlvs, err = parse_tlv_fields(r)

		if err != nil {
			return nil, nil, err
		}
	}

	return fields, tlvs, nil
}

func parse_tlv_fields(r *bytes.Buffer) (map[uint16]*TLVField, error) {
	tlvs := map[uint16]*TLVField{}

	for {
		p := make([]byte, 4)
		_, err := r.Read(p)

		if err == io.EOF {
			break
		} else if err != nil {
			return nil, err
		}

		// length
		l := unpackUi16(p[2:4])

		// Get Value
		v := make([]byte, l)

		_, err = r.Read(v)
		if err != nil {
			return nil, err
		}

		tlvs[unpackUi16(p[0:2])] = &TLVField{
			unpackUi16(p[0:2]),
			unpackUi16(p[2:4]),
			v,
		}
	}

	return tlvs, nil
}

func validate_pdu_field(f string, v interface{}) bool {
	switch f {
	case SOURCE_ADDR_TON, SOURCE_ADDR_NPI, DEST_ADDR_TON, DEST_ADDR_NPI, ESM_CLASS, PROTOCOL_ID, PRIORITY_FLAG, REGISTERED_DELIVERY, REPLACE_IF_PRESENT_FLAG, DATA_CODING, SM_DEFAULT_MSG_ID, INTERFACE_VERSION, ADDR_TON, ADDR_NPI, SM_LENGTH:
		if validate_pdu_field_type(0x00, v) {
			return true
		}
	case SERVICE_TYPE, SOURCE_ADDR, DESTINATION_ADDR, SCHEDULE_DELIVERY_TIME, VALIDITY_PERIOD, SYSTEM_ID, PASSWORD, SYSTEM_TYPE, ADDRESS_RANGE, MESSAGE_ID, SHORT_MESSAGE:
		if validate_pdu_field_type("string", v) {
			return true
		}
	}
	return false
}

func validate_pdu_field_type(t interface{}, v interface{}) bool {
	if reflect.TypeOf(t) == reflect.TypeOf(v) {
		return true
	}

	return false
}

func included_check(a []string, v string) bool {
	for _, k := range a {
		if k == v {
			return true
		}
	}
	return false
}

func unpackUi32(b []byte) (n uint32) {
	n = binary.BigEndian.Uint32(b)
	return
}

func packUi32(n uint32) (b []byte) {
	b = make([]byte, 4)
	binary.BigEndian.PutUint32(b, n)
	return
}

func unpackUi16(b []byte) (n uint16) {
	n = binary.BigEndian.Uint16(b)
	return
}

func packUi16(n uint16) (b []byte) {
	b = make([]byte, 2)
	binary.BigEndian.PutUint16(b, n)
	return
}

func packUi8(n uint8) (b []byte) {
	b = make([]byte, 2)
	binary.BigEndian.PutUint16(b, uint16(n))
	return b[1:]
}
<|endoftext|>


<commit_before>package market

import (
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"time"

	"github.com/nzai/stockrecorder/analyse"
	"github.com/nzai/stockrecorder/config"
	"github.com/nzai/stockrecorder/db"
	"github.com/nzai/stockrecorder/io"
)

type YahooJson struct {
	Chart YahooChart `json:"chart"`
}

type YahooChart struct {
	Result []YahooResult `json:"result"`
	Err    *YahooError   `json:"error"`
}

type YahooError struct {
	Code        string `json:"code"`
	Description string `json:"description"`
}

type YahooResult struct {
	Meta       YahooMeta       `json:"meta"`
	Timestamp  []int64         `json:"timestamp"`
	Indicators YahooIndicators `json:"indicators"`
}

type YahooMeta struct {
	Currency             string              `json:"currency"`
	Symbol               string              `json:"symbol"`
	ExchangeName         string              `json:"exchangeName"`
	InstrumentType       string              `json:"instrumentType"`
	FirstTradeDate       int64               `json:"firstTradeDate"`
	GMTOffset            int                 `json:"gmtoffset"`
	Timezone             string              `json:"timezone"`
	PreviousClose        float32             `json:"previousClose"`
	Scale                int                 `json:"scale"`
	CurrentTradingPeriod YahooTradingPeroid  `json:"currentTradingPeriod"`
	TradingPeriods       YahooTradingPeroids `json:"tradingPeriods"`
	DataGranularity      string              `json:"dataGranularity"`
	ValidRanges          []string            `json:"validRanges"`
}

type YahooTradingPeroid struct {
	Pre     YahooTradingPeroidSection `json:"pre"`
	Regular YahooTradingPeroidSection `json:"regular"`
	Post    YahooTradingPeroidSection `json:"post"`
}

type YahooTradingPeroids struct {
	Pres     [][]YahooTradingPeroidSection `json:"pre"`
	Regulars [][]YahooTradingPeroidSection `json:"regular"`
	Posts    [][]YahooTradingPeroidSection `json:"post"`
}

type YahooTradingPeroidSection struct {
	Timezone  string `json:"timezone"`
	Start     int64  `json:"start"`
	End       int64  `json:"end"`
	GMTOffset int    `json:"gmtoffset"`
}

type YahooIndicators struct {
	Quotes []YahooQuote `json:"quote"`
}

type YahooQuote struct {
	Open   []float32 `json:"open"`
	Close  []float32 `json:"close"`
	High   []float32 `json:"high"`
	Low    []float32 `json:"low"`
	Volume []int64   `json:"volume"`
}

//	从雅虎财经获取上市公司分时数据
func DownloadCompanyDaily(marketName, companyCode, queryCode string, day time.Time) error {
	//	文件保存路径
	fileName := fmt.Sprintf("%s_raw.txt", day.Format("20060102"))
	filePath := filepath.Join(config.Get().DataDir, marketName, companyCode, fileName)

	//	如果文件已存在就忽略
	var content []byte
	_, err := os.Stat(filePath)
	if os.IsNotExist(err) {
		//	如果不存在就抓取并保存
		start := time.Date(day.Year(), day.Month(), day.Day(), 0, 0, 0, 0, day.Location())
		end := start.Add(time.Hour * 24)

		pattern := "https://finance-yql.media.yahoo.com/v7/finance/chart/%s?period2=%d&period1=%d&interval=1m&indicators=quote&includeTimestamps=true&includePrePost=true&events=div%7Csplit%7Cearn&corsDomain=finance.yahoo.com"
		url := fmt.Sprintf(pattern, queryCode, end.Unix(), start.Unix())

		html, err := io.DownloadStringRetry(url, retryTimes, retryIntervalSeconds)
		if err != nil {
			return err
		}

		content = []byte(html)

		//	写入文件
		return io.WriteString(filePath, html)
	}

	//	如果不解析
	if !config.Get().EnableAnalyse {
		return nil
	}

	//	检查数据库是否解析过,解析过就忽略
	found, err := db.DailyExists(marketName, companyCode, day)
	if err != nil {
		return err
	}

	if !found {
		if content == nil {
			//	读取文件
			buffer, err := io.ReadAllBytes(filePath)
			if err != nil {
				return err
			}

			content = buffer
		}

		//	解析
		dar, err := parseDailyYahooJson(marketName, companyCode, day, content)
		if err != nil {
			return err
		}

		//	保存
		err = db.DailySave(dar)
		if err != nil {
			return err
		}
	}

	return nil
}

//	解析雅虎Json
func parseDailyYahooJson(marketName, companyCode string, date time.Time, buffer []byte) (*analyse.DailyAnalyzeResult, error) {

	yj := &YahooJson{}
	err := json.Unmarshal(buffer, &yj)
	if err != nil {
		return nil, fmt.Errorf("解析雅虎Json发生错误: %s", err)
	}
	//	log.Print("AAAAA:", yj)

	result := &analyse.DailyAnalyzeResult{
		DailyResult: analyse.DailyResult{
			Code:    companyCode,
			Market:  marketName,
			Date:    date,
			Error:   false,
			Message: ""},
		Pre:     make([]analyse.Peroid60, 0),
		Regular: make([]analyse.Peroid60, 0),
		Post:    make([]analyse.Peroid60, 0)}

	//	检查数据
	err = validateDailyYahooJson(yj)
	if err != nil {
		result.DailyResult.Error = true
		result.DailyResult.Message = err.Error()

		return result, nil
	}

	periods, quote := yj.Chart.Result[0].Meta.TradingPeriods, yj.Chart.Result[0].Indicators.Quotes[0]
	for index, ts := range yj.Chart.Result[0].Timestamp {

		p := analyse.Peroid60{
			Code:   companyCode,
			Market: marketName,
			Start:  time.Unix(ts, 0),
			End:    time.Unix(ts+60, 0),
			Open:   quote.Open[index],
			Close:  quote.Close[index],
			High:   quote.High[index],
			Low:    quote.Low[index],
			Volume: quote.Volume[index]}

		//	Pre, Regular, Post
		if ts >= periods.Pres[0][0].Start && ts < periods.Pres[0][0].End {
			result.Pre = append(result.Pre, p)
		} else if ts >= periods.Regulars[0][0].Start && ts < periods.Regulars[0][0].End {
			result.Regular = append(result.Regular, p)
		} else if ts >= periods.Posts[0][0].Start && ts < periods.Posts[0][0].End {
			result.Post = append(result.Regular, p)
		}
	}

	return result, nil
}

func validateDailyYahooJson(yj *YahooJson) error {

	if yj.Chart.Err != nil {
		return fmt.Errorf("[%s]%s", yj.Chart.Err.Code, yj.Chart.Err.Description)
	}

	if yj.Chart.Result == nil || len(yj.Chart.Result) == 0 {
		return fmt.Errorf("Result为空")
	}

	if yj.Chart.Result[0].Indicators.Quotes == nil || len(yj.Chart.Result[0].Indicators.Quotes) == 0 {
		return fmt.Errorf("Quotes为空")
	}

	result, quote := yj.Chart.Result[0], yj.Chart.Result[0].Indicators.Quotes[0]
	if len(result.Timestamp) != len(quote.Open) ||
		len(result.Timestamp) != len(quote.Close) ||
		len(result.Timestamp) != len(quote.High) ||
		len(result.Timestamp) != len(quote.Low) ||
		len(result.Timestamp) != len(quote.Volume) {
		return fmt.Errorf("Quotes数量不正确")
	}

	if len(result.Meta.TradingPeriods.Pres) == 0 ||
		len(result.Meta.TradingPeriods.Pres[0]) == 0 ||
		len(result.Meta.TradingPeriods.Posts) == 0 ||
		len(result.Meta.TradingPeriods.Posts[0]) == 0 ||
		len(result.Meta.TradingPeriods.Regulars) == 0 ||
		len(result.Meta.TradingPeriods.Regulars[0]) == 0 {
		return fmt.Errorf("TradingPeriods数量不正确")
	}
	return nil
}
<commit_msg>调整雅虎财经数据的解析及保存方法，有问题的数据才以文件形式存盘，解析过的文件一律删除，将数据从文件系统全部搬移到数据库中<commit_after>package market

import (
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"time"

	"github.com/nzai/stockrecorder/config"
	"github.com/nzai/stockrecorder/db"
	"github.com/nzai/stockrecorder/io"
)

type YahooJson struct {
	Chart YahooChart `json:"chart"`
}

type YahooChart struct {
	Result []YahooResult `json:"result"`
	Err    *YahooError   `json:"error"`
}

type YahooError struct {
	Code        string `json:"code"`
	Description string `json:"description"`
}

type YahooResult struct {
	Meta       YahooMeta       `json:"meta"`
	Timestamp  []int64         `json:"timestamp"`
	Indicators YahooIndicators `json:"indicators"`
}

type YahooMeta struct {
	Currency             string              `json:"currency"`
	Symbol               string              `json:"symbol"`
	ExchangeName         string              `json:"exchangeName"`
	InstrumentType       string              `json:"instrumentType"`
	FirstTradeDate       int64               `json:"firstTradeDate"`
	GMTOffset            int                 `json:"gmtoffset"`
	Timezone             string              `json:"timezone"`
	PreviousClose        float32             `json:"previousClose"`
	Scale                int                 `json:"scale"`
	CurrentTradingPeriod YahooTradingPeroid  `json:"currentTradingPeriod"`
	TradingPeriods       YahooTradingPeroids `json:"tradingPeriods"`
	DataGranularity      string              `json:"dataGranularity"`
	ValidRanges          []string            `json:"validRanges"`
}

type YahooTradingPeroid struct {
	Pre     YahooTradingPeroidSection `json:"pre"`
	Regular YahooTradingPeroidSection `json:"regular"`
	Post    YahooTradingPeroidSection `json:"post"`
}

type YahooTradingPeroids struct {
	Pres     [][]YahooTradingPeroidSection `json:"pre"`
	Regulars [][]YahooTradingPeroidSection `json:"regular"`
	Posts    [][]YahooTradingPeroidSection `json:"post"`
}

type YahooTradingPeroidSection struct {
	Timezone  string `json:"timezone"`
	Start     int64  `json:"start"`
	End       int64  `json:"end"`
	GMTOffset int    `json:"gmtoffset"`
}

type YahooIndicators struct {
	Quotes []YahooQuote `json:"quote"`
}

type YahooQuote struct {
	Open   []float32 `json:"open"`
	Close  []float32 `json:"close"`
	High   []float32 `json:"high"`
	Low    []float32 `json:"low"`
	Volume []int64   `json:"volume"`
}

//	从雅虎财经获取上市公司分时数据
func DownloadCompanyDaily(marketName, companyCode, queryCode string, day time.Time) error {

	//	检查数据库是否解析过
	found, err := db.DailyExists(marketName, companyCode, day)
	if err != nil {
		return err
	}

	//	解析过的不再重复解析
	if found {
		return nil
	}

	//	文件保存路径
	fileName := fmt.Sprintf("%s_raw.txt", day.Format("20060102"))
	filePath := filepath.Join(config.Get().DataDir, marketName, companyCode, fileName)

	var buffer []byte
	fileExists := false

	//	检查磁盘上数据文件是否已经存在
	_, err = os.Stat(filePath)
	if os.IsNotExist(err) {

		//	如果不存在就抓取
		start := time.Date(day.Year(), day.Month(), day.Day(), 0, 0, 0, 0, day.Location())
		end := start.Add(time.Hour * 24)

		pattern := "https://finance-yql.media.yahoo.com/v7/finance/chart/%s?period2=%d&period1=%d&interval=1m&indicators=quote&includeTimestamps=true&includePrePost=true&events=div%7Csplit%7Cearn&corsDomain=finance.yahoo.com"
		url := fmt.Sprintf(pattern, queryCode, end.Unix(), start.Unix())

		//	查询Yahoo财经接口,返回股票分时数据
		content, err := io.DownloadStringRetry(url, retryTimes, retryIntervalSeconds)
		if err != nil {
			return err
		}

		buffer = []byte(content)
	} else {
		fileExists = true

		//	如果已经存在就读取文件
		buffer, err = io.ReadAllBytes(filePath)
		if err != nil {
			return err
		}
	}

	//	解析
	dar, err := ParseDailyYahooJson(marketName, companyCode, day, buffer)
	if err != nil {

		//	解析错误的先保存为文件
		err1 := saveDaily(marketName, companyCode, day, buffer)
		if err1 != nil {
			return err1
		}

		return err
	}

	//	保存
	err = db.DailySave(dar)
	if err != nil {

		//	保存错误的先保存为文件
		err1 := saveDaily(marketName, companyCode, day, buffer)
		if err1 != nil {
			return err1
		}

		return err
	}

	if fileExists {
		//	解析成功就删除文件
		return os.Remove(filePath)
	}

	return nil
}

//	解析雅虎Json
func ParseDailyYahooJson(marketName, companyCode string, date time.Time, buffer []byte) (*db.DailyAnalyzeResult, error) {

	yj := &YahooJson{}
	err := json.Unmarshal(buffer, &yj)
	if err != nil {
		return nil, fmt.Errorf("解析雅虎Json发生错误: %s", err)
	}

	result := &db.DailyAnalyzeResult{
		DailyResult: db.DailyResult{
			Code:    companyCode,
			Market:  marketName,
			Date:    date,
			Error:   false,
			Message: ""},
		Pre:     make([]db.Peroid60, 0),
		Regular: make([]db.Peroid60, 0),
		Post:    make([]db.Peroid60, 0)}

	//	检查数据
	err = validateDailyYahooJson(yj)
	if err != nil {
		result.DailyResult.Error = true
		result.DailyResult.Message = err.Error()

		return result, nil
	}

	periods, quote := yj.Chart.Result[0].Meta.TradingPeriods, yj.Chart.Result[0].Indicators.Quotes[0]
	for index, ts := range yj.Chart.Result[0].Timestamp {

		p := db.Peroid60{
			Code:   companyCode,
			Market: marketName,
			Start:  time.Unix(ts, 0),
			End:    time.Unix(ts+60, 0),
			Open:   quote.Open[index],
			Close:  quote.Close[index],
			High:   quote.High[index],
			Low:    quote.Low[index],
			Volume: quote.Volume[index]}

		//	Pre, Regular, Post
		if ts >= periods.Pres[0][0].Start && ts < periods.Pres[0][0].End {
			result.Pre = append(result.Pre, p)
		} else if ts >= periods.Regulars[0][0].Start && ts < periods.Regulars[0][0].End {
			result.Regular = append(result.Regular, p)
		} else if ts >= periods.Posts[0][0].Start && ts < periods.Posts[0][0].End {
			result.Post = append(result.Regular, p)
		}
	}

	return result, nil
}

//	验证雅虎Json
func validateDailyYahooJson(yj *YahooJson) error {

	if yj.Chart.Err != nil {
		return fmt.Errorf("[%s]%s", yj.Chart.Err.Code, yj.Chart.Err.Description)
	}

	if yj.Chart.Result == nil || len(yj.Chart.Result) == 0 {
		return fmt.Errorf("Result为空")
	}

	if yj.Chart.Result[0].Indicators.Quotes == nil || len(yj.Chart.Result[0].Indicators.Quotes) == 0 {
		return fmt.Errorf("Quotes为空")
	}

	result, quote := yj.Chart.Result[0], yj.Chart.Result[0].Indicators.Quotes[0]
	if len(result.Timestamp) != len(quote.Open) ||
		len(result.Timestamp) != len(quote.Close) ||
		len(result.Timestamp) != len(quote.High) ||
		len(result.Timestamp) != len(quote.Low) ||
		len(result.Timestamp) != len(quote.Volume) {
		return fmt.Errorf("Quotes数量不正确")
	}

	if len(result.Meta.TradingPeriods.Pres) == 0 ||
		len(result.Meta.TradingPeriods.Pres[0]) == 0 ||
		len(result.Meta.TradingPeriods.Posts) == 0 ||
		len(result.Meta.TradingPeriods.Posts[0]) == 0 ||
		len(result.Meta.TradingPeriods.Regulars) == 0 ||
		len(result.Meta.TradingPeriods.Regulars[0]) == 0 {
		return fmt.Errorf("TradingPeriods数量不正确")
	}
	return nil
}

//	保存到文件
func saveDaily(marketName, companyCode string, day time.Time, buffer []byte) error {

	//	文件保存路径
	fileName := fmt.Sprintf("%s_raw.txt", day.Format("20060102"))
	filePath := filepath.Join(config.Get().DataDir, marketName, companyCode, fileName)

	//	不覆盖原文件
	_, err := os.Stat(filePath)
	if os.IsNotExist(err) {
		return io.WriteBytes(filePath, buffer)
	}

	return nil
}
<|endoftext|>


<commit_before>package session

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/breml/logstash-config/ast"
	"github.com/breml/logstash-config/ast/astutil"
	"github.com/pkg/errors"

	"github.com/magnusbaeck/logstash-filter-verifier/v2/internal/daemon/idgen"
	"github.com/magnusbaeck/logstash-filter-verifier/v2/internal/daemon/logstashconfig"
	"github.com/magnusbaeck/logstash-filter-verifier/v2/internal/daemon/pipeline"
	"github.com/magnusbaeck/logstash-filter-verifier/v2/internal/daemon/pool"
	"github.com/magnusbaeck/logstash-filter-verifier/v2/internal/daemon/template"
	"github.com/magnusbaeck/logstash-filter-verifier/v2/internal/logging"
	"github.com/magnusbaeck/logstash-filter-verifier/v2/internal/testcase"
)

type Session struct {
	id string

	logstashController         pool.LogstashController
	isOrderedPipelineSupported bool

	baseDir    string
	sessionDir string

	pipelines         pipeline.Pipelines
	inputPluginCodecs map[string]string
	testexec          int

	noCleanup bool

	log logging.Logger
}

func newSession(baseDir string, logstashController pool.LogstashController, noCleanup bool, isOrderedPipelineSupported bool, log logging.Logger) *Session {
	sessionID := idgen.New()
	sessionDir := fmt.Sprintf("%s/session/%s", baseDir, sessionID)
	return &Session{
		id:                         sessionID,
		baseDir:                    baseDir,
		sessionDir:                 sessionDir,
		logstashController:         logstashController,
		isOrderedPipelineSupported: isOrderedPipelineSupported,
		noCleanup:                  noCleanup,
		inputPluginCodecs:          map[string]string{},
		log:                        log,
	}
}

// ID returns the id of the session.
func (s Session) ID() string {
	return s.id
}

// setupTest prepares the Logstash configuration for a new test run.
func (s *Session) setupTest(pipelines pipeline.Pipelines, configFiles []logstashconfig.File) error {
	err := os.MkdirAll(s.sessionDir, 0700)
	if err != nil {
		return err
	}

	sutConfigDir := filepath.Join(s.sessionDir, "sut")

	// adjust pipeline names and config directories to session
	for i := range pipelines {
		pipelineName := fmt.Sprintf("lfv_%s_%s", s.id, pipelines[i].ID)

		pipelines[i].ID = pipelineName
		pipelines[i].Config = filepath.Join(sutConfigDir, pipelines[i].Config)
		if s.isOrderedPipelineSupported {
			pipelines[i].Ordered = "true"
		}
		pipelines[i].Workers = 1
	}

	// Preprocess and Save Config Files
	for _, configFile := range configFiles {
		inputCodecs, err := configFile.ReplaceInputs(s.id)
		if err != nil {
			return err
		}
		s.inputPluginCodecs = inputCodecs

		outputs, err := configFile.ReplaceOutputs()
		if err != nil {
			return err
		}

		err = configFile.Save(sutConfigDir)
		if err != nil {
			return err
		}

		outputPipelines, err := s.createOutputPipelines(outputs)
		if err != nil {
			return err
		}

		pipelines = append(pipelines, outputPipelines...)
	}

	// Reload Logstash Config
	s.pipelines = pipelines
	// err = s.logstash.ReloadPipelines(pipelines)
	err = s.logstashController.SetupTest(pipelines)
	if err != nil {
		s.log.Errorf("failed to reload Logstash config: %v", err)
	}

	return nil
}

func (s *Session) createOutputPipelines(outputs []string) ([]pipeline.Pipeline, error) {
	lfvOutputsDir := filepath.Join(s.sessionDir, "lfv_outputs")
	err := os.MkdirAll(lfvOutputsDir, 0700)
	if err != nil {
		return nil, err
	}

	pipelines := make([]pipeline.Pipeline, 0)
	for _, output := range outputs {
		pipelineName := fmt.Sprintf("lfv_output_%s", output)

		templateData := struct {
			PipelineName     string
			PipelineOrigName string
		}{
			PipelineName:     pipelineName,
			PipelineOrigName: output,
		}

		err = template.ToFile(filepath.Join(lfvOutputsDir, output+".conf"), outputPipeline, templateData, 0644)
		if err != nil {
			return nil, err
		}

		pipeline := pipeline.Pipeline{
			ID:      pipelineName,
			Config:  filepath.Join(lfvOutputsDir, output+".conf"),
			Workers: 1,
		}
		if s.isOrderedPipelineSupported {
			pipeline.Ordered = "true"
		}
		pipelines = append(pipelines, pipeline)
	}

	return pipelines, nil
}

// ExecuteTest runs a test case set against the Logstash configuration, that has
// been loaded previously with SetupTest.
func (s *Session) ExecuteTest(inputPlugin string, inputLines []string, inEvents []map[string]interface{}, expectedEvents int) error {
	s.testexec++
	pipelineName := fmt.Sprintf("lfv_input_%d", s.testexec)
	inputDir := filepath.Join(s.sessionDir, "lfv_inputs", strconv.Itoa(s.testexec))
	inputPluginName := fmt.Sprintf("%s_%s_%s", "__lfv_input", s.id, inputPlugin)
	inputCodec, ok := s.inputPluginCodecs[inputPlugin]
	if !ok {
		inputCodec = "codec => plain"
	}

	// Prepare input directory
	err := os.MkdirAll(inputDir, 0700)
	if err != nil {
		return err
	}

	fieldsFilename := filepath.Join(inputDir, "fields.json")
	err = prepareFields(fieldsFilename, inEvents)
	if err != nil {
		return err
	}

	pipelineFilename := filepath.Join(inputDir, "input.conf")
	err = createInput(pipelineFilename, fieldsFilename, inputPluginName, inputLines, inputCodec)
	if err != nil {
		return err
	}

	pipeline := pipeline.Pipeline{
		ID:      pipelineName,
		Config:  pipelineFilename,
		Workers: 1,
	}
	if s.isOrderedPipelineSupported {
		pipeline.Ordered = "true"
	}
	pipelines := append(s.pipelines, pipeline)
	err = s.logstashController.ExecuteTest(pipelines, expectedEvents)
	if err != nil {
		return err
	}

	return nil
}

func prepareFields(fieldsFilename string, inEvents []map[string]interface{}) error {
	fields := make(map[string]map[string]interface{})

	for i, event := range inEvents {
		id := fmt.Sprintf("%d", i)
		fields[id] = event
	}

	bfields, err := json.Marshal(fields)
	if err != nil {
		return err
	}
	err = ioutil.WriteFile(fieldsFilename, bfields, 0600)
	if err != nil {
		return err
	}

	return nil
}

func createInput(pipelineFilename string, fieldsFilename string, inputPluginName string, inputLines []string, inputCodec string) error {
	for i := range inputLines {
		var err error
		inputLine, err := astutil.Quote(inputLines[i], ast.DoubleQuoted)
		if err != nil {
			inputLine = astutil.QuoteWithEscape(inputLines[i], ast.SingleQuoted)
		}
		inputLines[i] = inputLine
	}

	templateData := struct {
		InputPluginName          string
		InputLines               string
		InputCodec               string
		FieldsFilename           string
		DummyEventInputIndicator string
	}{
		InputPluginName:          inputPluginName,
		InputLines:               strings.Join(inputLines, ", "),
		InputCodec:               inputCodec,
		FieldsFilename:           fieldsFilename,
		DummyEventInputIndicator: testcase.DummyEventInputIndicator,
	}
	err := template.ToFile(pipelineFilename, inputGenerator, templateData, 0600)
	if err != nil {
		return err
	}

	return nil
}

// GetResults returns the returned events from Logstash.
func (s *Session) GetResults() ([]string, error) {
	results, err := s.logstashController.GetResults()
	if err != nil {
		return nil, err
	}
	return results, nil
}

// GetStats returns the statistics for a test suite.
func (s *Session) GetStats() {
	panic("not implemented")
}

func (s *Session) teardown() error {
	// TODO: Perform a reset of the Logstash instance including Stdin Buffer, etc.
	err1 := s.logstashController.Teardown()
	var err2 error
	if !s.noCleanup {
		err2 = os.RemoveAll(s.sessionDir)
	}
	if err1 != nil || err2 != nil {
		return errors.Errorf("session teardown failed: %v, %v", err1, err2)
	}

	return nil
}
<commit_msg>Fix handling of input codecs<commit_after>package session

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/breml/logstash-config/ast"
	"github.com/breml/logstash-config/ast/astutil"
	"github.com/pkg/errors"

	"github.com/magnusbaeck/logstash-filter-verifier/v2/internal/daemon/idgen"
	"github.com/magnusbaeck/logstash-filter-verifier/v2/internal/daemon/logstashconfig"
	"github.com/magnusbaeck/logstash-filter-verifier/v2/internal/daemon/pipeline"
	"github.com/magnusbaeck/logstash-filter-verifier/v2/internal/daemon/pool"
	"github.com/magnusbaeck/logstash-filter-verifier/v2/internal/daemon/template"
	"github.com/magnusbaeck/logstash-filter-verifier/v2/internal/logging"
	"github.com/magnusbaeck/logstash-filter-verifier/v2/internal/testcase"
)

type Session struct {
	id string

	logstashController         pool.LogstashController
	isOrderedPipelineSupported bool

	baseDir    string
	sessionDir string

	pipelines         pipeline.Pipelines
	inputPluginCodecs map[string]string
	testexec          int

	noCleanup bool

	log logging.Logger
}

func newSession(baseDir string, logstashController pool.LogstashController, noCleanup bool, isOrderedPipelineSupported bool, log logging.Logger) *Session {
	sessionID := idgen.New()
	sessionDir := fmt.Sprintf("%s/session/%s", baseDir, sessionID)
	return &Session{
		id:                         sessionID,
		baseDir:                    baseDir,
		sessionDir:                 sessionDir,
		logstashController:         logstashController,
		isOrderedPipelineSupported: isOrderedPipelineSupported,
		noCleanup:                  noCleanup,
		inputPluginCodecs:          map[string]string{},
		log:                        log,
	}
}

// ID returns the id of the session.
func (s Session) ID() string {
	return s.id
}

// setupTest prepares the Logstash configuration for a new test run.
func (s *Session) setupTest(pipelines pipeline.Pipelines, configFiles []logstashconfig.File) error {
	err := os.MkdirAll(s.sessionDir, 0700)
	if err != nil {
		return err
	}

	sutConfigDir := filepath.Join(s.sessionDir, "sut")

	// adjust pipeline names and config directories to session
	for i := range pipelines {
		pipelineName := fmt.Sprintf("lfv_%s_%s", s.id, pipelines[i].ID)

		pipelines[i].ID = pipelineName
		pipelines[i].Config = filepath.Join(sutConfigDir, pipelines[i].Config)
		if s.isOrderedPipelineSupported {
			pipelines[i].Ordered = "true"
		}
		pipelines[i].Workers = 1
	}

	// Preprocess and Save Config Files
	for _, configFile := range configFiles {
		inputCodecs, err := configFile.ReplaceInputs(s.id)
		if err != nil {
			return err
		}
		for id, codec := range inputCodecs {
			s.inputPluginCodecs[id] = codec
		}

		outputs, err := configFile.ReplaceOutputs()
		if err != nil {
			return err
		}

		err = configFile.Save(sutConfigDir)
		if err != nil {
			return err
		}

		outputPipelines, err := s.createOutputPipelines(outputs)
		if err != nil {
			return err
		}

		pipelines = append(pipelines, outputPipelines...)
	}

	// Reload Logstash Config
	s.pipelines = pipelines
	// err = s.logstash.ReloadPipelines(pipelines)
	err = s.logstashController.SetupTest(pipelines)
	if err != nil {
		s.log.Errorf("failed to reload Logstash config: %v", err)
	}

	return nil
}

func (s *Session) createOutputPipelines(outputs []string) ([]pipeline.Pipeline, error) {
	lfvOutputsDir := filepath.Join(s.sessionDir, "lfv_outputs")
	err := os.MkdirAll(lfvOutputsDir, 0700)
	if err != nil {
		return nil, err
	}

	pipelines := make([]pipeline.Pipeline, 0)
	for _, output := range outputs {
		pipelineName := fmt.Sprintf("lfv_output_%s", output)

		templateData := struct {
			PipelineName     string
			PipelineOrigName string
		}{
			PipelineName:     pipelineName,
			PipelineOrigName: output,
		}

		err = template.ToFile(filepath.Join(lfvOutputsDir, output+".conf"), outputPipeline, templateData, 0644)
		if err != nil {
			return nil, err
		}

		pipeline := pipeline.Pipeline{
			ID:      pipelineName,
			Config:  filepath.Join(lfvOutputsDir, output+".conf"),
			Workers: 1,
		}
		if s.isOrderedPipelineSupported {
			pipeline.Ordered = "true"
		}
		pipelines = append(pipelines, pipeline)
	}

	return pipelines, nil
}

// ExecuteTest runs a test case set against the Logstash configuration, that has
// been loaded previously with SetupTest.
func (s *Session) ExecuteTest(inputPlugin string, inputLines []string, inEvents []map[string]interface{}, expectedEvents int) error {
	s.testexec++
	pipelineName := fmt.Sprintf("lfv_input_%d", s.testexec)
	inputDir := filepath.Join(s.sessionDir, "lfv_inputs", strconv.Itoa(s.testexec))
	inputPluginName := fmt.Sprintf("%s_%s_%s", "__lfv_input", s.id, inputPlugin)
	inputCodec, ok := s.inputPluginCodecs[inputPlugin]
	if !ok {
		inputCodec = "codec => plain"
	}

	// Prepare input directory
	err := os.MkdirAll(inputDir, 0700)
	if err != nil {
		return err
	}

	fieldsFilename := filepath.Join(inputDir, "fields.json")
	err = prepareFields(fieldsFilename, inEvents)
	if err != nil {
		return err
	}

	pipelineFilename := filepath.Join(inputDir, "input.conf")
	err = createInput(pipelineFilename, fieldsFilename, inputPluginName, inputLines, inputCodec)
	if err != nil {
		return err
	}

	pipeline := pipeline.Pipeline{
		ID:      pipelineName,
		Config:  pipelineFilename,
		Workers: 1,
	}
	if s.isOrderedPipelineSupported {
		pipeline.Ordered = "true"
	}
	pipelines := append(s.pipelines, pipeline)
	err = s.logstashController.ExecuteTest(pipelines, expectedEvents)
	if err != nil {
		return err
	}

	return nil
}

func prepareFields(fieldsFilename string, inEvents []map[string]interface{}) error {
	fields := make(map[string]map[string]interface{})

	for i, event := range inEvents {
		id := fmt.Sprintf("%d", i)
		fields[id] = event
	}

	bfields, err := json.Marshal(fields)
	if err != nil {
		return err
	}
	err = ioutil.WriteFile(fieldsFilename, bfields, 0600)
	if err != nil {
		return err
	}

	return nil
}

func createInput(pipelineFilename string, fieldsFilename string, inputPluginName string, inputLines []string, inputCodec string) error {
	for i := range inputLines {
		var err error
		inputLine, err := astutil.Quote(inputLines[i], ast.DoubleQuoted)
		if err != nil {
			inputLine = astutil.QuoteWithEscape(inputLines[i], ast.SingleQuoted)
		}
		inputLines[i] = inputLine
	}

	templateData := struct {
		InputPluginName          string
		InputLines               string
		InputCodec               string
		FieldsFilename           string
		DummyEventInputIndicator string
	}{
		InputPluginName:          inputPluginName,
		InputLines:               strings.Join(inputLines, ", "),
		InputCodec:               inputCodec,
		FieldsFilename:           fieldsFilename,
		DummyEventInputIndicator: testcase.DummyEventInputIndicator,
	}
	err := template.ToFile(pipelineFilename, inputGenerator, templateData, 0600)
	if err != nil {
		return err
	}

	return nil
}

// GetResults returns the returned events from Logstash.
func (s *Session) GetResults() ([]string, error) {
	results, err := s.logstashController.GetResults()
	if err != nil {
		return nil, err
	}
	return results, nil
}

// GetStats returns the statistics for a test suite.
func (s *Session) GetStats() {
	panic("not implemented")
}

func (s *Session) teardown() error {
	// TODO: Perform a reset of the Logstash instance including Stdin Buffer, etc.
	err1 := s.logstashController.Teardown()
	var err2 error
	if !s.noCleanup {
		err2 = os.RemoveAll(s.sessionDir)
	}
	if err1 != nil || err2 != nil {
		return errors.Errorf("session teardown failed: %v, %v", err1, err2)
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net"
	"time"
)

const (
	WIN = 100
)

func main() {

	rand.Seed(time.Now().Unix())

	l, err := net.Listen("tcp", "localhost:4000")
	if err != nil {
		log.Fatal(err)
	}

	for {
		conn, err := l.Accept()
		if err != nil {
			log.Fatal(err)
		}
		enc := json.NewEncoder(conn)
		dec := json.NewDecoder(conn)
		var player user
		conn.Write([]byte("input your name plase: {\"Name\":\"gwy\"}\n"))
		if err := dec.Decode(&player); err == io.EOF {
			enc.Encode(fmt.Sprintf("err: %v", err))
		} else if err != nil {
			enc.Encode(fmt.Sprintf("err: %v", err))
		} else {
			player.Score = 0
			player.Rwc = conn
		}
		go match(&player)
	}
}

var players = make(chan Player)

func match(p Player) {
	p.Write([]byte("Waiting for a player..."))
	//fmt.Fprint(p "Waiting for a player...")

	select {
	case players <- p:
		// now handled by the other goroutine
	case p2 := <-players:
		Play(p, p2)
	case <-time.After(5 * time.Second):
		//r, w := io.Pipe()
		w := ioutil.Discard
		c := &computer{w, "computer1", 0, 0, 0, 27}
		Play(p, c)
	}
}

func Play(a, b Player) {
	defer a.Close()
	defer b.Close()

again:
	a.Reset()
	b.Reset()
	a.Write([]byte("Game Begin\n"))
	a.Write([]byte("----------------------------\n"))
	b.Write([]byte("Game Begin\n"))
	b.Write([]byte("----------------------------\n"))
	var players = make(chan Player, 2)
	players <- a
	players <- b
	for {
		player := <-players
		a.Write([]byte(fmt.Sprintf("----------Now %s turn----------\n", player.GetName())))
		b.Write([]byte(fmt.Sprintf("----------Now %s turn----------\n", player.GetName())))
		score := player.Play()
		if score > WIN {
			a.Write([]byte(fmt.Sprintf("GAME OVER.\n %s win.\n", player.GetName())))
			a.Write([]byte(fmt.Sprintf("Play again? y/n\n")))
			b.Write([]byte(fmt.Sprintf("GAME OVER.\n %s win.\n", player.GetName())))
			b.Write([]byte(fmt.Sprintf("Play again? y/n\n")))

			var buffer []byte
			buffer = make([]byte, 2)
			_, err := a.Read(buffer)
			if err != nil {
				fmt.Println(err)
				break
			}
			if string(buffer) == "n\n" {
				break
			} else {
				players <- player
				goto again //need modfiy here
			}
		}
		a.Write([]byte(fmt.Sprintf("Player %s get %d in this turn, now total score is %d\n",
			player.GetName(), player.GetThisTurn(), player.GetScore())))
		b.Write([]byte(fmt.Sprintf("Player %s get %d in this turn, now total score is %d\n",
			player.GetName(), player.GetThisTurn(), player.GetScore())))
		players <- player
	}

}

type Player interface {
	io.ReadWriteCloser
	Play() int
	GetName() string
	GetThisTurn() int
	GetScore() int
	Reset()
}

type computer struct {
	//Rc        io.ReadCloser
	Wt        io.Writer
	Name      string
	Score     int
	ThisTurn  int
	RollScore int
	max       int
}

func (player *computer) Read(p []byte) (n int, err error) {
	return 0, nil //player.Rc.Read(p)
}
func (player *computer) Write(p []byte) (n int, err error) {
	return player.Wt.Write(p)
}
func (player *computer) Close() error {
	return nil //player.Rc.Close()
}
func (player *computer) GetName() string {
	return player.Name
}
func (player *computer) GetThisTurn() int {
	return player.ThisTurn
}
func (player *computer) GetScore() int {
	return player.Score
}
func (player *computer) Reset() {
	player.Score = 0
	player.ThisTurn = 0
	player.RollScore = 0
	return
}
func (player *computer) Play() int {
	player.ThisTurn = 0
	for player.ThisTurn < player.max {
		player.RollScore = rand.Intn(6) + 1
		player.Wt.Write([]byte(fmt.Sprintf("This turn %d, rolling... get %d.\n", player.ThisTurn, player.RollScore)))
		if player.RollScore == 1 {
			player.ThisTurn = 0
			break
		} else {
			player.ThisTurn += player.RollScore
		}
	}
	player.Score += player.ThisTurn
	player.Wt.Write([]byte(fmt.Sprintf("Finally, total score %d, this turn %d.\n", player.Score, player.ThisTurn)))
	return player.Score
}

type CmdMessage struct {
	CMD uint32 //ROLL or STAY
}

const (
	ROLL = iota
	STAY
)

type user struct {
	Rwc       io.ReadWriteCloser
	Name      string
	Score     int
	ThisTurn  int
	RollScore int
}

func (player *user) Read(p []byte) (n int, err error) {
	return player.Rwc.Read(p)
}
func (player *user) Write(p []byte) (n int, err error) {
	return player.Rwc.Write(p)
}
func (player *user) Close() error {
	return player.Rwc.Close()
}

func (player *user) GetName() string {
	return player.Name
}
func (player *user) GetThisTurn() int {
	return player.ThisTurn
}
func (player *user) GetScore() int {
	return player.Score
}
func (player *user) Reset() {
	player.Score = 0
	player.ThisTurn = 0
	player.RollScore = 0
	return
}

func (player *user) Roll() (end bool) {
	end = false
	player.RollScore = rand.Intn(6) + 1
	if player.RollScore == 1 {
		player.ThisTurn = 0
		end = true
	} else {
		player.ThisTurn += player.RollScore
	}
	return
}
func (player *user) Play() int {
	dec := json.NewDecoder(player.Rwc)
	enc := json.NewEncoder(player.Rwc)
	end := false
	player.ThisTurn = 0
	player.RollScore = 0
	for !end {
		enc.Encode(player)
		player.Rwc.Write([]byte(fmt.Sprintf("roll: {\"CMD\":0} or stay: {\"CMD\":1}?\n")))
		var msg CmdMessage
		if err := dec.Decode(&msg); err == io.EOF {
			break
		} else if err != nil {
			player.Rwc.Write([]byte(fmt.Sprintf("%v", err)))
			log.Fatal("Error when decode: ", err)
		} else {
			switch msg.CMD {
			case ROLL:
				player.Rwc.Write([]byte(fmt.Sprintf("Rolling... \n")))
				end = player.Roll()
			case STAY:
				end = true
			default:
				player.Rwc.Write([]byte(fmt.Sprintf("Invalid input, roll: {\"CMD\":0} or stay: {\"CMD\":1}?")))
			}
		}
	}
	player.Score += player.ThisTurn
	enc.Encode(player)
	return player.Score
}
<commit_msg>version 0.9.3     output notice to opponnet when playing.<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io"
	//"io/ioutil"
	"log"
	"math/rand"
	"net"
	"time"
)

const (
	WIN = 100
)

func main() {

	rand.Seed(time.Now().Unix())

	l, err := net.Listen("tcp", "localhost:4000")
	if err != nil {
		log.Fatal(err)
	}

	for {
		conn, err := l.Accept()
		if err != nil {
			log.Fatal(err)
		}
		enc := json.NewEncoder(conn)
		dec := json.NewDecoder(conn)
		var player user
		conn.Write([]byte("input your name plase: {\"Name\":\"gwy\"}\n"))
		if err := dec.Decode(&player); err == io.EOF {
			enc.Encode(fmt.Sprintf("err: %v", err))
		} else if err != nil {
			enc.Encode(fmt.Sprintf("err: %v", err))
		} else {
			player.Score = 0
			player.Rwc = conn
		}
		go match(&player)
	}
}

var players = make(chan Player)

func match(p Player) {
	p.Write([]byte("Waiting for a player...\n"))

	select {
	case players <- p:
		// now handled by the other goroutine
	case p2 := <-players:
		Play(p, p2)
	case <-time.After(5 * time.Second):
		//w := ioutil.Discard
		c := &computer{Name: "computer1", max: 27}
		Play(p, c)
	}
}

func Play(a, b Player) {
	defer a.Close()
	defer b.Close()

again:
	a.Reset()
	b.Reset()
	a.Write([]byte(fmt.Sprintf("%s VS %s, Game Begin\n", a.GetName(), b.GetName())))
	a.Write([]byte("----------------------------\n"))
	b.Write([]byte(fmt.Sprintf("%s VS %s, Game Begin\n", a.GetName(), b.GetName())))
	b.Write([]byte("----------------------------\n"))
	current, opponent := a, b
	for {
		score := current.Play(opponent)
		if score > WIN {
			a.Write([]byte(fmt.Sprintf("GAME OVER.\n %s win.\n", current.GetName())))
			a.Write([]byte(fmt.Sprintf("Play again? y/n\n")))
			b.Write([]byte(fmt.Sprintf("GAME OVER.\n %s win.\n", current.GetName())))
			b.Write([]byte(fmt.Sprintf("Play again? y/n\n")))

			var buffer []byte
			buffer = make([]byte, 2)
			_, err := a.Read(buffer)
			if err != nil {
				fmt.Println(err)
				break
			}
			if string(buffer) == "n\n" {
				break
			} else {
				goto again //need modfiy here
			}
		}
		current, opponent = opponent, current
	}

}

type Player interface {
	io.ReadWriteCloser
	Play(opponent Player) int
	GetName() string
	GetThisTurn() int
	GetScore() int
	Reset()
}

type computer struct {
	//Rc        io.ReadCloser
	//Wt        io.Writer
	Name      string
	Score     int
	ThisTurn  int
	RollScore int
	max       int
}

func (player *computer) Read(p []byte) (n int, err error) {
	return 0, nil //player.Rc.Read(p)
}
func (player *computer) Write(p []byte) (n int, err error) {
	return 9, nil //player.Wt.Write(p)
}
func (player *computer) Close() error {
	return nil //player.Rc.Close()
}
func (player *computer) GetName() string {
	return player.Name
}
func (player *computer) GetThisTurn() int {
	return player.ThisTurn
}
func (player *computer) GetScore() int {
	return player.Score
}
func (player *computer) Reset() {
	player.Score = 0
	player.ThisTurn = 0
	player.RollScore = 0
	return
}
func (player *computer) Play(opponent Player) int {
	opponent.Write([]byte(fmt.Sprintf("----------Now %s turn----------\n", player.GetName())))
	player.ThisTurn = 0
	end := false
	for player.ThisTurn < player.max && !end {
		player.RollScore = rand.Intn(6) + 1
		if player.RollScore == 1 {
			player.ThisTurn = 0
			time.Sleep(100 * time.Millisecond)
			end = true
		} else {
			player.ThisTurn += player.RollScore
		}
		opponent.Write([]byte(fmt.Sprintf("Rolling... get %d, this turn %d.\n",
			player.RollScore, player.ThisTurn)))
	}
	player.Score += player.ThisTurn
	opponent.Write([]byte(fmt.Sprintf("Finally, this turn %d, total score (%d vs %d).\n",
		player.ThisTurn, player.Score, opponent.GetScore())))
	return player.Score
}

type CmdMessage struct {
	CMD uint32 //ROLL or STAY
}

const (
	ROLL = iota
	STAY
)

type user struct {
	Rwc       io.ReadWriteCloser
	Name      string
	Score     int
	ThisTurn  int
	RollScore int
}

func (player *user) Read(p []byte) (n int, err error) {
	return player.Rwc.Read(p)
}
func (player *user) Write(p []byte) (n int, err error) {
	return player.Rwc.Write(p)
}
func (player *user) Close() error {
	return player.Rwc.Close()
}

func (player *user) GetName() string {
	return player.Name
}
func (player *user) GetThisTurn() int {
	return player.ThisTurn
}
func (player *user) GetScore() int {
	return player.Score
}
func (player *user) Reset() {
	player.Score = 0
	player.ThisTurn = 0
	player.RollScore = 0
	return
}

func (player *user) Roll() (end bool) {
	end = false
	player.RollScore = rand.Intn(6) + 1
	if player.RollScore == 1 {
		player.ThisTurn = 0
		end = true
	} else {
		player.ThisTurn += player.RollScore
		time.Sleep(100 * time.Millisecond)
	}
	return
}
func (player *user) Play(opponent Player) int {
	player.Write([]byte(fmt.Sprintf("----------Now %s turn----------\n", player.GetName())))
	opponent.Write([]byte(fmt.Sprintf("----------Now %s turn----------\n", player.GetName())))
	dec := json.NewDecoder(player)
	//enc := json.NewEncoder(player)
	player.ThisTurn = 0
	player.RollScore = 0
	end := false
	for !end {
		//enc.Encode(player)
		player.Write([]byte(fmt.Sprintf("roll: {\"CMD\":0} or stay: {\"CMD\":1}?\n")))
		var msg CmdMessage
		if err := dec.Decode(&msg); err == io.EOF {
			break
		} else if err != nil {
			player.Write([]byte(fmt.Sprintf("%v", err)))
			log.Fatal("Error when decode: ", err)
		} else {
			switch msg.CMD {
			case ROLL:
				end = player.Roll()
				player.Write([]byte(fmt.Sprintf("Rolling... get %d, this turn %d.\n",
					player.RollScore, player.ThisTurn)))
				opponent.Write([]byte(fmt.Sprintf("Rolling... get %d, this turn %d.\n",
					player.RollScore, player.ThisTurn)))
			case STAY:
				end = true
			default:
				player.Write([]byte(fmt.Sprintf("Invalid input, roll: {\"CMD\":0} or stay: {\"CMD\":1}?")))
			}
		}
	}
	player.Score += player.ThisTurn
	player.Write([]byte(fmt.Sprintf("Finally, this turn %d, total score (%d vs %d).\n",
		player.ThisTurn, player.Score, opponent.GetScore())))
	opponent.Write([]byte(fmt.Sprintf("Finally, this turn %d, total score (%d vs %d).\n",
		player.ThisTurn, player.Score, opponent.GetScore())))
	return player.Score
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"errors"
	ggio "github.com/gogo/protobuf/io"
	p2p_peer "github.com/ipfs/go-libp2p-peer"
	p2p_pstore "github.com/ipfs/go-libp2p-peerstore"
	multiaddr "github.com/jbenet/go-multiaddr"
	p2p_net "github.com/libp2p/go-libp2p/p2p/net"
	mc "github.com/mediachain/concat/mc"
	pb "github.com/mediachain/concat/proto"
	"log"
	"time"
)

var NodeOffline = errors.New("Node is offline")
var NoDirectory = errors.New("No directory server")
var UnknownPeer = errors.New("Unknown peer")
var IllegalState = errors.New("Illegal node state")

// goOffline stops the network
func (node *Node) goOffline() error {
	node.mx.Lock()
	defer node.mx.Unlock()

	switch node.status {
	case StatusPublic:
		node.dirCancel()
		fallthrough
	case StatusOnline:
		err := node.host.Close()
		node.status = StatusOffline
		log.Println("Node is offline")
		return err

	default:
		return nil
	}
}

// goOnline starts the network; if the node is already public, it stays public
func (node *Node) goOnline() error {
	node.mx.Lock()
	defer node.mx.Unlock()

	switch node.status {
	case StatusOffline:
		err := node._goOnline()
		if err != nil {
			return err
		}

		node.status = StatusOnline
		log.Println("Node is online")
		return nil

	default:
		return nil
	}
}

func (node *Node) _goOnline() error {
	host, err := mc.NewHost(node.Identity, node.laddr)
	if err != nil {
		return err
	}

	host.SetStreamHandler("/mediachain/node/ping", node.pingHandler)
	node.host = host

	return nil
}

// goPublic starts the network if it's not already up and registers with the
// directory; fails with NoDirectory if that hasn't been configured.
func (node *Node) goPublic() error {
	if node.dir == nil {
		return NoDirectory
	}

	node.mx.Lock()
	defer node.mx.Unlock()

	switch node.status {
	case StatusOffline:
		err := node._goOnline()
		if err != nil {
			return err
		}
		fallthrough

	case StatusOnline:
		ctx, cancel := context.WithCancel(context.Background())
		go node.registerPeer(ctx, node.laddr)

		node.dirCancel = cancel
		node.status = StatusPublic

		log.Println("Node is public")
		return nil

	default:
		return nil
	}
}

func (node *Node) pingHandler(s p2p_net.Stream) {
	defer s.Close()

	pid := s.Conn().RemotePeer()
	log.Printf("node/ping: new stream from %s", pid.Pretty())

	var ping pb.Ping
	var pong pb.Pong
	r := ggio.NewDelimitedReader(s, mc.MaxMessageSize)
	w := ggio.NewDelimitedWriter(s)

	for {
		err := r.ReadMsg(&ping)
		if err != nil {
			return
		}

		log.Printf("node/ping: ping from %s; ponging", pid.Pretty())

		err = w.WriteMsg(&pong)
		if err != nil {
			return
		}
	}
}

func (node *Node) registerPeer(ctx context.Context, addrs ...multiaddr.Multiaddr) {
	for {
		err := node.registerPeerImpl(ctx, addrs...)
		if err == nil {
			return
		}

		// sleep and retry
		select {
		case <-ctx.Done():
			return

		case <-time.After(5 * time.Minute):
			log.Println("Retrying to register with directory")
			continue
		}
	}
}

func (node *Node) registerPeerImpl(ctx context.Context, addrs ...multiaddr.Multiaddr) error {
	err := node.host.Connect(ctx, *node.dir)
	if err != nil {
		log.Printf("Failed to connect to directory: %s", err.Error())
		return err
	}

	s, err := node.host.NewStream(ctx, node.dir.ID, "/mediachain/dir/register")
	if err != nil {
		log.Printf("Failed to open directory stream: %s", err.Error())
		return err
	}
	defer s.Close()

	pinfo := p2p_pstore.PeerInfo{node.ID, addrs}
	var pbpi pb.PeerInfo
	mc.PBFromPeerInfo(&pbpi, pinfo)
	msg := pb.RegisterPeer{&pbpi}

	w := ggio.NewDelimitedWriter(s)
	for {
		log.Printf("Registering with directory")
		err = w.WriteMsg(&msg)
		if err != nil {
			log.Printf("Failed to register with directory: %s", err.Error())
			return err
		}

		select {
		case <-ctx.Done():
			return nil

		case <-time.After(5 * time.Minute):
			continue
		}
	}
}

func (node *Node) doPing(ctx context.Context, pid p2p_peer.ID) error {
	if node.status == StatusOffline {
		return NodeOffline
	}

	pinfo, err := node.doLookup(ctx, pid)
	if err != nil {
		return err
	}

	err = node.host.Connect(ctx, pinfo)
	if err != nil {
		return err
	}

	s, err := node.host.NewStream(ctx, pinfo.ID, "/mediachain/node/ping")
	if err != nil {
		return err
	}
	defer s.Close()

	var ping pb.Ping
	w := ggio.NewDelimitedWriter(s)
	err = w.WriteMsg(&ping)
	if err != nil {
		return err
	}

	var pong pb.Pong
	r := ggio.NewDelimitedReader(s, mc.MaxMessageSize)
	err = r.ReadMsg(&pong)

	return err
}

func (node *Node) doLookup(ctx context.Context, pid p2p_peer.ID) (empty p2p_pstore.PeerInfo, err error) {
	if node.status == StatusOffline {
		return empty, NodeOffline
	}

	if node.dir == nil {
		return empty, NoDirectory
	}

	node.host.Connect(ctx, *node.dir)
	if err != nil {
		return empty, err
	}

	s, err := node.host.NewStream(ctx, node.dir.ID, "/mediachain/dir/lookup")
	if err != nil {
		return empty, err
	}
	defer s.Close()

	req := pb.LookupPeerRequest{pid.Pretty()}
	w := ggio.NewDelimitedWriter(s)
	err = w.WriteMsg(&req)
	if err != nil {
		return empty, err
	}

	var resp pb.LookupPeerResponse
	r := ggio.NewDelimitedReader(s, mc.MaxMessageSize)
	err = r.ReadMsg(&resp)
	if err != nil {
		return empty, err
	}

	if resp.Peer == nil {
		return empty, UnknownPeer
	}

	pinfo, err := mc.PBToPeerInfo(resp.Peer)
	if err != nil {
		return empty, err
	}

	return pinfo, nil
}
<commit_msg>mcnode/proto: variable cosmetics<commit_after>package main

import (
	"context"
	"errors"
	ggio "github.com/gogo/protobuf/io"
	p2p_peer "github.com/ipfs/go-libp2p-peer"
	p2p_pstore "github.com/ipfs/go-libp2p-peerstore"
	multiaddr "github.com/jbenet/go-multiaddr"
	p2p_net "github.com/libp2p/go-libp2p/p2p/net"
	mc "github.com/mediachain/concat/mc"
	pb "github.com/mediachain/concat/proto"
	"log"
	"time"
)

var (
	NodeOffline  = errors.New("Node is offline")
	NoDirectory  = errors.New("No directory server")
	UnknownPeer  = errors.New("Unknown peer")
	IllegalState = errors.New("Illegal node state")
)

// goOffline stops the network
func (node *Node) goOffline() error {
	node.mx.Lock()
	defer node.mx.Unlock()

	switch node.status {
	case StatusPublic:
		node.dirCancel()
		fallthrough
	case StatusOnline:
		err := node.host.Close()
		node.status = StatusOffline
		log.Println("Node is offline")
		return err

	default:
		return nil
	}
}

// goOnline starts the network; if the node is already public, it stays public
func (node *Node) goOnline() error {
	node.mx.Lock()
	defer node.mx.Unlock()

	switch node.status {
	case StatusOffline:
		err := node._goOnline()
		if err != nil {
			return err
		}

		node.status = StatusOnline
		log.Println("Node is online")
		return nil

	default:
		return nil
	}
}

func (node *Node) _goOnline() error {
	host, err := mc.NewHost(node.Identity, node.laddr)
	if err != nil {
		return err
	}

	host.SetStreamHandler("/mediachain/node/ping", node.pingHandler)
	node.host = host

	return nil
}

// goPublic starts the network if it's not already up and registers with the
// directory; fails with NoDirectory if that hasn't been configured.
func (node *Node) goPublic() error {
	if node.dir == nil {
		return NoDirectory
	}

	node.mx.Lock()
	defer node.mx.Unlock()

	switch node.status {
	case StatusOffline:
		err := node._goOnline()
		if err != nil {
			return err
		}
		fallthrough

	case StatusOnline:
		ctx, cancel := context.WithCancel(context.Background())
		go node.registerPeer(ctx, node.laddr)

		node.dirCancel = cancel
		node.status = StatusPublic

		log.Println("Node is public")
		return nil

	default:
		return nil
	}
}

func (node *Node) pingHandler(s p2p_net.Stream) {
	defer s.Close()

	pid := s.Conn().RemotePeer()
	log.Printf("node/ping: new stream from %s", pid.Pretty())

	var ping pb.Ping
	var pong pb.Pong
	r := ggio.NewDelimitedReader(s, mc.MaxMessageSize)
	w := ggio.NewDelimitedWriter(s)

	for {
		err := r.ReadMsg(&ping)
		if err != nil {
			return
		}

		log.Printf("node/ping: ping from %s; ponging", pid.Pretty())

		err = w.WriteMsg(&pong)
		if err != nil {
			return
		}
	}
}

func (node *Node) registerPeer(ctx context.Context, addrs ...multiaddr.Multiaddr) {
	for {
		err := node.registerPeerImpl(ctx, addrs...)
		if err == nil {
			return
		}

		// sleep and retry
		select {
		case <-ctx.Done():
			return

		case <-time.After(5 * time.Minute):
			log.Println("Retrying to register with directory")
			continue
		}
	}
}

func (node *Node) registerPeerImpl(ctx context.Context, addrs ...multiaddr.Multiaddr) error {
	err := node.host.Connect(ctx, *node.dir)
	if err != nil {
		log.Printf("Failed to connect to directory: %s", err.Error())
		return err
	}

	s, err := node.host.NewStream(ctx, node.dir.ID, "/mediachain/dir/register")
	if err != nil {
		log.Printf("Failed to open directory stream: %s", err.Error())
		return err
	}
	defer s.Close()

	pinfo := p2p_pstore.PeerInfo{node.ID, addrs}
	var pbpi pb.PeerInfo
	mc.PBFromPeerInfo(&pbpi, pinfo)
	msg := pb.RegisterPeer{&pbpi}

	w := ggio.NewDelimitedWriter(s)
	for {
		log.Printf("Registering with directory")
		err = w.WriteMsg(&msg)
		if err != nil {
			log.Printf("Failed to register with directory: %s", err.Error())
			return err
		}

		select {
		case <-ctx.Done():
			return nil

		case <-time.After(5 * time.Minute):
			continue
		}
	}
}

func (node *Node) doPing(ctx context.Context, pid p2p_peer.ID) error {
	if node.status == StatusOffline {
		return NodeOffline
	}

	pinfo, err := node.doLookup(ctx, pid)
	if err != nil {
		return err
	}

	err = node.host.Connect(ctx, pinfo)
	if err != nil {
		return err
	}

	s, err := node.host.NewStream(ctx, pinfo.ID, "/mediachain/node/ping")
	if err != nil {
		return err
	}
	defer s.Close()

	var ping pb.Ping
	w := ggio.NewDelimitedWriter(s)
	err = w.WriteMsg(&ping)
	if err != nil {
		return err
	}

	var pong pb.Pong
	r := ggio.NewDelimitedReader(s, mc.MaxMessageSize)
	err = r.ReadMsg(&pong)

	return err
}

func (node *Node) doLookup(ctx context.Context, pid p2p_peer.ID) (empty p2p_pstore.PeerInfo, err error) {
	if node.status == StatusOffline {
		return empty, NodeOffline
	}

	if node.dir == nil {
		return empty, NoDirectory
	}

	node.host.Connect(ctx, *node.dir)
	if err != nil {
		return empty, err
	}

	s, err := node.host.NewStream(ctx, node.dir.ID, "/mediachain/dir/lookup")
	if err != nil {
		return empty, err
	}
	defer s.Close()

	req := pb.LookupPeerRequest{pid.Pretty()}
	w := ggio.NewDelimitedWriter(s)
	err = w.WriteMsg(&req)
	if err != nil {
		return empty, err
	}

	var resp pb.LookupPeerResponse
	r := ggio.NewDelimitedReader(s, mc.MaxMessageSize)
	err = r.ReadMsg(&resp)
	if err != nil {
		return empty, err
	}

	if resp.Peer == nil {
		return empty, UnknownPeer
	}

	pinfo, err := mc.PBToPeerInfo(resp.Peer)
	if err != nil {
		return empty, err
	}

	return pinfo, nil
}
<|endoftext|>


<commit_before>package main

// The Command interface must be implemented by the different command types. It is used to run the command in different
// contexts, i.e. either shell, docker, or logging. The last one comes in handy if a command's underlying actions are
// rather lengthy or cryptic, but the intent is described easily (like writing assets or files for example).
type Command interface {
	Shell() string   // Used for executing the action in a shell (locally or via ssh).
	Logging() string // Get string used for logging.
}

type DockerCommand interface {
	Docker() string // Used for executing the action in a docker context.
}

type Renderer interface {
	Render(i interface{})
}

type Validator interface {
	Validate() error
}
<commit_msg>removed Command interface from urknall binary assets<commit_after><|endoftext|>


<commit_before>package tree
/*
#cgo LDFLAGS: -lxml2
#cgo CFLAGS: -I/usr/include/libxml2
#include <libxml/tree.h>
*/
import "C"
import "unsafe"

type Element struct {
	*XmlNode
}

func (node *Element) ElementType() int {
	elem := (*C.xmlElement)(unsafe.Pointer(node.ptr()))
	return int(elem.etype)
}

func (node *Element) new(ptr *C.xmlNode) *Element {
	if ptr == nil {
		return nil
	}
	return NewNode(unsafe.Pointer(ptr), node.Doc()).(*Element)
}

func (node *Element) NextElement() *Element {
	return node.new(C.xmlNextElementSibling(node.NodePtr))
}

func (node *Element) PrevElement() *Element {
	return node.new(C.xmlPreviousElementSibling(node.NodePtr))
}

func (node *Element) FirstElement() *Element {
	return node.new(C.xmlFirstElementChild(node.NodePtr))
}

func (node *Element) LastElement() *Element {
	return node.new(C.xmlLastElementChild(node.NodePtr))
}

func (node *Element) Clear() {
	// Remember, as we delete them, the last one moves to the front
	child := node.First()
	for child != nil {
		child.Remove()
		child.Free()
		child = node.First()
	}
}

func (node *Element) Content() string {
	child := node.First()
	output := ""
	for child != nil {
		output = output + child.DumpHTML()
		child = child.Next()
	}
	return output
}

func (node *Element) SetContent(content string) {
	node.Clear()
	node.AppendContent(content)
}

func (node *Element) AppendContent(content string) {
	child := node.Doc().ParseFragment(content)
	for child != nil {
		node.AppendChildNode(child)
		child = child.Next()
	}
}

func (node *Element) PrependContent(content string) {
	child := node.Doc().ParseFragment(content).Parent().Last()
	for child != nil {
		node.PrependChildNode(child)
		child = child.Prev()
	}
}
<commit_msg>adding in before/after content methods<commit_after>package tree
/*
#cgo LDFLAGS: -lxml2
#cgo CFLAGS: -I/usr/include/libxml2
#include <libxml/tree.h>
*/
import "C"
import "unsafe"

type Element struct {
	*XmlNode
}

func (node *Element) ElementType() int {
	elem := (*C.xmlElement)(unsafe.Pointer(node.ptr()))
	return int(elem.etype)
}

func (node *Element) new(ptr *C.xmlNode) *Element {
	if ptr == nil {
		return nil
	}
	return NewNode(unsafe.Pointer(ptr), node.Doc()).(*Element)
}

func (node *Element) NextElement() *Element {
	return node.new(C.xmlNextElementSibling(node.NodePtr))
}

func (node *Element) PrevElement() *Element {
	return node.new(C.xmlPreviousElementSibling(node.NodePtr))
}

func (node *Element) FirstElement() *Element {
	return node.new(C.xmlFirstElementChild(node.NodePtr))
}

func (node *Element) LastElement() *Element {
	return node.new(C.xmlLastElementChild(node.NodePtr))
}

func (node *Element) Clear() {
	// Remember, as we delete them, the last one moves to the front
	child := node.First()
	for child != nil {
		child.Remove()
		child.Free()
		child = node.First()
	}
}

func (node *Element) Content() string {
	child := node.First()
	output := ""
	for child != nil {
		output = output + child.DumpHTML()
		child = child.Next()
	}
	return output
}

func (node *Element) SetContent(content string) {
	node.Clear()
	node.AppendContent(content)
}

func (node *Element) AppendContent(content string) {
	child := node.Doc().ParseFragment(content)
	for child != nil {
		node.AppendChildNode(child)
		child = child.Next()
	}
}

func (node *Element) PrependContent(content string) {
	child := node.Doc().ParseFragment(content).Parent().Last()
	for child != nil {
		node.PrependChildNode(child)
		child = child.Prev()
	}
}

func (node *XmlNode) AddContentAfter(content string) {
	child := node.Doc().ParseFragment(content).Parent().Last()
	for child != nil {
		node.AddNodeAfter(child)
		child = child.Prev()
	}
}
func (node *XmlNode) AddContentBefore(content string) {
	child := node.Doc().ParseFragment(content).Parent().First()
	for child != nil {
		node.AddNodeBefore(child)
		child = child.Next()
	}
}

<|endoftext|>