<commit_before>package endly_test

import (
	"github.com/stretchr/testify/assert"
	"testing"
	"github.com/viant/endly"
	"github.com/viant/toolbox/url"
	"bytes"
	"fmt"
	"github.com/viant/toolbox/storage"
	"github.com/viant/toolbox"
	"strings"

	"path"
)

const code = `
	package main
	import "fmt"

	func main() {
		fmt.Println("Hello WebDriver!\n")
	}
`

func TestSeleniumService_Start(t *testing.T) {

	var credentialFile, err = GetDummyCredential()
	var target = url.NewResource("scp://127.0.0.1:22/", credentialFile)
	assert.Nil(t, err)
	var manager = endly.NewManager()
	var useCases = []struct {
		baseDir     string
		DataURLs    []string
		DataPayload []byte
		target      *url.Resource
		request     *endly.SeleniumServerStartRequest
		Pid         int
	}{
		{
			"test/selenium/start/inactive/darwin",
			[]string{
				"https://github.com/mozilla/geckodriver/releases/download/v0.19.1/geckodriver-v0.19.1-linux64.tar.gz",
				"http://selenium-release.storage.googleapis.com/3.4/selenium-server-standalone-3.4.0.jar",
			},
			[]byte("test"),
			url.NewResource("scp://127.0.0.1:22/", credentialFile),
			&endly.SeleniumServerStartRequest{
				Target:     target,
				Sdk:        "jdk",
				SdkVersion: "1.8",
				Version:    "3.4",
				Port:       8117,
			},
			28811,
		},
		{
			"test/selenium/start/active/darwin",
			[]string{
				"https://github.com/mozilla/geckodriver/releases/download/v0.19.1/geckodriver-v0.19.1-linux64.tar.gz",
				"http://selenium-release.storage.googleapis.com/3.4/selenium-server-standalone-3.4.0.jar",
			},
			[]byte("test"),
			url.NewResource("scp://127.0.0.1:22/", credentialFile),
			&endly.SeleniumServerStartRequest{
				Target:     target,
				Sdk:        "jdk",
				SdkVersion: "1.8",
				Version:    "3.4",
				Port:       8117,
			},
			28866,
		},
	}

	for _, useCase := range useCases {
		execService, err := GetReplayService(useCase.baseDir)
		if assert.Nil(t, err) {
			context, err := OpenTestContext(manager, useCase.target, execService)
			var state = context.State()

			if len(useCase.DataURLs) > 0 {
				storageService := storage.NewMemoryService()
				state.Put(endly.UseMemoryService, true)
				for _, setupURL := range useCase.DataURLs {
					err = storageService.Upload(setupURL, bytes.NewReader(useCase.DataPayload))
				}

				assert.Nil(t, err)
			}
			service, err := context.Service(endly.SeleniumServiceID)
			if !assert.Nil(t, err) {
				break;
			}

			defer context.Close()
			if assert.Nil(t, err) {
				serviceResponse := service.Run(context, useCase.request)

				var baseCase = useCase.baseDir
				assert.Equal(t, "", serviceResponse.Error, baseCase)
				response, ok := serviceResponse.Response.(*endly.SeleniumServerStartResponse)
				if !ok {
					assert.Fail(t, fmt.Sprintf("process serviceResponse was empty %v %T", baseCase, serviceResponse.Response))
					continue
				}
				var actual = response.Pid
				assert.Equal(t, actual, useCase.Pid, "PID "+baseCase)
			}
		}
	}

}

func StartSeleniumMockServer(port int) error {
	baseDir := toolbox.CallerDirectory(3)
	var sessionPath = path.Join(baseDir, "test/selenium/http/")

	return endly.StartHTTPServer(port, &endly.HTTPServerTrips{
		IndexKeys:     []string{endly.MethodKey, endly.URLKey, endly.BodyKey, endly.ContentTypeKey},
		BaseDirectory: sessionPath,
	})
}

func TestSeleniumService_Calls(t *testing.T) {

	StartSeleniumMockServer(8116)

	manager := endly.NewManager()
	service, err := manager.Service(endly.SeleniumServiceID)
	assert.Nil(t, err)
	assert.NotNil(t, service)

	context := manager.NewContext(toolbox.NewContext())
	var targetHost = "127.0.0.1:8116"
	var target = url.NewResource(fmt.Sprintf("http://%v/", targetHost))

	serviceResponse := service.Run(context, &endly.SeleniumOpenSessionRequest{
		RemoteSelenium: target,
		Browser:        "firefox",
	})

	if assert.Equal(t, "", serviceResponse.Error) {
		response, ok := serviceResponse.Response.(*endly.SeleniumOpenSessionResponse)
		if assert.True(t, ok) {
			assert.EqualValues(t, response.SessionID, targetHost)
		}
	}

	serviceResponse = service.Run(context, &endly.SeleniumWebDriverCallRequest{
		SessionID: targetHost,
		Call: &endly.SeleniumMethodCall{
			Method:     "Get",
			Parameters: []interface{}{"http://play.golang.org/?simple=1"},
		},
	})

	if assert.Equal(t, "", serviceResponse.Error) {

		_, ok := serviceResponse.Response.(*endly.SeleniumServiceCallResponse)
		if assert.True(t, ok) {

		}
	}

	serviceResponse = service.Run(context, &endly.SeleniumWebElementCallRequest{
		SessionID: targetHost,

		Selector: &endly.WebElementSelector{
			By:    "css selector",
			Value: "#dummay",
		},
		Call: &endly.SeleniumMethodCall{
			Method:     "Clear",
			Parameters: []interface{}{},
		},
	})
	assert.Equal(t, "failed to call web element: failed to lookup element: css selector #dummay", serviceResponse.Error)

	serviceResponse = service.Run(context, &endly.SeleniumWebElementCallRequest{
		SessionID: targetHost,

		Selector: &endly.WebElementSelector{
			By:    "css selector",
			Value: "#code",
		},
		Call: &endly.SeleniumMethodCall{
			Method:     "Clear",
			Parameters: []interface{}{},
		},
	})

	if assert.Equal(t, "", serviceResponse.Error) {
		_, ok := serviceResponse.Response.(*endly.SeleniumWebElementCallResponse)
		if assert.True(t, ok) {

		}
	}

	serviceResponse = service.Run(context, &endly.SeleniumWebElementCallRequest{
		SessionID: targetHost,

		Selector: &endly.WebElementSelector{
			By:    "css selector",
			Value: "#code",
		},
		Call: &endly.SeleniumMethodCall{
			Method: "SendKeys",
			Parameters: []interface{}{
				code,
			},
		},
	})
	if assert.Equal(t, "", serviceResponse.Error) {
		_, ok := serviceResponse.Response.(*endly.SeleniumWebElementCallResponse)
		if assert.True(t, ok) {

		}
	}

	serviceResponse = service.Run(context, &endly.SeleniumWebElementCallRequest{
		SessionID: targetHost,
		Call: &endly.SeleniumMethodCall{
			Method:     "Click",
			Parameters: []interface{}{},
			Wait:       &endly.SeleniumWait{SleepInMs: 1,},
		},
		Selector: &endly.WebElementSelector{
			By:    "css selector",
			Value: "#run",
		},
	})
	if assert.Equal(t, "", serviceResponse.Error) {
		_, ok := serviceResponse.Response.(*endly.SeleniumWebElementCallResponse)
		if assert.True(t, ok) {

		}
	}

	serviceResponse = service.Run(context, &endly.SeleniumWebElementCallRequest{
		SessionID: targetHost,

		Selector: &endly.WebElementSelector{
			By:    "css selector",
			Value: "#output",
		},
		Call: &endly.SeleniumMethodCall{
			Method:     "Text",
			Parameters: []interface{}{},
			Wait: &endly.SeleniumWait{
				Repeat:       20,
				SleepInMs:    100,
				ExitCriteria: "$value:/WebDriver/",
			},
		},
	})
	if assert.Equal(t, "", serviceResponse.Error) {
		callResponse, ok := serviceResponse.Response.(*endly.SeleniumWebElementCallResponse)
		if assert.True(t, ok) {
			assert.True(t, strings.Contains(toolbox.AsString(callResponse.Result[0]), "Hello WebDriver!"))
		}
	}

	serviceResponse = service.Run(context, &endly.SeleniumWebDriverCallRequest{
		SessionID: targetHost,
		Call: &endly.SeleniumMethodCall{
			Method:     "Close",
			Parameters: []interface{}{},
		},
	})

}

func TestSeleniumService_Run(t *testing.T) {

	StartSeleniumMockServer(8118)

	manager := endly.NewManager()
	service, err := manager.Service(endly.SeleniumServiceID)
	assert.Nil(t, err)
	assert.NotNil(t, service)

	context := manager.NewContext(toolbox.NewContext())
	var targetHost = "127.0.0.1:8118"
	var target = url.NewResource(fmt.Sprintf("http://%v/", targetHost))

	serviceResponse := service.Run(context, &endly.SeleniumRunRequest{
		RemoteSelenium: target,
		Browser:        "firefox",
		Actions:[]*endly.SeleniumAction{
			{
				Calls:[]*endly.SeleniumMethodCall{
					endly.NewSeleniumMethodCall("Get", nil, "http://play.golang.org/?simple=1"),
				},
			},
			{
				Selector:endly.NewWebElementSelector("", "#code"),
				Calls:[]*endly.SeleniumMethodCall{
					endly.NewSeleniumMethodCall("Clear", nil),
				},
			},
			{
				Selector:endly.NewWebElementSelector("", "#code"),
				Calls:[]*endly.SeleniumMethodCall{
					endly.NewSeleniumMethodCall("SendKeys", nil, code),
				},
			},
			{
				Selector:endly.NewWebElementSelector("", "#run"),
				Calls:[]*endly.SeleniumMethodCall{
					endly.NewSeleniumMethodCall("Click", nil),
				},
			},
			{
				Selector:endly.NewWebElementSelector("", "#output"),
				Calls:[]*endly.SeleniumMethodCall{
					endly.NewSeleniumMethodCall("Text",  &endly.SeleniumWait{
						Repeat:       20,
						SleepInMs:    100,
						ExitCriteria: "$value:/WebDriver/",
					}),
				},
			},
		},
	})
	if assert.Equal(t, "", serviceResponse.Error) {

		runResponse, ok := serviceResponse.Response.(*endly.SeleniumRunResponse)
		if assert.True(t, ok) {
			output, ok := runResponse.Data["#output"];
			if assert.True(t, ok) {
				ouputMap := toolbox.AsMap(output)
				assert.EqualValues(t, "Hello WebDriver!\n\n\nProgram exited.", ouputMap["Text"])
			}

		}
	}

	serviceResponse = service.Run(context, &endly.SeleniumCloseSessionRequest{
		SessionID: targetHost,
	})

}

<commit_msg>patched selenium test<commit_after>package endly_test

import (
	"github.com/stretchr/testify/assert"
	"testing"
	"github.com/viant/endly"
	"github.com/viant/toolbox/url"
	"bytes"
	"fmt"
	"github.com/viant/toolbox/storage"
	"github.com/viant/toolbox"
	"strings"

	"path"
)

const code = `
	package main
	import "fmt"

	func main() {
		fmt.Println("Hello WebDriver!\n")
	}
`

func TestSeleniumService_Start(t *testing.T) {

	var credentialFile, err = GetDummyCredential()
	var target = url.NewResource("scp://127.0.0.1:22/", credentialFile)
	assert.Nil(t, err)
	var manager = endly.NewManager()
	var useCases = []struct {
		baseDir     string
		DataURLs    []string
		DataPayload []byte
		target      *url.Resource
		request     *endly.SeleniumServerStartRequest
		Pid         int
	}{
		{
			"test/selenium/start/inactive/darwin",
			[]string{
				"https://github.com/mozilla/geckodriver/releases/download/v0.19.1/geckodriver-v0.19.1-linux64.tar.gz",
				"http://selenium-release.storage.googleapis.com/3.4/selenium-server-standalone-3.4.0.jar",
			},
			[]byte("test"),
			url.NewResource("scp://127.0.0.1:22/", credentialFile),
			&endly.SeleniumServerStartRequest{
				Target:     target,
				Sdk:        "jdk",
				SdkVersion: "1.8",
				Version:    "3.4",
				Port:       8117,
			},
			28811,
		},
		{
			"test/selenium/start/active/darwin",
			[]string{
				"https://github.com/mozilla/geckodriver/releases/download/v0.19.1/geckodriver-v0.19.1-linux64.tar.gz",
				"http://selenium-release.storage.googleapis.com/3.4/selenium-server-standalone-3.4.0.jar",
			},
			[]byte("test"),
			url.NewResource("scp://127.0.0.1:22/", credentialFile),
			&endly.SeleniumServerStartRequest{
				Target:     target,
				Sdk:        "jdk",
				SdkVersion: "1.8",
				Version:    "3.4",
				Port:       8117,
			},
			28866,
		},
	}

	for _, useCase := range useCases {
		execService, err := GetReplayService(useCase.baseDir)
		if assert.Nil(t, err) {
			context, err := OpenTestContext(manager, useCase.target, execService)
			var state = context.State()

			if len(useCase.DataURLs) > 0 {
				storageService := storage.NewMemoryService()
				state.Put(endly.UseMemoryService, true)
				for _, setupURL := range useCase.DataURLs {
					err = storageService.Upload(setupURL, bytes.NewReader(useCase.DataPayload))
				}

				assert.Nil(t, err)
			}
			service, err := context.Service(endly.SeleniumServiceID)
			if !assert.Nil(t, err) {
				break;
			}

			defer context.Close()
			if assert.Nil(t, err) {
				serviceResponse := service.Run(context, useCase.request)

				var baseCase = useCase.baseDir
				assert.Equal(t, "", serviceResponse.Error, baseCase)
				response, ok := serviceResponse.Response.(*endly.SeleniumServerStartResponse)
				if !ok {
					assert.Fail(t, fmt.Sprintf("process serviceResponse was empty %v %T", baseCase, serviceResponse.Response))
					continue
				}
				var actual = response.Pid
				assert.Equal(t, actual, useCase.Pid, "PID "+baseCase)
			}
		}
	}

}

func StartSeleniumMockServer(port int) error {
	baseDir := toolbox.CallerDirectory(3)
	var sessionPath = path.Join(baseDir, "test/selenium/http/")

	return endly.StartHTTPServer(port, &endly.HTTPServerTrips{
		IndexKeys:     []string{endly.MethodKey, endly.URLKey, endly.BodyKey, endly.ContentTypeKey},
		BaseDirectory: sessionPath,
	})
}

func TestSeleniumService_Calls(t *testing.T) {

	StartSeleniumMockServer(8116)

	manager := endly.NewManager()
	service, err := manager.Service(endly.SeleniumServiceID)
	assert.Nil(t, err)
	assert.NotNil(t, service)

	context := manager.NewContext(toolbox.NewContext())
	var targetHost = "127.0.0.1:8116"
	var target = url.NewResource(fmt.Sprintf("http://%v/", targetHost))

	serviceResponse := service.Run(context, &endly.SeleniumOpenSessionRequest{
		RemoteSelenium: target,
		Browser:        "firefox",
	})

	if assert.Equal(t, "", serviceResponse.Error) {
		response, ok := serviceResponse.Response.(*endly.SeleniumOpenSessionResponse)
		if assert.True(t, ok) {
			assert.EqualValues(t, response.SessionID, targetHost)
		}
	}

	serviceResponse = service.Run(context, &endly.SeleniumWebDriverCallRequest{
		SessionID: targetHost,
		Call: &endly.SeleniumMethodCall{
			Method:     "Get",
			Parameters: []interface{}{"http://play.golang.org/?simple=1"},
		},
	})

	if assert.Equal(t, "", serviceResponse.Error) {

		_, ok := serviceResponse.Response.(*endly.SeleniumServiceCallResponse)
		if assert.True(t, ok) {

		}
	}

	serviceResponse = service.Run(context, &endly.SeleniumWebElementCallRequest{
		SessionID: targetHost,

		Selector: &endly.WebElementSelector{
			By:    "css selector",
			Value: "#dummay",
		},
		Call: &endly.SeleniumMethodCall{
			Method:     "Clear",
			Parameters: []interface{}{},
		},
	})
	response, ok := serviceResponse.Response.(*endly.SeleniumWebElementCallResponse)
	if assert.True(t, ok ) {
		assert.Equal(t, "failed to lookup element: css selector #dummay", response.LookupError)
	}
	serviceResponse = service.Run(context, &endly.SeleniumWebElementCallRequest{
		SessionID: targetHost,

		Selector: &endly.WebElementSelector{
			By:    "css selector",
			Value: "#code",
		},
		Call: &endly.SeleniumMethodCall{
			Method:     "Clear",
			Parameters: []interface{}{},
		},
	})

	if assert.Equal(t, "", serviceResponse.Error) {
		_, ok := serviceResponse.Response.(*endly.SeleniumWebElementCallResponse)
		if assert.True(t, ok) {

		}
	}

	serviceResponse = service.Run(context, &endly.SeleniumWebElementCallRequest{
		SessionID: targetHost,

		Selector: &endly.WebElementSelector{
			By:    "css selector",
			Value: "#code",
		},
		Call: &endly.SeleniumMethodCall{
			Method: "SendKeys",
			Parameters: []interface{}{
				code,
			},
		},
	})
	if assert.Equal(t, "", serviceResponse.Error) {
		_, ok := serviceResponse.Response.(*endly.SeleniumWebElementCallResponse)
		if assert.True(t, ok) {

		}
	}

	serviceResponse = service.Run(context, &endly.SeleniumWebElementCallRequest{
		SessionID: targetHost,
		Call: &endly.SeleniumMethodCall{
			Method:     "Click",
			Parameters: []interface{}{},
			Wait:       &endly.SeleniumWait{SleepInMs: 1,},
		},
		Selector: &endly.WebElementSelector{
			By:    "css selector",
			Value: "#run",
		},
	})
	if assert.Equal(t, "", serviceResponse.Error) {
		_, ok := serviceResponse.Response.(*endly.SeleniumWebElementCallResponse)
		if assert.True(t, ok) {

		}
	}

	serviceResponse = service.Run(context, &endly.SeleniumWebElementCallRequest{
		SessionID: targetHost,

		Selector: &endly.WebElementSelector{
			By:    "css selector",
			Value: "#output",
		},
		Call: &endly.SeleniumMethodCall{
			Method:     "Text",
			Parameters: []interface{}{},
			Wait: &endly.SeleniumWait{
				Repeat:       20,
				SleepInMs:    100,
				ExitCriteria: "$value:/WebDriver/",
			},
		},
	})
	if assert.Equal(t, "", serviceResponse.Error) {
		callResponse, ok := serviceResponse.Response.(*endly.SeleniumWebElementCallResponse)
		if assert.True(t, ok) {
			assert.True(t, strings.Contains(toolbox.AsString(callResponse.Result[0]), "Hello WebDriver!"))
		}
	}

	serviceResponse = service.Run(context, &endly.SeleniumWebDriverCallRequest{
		SessionID: targetHost,
		Call: &endly.SeleniumMethodCall{
			Method:     "Close",
			Parameters: []interface{}{},
		},
	})

}

func TestSeleniumService_Run(t *testing.T) {

	StartSeleniumMockServer(8118)

	manager := endly.NewManager()
	service, err := manager.Service(endly.SeleniumServiceID)
	assert.Nil(t, err)
	assert.NotNil(t, service)

	context := manager.NewContext(toolbox.NewContext())
	var targetHost = "127.0.0.1:8118"
	var target = url.NewResource(fmt.Sprintf("http://%v/", targetHost))

	serviceResponse := service.Run(context, &endly.SeleniumRunRequest{
		RemoteSelenium: target,
		Browser:        "firefox",
		Actions:[]*endly.SeleniumAction{
			{
				Calls:[]*endly.SeleniumMethodCall{
					endly.NewSeleniumMethodCall("Get", nil, "http://play.golang.org/?simple=1"),
				},
			},
			{
				Selector:endly.NewWebElementSelector("", "#code"),
				Calls:[]*endly.SeleniumMethodCall{
					endly.NewSeleniumMethodCall("Clear", nil),
				},
			},
			{
				Selector:endly.NewWebElementSelector("", "#code"),
				Calls:[]*endly.SeleniumMethodCall{
					endly.NewSeleniumMethodCall("SendKeys", nil, code),
				},
			},
			{
				Selector:endly.NewWebElementSelector("", "#run"),
				Calls:[]*endly.SeleniumMethodCall{
					endly.NewSeleniumMethodCall("Click", nil),
				},
			},
			{
				Selector:endly.NewWebElementSelector("", "#output"),
				Calls:[]*endly.SeleniumMethodCall{
					endly.NewSeleniumMethodCall("Text",  &endly.SeleniumWait{
						Repeat:       20,
						SleepInMs:    100,
						ExitCriteria: "$value:/WebDriver/",
					}),
				},
			},
		},
	})
	if assert.Equal(t, "", serviceResponse.Error) {

		runResponse, ok := serviceResponse.Response.(*endly.SeleniumRunResponse)
		if assert.True(t, ok) {
			output, ok := runResponse.Data["#output"];
			if assert.True(t, ok) {
				ouputMap := toolbox.AsMap(output)
				assert.EqualValues(t, "Hello WebDriver!\n\n\nProgram exited.", ouputMap["Text"])
			}

		}
	}

	serviceResponse = service.Run(context, &endly.SeleniumCloseSessionRequest{
		SessionID: targetHost,
	})

}

<|endoftext|>


<commit_before>package meta

import (
	"compress/gzip"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/gogo/protobuf/proto"
	"github.com/hashicorp/raft"
	"github.com/influxdb/influxdb/services/meta/internal"
	"github.com/influxdb/influxdb/uuid"
)

// handler represents an HTTP handler for the meta service.
type handler struct {
	config  *Config
	Version string

	logger         *log.Logger
	loggingEnabled bool // Log every HTTP access.
	pprofEnabled   bool
	store          interface {
		afterIndex(index uint64) <-chan struct{}
		index() uint64
		leader() string
		leaderHTTP() string
		snapshot() (*Data, error)
		apply(b []byte) error
		join(n *NodeInfo) error
	}
	s *Service

	mu      sync.RWMutex
	closing chan struct{}
}

// newHandler returns a new instance of handler with routes.
func newHandler(c *Config, s *Service) *handler {
	h := &handler{
		s:              s,
		config:         c,
		logger:         log.New(os.Stderr, "[meta-http] ", log.LstdFlags),
		loggingEnabled: c.LoggingEnabled,
		closing:        make(chan struct{}),
	}

	return h
}

// SetRoutes sets the provided routes on the handler.
func (h *handler) WrapHandler(name string, hf http.HandlerFunc) http.Handler {
	var handler http.Handler
	handler = http.HandlerFunc(hf)
	handler = gzipFilter(handler)
	handler = versionHeader(handler, h)
	handler = requestID(handler)
	if h.loggingEnabled {
		handler = logging(handler, name, h.logger)
	}
	handler = recovery(handler, name, h.logger) // make sure recovery is always last

	return handler
}

// ServeHTTP responds to HTTP request to the handler.
func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "HEAD":
		h.WrapHandler("ping", h.servePing).ServeHTTP(w, r)
	case "GET":
		h.WrapHandler("snapshot", h.serveSnapshot).ServeHTTP(w, r)
	case "POST":
		h.WrapHandler("execute", h.serveExec).ServeHTTP(w, r)
	default:
		http.Error(w, "", http.StatusBadRequest)
	}
}

func (h *handler) Close() error {
	h.mu.Lock()
	defer h.mu.Unlock()
	select {
	case <-h.closing:
		// do nothing here
	default:
		close(h.closing)
	}
	return nil
}

func (h *handler) isClosed() error {
	h.mu.RLock()
	defer h.mu.RUnlock()
	select {
	case <-h.closing:
		return fmt.Errorf("server closed")
	default:
		return nil
	}
}

// serveExec executes the requested command.
func (h *handler) serveExec(w http.ResponseWriter, r *http.Request) {
	if err := h.isClosed(); err != nil {
		h.httpError(err, w, http.StatusInternalServerError)
		return
	}

	// Read the command from the request body.
	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		h.httpError(err, w, http.StatusInternalServerError)
		return
	}

	if r.URL.Path == "/join" {
		n := &NodeInfo{}
		if err := json.Unmarshal(body, n); err != nil {
			h.httpError(err, w, http.StatusInternalServerError)
			return
		}
		err := h.store.join(n)
		if err == raft.ErrNotLeader {
			l := h.store.leaderHTTP()
			if l == "" {
				// No cluster leader. Client will have to try again later.
				h.httpError(errors.New("no leader"), w, http.StatusServiceUnavailable)
				return
			}
			scheme := "http://"
			if h.config.HTTPSEnabled {
				scheme = "https://"
			}

			l = scheme + l + "/join"
			http.Redirect(w, r, l, http.StatusTemporaryRedirect)
			return
		}

		if err != nil {
			h.httpError(err, w, http.StatusInternalServerError)
			return
		}

		return
	}

	// Make sure it's a valid command.
	if err := validateCommand(body); err != nil {
		h.httpError(err, w, http.StatusBadRequest)
		return
	}

	// Apply the command to the store.
	var resp *internal.Response
	if err := h.store.apply(body); err != nil {
		// If we aren't the leader, redirect client to the leader.
		if err == raft.ErrNotLeader {
			l := h.store.leaderHTTP()
			if l == "" {
				// No cluster leader. Client will have to try again later.
				h.httpError(errors.New("no leader"), w, http.StatusServiceUnavailable)
				return
			}
			scheme := "http://"
			if h.config.HTTPSEnabled {
				scheme = "https://"
			}

			l = scheme + l + "/execute"
			http.Redirect(w, r, l, http.StatusTemporaryRedirect)
			return
		}

		// Error wasn't a leadership error so pass it back to client.
		resp = &internal.Response{
			OK:    proto.Bool(false),
			Error: proto.String(err.Error()),
		}
	} else {
		// Apply was successful. Return the new store index to the client.
		resp = &internal.Response{
			OK:    proto.Bool(false),
			Index: proto.Uint64(h.store.index()),
		}
	}

	// Marshal the response.
	b, err := proto.Marshal(resp)
	if err != nil {
		h.httpError(err, w, http.StatusInternalServerError)
		return
	}

	// Send response to client.
	w.Header().Add("Content-Type", "application/octet-stream")
	w.Write(b)
}

func validateCommand(b []byte) error {
	// Ensure command can be deserialized before applying.
	if err := proto.Unmarshal(b, &internal.Command{}); err != nil {
		return fmt.Errorf("unable to unmarshal command: %s", err)
	}

	return nil
}

// serveSnapshot is a long polling http connection to server cache updates
func (h *handler) serveSnapshot(w http.ResponseWriter, r *http.Request) {
	if err := h.isClosed(); err != nil {
		h.httpError(err, w, http.StatusInternalServerError)
		return
	}

	// get the current index that client has
	index, err := strconv.ParseUint(r.URL.Query().Get("index"), 10, 64)
	if err != nil {
		http.Error(w, "error parsing index", http.StatusBadRequest)
	}

	select {
	case <-h.store.afterIndex(index):
		// Send updated snapshot to client.
		ss, err := h.store.snapshot()
		if err != nil {
			h.httpError(err, w, http.StatusInternalServerError)
			return
		}
		b, err := ss.MarshalBinary()
		if err != nil {
			h.httpError(err, w, http.StatusInternalServerError)
			return
		}
		w.Write(b)
		return
	case <-w.(http.CloseNotifier).CloseNotify():
		// Client closed the connection so we're done.
		return
	case <-h.closing:
		h.httpError(fmt.Errorf("server closed"), w, http.StatusInternalServerError)
		return
	}
}

// servePing returns a simple response to let the client know the server is running.
func (h *handler) servePing(w http.ResponseWriter, r *http.Request) {
	w.Write([]byte("ACK"))
}

type gzipResponseWriter struct {
	io.Writer
	http.ResponseWriter
}

func (w gzipResponseWriter) Write(b []byte) (int, error) {
	return w.Writer.Write(b)
}

func (w gzipResponseWriter) Flush() {
	w.Writer.(*gzip.Writer).Flush()
}

func (w gzipResponseWriter) CloseNotify() <-chan bool {
	return w.ResponseWriter.(http.CloseNotifier).CloseNotify()
}

// determines if the client can accept compressed responses, and encodes accordingly
func gzipFilter(inner http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
			inner.ServeHTTP(w, r)
			return
		}
		w.Header().Set("Content-Encoding", "gzip")
		gz := gzip.NewWriter(w)
		defer gz.Close()
		gzw := gzipResponseWriter{Writer: gz, ResponseWriter: w}
		inner.ServeHTTP(gzw, r)
	})
}

// versionHeader takes a HTTP handler and returns a HTTP handler
// and adds the X-INFLUXBD-VERSION header to outgoing responses.
func versionHeader(inner http.Handler, h *handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Add("X-InfluxDB-Version", h.Version)
		inner.ServeHTTP(w, r)
	})
}

func requestID(inner http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		uid := uuid.TimeUUID()
		r.Header.Set("Request-Id", uid.String())
		w.Header().Set("Request-Id", r.Header.Get("Request-Id"))

		inner.ServeHTTP(w, r)
	})
}

func logging(inner http.Handler, name string, weblog *log.Logger) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		start := time.Now()
		l := &responseLogger{w: w}
		inner.ServeHTTP(l, r)
		logLine := buildLogLine(l, r, start)
		weblog.Println(logLine)
	})
}

func recovery(inner http.Handler, name string, weblog *log.Logger) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		start := time.Now()
		l := &responseLogger{w: w}

		defer func() {
			if err := recover(); err != nil {
				logLine := buildLogLine(l, r, start)
				logLine = fmt.Sprintf(`%s [panic:%s]`, logLine, err)
				weblog.Println(logLine)
			}
		}()

		inner.ServeHTTP(l, r)
	})
}

func (h *handler) httpError(err error, w http.ResponseWriter, status int) {
	if h.loggingEnabled {
		h.logger.Println(err)
	}
	http.Error(w, "", status)
}
<commit_msg>Update close handling on meta service<commit_after>package meta

import (
	"compress/gzip"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/gogo/protobuf/proto"
	"github.com/hashicorp/raft"
	"github.com/influxdb/influxdb/services/meta/internal"
	"github.com/influxdb/influxdb/uuid"
)

// handler represents an HTTP handler for the meta service.
type handler struct {
	config  *Config
	Version string

	logger         *log.Logger
	loggingEnabled bool // Log every HTTP access.
	pprofEnabled   bool
	store          interface {
		afterIndex(index uint64) <-chan struct{}
		index() uint64
		leader() string
		leaderHTTP() string
		snapshot() (*Data, error)
		apply(b []byte) error
		join(n *NodeInfo) error
	}
	s *Service

	mu      sync.RWMutex
	closing chan struct{}
}

// newHandler returns a new instance of handler with routes.
func newHandler(c *Config, s *Service) *handler {
	h := &handler{
		s:              s,
		config:         c,
		logger:         log.New(os.Stderr, "[meta-http] ", log.LstdFlags),
		loggingEnabled: c.LoggingEnabled,
		closing:        make(chan struct{}),
	}

	return h
}

// SetRoutes sets the provided routes on the handler.
func (h *handler) WrapHandler(name string, hf http.HandlerFunc) http.Handler {
	var handler http.Handler
	handler = http.HandlerFunc(hf)
	handler = gzipFilter(handler)
	handler = versionHeader(handler, h)
	handler = requestID(handler)
	if h.loggingEnabled {
		handler = logging(handler, name, h.logger)
	}
	handler = recovery(handler, name, h.logger) // make sure recovery is always last

	return handler
}

// ServeHTTP responds to HTTP request to the handler.
func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "HEAD":
		h.WrapHandler("ping", h.servePing).ServeHTTP(w, r)
	case "GET":
		h.WrapHandler("snapshot", h.serveSnapshot).ServeHTTP(w, r)
	case "POST":
		h.WrapHandler("execute", h.serveExec).ServeHTTP(w, r)
	default:
		http.Error(w, "", http.StatusBadRequest)
	}
}

func (h *handler) Close() error {
	h.mu.Lock()
	defer h.mu.Unlock()
	select {
	case <-h.closing:
		// do nothing here
	default:
		close(h.closing)
	}
	return nil
}

func (h *handler) isClosed() bool {
	h.mu.RLock()
	defer h.mu.RUnlock()
	select {
	case <-h.closing:
		return true
	default:
		return false
	}
}

// serveExec executes the requested command.
func (h *handler) serveExec(w http.ResponseWriter, r *http.Request) {
	if h.isClosed() {
		h.httpError(fmt.Errorf("server closed"), w, http.StatusInternalServerError)
		return
	}

	// Read the command from the request body.
	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		h.httpError(err, w, http.StatusInternalServerError)
		return
	}

	if r.URL.Path == "/join" {
		n := &NodeInfo{}
		if err := json.Unmarshal(body, n); err != nil {
			h.httpError(err, w, http.StatusInternalServerError)
			return
		}
		err := h.store.join(n)
		if err == raft.ErrNotLeader {
			l := h.store.leaderHTTP()
			if l == "" {
				// No cluster leader. Client will have to try again later.
				h.httpError(errors.New("no leader"), w, http.StatusServiceUnavailable)
				return
			}
			scheme := "http://"
			if h.config.HTTPSEnabled {
				scheme = "https://"
			}

			l = scheme + l + "/join"
			http.Redirect(w, r, l, http.StatusTemporaryRedirect)
			return
		}

		if err != nil {
			h.httpError(err, w, http.StatusInternalServerError)
			return
		}

		return
	}

	// Make sure it's a valid command.
	if err := validateCommand(body); err != nil {
		h.httpError(err, w, http.StatusBadRequest)
		return
	}

	// Apply the command to the store.
	var resp *internal.Response
	if err := h.store.apply(body); err != nil {
		// If we aren't the leader, redirect client to the leader.
		if err == raft.ErrNotLeader {
			l := h.store.leaderHTTP()
			if l == "" {
				// No cluster leader. Client will have to try again later.
				h.httpError(errors.New("no leader"), w, http.StatusServiceUnavailable)
				return
			}
			scheme := "http://"
			if h.config.HTTPSEnabled {
				scheme = "https://"
			}

			l = scheme + l + "/execute"
			http.Redirect(w, r, l, http.StatusTemporaryRedirect)
			return
		}

		// Error wasn't a leadership error so pass it back to client.
		resp = &internal.Response{
			OK:    proto.Bool(false),
			Error: proto.String(err.Error()),
		}
	} else {
		// Apply was successful. Return the new store index to the client.
		resp = &internal.Response{
			OK:    proto.Bool(false),
			Index: proto.Uint64(h.store.index()),
		}
	}

	// Marshal the response.
	b, err := proto.Marshal(resp)
	if err != nil {
		h.httpError(err, w, http.StatusInternalServerError)
		return
	}

	// Send response to client.
	w.Header().Add("Content-Type", "application/octet-stream")
	w.Write(b)
}

func validateCommand(b []byte) error {
	// Ensure command can be deserialized before applying.
	if err := proto.Unmarshal(b, &internal.Command{}); err != nil {
		return fmt.Errorf("unable to unmarshal command: %s", err)
	}

	return nil
}

// serveSnapshot is a long polling http connection to server cache updates
func (h *handler) serveSnapshot(w http.ResponseWriter, r *http.Request) {
	if h.isClosed() {
		h.httpError(fmt.Errorf("server closed"), w, http.StatusInternalServerError)
		return
	}

	// get the current index that client has
	index, err := strconv.ParseUint(r.URL.Query().Get("index"), 10, 64)
	if err != nil {
		http.Error(w, "error parsing index", http.StatusBadRequest)
	}

	select {
	case <-h.store.afterIndex(index):
		// Send updated snapshot to client.
		ss, err := h.store.snapshot()
		if err != nil {
			h.httpError(err, w, http.StatusInternalServerError)
			return
		}
		b, err := ss.MarshalBinary()
		if err != nil {
			h.httpError(err, w, http.StatusInternalServerError)
			return
		}
		w.Write(b)
		return
	case <-w.(http.CloseNotifier).CloseNotify():
		// Client closed the connection so we're done.
		return
	case <-h.closing:
		h.httpError(fmt.Errorf("server closed"), w, http.StatusInternalServerError)
		return
	}
}

// servePing returns a simple response to let the client know the server is running.
func (h *handler) servePing(w http.ResponseWriter, r *http.Request) {
	w.Write([]byte("ACK"))
}

type gzipResponseWriter struct {
	io.Writer
	http.ResponseWriter
}

func (w gzipResponseWriter) Write(b []byte) (int, error) {
	return w.Writer.Write(b)
}

func (w gzipResponseWriter) Flush() {
	w.Writer.(*gzip.Writer).Flush()
}

func (w gzipResponseWriter) CloseNotify() <-chan bool {
	return w.ResponseWriter.(http.CloseNotifier).CloseNotify()
}

// determines if the client can accept compressed responses, and encodes accordingly
func gzipFilter(inner http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
			inner.ServeHTTP(w, r)
			return
		}
		w.Header().Set("Content-Encoding", "gzip")
		gz := gzip.NewWriter(w)
		defer gz.Close()
		gzw := gzipResponseWriter{Writer: gz, ResponseWriter: w}
		inner.ServeHTTP(gzw, r)
	})
}

// versionHeader takes a HTTP handler and returns a HTTP handler
// and adds the X-INFLUXBD-VERSION header to outgoing responses.
func versionHeader(inner http.Handler, h *handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Add("X-InfluxDB-Version", h.Version)
		inner.ServeHTTP(w, r)
	})
}

func requestID(inner http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		uid := uuid.TimeUUID()
		r.Header.Set("Request-Id", uid.String())
		w.Header().Set("Request-Id", r.Header.Get("Request-Id"))

		inner.ServeHTTP(w, r)
	})
}

func logging(inner http.Handler, name string, weblog *log.Logger) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		start := time.Now()
		l := &responseLogger{w: w}
		inner.ServeHTTP(l, r)
		logLine := buildLogLine(l, r, start)
		weblog.Println(logLine)
	})
}

func recovery(inner http.Handler, name string, weblog *log.Logger) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		start := time.Now()
		l := &responseLogger{w: w}

		defer func() {
			if err := recover(); err != nil {
				logLine := buildLogLine(l, r, start)
				logLine = fmt.Sprintf(`%s [panic:%s]`, logLine, err)
				weblog.Println(logLine)
			}
		}()

		inner.ServeHTTP(l, r)
	})
}

func (h *handler) httpError(err error, w http.ResponseWriter, status int) {
	if h.loggingEnabled {
		h.logger.Println(err)
	}
	http.Error(w, "", status)
}
<|endoftext|>


<commit_before>package cfclient

import (
	"encoding/json"
	"fmt"
	"net/url"
)

// ProcessListResponse is the json body returned from the API
type ProcessListResponse struct {
	Pagination Pagination `json:"pagination"`
	Processes  []Process  `json:"resources"`
}

// Process represents a running process in a container.
type Process struct {
	GUID        string `json:"guid"`
	Type        string `json:"type"`
	Instances   int    `json:"instances"`
	MemoryInMB  int    `json:"memory_in_mb"`
	DiskInMB    int    `json:"disk_in_mb"`
	Ports       []int  `json:"ports,omitempty"`
	CreatedAt   string `json:"created_at"`
	UpdatedAt   string `json:"updated_at"`
	HealthCheck struct {
		Type string `json:"type"`
		Data struct {
			Timeout           int    `json:"timeout"`
			InvocationTimeout int    `json:"invocation_timeout"`
			Endpoint          string `json:"endpoint"`
		} `json:"data"`
	} `json:"health_check"`
	Links struct {
		Self  Link `json:"self"`
		Scale Link `json:"scale"`
		App   Link `json:"app"`
		Space Link `json:"space"`
		Stats Link `json:"stats"`
	} `json:"links"`
}

// ListAllProcesses will call the v3 processes api
func (c *Client) ListAllProcesses() ([]Process, error) {
	return c.ListAllProcessesByQuery(url.Values{})
}

// ListAllProcessesByQuery will call the v3 processes api
func (c *Client) ListAllProcessesByQuery(query url.Values) ([]Process, error) {
	var allProcesses []Process

	urlPath := "/v3/processes"
	for {
		resp, err := c.getProcessPage(urlPath, query)
		if err != nil {
			return nil, err
		}

		if resp.Pagination.TotalResults == 0 {
			return nil, nil
		}

		if allProcesses == nil {
			allProcesses = make([]Process, 0, resp.Pagination.TotalResults)
		}

		allProcesses = append(allProcesses, resp.Processes...)
		if resp.Pagination.Next.Href == "" {
			return allProcesses, nil
		}

		nextURL := resp.Pagination.Next.Href
		if nextURL == "" {
			return allProcesses, nil
		}

		// TODO: Use extractPathFromURL to standardize url parsing for paging
		u, err := url.Parse(nextURL)
		if err != nil {
			return nil, err
		}

		urlPath = u.Path
		query, err = url.ParseQuery(u.RawQuery)
		if err != nil {
			return nil, err
		}
	}
}

func (c *Client) getProcessPage(urlPath string, query url.Values) (*ProcessListResponse, error) {
	req := c.NewRequest("GET", fmt.Sprintf("%s?%s", urlPath, query.Encode()))

	resp, err := c.DoRequest(req)
	if err != nil {
		return nil, err
	}

	procResp := new(ProcessListResponse)
	defer resp.Body.Close()
	err = json.NewDecoder(resp.Body).Decode(procResp)
	if err != nil {
		return nil, err
	}

	return procResp, nil
}
<commit_msg>Refactor processes pagination for consistency with apps and packages pagination.<commit_after>package cfclient

import (
	"encoding/json"
	"net/url"
)

// ProcessListResponse is the json body returned from the API
type ProcessListResponse struct {
	Pagination Pagination `json:"pagination"`
	Processes  []Process  `json:"resources"`
}

// Process represents a running process in a container.
type Process struct {
	GUID        string `json:"guid"`
	Type        string `json:"type"`
	Instances   int    `json:"instances"`
	MemoryInMB  int    `json:"memory_in_mb"`
	DiskInMB    int    `json:"disk_in_mb"`
	Ports       []int  `json:"ports,omitempty"`
	CreatedAt   string `json:"created_at"`
	UpdatedAt   string `json:"updated_at"`
	HealthCheck struct {
		Type string `json:"type"`
		Data struct {
			Timeout           int    `json:"timeout"`
			InvocationTimeout int    `json:"invocation_timeout"`
			Endpoint          string `json:"endpoint"`
		} `json:"data"`
	} `json:"health_check"`
	Links struct {
		Self  Link `json:"self"`
		Scale Link `json:"scale"`
		App   Link `json:"app"`
		Space Link `json:"space"`
		Stats Link `json:"stats"`
	} `json:"links"`
}

// ListAllProcesses will call the v3 processes api
func (c *Client) ListAllProcesses() ([]Process, error) {
	return c.ListAllProcessesByQuery(url.Values{})
}

// ListAllProcessesByQuery will call the v3 processes api
func (c *Client) ListAllProcessesByQuery(query url.Values) ([]Process, error) {
	var allProcesses []Process

	requestURL := "/v3/processes"
	if e := query.Encode(); len(e) > 0 {
		requestURL += "?" + e
	}
	for {
		resp, err := c.getProcessPage(requestURL)
		if err != nil {
			return nil, err
		}

		if resp.Pagination.TotalResults == 0 {
			return nil, nil
		}

		if allProcesses == nil {
			allProcesses = make([]Process, 0, resp.Pagination.TotalResults)
		}

		allProcesses = append(allProcesses, resp.Processes...)
		if resp.Pagination.Next.Href == "" {
			break
		}

		requestURL := resp.Pagination.Next.Href
		if requestURL == "" {
			return allProcesses, nil
		}
		requestURL, err = extractPathFromURL(requestURL)
		if err != nil {
			return nil, err
		}
	}

	return allProcesses, nil
}

func (c *Client) getProcessPage(requestURL string) (*ProcessListResponse, error) {
	req := c.NewRequest("GET", requestURL)

	resp, err := c.DoRequest(req)
	if err != nil {
		return nil, err
	}

	procResp := new(ProcessListResponse)
	defer resp.Body.Close()
	err = json.NewDecoder(resp.Body).Decode(procResp)
	if err != nil {
		return nil, err
	}

	return procResp, nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Dave Collins
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package main

import (
	"github.com/davecgh/dcrstakesim/internal/tickettreap"
)

// calcNextStakeDiffProposal1 returns the required stake difficulty (aka ticket
// price) for the block after the current tip block the simulator is associated
// with using the algorithm proposed by raedah in
// https://github.com/decred/dcrd/issues/584
func (s *simulator) calcNextStakeDiffProposal1() int64 {
	// Stake difficulty before any tickets could possibly be purchased is
	// the minimum value.
	nextHeight := int32(0)
	if s.tip != nil {
		nextHeight = s.tip.height + 1
	}
	stakeDiffStartHeight := int32(s.params.CoinbaseMaturity) + 1
	if nextHeight < stakeDiffStartHeight {
		return s.params.MinimumStakeDiff
	}

	// Return the previous block's difficulty requirements if the next block
	// is not at a difficulty retarget interval.
	intervalSize := s.params.StakeDiffWindowSize
	curDiff := s.tip.ticketPrice
	if int64(nextHeight)%intervalSize != 0 {
		return curDiff
	}

	// Attempt to get the ticket price and pool size from the previous
	// retarget interval.
	var prevPoolSize int64
	prevRetargetHeight := nextHeight - int32(intervalSize)
	node := s.ancestorNode(s.tip, prevRetargetHeight, nil)
	if node != nil {
		prevPoolSize = int64(node.poolSize)
	}

	// Return the existing ticket price for the first interval.
	if prevPoolSize == 0 {
		return curDiff
	}

	curPoolSize := int64(s.tip.poolSize)
	ratio := float64(curPoolSize) / float64(prevPoolSize)
	return int64(float64(curDiff) * ratio)
}

// calcNextStakeDiffProposal2 returns the required stake difficulty (aka ticket
// price) for the block after the current tip block the simulator is associated
// with using the algorithm proposed by animedow in
// https://github.com/decred/dcrd/issues/584
func (s *simulator) calcNextStakeDiffProposal2() int64 {
	// Stake difficulty before any tickets could possibly be purchased is
	// the minimum value.
	nextHeight := int32(0)
	if s.tip != nil {
		nextHeight = s.tip.height + 1
	}
	stakeDiffStartHeight := int32(s.params.CoinbaseMaturity) + 1
	if nextHeight < stakeDiffStartHeight {
		return s.params.MinimumStakeDiff
	}

	// Return the previous block's difficulty requirements if the next block
	// is not at a difficulty retarget interval.
	intervalSize := s.params.StakeDiffWindowSize
	curDiff := s.tip.ticketPrice
	if int64(nextHeight)%intervalSize != 0 {
		return curDiff
	}

	//                ax
	// f(x) = - ---------------- + d
	//           (x - b)(x + c)
	//
	// x = amount of ticket deviation from the target pool size;
	// a = a modifier controlling the slope of the function;
	// b = the maximum boundary;
	// c = the minimum boundary;
	// d = the average ticket price in pool.
	x := int64(s.tip.poolSize) - (int64(s.params.TicketsPerBlock) *
		int64(s.params.TicketPoolSize))
	a := int64(100000)
	b := int64(2880)
	c := int64(2880)
	var d int64
	var totalSpent int64
	totalTickets := int64(len(s.immatureTickets) + s.liveTickets.Len())
	if totalTickets != 0 {
		for _, ticket := range s.immatureTickets {
			totalSpent += int64(ticket.price)
		}
		s.liveTickets.ForEach(func(k tickettreap.Key, v *tickettreap.Value) bool {
			totalSpent += v.PurchasePrice
			return true
		})
		d = totalSpent / totalTickets
	}
	price := int64(float64(d) - 100000000*(float64(a*x)/float64((x-b)*(x+c))))
	if price < s.params.MinimumStakeDiff {
		price = s.params.MinimumStakeDiff
	}
	return price
}

// calcNextStakeDiffProposal3 returns the required stake difficulty (aka ticket
// price) for the block after the current tip block the simulator is associated
// with using the algorithm proposed by coblee in
// https://github.com/decred/dcrd/issues/584
func (s *simulator) calcNextStakeDiffProposal3() int64 {
	// Stake difficulty before any tickets could possibly be purchased is
	// the minimum value.
	nextHeight := int32(0)
	if s.tip != nil {
		nextHeight = s.tip.height + 1
	}
	stakeDiffStartHeight := int32(s.params.CoinbaseMaturity) + 1
	if nextHeight < stakeDiffStartHeight {
		return s.params.MinimumStakeDiff
	}

	// Return the previous block's difficulty requirements if the next block
	// is not at a difficulty retarget interval.
	intervalSize := s.params.StakeDiffWindowSize
	curDiff := s.tip.ticketPrice
	if int64(nextHeight)%intervalSize != 0 {
		return curDiff
	}

	// f(x) = x*(locked/target_pool_size) + (1-x)*(locked/pool_size_actual)
	ticketsPerBlock := int64(s.params.TicketsPerBlock)
	targetPoolSize := ticketsPerBlock * int64(s.params.TicketPoolSize)
	lockedSupply := s.tip.stakedCoins
	x := int64(1)
	var price int64
	if s.tip.poolSize == 0 {
		price = int64(lockedSupply) / targetPoolSize
	} else {
		price = x*int64(lockedSupply)/targetPoolSize +
			(1-x)*(int64(lockedSupply)/int64(s.tip.poolSize))
	}
	if price < s.params.MinimumStakeDiff {
		price = s.params.MinimumStakeDiff
	}
	return price
}
<commit_msg>Correct comment typo in proposal 1.<commit_after>// Copyright (c) 2017 Dave Collins
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package main

import (
	"github.com/davecgh/dcrstakesim/internal/tickettreap"
)

// calcNextStakeDiffProposal1 returns the required stake difficulty (aka ticket
// price) for the block after the current tip block the simulator is associated
// with using the algorithm proposed by raedah in
// https://github.com/decred/dcrd/issues/584
func (s *simulator) calcNextStakeDiffProposal1() int64 {
	// Stake difficulty before any tickets could possibly be purchased is
	// the minimum value.
	nextHeight := int32(0)
	if s.tip != nil {
		nextHeight = s.tip.height + 1
	}
	stakeDiffStartHeight := int32(s.params.CoinbaseMaturity) + 1
	if nextHeight < stakeDiffStartHeight {
		return s.params.MinimumStakeDiff
	}

	// Return the previous block's difficulty requirements if the next block
	// is not at a difficulty retarget interval.
	intervalSize := s.params.StakeDiffWindowSize
	curDiff := s.tip.ticketPrice
	if int64(nextHeight)%intervalSize != 0 {
		return curDiff
	}

	// Attempt to get the pool size from the previous retarget interval.
	var prevPoolSize int64
	prevRetargetHeight := nextHeight - int32(intervalSize)
	node := s.ancestorNode(s.tip, prevRetargetHeight, nil)
	if node != nil {
		prevPoolSize = int64(node.poolSize)
	}

	// Return the existing ticket price for the first interval.
	if prevPoolSize == 0 {
		return curDiff
	}

	curPoolSize := int64(s.tip.poolSize)
	ratio := float64(curPoolSize) / float64(prevPoolSize)
	return int64(float64(curDiff) * ratio)
}

// calcNextStakeDiffProposal2 returns the required stake difficulty (aka ticket
// price) for the block after the current tip block the simulator is associated
// with using the algorithm proposed by animedow in
// https://github.com/decred/dcrd/issues/584
func (s *simulator) calcNextStakeDiffProposal2() int64 {
	// Stake difficulty before any tickets could possibly be purchased is
	// the minimum value.
	nextHeight := int32(0)
	if s.tip != nil {
		nextHeight = s.tip.height + 1
	}
	stakeDiffStartHeight := int32(s.params.CoinbaseMaturity) + 1
	if nextHeight < stakeDiffStartHeight {
		return s.params.MinimumStakeDiff
	}

	// Return the previous block's difficulty requirements if the next block
	// is not at a difficulty retarget interval.
	intervalSize := s.params.StakeDiffWindowSize
	curDiff := s.tip.ticketPrice
	if int64(nextHeight)%intervalSize != 0 {
		return curDiff
	}

	//                ax
	// f(x) = - ---------------- + d
	//           (x - b)(x + c)
	//
	// x = amount of ticket deviation from the target pool size;
	// a = a modifier controlling the slope of the function;
	// b = the maximum boundary;
	// c = the minimum boundary;
	// d = the average ticket price in pool.
	x := int64(s.tip.poolSize) - (int64(s.params.TicketsPerBlock) *
		int64(s.params.TicketPoolSize))
	a := int64(100000)
	b := int64(2880)
	c := int64(2880)
	var d int64
	var totalSpent int64
	totalTickets := int64(len(s.immatureTickets) + s.liveTickets.Len())
	if totalTickets != 0 {
		for _, ticket := range s.immatureTickets {
			totalSpent += int64(ticket.price)
		}
		s.liveTickets.ForEach(func(k tickettreap.Key, v *tickettreap.Value) bool {
			totalSpent += v.PurchasePrice
			return true
		})
		d = totalSpent / totalTickets
	}
	price := int64(float64(d) - 100000000*(float64(a*x)/float64((x-b)*(x+c))))
	if price < s.params.MinimumStakeDiff {
		price = s.params.MinimumStakeDiff
	}
	return price
}

// calcNextStakeDiffProposal3 returns the required stake difficulty (aka ticket
// price) for the block after the current tip block the simulator is associated
// with using the algorithm proposed by coblee in
// https://github.com/decred/dcrd/issues/584
func (s *simulator) calcNextStakeDiffProposal3() int64 {
	// Stake difficulty before any tickets could possibly be purchased is
	// the minimum value.
	nextHeight := int32(0)
	if s.tip != nil {
		nextHeight = s.tip.height + 1
	}
	stakeDiffStartHeight := int32(s.params.CoinbaseMaturity) + 1
	if nextHeight < stakeDiffStartHeight {
		return s.params.MinimumStakeDiff
	}

	// Return the previous block's difficulty requirements if the next block
	// is not at a difficulty retarget interval.
	intervalSize := s.params.StakeDiffWindowSize
	curDiff := s.tip.ticketPrice
	if int64(nextHeight)%intervalSize != 0 {
		return curDiff
	}

	// f(x) = x*(locked/target_pool_size) + (1-x)*(locked/pool_size_actual)
	ticketsPerBlock := int64(s.params.TicketsPerBlock)
	targetPoolSize := ticketsPerBlock * int64(s.params.TicketPoolSize)
	lockedSupply := s.tip.stakedCoins
	x := int64(1)
	var price int64
	if s.tip.poolSize == 0 {
		price = int64(lockedSupply) / targetPoolSize
	} else {
		price = x*int64(lockedSupply)/targetPoolSize +
			(1-x)*(int64(lockedSupply)/int64(s.tip.poolSize))
	}
	if price < s.params.MinimumStakeDiff {
		price = s.params.MinimumStakeDiff
	}
	return price
}
<|endoftext|>


<commit_before>package proxy

import (
	"github.com/coyove/goflyway/pkg/logg"

	"encoding/binary"
	"io"
	"net"
	"strconv"
	"time"
)

const (
	UOT_HEADER = byte(0x07)
)

func derr(err error) bool {
	if ne, ok := err.(net.Error); ok {
		if ne.Timeout() {
			return false
		}
	}

	return true
}

func (proxy *ProxyUpstream) HandleTCPtoUDP(c net.Conn) {
	defer c.Close()

	readFromTCP := func() (string, []byte) {
		xbuf := make([]byte, 2)

		c.SetReadDeadline(time.Now().Add(time.Duration(TCP_TIMEOUT) * time.Second))
		if _, err := io.ReadAtLeast(c, xbuf, 2); err != nil {
			if derr(err) {
				logg.E(CANNOT_READ_BUF, err)
			}
			return "", nil
		}

		hostlen := int(xbuf[1])
		hostbuf := make([]byte, hostlen)

		if _, err := io.ReadAtLeast(c, hostbuf, hostlen); err != nil {
			if derr(err) {
				logg.E(CANNOT_READ_BUF, err)
			}
			return "", nil
		}

		if hostlen < 4 {
			logg.E("[TtU] invalid hostlen")
			return "", nil
		}

		host := string(proxy.GCipher.Decrypt(hostbuf[:hostlen-4]))
		port := int(binary.BigEndian.Uint16(hostbuf[hostlen-4 : hostlen-2]))
		host = host + ":" + strconv.Itoa(port)

		payloadlen := int(binary.BigEndian.Uint16(hostbuf[hostlen-2:]))
		payload := make([]byte, payloadlen)

		if _, err := io.ReadAtLeast(c, payload, payloadlen); err != nil {
			if derr(err) {
				logg.E(CANNOT_READ_BUF, err)
			}
			return "", nil
		}

		payload = proxy.GCipher.Decrypt(payload)
		return host, payload
	}

	host, payload := readFromTCP()

	uaddr, _ := net.ResolveUDPAddr("udp", host)
	rconn, err := net.DialUDP("udp", nil, uaddr)
	if err != nil {
		logg.E("[UDP] dial - ", err)
		return
	}

	rconn.SetWriteDeadline(time.Now().Add(time.Duration(UDP_TIMEOUT) * time.Second))
	if _, err := rconn.Write(payload); err != nil {
		if derr(err) {
			logg.E("[TtU] write to target - ", err)
		}
		return
	}

	quit := make(chan bool)
	go func() { // goroutine: read from downstream tcp, write to target host udp
	READ:
		for {
			select {
			case <-quit:
				break READ
			default:
				if _, buf := readFromTCP(); buf != nil {
					rconn.SetWriteDeadline(time.Now().Add(time.Duration(UDP_TIMEOUT) * time.Second))
					if _, err := rconn.Write(buf); err != nil {
						if derr(err) {
							logg.E("[TtU] write to target - ", err)
						}
						break READ
					}
				} else {
					break READ
				}
			}
		}

		c.Close()     // may double-close, but fine
		rconn.Close() // may double-close, but fine
	}()

	buf := make([]byte, 2048)
	for { // read from target host udp, write to downstream tcp
		rconn.SetReadDeadline(time.Now().Add(time.Duration(UDP_TIMEOUT) * time.Second))
		n, _, err := rconn.ReadFrom(buf)
		// logg.L(n, ad.String(), err)

		if n > 0 {
			ybuf := proxy.GCipher.Encrypt(buf[:n])
			payload := append([]byte{UOT_HEADER, 0, 0}, ybuf...)
			binary.BigEndian.PutUint16(payload[1:3], uint16(len(ybuf)))

			c.SetWriteDeadline(time.Now().Add(time.Duration(UDP_TIMEOUT) * time.Second))
			_, err := c.Write(payload)
			if err != nil {
				if derr(err) {
					logg.E("[TtU] write to downstream - ", err)
				}

				break
			}
		}

		if err != nil {
			if derr(err) {
				logg.E("[TtU] readfrom - ", err)
			}

			break
		}
	}

	quit <- true
	rconn.Close()
}

func (proxy *ProxyClient) dialForUDP(client net.Addr, dst string) (net.Conn, string, bool) {
	proxy.udp.upstream.Lock()
	defer proxy.udp.upstream.Unlock()

	if proxy.udp.upstream.conns == nil {
		proxy.udp.upstream.conns = make(map[string]net.Conn)
	}

	str := client.String() + "-" + dst + "-" + strconv.Itoa(proxy.GCipher.Rand.Intn(proxy.UDPRelayCoconn))
	if conn, ok := proxy.udp.upstream.conns[str]; ok {
		return conn, str, false
	}

	u, _, _ := net.SplitHostPort(proxy.Upstream)
	upstreamConn, err := net.Dial("tcp", u+":"+strconv.Itoa(proxy.UDPRelayPort))
	if err != nil {
		logg.E("[UPSTREAM] udp - ", err)
		return nil, "", false
	}

	proxy.udp.upstream.conns[str] = upstreamConn
	return upstreamConn, str, true
}

func (proxy *ProxyClient) HandleUDPtoTCP(b []byte, src net.Addr) {
	_, dst, ok := ParseDstFrom(nil, b, true)
	if !ok {
		return
	}

	upstreamConn, token, firstTime := proxy.dialForUDP(src, dst.String())
	if upstreamConn == nil {
		return
	}

	// prepare the payload
	buf := proxy.GCipher.Encrypt(b[dst.size:])
	enchost := proxy.GCipher.Encrypt([]byte(dst.HostString()))

	//                                   +-------------- hostlen -------------+
	// | 0x07 (1b header) | hostlen (1b) | host | port (2b) | payloadlen (2b) |
	payload := make([]byte, 2+len(enchost)+2+2+len(buf))

	payload[0], payload[1] = UOT_HEADER, byte(len(enchost)+2+2)

	copy(payload[2:], enchost)

	binary.BigEndian.PutUint16(payload[2+len(enchost):], uint16(dst.port))
	binary.BigEndian.PutUint16(payload[2+len(enchost)+2:], uint16(len(buf)))

	copy(payload[2+len(enchost)+2+2:], buf)

	upstreamConn.Write(payload)

	if !firstTime {
		// we are not the first one using this connection, so just return here
		return
	}

	xbuf := make([]byte, 2048)
	for {
		readFromTCP := func() []byte {
			xbuf := make([]byte, 3)
			upstreamConn.SetReadDeadline(time.Now().Add(time.Duration(TCP_TIMEOUT) * time.Second))

			if _, err := io.ReadAtLeast(upstreamConn, xbuf, 3); err != nil {
				if err != io.EOF && derr(err) {
					logg.E(CANNOT_READ_BUF, err)
				}

				return nil
			}

			payloadlen := int(binary.BigEndian.Uint16(xbuf[1:3]))
			payload := make([]byte, payloadlen)
			if _, err := io.ReadAtLeast(upstreamConn, payload, payloadlen); err != nil {
				if derr(err) {
					logg.E(CANNOT_READ_BUF, err)
				}
				return nil
			}

			return proxy.GCipher.Decrypt(payload)
		}

		// read from upstream
		buf := readFromTCP()

		if buf != nil && len(buf) > 0 {
			logg.D("[UtT] receive - ", len(buf))

			var err error

			if proxy.UDPRelayNoHdr {
				_, err = proxy.udp.relay.WriteTo(buf, src)
			} else {
				copy(xbuf, UDP_REQUEST_HEADER)
				copy(xbuf[len(UDP_REQUEST_HEADER):], buf)

				_, err = proxy.udp.relay.WriteTo(xbuf[:len(buf)+len(UDP_REQUEST_HEADER)], src)
			}

			if err != nil {
				logg.E("[UtT] write - ", err)
				break
			}
		} else {
			break
		}
	}

	proxy.udp.upstream.Lock()
	delete(proxy.udp.upstream.conns, token)
	proxy.udp.upstream.Unlock()

	upstreamConn.Close()
}
<commit_msg>Fix bug in udp relay<commit_after>package proxy

import (
	"github.com/coyove/goflyway/pkg/logg"

	"encoding/binary"
	"io"
	"net"
	"strconv"
	"time"
)

const (
	UOT_HEADER = byte(0x07)
)

func derr(err error) bool {
	if ne, ok := err.(net.Error); ok {
		if ne.Timeout() {
			return false
		}
	}

	return true
}

func (proxy *ProxyUpstream) HandleTCPtoUDP(c net.Conn) {
	defer c.Close()

	readFromTCP := func() (string, []byte) {
		xbuf := make([]byte, 2)

		c.SetReadDeadline(time.Now().Add(time.Duration(TCP_TIMEOUT) * time.Second))
		if _, err := io.ReadAtLeast(c, xbuf, 2); err != nil {
			if derr(err) {
				logg.E(CANNOT_READ_BUF, err)
			}
			return "", nil
		}

		hostlen := int(xbuf[1])
		hostbuf := make([]byte, hostlen)

		if _, err := io.ReadAtLeast(c, hostbuf, hostlen); err != nil {
			if derr(err) {
				logg.E(CANNOT_READ_BUF, err)
			}
			return "", nil
		}

		if hostlen < 4 {
			logg.E("[TtU] invalid hostlen")
			return "", nil
		}

		host := string(proxy.GCipher.Decrypt(hostbuf[:hostlen-4]))
		port := int(binary.BigEndian.Uint16(hostbuf[hostlen-4 : hostlen-2]))
		host = host + ":" + strconv.Itoa(port)

		payloadlen := int(binary.BigEndian.Uint16(hostbuf[hostlen-2:]))
		payload := make([]byte, payloadlen)

		if _, err := io.ReadAtLeast(c, payload, payloadlen); err != nil {
			if derr(err) {
				logg.E(CANNOT_READ_BUF, err)
			}
			return "", nil
		}

		payload = proxy.GCipher.Decrypt(payload)
		return host, payload
	}

	host, payload := readFromTCP()

	uaddr, _ := net.ResolveUDPAddr("udp", host)
	rconn, err := net.DialUDP("udp", nil, uaddr)
	if err != nil {
		logg.E("[UDP] dial - ", err)
		return
	}

	rconn.SetWriteDeadline(time.Now().Add(time.Duration(UDP_TIMEOUT) * time.Second))
	if _, err := rconn.Write(payload); err != nil {
		if derr(err) {
			logg.E("[TtU] write to target - ", err)
		}
		return
	}

	quit := make(chan bool)
	go func() { // goroutine: read from downstream tcp, write to target host udp
	READ:
		for {
			select {
			case <-quit:
				break READ
			default:
				if _, buf := readFromTCP(); buf != nil {
					rconn.SetWriteDeadline(time.Now().Add(time.Duration(UDP_TIMEOUT) * time.Second))
					if _, err := rconn.Write(buf); err != nil {
						if derr(err) {
							logg.E("[TtU] write to target - ", err)
						}
						break READ
					}
				} else {
					break READ
				}
			}
		}

		c.Close()     // may double-close, but fine
		rconn.Close() // may double-close, but fine
	}()

	buf := make([]byte, 2048)
	for { // read from target host udp, write to downstream tcp
		rconn.SetReadDeadline(time.Now().Add(time.Duration(UDP_TIMEOUT) * time.Second))
		n, _, err := rconn.ReadFrom(buf)
		// logg.L(n, ad.String(), err)

		if n > 0 {
			ybuf := proxy.GCipher.Encrypt(buf[:n])
			payload := append([]byte{UOT_HEADER, 0, 0}, ybuf...)
			binary.BigEndian.PutUint16(payload[1:3], uint16(len(ybuf)))

			c.SetWriteDeadline(time.Now().Add(time.Duration(UDP_TIMEOUT) * time.Second))
			_, err := c.Write(payload)
			if err != nil {
				if derr(err) {
					logg.E("[TtU] write to downstream - ", err)
				}

				break
			}
		}

		if err != nil {
			if derr(err) {
				logg.E("[TtU] readfrom - ", err)
			}

			break
		}
	}

	quit <- true
	rconn.Close()
}

func (proxy *ProxyClient) dialForUDP(client net.Addr, dst string) (net.Conn, string, bool) {
	proxy.udp.upstream.Lock()
	defer proxy.udp.upstream.Unlock()

	if proxy.udp.upstream.conns == nil {
		proxy.udp.upstream.conns = make(map[string]net.Conn)
	}

	str := client.String() + "-" + dst + "-" + strconv.Itoa(proxy.GCipher.Rand.Intn(proxy.UDPRelayCoconn))
	if conn, ok := proxy.udp.upstream.conns[str]; ok {
		return conn, str, false
	}

	u, _, _ := net.SplitHostPort(proxy.Upstream)
	upstreamConn, err := net.Dial("tcp", u+":"+strconv.Itoa(proxy.UDPRelayPort))
	if err != nil {
		logg.E("[UPSTREAM] udp - ", err)
		return nil, "", false
	}

	proxy.udp.upstream.conns[str] = upstreamConn
	return upstreamConn, str, true
}

func (proxy *ProxyClient) HandleUDPtoTCP(b []byte, src net.Addr) {
	_, dst, ok := ParseDstFrom(nil, b, true)
	if !ok {
		return
	}

	upstreamConn, token, firstTime := proxy.dialForUDP(src, dst.String())
	if upstreamConn == nil {
		return
	}

	// prepare the payload
	buf := proxy.GCipher.Encrypt(b[dst.size:])
	enchost := proxy.GCipher.Encrypt([]byte(dst.HostString()))

	//                                   +-------------- hostlen -------------+
	// | 0x07 (1b header) | hostlen (1b) | host | port (2b) | payloadlen (2b) |
	payload := make([]byte, 2+len(enchost)+2+2+len(buf))

	payload[0], payload[1] = UOT_HEADER, byte(len(enchost)+2+2)

	copy(payload[2:], enchost)

	binary.BigEndian.PutUint16(payload[2+len(enchost):], uint16(dst.port))
	binary.BigEndian.PutUint16(payload[2+len(enchost)+2:], uint16(len(buf)))

	copy(payload[2+len(enchost)+2+2:], buf)

	upstreamConn.Write(payload)

	if !firstTime {
		// we are not the first one using this connection, so just return here
		return
	}

	xbuf := make([]byte, 2048)
	for {
		readFromTCP := func() []byte {
			xbuf := make([]byte, 3)
			upstreamConn.SetReadDeadline(time.Now().Add(time.Duration(TCP_TIMEOUT) * time.Second))

			if _, err := io.ReadAtLeast(upstreamConn, xbuf, 3); err != nil {
				if err != io.EOF && derr(err) {
					logg.E(CANNOT_READ_BUF, err)
				}

				return nil
			}

			payloadlen := int(binary.BigEndian.Uint16(xbuf[1:3]))
			payload := make([]byte, payloadlen)
			if _, err := io.ReadAtLeast(upstreamConn, payload, payloadlen); err != nil {
				if derr(err) {
					logg.E(CANNOT_READ_BUF, err)
				}
				return nil
			}

			return proxy.GCipher.Decrypt(payload)
		}

		// read from upstream
		buf := readFromTCP()

		if buf != nil && len(buf) > 0 {
			logg.D("[UtT] receive - ", len(buf))

			var err error

			if proxy.UDPRelayNoHdr {
				_, err = proxy.udp.relay.WriteTo(buf, src)
			} else {
				// prepare the response header
				if len(dst.ip) == net.IPv4len {
					copy(xbuf, UDP_REQUEST_HEADER)
					copy(xbuf[4:8], dst.ip)
					binary.BigEndian.PutUint16(xbuf[8:], uint16(dst.port))
					copy(xbuf[len(UDP_REQUEST_HEADER):], buf)
				} else {
					copy(xbuf, UDP_REQUEST_HEADER6)
					copy(xbuf[4:20], dst.ip)
					binary.BigEndian.PutUint16(xbuf[20:], uint16(dst.port))
					copy(xbuf[len(UDP_REQUEST_HEADER6):], buf)
				}

				_, err = proxy.udp.relay.WriteTo(xbuf[:len(buf)+len(UDP_REQUEST_HEADER)], src)
			}

			if err != nil {
				logg.E("[UtT] write - ", err)
				break
			}
		} else {
			break
		}
	}

	proxy.udp.upstream.Lock()
	delete(proxy.udp.upstream.conns, token)
	proxy.udp.upstream.Unlock()

	upstreamConn.Close()
}
<|endoftext|>


<commit_before>// Copyright 2016 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

// +build docker

package test

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"strings"
	"testing"

	//"github.com/samalba/dockerclient"
)

func createCommand(c string, args ...string) *exec.Cmd {
	fmt.Println(strings.Join(append([]string{"$", c}, args...), " "))
	cmd := exec.Command(c, args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd
}

func dockerExec(container string, command ...string) error {
	args := append([]string{
		"exec",
		container,
	}, command...)
	cmd := createCommand("docker", args...)
	return cmd.Run()
}

func getDockerIds(f string) ([]string, error) {
	cmd := exec.Command("docker-compose", "ps", "-q", f)
	var buf bytes.Buffer
	cmd.Stdout = &buf
	cmd.Stderr = &buf
	err := cmd.Run()
	if err != nil {
		return nil, err
	}
	return strings.Split(strings.TrimSpace(buf.String()), "\n"), nil
}

func startDockers() error {
	cmd := createCommand(fmt.Sprintf("%s/src/github.com/keybase/kbfs/test/run_dockers.sh", os.Getenv("GOPATH")))
	return cmd.Run()
}

func stopDockers() error {
	cmd := createCommand("docker-compose", "down")
	return cmd.Run()
}

func resetService(n int) (map[string]string, error) {
	err := stopDockers()
	if err != nil {
		return nil, fmt.Errorf("Unable to scale down the service: %v", err)
	}
	err = startDockers()
	if err != nil {
		return nil, fmt.Errorf("Unable to start the service: %v", err)
	}
	err = createCommand("docker-compose", "scale", fmt.Sprintf("keybase=%d", n)).Run()
	if err != nil {
		return nil, fmt.Errorf("Unable to scale up the service: %v", err)
	}
	containers, err := getDockerIds("keybase")
	if err != nil {
		return nil, fmt.Errorf("Unable to obtain docker IDs: %v", err)
	}
	return signupContainers(containers)
}

func signupContainer(container string, username string) error {
	email := fmt.Sprintf("%s@keyba.se", username)

	cmd := createCommand("docker", "exec", container,
		"keybase", "signup", "-c", "202020202020202020202020",
		"--email", email, "--username", username,
		"-p", "strong passphrase", "-d", "dev1", "-b", "--devel")

	err := cmd.Run()
	if err != nil {
		return fmt.Errorf("Unable to signup user %s on container %s. Error: %v", username, container, err)
	}
	return nil
}

func signupContainers(containers []string) (map[string]string, error) {
	containersByUsername := make(map[string]string)

	for i, container := range containers {

		username := fmt.Sprintf("test%d", i)
		err := signupContainer(container, username)
		if err != nil {
			return nil, err
		}

		containersByUsername[username] = container
	}
	return containersByUsername, nil
}

func getTLF(writers []string, readers []string) string {
	content := []string{strings.Join(writers, ",")}
	if len(readers) > 0 {
		content = append(content, strings.Join(readers, "#"))
	}
	return fmt.Sprintf("/keybase/private/%s", strings.Join(content, "#"))
}

func listFolder(container string, tlf string) error {
	err := dockerExec(container, "ls", tlf)

	if err != nil {
		return fmt.Errorf("Unable to list folder %s on container %s. Error: %v", tlf, container)
	}
	return nil
}

func writeToFile(container string, tlf string, filename string, text string) error {
	err := dockerExec(container, "sh", "-c", fmt.Sprintf("echo \"%s\" >> %s", text, fmt.Sprintf("%s/%s", tlf, filename)))

	if err != nil {
		return fmt.Errorf("Unable to write to file %s/%s on container %s", tlf, filename, container)
	}
	return nil
}

func readFromFile(container string, tlf string, filename string) error {
	err := dockerExec(container, "cat", fmt.Sprintf("%s/%s", tlf, filename))

	if err != nil {
		return fmt.Errorf("Unable to read from file %s/%s on container %s", tlf, filename, container)
	}
	return nil
}

func TestSharedFileWrite(t *testing.T) {
	containers, err := resetService(2)
	if err != nil {
		t.Fatalf("Failed to reset service: %v", err)
	}
	tlf := getTLF([]string{"test0", "test1"}, []string{})
	err = writeToFile(
		containers["test0"],
		tlf,
		"hello.txt",
		"world",
	)
	if err != nil {
		stopDockers()
		t.Fatalf("Failed to write to file: %v", err)
	}
	err = listFolder(
		containers["test1"],
		tlf,
	)
	if err != nil {
		stopDockers()
		t.Fatalf("Failed to list folder %s. Error: %v", tlf, err)
	}
	err = readFromFile(
		containers["test1"],
		tlf,
		"hello.txt",
	)
	if err != nil {
		stopDockers()
		t.Fatalf("Failed to write to file: %v", err)
	}

	err = stopDockers()
	if err != nil {
		t.Fatalf("Failed to stop dockers: %v", err)
	}
}
<commit_msg>Cleaned up test output<commit_after>// Copyright 2016 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

// +build docker

package test

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strings"
	"testing"

	//"github.com/samalba/dockerclient"
)

func createCommand(out io.Writer, err io.Writer, c string, args ...string) *exec.Cmd {
	fmt.Println(strings.Join(append([]string{"$", c}, args...), " "))
	cmd := exec.Command(c, args...)
	cmd.Stdout = out
	cmd.Stderr = err
	return cmd
}

func createStdoutCommand(c string, args ...string) *exec.Cmd {
	cmd := createCommand(os.Stdout, os.Stderr, c, args...)
	return cmd
}

func dockerExec(out io.Writer, err io.Writer, container string, command ...string) error {
	args := append([]string{
		"exec",
		container,
	}, command...)
	cmd := createCommand(out, err, "docker", args...)
	return cmd.Run()
}

func getDockerIds(f string) ([]string, error) {
	cmd := exec.Command("docker-compose", "ps", "-q", f)
	var buf bytes.Buffer
	cmd.Stdout = &buf
	cmd.Stderr = &buf
	err := cmd.Run()
	if err != nil {
		return nil, err
	}
	return strings.Split(strings.TrimSpace(buf.String()), "\n"), nil
}

func startDockers() error {
	cmd := createStdoutCommand(fmt.Sprintf("%s/src/github.com/keybase/kbfs/test/run_dockers.sh", os.Getenv("GOPATH")))
	return cmd.Run()
}

func stopDockers() error {
	cmd := createStdoutCommand("docker-compose", "down")
	return cmd.Run()
}

func resetService(n int) (map[string]string, error) {
	err := stopDockers()
	if err != nil {
		return nil, fmt.Errorf("Unable to scale down the service: %v", err)
	}
	err = startDockers()
	if err != nil {
		return nil, fmt.Errorf("Unable to start the service: %v", err)
	}
	err = createStdoutCommand("docker-compose", "scale", fmt.Sprintf("keybase=%d", n)).Run()
	if err != nil {
		return nil, fmt.Errorf("Unable to scale up the service: %v", err)
	}
	containers, err := getDockerIds("keybase")
	if err != nil {
		return nil, fmt.Errorf("Unable to obtain docker IDs: %v", err)
	}
	return signupContainers(containers)
}

func signupContainer(container string, username string) error {
	email := fmt.Sprintf("%s@keyba.se", username)

	cmd := createCommand(nil, nil, "docker", "exec", container,
		"keybase", "signup", "-c", "202020202020202020202020",
		"--email", email, "--username", username,
		"-p", "strong passphrase", "-d", "dev1", "-b", "--devel")

	err := cmd.Run()
	if err != nil {
		return fmt.Errorf("Unable to signup user %s on container %s. Error: %v", username, container, err)
	}
	return nil
}

func signupContainers(containers []string) (map[string]string, error) {
	containersByUsername := make(map[string]string)

	for i, container := range containers {

		username := fmt.Sprintf("test%d", i)
		err := signupContainer(container, username)
		if err != nil {
			return nil, err
		}

		containersByUsername[username] = container
	}
	return containersByUsername, nil
}

func getTLF(writers []string, readers []string) string {
	content := []string{strings.Join(writers, ",")}
	if len(readers) > 0 {
		content = append(content, strings.Join(readers, "#"))
	}
	return fmt.Sprintf("/keybase/private/%s", strings.Join(content, "#"))
}

func listFolder(container string, tlf string) error {
	err := dockerExec(os.Stdout, os.Stderr, container, "ls", tlf)

	if err != nil {
		return fmt.Errorf("Unable to list folder %s on container %s. Error: %v", tlf, container)
	}
	return nil
}

func writeToFile(container string, tlf string, filename string, text string) error {
	err := dockerExec(os.Stdout, os.Stderr, container, "sh", "-c", fmt.Sprintf("echo \"%s\" >> %s", text, fmt.Sprintf("%s/%s", tlf, filename)))

	if err != nil {
		return fmt.Errorf("Unable to write to file %s/%s on container %s", tlf, filename, container)
	}
	return nil
}

func readFromFile(container string, tlf string, filename string) error {
	err := dockerExec(os.Stdout, os.Stderr, container, "cat", fmt.Sprintf("%s/%s", tlf, filename))

	if err != nil {
		return fmt.Errorf("Unable to read from file %s/%s on container %s", tlf, filename, container)
	}
	return nil
}

func TestSharedFileWrite(t *testing.T) {
	containers, err := resetService(2)
	if err != nil {
		t.Fatalf("Failed to reset service: %v", err)
	}
	tlf := getTLF([]string{"test0", "test1"}, []string{})
	err = writeToFile(
		containers["test0"],
		tlf,
		"hello.txt",
		"world",
	)
	if err != nil {
		stopDockers()
		t.Fatalf("Failed to write to file: %v", err)
	}
	err = listFolder(
		containers["test1"],
		tlf,
	)
	if err != nil {
		stopDockers()
		t.Fatalf("Failed to list folder %s. Error: %v", tlf, err)
	}
	err = readFromFile(
		containers["test1"],
		tlf,
		"hello.txt",
	)
	if err != nil {
		stopDockers()
		t.Fatalf("Failed to write to file: %v", err)
	}

	err = stopDockers()
	if err != nil {
		t.Fatalf("Failed to stop dockers: %v", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"

	"github.com/piotrkowalczuk/charon/charond"
)

const VERSION = "0.1.2"

type configuration struct {
	host      string
	port      int
	namespace string
	subsystem string
	test      bool
	logger    struct {
		adapter string
		format  string
		level   int
	}
	mnemosyne struct {
		address string
	}
	password struct {
		strategy string
		bcrypt   struct {
			cost int
		}
	}
	monitoring struct {
		engine string
	}
	postgres struct {
		address string
	}
	tls struct {
		enabled  bool
		certFile string
		keyFile  string
	}
}

func (c *configuration) init() {
	if c == nil {
		*c = configuration{}
	}

	flag.StringVar(&c.host, "host", "127.0.0.1", "host")
	flag.IntVar(&c.port, "port", 8080, "port")
	flag.StringVar(&c.namespace, "namespace", "", "namespace")
	flag.StringVar(&c.subsystem, "subsystem", "charon", "subsystem")
	flag.BoolVar(&c.test, "test", false, "determines in what mode application starts")
	flag.StringVar(&c.logger.adapter, "l.adapter", loggerAdapterStdOut, "logger adapter")
	flag.StringVar(&c.logger.format, "l.format", loggerFormatJSON, "logger format")
	flag.IntVar(&c.logger.level, "l.level", 6, "logger level")
	flag.StringVar(&c.mnemosyne.address, "mnemo.address", "", "mnemosyne session store connection address")
	flag.StringVar(&c.password.strategy, "pwd.strategy", "bcrypt", "strategy how password will be stored")
	flag.IntVar(&c.password.bcrypt.cost, "pwd.bcryptcost", 10, "bcrypt cost, bigget than safer (and longer to create)")
	flag.StringVar(&c.monitoring.engine, "m.engine", charond.MonitoringEnginePrometheus, "monitoring engine")
	flag.StringVar(&c.postgres.address, "p.address", "postgres://localhost:5432?sslmode=disable", "postgres connection string")
	flag.BoolVar(&c.tls.enabled, "tls", false, "tls enable flag")
	flag.StringVar(&c.tls.certFile, "tls.certfile", "", "path to tls cert file")
	flag.StringVar(&c.tls.keyFile, "tls.keyfile", "", "path to tls key file")
}

func (c *configuration) parse() {
	if !flag.Parsed() {
		ver := flag.Bool("version", false, "print version and exit")
		flag.Parse()
		if *ver {
			fmt.Printf("%s", VERSION)
			os.Exit(0)
		}
	}
}
<commit_msg>proper version displaying<commit_after>package main

import (
	"flag"
	"fmt"
	"os"

	"github.com/piotrkowalczuk/charon/charond"
)

const VERSION = "0.3.1"

type configuration struct {
	host      string
	port      int
	namespace string
	subsystem string
	test      bool
	logger    struct {
		adapter string
		format  string
		level   int
	}
	mnemosyne struct {
		address string
	}
	password struct {
		strategy string
		bcrypt   struct {
			cost int
		}
	}
	monitoring struct {
		engine string
	}
	postgres struct {
		address string
	}
	tls struct {
		enabled  bool
		certFile string
		keyFile  string
	}
}

func (c *configuration) init() {
	if c == nil {
		*c = configuration{}
	}

	flag.StringVar(&c.host, "host", "127.0.0.1", "host")
	flag.IntVar(&c.port, "port", 8080, "port")
	flag.StringVar(&c.namespace, "namespace", "", "namespace")
	flag.StringVar(&c.subsystem, "subsystem", "charon", "subsystem")
	flag.BoolVar(&c.test, "test", false, "determines in what mode application starts")
	flag.StringVar(&c.logger.adapter, "l.adapter", loggerAdapterStdOut, "logger adapter")
	flag.StringVar(&c.logger.format, "l.format", loggerFormatJSON, "logger format")
	flag.IntVar(&c.logger.level, "l.level", 6, "logger level")
	flag.StringVar(&c.mnemosyne.address, "mnemo.address", "", "mnemosyne session store connection address")
	flag.StringVar(&c.password.strategy, "pwd.strategy", "bcrypt", "strategy how password will be stored")
	flag.IntVar(&c.password.bcrypt.cost, "pwd.bcryptcost", 10, "bcrypt cost, bigget than safer (and longer to create)")
	flag.StringVar(&c.monitoring.engine, "m.engine", charond.MonitoringEnginePrometheus, "monitoring engine")
	flag.StringVar(&c.postgres.address, "p.address", "postgres://localhost:5432?sslmode=disable", "postgres connection string")
	flag.BoolVar(&c.tls.enabled, "tls", false, "tls enable flag")
	flag.StringVar(&c.tls.certFile, "tls.certfile", "", "path to tls cert file")
	flag.StringVar(&c.tls.keyFile, "tls.keyfile", "", "path to tls key file")
}

func (c *configuration) parse() {
	if !flag.Parsed() {
		ver := flag.Bool("version", false, "print version and exit")
		flag.Parse()
		if *ver {
			fmt.Printf("%s", VERSION)
			os.Exit(0)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"github.com/clawio/clawiod/root"
	"github.com/go-kit/kit/log/levels"
	"github.com/gorilla/handlers"
	"github.com/gorilla/mux"
	"github.com/prometheus/client_golang/prometheus"
	"io"
	"net/http"
	"os"
	"time"
)

type server struct {
	logger         levels.Levels
	router         http.Handler
	config         root.Configuration
	httpLogger     io.Writer
	registryDriver root.RegistryDriver
	webServices    map[string]root.WebService
}

func newServer(config root.Configuration) (*server, error) {
	logger, err := getLogger(config)
	if err != nil {
		return nil, err
	}
	registryDriver, err := getRegistryDriver(config)
	if err != nil {
		return nil, err
	}
	s := &server{logger: logger, config: config, registryDriver: registryDriver}
	err = s.configureRouter()
	if err != nil {
		return nil, err
	}
	// do the register in other routine repeatedly to avoid the node
	// being removed by the TTL constraint
	go func() {
		err = s.registerNode()
		if err != nil {
			s.logger.Error().Log("error", "error registering node")
		}
		for range time.Tick(time.Second * 5) {
			s.logger.Info().Log("msg", "keep alive is issued every 5 seconds: re-registering node")
			err = s.registerNode()
			if err != nil {
				s.logger.Error().Log("error", "error registering node")
			}
		}
	}()

	return s, nil
}

func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	handlers.CombinedLoggingHandler(s.httpLogger, s.router).ServeHTTP(w, r)
}

func (s *server) registerNode() error {
	hostname, err := os.Hostname()
	if err != nil {
		s.logger.Error().Log("error", err)
		return err
	}
	for key, ws := range s.webServices {
		rol := key + "-node"
		if ws.IsProxy() {
			rol = rol + "-proxy"
		}

		url := fmt.Sprintf("%s:%d", hostname, s.config.GetPort())
		if s.config.IsTLSEnabled() {
			url = fmt.Sprintf("https://%s", url)
		} else {
			url = fmt.Sprintf("http://%s", url)
		}

		node := &node{
			xhost:    fmt.Sprintf("%s:%d", hostname, s.config.GetPort()),
			xid:      fmt.Sprintf("%s:%d", hostname, s.config.GetPort()),
			xrol:     rol,
			xurl:     url,
			xversion: "TODO"}
		err := s.registryDriver.Register(context.Background(), node)
		if err != nil {
			s.logger.Error().Log("error", err)
			return err
		}
	}
	return nil
}

func (s *server) configureRouter() error {
	config := s.config

	httpLogger, err := getHTTPLogger(config)
	if err != nil {
		s.logger.Error().Log("error", err)
		return err
	}
	s.httpLogger = httpLogger

	loggerMiddleware, err := getLoggerMiddleware(config)
	if err != nil {
		s.logger.Error().Log("error", err)
		return err
	}

	corsMiddleware, err := getCORSMiddleware(config)
	if err != nil {
		s.logger.Error().Log("error", err)
		return err
	}

	webServices, err := getWebServices(config)
	if err != nil {
		s.logger.Error().Log("error", err)
		return err
	}
	s.logger.Info().Log("msg", "web services enabled", "webservices", config.GetEnabledWebServices())
	s.webServices = webServices

	router := mux.NewRouter()
	router.Handle("/metrics", prometheus.Handler()).Methods("GET")
	s.logger.Info().Log("method", "GET", "endpoint", "/metrics", "msg", "endpoint available - created by prometheus")
	for key, service := range webServices {
		s.logger.Info().Log("msg", key+" web service enabled")
		for path, methods := range service.Endpoints() {
			for method, handlerFunc := range methods {
				handlerFunc = loggerMiddleware.HandlerFunc(handlerFunc)
				handlerFunc := http.HandlerFunc(handlerFunc)
				var handler http.Handler
				if config.IsCORSMiddlewareEnabled() {
					handler = handlerFunc
					handler = corsMiddleware.Handler(handler)
					if method == "*" {
						router.Handle(path, handler)
					} else {
						router.Handle(path, handler).Methods(method)
					}

					prometheus.InstrumentHandler(path, handler)
					s.logger.Info().Log("method", method, "endpoint", path, "msg", "endpoint available")
					router.Handle(path, handler).Methods("OPTIONS")
					s.logger.Info().Log("method", "OPTIONS", "endpoint", path, "msg", "endpoint available - created by corsmiddleware")
				} else {
					handler = handlerFunc
					if method == "*" {
						router.Handle(path, handler)
					} else {
						router.Handle(path, handler).Methods(method)
					}
					prometheus.InstrumentHandler(path, handler)
					s.logger.Info().Log("method", method, "endpoint", path, "msg", "endpoint available")
				}
			}
		}
	}
	s.router = router
	return nil
}

type node struct {
	xid      string
	xrol     string
	xhost    string
	xversion string
	xurl     string
}

func (n *node) ID() string {
	return n.xid
}
func (n *node) Rol() string {
	return n.xrol
}
func (n *node) Host() string {
	return n.xhost
}
func (n *node) Version() string {
	return n.xversion
}
func (n *node) URL() string {
	return n.xurl
}
<commit_msg>Fix cache time expiration<commit_after>package main

import (
	"context"
	"fmt"
	"github.com/clawio/clawiod/root"
	"github.com/go-kit/kit/log/levels"
	"github.com/gorilla/handlers"
	"github.com/gorilla/mux"
	"github.com/prometheus/client_golang/prometheus"
	"io"
	"net/http"
	"os"
	"time"
)

type server struct {
	logger         levels.Levels
	router         http.Handler
	config         root.Configuration
	httpLogger     io.Writer
	registryDriver root.RegistryDriver
	webServices    map[string]root.WebService
}

func newServer(config root.Configuration) (*server, error) {
	logger, err := getLogger(config)
	if err != nil {
		return nil, err
	}
	registryDriver, err := getRegistryDriver(config)
	if err != nil {
		return nil, err
	}
	s := &server{logger: logger, config: config, registryDriver: registryDriver}
	err = s.configureRouter()
	if err != nil {
		return nil, err
	}
	// do the register in other routine repeatedly to avoid the node
	// being removed by the TTL constraint
	go func() {
		err = s.registerNode()
		if err != nil {
			s.logger.Error().Log("error", "error registering node")
		}
		for range time.Tick(time.Second * 5) {
			err = s.registerNode()
			if err != nil {
				s.logger.Error().Log("error", "error registering node")
			}
		}
	}()

	return s, nil
}

func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	handlers.CombinedLoggingHandler(s.httpLogger, s.router).ServeHTTP(w, r)
}

func (s *server) registerNode() error {
	hostname, err := os.Hostname()
	if err != nil {
		s.logger.Error().Log("error", err)
		return err
	}
	for key, ws := range s.webServices {
		rol := key + "-node"
		if ws.IsProxy() {
			rol = rol + "-proxy"
		}

		url := fmt.Sprintf("%s:%d", hostname, s.config.GetPort())
		if s.config.IsTLSEnabled() {
			url = fmt.Sprintf("https://%s", url)
		} else {
			url = fmt.Sprintf("http://%s", url)
		}

		node := &node{
			xhost:    fmt.Sprintf("%s:%d", hostname, s.config.GetPort()),
			xid:      fmt.Sprintf("%s:%d", hostname, s.config.GetPort()),
			xrol:     rol,
			xurl:     url,
			xversion: "TODO"}
		err := s.registryDriver.Register(context.Background(), node)
		if err != nil {
			s.logger.Error().Log("error", err)
			return err
		}
	}
	return nil
}

func (s *server) configureRouter() error {
	config := s.config

	httpLogger, err := getHTTPLogger(config)
	if err != nil {
		s.logger.Error().Log("error", err)
		return err
	}
	s.httpLogger = httpLogger

	loggerMiddleware, err := getLoggerMiddleware(config)
	if err != nil {
		s.logger.Error().Log("error", err)
		return err
	}

	corsMiddleware, err := getCORSMiddleware(config)
	if err != nil {
		s.logger.Error().Log("error", err)
		return err
	}

	webServices, err := getWebServices(config)
	if err != nil {
		s.logger.Error().Log("error", err)
		return err
	}
	s.logger.Info().Log("msg", "web services enabled", "webservices", config.GetEnabledWebServices())
	s.webServices = webServices

	router := mux.NewRouter()
	router.Handle("/metrics", prometheus.Handler()).Methods("GET")
	s.logger.Info().Log("method", "GET", "endpoint", "/metrics", "msg", "endpoint available - created by prometheus")
	for key, service := range webServices {
		s.logger.Info().Log("msg", key+" web service enabled")
		for path, methods := range service.Endpoints() {
			for method, handlerFunc := range methods {
				handlerFunc = loggerMiddleware.HandlerFunc(handlerFunc)
				handlerFunc := http.HandlerFunc(handlerFunc)
				var handler http.Handler
				if config.IsCORSMiddlewareEnabled() {
					handler = handlerFunc
					handler = corsMiddleware.Handler(handler)
					if method == "*" {
						router.Handle(path, handler)
					} else {
						router.Handle(path, handler).Methods(method)
					}

					prometheus.InstrumentHandler(path, handler)
					s.logger.Info().Log("method", method, "endpoint", path, "msg", "endpoint available")
					router.Handle(path, handler).Methods("OPTIONS")
					s.logger.Info().Log("method", "OPTIONS", "endpoint", path, "msg", "endpoint available - created by corsmiddleware")
				} else {
					handler = handlerFunc
					if method == "*" {
						router.Handle(path, handler)
					} else {
						router.Handle(path, handler).Methods(method)
					}
					prometheus.InstrumentHandler(path, handler)
					s.logger.Info().Log("method", method, "endpoint", path, "msg", "endpoint available")
				}
			}
		}
	}
	s.router = router
	return nil
}

type node struct {
	xid      string
	xrol     string
	xhost    string
	xversion string
	xurl     string
}

func (n *node) ID() string {
	return n.xid
}
func (n *node) Rol() string {
	return n.xrol
}
func (n *node) Host() string {
	return n.xhost
}
func (n *node) Version() string {
	return n.xversion
}
func (n *node) URL() string {
	return n.xurl
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"

	"github.com/golang/glog"
	yaml "gopkg.in/yaml.v2"

	"strings"

	"k8s.io/publishing-bot/cmd/publishing-bot/config"
)

const (
	depCommit   = "7c44971bbb9f0ed87db40b601f2d9fe4dffb750d"
	godepCommit = "tags/v79"
)

var (
	SystemGoPath = os.Getenv("GOPATH")
	BaseRepoPath = filepath.Join(SystemGoPath, "src", "k8s.io")
)

func Usage() {
	fmt.Fprintf(os.Stderr, `
Usage: %s [-config <config-yaml-file>] [-source-repo <repo>] [-source-org <org>] [-rules-file <file> ] [-skip-godep|skip-dep] [-target-org <org>]

Command line flags override config values.
`, os.Args[0])
	flag.PrintDefaults()
}

func main() {
	configFilePath := flag.String("config", "", "the config file in yaml format")
	repoName := flag.String("source-repo", "", "the name of the source repository (eg. kubernetes)")
	repoOrg := flag.String("source-org", "", "the name of the source repository organization, (eg. kubernetes)")
	rulesFile := flag.String("rules-file", "", "the file with repository rules")
	targetOrg := flag.String("target-org", "", `the target organization to publish into (e.g. "k8s-publishing-bot")`)
	skipGodep := flag.Bool("skip-godep", false, `skip godeps installation and godeps-restore`)
	skipDep := flag.Bool("skip-dep", false, `skip 'dep'' installation`)

	flag.Usage = Usage
	flag.Parse()

	cfg := config.Config{}
	if *configFilePath != "" {
		bs, err := ioutil.ReadFile(*configFilePath)
		if err != nil {
			glog.Fatalf("Failed to load config file from %q: %v", *configFilePath, err)
		}
		if err := yaml.Unmarshal(bs, &cfg); err != nil {
			glog.Fatalf("Failed to parse config file at %q: %v", *configFilePath, err)
		}
	}

	if *targetOrg != "" {
		cfg.TargetOrg = *targetOrg
	}
	if *repoName != "" {
		cfg.SourceRepo = *repoName
	}
	if *repoOrg != "" {
		cfg.SourceOrg = *repoOrg
	}

	if cfg.SourceRepo != "kubernetes" {
		BaseRepoPath = filepath.Join(SystemGoPath, "src", "github.com", cfg.TargetOrg)
	}

	if *rulesFile != "" {
		cfg.RulesFile = *rulesFile
	}

	if len(cfg.SourceRepo) == 0 || len(cfg.SourceOrg) == 0 {
		glog.Fatalf("source-org and source-repo cannot be empty")
	}

	if len(cfg.TargetOrg) == 0 {
		glog.Fatalf("Target organization cannot be empty")
	}

	// If RULE_FILE_PATH is detected, check if the source repository include rules files.
	if len(os.Getenv("RULE_FILE_PATH")) > 0 {
		cfg.RulesFile = filepath.Join(BaseRepoPath, cfg.SourceRepo, os.Getenv("RULE_FILE_PATH"))
	}

	if len(cfg.RulesFile) == 0 {
		glog.Fatalf("No rules file provided")
	}
	rules, err := config.LoadRules(cfg.RulesFile)
	if err != nil {
		glog.Fatalf("Failed to load rules: %v", err)
	}

	if err := os.MkdirAll(BaseRepoPath, os.ModePerm); err != nil {
		glog.Fatalf("Failed to create source repo directory %s: %v", BaseRepoPath, err)
	}

	if !*skipGodep {
		installGodeps()
	}
	if !*skipDep {
		installDep()
	}

	cloneSourceRepo(cfg, *skipGodep)
	for _, rule := range rules.Rules {
		cloneForkRepo(cfg, rule.DestinationRepository)
	}
}

func cloneForkRepo(cfg config.Config, repoName string) {
	forkRepoLocation := "https://github.com/" + cfg.TargetOrg + "/" + repoName
	repoDir := filepath.Join(BaseRepoPath, repoName)

	if _, err := os.Stat(repoDir); err == nil {
		glog.Infof("Fork repository %q already cloned to %s, resetting remote URL ...", repoName, repoDir)
		setUrlCmd := exec.Command("git", "remote", "set-url", "origin", forkRepoLocation)
		setUrlCmd.Dir = repoDir
		run(setUrlCmd)
		os.Remove(filepath.Join(repoDir, ".git", "index.lock"))
		return
	}

	glog.Infof("Cloning fork repository %s ...", forkRepoLocation)
	run(exec.Command("git", "clone", forkRepoLocation))

	// TODO: This can be set as an env variable for the container
	setUsernameCmd := exec.Command("git", "config", "user.name", os.Getenv("GIT_COMMITTER_NAME"))
	setUsernameCmd.Dir = repoDir
	run(setUsernameCmd)

	// TODO: This can be set as an env variable for the container
	setEmailCmd := exec.Command("git", "config", "user.email", os.Getenv("GIT_COMMITTER_EMAIL"))
	setEmailCmd.Dir = repoDir
	run(setEmailCmd)
}

func installGodeps() {
	if _, err := exec.LookPath("godep"); err == nil {
		glog.Infof("Already installed: godep")
		return
	}
	glog.Infof("Installing github.com/tools/godep#%s ...", godepCommit)
	run(exec.Command("go", "get", "github.com/tools/godep"))

	godepDir := filepath.Join(SystemGoPath, "src", "github.com", "tools", "godep")
	godepCheckoutCmd := exec.Command("git", "checkout", godepCommit)
	godepCheckoutCmd.Dir = godepDir
	run(godepCheckoutCmd)

	godepInstallCmd := exec.Command("go", "install", "./...")
	godepInstallCmd.Dir = godepDir
	run(godepInstallCmd)
}

func installDep() {
	if _, err := exec.LookPath("dep"); err == nil {
		glog.Infof("Already installed: dep")
		return
	}
	glog.Infof("Installing github.com/golang/dep#%s ...", depCommit)
	depGoGetCmd := exec.Command("go", "get", "github.com/golang/dep")
	run(depGoGetCmd)

	depDir := filepath.Join(SystemGoPath, "src", "github.com", "golang", "dep")
	depCheckoutCmd := exec.Command("git", "checkout", depCommit)
	depCheckoutCmd.Dir = depDir
	run(depCheckoutCmd)

	depInstallCmd := exec.Command("go", "install", "./cmd/dep")
	depInstallCmd.Dir = depDir
	run(depInstallCmd)
}

// run wraps the cmd.Run() command and sets the standard output and common environment variables.
// if the c.Dir is not set, the BaseRepoPath will be used as a base directory for the command.
func run(c *exec.Cmd) {
	c.Stdout = os.Stdout
	c.Stderr = os.Stderr
	if len(c.Dir) == 0 {
		c.Dir = BaseRepoPath
	}
	if err := c.Run(); err != nil {
		glog.Fatalf("Command %q failed: %v", strings.Join(c.Args, " "), err)
	}
}

func cloneSourceRepo(cfg config.Config, runGodepRestore bool) {
	if _, err := os.Stat(filepath.Join(BaseRepoPath, cfg.SourceRepo)); err == nil {
		glog.Infof("Source repository %q already cloned, skipping", cfg.SourceRepo)
		return
	}

	repoLocation := "https://" + filepath.Join("github.com", cfg.SourceOrg, cfg.SourceRepo)
	glog.Infof("Cloning source repository %s ...", repoLocation)
	cloneCmd := exec.Command("git", "clone", repoLocation)
	run(cloneCmd)

	if runGodepRestore {
		glog.Infof("Running hack/godep-restore.sh ...")
		restoreCmd := exec.Command("bash", "-x", "hack/godep-restore.sh")
		restoreCmd.Dir = filepath.Join(BaseRepoPath, cfg.SourceRepo)
		run(restoreCmd)
	}
}
<commit_msg>Bump godep version to v80<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"

	"github.com/golang/glog"
	yaml "gopkg.in/yaml.v2"

	"strings"

	"k8s.io/publishing-bot/cmd/publishing-bot/config"
)

const (
	depCommit   = "7c44971bbb9f0ed87db40b601f2d9fe4dffb750d"
	godepCommit = "tags/v80"
)

var (
	SystemGoPath = os.Getenv("GOPATH")
	BaseRepoPath = filepath.Join(SystemGoPath, "src", "k8s.io")
)

func Usage() {
	fmt.Fprintf(os.Stderr, `
Usage: %s [-config <config-yaml-file>] [-source-repo <repo>] [-source-org <org>] [-rules-file <file> ] [-skip-godep|skip-dep] [-target-org <org>]

Command line flags override config values.
`, os.Args[0])
	flag.PrintDefaults()
}

func main() {
	configFilePath := flag.String("config", "", "the config file in yaml format")
	repoName := flag.String("source-repo", "", "the name of the source repository (eg. kubernetes)")
	repoOrg := flag.String("source-org", "", "the name of the source repository organization, (eg. kubernetes)")
	rulesFile := flag.String("rules-file", "", "the file with repository rules")
	targetOrg := flag.String("target-org", "", `the target organization to publish into (e.g. "k8s-publishing-bot")`)
	skipGodep := flag.Bool("skip-godep", false, `skip godeps installation and godeps-restore`)
	skipDep := flag.Bool("skip-dep", false, `skip 'dep'' installation`)

	flag.Usage = Usage
	flag.Parse()

	cfg := config.Config{}
	if *configFilePath != "" {
		bs, err := ioutil.ReadFile(*configFilePath)
		if err != nil {
			glog.Fatalf("Failed to load config file from %q: %v", *configFilePath, err)
		}
		if err := yaml.Unmarshal(bs, &cfg); err != nil {
			glog.Fatalf("Failed to parse config file at %q: %v", *configFilePath, err)
		}
	}

	if *targetOrg != "" {
		cfg.TargetOrg = *targetOrg
	}
	if *repoName != "" {
		cfg.SourceRepo = *repoName
	}
	if *repoOrg != "" {
		cfg.SourceOrg = *repoOrg
	}

	if cfg.SourceRepo != "kubernetes" {
		BaseRepoPath = filepath.Join(SystemGoPath, "src", "github.com", cfg.TargetOrg)
	}

	if *rulesFile != "" {
		cfg.RulesFile = *rulesFile
	}

	if len(cfg.SourceRepo) == 0 || len(cfg.SourceOrg) == 0 {
		glog.Fatalf("source-org and source-repo cannot be empty")
	}

	if len(cfg.TargetOrg) == 0 {
		glog.Fatalf("Target organization cannot be empty")
	}

	// If RULE_FILE_PATH is detected, check if the source repository include rules files.
	if len(os.Getenv("RULE_FILE_PATH")) > 0 {
		cfg.RulesFile = filepath.Join(BaseRepoPath, cfg.SourceRepo, os.Getenv("RULE_FILE_PATH"))
	}

	if len(cfg.RulesFile) == 0 {
		glog.Fatalf("No rules file provided")
	}
	rules, err := config.LoadRules(cfg.RulesFile)
	if err != nil {
		glog.Fatalf("Failed to load rules: %v", err)
	}

	if err := os.MkdirAll(BaseRepoPath, os.ModePerm); err != nil {
		glog.Fatalf("Failed to create source repo directory %s: %v", BaseRepoPath, err)
	}

	if !*skipGodep {
		installGodeps()
	}
	if !*skipDep {
		installDep()
	}

	cloneSourceRepo(cfg, *skipGodep)
	for _, rule := range rules.Rules {
		cloneForkRepo(cfg, rule.DestinationRepository)
	}
}

func cloneForkRepo(cfg config.Config, repoName string) {
	forkRepoLocation := "https://github.com/" + cfg.TargetOrg + "/" + repoName
	repoDir := filepath.Join(BaseRepoPath, repoName)

	if _, err := os.Stat(repoDir); err == nil {
		glog.Infof("Fork repository %q already cloned to %s, resetting remote URL ...", repoName, repoDir)
		setUrlCmd := exec.Command("git", "remote", "set-url", "origin", forkRepoLocation)
		setUrlCmd.Dir = repoDir
		run(setUrlCmd)
		os.Remove(filepath.Join(repoDir, ".git", "index.lock"))
		return
	}

	glog.Infof("Cloning fork repository %s ...", forkRepoLocation)
	run(exec.Command("git", "clone", forkRepoLocation))

	// TODO: This can be set as an env variable for the container
	setUsernameCmd := exec.Command("git", "config", "user.name", os.Getenv("GIT_COMMITTER_NAME"))
	setUsernameCmd.Dir = repoDir
	run(setUsernameCmd)

	// TODO: This can be set as an env variable for the container
	setEmailCmd := exec.Command("git", "config", "user.email", os.Getenv("GIT_COMMITTER_EMAIL"))
	setEmailCmd.Dir = repoDir
	run(setEmailCmd)
}

func installGodeps() {
	if _, err := exec.LookPath("godep"); err == nil {
		glog.Infof("Already installed: godep")
		return
	}
	glog.Infof("Installing github.com/tools/godep#%s ...", godepCommit)
	run(exec.Command("go", "get", "github.com/tools/godep"))

	godepDir := filepath.Join(SystemGoPath, "src", "github.com", "tools", "godep")
	godepCheckoutCmd := exec.Command("git", "checkout", godepCommit)
	godepCheckoutCmd.Dir = godepDir
	run(godepCheckoutCmd)

	godepInstallCmd := exec.Command("go", "install", "./...")
	godepInstallCmd.Dir = godepDir
	run(godepInstallCmd)
}

func installDep() {
	if _, err := exec.LookPath("dep"); err == nil {
		glog.Infof("Already installed: dep")
		return
	}
	glog.Infof("Installing github.com/golang/dep#%s ...", depCommit)
	depGoGetCmd := exec.Command("go", "get", "github.com/golang/dep")
	run(depGoGetCmd)

	depDir := filepath.Join(SystemGoPath, "src", "github.com", "golang", "dep")
	depCheckoutCmd := exec.Command("git", "checkout", depCommit)
	depCheckoutCmd.Dir = depDir
	run(depCheckoutCmd)

	depInstallCmd := exec.Command("go", "install", "./cmd/dep")
	depInstallCmd.Dir = depDir
	run(depInstallCmd)
}

// run wraps the cmd.Run() command and sets the standard output and common environment variables.
// if the c.Dir is not set, the BaseRepoPath will be used as a base directory for the command.
func run(c *exec.Cmd) {
	c.Stdout = os.Stdout
	c.Stderr = os.Stderr
	if len(c.Dir) == 0 {
		c.Dir = BaseRepoPath
	}
	if err := c.Run(); err != nil {
		glog.Fatalf("Command %q failed: %v", strings.Join(c.Args, " "), err)
	}
}

func cloneSourceRepo(cfg config.Config, runGodepRestore bool) {
	if _, err := os.Stat(filepath.Join(BaseRepoPath, cfg.SourceRepo)); err == nil {
		glog.Infof("Source repository %q already cloned, skipping", cfg.SourceRepo)
		return
	}

	repoLocation := "https://" + filepath.Join("github.com", cfg.SourceOrg, cfg.SourceRepo)
	glog.Infof("Cloning source repository %s ...", repoLocation)
	cloneCmd := exec.Command("git", "clone", repoLocation)
	run(cloneCmd)

	if runGodepRestore {
		glog.Infof("Running hack/godep-restore.sh ...")
		restoreCmd := exec.Command("bash", "-x", "hack/godep-restore.sh")
		restoreCmd.Dir = filepath.Join(BaseRepoPath, cfg.SourceRepo)
		run(restoreCmd)
	}
}
<|endoftext|>