<commit_before>package utils

import (
	"fmt"
	"io/ioutil"
	//"log"
	"net/http"
	"net/url"
	"regexp"
	"strings"
)

// JobqueueHTTPClient provides some utility methods to communicate with the Task Manager runner via HTTP
type JobqueueHTTPClient struct {
	URL url.URL
}

func (client *JobqueueHTTPClient) setDefaultURL() {
	u, _ := url.Parse("http://localhost:8000/")
	client.URL = *u
}

//TODO: support JSON content type

// Open an HTTP connection to control the task manager runner
func (client *JobqueueHTTPClient) Open(url string) (string, error) {
	re1, err := regexp.Compile(`([hftps]+)?://([\w\.]+)?(:\d+)?(/[^\?]+)?(?:\?(.*))?`)
	result := re1.FindAllStringSubmatch(url, -1)[0]
	if nil != err {
		return "", err
	}
	client.setDefaultURL()

	if "" != result[1] {
		client.URL.Scheme = result[1]
	}
	if "" != result[2] {
		client.URL.Host = result[2] + result[3]
	} else if "" != result[3] {
		client.URL.Host = "localhost" + result[3]
	}
	if "" != result[4] {
		client.URL.Path = result[4]
	}
	if "" != result[5] {
		client.URL.RawQuery = result[5]
	}

	return "Connected to " + client.URL.String(), err
}

// get the full URL from the path
func (client *JobqueueHTTPClient) getAddress(path string) string {
	u := client.URL
	u.Path = path
	return u.String()
}

// List the task managers listening at this address
func (client *JobqueueHTTPClient) List() (string, error) {
	resp, err := http.Get(client.getAddress("/list"))
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	bytes, err := ioutil.ReadAll(resp.Body)
	return string(bytes), err
}

// ListWorkers gets the status of each worker process for a given task
func (client *JobqueueHTTPClient) ListWorkers(name string) (string, error) {
	//log.Println("Listing workers for task", name)
	resp, err := http.Get(client.getAddress("/tasks/" + name + "/list"))
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	bytes, err := ioutil.ReadAll(resp.Body)
	return string(bytes), err
}

// ListAsList returns a list of task names (as a string slice)
func (client *JobqueueHTTPClient) ListAsList() []string {
	res, err := client.List()
	if err != nil {
		return []string{}
	}
	return strings.Split(res, "\n")
}

// Set an option on a certain task
func (client *JobqueueHTTPClient) Set(name string, param string, value string) (string, error) {
	//log.Println("Stopping task", name)
	path := fmt.Sprintf("/tasks/%s/set/%s/%s", name, param, value)
	resp, err := http.PostForm(client.getAddress(path), nil)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	bytes, err := ioutil.ReadAll(resp.Body)
	return string(bytes), err
}

// Start a stopped task
func (client *JobqueueHTTPClient) Start(name string) (string, error) {
	//log.Println("Starting task", name)
	resp, err := http.PostForm(client.getAddress("/tasks/"+name+"/start"), nil)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	bytes, err := ioutil.ReadAll(resp.Body)
	return string(bytes), err
}

// Stop a running task, or all of them
func (client *JobqueueHTTPClient) Stop(name string) (string, error) {
	//log.Println("Stopping task", name)
	//resp, err := http.PostForm(client.getAddress("/tasks/"+name+"/stop"), nil)
	req, err := http.NewRequest("DELETE", client.getAddress("/tasks/"+name), nil)
	if err != nil {
		return "", err
	}
	httpClient := &http.Client{}
	resp, err := httpClient.Do(req)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	bytes, err := ioutil.ReadAll(resp.Body)
	return string(bytes), err
}

// Status gets some information about the status of a task (or all of them)
func (client *JobqueueHTTPClient) Status(name string) (string, error) {
	//log.Println("Status")
	resp, err := http.Get(client.getAddress("/tasks/" + name))
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	bytes, err := ioutil.ReadAll(resp.Body)
	return string(bytes), err
}
<commit_msg>accept host with http:// prefix<commit_after>package utils

import (
	"fmt"
	"io/ioutil"
	//"log"
	"net/http"
	"net/url"
	"regexp"
	"strings"
)

// JobqueueHTTPClient provides some utility methods to communicate with the Task Manager runner via HTTP
type JobqueueHTTPClient struct {
	URL url.URL
}

func (client *JobqueueHTTPClient) setDefaultURL() {
	u, _ := url.Parse("http://localhost:8000/")
	client.URL = *u
}

//TODO: support JSON content type

// Open an HTTP connection to control the task manager runner
func (client *JobqueueHTTPClient) Open(url string) (string, error) {
	if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") {
		url = "http://" + url
	}
	re1, err := regexp.Compile(`([hftps]+)?://([\w\.]+)?(:\d+)?(/[^\?]+)?(?:\?(.*))?`)
	if nil != err {
		return "", err
	}
	res := re1.FindAllStringSubmatch(url, -1)
	if nil == res {
		return "", fmt.Errorf("Cannot parse url %s", url)
	}
	result := res[0]
	client.setDefaultURL()

	if "" != result[1] {
		client.URL.Scheme = result[1]
	}
	if "" != result[2] {
		client.URL.Host = result[2] + result[3]
	} else if "" != result[3] {
		client.URL.Host = "localhost" + result[3]
	}
	if "" != result[4] {
		client.URL.Path = result[4]
	}
	if "" != result[5] {
		client.URL.RawQuery = result[5]
	}

	return "Connected to " + client.URL.String(), err
}

// get the full URL from the path
func (client *JobqueueHTTPClient) getAddress(path string) string {
	u := client.URL
	u.Path = path
	return u.String()
}

// List the task managers listening at this address
func (client *JobqueueHTTPClient) List() (string, error) {
	resp, err := http.Get(client.getAddress("/list"))
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	bytes, err := ioutil.ReadAll(resp.Body)
	return string(bytes), err
}

// ListWorkers gets the status of each worker process for a given task
func (client *JobqueueHTTPClient) ListWorkers(name string) (string, error) {
	//log.Println("Listing workers for task", name)
	resp, err := http.Get(client.getAddress("/tasks/" + name + "/list"))
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	bytes, err := ioutil.ReadAll(resp.Body)
	return string(bytes), err
}

// ListAsList returns a list of task names (as a string slice)
func (client *JobqueueHTTPClient) ListAsList() []string {
	res, err := client.List()
	if err != nil {
		return []string{}
	}
	return strings.Split(res, "\n")
}

// Set an option on a certain task
func (client *JobqueueHTTPClient) Set(name string, param string, value string) (string, error) {
	//log.Println("Stopping task", name)
	path := fmt.Sprintf("/tasks/%s/set/%s/%s", name, param, value)
	resp, err := http.PostForm(client.getAddress(path), nil)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	bytes, err := ioutil.ReadAll(resp.Body)
	return string(bytes), err
}

// Start a stopped task
func (client *JobqueueHTTPClient) Start(name string) (string, error) {
	//log.Println("Starting task", name)
	resp, err := http.PostForm(client.getAddress("/tasks/"+name+"/start"), nil)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	bytes, err := ioutil.ReadAll(resp.Body)
	return string(bytes), err
}

// Stop a running task, or all of them
func (client *JobqueueHTTPClient) Stop(name string) (string, error) {
	//log.Println("Stopping task", name)
	//resp, err := http.PostForm(client.getAddress("/tasks/"+name+"/stop"), nil)
	req, err := http.NewRequest("DELETE", client.getAddress("/tasks/"+name), nil)
	if err != nil {
		return "", err
	}
	httpClient := &http.Client{}
	resp, err := httpClient.Do(req)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	bytes, err := ioutil.ReadAll(resp.Body)
	return string(bytes), err
}

// Status gets some information about the status of a task (or all of them)
func (client *JobqueueHTTPClient) Status(name string) (string, error) {
	//log.Println("Status")
	resp, err := http.Get(client.getAddress("/tasks/" + name))
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	bytes, err := ioutil.ReadAll(resp.Body)
	return string(bytes), err
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd

import (
	"bytes"
	"errors"
	"io"
	"io/ioutil"
	"testing"

	"github.com/GoogleContainerTools/skaffold/cmd/skaffold/app/flags"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build"
	"github.com/GoogleContainerTools/skaffold/testutil"
)

func TestQuietFlag(t *testing.T) {
	mockCreateRunner := func(buildOut io.Writer) ([]build.Artifact, error) {
		return []build.Artifact{{
			ImageName: "gcr.io/skaffold/example",
			Tag:       "test",
		}}, nil
	}

	orginalCreateRunner := createRunnerAndBuildFunc
	defer func(c func(buildOut io.Writer) ([]build.Artifact, error)) { createRunnerAndBuildFunc = c }(orginalCreateRunner)
	var tests = []struct {
		name           string
		template       string
		expectedOutput []byte
		mock           func(io.Writer) ([]build.Artifact, error)
		shdErr         bool
	}{
		{
			name:           "quiet flag print build images with no template",
			expectedOutput: []byte("gcr.io/skaffold/example -> test\n"),
			shdErr:         false,
			mock:           mockCreateRunner,
		},
		{
			name:           "quiet flag print build images applies pattern specified in template ",
			template:       "{{.}}",
			expectedOutput: []byte("{[{gcr.io/skaffold/example test}]}"),
			shdErr:         false,
			mock:           mockCreateRunner,
		},
		{
			name:           "build errors out when incorrect template specified",
			template:       "{{.Incorrect}}",
			expectedOutput: nil,
			shdErr:         true,
			mock:           mockCreateRunner,
		},
	}

	for _, test := range tests {
		quietFlag = true
		if test.template != "" {
			buildFormatFlag = flags.NewTemplateFlag(test.template, BuildOutput{})
		}
		createRunnerAndBuildFunc = test.mock
		var output bytes.Buffer
		err := runBuild(&output)
		testutil.CheckErrorAndDeepEqual(t, test.shdErr, err, string(test.expectedOutput), output.String())
	}
}

func TestRunBuild(t *testing.T) {
	mockCreateRunner := func(buildOut io.Writer) ([]build.Artifact, error) {
		return []build.Artifact{{
			ImageName: "gcr.io/skaffold/example",
			Tag:       "test",
		}}, nil
	}
	errRunner := func(buildOut io.Writer) ([]build.Artifact, error) {
		return nil, errors.New("some error")
	}

	orginalCreateRunner := createRunnerAndBuildFunc
	defer func(c func(buildOut io.Writer) ([]build.Artifact, error)) { createRunnerAndBuildFunc = c }(orginalCreateRunner)

	var tests = []struct {
		name   string
		mock   func(io.Writer) ([]build.Artifact, error)
		shdErr bool
	}{
		{
			name:   "buod should return successfully when runner is successful.",
			shdErr: false,
			mock:   mockCreateRunner,
		},
		{
			name:   "build errors out when there was runner error.",
			shdErr: true,
			mock:   errRunner,
		},
	}
	for _, test := range tests {
		createRunnerAndBuildFunc = test.mock
		err := runBuild(ioutil.Discard)
		testutil.CheckError(t, test.shdErr, err)
	}

}
<commit_msg>Fix global variables race condition<commit_after>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd

import (
	"bytes"
	"errors"
	"io"
	"io/ioutil"
	"testing"

	"github.com/GoogleContainerTools/skaffold/cmd/skaffold/app/flags"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build"
	"github.com/GoogleContainerTools/skaffold/testutil"
)

func TestQuietFlag(t *testing.T) {
	mockCreateRunner := func(buildOut io.Writer) ([]build.Artifact, error) {
		return []build.Artifact{{
			ImageName: "gcr.io/skaffold/example",
			Tag:       "test",
		}}, nil
	}

	orginalCreateRunner := createRunnerAndBuildFunc
	defer func(c func(buildOut io.Writer) ([]build.Artifact, error)) { createRunnerAndBuildFunc = c }(orginalCreateRunner)
	var tests = []struct {
		name           string
		template       string
		expectedOutput []byte
		mock           func(io.Writer) ([]build.Artifact, error)
		shdErr         bool
	}{
		{
			name:           "quiet flag print build images with no template",
			expectedOutput: []byte("gcr.io/skaffold/example -> test\n"),
			shdErr:         false,
			mock:           mockCreateRunner,
		},
		{
			name:           "quiet flag print build images applies pattern specified in template ",
			template:       "{{.}}",
			expectedOutput: []byte("{[{gcr.io/skaffold/example test}]}"),
			shdErr:         false,
			mock:           mockCreateRunner,
		},
		{
			name:           "build errors out when incorrect template specified",
			template:       "{{.Incorrect}}",
			expectedOutput: nil,
			shdErr:         true,
			mock:           mockCreateRunner,
		},
	}

	for _, test := range tests {
		quietFlag = true
		defer func() { quietFlag = false }()
		if test.template != "" {
			buildFormatFlag = flags.NewTemplateFlag(test.template, BuildOutput{})
		}
		defer func() { buildFormatFlag = nil }()
		createRunnerAndBuildFunc = test.mock
		var output bytes.Buffer
		err := runBuild(&output)
		testutil.CheckErrorAndDeepEqual(t, test.shdErr, err, string(test.expectedOutput), output.String())
	}
}

func TestRunBuild(t *testing.T) {
	mockCreateRunner := func(buildOut io.Writer) ([]build.Artifact, error) {
		return []build.Artifact{{
			ImageName: "gcr.io/skaffold/example",
			Tag:       "test",
		}}, nil
	}
	errRunner := func(buildOut io.Writer) ([]build.Artifact, error) {
		return nil, errors.New("some error")
	}

	orginalCreateRunner := createRunnerAndBuildFunc
	defer func(c func(buildOut io.Writer) ([]build.Artifact, error)) { createRunnerAndBuildFunc = c }(orginalCreateRunner)

	var tests = []struct {
		name   string
		mock   func(io.Writer) ([]build.Artifact, error)
		shdErr bool
	}{
		{
			name:   "buod should return successfully when runner is successful.",
			shdErr: false,
			mock:   mockCreateRunner,
		},
		{
			name:   "build errors out when there was runner error.",
			shdErr: true,
			mock:   errRunner,
		},
	}
	for _, test := range tests {
		createRunnerAndBuildFunc = test.mock
		err := runBuild(ioutil.Discard)
		testutil.CheckError(t, test.shdErr, err)
	}

}
<|endoftext|>


<commit_before>package config

import (
	"crypto/tls"
	"fmt"
	"strings"
	"time"

	"cloud.google.com/go/pubsub"
	"github.com/aws/aws-sdk-go/service/dynamodb"
	"github.com/aws/aws-sdk-go/service/sqs"
	"go.mongodb.org/mongo-driver/mongo"
)

const (
	// DefaultResultsExpireIn is a default time used to expire task states and group metadata from the backend
	DefaultResultsExpireIn = 24 * 3600
)

var (
	// Start with sensible default values
	defaultCnf = &Config{
		Broker:          "amqp://guest:guest@localhost:5672/",
		DefaultQueue:    "machinery_tasks",
		ResultBackend:   "amqp://guest:guest@localhost:5672/",
		ResultsExpireIn: DefaultResultsExpireIn,
		AMQP: &AMQPConfig{
			Exchange:      "machinery_exchange",
			ExchangeType:  "direct",
			BindingKey:    "machinery_task",
			PrefetchCount: 3,
		},
		DynamoDB: &DynamoDBConfig{
			TaskStatesTable: "task_states",
			GroupMetasTable: "group_metas",
		},
		Redis: &RedisConfig{
			MaxIdle:                3,
			IdleTimeout:            240,
			ReadTimeout:            15,
			WriteTimeout:           15,
			ConnectTimeout:         15,
			DelayedTasksPollPeriod: 20,
		},
		GCPPubSub: &GCPPubSubConfig{
			Client: nil,
		},
	}

	reloadDelay = time.Second * 10
)

// Config holds all configuration for our program
type Config struct {
	Broker          string           `yaml:"broker" envconfig:"BROKER"`
	DefaultQueue    string           `yaml:"default_queue" envconfig:"DEFAULT_QUEUE"`
	ResultBackend   string           `yaml:"result_backend" envconfig:"RESULT_BACKEND"`
	ResultsExpireIn int              `yaml:"results_expire_in" envconfig:"RESULTS_EXPIRE_IN"`
	AMQP            *AMQPConfig      `yaml:"amqp"`
	SQS             *SQSConfig       `yaml:"sqs"`
	Redis           *RedisConfig     `yaml:"redis"`
	GCPPubSub       *GCPPubSubConfig `yaml:"-" ignored:"true"`
	MongoDB         *MongoDBConfig   `yamk:"-" ignored:"true"`
	TLSConfig       *tls.Config
	// NoUnixSignals - when set disables signal handling in machinery
	NoUnixSignals bool            `yaml:"no_unix_signals" envconfig:"NO_UNIX_SIGNALS"`
	DynamoDB      *DynamoDBConfig `yaml:"dynamodb"`
}

// QueueBindingArgs arguments which are used when binding to the exchange
type QueueBindingArgs map[string]interface{}

// AMQPConfig wraps RabbitMQ related configuration
type AMQPConfig struct {
	Exchange         string           `yaml:"exchange" envconfig:"AMQP_EXCHANGE"`
	ExchangeType     string           `yaml:"exchange_type" envconfig:"AMQP_EXCHANGE_TYPE"`
	QueueBindingArgs QueueBindingArgs `yaml:"queue_binding_args" envconfig:"AMQP_QUEUE_BINDING_ARGS"`
	BindingKey       string           `yaml:"binding_key" envconfig:"AMQP_BINDING_KEY"`
	PrefetchCount    int              `yaml:"prefetch_count" envconfig:"AMQP_PREFETCH_COUNT"`
}

// DynamoDBConfig wraps DynamoDB related configuration
type DynamoDBConfig struct {
	Client          *dynamodb.DynamoDB
	TaskStatesTable string `yaml:"task_states_table" envconfig:"TASK_STATES_TABLE"`
	GroupMetasTable string `yaml:"group_metas_table" envconfig:"GROUP_METAS_TABLE"`
}

// SQSConfig wraps SQS related configuration
type SQSConfig struct {
	Client          *sqs.SQS
	WaitTimeSeconds int `yaml:"receive_wait_time_seconds" envconfig:"SQS_WAIT_TIME_SECONDS"`
	// https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html
	// visibility timeout should default to nil to use the overall visibility timeout for the queue
	VisibilityTimeout *int `yaml:"receive_visibility_timeout" envconfig:"SQS_VISIBILITY_TIMEOUT"`
}

// RedisConfig ...
type RedisConfig struct {
	// Maximum number of idle connections in the pool.
	MaxIdle int `yaml:"max_idle" envconfig:"REDIS_MAX_IDLE"`

	// Maximum number of connections allocated by the pool at a given time.
	// When zero, there is no limit on the number of connections in the pool.
	MaxActive int `yaml:"max_active" envconfig:"REDIS_MAX_ACTIVE"`

	// Close connections after remaining idle for this duration in seconds. If the value
	// is zero, then idle connections are not closed. Applications should set
	// the timeout to a value less than the server's timeout.
	IdleTimeout int `yaml:"max_idle_timeout" envconfig:"REDIS_IDLE_TIMEOUT"`

	// If Wait is true and the pool is at the MaxActive limit, then Get() waits
	// for a connection to be returned to the pool before returning.
	Wait bool `yaml:"wait" envconfig:"REDIS_WAIT"`

	// ReadTimeout specifies the timeout in seconds for reading a single command reply.
	ReadTimeout int `yaml:"read_timeout" envconfig:"REDIS_READ_TIMEOUT"`

	// WriteTimeout specifies the timeout in seconds for writing a single command.
	WriteTimeout int `yaml:"write_timeout" envconfig:"REDIS_WRITE_TIMEOUT"`

	// ConnectTimeout specifies the timeout in seconds for connecting to the Redis server when
	// no DialNetDial option is specified.
	ConnectTimeout int `yaml:"connect_timeout" envconfig:"REDIS_CONNECT_TIMEOUT"`

	// DelayedTasksPollPeriod specifies the period in milliseconds when polling redis for delayed tasks
	DelayedTasksPollPeriod int `yaml:"delayed_tasks_poll_period" envconfig:"REDIS_DELAYED_TASKS_POLL_PERIOD"`
}

// GCPPubSubConfig wraps GCP PubSub related configuration
type GCPPubSubConfig struct {
	Client       *pubsub.Client
	MaxExtension time.Duration
}

// MongoDBConfig ...
type MongoDBConfig struct {
	Client   *mongo.Client
	Database string
}

// Decode from yaml to map (any field whose type or pointer-to-type implements
// envconfig.Decoder can control its own deserialization)
func (args *QueueBindingArgs) Decode(value string) error {
	pairs := strings.Split(value, ",")
	mp := make(map[string]interface{}, len(pairs))
	for _, pair := range pairs {
		kvpair := strings.Split(pair, ":")
		if len(kvpair) != 2 {
			return fmt.Errorf("invalid map item: %q", pair)
		}
		mp[kvpair[0]] = kvpair[1]
	}
	*args = QueueBindingArgs(mp)
	return nil
}
<commit_msg>the default expire time is 1 day, not 1 hour<commit_after>package config

import (
	"crypto/tls"
	"fmt"
	"strings"
	"time"

	"cloud.google.com/go/pubsub"
	"github.com/aws/aws-sdk-go/service/dynamodb"
	"github.com/aws/aws-sdk-go/service/sqs"
	"go.mongodb.org/mongo-driver/mongo"
)

const (
	// DefaultResultsExpireIn is a default time used to expire task states and group metadata from the backend
	DefaultResultsExpireIn = 3600
)

var (
	// Start with sensible default values
	defaultCnf = &Config{
		Broker:          "amqp://guest:guest@localhost:5672/",
		DefaultQueue:    "machinery_tasks",
		ResultBackend:   "amqp://guest:guest@localhost:5672/",
		ResultsExpireIn: DefaultResultsExpireIn,
		AMQP: &AMQPConfig{
			Exchange:      "machinery_exchange",
			ExchangeType:  "direct",
			BindingKey:    "machinery_task",
			PrefetchCount: 3,
		},
		DynamoDB: &DynamoDBConfig{
			TaskStatesTable: "task_states",
			GroupMetasTable: "group_metas",
		},
		Redis: &RedisConfig{
			MaxIdle:                3,
			IdleTimeout:            240,
			ReadTimeout:            15,
			WriteTimeout:           15,
			ConnectTimeout:         15,
			DelayedTasksPollPeriod: 20,
		},
		GCPPubSub: &GCPPubSubConfig{
			Client: nil,
		},
	}

	reloadDelay = time.Second * 10
)

// Config holds all configuration for our program
type Config struct {
	Broker          string           `yaml:"broker" envconfig:"BROKER"`
	DefaultQueue    string           `yaml:"default_queue" envconfig:"DEFAULT_QUEUE"`
	ResultBackend   string           `yaml:"result_backend" envconfig:"RESULT_BACKEND"`
	ResultsExpireIn int              `yaml:"results_expire_in" envconfig:"RESULTS_EXPIRE_IN"`
	AMQP            *AMQPConfig      `yaml:"amqp"`
	SQS             *SQSConfig       `yaml:"sqs"`
	Redis           *RedisConfig     `yaml:"redis"`
	GCPPubSub       *GCPPubSubConfig `yaml:"-" ignored:"true"`
	MongoDB         *MongoDBConfig   `yamk:"-" ignored:"true"`
	TLSConfig       *tls.Config
	// NoUnixSignals - when set disables signal handling in machinery
	NoUnixSignals bool            `yaml:"no_unix_signals" envconfig:"NO_UNIX_SIGNALS"`
	DynamoDB      *DynamoDBConfig `yaml:"dynamodb"`
}

// QueueBindingArgs arguments which are used when binding to the exchange
type QueueBindingArgs map[string]interface{}

// AMQPConfig wraps RabbitMQ related configuration
type AMQPConfig struct {
	Exchange         string           `yaml:"exchange" envconfig:"AMQP_EXCHANGE"`
	ExchangeType     string           `yaml:"exchange_type" envconfig:"AMQP_EXCHANGE_TYPE"`
	QueueBindingArgs QueueBindingArgs `yaml:"queue_binding_args" envconfig:"AMQP_QUEUE_BINDING_ARGS"`
	BindingKey       string           `yaml:"binding_key" envconfig:"AMQP_BINDING_KEY"`
	PrefetchCount    int              `yaml:"prefetch_count" envconfig:"AMQP_PREFETCH_COUNT"`
}

// DynamoDBConfig wraps DynamoDB related configuration
type DynamoDBConfig struct {
	Client          *dynamodb.DynamoDB
	TaskStatesTable string `yaml:"task_states_table" envconfig:"TASK_STATES_TABLE"`
	GroupMetasTable string `yaml:"group_metas_table" envconfig:"GROUP_METAS_TABLE"`
}

// SQSConfig wraps SQS related configuration
type SQSConfig struct {
	Client          *sqs.SQS
	WaitTimeSeconds int `yaml:"receive_wait_time_seconds" envconfig:"SQS_WAIT_TIME_SECONDS"`
	// https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html
	// visibility timeout should default to nil to use the overall visibility timeout for the queue
	VisibilityTimeout *int `yaml:"receive_visibility_timeout" envconfig:"SQS_VISIBILITY_TIMEOUT"`
}

// RedisConfig ...
type RedisConfig struct {
	// Maximum number of idle connections in the pool.
	MaxIdle int `yaml:"max_idle" envconfig:"REDIS_MAX_IDLE"`

	// Maximum number of connections allocated by the pool at a given time.
	// When zero, there is no limit on the number of connections in the pool.
	MaxActive int `yaml:"max_active" envconfig:"REDIS_MAX_ACTIVE"`

	// Close connections after remaining idle for this duration in seconds. If the value
	// is zero, then idle connections are not closed. Applications should set
	// the timeout to a value less than the server's timeout.
	IdleTimeout int `yaml:"max_idle_timeout" envconfig:"REDIS_IDLE_TIMEOUT"`

	// If Wait is true and the pool is at the MaxActive limit, then Get() waits
	// for a connection to be returned to the pool before returning.
	Wait bool `yaml:"wait" envconfig:"REDIS_WAIT"`

	// ReadTimeout specifies the timeout in seconds for reading a single command reply.
	ReadTimeout int `yaml:"read_timeout" envconfig:"REDIS_READ_TIMEOUT"`

	// WriteTimeout specifies the timeout in seconds for writing a single command.
	WriteTimeout int `yaml:"write_timeout" envconfig:"REDIS_WRITE_TIMEOUT"`

	// ConnectTimeout specifies the timeout in seconds for connecting to the Redis server when
	// no DialNetDial option is specified.
	ConnectTimeout int `yaml:"connect_timeout" envconfig:"REDIS_CONNECT_TIMEOUT"`

	// DelayedTasksPollPeriod specifies the period in milliseconds when polling redis for delayed tasks
	DelayedTasksPollPeriod int `yaml:"delayed_tasks_poll_period" envconfig:"REDIS_DELAYED_TASKS_POLL_PERIOD"`
}

// GCPPubSubConfig wraps GCP PubSub related configuration
type GCPPubSubConfig struct {
	Client       *pubsub.Client
	MaxExtension time.Duration
}

// MongoDBConfig ...
type MongoDBConfig struct {
	Client   *mongo.Client
	Database string
}

// Decode from yaml to map (any field whose type or pointer-to-type implements
// envconfig.Decoder can control its own deserialization)
func (args *QueueBindingArgs) Decode(value string) error {
	pairs := strings.Split(value, ",")
	mp := make(map[string]interface{}, len(pairs))
	for _, pair := range pairs {
		kvpair := strings.Split(pair, ":")
		if len(kvpair) != 2 {
			return fmt.Errorf("invalid map item: %q", pair)
		}
		mp[kvpair[0]] = kvpair[1]
	}
	*args = QueueBindingArgs(mp)
	return nil
}
<|endoftext|>


<commit_before>package httputils

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"runtime"
	"strconv"
	"time"

	// Below is a port of the exponential backoff implementation from
	// google-http-java-client.
	"github.com/cenkalti/backoff"
	"github.com/fiorix/go-web/autogzip"
	"github.com/skia-dev/glog"
	"go.skia.org/infra/go/metrics2"
	"go.skia.org/infra/go/timer"
	"go.skia.org/infra/go/util"
)

const (
	DIAL_TIMEOUT    = time.Minute
	REQUEST_TIMEOUT = 5 * time.Minute

	// Exponential backoff defaults.
	INITIAL_INTERVAL     = 500 * time.Millisecond
	RANDOMIZATION_FACTOR = 0.5
	BACKOFF_MULTIPLIER   = 1.5
	MAX_INTERVAL         = 60 * time.Second
	MAX_ELAPSED_TIME     = 5 * time.Minute
)

// DialTimeout is a dialer that sets a timeout.
func DialTimeout(network, addr string) (net.Conn, error) {
	return net.DialTimeout(network, addr, DIAL_TIMEOUT)
}

// NewTimeoutClient creates a new http.Client with both a dial timeout and a
// request timeout.
func NewTimeoutClient() *http.Client {
	return &http.Client{
		Transport: &http.Transport{
			Dial: DialTimeout,
		},
		Timeout: REQUEST_TIMEOUT,
	}
}

type BackOffConfig struct {
	initialInterval     time.Duration
	maxInterval         time.Duration
	maxElapsedTime      time.Duration
	randomizationFactor float64
	backOffMultiplier   float64
}

// NewBackOffTransport creates a BackOffTransport with default values. Look at
// NewConfiguredBackOffTransport for an example of how the values impact behavior.
func NewBackOffTransport() http.RoundTripper {
	config := &BackOffConfig{
		initialInterval:     INITIAL_INTERVAL,
		maxInterval:         MAX_INTERVAL,
		maxElapsedTime:      MAX_ELAPSED_TIME,
		randomizationFactor: RANDOMIZATION_FACTOR,
		backOffMultiplier:   BACKOFF_MULTIPLIER,
	}
	return NewConfiguredBackOffTransport(config)
}

type BackOffTransport struct {
	http.Transport
	backOffConfig *BackOffConfig
}

type ResponsePagination struct {
	Offset int `json:"offset"`
	Size   int `json:"size"`
	Total  int `json:"total"`
}

// NewBackOffTransport creates a BackOffTransport with the specified config.
//
// Example: The default retry_interval is .5 seconds, default randomization_factor
// is 0.5, default multiplier is 1.5 and the default max_interval is 1 minute. For
// 10 tries the sequence will be (values in seconds) and assuming we go over the
// max_elapsed_time on the 10th try:
//
//  request#     retry_interval     randomized_interval
//  1             0.5                [0.25,   0.75]
//  2             0.75               [0.375,  1.125]
//  3             1.125              [0.562,  1.687]
//  4             1.687              [0.8435, 2.53]
//  5             2.53               [1.265,  3.795]
//  6             3.795              [1.897,  5.692]
//  7             5.692              [2.846,  8.538]
//  8             8.538              [4.269, 12.807]
//  9            12.807              [6.403, 19.210]
//  10           19.210              backoff.Stop
func NewConfiguredBackOffTransport(config *BackOffConfig) http.RoundTripper {
	return &BackOffTransport{
		Transport:     http.Transport{Dial: DialTimeout},
		backOffConfig: config,
	}
}

// RoundTrip implements the RoundTripper interface.
func (t *BackOffTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	// Initialize the exponential backoff client.
	backOffClient := &backoff.ExponentialBackOff{
		InitialInterval:     t.backOffConfig.initialInterval,
		RandomizationFactor: t.backOffConfig.randomizationFactor,
		Multiplier:          t.backOffConfig.backOffMultiplier,
		MaxInterval:         t.backOffConfig.maxInterval,
		MaxElapsedTime:      t.backOffConfig.maxElapsedTime,
		Clock:               backoff.SystemClock,
	}
	// Make a copy of the request's Body so that we can reuse it if the request
	// needs to be backed off and retried.
	bodyBuf := bytes.Buffer{}
	if req.Body != nil {
		if _, err := bodyBuf.ReadFrom(req.Body); err != nil {
			return nil, fmt.Errorf("Failed to read request body: %v", err)
		}
	}

	var resp *http.Response
	var err error
	roundTripOp := func() error {
		if req.Body != nil {
			req.Body = ioutil.NopCloser(bytes.NewBufferString(bodyBuf.String()))
		}
		resp, err = t.Transport.RoundTrip(req)
		if err != nil {
			return fmt.Errorf("Error while making the round trip: %s", err)
		}
		if resp != nil {
			if resp.StatusCode >= 500 && resp.StatusCode <= 599 {
				if resp.Body != nil {
					util.Close(resp.Body)
				}
				return fmt.Errorf("Got server error statuscode %d while making the HTTP %s request to %s", resp.StatusCode, req.Method, req.URL)
			} else if resp.StatusCode < 200 || resp.StatusCode > 299 {
				if resp.Body != nil {
					util.Close(resp.Body)
				}
				// Stop backing off if there are non server errors.
				backOffClient.MaxElapsedTime = backoff.Stop
				return fmt.Errorf("Got non server error statuscode %d while making the HTTP %s request to %s", resp.StatusCode, req.Method, req.URL)
			}
		}
		return nil
	}
	notifyFunc := func(err error, wait time.Duration) {
		glog.Warningf("Got error: %s. Retrying HTTP request after sleeping for %s", err, wait)
	}

	if err := backoff.RetryNotify(roundTripOp, backOffClient, notifyFunc); err != nil {
		return nil, fmt.Errorf("HTTP request failed inspite of exponential backoff: %s", err)
	}
	return resp, nil
}

// TODO(stephana): Remove 'r' from the argument list since it's not used. It would
// be also useful if we could specify a return status explicitly.

// ReportError formats an HTTP error response and also logs the detailed error message.
// The message parameter is returned in the HTTP response. If it is not provided then
// "Unknown error" will be returned instead.
func ReportError(w http.ResponseWriter, r *http.Request, err error, message string) {
	glog.Errorln(message, err)
	if err != io.ErrClosedPipe {
		httpErrMsg := message
		if message == "" {
			httpErrMsg = "Unknown error"
		}
		http.Error(w, httpErrMsg, 500)
	}
}

// responseProxy implements http.ResponseWriter and records the status codes.
type responseProxy struct {
	http.ResponseWriter
	wroteHeader bool
}

func (rp *responseProxy) WriteHeader(code int) {
	if !rp.wroteHeader {
		glog.Infof("Response Code: %d", code)
		metrics2.GetCounter("http.response", map[string]string{"statuscode": strconv.Itoa(code)}).Inc(1)
		rp.ResponseWriter.WriteHeader(code)
		rp.wroteHeader = true
	}
}

// recordResponse returns a wrapped http.Handler that records the status codes of the
// responses.
//
// Note that if a handler doesn't explicitly set a response code and goes with
// the default of 200 then this will never record anything.
func recordResponse(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		h.ServeHTTP(&responseProxy{ResponseWriter: w}, r)
	})
}

// LoggingGzipRequestResponse records parts of the request and the response to the logs.
func LoggingGzipRequestResponse(h http.Handler) http.Handler {
	// Closure to capture the request.
	f := func(w http.ResponseWriter, r *http.Request) {
		defer func() {
			if err := recover(); err != nil {
				const size = 64 << 10
				buf := make([]byte, size)
				buf = buf[:runtime.Stack(buf, false)]
				glog.Errorf("panic serving %v: %v\n%s", r.URL.Path, err, buf)
			}
		}()
		defer timer.New(fmt.Sprintf("Request: %s %s %#v Content Length: %d Latency:", r.URL.Path, r.Method, r.URL, r.ContentLength)).Stop()
		h.ServeHTTP(w, r)
	}

	return autogzip.Handle(recordResponse(http.HandlerFunc(f)))
}

// MakeResourceHandler is an HTTP handler function designed for serving files.
func MakeResourceHandler(resourcesDir string) func(http.ResponseWriter, *http.Request) {
	fileServer := http.FileServer(http.Dir(resourcesDir))
	return func(w http.ResponseWriter, r *http.Request) {
		w.Header().Add("Cache-Control", "max-age=300")
		fileServer.ServeHTTP(w, r)
	}
}

// CorsHandler is an HTTP handler function which adds the necessary header for CORS.
func CorsHandler(h func(http.ResponseWriter, *http.Request)) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		w.Header().Add("Access-Control-Allow-Origin", "*")
		h(w, r)
	}
}

// PaginationParams is helper function to extract pagination parameters from a
// URL query string. It assumes that 'offset' and 'size' are the query parameters
// used for pagination. It parses the values and returns an error if they are
// not integers. If the params are not set the defaults are proviced.
// Further it ensures that size is never above max size.
func PaginationParams(query url.Values, defaultOffset, defaultSize, maxSize int) (int, int, error) {
	size, err := getPositiveInt(query, "size", defaultSize)
	if err != nil {
		return 0, 0, err
	}

	offset, err := getPositiveInt(query, "offset", defaultOffset)
	if err != nil {
		return 0, 0, err
	}

	return offset, util.MinInt(size, maxSize), nil
}

// getPositiveInt parses the param in query and ensures it is >= 0 using
// default value when necessary.
func getPositiveInt(query url.Values, param string, defaultVal int) (int, error) {
	var val int
	var err error
	if valStr := query.Get(param); valStr == "" {
		return defaultVal, nil
	} else {
		val, err = strconv.Atoi(valStr)
		if err != nil {
			return 0, fmt.Errorf("Not a valid integer value.")
		}
	}
	if val < 0 {
		return defaultVal, nil
	}
	return val, nil
}
<commit_msg>Make the backoff Transport output the response message on failure<commit_after>package httputils

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"runtime"
	"strconv"
	"time"

	// Below is a port of the exponential backoff implementation from
	// google-http-java-client.
	"github.com/cenkalti/backoff"
	"github.com/fiorix/go-web/autogzip"
	"github.com/skia-dev/glog"
	"go.skia.org/infra/go/metrics2"
	"go.skia.org/infra/go/timer"
	"go.skia.org/infra/go/util"
)

const (
	DIAL_TIMEOUT    = time.Minute
	REQUEST_TIMEOUT = 5 * time.Minute

	// Exponential backoff defaults.
	INITIAL_INTERVAL     = 500 * time.Millisecond
	RANDOMIZATION_FACTOR = 0.5
	BACKOFF_MULTIPLIER   = 1.5
	MAX_INTERVAL         = 60 * time.Second
	MAX_ELAPSED_TIME     = 5 * time.Minute

	MAX_BYTES_IN_RESPONSE_BODY = 10 * 1024 //10 KB
)

// DialTimeout is a dialer that sets a timeout.
func DialTimeout(network, addr string) (net.Conn, error) {
	return net.DialTimeout(network, addr, DIAL_TIMEOUT)
}

// NewTimeoutClient creates a new http.Client with both a dial timeout and a
// request timeout.
func NewTimeoutClient() *http.Client {
	return &http.Client{
		Transport: &http.Transport{
			Dial: DialTimeout,
		},
		Timeout: REQUEST_TIMEOUT,
	}
}

type BackOffConfig struct {
	initialInterval     time.Duration
	maxInterval         time.Duration
	maxElapsedTime      time.Duration
	randomizationFactor float64
	backOffMultiplier   float64
}

// NewBackOffTransport creates a BackOffTransport with default values. Look at
// NewConfiguredBackOffTransport for an example of how the values impact behavior.
func NewBackOffTransport() http.RoundTripper {
	config := &BackOffConfig{
		initialInterval:     INITIAL_INTERVAL,
		maxInterval:         MAX_INTERVAL,
		maxElapsedTime:      MAX_ELAPSED_TIME,
		randomizationFactor: RANDOMIZATION_FACTOR,
		backOffMultiplier:   BACKOFF_MULTIPLIER,
	}
	return NewConfiguredBackOffTransport(config)
}

type BackOffTransport struct {
	http.Transport
	backOffConfig *BackOffConfig
}

type ResponsePagination struct {
	Offset int `json:"offset"`
	Size   int `json:"size"`
	Total  int `json:"total"`
}

// NewBackOffTransport creates a BackOffTransport with the specified config.
//
// Example: The default retry_interval is .5 seconds, default randomization_factor
// is 0.5, default multiplier is 1.5 and the default max_interval is 1 minute. For
// 10 tries the sequence will be (values in seconds) and assuming we go over the
// max_elapsed_time on the 10th try:
//
//  request#     retry_interval     randomized_interval
//  1             0.5                [0.25,   0.75]
//  2             0.75               [0.375,  1.125]
//  3             1.125              [0.562,  1.687]
//  4             1.687              [0.8435, 2.53]
//  5             2.53               [1.265,  3.795]
//  6             3.795              [1.897,  5.692]
//  7             5.692              [2.846,  8.538]
//  8             8.538              [4.269, 12.807]
//  9            12.807              [6.403, 19.210]
//  10           19.210              backoff.Stop
func NewConfiguredBackOffTransport(config *BackOffConfig) http.RoundTripper {
	return &BackOffTransport{
		Transport:     http.Transport{Dial: DialTimeout},
		backOffConfig: config,
	}
}

// RoundTrip implements the RoundTripper interface.
func (t *BackOffTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	// Initialize the exponential backoff client.
	backOffClient := &backoff.ExponentialBackOff{
		InitialInterval:     t.backOffConfig.initialInterval,
		RandomizationFactor: t.backOffConfig.randomizationFactor,
		Multiplier:          t.backOffConfig.backOffMultiplier,
		MaxInterval:         t.backOffConfig.maxInterval,
		MaxElapsedTime:      t.backOffConfig.maxElapsedTime,
		Clock:               backoff.SystemClock,
	}
	// Make a copy of the request's Body so that we can reuse it if the request
	// needs to be backed off and retried.
	bodyBuf := bytes.Buffer{}
	if req.Body != nil {
		if _, err := bodyBuf.ReadFrom(req.Body); err != nil {
			return nil, fmt.Errorf("Failed to read request body: %v", err)
		}
	}

	var resp *http.Response
	var err error
	roundTripOp := func() error {
		if req.Body != nil {
			req.Body = ioutil.NopCloser(bytes.NewBufferString(bodyBuf.String()))
		}
		resp, err = t.Transport.RoundTrip(req)
		if err != nil {
			return fmt.Errorf("Error while making the round trip: %s", err)
		}
		if resp != nil {
			if resp.StatusCode >= 500 && resp.StatusCode <= 599 {
				// We can't close the resp.Body on success, so we must do it in each of the failure cases.
				return fmt.Errorf("Got server error statuscode %d while making the HTTP %s request to %s\nResponse: %s", resp.StatusCode, req.Method, req.URL, readAndClose(resp.Body))
			} else if resp.StatusCode < 200 || resp.StatusCode > 299 {
				// We can't close the resp.Body on success, so we must do it in each of the failure cases.
				// Stop backing off if there are non server errors.
				backOffClient.MaxElapsedTime = backoff.Stop
				return fmt.Errorf("Got non server error statuscode %d while making the HTTP %s request to %s\nResponse: %s", resp.StatusCode, req.Method, req.URL, readAndClose(resp.Body))
			}
		}
		return nil
	}
	notifyFunc := func(err error, wait time.Duration) {
		glog.Warningf("Got error: %s. Retrying HTTP request after sleeping for %s", err, wait)
	}

	if err := backoff.RetryNotify(roundTripOp, backOffClient, notifyFunc); err != nil {
		return nil, fmt.Errorf("HTTP request failed inspite of exponential backoff: %s", err)
	}
	return resp, nil
}

// readAndClose reads the content of a ReadCloser (e.g. http Response), and returns it as a string.
// If the response was nil or there was a problem, it will return empty string.  The reader,
//if non-null, will be closed by this function.
func readAndClose(r io.ReadCloser) string {
	if r != nil {
		defer util.Close(r)
		if b, err := ioutil.ReadAll(io.LimitReader(r, MAX_BYTES_IN_RESPONSE_BODY)); err != nil {
			glog.Warningf("There was a potential problem reading the response body: %s", err)
		} else {
			return fmt.Sprintf("%q", string(b))
		}
	}
	return ""
}

// TODO(stephana): Remove 'r' from the argument list since it's not used. It would
// be also useful if we could specify a return status explicitly.

// ReportError formats an HTTP error response and also logs the detailed error message.
// The message parameter is returned in the HTTP response. If it is not provided then
// "Unknown error" will be returned instead.
func ReportError(w http.ResponseWriter, r *http.Request, err error, message string) {
	glog.Errorln(message, err)
	if err != io.ErrClosedPipe {
		httpErrMsg := message
		if message == "" {
			httpErrMsg = "Unknown error"
		}
		http.Error(w, httpErrMsg, 500)
	}
}

// responseProxy implements http.ResponseWriter and records the status codes.
type responseProxy struct {
	http.ResponseWriter
	wroteHeader bool
}

func (rp *responseProxy) WriteHeader(code int) {
	if !rp.wroteHeader {
		glog.Infof("Response Code: %d", code)
		metrics2.GetCounter("http.response", map[string]string{"statuscode": strconv.Itoa(code)}).Inc(1)
		rp.ResponseWriter.WriteHeader(code)
		rp.wroteHeader = true
	}
}

// recordResponse returns a wrapped http.Handler that records the status codes of the
// responses.
//
// Note that if a handler doesn't explicitly set a response code and goes with
// the default of 200 then this will never record anything.
func recordResponse(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		h.ServeHTTP(&responseProxy{ResponseWriter: w}, r)
	})
}

// LoggingGzipRequestResponse records parts of the request and the response to the logs.
func LoggingGzipRequestResponse(h http.Handler) http.Handler {
	// Closure to capture the request.
	f := func(w http.ResponseWriter, r *http.Request) {
		defer func() {
			if err := recover(); err != nil {
				const size = 64 << 10
				buf := make([]byte, size)
				buf = buf[:runtime.Stack(buf, false)]
				glog.Errorf("panic serving %v: %v\n%s", r.URL.Path, err, buf)
			}
		}()
		defer timer.New(fmt.Sprintf("Request: %s %s %#v Content Length: %d Latency:", r.URL.Path, r.Method, r.URL, r.ContentLength)).Stop()
		h.ServeHTTP(w, r)
	}

	return autogzip.Handle(recordResponse(http.HandlerFunc(f)))
}

// MakeResourceHandler is an HTTP handler function designed for serving files.
func MakeResourceHandler(resourcesDir string) func(http.ResponseWriter, *http.Request) {
	fileServer := http.FileServer(http.Dir(resourcesDir))
	return func(w http.ResponseWriter, r *http.Request) {
		w.Header().Add("Cache-Control", "max-age=300")
		fileServer.ServeHTTP(w, r)
	}
}

// CorsHandler is an HTTP handler function which adds the necessary header for CORS.
func CorsHandler(h func(http.ResponseWriter, *http.Request)) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		w.Header().Add("Access-Control-Allow-Origin", "*")
		h(w, r)
	}
}

// PaginationParams is helper function to extract pagination parameters from a
// URL query string. It assumes that 'offset' and 'size' are the query parameters
// used for pagination. It parses the values and returns an error if they are
// not integers. If the params are not set the defaults are proviced.
// Further it ensures that size is never above max size.
func PaginationParams(query url.Values, defaultOffset, defaultSize, maxSize int) (int, int, error) {
	size, err := getPositiveInt(query, "size", defaultSize)
	if err != nil {
		return 0, 0, err
	}

	offset, err := getPositiveInt(query, "offset", defaultOffset)
	if err != nil {
		return 0, 0, err
	}

	return offset, util.MinInt(size, maxSize), nil
}

// getPositiveInt parses the param in query and ensures it is >= 0 using
// default value when necessary.
func getPositiveInt(query url.Values, param string, defaultVal int) (int, error) {
	var val int
	var err error
	if valStr := query.Get(param); valStr == "" {
		return defaultVal, nil
	} else {
		val, err = strconv.Atoi(valStr)
		if err != nil {
			return 0, fmt.Errorf("Not a valid integer value.")
		}
	}
	if val < 0 {
		return defaultVal, nil
	}
	return val, nil
}
<|endoftext|>


<commit_before>// Copyright 2014 The roc Author. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package rocserv

import (
	"fmt"
	"time"
	"encoding/json"
	// now use 73a8ef737e8ea002281a28b4cb92a1de121ad4c6
    //"github.com/coreos/go-etcd/etcd"
    etcd "github.com/coreos/etcd/client"

	"github.com/shawnfeng/sutil/slowid"
	"github.com/shawnfeng/sutil/sconf"
	"github.com/shawnfeng/sutil/slog"
	"github.com/shawnfeng/sutil/ssync"

	"github.com/shawnfeng/dbrouter"

	"golang.org/x/net/context"
)

const (
	BASE_LOC_DIST = "dist"
	// 调整了服务注册结构，为兼容老版本，BASE_LOC_DIST下也要完成之前方式的注册
	// dist2 2为版本2
	BASE_LOC_DIST_V2 = "dist2"
	BASE_LOC_ETC = "etc"
	BASE_LOC_ETC_GLOBAL = "etc/global"
	BASE_LOC_SKEY = "skey"
	BASE_LOC_OP = "op"
	BASE_LOC_DB = "db/route"
	// 服务内分布式锁，只在单个服务副本之间使用
	BASE_LOC_LOCAL_DIST_LOCK = "lock/local"
	// 全局分布式锁，跨服务使用
	BASE_LOC_GLOBAL_DIST_LOCK = "lock/global"

	// 服务注册的位置
	BASE_LOC_REG_SERV = "serve"
	// 后门注册的位置
	BASE_LOC_REG_BACKDOOR = "backdoor"

	// 服务手动配置位置
	BASE_LOC_REG_MANUAL = "manual"
)

type configEtcd struct {
	etcdAddrs []string
	useBaseloc string
}

type ServBaseV2 struct {
	IdGenerator

	confEtcd configEtcd

	dbLocation string
	servLocation string
	copyName string
	sessKey string

	etcdClient etcd.KeysAPI
	servId int

	dbRouter *dbrouter.Router

	muLocks ssync.Mutex
	locks map[string]*ssync.Mutex

	muHearts ssync.Mutex
	hearts map[string]*distLockHeart
}

func (m *ServBaseV2) RegisterBackDoor(servs map[string]*ServInfo) error {
	fun := "ServBaseV2.RegisterBackDoor -->"
	rd := &RegData {
		Servs: servs,
	}

	js, err := json.Marshal(rd)
	if err != nil {
		return err
	}

	slog.Infof("%s servs:%s", fun, js)

	path := fmt.Sprintf("%s/%s/%s/%d/%s", m.confEtcd.useBaseloc, BASE_LOC_DIST_V2, m.servLocation, m.servId, BASE_LOC_REG_BACKDOOR)

	return m.doRegister(path, string(js), true)

}

// {type:http/thrift, addr:10.3.3.3:23233, processor:fuck}
func (m *ServBaseV2) RegisterService(servs map[string]*ServInfo) error {
	fun := "ServBaseV2.RegisterService -->"
	err :=  m.RegisterServiceV2(servs)
	if err != nil {
		slog.Errorf("%s reg v2 err:%s", fun, err)
		return err
	}

	err = m.RegisterServiceV1(servs)
	if err != nil {
		slog.Errorf("%s reg v1 err:%s", fun, err)
		return err
	}

	slog.Errorf("%s regist ok", fun)

	return nil
}

func (m *ServBaseV2) RegisterServiceV2(servs map[string]*ServInfo) error {
	fun := "ServBaseV2.RegisterServiceV2 -->"

	rd := &RegData {
		Servs: servs,
	}

	js, err := json.Marshal(rd)
	if err != nil {
		return err
	}

	slog.Infof("%s servs:%s", fun, js)

	path := fmt.Sprintf("%s/%s/%s/%d/%s", m.confEtcd.useBaseloc, BASE_LOC_DIST_V2, m.servLocation, m.servId, BASE_LOC_REG_SERV)

	return m.doRegister(path, string(js), true)
}

// 为兼容老的client发现服务，保留的
func (m *ServBaseV2) RegisterServiceV1(servs map[string]*ServInfo) error {
	fun := "ServBaseV2.RegisterServiceV1 -->"

	js, err := json.Marshal(servs)
	if err != nil {
		return err
	}

	slog.Infof("%s servs:%s", fun, js)

	path := fmt.Sprintf("%s/%s/%s/%d", m.confEtcd.useBaseloc, BASE_LOC_DIST, m.servLocation, m.servId)

	return m.doRegister(path, string(js), false)
}

func (m *ServBaseV2) doRegister(path, js string, refresh bool) error {
	fun := "ServBaseV2.doRegister -->"
	// 创建完成标志
	var iscreate bool

	go func() {

		for i := 0; ; i++ {
			var err error
			var r *etcd.Response
			if !iscreate {
				slog.Warnf("%s create idx:%d servs:%s", fun, i, js)
				r, err = m.etcdClient.Set(context.Background(), path, js, &etcd.SetOptions {
					TTL: time.Second*120,
				})
			} else {
				if refresh {
					// 在刷新ttl时候，不允许变更value
					// 节点超时时间为120秒
					slog.Infof("%s refresh ttl idx:%d servs:%s", fun, i, js)
					r, err = m.etcdClient.Set(context.Background(), path, "", &etcd.SetOptions {
						PrevExist: etcd.PrevExist,
						TTL: time.Second*120,
						Refresh: true,
					})
				} else {
					r, err = m.etcdClient.Set(context.Background(), path, js, &etcd.SetOptions {
						TTL: time.Second*120,
					})
				}

			}

			if err != nil {
				iscreate = false
				slog.Errorf("%s reg idx:%d err:%s", fun, i, err)

			} else {
				iscreate = true
				jr, _ := json.Marshal(r)
				slog.Infof("%s reg idx:%d ok:%s", fun, i, jr)
			}

			// 每分发起一次注册
			time.Sleep(time.Second * 60)
		}

	}()

	return nil
}

func (m *ServBaseV2) Servid() int {
	return m.servId
}

func (m *ServBaseV2) Copyname() string {
	return fmt.Sprintf("%s%d", m.servLocation, m.servId)

}

func (m *ServBaseV2) Servname() string {
	return m.servLocation
}

func (m *ServBaseV2) Dbrouter() *dbrouter.Router {
	return m.dbRouter
}

func (m *ServBaseV2) ServConfig(cfg interface{}) error {
	fun := "ServBaseV2.ServConfig -->"
	// 获取全局配置
	path := fmt.Sprintf("%s/%s", m.confEtcd.useBaseloc, BASE_LOC_ETC_GLOBAL)
	scfg_global, err := getValue(m.etcdClient, path)
	if err != nil {
		slog.Warnf("%s serv config global value path:%s err:%s", fun, path, err)
	}
	slog.Infof("%s global cfg:%s path:%s", fun, scfg_global, path)

	path = fmt.Sprintf("%s/%s/%s", m.confEtcd.useBaseloc, BASE_LOC_ETC, m.servLocation)
	scfg, err := getValue(m.etcdClient, path)
	if err != nil {
		slog.Warnf("%s serv config value path:%s err:%s", fun, path, err)
	}

	slog.Infof("%s cfg:%s path:%s", fun, scfg, path)
	tf := sconf.NewTierConf()
	err = tf.Load(scfg_global)
	if err != nil {
		return err
	}

	err = tf.Load(scfg)
	if err != nil {
		return err
	}

	err = tf.Unmarshal(cfg)
	if err != nil {
		return err
	}

	return nil
}

// etcd v2 接口
func NewServBaseV2(confEtcd configEtcd, servLocation, skey string) (*ServBaseV2, error) {
	fun := "NewServBaseV2 -->"

	cfg := etcd.Config{
		Endpoints: confEtcd.etcdAddrs,
		Transport: etcd.DefaultTransport,
	}

	c, err := etcd.New(cfg)
	if err != nil {
		return nil, fmt.Errorf("create etchd client cfg error")
	}

    client := etcd.NewKeysAPI(c)
	if client == nil {
		return nil, fmt.Errorf("create etchd api error")
	}

	path := fmt.Sprintf("%s/%s/%s", confEtcd.useBaseloc, BASE_LOC_SKEY, servLocation)

	sid, err := retryGenSid(client, path, skey, 3)
	if err != nil {
		return nil, err
	}

	slog.Infof("%s path:%s sid:%d skey:%s", fun, path, sid, skey)

	dbloc := fmt.Sprintf("%s/%s", confEtcd.useBaseloc, BASE_LOC_DB)

	var dr *dbrouter.Router
	jscfg, err := getValue(client, dbloc)
	if err != nil {
		slog.Warnf("%s db:%s config notfound", fun, dbloc)
	} else {
		dr, err = dbrouter.NewRouter(jscfg)
		if err != nil {
			return nil, err
		}
	}

	reg := &ServBaseV2 {
		confEtcd: confEtcd,
		dbLocation: dbloc,
		servLocation: servLocation,
		sessKey: skey,
		etcdClient: client,
		servId: sid,
		locks: make(map[string]*ssync.Mutex),
		hearts: make(map[string]*distLockHeart),

		dbRouter: dr,

	}

	sf, err := initSnowflake(sid)
	if err != nil {
		return nil, err
	}

	reg.IdGenerator.snow = sf
	reg.IdGenerator.slow = make(map[string]*slowid.Slowid)
	reg.IdGenerator.servId = sid

	return reg, nil

}

// mutex

<commit_msg>close new version service register<commit_after>// Copyright 2014 The roc Author. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package rocserv

import (
	"fmt"
	"time"
	"encoding/json"
	// now use 73a8ef737e8ea002281a28b4cb92a1de121ad4c6
    //"github.com/coreos/go-etcd/etcd"
    etcd "github.com/coreos/etcd/client"

	"github.com/shawnfeng/sutil/slowid"
	"github.com/shawnfeng/sutil/sconf"
	"github.com/shawnfeng/sutil/slog"
	"github.com/shawnfeng/sutil/ssync"

	"github.com/shawnfeng/dbrouter"

	"golang.org/x/net/context"
)

const (
	BASE_LOC_DIST = "dist"
	// 调整了服务注册结构，为兼容老版本，BASE_LOC_DIST下也要完成之前方式的注册
	// dist2 2为版本2
	BASE_LOC_DIST_V2 = "dist2"
	BASE_LOC_ETC = "etc"
	BASE_LOC_ETC_GLOBAL = "etc/global"
	BASE_LOC_SKEY = "skey"
	BASE_LOC_OP = "op"
	BASE_LOC_DB = "db/route"
	// 服务内分布式锁，只在单个服务副本之间使用
	BASE_LOC_LOCAL_DIST_LOCK = "lock/local"
	// 全局分布式锁，跨服务使用
	BASE_LOC_GLOBAL_DIST_LOCK = "lock/global"

	// 服务注册的位置
	BASE_LOC_REG_SERV = "serve"
	// 后门注册的位置
	BASE_LOC_REG_BACKDOOR = "backdoor"

	// 服务手动配置位置
	BASE_LOC_REG_MANUAL = "manual"
)

type configEtcd struct {
	etcdAddrs []string
	useBaseloc string
}

type ServBaseV2 struct {
	IdGenerator

	confEtcd configEtcd

	dbLocation string
	servLocation string
	copyName string
	sessKey string

	etcdClient etcd.KeysAPI
	servId int

	dbRouter *dbrouter.Router

	muLocks ssync.Mutex
	locks map[string]*ssync.Mutex

	muHearts ssync.Mutex
	hearts map[string]*distLockHeart
}

func (m *ServBaseV2) RegisterBackDoor(servs map[string]*ServInfo) error {
	return nil
	fun := "ServBaseV2.RegisterBackDoor -->"
	rd := &RegData {
		Servs: servs,
	}

	js, err := json.Marshal(rd)
	if err != nil {
		return err
	}

	slog.Infof("%s servs:%s", fun, js)

	path := fmt.Sprintf("%s/%s/%s/%d/%s", m.confEtcd.useBaseloc, BASE_LOC_DIST_V2, m.servLocation, m.servId, BASE_LOC_REG_BACKDOOR)

	return m.doRegister(path, string(js), true)

}

// {type:http/thrift, addr:10.3.3.3:23233, processor:fuck}
func (m *ServBaseV2) RegisterService(servs map[string]*ServInfo) error {
	fun := "ServBaseV2.RegisterService -->"
	err :=  m.RegisterServiceV2(servs)
	if err != nil {
		slog.Errorf("%s reg v2 err:%s", fun, err)
		return err
	}

	err = m.RegisterServiceV1(servs)
	if err != nil {
		slog.Errorf("%s reg v1 err:%s", fun, err)
		return err
	}

	slog.Errorf("%s regist ok", fun)

	return nil
}

func (m *ServBaseV2) RegisterServiceV2(servs map[string]*ServInfo) error {
	return nil
	fun := "ServBaseV2.RegisterServiceV2 -->"

	rd := &RegData {
		Servs: servs,
	}

	js, err := json.Marshal(rd)
	if err != nil {
		return err
	}

	slog.Infof("%s servs:%s", fun, js)

	path := fmt.Sprintf("%s/%s/%s/%d/%s", m.confEtcd.useBaseloc, BASE_LOC_DIST_V2, m.servLocation, m.servId, BASE_LOC_REG_SERV)

	return m.doRegister(path, string(js), true)
}

// 为兼容老的client发现服务，保留的
func (m *ServBaseV2) RegisterServiceV1(servs map[string]*ServInfo) error {
	fun := "ServBaseV2.RegisterServiceV1 -->"

	js, err := json.Marshal(servs)
	if err != nil {
		return err
	}

	slog.Infof("%s servs:%s", fun, js)

	path := fmt.Sprintf("%s/%s/%s/%d", m.confEtcd.useBaseloc, BASE_LOC_DIST, m.servLocation, m.servId)

	return m.doRegister(path, string(js), false)
}

func (m *ServBaseV2) doRegister(path, js string, refresh bool) error {
	fun := "ServBaseV2.doRegister -->"
	// 创建完成标志
	var iscreate bool

	go func() {

		for i := 0; ; i++ {
			var err error
			var r *etcd.Response
			if !iscreate {
				slog.Warnf("%s create idx:%d servs:%s", fun, i, js)
				r, err = m.etcdClient.Set(context.Background(), path, js, &etcd.SetOptions {
					TTL: time.Second*120,
				})
			} else {
				if refresh {
					// 在刷新ttl时候，不允许变更value
					// 节点超时时间为120秒
					slog.Infof("%s refresh ttl idx:%d servs:%s", fun, i, js)
					r, err = m.etcdClient.Set(context.Background(), path, "", &etcd.SetOptions {
						PrevExist: etcd.PrevExist,
						TTL: time.Second*120,
						Refresh: true,
					})
				} else {
					r, err = m.etcdClient.Set(context.Background(), path, js, &etcd.SetOptions {
						TTL: time.Second*120,
					})
				}

			}

			if err != nil {
				iscreate = false
				slog.Errorf("%s reg idx:%d err:%s", fun, i, err)

			} else {
				iscreate = true
				jr, _ := json.Marshal(r)
				slog.Infof("%s reg idx:%d ok:%s", fun, i, jr)
			}

			// 每分发起一次注册
			time.Sleep(time.Second * 60)
		}

	}()

	return nil
}

func (m *ServBaseV2) Servid() int {
	return m.servId
}

func (m *ServBaseV2) Copyname() string {
	return fmt.Sprintf("%s%d", m.servLocation, m.servId)

}

func (m *ServBaseV2) Servname() string {
	return m.servLocation
}

func (m *ServBaseV2) Dbrouter() *dbrouter.Router {
	return m.dbRouter
}

func (m *ServBaseV2) ServConfig(cfg interface{}) error {
	fun := "ServBaseV2.ServConfig -->"
	// 获取全局配置
	path := fmt.Sprintf("%s/%s", m.confEtcd.useBaseloc, BASE_LOC_ETC_GLOBAL)
	scfg_global, err := getValue(m.etcdClient, path)
	if err != nil {
		slog.Warnf("%s serv config global value path:%s err:%s", fun, path, err)
	}
	slog.Infof("%s global cfg:%s path:%s", fun, scfg_global, path)

	path = fmt.Sprintf("%s/%s/%s", m.confEtcd.useBaseloc, BASE_LOC_ETC, m.servLocation)
	scfg, err := getValue(m.etcdClient, path)
	if err != nil {
		slog.Warnf("%s serv config value path:%s err:%s", fun, path, err)
	}

	slog.Infof("%s cfg:%s path:%s", fun, scfg, path)
	tf := sconf.NewTierConf()
	err = tf.Load(scfg_global)
	if err != nil {
		return err
	}

	err = tf.Load(scfg)
	if err != nil {
		return err
	}

	err = tf.Unmarshal(cfg)
	if err != nil {
		return err
	}

	return nil
}

// etcd v2 接口
func NewServBaseV2(confEtcd configEtcd, servLocation, skey string) (*ServBaseV2, error) {
	fun := "NewServBaseV2 -->"

	cfg := etcd.Config{
		Endpoints: confEtcd.etcdAddrs,
		Transport: etcd.DefaultTransport,
	}

	c, err := etcd.New(cfg)
	if err != nil {
		return nil, fmt.Errorf("create etchd client cfg error")
	}

    client := etcd.NewKeysAPI(c)
	if client == nil {
		return nil, fmt.Errorf("create etchd api error")
	}

	path := fmt.Sprintf("%s/%s/%s", confEtcd.useBaseloc, BASE_LOC_SKEY, servLocation)

	sid, err := retryGenSid(client, path, skey, 3)
	if err != nil {
		return nil, err
	}

	slog.Infof("%s path:%s sid:%d skey:%s", fun, path, sid, skey)

	dbloc := fmt.Sprintf("%s/%s", confEtcd.useBaseloc, BASE_LOC_DB)

	var dr *dbrouter.Router
	jscfg, err := getValue(client, dbloc)
	if err != nil {
		slog.Warnf("%s db:%s config notfound", fun, dbloc)
	} else {
		dr, err = dbrouter.NewRouter(jscfg)
		if err != nil {
			return nil, err
		}
	}

	reg := &ServBaseV2 {
		confEtcd: confEtcd,
		dbLocation: dbloc,
		servLocation: servLocation,
		sessKey: skey,
		etcdClient: client,
		servId: sid,
		locks: make(map[string]*ssync.Mutex),
		hearts: make(map[string]*distLockHeart),

		dbRouter: dr,

	}

	sf, err := initSnowflake(sid)
	if err != nil {
		return nil, err
	}

	reg.IdGenerator.snow = sf
	reg.IdGenerator.slow = make(map[string]*slowid.Slowid)
	reg.IdGenerator.servId = sid

	return reg, nil

}

// mutex

<|endoftext|>


<commit_before>package netcode

import (
	"io"
	"math"
)

// Buffer is a helper struct for serializing and deserializing as the caller
// does not need to externally manage where in the buffer they are currently reading
// or writing to.
type Buffer struct {
	Buf []byte // the backing byte slice
	Pos int    // current position in read/write
}

// Creates a new Buffer with a backing byte slice of the provided size
func NewBuffer(size int) *Buffer {
	b := &Buffer{}
	b.Buf = make([]byte, size)
	return b
}

// Creates a new Buffer using the original backing slice
func NewBufferFromRef(buf []byte) *Buffer {
	b := &Buffer{}
	b.Buf = buf
	b.Pos = 0
	return b
}

// Creates a new buffer from a byte slice
func NewBufferFromBytes(buf []byte) *Buffer {
	b := &Buffer{}
	b.Buf = make([]byte, len(buf))
	copy(b.Buf, buf)
	return b
}

// Returns a copy of Buffer
func (b *Buffer) Copy() *Buffer {
	c := NewBuffer(len(b.Buf))
	copy(c.Buf, b.Buf)
	return c
}

// Gets the length of the backing byte slice
func (b *Buffer) Len() int {
	return len(b.Buf)
}

// Returns the backing byte slice
func (b *Buffer) Bytes() []byte {
	return b.Buf
}

// Resets the position back to beginning of buffer
func (b *Buffer) Reset() {
	b.Pos = 0
}

// GetByte decodes a little-endian byte
func (b *Buffer) GetByte() (byte, error) {
	return b.GetUint8()
}

// GetBytes returns a byte slice possibly smaller than length if bytes are not available from the
// reader.
func (b *Buffer) GetBytes(length int) ([]byte, error) {
	if len(b.Buf) < length {
		return nil, io.EOF
	}
	value := b.Buf[b.Pos : b.Pos+length]
	b.Pos += length
	return value, nil
}

// GetUint8 decodes a little-endian uint8 from the buffer
func (b *Buffer) GetUint8() (uint8, error) {
	buf, err := b.GetBytes(SizeUint8)
	if err != nil {
		return 0, nil
	}
	return uint8(buf[0]), nil
}

// GetUint16 decodes a little-endian uint16 from the buffer
func (b *Buffer) GetUint16() (uint16, error) {
	var n uint16
	buf, err := b.GetBytes(SizeUint16)
	if err != nil {
		return 0, nil
	}
	n |= uint16(buf[0])
	n |= uint16(buf[1]) << 8
	return n, nil
}

// GetUint32 decodes a little-endian uint32 from the buffer
func (b *Buffer) GetUint32() (uint32, error) {
	var n uint32
	buf, err := b.GetBytes(SizeUint32)
	if err != nil {
		return 0, nil
	}
	n |= uint32(buf[0])
	n |= uint32(buf[1]) << 8
	n |= uint32(buf[2]) << 16
	n |= uint32(buf[3]) << 24
	return n, nil
}

// GetUint64 decodes a little-endian uint64 from the buffer
func (b *Buffer) GetUint64() (uint64, error) {
	var n uint64
	buf, err := b.GetBytes(SizeUint64)
	if err != nil {
		return 0, nil
	}
	n |= uint64(buf[0])
	n |= uint64(buf[1]) << 8
	n |= uint64(buf[2]) << 16
	n |= uint64(buf[3]) << 24
	n |= uint64(buf[4]) << 32
	n |= uint64(buf[5]) << 40
	n |= uint64(buf[6]) << 48
	n |= uint64(buf[7]) << 56
	return n, nil
}

// GetInt8 decodes a little-endian int8 from the buffer
func (b *Buffer) GetInt8() (int8, error) {
	buf, err := b.GetBytes(SizeInt8)
	if err != nil {
		return 0, nil
	}
	return int8(buf[0]), nil
}

// GetInt16 decodes a little-endian int16 from the buffer
func (b *Buffer) GetInt16() (int16, error) {
	var n int16
	buf, err := b.GetBytes(SizeInt16)
	if err != nil {
		return 0, nil
	}
	n |= int16(buf[0])
	n |= int16(buf[1]) << 8
	return n, nil
}

// GetInt32 decodes a little-endian int32 from the buffer
func (b *Buffer) GetInt32() (int32, error) {
	var n int32
	buf, err := b.GetBytes(SizeInt32)
	if err != nil {
		return 0, nil
	}
	n |= int32(buf[0])
	n |= int32(buf[1]) << 8
	n |= int32(buf[2]) << 16
	n |= int32(buf[3]) << 24
	return n, nil
}

// GetInt64 decodes a little-endian int64 from the buffer
func (b *Buffer) GetInt64() (int64, error) {
	var n int64
	buf, err := b.GetBytes(SizeInt64)
	if err != nil {
		return 0, nil
	}
	n |= int64(buf[0])
	n |= int64(buf[1]) << 8
	n |= int64(buf[2]) << 16
	n |= int64(buf[3]) << 24
	n |= int64(buf[4]) << 32
	n |= int64(buf[5]) << 40
	n |= int64(buf[6]) << 48
	n |= int64(buf[7]) << 56
	return n, nil
}

// WriteByte encodes a little-endian uint8 into the buffer.
func (b *Buffer) WriteByte(n byte) {
	b.Buf[b.Pos] = uint8(n)
	b.Pos++
}

// WriteBytes encodes a little-endian byte slice into the buffer
func (b *Buffer) WriteBytes(src []byte) {
	for i := 0; i < len(src); i += 1 {
		b.WriteByte(uint8(src[i]))
	}
}

// WriteBytes encodes a little-endian byte slice into the buffer
func (b *Buffer) WriteBytesN(src []byte, length int) {
	for i := 0; i < length; i += 1 {
		b.WriteByte(uint8(src[i]))
	}
}

// WriteUint8 encodes a little-endian uint8 into the buffer.
func (b *Buffer) WriteUint8(n uint8) {
	b.Buf[b.Pos] = byte(n)
	b.Pos++
}

// WriteUint16 encodes a little-endian uint16 into the buffer.
func (b *Buffer) WriteUint16(n uint16) {
	b.Buf[b.Pos] = byte(n)
	b.Pos++
	b.Buf[b.Pos] = byte(n >> 8)
	b.Pos++
}

// WriteUint32 encodes a little-endian uint32 into the buffer.
func (b *Buffer) WriteUint32(n uint32) {
	b.Buf[b.Pos] = byte(n)
	b.Pos++
	b.Buf[b.Pos] = byte(n >> 8)
	b.Pos++
	b.Buf[b.Pos] = byte(n >> 16)
	b.Pos++
	b.Buf[b.Pos] = byte(n >> 24)
	b.Pos++
}

// WriteUint64 encodes a little-endian uint64 into the buffer.
func (b *Buffer) WriteUint64(n uint64) {
	for i := uint(0); i < uint(SizeUint64); i++ {
		b.Buf[b.Pos] = byte(n >> (i * 8))
		b.Pos++
	}
}

// WriteInt8 encodes a little-endian int8 into the buffer.
func (b *Buffer) WriteInt8(n int8) {
	b.Buf[b.Pos] = byte(n)
	b.Pos++
}

// WriteInt16 encodes a little-endian int16 into the buffer.
func (b *Buffer) WriteInt16(n int16) {
	b.Buf[b.Pos] = byte(n)
	b.Pos++
	b.Buf[b.Pos] = byte(n >> 8)
	b.Pos++
}

// WriteInt32 encodes a little-endian int32 into the buffer.
func (b *Buffer) WriteInt32(n int32) {
	b.Buf[b.Pos] = byte(n)
	b.Pos++
	b.Buf[b.Pos] = byte(n >> 8)
	b.Pos++
	b.Buf[b.Pos] = byte(n >> 16)
	b.Pos++
	b.Buf[b.Pos] = byte(n >> 24)
	b.Pos++
}

// WriteInt64 encodes a little-endian int64 into the buffer.
func (b *Buffer) WriteInt64(n int64) {
	for i := uint(0); i < uint(SizeInt64); i++ {
		b.Buf[b.Pos] = byte(n >> (i * 8))
		b.Pos++
	}
}

// WriteFloat32 encodes a little-endian float32 into the buffer.
func (b *Buffer) WriteFloat32(n float32) {
	b.WriteUint32(math.Float32bits(n))
}

// WriteFloat64 encodes a little-endian float64 into the buffer.
func (b *Buffer) WriteFloat64(buf []byte, n float64) {
	b.WriteUint64(math.Float64bits(n))
}
<commit_msg>Remove unnecessary type conversions<commit_after>package netcode

import (
	"io"
	"math"
)

// Buffer is a helper struct for serializing and deserializing as the caller
// does not need to externally manage where in the buffer they are currently reading
// or writing to.
type Buffer struct {
	Buf []byte // the backing byte slice
	Pos int    // current position in read/write
}

// Creates a new Buffer with a backing byte slice of the provided size
func NewBuffer(size int) *Buffer {
	b := &Buffer{}
	b.Buf = make([]byte, size)
	return b
}

// Creates a new Buffer using the original backing slice
func NewBufferFromRef(buf []byte) *Buffer {
	b := &Buffer{}
	b.Buf = buf
	b.Pos = 0
	return b
}

// Creates a new buffer from a byte slice
func NewBufferFromBytes(buf []byte) *Buffer {
	b := &Buffer{}
	b.Buf = make([]byte, len(buf))
	copy(b.Buf, buf)
	return b
}

// Returns a copy of Buffer
func (b *Buffer) Copy() *Buffer {
	c := NewBuffer(len(b.Buf))
	copy(c.Buf, b.Buf)
	return c
}

// Gets the length of the backing byte slice
func (b *Buffer) Len() int {
	return len(b.Buf)
}

// Returns the backing byte slice
func (b *Buffer) Bytes() []byte {
	return b.Buf
}

// Resets the position back to beginning of buffer
func (b *Buffer) Reset() {
	b.Pos = 0
}

// GetByte decodes a little-endian byte
func (b *Buffer) GetByte() (byte, error) {
	return b.GetUint8()
}

// GetBytes returns a byte slice possibly smaller than length if bytes are not available from the
// reader.
func (b *Buffer) GetBytes(length int) ([]byte, error) {
	if len(b.Buf) < length {
		return nil, io.EOF
	}
	value := b.Buf[b.Pos : b.Pos+length]
	b.Pos += length
	return value, nil
}

// GetUint8 decodes a little-endian uint8 from the buffer
func (b *Buffer) GetUint8() (uint8, error) {
	buf, err := b.GetBytes(SizeUint8)
	if err != nil {
		return 0, nil
	}
	return uint8(buf[0]), nil
}

// GetUint16 decodes a little-endian uint16 from the buffer
func (b *Buffer) GetUint16() (uint16, error) {
	var n uint16
	buf, err := b.GetBytes(SizeUint16)
	if err != nil {
		return 0, nil
	}
	n |= uint16(buf[0])
	n |= uint16(buf[1]) << 8
	return n, nil
}

// GetUint32 decodes a little-endian uint32 from the buffer
func (b *Buffer) GetUint32() (uint32, error) {
	var n uint32
	buf, err := b.GetBytes(SizeUint32)
	if err != nil {
		return 0, nil
	}
	n |= uint32(buf[0])
	n |= uint32(buf[1]) << 8
	n |= uint32(buf[2]) << 16
	n |= uint32(buf[3]) << 24
	return n, nil
}

// GetUint64 decodes a little-endian uint64 from the buffer
func (b *Buffer) GetUint64() (uint64, error) {
	var n uint64
	buf, err := b.GetBytes(SizeUint64)
	if err != nil {
		return 0, nil
	}
	n |= uint64(buf[0])
	n |= uint64(buf[1]) << 8
	n |= uint64(buf[2]) << 16
	n |= uint64(buf[3]) << 24
	n |= uint64(buf[4]) << 32
	n |= uint64(buf[5]) << 40
	n |= uint64(buf[6]) << 48
	n |= uint64(buf[7]) << 56
	return n, nil
}

// GetInt8 decodes a little-endian int8 from the buffer
func (b *Buffer) GetInt8() (int8, error) {
	buf, err := b.GetBytes(SizeInt8)
	if err != nil {
		return 0, nil
	}
	return int8(buf[0]), nil
}

// GetInt16 decodes a little-endian int16 from the buffer
func (b *Buffer) GetInt16() (int16, error) {
	var n int16
	buf, err := b.GetBytes(SizeInt16)
	if err != nil {
		return 0, nil
	}
	n |= int16(buf[0])
	n |= int16(buf[1]) << 8
	return n, nil
}

// GetInt32 decodes a little-endian int32 from the buffer
func (b *Buffer) GetInt32() (int32, error) {
	var n int32
	buf, err := b.GetBytes(SizeInt32)
	if err != nil {
		return 0, nil
	}
	n |= int32(buf[0])
	n |= int32(buf[1]) << 8
	n |= int32(buf[2]) << 16
	n |= int32(buf[3]) << 24
	return n, nil
}

// GetInt64 decodes a little-endian int64 from the buffer
func (b *Buffer) GetInt64() (int64, error) {
	var n int64
	buf, err := b.GetBytes(SizeInt64)
	if err != nil {
		return 0, nil
	}
	n |= int64(buf[0])
	n |= int64(buf[1]) << 8
	n |= int64(buf[2]) << 16
	n |= int64(buf[3]) << 24
	n |= int64(buf[4]) << 32
	n |= int64(buf[5]) << 40
	n |= int64(buf[6]) << 48
	n |= int64(buf[7]) << 56
	return n, nil
}

// WriteByte encodes a little-endian uint8 into the buffer.
func (b *Buffer) WriteByte(n byte) {
	b.Buf[b.Pos] = n
	b.Pos++
}

// WriteBytes encodes a little-endian byte slice into the buffer
func (b *Buffer) WriteBytes(src []byte) {
	for i := 0; i < len(src); i += 1 {
		b.WriteByte(src[i])
	}
}

// WriteBytes encodes a little-endian byte slice into the buffer
func (b *Buffer) WriteBytesN(src []byte, length int) {
	for i := 0; i < length; i += 1 {
		b.WriteByte(src[i])
	}
}

// WriteUint8 encodes a little-endian uint8 into the buffer.
func (b *Buffer) WriteUint8(n uint8) {
	b.Buf[b.Pos] = byte(n)
	b.Pos++
}

// WriteUint16 encodes a little-endian uint16 into the buffer.
func (b *Buffer) WriteUint16(n uint16) {
	b.Buf[b.Pos] = byte(n)
	b.Pos++
	b.Buf[b.Pos] = byte(n >> 8)
	b.Pos++
}

// WriteUint32 encodes a little-endian uint32 into the buffer.
func (b *Buffer) WriteUint32(n uint32) {
	b.Buf[b.Pos] = byte(n)
	b.Pos++
	b.Buf[b.Pos] = byte(n >> 8)
	b.Pos++
	b.Buf[b.Pos] = byte(n >> 16)
	b.Pos++
	b.Buf[b.Pos] = byte(n >> 24)
	b.Pos++
}

// WriteUint64 encodes a little-endian uint64 into the buffer.
func (b *Buffer) WriteUint64(n uint64) {
	for i := uint(0); i < uint(SizeUint64); i++ {
		b.Buf[b.Pos] = byte(n >> (i * 8))
		b.Pos++
	}
}

// WriteInt8 encodes a little-endian int8 into the buffer.
func (b *Buffer) WriteInt8(n int8) {
	b.Buf[b.Pos] = byte(n)
	b.Pos++
}

// WriteInt16 encodes a little-endian int16 into the buffer.
func (b *Buffer) WriteInt16(n int16) {
	b.Buf[b.Pos] = byte(n)
	b.Pos++
	b.Buf[b.Pos] = byte(n >> 8)
	b.Pos++
}

// WriteInt32 encodes a little-endian int32 into the buffer.
func (b *Buffer) WriteInt32(n int32) {
	b.Buf[b.Pos] = byte(n)
	b.Pos++
	b.Buf[b.Pos] = byte(n >> 8)
	b.Pos++
	b.Buf[b.Pos] = byte(n >> 16)
	b.Pos++
	b.Buf[b.Pos] = byte(n >> 24)
	b.Pos++
}

// WriteInt64 encodes a little-endian int64 into the buffer.
func (b *Buffer) WriteInt64(n int64) {
	for i := uint(0); i < uint(SizeInt64); i++ {
		b.Buf[b.Pos] = byte(n >> (i * 8))
		b.Pos++
	}
}

// WriteFloat32 encodes a little-endian float32 into the buffer.
func (b *Buffer) WriteFloat32(n float32) {
	b.WriteUint32(math.Float32bits(n))
}

// WriteFloat64 encodes a little-endian float64 into the buffer.
func (b *Buffer) WriteFloat64(buf []byte, n float64) {
	b.WriteUint64(math.Float64bits(n))
}
<|endoftext|>


<commit_before>// Copyright 2018 PingCAP, Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSES/QL-LICENSE file.

// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package timeutil

import (
	"os"
	"path/filepath"
	"strings"
	"testing"

	. "github.com/pingcap/check"
)

var _ = Suite(&testTimeSuite{})

func TestT(t *testing.T) {
	TestingT(t)
}

type testTimeSuite struct{}

func (s *testTimeSuite) TestgetTZNameFromFileName(c *C) {
	tz, err := inferTZNameFromFileName("/usr/share/zoneinfo/Asia/Shanghai")

	c.Assert(err, IsNil)
	c.Assert(tz, Equals, "Asia/Shanghai")

	tz, err = inferTZNameFromFileName("/usr/share/zoneinfo.default/Asia/Shanghai")

	c.Assert(err, IsNil)
	c.Assert(tz, Equals, "Asia/Shanghai")
}

func (s *testTimeSuite) TestLocal(c *C) {
	os.Setenv("TZ", "Asia/Shanghai")
	systemTZ.Store(InferSystemTZ())
	loc := SystemLocation()
	c.Assert(systemTZ.Load(), Equals, "Asia/Shanghai")
	c.Assert(loc.String(), Equals, "Asia/Shanghai")

	os.Setenv("TZ", "UTC")
	// reset systemTZ
	systemTZ.Store(InferSystemTZ())
	loc = SystemLocation()
	c.Assert(loc.String(), Equals, "UTC")

	os.Setenv("TZ", "")
	// reset systemTZ
	systemTZ.Store(InferSystemTZ())
	loc = SystemLocation()
	c.Assert(loc.String(), Equals, "UTC")
	os.Unsetenv("TZ")
}

func (s *testTimeSuite) TestInferOneStepLinkForPath(c *C) {
	os.Remove("/tmp/testlink1")
	os.Remove("/tmp/testlink2")
	os.Remove("/tmp/testlink3")
	var link2, link3 string
	var err error
	var link1 *os.File
	link1, err = os.Create("/tmp/testlink1")
	c.Assert(err, IsNil)
	err = os.Symlink(link1.Name(), "/tmp/testlink2")
	c.Assert(err, IsNil)
	err = os.Symlink("/tmp/testlink2", "/tmp/testlink3")
	c.Assert(err, IsNil)
	link2, err = inferOneStepLinkForPath("/tmp/testlink3")
	c.Assert(err, IsNil)
	c.Assert(link2, Equals, "/tmp/testlink2")
	link3, err = filepath.EvalSymlinks("/tmp/testlink3")
	c.Assert(err, IsNil)
	c.Assert(strings.Index(link3, link1.Name()), Not(Equals), -1)
}
<commit_msg>util/timeutil: fix timeutil/time_test.go on Windows (#22758)<commit_after>// Copyright 2018 PingCAP, Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSES/QL-LICENSE file.

// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package timeutil

import (
	"os"
	"path/filepath"
	"strings"
	"testing"

	. "github.com/pingcap/check"
)

var _ = Suite(&testTimeSuite{})

func TestT(t *testing.T) {
	TestingT(t)
}

type testTimeSuite struct{}

func (s *testTimeSuite) TestgetTZNameFromFileName(c *C) {
	tz, err := inferTZNameFromFileName("/usr/share/zoneinfo/Asia/Shanghai")

	c.Assert(err, IsNil)
	c.Assert(tz, Equals, "Asia/Shanghai")

	tz, err = inferTZNameFromFileName("/usr/share/zoneinfo.default/Asia/Shanghai")

	c.Assert(err, IsNil)
	c.Assert(tz, Equals, "Asia/Shanghai")
}

func (s *testTimeSuite) TestLocal(c *C) {
	os.Setenv("TZ", "Asia/Shanghai")
	systemTZ.Store(InferSystemTZ())
	loc := SystemLocation()
	c.Assert(systemTZ.Load(), Equals, "Asia/Shanghai")
	c.Assert(loc.String(), Equals, "Asia/Shanghai")

	os.Setenv("TZ", "UTC")
	// reset systemTZ
	systemTZ.Store(InferSystemTZ())
	loc = SystemLocation()
	c.Assert(loc.String(), Equals, "UTC")

	os.Setenv("TZ", "")
	// reset systemTZ
	systemTZ.Store(InferSystemTZ())
	loc = SystemLocation()
	c.Assert(loc.String(), Equals, "UTC")
	os.Unsetenv("TZ")
}

func (s *testTimeSuite) TestInferOneStepLinkForPath(c *C) {
	os.Remove(filepath.Join(os.TempDir(), "testlink1"))
	os.Remove(filepath.Join(os.TempDir(), "testlink2"))
	os.Remove(filepath.Join(os.TempDir(), "testlink3"))
	var link2, link3 string
	var err error
	var link1 *os.File
	link1, err = os.Create(filepath.Join(os.TempDir(), "testlink1"))
	c.Assert(err, IsNil)
	err = os.Symlink(link1.Name(), filepath.Join(os.TempDir(), "testlink2"))
	c.Assert(err, IsNil)
	err = os.Symlink(filepath.Join(os.TempDir(), "testlink2"), filepath.Join(os.TempDir(), "testlink3"))
	c.Assert(err, IsNil)
	link2, err = inferOneStepLinkForPath(filepath.Join(os.TempDir(), "testlink3"))
	c.Assert(err, IsNil)
	c.Assert(link2, Equals, filepath.Join(os.TempDir(), "testlink2"))
	link3, err = filepath.EvalSymlinks(filepath.Join(os.TempDir(), "testlink3"))
	c.Assert(err, IsNil)
	c.Assert(strings.Index(link3, link1.Name()), Not(Equals), -1)
}
<|endoftext|>


<commit_before>package main

import (
  "fmt"
  "strings"
  "crypto/hmac"
  "crypto/sha1"
  "encoding/hex"
  "net/http"
  "os"
  "os/exec"
  "io/ioutil"
  "gopkg.in/robfig/cron.v2"
  "time"
  "strconv"
  "errors"
  "log"
)

type webhook struct {
  Url string
  Repo string
  Mirror_repo string
  Name string
}

func (hook *webhook) init () {
  if hook.Name == "" {
    parts := strings.Split(hook.Repo, "/")
    hook.Name = parts[len(parts) - 1]
  }
  if hook.Url == "" {
    parts := strings.Split(hook.Name, ".git")
    hook.Url = "/" + parts[0]
  }
  if hook.Repo == "" || hook.Mirror_repo == "" {
    err := errors.New("webhook configuration must contain both repo and mirror_repo options")
    log.Fatal(err)
  }

  hook.createCron()
  hook.createRoute()
  hook.setUpRepo()
}

func (hook *webhook) createCron () {
  interval := envDefault("CRON", "@hourly")

  if strings.ToLower(interval) == "false" {
    return
  }
  c := cron.New()
  c.AddFunc(interval, hook.mirrorRepo)
  c.Start()
}

func (hook *webhook) createRoute () {
  http.HandleFunc(hook.Url, hook.ServeHTTP)
}

func (hook *webhook) ServeHTTP(res http.ResponseWriter, req *http.Request) {
  if (verifyRequest(req)){
    go hook.mirrorRepo()
    fmt.Fprintf(res, hook.Repo)
  } else {
    http.Error(res, "400 Bad Request - Missing X-GitHub-Event Header", http.StatusBadRequest)
    return
  }
}

func (hook *webhook) setUpRepo () {
  repoExist, _ := exists (hook.Name)
  if !repoExist {
    fmt.Println("Cloning", hook.Repo)
    runCmd ("git",  []string{"clone", "--mirror", hook.Repo})
  }
  fmt.Println("Setting push remote to ", hook.Mirror_repo)
  runCmd ("git", []string{"remote", "set-url", "--push", "origin", hook.Mirror_repo}, hook.Name)
  hook.mirrorRepo()
}

func (hook *webhook) mirrorRepo () {
  fmt.Println("Pulling", hook.Repo)
  runCmd ("git", []string{"fetch", "-p","origin"}, hook.Name)
  fmt.Println("Pushing", hook.Mirror_repo)
  runCmd ("git", []string{"push", "--mirror"}, hook.Name)
}

func verifyRequest (req *http.Request) bool {
  body, err := ioutil.ReadAll(req.Body)
  handleError(err)

  secret := os.Getenv("SECRET")
  if secret != ""{
    const signaturePrefix = "sha1="
    const signatureLength = 45 // len(SignaturePrefix) + len(hex(sha1))

    sig := req.Header.Get("X-Hub-Signature")
    gitlabToken := req.Header.Get("X-Gitlab-Token")

    if sig == "" && gitlabToken != ""{
      if gitlabToken == secret {
        return true
      } else {
        return false
      }
    } else if len(sig) != signatureLength || !strings.HasPrefix(sig, signaturePrefix) {
      return false
    }

    mac := hmac.New(sha1.New, []byte(secret))
    mac.Write(body)
    expectedMac := mac.Sum(nil)
    expectedSig := "sha1=" + hex.EncodeToString(expectedMac)

    return hmac.Equal([]byte(expectedSig), []byte(sig))
  }
  return true
}

func runCmd (cmd string, args []string, dir ...string) {
  timeoutInt, err := strconv.Atoi(envDefault("TIMEOUT", "10000"))
  handleError(err)

  timeout := time.Duration(timeoutInt) * time.Millisecond

  command := exec.Command(cmd, args...)
  if len(dir) > 0 && dir[0] != "" {
    command.Dir = dir[0]
  }

  err = command.Start();
  handleError(err)

  done := make(chan error)
  go func() { done <- command.Wait() }()

  select {
  case err := <-done:
    handleError(err)
  case <-time.After(timeout):
    command.Process.Kill()
    fmt.Println("timeout")
  }
}
<commit_msg>Fixed only last repo config item url working<commit_after>package main

import (
  "fmt"
  "strings"
  "crypto/hmac"
  "crypto/sha1"
  "encoding/hex"
  "net/http"
  "os"
  "os/exec"
  "io/ioutil"
  "gopkg.in/robfig/cron.v2"
  "time"
  "strconv"
  "errors"
  "log"
)

type webhook struct {
  Url string
  Repo string
  Mirror_repo string
  Name string
}

func (hook *webhook) init () {
  if hook.Name == "" {
    parts := strings.Split(hook.Repo, "/")
    hook.Name = parts[len(parts) - 1]
  }
  if hook.Url == "" {
    parts := strings.Split(hook.Name, ".git")
    hook.Url = "/" + parts[0]
  }
  if hook.Repo == "" || hook.Mirror_repo == "" {
    err := errors.New("webhook configuration must contain both repo and mirror_repo options")
    log.Fatal(err)
  }

  hook.createCron()
  hook.createRoute()
  hook.setUpRepo()
}

func (hook webhook) createCron () {
  interval := envDefault("CRON", "* * 1 * * *")

  if strings.ToLower(interval) == "false" {
    return
  }
  c := cron.New()
  c.AddFunc(interval, hook.mirrorRepo)
  c.Start()
}

func (hook webhook) createRoute () {
  http.HandleFunc(hook.Url, hook.ServeHTTP)
}

func (hook *webhook) ServeHTTP(res http.ResponseWriter, req *http.Request) {
  if (verifyRequest(req)){
    go hook.mirrorRepo()
    fmt.Fprintf(res, hook.Repo)
  } else {
    http.Error(res, "400 Bad Request - Missing X-GitHub-Event Header", http.StatusBadRequest)
    return
  }
}

func (hook *webhook) setUpRepo () {
  repoExist, _ := exists (hook.Name)
  if !repoExist {
    fmt.Println("Cloning", hook.Repo)
    runCmd ("git",  []string{"clone", "--mirror", hook.Repo})
  }
  fmt.Println("Setting push remote to ", hook.Mirror_repo)
  runCmd ("git", []string{"remote", "set-url", "--push", "origin", hook.Mirror_repo}, hook.Name)
  hook.mirrorRepo()
}

func (hook *webhook) mirrorRepo () {
  fmt.Println("Pulling", hook.Repo)
  runCmd ("git", []string{"fetch", "-p","origin"}, hook.Name)
  fmt.Println("Pushing", hook.Mirror_repo)
  runCmd ("git", []string{"push", "--mirror"}, hook.Name)
}

func verifyRequest (req *http.Request) bool {
  body, err := ioutil.ReadAll(req.Body)
  handleError(err)

  secret := os.Getenv("SECRET")
  if secret != ""{
    const signaturePrefix = "sha1="
    const signatureLength = 45 // len(SignaturePrefix) + len(hex(sha1))

    sig := req.Header.Get("X-Hub-Signature")
    gitlabToken := req.Header.Get("X-Gitlab-Token")

    if sig == "" && gitlabToken != ""{
      if gitlabToken == secret {
        return true
      } else {
        return false
      }
    } else if len(sig) != signatureLength || !strings.HasPrefix(sig, signaturePrefix) {
      return false
    }

    mac := hmac.New(sha1.New, []byte(secret))
    mac.Write(body)
    expectedMac := mac.Sum(nil)
    expectedSig := "sha1=" + hex.EncodeToString(expectedMac)

    return hmac.Equal([]byte(expectedSig), []byte(sig))
  }
  return true
}

func runCmd (cmd string, args []string, dir ...string) {
  timeoutInt, err := strconv.Atoi(envDefault("TIMEOUT", "10000"))
  handleError(err)

  timeout := time.Duration(timeoutInt) * time.Millisecond

  command := exec.Command(cmd, args...)
  if len(dir) > 0 && dir[0] != "" {
    command.Dir = dir[0]
  }

  err = command.Start();
  handleError(err)

  done := make(chan error)
  go func() { done <- command.Wait() }()

  select {
  case err := <-done:
    handleError(err)
  case <-time.After(timeout):
    command.Process.Kill()
    fmt.Println("timeout")
  }
}
<|endoftext|>