<commit_before>// Copyright 2018 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package resources

import (
	"github.com/gohugoio/hugo/deps"
	"github.com/gohugoio/hugo/tpl/internal"
)

const name = "resources"

func init() {
	f := func(d *deps.Deps) *internal.TemplateFuncsNamespace {
		ctx, err := New(d)
		if err != nil {
			// TODO(bep) no panic.
			panic(err)
		}

		ns := &internal.TemplateFuncsNamespace{
			Name:    name,
			Context: func(args ...interface{}) (interface{}, error) { return ctx, nil },
		}

		ns.AddMethodMapping(ctx.Get,
			nil,
			[][2]string{},
		)

		// Add aliases for the most common transformations.

		ns.AddMethodMapping(ctx.Fingerprint,
			[]string{"fingerprint"},
			[][2]string{},
		)

		ns.AddMethodMapping(ctx.Minify,
			[]string{"minify"},
			[][2]string{},
		)

		ns.AddMethodMapping(ctx.ToCSS,
			[]string{"toCSS"},
			[][2]string{},
		)

		ns.AddMethodMapping(ctx.PostCSS,
			[]string{"postCSS"},
			[][2]string{},
		)

		ns.AddMethodMapping(ctx.Babel,
			[]string{"babel"},
			[][2]string{},
		)

		return ns
	}

	internal.AddTemplateFuncsNamespace(f)
}
<commit_msg>tpl/resources: Add empty method mapping for GetRemote<commit_after>// Copyright 2018 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package resources

import (
	"github.com/gohugoio/hugo/deps"
	"github.com/gohugoio/hugo/tpl/internal"
)

const name = "resources"

func init() {
	f := func(d *deps.Deps) *internal.TemplateFuncsNamespace {
		ctx, err := New(d)
		if err != nil {
			// TODO(bep) no panic.
			panic(err)
		}

		ns := &internal.TemplateFuncsNamespace{
			Name:    name,
			Context: func(args ...interface{}) (interface{}, error) { return ctx, nil },
		}

		ns.AddMethodMapping(ctx.Get,
			nil,
			[][2]string{},
		)

		ns.AddMethodMapping(ctx.GetRemote,
			nil,
			[][2]string{},
		)

		// Add aliases for the most common transformations.

		ns.AddMethodMapping(ctx.Fingerprint,
			[]string{"fingerprint"},
			[][2]string{},
		)

		ns.AddMethodMapping(ctx.Minify,
			[]string{"minify"},
			[][2]string{},
		)

		ns.AddMethodMapping(ctx.ToCSS,
			[]string{"toCSS"},
			[][2]string{},
		)

		ns.AddMethodMapping(ctx.PostCSS,
			[]string{"postCSS"},
			[][2]string{},
		)

		ns.AddMethodMapping(ctx.Babel,
			[]string{"babel"},
			[][2]string{},
		)

		return ns
	}

	internal.AddTemplateFuncsNamespace(f)
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/rand"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
)

// Job to add a Todo to the database
type AddTodoJob struct {
	toAdd    Todo
	created  chan Todo
	exitChan chan error
}

func NewAddTodoJob(todo Todo) *AddTodoJob {
	return &AddTodoJob{
		toAdd:    todo,
		created:  make(chan Todo, 1),
		exitChan: make(chan error, 1),
	}
}
func (j AddTodoJob) ExitChan() chan error {
	return j.exitChan
}
func (j AddTodoJob) Run(db string) error {
	todos := make(map[string]Todo, 0)
	content, err := ioutil.ReadFile(db)
	if err != nil {
		return err
	}

	if err = json.Unmarshal(content, &todos); err != nil {
		return err
	}

	id, err := newUUID()
	todo := Todo{Id: id, Value: j.toAdd.Value}
	todos[id] = todo

	b, err := json.Marshal(todos)
	if err != nil {
		return err
	}

	if err = ioutil.WriteFile(db, b, 0644); err != nil {
		return err
	}

	j.created <- todo
	return nil
}

func newUUID() (string, error) {
	uuid := make([]byte, 16)
	n, err := io.ReadFull(rand.Reader, uuid)
	if n != len(uuid) || err != nil {
		return "", err
	}
	// variant bits; see section 4.1.1
	uuid[8] = uuid[8]&^0xc0 | 0x80
	// version 4 (pseudo-random); see section 4.1.3
	uuid[6] = uuid[6]&^0xf0 | 0x40
	return fmt.Sprintf("%x-%x-%x-%x-%x", uuid[0:4], uuid[4:6], uuid[6:8], uuid[8:10], uuid[10:]), nil
}
<commit_msg>docs(addJob): documenting uuid function<commit_after>package main

import (
	"crypto/rand"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
)

// Job to add a Todo to the database
type AddTodoJob struct {
	toAdd    Todo
	created  chan Todo
	exitChan chan error
}

func NewAddTodoJob(todo Todo) *AddTodoJob {
	return &AddTodoJob{
		toAdd:    todo,
		created:  make(chan Todo, 1),
		exitChan: make(chan error, 1),
	}
}
func (j AddTodoJob) ExitChan() chan error {
	return j.exitChan
}
func (j AddTodoJob) Run(db string) error {
	todos := make(map[string]Todo, 0)
	content, err := ioutil.ReadFile(db)
	if err != nil {
		return err
	}

	if err = json.Unmarshal(content, &todos); err != nil {
		return err
	}

	id, err := newUUID()
	todo := Todo{Id: id, Value: j.toAdd.Value}
	todos[id] = todo

	b, err := json.Marshal(todos)
	if err != nil {
		return err
	}

	if err = ioutil.WriteFile(db, b, 0644); err != nil {
		return err
	}

	j.created <- todo
	return nil
}

// Generate a uuid to use as a unique identifier for each Todo
// http://play.golang.org/p/4FkNSiUDMg
func newUUID() (string, error) {
	uuid := make([]byte, 16)
	n, err := io.ReadFull(rand.Reader, uuid)
	if n != len(uuid) || err != nil {
		return "", err
	}
	// variant bits; see section 4.1.1
	uuid[8] = uuid[8]&^0xc0 | 0x80
	// version 4 (pseudo-random); see section 4.1.3
	uuid[6] = uuid[6]&^0xf0 | 0x40
	return fmt.Sprintf("%x-%x-%x-%x-%x", uuid[0:4], uuid[4:6], uuid[6:8], uuid[8:10], uuid[10:]), nil
}
<|endoftext|>


<commit_before>package finder

import (
	"bytes"
	"context"
	"fmt"
	"strings"
	"time"

	"github.com/lomik/graphite-clickhouse/helper/clickhouse"
)

const ReverseLevelOffset = 10000
const TreeLevelOffset = 20000
const ReverseTreeLevelOffset = 30000

const DefaultTreeDate = "1970-02-12"

type IndexFinder struct {
	url          string             // clickhouse dsn
	table        string             // graphite_tree table
	opts         clickhouse.Options // timeout, connectTimeout
	dailyEnabled bool
	body         []byte // clickhouse response body
	useReverse   bool
	useDaily     bool
}

func NewIndex(url string, table string, dailyEnabled bool, opts clickhouse.Options) Finder {
	return &IndexFinder{
		url:          url,
		table:        table,
		opts:         opts,
		dailyEnabled: dailyEnabled,
	}
}

func (idx *IndexFinder) where(query string, levelOffset int) *Where {
	level := strings.Count(query, ".") + 1

	w := NewWhere()

	w.Andf("Level = %d", level+levelOffset)

	if query == "*" {
		return w
	}

	// simple metric
	if !HasWildcard(query) {
		w.Andf("Path = %s OR Path = %s", Q(query), Q(query+"."))
		return w
	}

	// before any wildcard symbol
	simplePrefix := query[:strings.IndexAny(query, "[]{}*?")]

	if len(simplePrefix) > 0 {
		w.Andf("Path LIKE %s", Q(likeEscape(simplePrefix)+`%`))
	}

	// prefix search like "metric.name.xx*"
	if len(simplePrefix) == len(query)-1 && query[len(query)-1] == '*' {
		return w
	}

	// Q() replaces \ with \\, so using \. does not work here.
	// work around with [.]
	w.Andf("match(Path, %s)", Q(`^`+GlobToRegexp(query)+`[.]?$`))
	return w
}

func (idx *IndexFinder) Execute(ctx context.Context, query string, from int64, until int64) (err error) {
	p := strings.LastIndexByte(query, '.')

	if p < 0 || p >= len(query)-1 || HasWildcard(query[p+1:]) {
		idx.useReverse = false
	} else {
		idx.useReverse = true
	}

	if idx.dailyEnabled && from > 0 && until > 0 {
		idx.useDaily = true
	} else {
		idx.useDaily = false
	}

	var levelOffset int
	if idx.useDaily {
		if idx.useReverse {
			levelOffset = ReverseLevelOffset
		}
	} else {
		if idx.useReverse {
			levelOffset = ReverseTreeLevelOffset
		} else {
			levelOffset = TreeLevelOffset
		}
	}

	if idx.useReverse {
		query = ReverseString(query)
	}

	where := idx.where(query, levelOffset)

	if idx.useDaily {
		where.Andf(
			"Date >='%s' AND Date <= '%s'",
			time.Unix(from, 0).Format("2006-01-02"),
			time.Unix(until, 0).Format("2006-01-02"),
		)
	}

	idx.body, err = clickhouse.Query(
		ctx,
		idx.url,
		fmt.Sprintf("SELECT Path FROM %s WHERE %s GROUP BY Path", idx.table, where),
		idx.table,
		idx.opts,
	)

	return
}

func (idx *IndexFinder) Abs(v []byte) []byte {
	return v
}

func (idx *IndexFinder) makeList(onlySeries bool) [][]byte {
	if idx.body == nil {
		return [][]byte{}
	}

	rows := bytes.Split(idx.body, []byte{'\n'})

	skip := 0
	for i := 0; i < len(rows); i++ {
		if len(rows[i]) == 0 {
			skip++
			continue
		}
		if onlySeries && rows[i][len(rows[i])-1] == '.' {
			skip++
			continue
		}
		if skip > 0 {
			rows[i-skip] = rows[i]
		}
	}

	rows = rows[:len(rows)-skip]

	if idx.useReverse {
		for i := 0; i < len(rows); i++ {
			rows[i] = ReverseBytes(rows[i])
		}
	}

	return rows
}

func (idx *IndexFinder) List() [][]byte {
	return idx.makeList(false)
}

func (idx *IndexFinder) Series() [][]byte {
	return idx.makeList(true)
}
<commit_msg>date in tree queries<commit_after>package finder

import (
	"bytes"
	"context"
	"fmt"
	"strings"
	"time"

	"github.com/lomik/graphite-clickhouse/helper/clickhouse"
)

const ReverseLevelOffset = 10000
const TreeLevelOffset = 20000
const ReverseTreeLevelOffset = 30000

const DefaultTreeDate = "1970-02-12"

type IndexFinder struct {
	url          string             // clickhouse dsn
	table        string             // graphite_tree table
	opts         clickhouse.Options // timeout, connectTimeout
	dailyEnabled bool
	body         []byte // clickhouse response body
	useReverse   bool
	useDaily     bool
}

func NewIndex(url string, table string, dailyEnabled bool, opts clickhouse.Options) Finder {
	return &IndexFinder{
		url:          url,
		table:        table,
		opts:         opts,
		dailyEnabled: dailyEnabled,
	}
}

func (idx *IndexFinder) where(query string, levelOffset int) *Where {
	level := strings.Count(query, ".") + 1

	w := NewWhere()

	w.Andf("Level = %d", level+levelOffset)

	if query == "*" {
		return w
	}

	// simple metric
	if !HasWildcard(query) {
		w.Andf("Path = %s OR Path = %s", Q(query), Q(query+"."))
		return w
	}

	// before any wildcard symbol
	simplePrefix := query[:strings.IndexAny(query, "[]{}*?")]

	if len(simplePrefix) > 0 {
		w.Andf("Path LIKE %s", Q(likeEscape(simplePrefix)+`%`))
	}

	// prefix search like "metric.name.xx*"
	if len(simplePrefix) == len(query)-1 && query[len(query)-1] == '*' {
		return w
	}

	// Q() replaces \ with \\, so using \. does not work here.
	// work around with [.]
	w.Andf("match(Path, %s)", Q(`^`+GlobToRegexp(query)+`[.]?$`))
	return w
}

func (idx *IndexFinder) Execute(ctx context.Context, query string, from int64, until int64) (err error) {
	p := strings.LastIndexByte(query, '.')

	if p < 0 || p >= len(query)-1 || HasWildcard(query[p+1:]) {
		idx.useReverse = false
	} else {
		idx.useReverse = true
	}

	if idx.dailyEnabled && from > 0 && until > 0 {
		idx.useDaily = true
	} else {
		idx.useDaily = false
	}

	var levelOffset int
	if idx.useDaily {
		if idx.useReverse {
			levelOffset = ReverseLevelOffset
		}
	} else {
		if idx.useReverse {
			levelOffset = ReverseTreeLevelOffset
		} else {
			levelOffset = TreeLevelOffset
		}
	}

	if idx.useReverse {
		query = ReverseString(query)
	}

	where := idx.where(query, levelOffset)

	if idx.useDaily {
		where.Andf(
			"Date >='%s' AND Date <= '%s'",
			time.Unix(from, 0).Format("2006-01-02"),
			time.Unix(until, 0).Format("2006-01-02"),
		)
	} else {
		where.Andf("Date = '%s'", DefaultTreeDate)
	}

	idx.body, err = clickhouse.Query(
		ctx,
		idx.url,
		fmt.Sprintf("SELECT Path FROM %s WHERE %s GROUP BY Path", idx.table, where),
		idx.table,
		idx.opts,
	)

	return
}

func (idx *IndexFinder) Abs(v []byte) []byte {
	return v
}

func (idx *IndexFinder) makeList(onlySeries bool) [][]byte {
	if idx.body == nil {
		return [][]byte{}
	}

	rows := bytes.Split(idx.body, []byte{'\n'})

	skip := 0
	for i := 0; i < len(rows); i++ {
		if len(rows[i]) == 0 {
			skip++
			continue
		}
		if onlySeries && rows[i][len(rows[i])-1] == '.' {
			skip++
			continue
		}
		if skip > 0 {
			rows[i-skip] = rows[i]
		}
	}

	rows = rows[:len(rows)-skip]

	if idx.useReverse {
		for i := 0; i < len(rows); i++ {
			rows[i] = ReverseBytes(rows[i])
		}
	}

	return rows
}

func (idx *IndexFinder) List() [][]byte {
	return idx.makeList(false)
}

func (idx *IndexFinder) Series() [][]byte {
	return idx.makeList(true)
}
<|endoftext|>


<commit_before>package docker

import (
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/fsouza/go-dockerclient"
	"github.com/iron-io/worker/runner/agent"
	"golang.org/x/net/context"
)

// wrap docker client calls so we can retry 500s, kind of sucks but fsouza doesn't
// bake in retries we can use internally, could contribute it at some point, would
// be much more convenient if we didn't have to do this, but it's better than ad hoc retries.
// also adds timeouts to many operations, varying by operation
// TODO could generate this, maybe not worth it, may not change often
type dockerClient interface {
	// Each of these are github.com/fsouza/go-dockerclient methods

	AttachToContainerNonBlocking(opts docker.AttachToContainerOptions) (docker.CloseWaiter, error)
	WaitContainer(ctx context.Context, id string) (int, error)
	StartContainer(id string, hostConfig *docker.HostConfig) error
	CreateContainer(opts docker.CreateContainerOptions) (*docker.Container, error)
	RemoveContainer(opts docker.RemoveContainerOptions) error
	PullImage(opts docker.PullImageOptions, auth docker.AuthConfiguration) error
	InspectImage(name string) (*docker.Image, error)
	InspectContainer(id string) (*docker.Container, error)
	StopContainer(id string, timeout uint) error
	Stats(opts docker.StatsOptions) error
}

// TODO: switch to github.com/docker/engine-api
func newClient() dockerClient {
	// docker, err := docker.NewClient(conf.Docker)
	client, err := docker.NewClientFromEnv()
	if err != nil {
		logrus.WithError(err).Fatal("couldn't create docker client")
	}

	// NOTE add granularity to things like pull, should not effect
	// hijacked / streaming endpoints
	client.SetTimeout(120 * time.Second)

	return &dockerWrap{client}
}

type dockerWrap struct {
	docker *docker.Client
}

func retry(f func() error) error {
	var b agent.Backoff
	then := time.Now()
	limit := 10 * time.Minute
	var err error
	for time.Since(then) < limit {
		err = filter(f())
		if agent.IsTemporary(err) || isDocker500(err) {
			logrus.WithError(err).Warn("docker temporary error, retrying")
			b.Sleep()
			continue
		}
		return err
	}
	logrus.WithError(err).Warnf("retrying on docker errors exceeded %s, restart docker or rotate this instance?", limit)
	return err
}

func isDocker500(err error) bool {
	derr, ok := err.(*docker.Error)
	return ok && derr.Status >= 500
}

// some 500s are totally cool
func filter(err error) error {
	// "API error (500): {\"message\":\"service endpoint with name task-57d722ecdecb9e7be16aff17 already exists\"}\n" -> ok since container exists
	switch {
	default:
		return err
	case err == nil:
		return err
	case strings.Contains(err.Error(), "service endpoint with name"):
	}
	return nil
}

func (d *dockerWrap) AttachToContainerNonBlocking(opts docker.AttachToContainerOptions) (w docker.CloseWaiter, err error) {
	err = retry(func() error {
		w, err = d.docker.AttachToContainerNonBlocking(opts)
		return err
	})
	return w, err
}

func (d *dockerWrap) WaitContainer(ctx context.Context, id string) (code int, err error) {
	// special one, since fsouza doesn't have context on this one and tasks can
	// take longer than 20 minutes
	for {
		// backup bail mechanism so this doesn't sit here forever
		select {
		case <-ctx.Done():
		default:
		}

		err = retry(func() error {
			code, err = d.docker.WaitContainer(id)
			return err
		})
		err = filterNoSuchContainer(err)
		if err == nil {
			break
		}
		logrus.WithError(err).Warn("retrying wait container (this is ok)")
	}
	return code, err
}

func filterNoSuchContainer(err error) error {
	if err == nil {
		return nil
	}
	_, containerNotFound := err.(*docker.NoSuchContainer)
	dockerErr, ok := err.(*docker.Error)
	if containerNotFound || (ok && dockerErr.Status == 404) {
		return nil
	}
	return err
}

func filterNotRunning(err error) error {
	if err == nil {
		return nil
	}

	_, containerNotRunning := err.(*docker.ContainerNotRunning)
	dockerErr, ok := err.(*docker.Error)
	if containerNotRunning || (ok && dockerErr.Status == 304) {
		return nil
	}

	return err
}

func (d *dockerWrap) StartContainer(id string, hostConfig *docker.HostConfig) (err error) {
	err = retry(func() error {
		err = d.docker.StartContainer(id, hostConfig)
		return err
	})
	return err
}

func (d *dockerWrap) CreateContainer(opts docker.CreateContainerOptions) (c *docker.Container, err error) {
	err = retry(func() error {
		c, err = d.docker.CreateContainer(opts)
		return err
	})
	return c, err
}

func (d *dockerWrap) RemoveContainer(opts docker.RemoveContainerOptions) (err error) {
	err = retry(func() error {
		err = d.docker.RemoveContainer(opts)
		return err
	})
	return filterNoSuchContainer(err)
}

func (d *dockerWrap) PullImage(opts docker.PullImageOptions, auth docker.AuthConfiguration) (err error) {
	err = retry(func() error {
		err = d.docker.PullImage(opts, auth)
		return err
	})
	return err
}

func (d *dockerWrap) InspectImage(name string) (i *docker.Image, err error) {
	err = retry(func() error {
		i, err = d.docker.InspectImage(name)
		return err
	})
	return i, err
}

func (d *dockerWrap) InspectContainer(id string) (c *docker.Container, err error) {
	err = retry(func() error {
		c, err = d.docker.InspectContainer(id)
		return err
	})
	return c, err
}

func (d *dockerWrap) StopContainer(id string, timeout uint) (err error) {
	err = retry(func() error {
		err = d.docker.StopContainer(id, timeout)
		return err
	})
	return filterNotRunning(filterNoSuchContainer(err))
}

func (d *dockerWrap) Stats(opts docker.StatsOptions) (err error) {
	err = retry(func() error {
		err = d.docker.Stats(opts)
		return err
	})
	return err
}
<commit_msg>log filtered errors<commit_after>package docker

import (
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/fsouza/go-dockerclient"
	"github.com/iron-io/worker/runner/agent"
	"golang.org/x/net/context"
)

// wrap docker client calls so we can retry 500s, kind of sucks but fsouza doesn't
// bake in retries we can use internally, could contribute it at some point, would
// be much more convenient if we didn't have to do this, but it's better than ad hoc retries.
// also adds timeouts to many operations, varying by operation
// TODO could generate this, maybe not worth it, may not change often
type dockerClient interface {
	// Each of these are github.com/fsouza/go-dockerclient methods

	AttachToContainerNonBlocking(opts docker.AttachToContainerOptions) (docker.CloseWaiter, error)
	WaitContainer(ctx context.Context, id string) (int, error)
	StartContainer(id string, hostConfig *docker.HostConfig) error
	CreateContainer(opts docker.CreateContainerOptions) (*docker.Container, error)
	RemoveContainer(opts docker.RemoveContainerOptions) error
	PullImage(opts docker.PullImageOptions, auth docker.AuthConfiguration) error
	InspectImage(name string) (*docker.Image, error)
	InspectContainer(id string) (*docker.Container, error)
	StopContainer(id string, timeout uint) error
	Stats(opts docker.StatsOptions) error
}

// TODO: switch to github.com/docker/engine-api
func newClient() dockerClient {
	// docker, err := docker.NewClient(conf.Docker)
	client, err := docker.NewClientFromEnv()
	if err != nil {
		logrus.WithError(err).Fatal("couldn't create docker client")
	}

	// NOTE add granularity to things like pull, should not effect
	// hijacked / streaming endpoints
	client.SetTimeout(120 * time.Second)

	return &dockerWrap{client}
}

type dockerWrap struct {
	docker *docker.Client
}

func retry(f func() error) error {
	var b agent.Backoff
	then := time.Now()
	limit := 10 * time.Minute
	var err error
	for time.Since(then) < limit {
		err = filter(f())
		if agent.IsTemporary(err) || isDocker500(err) {
			logrus.WithError(err).Warn("docker temporary error, retrying")
			b.Sleep()
			continue
		}
		return err
	}
	logrus.WithError(err).Warnf("retrying on docker errors exceeded %s, restart docker or rotate this instance?", limit)
	return err
}

func isDocker500(err error) bool {
	derr, ok := err.(*docker.Error)
	return ok && derr.Status >= 500
}

// some 500s are totally cool
func filter(err error) error {
	// "API error (500): {\"message\":\"service endpoint with name task-57d722ecdecb9e7be16aff17 already exists\"}\n" -> ok since container exists
	switch {
	default:
		return err
	case err == nil:
		return err
	case strings.Contains(err.Error(), "service endpoint with name"):
	}
	logrus.WithError(err).Warn("filtering error")
	return nil
}

func (d *dockerWrap) AttachToContainerNonBlocking(opts docker.AttachToContainerOptions) (w docker.CloseWaiter, err error) {
	err = retry(func() error {
		w, err = d.docker.AttachToContainerNonBlocking(opts)
		return err
	})
	return w, err
}

func (d *dockerWrap) WaitContainer(ctx context.Context, id string) (code int, err error) {
	// special one, since fsouza doesn't have context on this one and tasks can
	// take longer than 20 minutes
	for {
		// backup bail mechanism so this doesn't sit here forever
		select {
		case <-ctx.Done():
		default:
		}

		err = retry(func() error {
			code, err = d.docker.WaitContainer(id)
			return err
		})
		err = filterNoSuchContainer(err)
		if err == nil {
			break
		}
		logrus.WithError(err).Warn("retrying wait container (this is ok)")
	}
	return code, err
}

func filterNoSuchContainer(err error) error {
	if err == nil {
		return nil
	}
	_, containerNotFound := err.(*docker.NoSuchContainer)
	dockerErr, ok := err.(*docker.Error)
	if containerNotFound || (ok && dockerErr.Status == 404) {
		return nil
	}
	return err
}

func filterNotRunning(err error) error {
	if err == nil {
		return nil
	}

	_, containerNotRunning := err.(*docker.ContainerNotRunning)
	dockerErr, ok := err.(*docker.Error)
	if containerNotRunning || (ok && dockerErr.Status == 304) {
		return nil
	}

	return err
}

func (d *dockerWrap) StartContainer(id string, hostConfig *docker.HostConfig) (err error) {
	err = retry(func() error {
		err = d.docker.StartContainer(id, hostConfig)
		return err
	})
	return err
}

func (d *dockerWrap) CreateContainer(opts docker.CreateContainerOptions) (c *docker.Container, err error) {
	err = retry(func() error {
		c, err = d.docker.CreateContainer(opts)
		return err
	})
	return c, err
}

func (d *dockerWrap) RemoveContainer(opts docker.RemoveContainerOptions) (err error) {
	err = retry(func() error {
		err = d.docker.RemoveContainer(opts)
		return err
	})
	return filterNoSuchContainer(err)
}

func (d *dockerWrap) PullImage(opts docker.PullImageOptions, auth docker.AuthConfiguration) (err error) {
	err = retry(func() error {
		err = d.docker.PullImage(opts, auth)
		return err
	})
	return err
}

func (d *dockerWrap) InspectImage(name string) (i *docker.Image, err error) {
	err = retry(func() error {
		i, err = d.docker.InspectImage(name)
		return err
	})
	return i, err
}

func (d *dockerWrap) InspectContainer(id string) (c *docker.Container, err error) {
	err = retry(func() error {
		c, err = d.docker.InspectContainer(id)
		return err
	})
	return c, err
}

func (d *dockerWrap) StopContainer(id string, timeout uint) (err error) {
	err = retry(func() error {
		err = d.docker.StopContainer(id, timeout)
		return err
	})
	return filterNotRunning(filterNoSuchContainer(err))
}

func (d *dockerWrap) Stats(opts docker.StatsOptions) (err error) {
	err = retry(func() error {
		err = d.docker.Stats(opts)
		return err
	})
	return err
}
<|endoftext|>


<commit_before>//Package server implements the http frontend
package server

import (
	"fmt"
	"log"
	"net/http"
	"strconv"
	"strings"
	"sync"

	"github.com/golang/protobuf/proto"
	"github.com/terranodo/tegola"
	"github.com/terranodo/tegola/basic"
	"github.com/terranodo/tegola/mvt"
)

const (
	//	MaxTileSize is 500k. Currently just throws a warning when tile
	//	is larger than MaxTileSize
	MaxTileSize = 500000
	//	MaxZoom will not render tile beyond this zoom level
	MaxZoom = 20
)

//	URI scheme: /maps/:map_name/:z/:x/:y
//	map_name - map name in the config file
//	z, x, y - tile coordinates as described in the Slippy Map Tilenames specification
//		z - zoom level
//		x - row
//		y - column
func handleZXY(w http.ResponseWriter, r *http.Request) {
	//	check http verb
	switch r.Method {
	//	preflight check for CORS request
	case "OPTIONS":
		//	TODO: how configurable do we want the CORS policy to be?
		//	set CORS header
		w.Header().Add("Access-Control-Allow-Origin", "*")

		//	options call does not have a body
		w.Write(nil)
		return
	//	tile request
	case "GET":
		//	pop off URI prefix
		uri := r.URL.Path[len("/maps/"):]

		//	break apart our URI
		uriParts := strings.Split(uri, "/")

		//	check that we have the correct number of arguments in our URI
		if len(uriParts) != 4 {
			http.Error(w, "uri requires four params: /:map_id/:z/:x/:y", http.StatusBadRequest)
			return
		}

		//	lookup our map layers
		layers, ok := maps[uriParts[0]]
		if !ok {
			log.Printf("map (%v) not configured. check your config file", uriParts[0])
			http.Error(w, "no map configured: "+uriParts[0], http.StatusBadRequest)
			return
		}

		//	trim the "y" param in the url in case it has an extension
		yparts := strings.Split(uriParts[3], ".")
		uriParts[3] = yparts[0]

		//	parse our URL vals to ints
		z, err := strconv.Atoi(uriParts[1])
		if err != nil {
			http.Error(w, "invalid z value: "+uriParts[1], http.StatusBadRequest)
			return
		}

		x, err := strconv.Atoi(uriParts[2])
		if err != nil {
			http.Error(w, "invalid x value: "+uriParts[2], http.StatusBadRequest)
			return
		}

		y, err := strconv.Atoi(uriParts[3])
		if err != nil {
			http.Error(w, "invalid y value: "+uriParts[3], http.StatusBadRequest)
			return
		}

		//	new tile
		tile := tegola.Tile{
			Z: z,
			X: x,
			Y: y,
		}

		//	generate a tile
		var mvtTile mvt.Tile

		//	check that our request is below max zoom and we have layers to render
		if tile.Z <= MaxZoom && len(layers) != 0 {

			//	wait group for concurrent layer fetching
			var wg sync.WaitGroup
			//	filter down the layers we need for this zoom
			ls := layers.FilterByZoom(tile.Z)

			//	layer stack
			mvtLayers := make([]*mvt.Layer, len(ls))

			//	iterate our layers
			for i, l := range ls {
				//	incriment our waitgroup
				wg.Add(1)
				//	go routine for rendering the layer
				go func(i int, l Layer) {
					//	on completion let the wait group know
					defer wg.Done()

					//	fetch layer from data provider
					mvtLayer, err := l.Provider.MVTLayer(l.Name, tile, l.DefaultTags)
					if err != nil {
						log.Printf("Error Getting MVTLayer: %v", err)
						http.Error(w, fmt.Sprintf("Error Getting MVTLayer: %v", err.Error()), http.StatusBadRequest)
						return
					}
					//	add the layer to the slice position
					mvtLayers[i] = mvtLayer
				}(i, l)
			}

			//	wait for the waitgroup to finish
			wg.Wait()

			//	add layers to our tile
			mvtTile.AddLayers(mvtLayers...)
		}

		//	check for the debug query string
		debug := r.URL.Query().Get("debug")
		if debug == "true" {
			//	add debug layer
			debugLayer := debugLayer(tile)
			mvtTile.AddLayers(debugLayer)
		}

		//	generate our vector tile
		vtile, err := mvtTile.VTile(tile.BoundingBox())
		if err != nil {
			log.Printf("Error Getting VTile: %v", err)
			http.Error(w, fmt.Sprintf("Error Getting VTile: %v", err.Error()), http.StatusBadRequest)
			return
		}

		//	marshal our tile into a protocol buffer
		var pbyte []byte
		pbyte, err = proto.Marshal(vtile)
		if err != nil {
			log.Printf("Error marshalling tile: %v", err)
			http.Error(w, "Error marshalling tile", http.StatusInternalServerError)
			return
		}

		//	TODO: how configurable do we want the CORS policy to be?
		//	set CORS header
		w.Header().Add("Access-Control-Allow-Origin", "*")

		//	mimetype for protocol buffers
		w.Header().Add("Content-Type", "application/x-protobuf")

		w.Write(pbyte)

		//	check for tile size warnings
		if len(pbyte) > MaxTileSize {
			log.Printf("tile z:%v, x:%v, y:%v is rather large - %v", tile.Z, tile.X, tile.Y, len(pbyte))
		}

		//	log the request
		L.Log(logItem{
			X:         tile.X,
			Y:         tile.Y,
			Z:         tile.Z,
			RequestIP: r.RemoteAddr,
		})

	default:
		http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
		return
	}
}

//	creates a debug layer with z/x/y encoded as a point
func debugLayer(tile tegola.Tile) *mvt.Layer {
	//	get tile bounding box
	ext := tile.BoundingBox()

	//	create a new layer and name it
	layer := mvt.Layer{
		Name: "debug",
	}
	xlen := ext.Maxx - ext.Minx
	ylen := ext.Maxy - ext.Miny

	//	tile outlines
	outline := mvt.Feature{
		Tags: map[string]interface{}{
			"type": "debug_outline",
		},
		Geometry: &basic.Line{ //	tile outline
			basic.Point{ext.Minx, ext.Miny},
			basic.Point{ext.Maxx, ext.Miny},
			basic.Point{ext.Maxx, ext.Maxy},
			basic.Point{ext.Minx, ext.Maxy},
		},
	}

	//	new feature
	zxy := mvt.Feature{
		Tags: map[string]interface{}{
			"type": "debug_text",
			"zxy":  fmt.Sprintf("Z:%v, X:%v, Y:%v", tile.Z, tile.X, tile.Y),
		},
		Geometry: &basic.Point{ //	middle of the tile
			ext.Minx + (xlen / 2),
			ext.Miny + (ylen / 2),
		},
	}

	layer.AddFeatures(outline, zxy)

	return &layer
}
<commit_msg>minor waitgroup optimization in parallel layer fetching<commit_after>//Package server implements the http frontend
package server

import (
	"fmt"
	"log"
	"net/http"
	"strconv"
	"strings"
	"sync"

	"github.com/golang/protobuf/proto"
	"github.com/terranodo/tegola"
	"github.com/terranodo/tegola/basic"
	"github.com/terranodo/tegola/mvt"
)

const (
	//	MaxTileSize is 500k. Currently just throws a warning when tile
	//	is larger than MaxTileSize
	MaxTileSize = 500000
	//	MaxZoom will not render tile beyond this zoom level
	MaxZoom = 20
)

//	URI scheme: /maps/:map_name/:z/:x/:y
//	map_name - map name in the config file
//	z, x, y - tile coordinates as described in the Slippy Map Tilenames specification
//		z - zoom level
//		x - row
//		y - column
func handleZXY(w http.ResponseWriter, r *http.Request) {
	//	check http verb
	switch r.Method {
	//	preflight check for CORS request
	case "OPTIONS":
		//	TODO: how configurable do we want the CORS policy to be?
		//	set CORS header
		w.Header().Add("Access-Control-Allow-Origin", "*")

		//	options call does not have a body
		w.Write(nil)
		return
	//	tile request
	case "GET":
		//	pop off URI prefix
		uri := r.URL.Path[len("/maps/"):]

		//	break apart our URI
		uriParts := strings.Split(uri, "/")

		//	check that we have the correct number of arguments in our URI
		if len(uriParts) != 4 {
			http.Error(w, "uri requires four params: /:map_id/:z/:x/:y", http.StatusBadRequest)
			return
		}

		//	lookup our map layers
		layers, ok := maps[uriParts[0]]
		if !ok {
			log.Printf("map (%v) not configured. check your config file", uriParts[0])
			http.Error(w, "no map configured: "+uriParts[0], http.StatusBadRequest)
			return
		}

		//	trim the "y" param in the url in case it has an extension
		yparts := strings.Split(uriParts[3], ".")
		uriParts[3] = yparts[0]

		//	parse our URL vals to ints
		z, err := strconv.Atoi(uriParts[1])
		if err != nil {
			http.Error(w, "invalid z value: "+uriParts[1], http.StatusBadRequest)
			return
		}

		x, err := strconv.Atoi(uriParts[2])
		if err != nil {
			http.Error(w, "invalid x value: "+uriParts[2], http.StatusBadRequest)
			return
		}

		y, err := strconv.Atoi(uriParts[3])
		if err != nil {
			http.Error(w, "invalid y value: "+uriParts[3], http.StatusBadRequest)
			return
		}

		//	new tile
		tile := tegola.Tile{
			Z: z,
			X: x,
			Y: y,
		}

		//	generate a tile
		var mvtTile mvt.Tile

		//	check that our request is below max zoom and we have layers to render
		if tile.Z <= MaxZoom && len(layers) != 0 {

			//	wait group for concurrent layer fetching
			var wg sync.WaitGroup
			//	filter down the layers we need for this zoom
			ls := layers.FilterByZoom(tile.Z)

			//	layer stack
			mvtLayers := make([]*mvt.Layer, len(ls))

			//	set our waitgroup count
			wg.Add(len(ls))

			//	iterate our layers
			for i, l := range ls {
				//	go routine for rendering the layer
				go func(i int, l Layer) {
					//	on completion let the wait group know
					defer wg.Done()

					//	fetch layer from data provider
					mvtLayer, err := l.Provider.MVTLayer(l.Name, tile, l.DefaultTags)
					if err != nil {
						log.Printf("Error Getting MVTLayer: %v", err)
						http.Error(w, fmt.Sprintf("Error Getting MVTLayer: %v", err.Error()), http.StatusBadRequest)
						return
					}
					//	add the layer to the slice position
					mvtLayers[i] = mvtLayer
				}(i, l)
			}

			//	wait for the waitgroup to finish
			wg.Wait()

			//	add layers to our tile
			mvtTile.AddLayers(mvtLayers...)
		}

		//	check for the debug query string
		debug := r.URL.Query().Get("debug")
		if debug == "true" {
			//	add debug layer
			debugLayer := debugLayer(tile)
			mvtTile.AddLayers(debugLayer)
		}

		//	generate our vector tile
		vtile, err := mvtTile.VTile(tile.BoundingBox())
		if err != nil {
			log.Printf("Error Getting VTile: %v", err)
			http.Error(w, fmt.Sprintf("Error Getting VTile: %v", err.Error()), http.StatusBadRequest)
			return
		}

		//	marshal our tile into a protocol buffer
		var pbyte []byte
		pbyte, err = proto.Marshal(vtile)
		if err != nil {
			log.Printf("Error marshalling tile: %v", err)
			http.Error(w, "Error marshalling tile", http.StatusInternalServerError)
			return
		}

		//	TODO: how configurable do we want the CORS policy to be?
		//	set CORS header
		w.Header().Add("Access-Control-Allow-Origin", "*")

		//	mimetype for protocol buffers
		w.Header().Add("Content-Type", "application/x-protobuf")

		w.Write(pbyte)

		//	check for tile size warnings
		if len(pbyte) > MaxTileSize {
			log.Printf("tile z:%v, x:%v, y:%v is rather large - %v", tile.Z, tile.X, tile.Y, len(pbyte))
		}

		//	log the request
		L.Log(logItem{
			X:         tile.X,
			Y:         tile.Y,
			Z:         tile.Z,
			RequestIP: r.RemoteAddr,
		})

	default:
		http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
		return
	}
}

//	creates a debug layer with z/x/y encoded as a point
func debugLayer(tile tegola.Tile) *mvt.Layer {
	//	get tile bounding box
	ext := tile.BoundingBox()

	//	create a new layer and name it
	layer := mvt.Layer{
		Name: "debug",
	}
	xlen := ext.Maxx - ext.Minx
	ylen := ext.Maxy - ext.Miny

	//	tile outlines
	outline := mvt.Feature{
		Tags: map[string]interface{}{
			"type": "debug_outline",
		},
		Geometry: &basic.Line{ //	tile outline
			basic.Point{ext.Minx, ext.Miny},
			basic.Point{ext.Maxx, ext.Miny},
			basic.Point{ext.Maxx, ext.Maxy},
			basic.Point{ext.Minx, ext.Maxy},
		},
	}

	//	new feature
	zxy := mvt.Feature{
		Tags: map[string]interface{}{
			"type": "debug_text",
			"zxy":  fmt.Sprintf("Z:%v, X:%v, Y:%v", tile.Z, tile.X, tile.Y),
		},
		Geometry: &basic.Point{ //	middle of the tile
			ext.Minx + (xlen / 2),
			ext.Miny + (ylen / 2),
		},
	}

	layer.AddFeatures(outline, zxy)

	return &layer
}
<|endoftext|>


<commit_before>package gocbcore

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strings"
	"sync/atomic"
	"time"
)

type configStreamBlock struct {
	Bytes []byte
}

func (i *configStreamBlock) UnmarshalJSON(data []byte) error {
	i.Bytes = make([]byte, len(data))
	copy(i.Bytes, data)
	return nil
}

func hostnameFromUri(uri string) string {
	uriInfo, err := url.Parse(uri)
	if err != nil {
		return uri
	}
	return strings.Split(uriInfo.Host, ":")[0]
}

func (agent *Agent) httpLooper(firstCfgFn func(*cfgBucket, string, error) bool) {
	waitPeriod := agent.confHttpRetryDelay
	maxConnPeriod := agent.confHttpRedialPeriod

	var iterNum uint64 = 1
	iterSawConfig := false
	seenNodes := make(map[string]uint64)
	isFirstTry := true

	logDebugf("HTTP Looper starting.")
	for {
		routingInfo := agent.routingInfo.Get()
		if routingInfo == nil {
			// Shutdown the looper if the agent is shutdown
			break
		}

		var pickedSrv string
		for _, srv := range routingInfo.mgmtEpList {
			if seenNodes[srv] >= iterNum {
				continue
			}
			pickedSrv = srv
			break
		}

		if pickedSrv == "" {
			logDebugf("Pick Failed.")
			// All servers have been visited during this iteration
			if isFirstTry {
				logDebugf("Could not find any alive http hosts.")
				firstCfgFn(nil, "", ErrBadHosts)
				break
			}

			if !iterSawConfig {
				logDebugf("Looper waiting...")
				// Wait for a period before trying again if there was a problem...
				// We also watch for the client being shut down.
				select {
				case <-time.After(waitPeriod):
				case <-agent.closeNotify:
				}
			}
			logDebugf("Looping again.")
			// Go to next iteration and try all servers again
			iterNum++
			iterSawConfig = false
			continue
		}

		logDebugf("Http Picked: %s.", pickedSrv)

		seenNodes[pickedSrv] = iterNum

		hostname := hostnameFromUri(pickedSrv)

		logDebugf("HTTP Hostname: %s.", pickedSrv)

		var resp *http.Response
		// 1 on success, 0 on failure for node, -1 for generic failure
		var doConfigRequest func(bool) int

		doConfigRequest = func(is2x bool) int {
			streamPath := "bs"
			if is2x {
				streamPath = "bucketsStreaming"
			}
			// HTTP request time!
			uri := fmt.Sprintf("%s/pools/default/%s/%s", pickedSrv, streamPath, agent.bucket)
			logDebugf("Requesting config from: %s.", uri)

			req, err := http.NewRequest("GET", uri, nil)
			if err != nil {
				logDebugf("Failed to build HTTP config request. %v", err)
				return 0
			}

			creds, err := getMgmtAuthCreds(agent.auth, pickedSrv)
			if err != nil {
				logDebugf("Failed to build get config credentials. %v", err)
				return 0
			}

			req.SetBasicAuth(creds.Username, creds.Password)

			resp, err = agent.httpCli.Do(req)
			if err != nil {
				logDebugf("Failed to connect to host. %v", err)
				return 0
			}

			if resp.StatusCode != 200 {
				if resp.StatusCode == 401 {
					logDebugf("Failed to connect to host, bad auth.")
					firstCfgFn(nil, "", ErrAuthError)
					return -1
				} else if resp.StatusCode == 404 {
					if is2x {
						logDebugf("Failed to connect to host, bad bucket.")
						firstCfgFn(nil, "", ErrAuthError)
						return -1
					}

					return doConfigRequest(true)
				}
				logDebugf("Failed to connect to host, unexpected status code: %v.", resp.StatusCode)
				return 0
			}
			return 1
		}

		switch doConfigRequest(false) {
		case 0:
			continue
		case -1:
			break
		}

		logDebugf("Connected.")

		var autoDisconnected int32

		// Autodisconnect eventually
		go func() {
			select {
			case <-time.After(maxConnPeriod):
			case <-agent.closeNotify:
			}

			logDebugf("Automatically resetting our HTTP connection")

			atomic.StoreInt32(&autoDisconnected, 1)

			err := resp.Body.Close()
			if err != nil {
				logErrorf("Socket close failed during auto-dc (%s)", err)
			}
		}()

		dec := json.NewDecoder(resp.Body)
		configBlock := new(configStreamBlock)
		for {
			err := dec.Decode(configBlock)
			if err != nil {
				if atomic.LoadInt32(&autoDisconnected) == 1 {
					// If we know we intentionally disconnected, we know we do not
					// need to close the client, nor log an error, since this was
					// expected behaviour
					break
				}

				logWarnf("Config block decode failure (%s)", err)

				if err != io.EOF {
					err = resp.Body.Close()
					if err != nil {
						logErrorf("Socket close failed after decode fail (%s)", err)
					}
				}

				break
			}

			logDebugf("Got Block: %v", string(configBlock.Bytes))

			bkCfg, err := parseConfig(configBlock.Bytes, hostname)
			if err != nil {
				logDebugf("Got error while parsing config: %v", err)

				err = resp.Body.Close()
				if err != nil {
					logErrorf("Socket close failed after parsing fail (%s)", err)
				}

				break
			}

			logDebugf("Got Config.")

			iterSawConfig = true
			if isFirstTry {
				logDebugf("HTTP Config Init")
				if !firstCfgFn(bkCfg, pickedSrv, nil) {
					logDebugf("Got error while activating first config")

					err = resp.Body.Close()
					if err != nil {
						logErrorf("Socket close failed after config init (%s)", err)
					}

					break
				}
				isFirstTry = false
			} else {
				logDebugf("HTTP Config Update")
				agent.updateConfig(bkCfg)
			}
		}

		logDebugf("HTTP, Setting %s to iter %d", pickedSrv, iterNum)
	}

	close(agent.httpLooperDoneSig)
}
<commit_msg>GOCBC-353: Handle raw IPv6 address in cluster map<commit_after>package gocbcore

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"sync/atomic"
	"time"
)

type configStreamBlock struct {
	Bytes []byte
}

func (i *configStreamBlock) UnmarshalJSON(data []byte) error {
	i.Bytes = make([]byte, len(data))
	copy(i.Bytes, data)
	return nil
}

func hostnameFromUri(uri string) string {
	uriInfo, err := url.Parse(uri)
	if err != nil {
		return uri
	}

	hostname, err := hostFromHostPort(uriInfo.Host)
	if err != nil {
		return uri
	}

	return hostname
}

func (agent *Agent) httpLooper(firstCfgFn func(*cfgBucket, string, error) bool) {
	waitPeriod := agent.confHttpRetryDelay
	maxConnPeriod := agent.confHttpRedialPeriod

	var iterNum uint64 = 1
	iterSawConfig := false
	seenNodes := make(map[string]uint64)
	isFirstTry := true

	logDebugf("HTTP Looper starting.")
	for {
		routingInfo := agent.routingInfo.Get()
		if routingInfo == nil {
			// Shutdown the looper if the agent is shutdown
			break
		}

		var pickedSrv string
		for _, srv := range routingInfo.mgmtEpList {
			if seenNodes[srv] >= iterNum {
				continue
			}
			pickedSrv = srv
			break
		}

		if pickedSrv == "" {
			logDebugf("Pick Failed.")
			// All servers have been visited during this iteration
			if isFirstTry {
				logDebugf("Could not find any alive http hosts.")
				firstCfgFn(nil, "", ErrBadHosts)
				break
			}

			if !iterSawConfig {
				logDebugf("Looper waiting...")
				// Wait for a period before trying again if there was a problem...
				// We also watch for the client being shut down.
				select {
				case <-time.After(waitPeriod):
				case <-agent.closeNotify:
				}
			}
			logDebugf("Looping again.")
			// Go to next iteration and try all servers again
			iterNum++
			iterSawConfig = false
			continue
		}

		logDebugf("Http Picked: %s.", pickedSrv)

		seenNodes[pickedSrv] = iterNum

		hostname := hostnameFromUri(pickedSrv)
		logDebugf("HTTP Hostname: %s.", hostname)

		var resp *http.Response
		// 1 on success, 0 on failure for node, -1 for generic failure
		var doConfigRequest func(bool) int

		doConfigRequest = func(is2x bool) int {
			streamPath := "bs"
			if is2x {
				streamPath = "bucketsStreaming"
			}
			// HTTP request time!
			uri := fmt.Sprintf("%s/pools/default/%s/%s", pickedSrv, streamPath, agent.bucket)
			logDebugf("Requesting config from: %s.", uri)

			req, err := http.NewRequest("GET", uri, nil)
			if err != nil {
				logDebugf("Failed to build HTTP config request. %v", err)
				return 0
			}

			creds, err := getMgmtAuthCreds(agent.auth, pickedSrv)
			if err != nil {
				logDebugf("Failed to build get config credentials. %v", err)
				return 0
			}

			req.SetBasicAuth(creds.Username, creds.Password)

			resp, err = agent.httpCli.Do(req)
			if err != nil {
				logDebugf("Failed to connect to host. %v", err)
				return 0
			}

			if resp.StatusCode != 200 {
				if resp.StatusCode == 401 {
					logDebugf("Failed to connect to host, bad auth.")
					firstCfgFn(nil, "", ErrAuthError)
					return -1
				} else if resp.StatusCode == 404 {
					if is2x {
						logDebugf("Failed to connect to host, bad bucket.")
						firstCfgFn(nil, "", ErrAuthError)
						return -1
					}

					return doConfigRequest(true)
				}
				logDebugf("Failed to connect to host, unexpected status code: %v.", resp.StatusCode)
				return 0
			}
			return 1
		}

		switch doConfigRequest(false) {
		case 0:
			continue
		case -1:
			break
		}

		logDebugf("Connected.")

		var autoDisconnected int32

		// Autodisconnect eventually
		go func() {
			select {
			case <-time.After(maxConnPeriod):
			case <-agent.closeNotify:
			}

			logDebugf("Automatically resetting our HTTP connection")

			atomic.StoreInt32(&autoDisconnected, 1)

			err := resp.Body.Close()
			if err != nil {
				logErrorf("Socket close failed during auto-dc (%s)", err)
			}
		}()

		dec := json.NewDecoder(resp.Body)
		configBlock := new(configStreamBlock)
		for {
			err := dec.Decode(configBlock)
			if err != nil {
				if atomic.LoadInt32(&autoDisconnected) == 1 {
					// If we know we intentionally disconnected, we know we do not
					// need to close the client, nor log an error, since this was
					// expected behaviour
					break
				}

				logWarnf("Config block decode failure (%s)", err)

				if err != io.EOF {
					err = resp.Body.Close()
					if err != nil {
						logErrorf("Socket close failed after decode fail (%s)", err)
					}
				}

				break
			}

			logDebugf("Got Block: %v", string(configBlock.Bytes))

			bkCfg, err := parseConfig(configBlock.Bytes, hostname)
			if err != nil {
				logDebugf("Got error while parsing config: %v", err)

				err = resp.Body.Close()
				if err != nil {
					logErrorf("Socket close failed after parsing fail (%s)", err)
				}

				break
			}

			logDebugf("Got Config.")

			iterSawConfig = true
			if isFirstTry {
				logDebugf("HTTP Config Init")
				if !firstCfgFn(bkCfg, pickedSrv, nil) {
					logDebugf("Got error while activating first config")

					err = resp.Body.Close()
					if err != nil {
						logErrorf("Socket close failed after config init (%s)", err)
					}

					break
				}
				isFirstTry = false
			} else {
				logDebugf("HTTP Config Update")
				agent.updateConfig(bkCfg)
			}
		}

		logDebugf("HTTP, Setting %s to iter %d", pickedSrv, iterNum)
	}

	close(agent.httpLooperDoneSig)
}
<|endoftext|>


<commit_before>package server

import (
    "testing"
)

var DATA = announceData{
    info_hash: "12345123451234512345",
    peer_id: "12345123451234512345",
    ip: "127.0.0.1",
    event: "STARTED",
    port: 6767,
    uploaded: 1024,
    downloaded: 512,
    left: 0,
    numwant: 30,
    compact: true,
    redisClient: OpenClient(),
}

func TestRedisSetIPMember(t *testing.T) {
    ret := RedisSetIPMember(&DATA)

    expectedReturn := 1

    if ret != expectedReturn {
        t.Fatalf("Expected %v, got %v", expectedReturn, ret)
    }
}

func TestRedisSetKeyVal(t *testing.T) {
    RedisSetKeyVal(DATA.redisClient, "test:1234", "1024")

    ret, _ := DATA.redisClient.SMembers("test:1234").Result()

    expectedReturn := ">1"

    if len(ret) == 0 {
        t.Fatalf("Expected %v, got %v", expectedReturn, len(ret))
    }
}

func TestRedisGetKeyVal(t *testing.T) {
    DATA.redisClient.SAdd("RedisGetKeyValTest:1024:complete", "1024")
    ret := RedisGetKeyVal(DATA.redisClient, "RedisGetKeyValTest:1024", &DATA)
    expectedReturn := ">1"

    if len(ret) == 0 {
        t.Fatalf("Expected %v, got %v", expectedReturn, len(ret))
    }
}

func TestRedisGetKeyValNoPreexistKey(t *testing.T) {
    DATA.redisClient.SAdd("RedisGetKeyValTest:1025", "1024")
    ret := RedisGetKeyVal(DATA.redisClient, "RedisGetKeyValTest:1025", &DATA)
    expectedReturn := 0

    if len(ret) != expectedReturn {
        t.Fatalf("Expected %v, got %v", expectedReturn, len(ret))
    }
}

func TestCreateIpPortPair(t *testing.T) {
    expectedReturn := "127.0.0.1:6767"
    ret := createIpPortPair(&DATA)

    if expectedReturn != ret {
        t.Fatalf("Expected %v, got %v", expectedReturn, ret)
    }
}

func TestConcatenateKeyMember(t *testing.T) {
    expectedReturn := "127.0.0.1:1024"
    ret := concatenateKeyMember("127.0.0.1", "1024")

    if expectedReturn != ret {
        t.Fatalf("Expected %v, got %v", expectedReturn, ret)
    }
}

func TestCreateNewTorrentKey(t *testing.T) {
    CreateNewTorrentKey(DATA.redisClient, "testTestCreateNewTorrentKey")

    ret, err := DATA.redisClient.Exists("testTestCreateNewTorrentKey").Result()
    if err != nil {
        t.Fatalf("%v", err)
    }
    if ret != true {
        t.Fatalf("CreateNewTorrentKey:complete failed to create")
    }

    ret, err = DATA.redisClient.SIsMember("testTestCreateNewTorrentKey", "complete").Result()
    if ret != true {
        t.Fatalf("testTestCreateNewTorrentKey:complete is not a member")
    }

    ret, err = DATA.redisClient.SIsMember("testTestCreateNewTorrentKey", "incomplete").Result()
    if ret != true {
        t.Fatalf("testTestCreateNewTorrentKey:incomplete is not a member")
    }

}

func TestRedisRemoveKeyValues(t *testing.T) {
    DATA.redisClient.SAdd("TestRedisRemoveKeyVal", "Test1")
    ret, err := DATA.redisClient.SIsMember("TestRedisRemoveKeyVal", "Test1").Result()
    if err != nil {
        t.Fatalf("%v", err)
    }
    if ret != true {
        t.Fatalf("Failed in setup of TestRedisRemoveKeyValues to add a key")
    }

    RedisRemoveKeysValue(DATA.redisClient, "TestRedisRemoveKeyVal", "Test1")
    ret, err = DATA.redisClient.SIsMember("TestRedisRemoveKeyVal", "Test1").Result()
    if err != nil {
        t.Fatalf("%v", err)
    }
    if ret == true {
        t.Fatalf("RedisRemoveKeyVal failed to remove the key")
    }

}

func TestRedisGetBoolKeyVal(t *testing.T) {
    RedisSetKeyVal(DATA.redisClient, "TestRedisGetBoolKeyVal", "1024")

    expectedReturn := true
    ret := RedisGetBoolKeyVal(DATA.redisClient, "TestRedisGetBoolKeyVal", "1024")

    if ret != expectedReturn {
        t.Fatalf("Expected %v, got %v", expectedReturn, ret)
    }
}

func TestRedisSetKeyIfNotExists(t *testing.T) {
    expectedReturn := true
    ret := RedisSetKeyIfNotExists(DATA.redisClient, "TestRedisSetKeyIfNotExists", "1024")

    if ret != expectedReturn {
        t.Fatalf("Expected %v, got %v", expectedReturn, ret)
    }
}

func TestRedisSetKeyIfNotExistsPreExistingKey(t *testing.T) {
    expectedReturn := true
    RedisSetKeyVal(DATA.redisClient, "TestRedisSetKeyIfNotExists", "1024")
    ret := RedisSetKeyIfNotExists(DATA.redisClient, "TestRedisSetKeyIfNotExists", "1024")

    if ret != expectedReturn {
        t.Fatalf("Expected %v, got %v", expectedReturn, ret)
    }
}

<commit_msg>Issue #96<commit_after>package server

import (
    "testing"
)

var DATA = announceData{
    info_hash: "12345123451234512345",
    peer_id: "12345123451234512345",
    ip: "127.0.0.1",
    event: "STARTED",
    port: 6767,
    uploaded: 1024,
    downloaded: 512,
    left: 0,
    numwant: 30,
    compact: true,
    redisClient: OpenClient(),
}

func TestRedisSetIPMember(t *testing.T) {
    ret := RedisSetIPMember(&DATA)

    expectedReturn := 1

    if ret != expectedReturn {
        t.Fatalf("Expected %v, got %v", expectedReturn, ret)
    }
}

func TestRedisSetKeyVal(t *testing.T) {
    RedisSetKeyVal(DATA.redisClient, "test:1234", "1024")

    ret, _ := DATA.redisClient.SMembers("test:1234").Result()

    expectedReturn := ">1"

    if len(ret) == 0 {
        t.Fatalf("Expected %v, got %v", expectedReturn, len(ret))
    }
}

func TestRedisGetKeyVal(t *testing.T) {
    DATA.redisClient.SAdd("RedisGetKeyValTest:1024:complete", "1024")
    ret := RedisGetKeyVal(DATA.redisClient, "RedisGetKeyValTest:1024", &DATA)
    expectedReturn := ">1"

    if len(ret) == 0 {
        t.Fatalf("Expected %v, got %v", expectedReturn, len(ret))
    }
}

func TestRedisGetKeyValNoPreexistKey(t *testing.T) {
    DATA.redisClient.SAdd("RedisGetKeyValTest:1025", "1024")
    ret := RedisGetKeyVal(DATA.redisClient, "RedisGetKeyValTest:1025", &DATA)
    expectedReturn := 0

    if len(ret) != expectedReturn {
        t.Fatalf("Expected %v, got %v", expectedReturn, len(ret))
    }
}

func TestCreateIpPortPair(t *testing.T) {
    expectedReturn := "127.0.0.1:6767"
    ret := createIpPortPair(&DATA)

    if expectedReturn != ret {
        t.Fatalf("Expected %v, got %v", expectedReturn, ret)
    }
}

func TestConcatenateKeyMember(t *testing.T) {
    expectedReturn := "127.0.0.1:1024"
    ret := concatenateKeyMember("127.0.0.1", "1024")

    if expectedReturn != ret {
        t.Fatalf("Expected %v, got %v", expectedReturn, ret)
    }
}

func TestCreateNewTorrentKey(t *testing.T) {
    CreateNewTorrentKey(DATA.redisClient, "testTestCreateNewTorrentKey")

    ret, err := DATA.redisClient.Exists("testTestCreateNewTorrentKey").Result()
    if err != nil {
        t.Fatalf("%v", err)
    }
    if ret != true {
        t.Fatalf("CreateNewTorrentKey:complete failed to create")
    }

    ret, err = DATA.redisClient.SIsMember("testTestCreateNewTorrentKey", "complete").Result()
    if ret != true {
        t.Fatalf("testTestCreateNewTorrentKey:complete is not a member")
    }

    ret, err = DATA.redisClient.SIsMember("testTestCreateNewTorrentKey", "incomplete").Result()
    if ret != true {
        t.Fatalf("testTestCreateNewTorrentKey:incomplete is not a member")
    }

}

func TestRedisRemoveKeyValues(t *testing.T) {
    DATA.redisClient.SAdd("TestRedisRemoveKeyVal", "Test1")
    ret, err := DATA.redisClient.SIsMember("TestRedisRemoveKeyVal", "Test1").Result()
    if err != nil {
        t.Fatalf("%v", err)
    }
    if ret != true {
        t.Fatalf("Failed in setup of TestRedisRemoveKeyValues to add a key")
    }

    RedisRemoveKeysValue(DATA.redisClient, "TestRedisRemoveKeyVal", "Test1")
    ret, err = DATA.redisClient.SIsMember("TestRedisRemoveKeyVal", "Test1").Result()
    if err != nil {
        t.Fatalf("%v", err)
    }
    if ret == true {
        t.Fatalf("RedisRemoveKeyVal failed to remove the key")
    }

}

func TestRedisGetBoolKeyVal(t *testing.T) {
    RedisSetKeyVal(DATA.redisClient, "TestRedisGetBoolKeyVal", "1024")

    expectedReturn := true
    ret := RedisGetBoolKeyVal(DATA.redisClient, "TestRedisGetBoolKeyVal", "1024")

    if ret != expectedReturn {
        t.Fatalf("Expected %v, got %v", expectedReturn, ret)
    }
}

func TestRedisSetKeyIfNotExists(t *testing.T) {
    expectedReturn := true
    ret := RedisSetKeyIfNotExists(DATA.redisClient, "TestRedisSetKeyIfNotExists", "1024")

    if ret != expectedReturn {
        t.Fatalf("Expected %v, got %v", expectedReturn, ret)
    }
}

func TestRedisSetKeyIfNotExistsPreExistingKey(t *testing.T) {
    expectedReturn := true
    RedisSetKeyVal(DATA.redisClient, "TestRedisSetKeyIfNotExists", "1024")
    ret := RedisSetKeyIfNotExists(DATA.redisClient, "TestRedisSetKeyIfNotExists", "1024")

    if ret != expectedReturn {
        t.Fatalf("Expected %v, got %v", expectedReturn, ret)
    }
}

func TestRedisGetCount(t *testing.T) {
    DATA.redisClient.SAdd("TestRedisGetCount", "Test")
    DATA.redisClient.SAdd("TestRedisGetCount:Test", "1235")
    DATA.redisClient.SAdd("TestRedisGetCount:Test", "1236")
    DATA.redisClient.SAdd("TestRedisGetCount:Test", "1237")
    DATA.redisClient.SAdd("TestRedisGetCount:Test", "1238")

    expectedReturn := 4
    ret, err := RedisGetCount(DATA.redisClient, "TestRedisGetCount", "Test")
    if err != nil {
        t.Fatalf("%v", err)
    }

    if ret != expectedReturn {
        t.Fatalf("Expected %v, got %v", expectedReturn, ret)
    }
}

func TestRedisGetAllPeers(t *testing.T) {
    DATA.redisClient.SAdd("TestRedisGetAllPeers", "complete")
    DATA.redisClient.SAdd("TestRedisGetAllPeers:complete", "1235")
    DATA.redisClient.SAdd("TestRedisGetAllPeers:complete", "1236")
    DATA.redisClient.SAdd("TestRedisGetAllPeers:complete", "1237")
    DATA.redisClient.SAdd("TestRedisGetAllPeers:complete", "1238")

    ret := RedisGetAllPeers(DATA.redisClient, "TestRedisGetAllPeers", &DATA)
    x := len(ret)

    if x != 4 {
        t.Fatalf("Expected 4 peers, got %v", x)
    }
}

func TestRedisGetAllPeersValGT30(t *testing.T) {
    DATA.redisClient.SAdd("TestRedisGetAllPeers1", "complete")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1201")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1202")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1203")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1204")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1205")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1206")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1207")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1208")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1209")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1210")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1211")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1212")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1213")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1214")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1215")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1216")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1217")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1218")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1209")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1200")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1201")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1202")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1203")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1204")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1205")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1216")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1217")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1218")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1219")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1220")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1221")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1222")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1221")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1222")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1223")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1224")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1225")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1226")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1227")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1228")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1229")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1230")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1231")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1232")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1233")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1234")
    DATA.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1235")

    ret := RedisGetAllPeers(DATA.redisClient, "TestRedisGetAllPeers1", &DATA)
    x := len(ret)

    if x != 30 {
        t.Fatalf("Expected 30 peers, got %v", x)
    }
}

func TestRedisGetAllPeersValLT30(t *testing.T) {
    DATA.redisClient.SAdd("TestRedisGetAllPeers2", "complete")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1201")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1202")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1203")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1204")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1205")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1206")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1207")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1208")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1209")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1210")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1211")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1212")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1213")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1214")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1215")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2", "incomplete")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1216")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1217")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1218")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1209")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1200")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1201")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1202")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1203")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1204")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1205")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1216")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1217")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1218")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1219")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1220")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1221")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1222")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1221")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1222")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1223")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1224")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1225")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1226")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1227")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1228")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1229")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1230")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1231")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1232")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1233")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1234")
    DATA.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1235")

    ret := RedisGetAllPeers(DATA.redisClient, "TestRedisGetAllPeers2", &DATA)
    x := len(ret)

    if x != 30 {
        t.Fatalf("Expected 30 peers, got %v", x)
    }
}
<|endoftext|>


<commit_before>package server

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/anacrolix/torrent"
	"github.com/anacrolix/torrent/metainfo"

	"github.com/jpillora/cloud-torrent/engine"
)

func (s *Server) api(r *http.Request) error {
	defer r.Body.Close()
	if r.Method != "POST" {
		return fmt.Errorf("Invalid request method (expecting POST)")
	}

	action := strings.TrimPrefix(r.URL.Path, "/api/")

	data, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return fmt.Errorf("Failed to download request body")
	}

	//convert url into torrent bytes
	if action == "url" {
		url := string(data)
		remote, err := http.Get(url)
		if err != nil {
			return fmt.Errorf("Invalid remote torrent URL: %s (%s)", err, url)
		}
		//TODO enforce max body size (32k?)
		data, err = ioutil.ReadAll(remote.Body)
		if err != nil {
			return fmt.Errorf("Failed to download remote torrent: %s", err)
		}
		action = "torrentfile"
	}

	//convert torrent bytes into magnet
	if action == "torrentfile" {
		reader := bytes.NewBuffer(data)
		info, err := metainfo.Load(reader)
		if err != nil {
			return err
		}
		spec := torrent.TorrentSpecFromMetaInfo(info)
		if err := s.engine.NewTorrent(spec); err != nil {
			return fmt.Errorf("Torrent error: %s", err)
		}
		return nil
	}

	//update after action completes
	defer s.state.Push()

	//interface with engine
	switch action {
	case "configure":
		c := engine.Config{}
		if err := json.Unmarshal(data, &c); err != nil {
			return err
		}
		if err := s.reconfigure(c); err != nil {
			return err
		}
	case "magnet":
		uri := string(data)
		if err := s.engine.NewMagnet(uri); err != nil {
			return fmt.Errorf("Magnet error: %s", err)
		}
	case "torrent":
		cmd := strings.SplitN(string(data), ":", 2)
		if len(cmd) != 2 {
			return fmt.Errorf("Invalid request")
		}
		state := cmd[0]
		infohash := cmd[1]
		if state == "start" {
			if err := s.engine.StartTorrent(infohash); err != nil {
				return err
			}
		} else if state == "stop" {
			if err := s.engine.StopTorrent(infohash); err != nil {
				return err
			}
		} else if state == "delete" {
			if err := s.engine.DeleteTorrent(infohash); err != nil {
				return err
			}
		} else {
			return fmt.Errorf("Invalid state: %s", state)
		}
	case "file":
		cmd := strings.SplitN(string(data), ":", 3)
		if len(cmd) != 3 {
			return fmt.Errorf("Invalid request")
		}
		state := cmd[0]
		infohash := cmd[1]
		filepath := cmd[2]
		if state == "start" {
			if err := s.engine.StartFile(infohash, filepath); err != nil {
				return err
			}
		} else if state == "stop" {
			if err := s.engine.StopFile(infohash, filepath); err != nil {
				return err
			}
		} else {
			return fmt.Errorf("Invalid state: %s", state)
		}
	default:
		return fmt.Errorf("Invalid action: %s", action)
	}
	return nil
}
<commit_msg>Update server_api.go<commit_after>package server

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/anacrolix/torrent"
	"github.com/anacrolix/torrent/metainfo"

	"github.com/gamerstt/Actsg/engine"
)

func (s *Server) api(r *http.Request) error {
	defer r.Body.Close()
	if r.Method != "POST" {
		return fmt.Errorf("Invalid request method (expecting POST)")
	}

	action := strings.TrimPrefix(r.URL.Path, "/api/")

	data, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return fmt.Errorf("Failed to download request body")
	}

	//convert url into torrent bytes
	if action == "url" {
		url := string(data)
		remote, err := http.Get(url)
		if err != nil {
			return fmt.Errorf("Invalid remote torrent URL: %s (%s)", err, url)
		}
		//TODO enforce max body size (32k?)
		data, err = ioutil.ReadAll(remote.Body)
		if err != nil {
			return fmt.Errorf("Failed to download remote torrent: %s", err)
		}
		action = "torrentfile"
	}

	//convert torrent bytes into magnet
	if action == "torrentfile" {
		reader := bytes.NewBuffer(data)
		info, err := metainfo.Load(reader)
		if err != nil {
			return err
		}
		spec := torrent.TorrentSpecFromMetaInfo(info)
		if err := s.engine.NewTorrent(spec); err != nil {
			return fmt.Errorf("Torrent error: %s", err)
		}
		return nil
	}

	//update after action completes
	defer s.state.Push()

	//interface with engine
	switch action {
	case "configure":
		c := engine.Config{}
		if err := json.Unmarshal(data, &c); err != nil {
			return err
		}
		if err := s.reconfigure(c); err != nil {
			return err
		}
	case "magnet":
		uri := string(data)
		if err := s.engine.NewMagnet(uri); err != nil {
			return fmt.Errorf("Magnet error: %s", err)
		}
	case "torrent":
		cmd := strings.SplitN(string(data), ":", 2)
		if len(cmd) != 2 {
			return fmt.Errorf("Invalid request")
		}
		state := cmd[0]
		infohash := cmd[1]
		if state == "start" {
			if err := s.engine.StartTorrent(infohash); err != nil {
				return err
			}
		} else if state == "stop" {
			if err := s.engine.StopTorrent(infohash); err != nil {
				return err
			}
		} else if state == "delete" {
			if err := s.engine.DeleteTorrent(infohash); err != nil {
				return err
			}
		} else {
			return fmt.Errorf("Invalid state: %s", state)
		}
	case "file":
		cmd := strings.SplitN(string(data), ":", 3)
		if len(cmd) != 3 {
			return fmt.Errorf("Invalid request")
		}
		state := cmd[0]
		infohash := cmd[1]
		filepath := cmd[2]
		if state == "start" {
			if err := s.engine.StartFile(infohash, filepath); err != nil {
				return err
			}
		} else if state == "stop" {
			if err := s.engine.StopFile(infohash, filepath); err != nil {
				return err
			}
		} else {
			return fmt.Errorf("Invalid state: %s", state)
		}
	default:
		return fmt.Errorf("Invalid action: %s", action)
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"io"
	"net"
	"strconv"
	"time"

	"github.com/nange/easypool"
	"github.com/nange/easyss/cipherstream"
	"github.com/nange/easyss/socks"
	"github.com/nange/easyss/utils"
	"github.com/pkg/errors"
	log "github.com/sirupsen/logrus"
)

func (ss *Easyss) Local() {
	listenAddr := ":" + strconv.Itoa(ss.config.LocalPort)
	ln, err := net.Listen("tcp", listenAddr)
	if err != nil {
		log.Fatal(err)
	}
	log.Infof("starting local socks5 server at %v", listenAddr)
	log.Debugf("config:%v", *ss.config)

	for {
		conn, err := ln.Accept()
		if err != nil {
			log.Error("accept:", err)
			continue
		}
		conn.(*net.TCPConn).SetKeepAlive(true)
		conn.(*net.TCPConn).SetKeepAlivePeriod(10 * time.Second)

		go func() {
			defer conn.Close()

			addr, err := socks.HandShake(conn)
			if err != nil {
				log.Errorf("local handshake err:%+v, remote:%v", err, addr)
				return
			}
			log.Debugf("target proxy addr is:%v", addr.String())

			var stream io.ReadWriteCloser
			if ss.config.EnableQuic {
				stream, err = ss.getStream()
			} else {
				stream, err = ss.tcpPool.Get()
			}
			if err != nil {
				log.Errorf("get stream err:%+v", err)
				return
			}
			defer stream.Close()

			header := utils.NewHTTP2DataFrameHeader(len(addr))
			gcm, err := cipherstream.NewAes256GCM([]byte(ss.config.Password))
			if err != nil {
				log.Errorf("cipherstream.NewAes256GCM err:%+v", err)
				return
			}

			headercipher, err := gcm.Encrypt(header)
			if err != nil {
				log.Errorf("gcm.Encrypt err:%+v", err)
				return
			}
			_, err = stream.Write(headercipher)
			if err != nil {
				log.Errorf("stream.Write err:%+v", errors.WithStack(err))
				return
			}

			payloadcipher, err := gcm.Encrypt([]byte(addr))
			if err != nil {
				log.Errorf("gcm.Encrypt err:%+v", err)
				return
			}
			_, err = stream.Write(payloadcipher)
			if err != nil {
				log.Errorf("stream.Write err:%+v", errors.WithStack(err))
				return
			}

			csStream, err := cipherstream.New(stream, ss.config.Password, ss.config.Method)
			if err != nil {
				log.Errorf("new cipherstream err:%+v, password:%v, method:%v",
					err, ss.config.Password, ss.config.Method)
				return
			}

			n1, n2, needclose := relay(csStream, conn, true)
			log.Infof("send %v bytes to %v, and recive %v bytes, needclose:%v", n1, addr, n2, needclose)
		}()
	}
}

// relay copies between cipherstream and plaintxtstream.
// return the number of bytes copies
// from plaintxtstream to cipherstream, from cipherstream to plaintxtstream, and any error occurred.
func relay(cipher, plaintxt io.ReadWriteCloser, islocal bool) (int64, int64, bool) {
	needclose := false
	ch := make(chan int64)

	go func() {
		n, err := io.Copy(plaintxt, cipher)
		if err == cipherstream.ErrDecrypt || err == cipherstream.ErrReadCipher {
			log.Warnf("io.Copy err:%+v, maybe underline connection have been closed", err)
			if cs, ok := cipher.(*cipherstream.CipherStream); ok {
				if pc, ok := cs.ReadWriteCloser.(*easypool.PoolConn); ok {
					log.Debug("mark cipher stream unusable")
					pc.MarkUnusable()
					needclose = true
				}
			}
		}
		if conn, ok := plaintxt.(*net.TCPConn); ok {
			log.Debugf("set tcp connection deadline to now, and free other goroutine")
			conn.SetDeadline(time.Now()) // wake up the other goroutine blocking on plaintxt conn
		}
		ch <- n
	}()

	n1, err := io.Copy(cipher, plaintxt)
	if err == cipherstream.ErrEncrypt || err == cipherstream.ErrWriteCipher {
		log.Warnf("io.Copy err:%+v, maybe underline connection have been closed", err)
		if cs, ok := cipher.(*cipherstream.CipherStream); ok {
			if pc, ok := cs.ReadWriteCloser.(*easypool.PoolConn); ok {
				log.Debug("mark cipher stream unusable")
				pc.MarkUnusable()
				needclose = true
			}
		}
	}
	if conn, ok := plaintxt.(*net.TCPConn); ok {
		log.Debugf("set tcp connection deadline to now, and free other goroutine")
		conn.SetDeadline(time.Now()) // wake up the other goroutine blocking on plaintxt conn
	}
	n2 := <-ch

	// TODO: if islocal && pc is not unusable, write RST_STREAM to remote server
	//	if islocal {
	//		log.Debug("write RST_STREAM to remote server")
	//		rstHeader := utils.NewHTTP2RstStreamHeader()
	//		cipher.Write(rstHeader)
	//	}

	return n1, n2, needclose
}
<commit_msg>update local logic<commit_after>package main

import (
	"io"
	"net"
	"strconv"
	"time"

	"github.com/nange/easypool"
	"github.com/nange/easyss/cipherstream"
	"github.com/nange/easyss/socks"
	"github.com/nange/easyss/utils"
	"github.com/pkg/errors"
	log "github.com/sirupsen/logrus"
)

func (ss *Easyss) Local() {
	listenAddr := ":" + strconv.Itoa(ss.config.LocalPort)
	ln, err := net.Listen("tcp", listenAddr)
	if err != nil {
		log.Fatal(err)
	}
	log.Infof("starting local socks5 server at %v", listenAddr)
	log.Debugf("config:%v", *ss.config)

	for {
		conn, err := ln.Accept()
		if err != nil {
			log.Error("accept:", err)
			continue
		}
		conn.(*net.TCPConn).SetKeepAlive(true)
		conn.(*net.TCPConn).SetKeepAlivePeriod(10 * time.Second)

		go func() {
			defer conn.Close()

			addr, err := socks.HandShake(conn)
			if err != nil {
				log.Errorf("local handshake err:%+v, remote:%v", err, addr)
				return
			}
			log.Debugf("target proxy addr is:%v", addr.String())

			var stream io.ReadWriteCloser
			if ss.config.EnableQuic {
				stream, err = ss.getStream()
			} else {
				stream, err = ss.tcpPool.Get()
			}
			if err != nil {
				log.Errorf("get stream err:%+v", err)
				return
			}
			defer stream.Close()

			header := utils.NewHTTP2DataFrameHeader(len(addr))
			gcm, err := cipherstream.NewAes256GCM([]byte(ss.config.Password))
			if err != nil {
				log.Errorf("cipherstream.NewAes256GCM err:%+v", err)
				return
			}

			headercipher, err := gcm.Encrypt(header)
			if err != nil {
				log.Errorf("gcm.Encrypt err:%+v", err)
				return
			}
			_, err = stream.Write(headercipher)
			if err != nil {
				log.Errorf("stream.Write err:%+v", errors.WithStack(err))
				return
			}

			payloadcipher, err := gcm.Encrypt([]byte(addr))
			if err != nil {
				log.Errorf("gcm.Encrypt err:%+v", err)
				return
			}
			_, err = stream.Write(payloadcipher)
			if err != nil {
				log.Errorf("stream.Write err:%+v", errors.WithStack(err))
				return
			}

			csStream, err := cipherstream.New(stream, ss.config.Password, ss.config.Method)
			if err != nil {
				log.Errorf("new cipherstream err:%+v, password:%v, method:%v",
					err, ss.config.Password, ss.config.Method)
				return
			}

			n1, n2, needclose := relay(csStream, conn, true)
			log.Infof("send %v bytes to %v, and recive %v bytes, needclose:%v", n1, addr, n2, needclose)
		}()
	}
}

// relay copies between cipherstream and plaintxtstream.
// return the number of bytes copies
// from plaintxtstream to cipherstream, from cipherstream to plaintxtstream, and any error occurred.
func relay(cipher, plaintxt io.ReadWriteCloser, islocal bool) (int64, int64, bool) {
	needclose := false
	ch := make(chan int64)

	go func() {
		n, err := io.Copy(plaintxt, cipher)
		if err == cipherstream.ErrDecrypt || err == cipherstream.ErrReadCipher {
			log.Warnf("io.Copy err:%+v, maybe underline connection have been closed", err)
			if cs, ok := cipher.(*cipherstream.CipherStream); ok {
				if pc, ok := cs.ReadWriteCloser.(*easypool.PoolConn); ok {
					log.Debug("mark cipher stream unusable")
					pc.MarkUnusable()
					needclose = true
				}
			}
		}
		if conn, ok := plaintxt.(*net.TCPConn); ok {
			log.Debugf("set tcp connection deadline to now, and free other goroutine")
			conn.SetDeadline(time.Now()) // wake up the other goroutine blocking on plaintxt conn
		}
		ch <- n
	}()

	n1, err := io.Copy(cipher, plaintxt)
	if err == cipherstream.ErrEncrypt || err == cipherstream.ErrWriteCipher {
		log.Warnf("io.Copy err:%+v, maybe underline connection have been closed", err)
		if cs, ok := cipher.(*cipherstream.CipherStream); ok {
			if pc, ok := cs.ReadWriteCloser.(*easypool.PoolConn); ok {
				log.Debug("mark cipher stream unusable")
				pc.MarkUnusable()
				needclose = true
			}
		}
	}
	if conn, ok := plaintxt.(*net.TCPConn); ok {
		log.Debugf("set tcp connection deadline to now, and free other goroutine")
		conn.SetDeadline(time.Now()) // wake up the other goroutine blocking on plaintxt conn
	}
	n2 := <-ch

	if islocal {
		if cs, ok := cipher.(*cipherstream.CipherStream); ok {
			if pc, ok := cs.ReadWriteCloser.(*easypool.PoolConn); ok {
				if pc.IsUnusable() {
					log.Debug("write RST_STREAM to remote server")
					rstHeader := utils.NewHTTP2RstStreamHeader()
					cipher.Write(rstHeader)
				}
			}
		}
	}

	return n1, n2, needclose
}
<|endoftext|>


<commit_before>package encrypted

import (
	"log"

	"github.com/tinzenite/shared"
)

/*
handleLockMessage handles the logic upon receiving a LockMessage. Notably this
includes allowing or disallowing a lock for a specific time frame.
*/
func (c *chaninterface) handleLockMessage(address string, lm *shared.LockMessage) {
	log.Println("TODO: received lock message!", lm.String())
	switch lm.Action {
	case shared.LoRequest:
		if c.enc.setLock(address) {
			// if successful notify peer of success
			accept := shared.CreateLockMessage(shared.LoAccept)
			c.enc.channel.Send(address, accept.JSON())
		}
		// if not successful we don't do anything, peer will retry
		return
	case shared.LoRelease:
		if c.enc.isLockedAddress(address) {
			c.enc.ClearLock()
			// TODO notify of clear?
			return
		}
		log.Println("WARNING: received release request from invalid peer!")
	default:
		log.Println("Invalid action in LockMessage received!")
	}
}

/*
handleRequestMessage handles the logic upon receiving a RequestMessage. NOTE:
will only be actually handled if Encrypted is currently locked.
*/
func (c *chaninterface) handleRequestMessage(address string, rm *shared.RequestMessage) {
	if !c.enc.checkLock(address) {
		log.Println("DEBUG: not locked to given address!")
		return
	}
	log.Println("TODO: received request message!", rm.String())
}

/*
handlePushMessage handles the logic upon receiving a PushMessage. NOTE: will
only be actually handled if Encrypted is currently locked.
*/
func (c *chaninterface) handlePushMessage(address string, pm *shared.PushMessage) {
	if !c.enc.checkLock(address) {
		log.Println("DEBUG: not locked to given address!")
		return
	}
	log.Println("TODO received push message!", pm.String())
}
<commit_msg>implemented first version of handling request messages<commit_after>package encrypted

import (
	"log"

	"github.com/tinzenite/shared"
)

/*
handleLockMessage handles the logic upon receiving a LockMessage. Notably this
includes allowing or disallowing a lock for a specific time frame.
*/
func (c *chaninterface) handleLockMessage(address string, lm *shared.LockMessage) {
	log.Println("TODO: received lock message!", lm.String())
	switch lm.Action {
	case shared.LoRequest:
		if c.enc.setLock(address) {
			// if successful notify peer of success
			accept := shared.CreateLockMessage(shared.LoAccept)
			c.enc.channel.Send(address, accept.JSON())
		}
		// if not successful we don't do anything, peer will retry
		return
	case shared.LoRelease:
		if c.enc.isLockedAddress(address) {
			c.enc.ClearLock()
			// TODO notify of clear?
			return
		}
		log.Println("WARNING: received release request from invalid peer!")
	default:
		log.Println("Invalid action in LockMessage received!")
	}
}

/*
handleRequestMessage handles the logic upon receiving a RequestMessage. NOTE:
will only be actually handled if Encrypted is currently locked.
*/
func (c *chaninterface) handleRequestMessage(address string, rm *shared.RequestMessage) {
	if !c.enc.checkLock(address) {
		log.Println("DEBUG: not locked to given address!")
		return
	}
	// path of file to send (will be set accordingly depending on ObjType)
	var filePath string
	// check what file to get and set filePath accordingly
	switch rm.ObjType {
	case shared.OtObject:
		filePath = c.enc.RootPath + "/" + rm.Identification
	case shared.OtModel:
		filePath = c.enc.RootPath + "/" + shared.MODELJSON
	default:
		// TODO maybe allow retrieval of this peer too?
		log.Println("Invalid ObjType requested!", rm.ObjType.String())
		return
	}
	// check that file exists
	if exists, _ := shared.FileExists(filePath); !exists {
		log.Println("DEBUG: file doesn't exist!", filePath)
		return
	}
	// send file
	err := c.enc.channel.SendFile(address, filePath, rm.Identification, func(success bool) {
		if !success {
			log.Println("Failed to send file on request!", filePath)
		}
	})
	// if error log
	if err != nil {
		log.Println("SendFile returned error:", err)
	}
}

/*
handlePushMessage handles the logic upon receiving a PushMessage. NOTE: will
only be actually handled if Encrypted is currently locked.
*/
func (c *chaninterface) handlePushMessage(address string, pm *shared.PushMessage) {
	if !c.enc.checkLock(address) {
		log.Println("DEBUG: not locked to given address!")
		return
	}
	log.Println("TODO received push message!", pm.String())
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
)

var cmdLogin = &Command{
	Run:   runLogin,
	Usage: "login",
	Short: "Log in to force.com",
	Long: `
Log in to force.com

Examples:

  force login
`,
}

func runLogin(cmd *Command, args []string) {
	var endpoint ForceEndpoint
	endpoint = EndpointProduction
	if len(args) > 0 {
		switch args[0] {
		case "test":
			endpoint = EndpointTest
		case "pre":
			endpoint = EndpointPrerelease
		default:
			ErrorAndExit("no such endpoint: %s", args[0])
		}
	}
	_, err := ForceLoginAndSave(endpoint)
	if err != nil {
		ErrorAndExit(err.Error())
	}
}

var cmdLogout = &Command{
	Run:   runLogout,
	Usage: "logout <account>",
	Short: "Log out from force.com",
	Long: `
Log out from force.com

Examples:

  force logout user@example.org
`,
}

func runLogout(cmd *Command, args []string) {
	if len(args) != 1 {
		ErrorAndExit("must specify account to log out")
	}
	account := args[0]
	Config.Delete("accounts", account)
	if active, _ := Config.Load("current", "account"); active == account {
		Config.Delete("current", "account")
		SetActiveAccountDefault()
	}
}

func ForceLoginAndSave(endpoint ForceEndpoint) (username string, err error) {
	creds, err := ForceLogin(endpoint)
	if err != nil {
		return
	}
	force := NewForce(creds)
	login, err := force.Get(creds.Id)
	if err != nil {
		return
	}
	body, err := json.Marshal(creds)
	if err != nil {
		return
	}
	username = login["username"].(string)
	Config.Save("accounts", username, string(body))
	Config.Save("current", "account", username)
	return
}
<commit_msg>Better examples for the login command.<commit_after>package main

import (
	"encoding/json"
)

var cmdLogin = &Command{
	Run:   runLogin,
	Usage: "login",
	Short: "Log in to force.com",
	Long: `
Log in to force.com

Examples:

  force login -- for Production and DE orgs
  force login test -- for Sandbox orgs
  force login pre -- for Pre-release feature orgs

`,
}

func runLogin(cmd *Command, args []string) {
	var endpoint ForceEndpoint
	endpoint = EndpointProduction
	if len(args) > 0 {
		switch args[0] {
		case "test":
			endpoint = EndpointTest
		case "pre":
			endpoint = EndpointPrerelease
		default:
			ErrorAndExit("no such endpoint: %s", args[0])
		}
	}
	_, err := ForceLoginAndSave(endpoint)
	if err != nil {
		ErrorAndExit(err.Error())
	}
}

var cmdLogout = &Command{
	Run:   runLogout,
	Usage: "logout <account>",
	Short: "Log out from force.com",
	Long: `
Log out from force.com

Examples:

  force logout user@example.org
`,
}

func runLogout(cmd *Command, args []string) {
	if len(args) != 1 {
		ErrorAndExit("must specify account to log out")
	}
	account := args[0]
	Config.Delete("accounts", account)
	if active, _ := Config.Load("current", "account"); active == account {
		Config.Delete("current", "account")
		SetActiveAccountDefault()
	}
}

func ForceLoginAndSave(endpoint ForceEndpoint) (username string, err error) {
	creds, err := ForceLogin(endpoint)
	if err != nil {
		return
	}
	force := NewForce(creds)
	login, err := force.Get(creds.Id)
	if err != nil {
		return
	}
	body, err := json.Marshal(creds)
	if err != nil {
		return
	}
	username = login["username"].(string)
	Config.Save("accounts", username, string(body))
	Config.Save("current", "account", username)
	return
}
<|endoftext|>


<commit_before>package secstring

import (
	"crypto/aes"
	"crypto/cipher"
	"crypto/rand"
	"errors"
	"strings"
	"syscall"
)

type SecString struct {
	String  []byte // Encrypted/Decrypted string
	Length  int    // Length of the target string
	Padding int    // Length of padding added
	iv      []byte
	cipher  cipher.Block
	encrypted bool
}

func memset(s []byte, c byte) {
	for i := 0; i < len(s); i++ {
		s[i] = c
	}
}

// Takes a []byte and builds a SecString out of it, wiping str in the
// process. The SecString.String is encrypted after this function.
//
// A SecString should be destroyed when it's no longer needed to prevent memory leaks.
// It is probably a good idea to defer SecString.Destroy()
func NewSecString(str []byte) (*SecString, error) {
	ret := &SecString{Length: len(str)}
	var err error

	if padding := ret.Length % aes.BlockSize; padding != 0 {
		ret.Padding = aes.BlockSize - padding
	}

	ret.String, err = syscall.Mmap(-1, 0, ret.Length+ret.Padding, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_ANON|syscall.MAP_PRIVATE)
	if err != nil {
		return nil, err
	}

	if err := syscall.Mlock(ret.String); err != nil {
		syscall.Munmap(ret.String)
		return nil, err
	}

	for i := 0; i < ret.Length; i++ {
		ret.String[i] = str[i]
		str[i] = 0
	}

	key := make([]byte, 32)
	if _, err := rand.Read(key); err != nil {
		memset(ret.String, 0)
		syscall.Munmap(ret.String)
		return nil, err
	}

	if ret.cipher, err = aes.NewCipher(key); err != nil {
		memset(ret.String, 0)
		syscall.Munmap(ret.String)
		return nil, err
	}

	if err := ret.Encrypt(); err != nil {
		memset(ret.String, 0)
		syscall.Munmap(ret.String)
		return nil, err
	}

	return ret, nil
}

// Makes a new SecString from a string reference. Destroys str after creating
// the secstring
func FromString(str *string) (*SecString, error) {
	b := make([]byte, len(*str))
	for i := 0; i < len(*str); i++ {
		b[i] = (*str)[i]
	}
	*str = strings.Repeat("x", len(*str))
	return NewSecString(b)
}

// Encrypts SecString.String
func (s *SecString) Encrypt() error {
	if s.encrypted {
		return errors.New("String already encrypted")
	}

	if err := syscall.Mprotect(s.String, syscall.PROT_READ|syscall.PROT_WRITE); err != nil {
		return err
	}

	s.iv = make([]byte, aes.BlockSize)
	if _, err := rand.Read(s.iv); err != nil {
		return err
	}

	encrypter := cipher.NewOFB(s.cipher, s.iv)
	encrypter.XORKeyStream(s.String, s.String)
	s.encrypted = true

	if err := syscall.Mprotect(s.String, syscall.PROT_READ); err != nil {
		return err
	}

	return nil
}

// Decrypt SecString.String for use
func (s *SecString) Decrypt() error {
	if ! s.encrypted {
		return errors.New("String already decrypted")
	}

	if err := syscall.Mprotect(s.String, syscall.PROT_READ|syscall.PROT_WRITE); err != nil {
		return err
	}

	decrypter := cipher.NewOFB(s.cipher, s.iv)
	decrypter.XORKeyStream(s.String, s.String)
	s.encrypted = false

	if err := syscall.Mprotect(s.String, syscall.PROT_READ); err != nil {
		return err
	}

	return nil
}

// Destroys the s. *MUST* be called to prevent memory leaks. Probably best to
// be called in a defer
func (s *SecString) Destroy() error {
	if err := syscall.Mprotect(s.String, syscall.PROT_READ|syscall.PROT_WRITE); err != nil {
		return err
	}

	memset(s.String, 0)

	if err := syscall.Munlock(s.String); err != nil {
		return err
	}

	if err := syscall.Munmap(s.String); err != nil {
		return err
	}

	s.String = nil
	return nil
}
<commit_msg>Use proper call for mmap<commit_after>package secstring

import (
	"crypto/aes"
	"crypto/cipher"
	"crypto/rand"
	"errors"
	"strings"
	"syscall"
)

type SecString struct {
	String  []byte // Encrypted/Decrypted string
	Length  int    // Length of the target string
	Padding int    // Length of padding added
	iv      []byte
	cipher  cipher.Block
	encrypted bool
}

func memset(s []byte, c byte) {
	for i := 0; i < len(s); i++ {
		s[i] = c
	}
}

// Takes a []byte and builds a SecString out of it, wiping str in the
// process. The SecString.String is encrypted after this function.
//
// A SecString should be destroyed when it's no longer needed to prevent memory leaks.
// It is probably a good idea to defer SecString.Destroy()
func NewSecString(str []byte) (*SecString, error) {
	ret := &SecString{Length: len(str)}
	var err error

	if padding := ret.Length % aes.BlockSize; padding != 0 {
		ret.Padding = aes.BlockSize - padding
	}

	ret.String, err = syscall.Mmap(0, 0, ret.Length+ret.Padding, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_ANON|syscall.MAP_PRIVATE)
	if err != nil {
		return nil, err
	}

	if err := syscall.Mlock(ret.String); err != nil {
		syscall.Munmap(ret.String)
		return nil, err
	}

	for i := 0; i < ret.Length; i++ {
		ret.String[i] = str[i]
		str[i] = 0
	}

	key := make([]byte, 32)
	if _, err := rand.Read(key); err != nil {
		memset(ret.String, 0)
		syscall.Munmap(ret.String)
		return nil, err
	}

	if ret.cipher, err = aes.NewCipher(key); err != nil {
		memset(ret.String, 0)
		syscall.Munmap(ret.String)
		return nil, err
	}

	if err := ret.Encrypt(); err != nil {
		memset(ret.String, 0)
		syscall.Munmap(ret.String)
		return nil, err
	}

	return ret, nil
}

// Makes a new SecString from a string reference. Destroys str after creating
// the secstring
func FromString(str *string) (*SecString, error) {
	b := make([]byte, len(*str))
	for i := 0; i < len(*str); i++ {
		b[i] = (*str)[i]
	}
	*str = strings.Repeat("x", len(*str))
	return NewSecString(b)
}

// Encrypts SecString.String
func (s *SecString) Encrypt() error {
	if s.encrypted {
		return errors.New("String already encrypted")
	}

	if err := syscall.Mprotect(s.String, syscall.PROT_READ|syscall.PROT_WRITE); err != nil {
		return err
	}

	s.iv = make([]byte, aes.BlockSize)
	if _, err := rand.Read(s.iv); err != nil {
		return err
	}

	encrypter := cipher.NewOFB(s.cipher, s.iv)
	encrypter.XORKeyStream(s.String, s.String)
	s.encrypted = true

	if err := syscall.Mprotect(s.String, syscall.PROT_READ); err != nil {
		return err
	}

	return nil
}

// Decrypt SecString.String for use
func (s *SecString) Decrypt() error {
	if ! s.encrypted {
		return errors.New("String already decrypted")
	}

	if err := syscall.Mprotect(s.String, syscall.PROT_READ|syscall.PROT_WRITE); err != nil {
		return err
	}

	decrypter := cipher.NewOFB(s.cipher, s.iv)
	decrypter.XORKeyStream(s.String, s.String)
	s.encrypted = false

	if err := syscall.Mprotect(s.String, syscall.PROT_READ); err != nil {
		return err
	}

	return nil
}

// Destroys the s. *MUST* be called to prevent memory leaks. Probably best to
// be called in a defer
func (s *SecString) Destroy() error {
	if err := syscall.Mprotect(s.String, syscall.PROT_READ|syscall.PROT_WRITE); err != nil {
		return err
	}

	memset(s.String, 0)

	if err := syscall.Munlock(s.String); err != nil {
		return err
	}

	if err := syscall.Munmap(s.String); err != nil {
		return err
	}

	s.String = nil
	return nil
}
<|endoftext|>


<commit_before>/*
 * Run TCP/UDP service loop: Create listener for incoming connection
 * requests as a go-routine and call user-defined service handler as
 * a go-routine to handle client sessions.
 *
 * (c) 2010 Bernd Fix   >Y<
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or (at
 * your option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

package network

///////////////////////////////////////////////////////////////////////
// Import external declarations.

import (
	"os"
	"net"
	"log"
)

///////////////////////////////////////////////////////////////////////
// Public interfaces

/*
 * User-defined service handler: Handle TCP/UDP client sessions.
 * The interface defines four methods:
 * - Process (conn): Main handler routine for connection
 * - GetName(): Return service name (for logging output)
 * - CanHandle (protocol): Check if handler can process given
 *       network protocol (TCP or UDP on IPv4 or IPv6)
 * - IsAllowed (addr): Checkk if remote address is allowed to
 *       be served by the service handler.
 */
type Service interface {
	Process (conn net.Conn)				// main handler routine
	GetName() string					// get symbolic name of service
	CanHandle (protocol string) bool	// check network protocol
	IsAllowed (remote string) bool		// check remote address
}

///////////////////////////////////////////////////////////////////////
// Public methods

/*
 * Run TCP/UDP network service with user-defined session handler.
 * @param network string - network identifier (TCP/UDP on IPv4/v6)
 * @param addr string - address:port specification of service
 * @param hdlr Service - implementation of service interface
 */
func RunService (network, addr string, hdlr Service) os.Error {

	// initialize control service
	service, err := net.Listen (network, addr)
	if err != nil {
		log.Println ("[" + hdlr.GetName() + "] service start-up failed: " + err.String())
		return err
	}

	// handle connection requests
	go func() {
		for {
			// wait for connection request
			client, err := service.Accept()
			if err != nil {
				log.Println ("[" + hdlr.GetName() + "] Accept(): " + err.String())
				continue
			}
			// check if connection is allowed:
			remote  := client.RemoteAddr().String()
			protocol := client.RemoteAddr().Network()
			// check for matching protocol
			if !hdlr.CanHandle (protocol)  {
				log.Printf("[" + hdlr.GetName() + "] rejected non-TCP connection from %v\n", protocol)
				client.Close()
				continue
			}
			// check for matching remote address
			if !hdlr.IsAllowed (remote)  {
				log.Printf("[" + hdlr.GetName() + "] rejected remote connection from %v\n", remote)
				client.Close()
				continue
			}
			// connection accepted
			log.Printf("[" + hdlr.GetName() + "] accepted %v\n", remote)

			// start handler
			go hdlr.Process (client)
		}
	}()

	// report success
	log.Println ("[" + hdlr.GetName() + "] service started...")
	return nil
}

///////////////////////////////////////////////////////////////////////
//	Revision history:
///////////////////////////////////////////////////////////////////////
//
//	Revision 1.0  2010-11-18 23:17:06  brf
//  Initial revision.
//
<commit_msg>Modified logging messages.<commit_after>/*
 * Run TCP/UDP service loop: Create listener for incoming connection
 * requests as a go-routine and call user-defined service handler as
 * a go-routine to handle client sessions.
 *
 * (c) 2010 Bernd Fix   >Y<
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or (at
 * your option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

package network

///////////////////////////////////////////////////////////////////////
// Import external declarations.

import (
	"os"
	"net"
	"log"
)

///////////////////////////////////////////////////////////////////////
// Public interfaces

/*
 * User-defined service handler: Handle TCP/UDP client sessions.
 * The interface defines four methods:
 * - Process (conn): Main handler routine for connection
 * - GetName(): Return service name (for logging output)
 * - CanHandle (protocol): Check if handler can process given
 *       network protocol (TCP or UDP on IPv4 or IPv6)
 * - IsAllowed (addr): Checkk if remote address is allowed to
 *       be served by the service handler.
 */
type Service interface {
	Process (conn net.Conn)				// main handler routine
	GetName() string					// get symbolic name of service
	CanHandle (protocol string) bool	// check network protocol
	IsAllowed (remote string) bool		// check remote address
}

///////////////////////////////////////////////////////////////////////
// Public methods

/*
 * Run TCP/UDP network service with user-defined session handler.
 * @param network string - network identifier (TCP/UDP on IPv4/v6)
 * @param addr string - address:port specification of service
 * @param hdlr Service - implementation of service interface
 */
func RunService (network, addr string, hdlr Service) os.Error {

	// initialize control service
	service, err := net.Listen (network, addr)
	if err != nil {
		log.Println ("[" + hdlr.GetName() + "] service start-up failed: " + err.String())
		return err
	}

	// handle connection requests
	go func() {
		for {
			// wait for connection request
			client, err := service.Accept()
			if err != nil {
				log.Println ("[" + hdlr.GetName() + "] Accept(): " + err.String())
				continue
			}
			// check if connection is allowed:
			remote  := client.RemoteAddr().String()
			protocol := client.RemoteAddr().Network()
			// check for matching protocol
			if !hdlr.CanHandle (protocol)  {
				log.Printf("[" + hdlr.GetName() + "] rejected connection protocol '%s' from %s\n", protocol, remote)
				client.Close()
				continue
			}
			// check for matching remote address
			if !hdlr.IsAllowed (remote)  {
				log.Printf("[" + hdlr.GetName() + "] rejected connection from %s\n", remote)
				client.Close()
				continue
			}
			// connection accepted
			log.Printf("[" + hdlr.GetName() + "] accepted connection from %s\n", remote)

			// start handler
			go hdlr.Process (client)
		}
	}()

	// report success
	log.Println ("[" + hdlr.GetName() + "] service started...")
	return nil
}

///////////////////////////////////////////////////////////////////////
//	Revision history:
///////////////////////////////////////////////////////////////////////
//
//	Revision 1.0  2010-11-18 23:17:06  brf
//  Initial revision.
//
<|endoftext|>