<commit_before>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/globocom/config"
	"github.com/globocom/tsuru/action"
	"github.com/globocom/tsuru/app"
	"github.com/globocom/tsuru/cmd"
	"github.com/globocom/tsuru/db"
	"github.com/globocom/tsuru/exec"
	"github.com/globocom/tsuru/log"
	"github.com/globocom/tsuru/provision"
	"github.com/globocom/tsuru/queue"
	"github.com/globocom/tsuru/router"
	_ "github.com/globocom/tsuru/router/hipache"
	_ "github.com/globocom/tsuru/router/testing"
	"io"
	"io/ioutil"
	"sync"
	"time"
)

func init() {
	provision.Register("docker", &dockerProvisioner{})
}

var (
	execut exec.Executor
	emutex sync.Mutex
)

func executor() exec.Executor {
	emutex.Lock()
	defer emutex.Unlock()
	if execut == nil {
		execut = exec.OsExecutor{}
	}
	return execut
}

func getRouter() (router.Router, error) {
	r, err := config.GetString("docker:router")
	if err != nil {
		return nil, err
	}
	return router.Get(r)
}

type dockerProvisioner struct{}

// Provision creates a route for the container
func (p *dockerProvisioner) Provision(app provision.App) error {
	r, err := getRouter()
	if err != nil {
		log.Fatalf("Failed to get router: %s", err)
		return err
	}
	err = app.Ready()
	if err != nil {
		return err
	}
	return r.AddBackend(app.GetName())
}

func (p *dockerProvisioner) Restart(app provision.App) error {
	containers, err := listAppContainers(app.GetName())
	if err != nil {
		log.Errorf("Got error while getting app containers: %s", err)
		return err
	}
	var buf bytes.Buffer
	for _, c := range containers {
		err = c.ssh(&buf, &buf, "/var/lib/tsuru/restart")
		if err != nil {
			log.Errorf("Failed to restart %q: %s.", app.GetName(), err)
			log.Debug("Command outputs:")
			log.Debugf("out: %s", &buf)
			log.Debugf("err: %s", &buf)
			return err
		}
		buf.Reset()
	}
	return nil
}

func injectEnvsAndRestart(a provision.App) {
	time.Sleep(5e9)
	err := a.SerializeEnvVars()
	if err != nil {
		log.Errorf("Failed to serialize env vars: %s.", err)
	}
	var buf bytes.Buffer
	w := app.LogWriter{App: a, Writer: &buf}
	err = a.Restart(&w)
	if err != nil {
		log.Errorf("Failed to restart app %q (%s): %s.", a.GetName(), err, buf.String())
	}
}

func startInBackground(a provision.App, c container, imageId string, w io.Writer, started chan bool) {
	newContainer, err := start(a, imageId, w)
	if err != nil {
		log.Errorf("error on start the app %s - %s", a.GetName(), err)
		started <- false
		return
	}
	msg := queue.Message{Action: app.BindService, Args: []string{a.GetName(), newContainer.ID}}
	go app.Enqueue(msg)
	if c.ID != "" {
		if a.RemoveUnit(c.ID) != nil {
			removeContainer(&c)
		}
	}
	started <- true
}

func (dockerProvisioner) Swap(app1, app2 provision.App) error {
	r, err := getRouter()
	if err != nil {
		return err
	}
	return r.Swap(app1.GetName(), app2.GetName())
}

func (p *dockerProvisioner) Deploy(a provision.App, version string, w io.Writer) error {
	imageId, err := build(a, version, w)
	if err != nil {
		return err
	}
	containers, err := listAppContainers(a.GetName())
	started := make(chan bool, len(containers))
	if err == nil && len(containers) > 0 {
		for _, c := range containers {
			go startInBackground(a, c, imageId, w, started)
		}
	} else {
		go startInBackground(a, container{}, imageId, w, started)
	}
	if <-started {
		fmt.Fprint(w, "\n ---> App will be restarted, please check its logs for more details...\n\n")
	} else {
		fmt.Fprint(w, "\n ---> App failed to start, please check its logs for more details...\n\n")
	}
	return nil
}

func (p *dockerProvisioner) Destroy(app provision.App) error {
	containers, _ := listAppContainers(app.GetName())
	for _, c := range containers {
		go func(c container) {
			removeContainer(&c)
		}(c)
	}
	go removeImage(assembleImageName(app.GetName()))
	r, err := getRouter()
	if err != nil {
		log.Errorf("Failed to get router: %s", err)
		return err
	}
	return r.RemoveBackend(app.GetName())
}

func (*dockerProvisioner) Addr(app provision.App) (string, error) {
	r, err := getRouter()
	if err != nil {
		log.Errorf("Failed to get router: %s", err)
		return "", err
	}
	addr, err := r.Addr(app.GetName())
	if err != nil {
		log.Errorf("Failed to obtain app %s address: %s", app.GetName(), err)
		return "", err
	}
	return addr, nil
}

func (*dockerProvisioner) AddUnits(a provision.App, units uint) ([]provision.Unit, error) {
	if units == 0 {
		return nil, errors.New("Cannot add 0 units")
	}
	containers, err := listAppContainers(a.GetName())
	if err != nil {
		return nil, err
	}
	if len(containers) < 1 {
		return nil, errors.New("New units can only be added after the first deployment")
	}
	writer := app.LogWriter{App: a, Writer: ioutil.Discard}
	result := make([]provision.Unit, int(units))
	imageId := getImage(a)
	for i := uint(0); i < units; i++ {
		container, err := start(a, imageId, &writer)
		if err != nil {
			return nil, err
		}
		result[i] = provision.Unit{
			Name:    container.ID,
			AppName: a.GetName(),
			Type:    a.GetPlatform(),
			Ip:      container.HostAddr,
			Status:  provision.StatusBuilding,
		}
	}
	return result, nil
}

func (*dockerProvisioner) RemoveUnit(a provision.App, unitName string) error {
	container, err := getContainer(unitName)
	if err != nil {
		return err
	}
	if container.AppName != a.GetName() {
		return errors.New("Unit does not belong to this app")
	}
	if err := removeContainer(container); err != nil {
		return err
	}
	return rebindWhenNeed(a.GetName(), container)
}

// rebindWhenNeed rebinds a unit to the app's services when it finds
// that the unit being removed has the same host that any
// of the units that still being used
func rebindWhenNeed(appName string, container *container) error {
	containers, err := listAppContainers(appName)
	if err != nil {
		return err
	}
	for _, c := range containers {
		if c.HostAddr == container.HostAddr && c.ID != container.ID {
			msg := queue.Message{Action: app.BindService, Args: []string{appName, c.ID}}
			go app.Enqueue(msg)
			break
		}
	}
	return nil
}

func removeContainer(c *container) error {
	err := c.stop()
	if err != nil {
		log.Errorf("error on stop unit %s - %s", c.ID, err)
	}
	err = c.remove()
	if err != nil {
		log.Errorf("error on remove container %s - %s", c.ID, err)
	}
	return err
}

func (*dockerProvisioner) InstallDeps(app provision.App, w io.Writer) error {
	return nil
}

func (*dockerProvisioner) ExecuteCommandOnce(stdout, stderr io.Writer, app provision.App, cmd string, args ...string) error {
	containers, err := listAppContainers(app.GetName())
	if err != nil {
		return err
	}
	if len(containers) == 0 {
		return errors.New("No containers for this app")
	}
	container := containers[0]
	return container.ssh(stdout, stderr, cmd, args...)
}

func (*dockerProvisioner) ExecuteCommand(stdout, stderr io.Writer, app provision.App, cmd string, args ...string) error {
	containers, err := listAppContainers(app.GetName())
	if err != nil {
		return err
	}
	if len(containers) == 0 {
		return errors.New("No containers for this app")
	}
	for _, c := range containers {
		err = c.ssh(stdout, stderr, cmd, args...)
		if err != nil {
			return err
		}
	}
	return nil
}

func (p *dockerProvisioner) SetCName(app provision.App, cname string) error {
	r, err := getRouter()
	if err != nil {
		return err
	}
	return r.SetCName(cname, app.GetName())
}

func (p *dockerProvisioner) UnsetCName(app provision.App, cname string) error {
	r, err := getRouter()
	if err != nil {
		return err
	}
	return r.UnsetCName(cname, app.GetName())
}

func (p *dockerProvisioner) Commands() []cmd.Command {
	return []cmd.Command{
		addNodeToSchedulerCmd{},
		removeNodeFromSchedulerCmd{},
		listNodesInTheSchedulerCmd{},
		&sshAgentCmd{},
	}
}

func collection() *db.Collection {
	name, err := config.GetString("docker:collection")
	if err != nil {
		log.Fatal(err.Error())
	}
	conn, err := db.Conn()
	if err != nil {
		log.Errorf("Failed to connect to the database: %s", err)
	}
	return conn.Collection(name)
}

func (p *dockerProvisioner) DeployPipeline() *action.Pipeline {
	actions := []*action.Action{
		&app.ProvisionerDeploy,
		&app.IncrementDeploy,
		&saveUnits,
		&injectEnvirons,
	}
	pipeline := action.NewPipeline(actions...)
	return pipeline
}
<commit_msg>docker: added bindService to DeployPipeline.<commit_after>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/globocom/config"
	"github.com/globocom/tsuru/action"
	"github.com/globocom/tsuru/app"
	"github.com/globocom/tsuru/cmd"
	"github.com/globocom/tsuru/db"
	"github.com/globocom/tsuru/exec"
	"github.com/globocom/tsuru/log"
	"github.com/globocom/tsuru/provision"
	"github.com/globocom/tsuru/queue"
	"github.com/globocom/tsuru/router"
	_ "github.com/globocom/tsuru/router/hipache"
	_ "github.com/globocom/tsuru/router/testing"
	"io"
	"io/ioutil"
	"sync"
	"time"
)

func init() {
	provision.Register("docker", &dockerProvisioner{})
}

var (
	execut exec.Executor
	emutex sync.Mutex
)

func executor() exec.Executor {
	emutex.Lock()
	defer emutex.Unlock()
	if execut == nil {
		execut = exec.OsExecutor{}
	}
	return execut
}

func getRouter() (router.Router, error) {
	r, err := config.GetString("docker:router")
	if err != nil {
		return nil, err
	}
	return router.Get(r)
}

type dockerProvisioner struct{}

// Provision creates a route for the container
func (p *dockerProvisioner) Provision(app provision.App) error {
	r, err := getRouter()
	if err != nil {
		log.Fatalf("Failed to get router: %s", err)
		return err
	}
	err = app.Ready()
	if err != nil {
		return err
	}
	return r.AddBackend(app.GetName())
}

func (p *dockerProvisioner) Restart(app provision.App) error {
	containers, err := listAppContainers(app.GetName())
	if err != nil {
		log.Errorf("Got error while getting app containers: %s", err)
		return err
	}
	var buf bytes.Buffer
	for _, c := range containers {
		err = c.ssh(&buf, &buf, "/var/lib/tsuru/restart")
		if err != nil {
			log.Errorf("Failed to restart %q: %s.", app.GetName(), err)
			log.Debug("Command outputs:")
			log.Debugf("out: %s", &buf)
			log.Debugf("err: %s", &buf)
			return err
		}
		buf.Reset()
	}
	return nil
}

func injectEnvsAndRestart(a provision.App) {
	time.Sleep(5e9)
	err := a.SerializeEnvVars()
	if err != nil {
		log.Errorf("Failed to serialize env vars: %s.", err)
	}
	var buf bytes.Buffer
	w := app.LogWriter{App: a, Writer: &buf}
	err = a.Restart(&w)
	if err != nil {
		log.Errorf("Failed to restart app %q (%s): %s.", a.GetName(), err, buf.String())
	}
}

func startInBackground(a provision.App, c container, imageId string, w io.Writer, started chan bool) {
	newContainer, err := start(a, imageId, w)
	if err != nil {
		log.Errorf("error on start the app %s - %s", a.GetName(), err)
		started <- false
		return
	}
	msg := queue.Message{Action: app.BindService, Args: []string{a.GetName(), newContainer.ID}}
	go app.Enqueue(msg)
	if c.ID != "" {
		if a.RemoveUnit(c.ID) != nil {
			removeContainer(&c)
		}
	}
	started <- true
}

func (dockerProvisioner) Swap(app1, app2 provision.App) error {
	r, err := getRouter()
	if err != nil {
		return err
	}
	return r.Swap(app1.GetName(), app2.GetName())
}

func (p *dockerProvisioner) Deploy(a provision.App, version string, w io.Writer) error {
	imageId, err := build(a, version, w)
	if err != nil {
		return err
	}
	containers, err := listAppContainers(a.GetName())
	started := make(chan bool, len(containers))
	if err == nil && len(containers) > 0 {
		for _, c := range containers {
			go startInBackground(a, c, imageId, w, started)
		}
	} else {
		go startInBackground(a, container{}, imageId, w, started)
	}
	if <-started {
		fmt.Fprint(w, "\n ---> App will be restarted, please check its logs for more details...\n\n")
	} else {
		fmt.Fprint(w, "\n ---> App failed to start, please check its logs for more details...\n\n")
	}
	return nil
}

func (p *dockerProvisioner) Destroy(app provision.App) error {
	containers, _ := listAppContainers(app.GetName())
	for _, c := range containers {
		go func(c container) {
			removeContainer(&c)
		}(c)
	}
	go removeImage(assembleImageName(app.GetName()))
	r, err := getRouter()
	if err != nil {
		log.Errorf("Failed to get router: %s", err)
		return err
	}
	return r.RemoveBackend(app.GetName())
}

func (*dockerProvisioner) Addr(app provision.App) (string, error) {
	r, err := getRouter()
	if err != nil {
		log.Errorf("Failed to get router: %s", err)
		return "", err
	}
	addr, err := r.Addr(app.GetName())
	if err != nil {
		log.Errorf("Failed to obtain app %s address: %s", app.GetName(), err)
		return "", err
	}
	return addr, nil
}

func (*dockerProvisioner) AddUnits(a provision.App, units uint) ([]provision.Unit, error) {
	if units == 0 {
		return nil, errors.New("Cannot add 0 units")
	}
	containers, err := listAppContainers(a.GetName())
	if err != nil {
		return nil, err
	}
	if len(containers) < 1 {
		return nil, errors.New("New units can only be added after the first deployment")
	}
	writer := app.LogWriter{App: a, Writer: ioutil.Discard}
	result := make([]provision.Unit, int(units))
	imageId := getImage(a)
	for i := uint(0); i < units; i++ {
		container, err := start(a, imageId, &writer)
		if err != nil {
			return nil, err
		}
		result[i] = provision.Unit{
			Name:    container.ID,
			AppName: a.GetName(),
			Type:    a.GetPlatform(),
			Ip:      container.HostAddr,
			Status:  provision.StatusBuilding,
		}
	}
	return result, nil
}

func (*dockerProvisioner) RemoveUnit(a provision.App, unitName string) error {
	container, err := getContainer(unitName)
	if err != nil {
		return err
	}
	if container.AppName != a.GetName() {
		return errors.New("Unit does not belong to this app")
	}
	if err := removeContainer(container); err != nil {
		return err
	}
	return rebindWhenNeed(a.GetName(), container)
}

// rebindWhenNeed rebinds a unit to the app's services when it finds
// that the unit being removed has the same host that any
// of the units that still being used
func rebindWhenNeed(appName string, container *container) error {
	containers, err := listAppContainers(appName)
	if err != nil {
		return err
	}
	for _, c := range containers {
		if c.HostAddr == container.HostAddr && c.ID != container.ID {
			msg := queue.Message{Action: app.BindService, Args: []string{appName, c.ID}}
			go app.Enqueue(msg)
			break
		}
	}
	return nil
}

func removeContainer(c *container) error {
	err := c.stop()
	if err != nil {
		log.Errorf("error on stop unit %s - %s", c.ID, err)
	}
	err = c.remove()
	if err != nil {
		log.Errorf("error on remove container %s - %s", c.ID, err)
	}
	return err
}

func (*dockerProvisioner) InstallDeps(app provision.App, w io.Writer) error {
	return nil
}

func (*dockerProvisioner) ExecuteCommandOnce(stdout, stderr io.Writer, app provision.App, cmd string, args ...string) error {
	containers, err := listAppContainers(app.GetName())
	if err != nil {
		return err
	}
	if len(containers) == 0 {
		return errors.New("No containers for this app")
	}
	container := containers[0]
	return container.ssh(stdout, stderr, cmd, args...)
}

func (*dockerProvisioner) ExecuteCommand(stdout, stderr io.Writer, app provision.App, cmd string, args ...string) error {
	containers, err := listAppContainers(app.GetName())
	if err != nil {
		return err
	}
	if len(containers) == 0 {
		return errors.New("No containers for this app")
	}
	for _, c := range containers {
		err = c.ssh(stdout, stderr, cmd, args...)
		if err != nil {
			return err
		}
	}
	return nil
}

func (p *dockerProvisioner) SetCName(app provision.App, cname string) error {
	r, err := getRouter()
	if err != nil {
		return err
	}
	return r.SetCName(cname, app.GetName())
}

func (p *dockerProvisioner) UnsetCName(app provision.App, cname string) error {
	r, err := getRouter()
	if err != nil {
		return err
	}
	return r.UnsetCName(cname, app.GetName())
}

func (p *dockerProvisioner) Commands() []cmd.Command {
	return []cmd.Command{
		addNodeToSchedulerCmd{},
		removeNodeFromSchedulerCmd{},
		listNodesInTheSchedulerCmd{},
		&sshAgentCmd{},
	}
}

func collection() *db.Collection {
	name, err := config.GetString("docker:collection")
	if err != nil {
		log.Fatal(err.Error())
	}
	conn, err := db.Conn()
	if err != nil {
		log.Errorf("Failed to connect to the database: %s", err)
	}
	return conn.Collection(name)
}

func (p *dockerProvisioner) DeployPipeline() *action.Pipeline {
	actions := []*action.Action{
		&app.ProvisionerDeploy,
		&app.IncrementDeploy,
		&saveUnits,
		&injectEnvirons,
		&bindService,
	}
	pipeline := action.NewPipeline(actions...)
	return pipeline
}
<|endoftext|>


<commit_before>package protocol

import (
	"sync"
	"time"

	"github.com/v2ray/v2ray-core/common/collect"
	"github.com/v2ray/v2ray-core/common/serial"
	"github.com/v2ray/v2ray-core/proxy/vmess"
)

const (
	updateIntervalSec = 10
	cacheDurationSec  = 120
)

type Timestamp int64

func (this Timestamp) Bytes() []byte {
	return serial.Int64Literal(this).Bytes()
}

func (this Timestamp) HashBytes() []byte {
	once := this.Bytes()
	bytes := make([]byte, 0, 32)
	bytes = append(bytes, once...)
	bytes = append(bytes, once...)
	bytes = append(bytes, once...)
	bytes = append(bytes, once...)
	return bytes
}

type idEntry struct {
	id      *vmess.ID
	userIdx int
	lastSec Timestamp
	hashes  *collect.SizedQueue
}

type UserSet interface {
	AddUser(user vmess.User) error
	GetUser(timeHash []byte) (vmess.User, Timestamp, bool)
}

type TimedUserSet struct {
	validUsers []vmess.User
	userHash   map[string]indexTimePair
	ids        []*idEntry
	access     sync.RWMutex
}

type indexTimePair struct {
	index   int
	timeSec Timestamp
}

func NewTimedUserSet() UserSet {
	tus := &TimedUserSet{
		validUsers: make([]vmess.User, 0, 16),
		userHash:   make(map[string]indexTimePair, 512),
		access:     sync.RWMutex{},
		ids:        make([]*idEntry, 0, 512),
	}
	go tus.updateUserHash(time.Tick(updateIntervalSec * time.Second))
	return tus
}

func (us *TimedUserSet) generateNewHashes(nowSec Timestamp, idx int, entry *idEntry) {
	for entry.lastSec <= nowSec {
		idHash := IDHash(entry.id.Bytes())
		idHash.Write(entry.lastSec.Bytes())
		idHashSlice := idHash.Sum(nil)
		hashValue := string(idHashSlice)
		us.access.Lock()
		us.userHash[hashValue] = indexTimePair{idx, entry.lastSec}
		us.access.Unlock()

		hash2Remove := entry.hashes.Put(hashValue)
		if hash2Remove != nil {
			us.access.Lock()
			delete(us.userHash, hash2Remove.(string))
			us.access.Unlock()
		}
		entry.lastSec++
	}
}

func (us *TimedUserSet) updateUserHash(tick <-chan time.Time) {
	for now := range tick {
		nowSec := Timestamp(now.Unix() + cacheDurationSec)
		for _, entry := range us.ids {
			us.generateNewHashes(nowSec, entry.userIdx, entry)
		}
	}
}

func (us *TimedUserSet) AddUser(user vmess.User) error {
	idx := len(us.validUsers)
	us.validUsers = append(us.validUsers, user)

	nowSec := time.Now().Unix()

	entry := &idEntry{
		id:      user.ID(),
		userIdx: idx,
		lastSec: Timestamp(nowSec - cacheDurationSec),
		hashes:  collect.NewSizedQueue(2*cacheDurationSec + 1),
	}
	us.generateNewHashes(Timestamp(nowSec+cacheDurationSec), idx, entry)
	us.ids = append(us.ids, entry)
	for _, alterid := range user.AlterIDs() {
		entry := &idEntry{
			id:      alterid,
			userIdx: idx,
			lastSec: Timestamp(nowSec - cacheDurationSec),
			hashes:  collect.NewSizedQueue(2*cacheDurationSec + 1),
		}
		us.generateNewHashes(Timestamp(nowSec+cacheDurationSec), idx, entry)
		us.ids = append(us.ids, entry)
	}

	return nil
}

func (us *TimedUserSet) GetUser(userHash []byte) (vmess.User, Timestamp, bool) {
	defer us.access.RUnlock()
	us.access.RLock()
	pair, found := us.userHash[string(userHash)]
	if found {
		return us.validUsers[pair.index], pair.timeSec, true
	}
	return nil, 0, false
}
<commit_msg>use [16]byte instead of string for faster hashing<commit_after>package protocol

import (
	"sync"
	"time"

	"github.com/v2ray/v2ray-core/common/collect"
	"github.com/v2ray/v2ray-core/common/serial"
	"github.com/v2ray/v2ray-core/proxy/vmess"
)

const (
	updateIntervalSec = 10
	cacheDurationSec  = 120
)

type Timestamp int64

func (this Timestamp) Bytes() []byte {
	return serial.Int64Literal(this).Bytes()
}

func (this Timestamp) HashBytes() []byte {
	once := this.Bytes()
	bytes := make([]byte, 0, 32)
	bytes = append(bytes, once...)
	bytes = append(bytes, once...)
	bytes = append(bytes, once...)
	bytes = append(bytes, once...)
	return bytes
}

type idEntry struct {
	id      *vmess.ID
	userIdx int
	lastSec Timestamp
	hashes  *collect.SizedQueue
}

type UserSet interface {
	AddUser(user vmess.User) error
	GetUser(timeHash []byte) (vmess.User, Timestamp, bool)
}

type TimedUserSet struct {
	validUsers []vmess.User
	userHash   map[[16]byte]indexTimePair
	ids        []*idEntry
	access     sync.RWMutex
}

type indexTimePair struct {
	index   int
	timeSec Timestamp
}

func NewTimedUserSet() UserSet {
	tus := &TimedUserSet{
		validUsers: make([]vmess.User, 0, 16),
		userHash:   make(map[[16]byte]indexTimePair, 512),
		access:     sync.RWMutex{},
		ids:        make([]*idEntry, 0, 512),
	}
	go tus.updateUserHash(time.Tick(updateIntervalSec * time.Second))
	return tus
}

func (us *TimedUserSet) generateNewHashes(nowSec Timestamp, idx int, entry *idEntry) {
	var hashValue [16]byte
	idHash := IDHash(entry.id.Bytes())
	for entry.lastSec <= nowSec {
		idHash.Write(entry.lastSec.Bytes())
		idHash.Sum(hashValue[:0])
		idHash.Reset()

		hash2Remove := entry.hashes.Put(hashValue)

		us.access.Lock()
		us.userHash[hashValue] = indexTimePair{idx, entry.lastSec}
		if hash2Remove != nil {
			delete(us.userHash, hash2Remove.([16]byte))
		}
		us.access.Unlock()
		entry.lastSec++
	}
}

func (us *TimedUserSet) updateUserHash(tick <-chan time.Time) {
	for now := range tick {
		nowSec := Timestamp(now.Unix() + cacheDurationSec)
		for _, entry := range us.ids {
			us.generateNewHashes(nowSec, entry.userIdx, entry)
		}
	}
}

func (us *TimedUserSet) AddUser(user vmess.User) error {
	idx := len(us.validUsers)
	us.validUsers = append(us.validUsers, user)

	nowSec := time.Now().Unix()

	entry := &idEntry{
		id:      user.ID(),
		userIdx: idx,
		lastSec: Timestamp(nowSec - cacheDurationSec),
		hashes:  collect.NewSizedQueue(2*cacheDurationSec + 1),
	}
	us.generateNewHashes(Timestamp(nowSec+cacheDurationSec), idx, entry)
	us.ids = append(us.ids, entry)
	for _, alterid := range user.AlterIDs() {
		entry := &idEntry{
			id:      alterid,
			userIdx: idx,
			lastSec: Timestamp(nowSec - cacheDurationSec),
			hashes:  collect.NewSizedQueue(2*cacheDurationSec + 1),
		}
		us.generateNewHashes(Timestamp(nowSec+cacheDurationSec), idx, entry)
		us.ids = append(us.ids, entry)
	}

	return nil
}

func (us *TimedUserSet) GetUser(userHash []byte) (vmess.User, Timestamp, bool) {
	defer us.access.RUnlock()
	us.access.RLock()
	var fixedSizeHash [16]byte
	copy(fixedSizeHash[:], userHash)
	pair, found := us.userHash[fixedSizeHash]
	if found {
		return us.validUsers[pair.index], pair.timeSec, true
	}
	return nil, 0, false
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

/*
This package is for loading different mailing list data types into Cloud Storage.
*/

package main

import (
	"context"
	"flag"
	"fmt"
	"log"
	"strings"
	"time"

	"github.com/google/project-OCEAN/1-raw-data/gcs"
	"github.com/google/project-OCEAN/1-raw-data/mailinglists/googlegroups"
	"github.com/google/project-OCEAN/1-raw-data/mailinglists/mailman"
	"github.com/google/project-OCEAN/1-raw-data/mailinglists/pipermail"
	"github.com/google/project-OCEAN/1-raw-data/utils"
)

var (
	//Below variables used if build run
	buildListRun = flag.Bool("build-list-run", false, "Use flag to run build list run vs manual command line run.")
	allListRun   = flag.Bool("all-list-run", false, "Use flag to get variables from command-line or do a full mailing list run or to do simple build test run of one mailing list.")
	allDateRun   = flag.Bool("all-date-run", false, "Use flag to get variables from command-line or do a full run")
	projectID    = flag.String("project-id", "", "GCP Project id.")

	//Below variables used if manual run
	bucketName   = flag.String("bucket-name", "test", "Bucket name to store files.")
	subDirectory = flag.String("subdirectory", "", "Subdirectory to store files. Enter 1 or more and use spaces to identify. CAUTION also enter the groupNames to load to in the same order.")
	mailingList  = flag.String("mailinglist", "", "Choose which mailing list to process either pipermail (default), mailman, googlegroups")
	groupNames   = flag.String("groupname", "", "Mailing list group name. Enter 1 or more and use spaces to identify. CAUTION also enter the buckets to load to in the same order.")
	startDate    = flag.String("start-date", "", "Start date in format of year-month-date and 4dig-2dig-2dig.")
	endDate      = flag.String("end-date", "", "End date in format of year-month-date and 4dig-2dig-2dig.")
	workerNum    = flag.Int("workers", 1, "Number of workers to use for goroutines.")
	subNames     []string
)

func getData(ctx context.Context, storage gcs.Connection, httpToDom utils.HttpDomResponse, workerNum, numMonths int, mailingList, groupName, startDateString, endDateString string) {
	switch mailingList {
	//TODO add start and end dates to pipermail
	case "pipermail":
		if err := pipermail.GetPipermailData(ctx, storage, groupName, startDateString, endDateString, httpToDom); err != nil {
			log.Fatalf("Mailman load failed: %v", err)
		}
	case "mailman":
		if err := mailman.GetMailmanData(ctx, storage, groupName, startDateString, endDateString, numMonths); err != nil {
			log.Fatalf("Mailman load failed: %v", err)
		}
		//TODO add start and end dates to google groups
	case "gg":
		if err := googlegroups.GetGoogleGroupsData(ctx, "", groupName, startDateString, endDateString, storage, workerNum); err != nil {
			log.Fatalf("GoogleGroups load failed: %v", err)
		}
	default:
		log.Fatalf("Mailing list %v is not an option. Change the option submitted.", mailingList)
	}
}

func main() {
	var err error
	numMonths := 1
	httpToDom := utils.DomResponse
	flag.Parse()
	fmt.Printf(*projectID)

	//Setup Storage connection
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	storageConn := gcs.StorageConnection{
		ProjectID: *projectID,
	}
	if err = storageConn.ConnectClient(ctx); err != nil {
		log.Fatalf("Connect GCS failes: %v", err)
	}

	if *buildListRun {
		//Build run to load mailing list data
		now := time.Now()
		//Set variables in build that aren't coming in on command line
		*bucketName = "mailinglists"
		groupName := ""

		// Setup bucket connection whether new or not
		storageConn.BucketName = *bucketName
		if err := storageConn.CreateBucket(ctx); err != nil {
			log.Fatalf("Create GCS Bucket failed: %v", err)
		}

		// Run Build to load all mailing lists
		if *allListRun {
			log.Printf("Build all lists ")
			mailListMap := map[string]string{"gg-angular": "2009-09-01", "gg-golang-announce": "2011-05-01", "gg-golang-checkins": "2009-11-01", " gg-golang-codereviews": "2013-12-01", "gg-golang-dev": "2009-11-01", "gg-golang-nuts": "2009-11-01", "gg-nodejs": "2009-06-01", "mailman-python-announce-list": "1999-04-01", "mailman-python-dev": "1999-04-01", "mailman-python-ideas": "2006-12-01", "pipermail-python-announce-list": "1999-04-01", "pipermail-python-dev": "1995-03-01", "pipermail-python-ideas": "2006-12-01", "pipermail-python-list": "1999-02-01"}

			for subName, origStartDate := range mailListMap {
				startDateResult, endDateResult := "", ""
				storageConn.SubDirectory = subName
				*mailingList = strings.SplitN(subName, "-", 2)[0]
				groupName = strings.SplitN(subName, "-", 2)[1]

				if *allDateRun {
					//Load all months
					log.Printf("All Date Cloud Run")
					//Set start and end dates with first mailing list date and current end date
					if startDateResult, endDateResult, err = utils.FixDate(origStartDate, *endDate); err != nil {
						log.Fatalf("Date error: %v", err)
					}
				} else {
					//Set start and end dates split by one month
					if startDateResult, endDateResult, err = utils.SplitDatesByMonth(*startDate, *endDate, 1); err != nil {
						log.Fatalf("Date error: %v", err)
					}
					log.Printf("One Month Run All MailingLists")
					startDateResult = now.AddDate(0, -1, 0).Format("2006-01-02")
					endDateResult = now.Format("2006-01-02")
				}

				//Get mailing list data and store
				getData(ctx, &storageConn, httpToDom, *workerNum, numMonths, *mailingList, groupName, startDateResult, endDateResult)
			}
		} else {
			log.Printf("Build test run with mailman")
			storageConn.SubDirectory = "mailman-python-announce-list"
			groupName = "python-announce-list"
			*startDate = now.AddDate(0, -1, 0).Format("2006-01-02")
			*endDate = now.AddDate(0, -1, 1).Format("2006-01-02")

			if err := mailman.GetMailmanData(ctx, &storageConn, groupName, *startDate, *endDate, numMonths); err != nil {
				log.Fatalf("Mailman test build load failed: %v", err)
			}
		}
	} else {

		//Manual run pulls variables from command line to load mailing list data
		storageConn.BucketName = *bucketName
		//Check and create bucket if needed
		if err := storageConn.CreateBucket(ctx); err != nil {
			log.Fatalf("Create GCS Bucket failed: %v", err)
		}

		if *subDirectory != "" {
			subNames = strings.Split(*subDirectory, " ")
		}

		for idx, groupName := range strings.Split(*groupNames, " ") {
			//Apply sub directory name to storageConn if it exists
			if *subDirectory != "" {
				storageConn.SubDirectory = subNames[idx]
			}

			//Get mailing list data and store
			getData(ctx, &storageConn, httpToDom, *workerNum, numMonths, *mailingList, groupName, *startDate, *endDate)
		}
	}
}
<commit_msg>Removing redundant code by using defaults.<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

/*
This package is for loading different mailing list data types into Cloud Storage.
*/

package main

import (
	"context"
	"flag"
	"fmt"
	"log"
	"strings"
	"time"

	"github.com/google/project-OCEAN/1-raw-data/gcs"
	"github.com/google/project-OCEAN/1-raw-data/mailinglists/googlegroups"
	"github.com/google/project-OCEAN/1-raw-data/mailinglists/mailman"
	"github.com/google/project-OCEAN/1-raw-data/mailinglists/pipermail"
	"github.com/google/project-OCEAN/1-raw-data/utils"
)

var (
	//Below variables used if build run
	buildListRun = flag.Bool("build-list-run", false, "Use flag to run build list run vs manual command line run.")
	allListRun   = flag.Bool("all-list-run", false, "Use flag to get variables from command-line or do a full mailing list run or to do simple build test run of one mailing list.")
	allDateRun   = flag.Bool("all-date-run", false, "Use flag to get variables from command-line or do a full run")
	projectID    = flag.String("project-id", "", "GCP Project id.")

	//Below variables used if manual run
	bucketName   = flag.String("bucket-name", "mailinglists", "Bucket name to store files.")
	subDirectory = flag.String("subdirectory", "", "Subdirectory to store files. Enter 1 or more and use spaces to identify. CAUTION also enter the groupNames to load to in the same order.")
	mailingList  = flag.String("mailinglist", "", "Choose which mailing list to process either pipermail (default), mailman, googlegroups")
	groupNames   = flag.String("groupname", "", "Mailing list group name. Enter 1 or more and use spaces to identify. CAUTION also enter the buckets to load to in the same order.")
	startDate    = flag.String("start-date", "", "Start date in format of year-month-date and 4dig-2dig-2dig.")
	endDate      = flag.String("end-date", "", "End date in format of year-month-date and 4dig-2dig-2dig.")
	workerNum    = flag.Int("workers", 1, "Number of workers to use for goroutines.")
	subNames     []string
)

func getData(ctx context.Context, storage gcs.Connection, httpToDom utils.HttpDomResponse, workerNum, numMonths int, mailingList, groupName, startDateString, endDateString string) {
	switch mailingList {
	//TODO add start and end dates to pipermail
	case "pipermail":
		if err := pipermail.GetPipermailData(ctx, storage, groupName, startDateString, endDateString, httpToDom); err != nil {
			log.Fatalf("Mailman load failed: %v", err)
		}
	case "mailman":
		if err := mailman.GetMailmanData(ctx, storage, groupName, startDateString, endDateString, numMonths); err != nil {
			log.Fatalf("Mailman load failed: %v", err)
		}
		//TODO add start and end dates to google groups
	case "gg":
		if err := googlegroups.GetGoogleGroupsData(ctx, "", groupName, startDateString, endDateString, storage, workerNum); err != nil {
			log.Fatalf("GoogleGroups load failed: %v", err)
		}
	default:
		log.Fatalf("Mailing list %v is not an option. Change the option submitted.", mailingList)
	}
}

func main() {
	var err error
	numMonths := 1
	httpToDom := utils.DomResponse
	flag.Parse()
	fmt.Printf(*projectID)

	//Setup Storage connection
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	storageConn := gcs.StorageConnection{
		ProjectID:  *projectID,
		BucketName: *bucketName,
	}
	if err = storageConn.ConnectClient(ctx); err != nil {
		log.Fatalf("Connect GCS failes: %v", err)
	}

	if *buildListRun {
		//Build run to load mailing list data
		now := time.Now()
		//Set variables in build that aren't coming in on command line
		groupName := ""

		// Setup bucket connection whether new or not
		storageConn.BucketName = *bucketName
		if err := storageConn.CreateBucket(ctx); err != nil {
			log.Fatalf("Create GCS Bucket failed: %v", err)
		}

		// Run Build to load all mailing lists
		if *allListRun {
			log.Printf("Build all lists ")
			mailListMap := map[string]string{"gg-angular": "2009-09-01", "gg-golang-announce": "2011-05-01", "gg-golang-checkins": "2009-11-01", " gg-golang-codereviews": "2013-12-01", "gg-golang-dev": "2009-11-01", "gg-golang-nuts": "2009-11-01", "gg-nodejs": "2009-06-01", "mailman-python-announce-list": "1999-04-01", "mailman-python-dev": "1999-04-01", "mailman-python-ideas": "2006-12-01", "pipermail-python-announce-list": "1999-04-01", "pipermail-python-dev": "1995-03-01", "pipermail-python-ideas": "2006-12-01", "pipermail-python-list": "1999-02-01"}

			for subName, origStartDate := range mailListMap {
				startDateResult, endDateResult := "", ""
				storageConn.SubDirectory = subName
				*mailingList = strings.SplitN(subName, "-", 2)[0]
				groupName = strings.SplitN(subName, "-", 2)[1]

				if *allDateRun {
					//Load all months
					log.Printf("All Date Cloud Run")
					//Set start and end dates with first mailing list date and current end date
					if startDateResult, endDateResult, err = utils.FixDate(origStartDate, *endDate); err != nil {
						log.Fatalf("Date error: %v", err)
					}
				} else {
					//Set start and end dates split by one month
					if startDateResult, endDateResult, err = utils.SplitDatesByMonth(*startDate, *endDate, 1); err != nil {
						log.Fatalf("Date error: %v", err)
					}
					log.Printf("One Month Run All MailingLists")
					startDateResult = now.AddDate(0, -1, 0).Format("2006-01-02")
					endDateResult = now.Format("2006-01-02")
				}

				//Get mailing list data and store
				getData(ctx, &storageConn, httpToDom, *workerNum, numMonths, *mailingList, groupName, startDateResult, endDateResult)
			}
		} else {
			log.Printf("Build test run with mailman")
			storageConn.SubDirectory = "mailman-python-announce-list"
			groupName = "python-announce-list"
			*startDate = now.AddDate(0, -1, 0).Format("2006-01-02")
			*endDate = now.AddDate(0, -1, 1).Format("2006-01-02")

			if err := mailman.GetMailmanData(ctx, &storageConn, groupName, *startDate, *endDate, numMonths); err != nil {
				log.Fatalf("Mailman test build load failed: %v", err)
			}
		}
	} else {

		//Check and create bucket if needed
		if err := storageConn.CreateBucket(ctx); err != nil {
			log.Fatalf("Create GCS Bucket failed: %v", err)
		}

		if *subDirectory != "" {
			subNames = strings.Split(*subDirectory, " ")
		}

		for idx, groupName := range strings.Split(*groupNames, " ") {
			//Apply sub directory name to storageConn if it exists
			if *subDirectory != "" {
				storageConn.SubDirectory = subNames[idx]
			}

			//Get mailing list data and store
			getData(ctx, &storageConn, httpToDom, *workerNum, numMonths, *mailingList, groupName, *startDate, *endDate)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2011 Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"encoding/gob"
	"errors"
	"flag"
	"fmt"
	"hash/fnv"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/httptest"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"code.google.com/p/goauth2/oauth"
)

var config = &oauth.Config{
	ClientId:     "", // Set by --clientid or --clientid_file
	ClientSecret: "", // Set by --secret or --secret_file
	Scope:        "", // filled in per-API
	AuthURL:      "https://accounts.google.com/o/oauth2/auth",
	TokenURL:     "https://accounts.google.com/o/oauth2/token",
}

// Flags
var (
	clientId     = flag.String("clientid", "", "OAuth Client ID.  If non-empty, overrides --clientid_file")
	clientIdFile = flag.String("clientid_file", "clientid.dat",
		"Name of a file containing just the project's OAuth Client ID from https://code.google.com/apis/console/")
	secret     = flag.String("secret", "", "OAuth Client Secret.  If non-empty, overrides --secret_file")
	secretFile = flag.String("secret_file", "clientsecret.dat",
		"Name of a file containing just the project's OAuth Client Secret from https://code.google.com/apis/console/")
	cacheToken = flag.Bool("cachetoken", true, "cache the OAuth token")
	debug      = flag.Bool("debug", false, "show HTTP traffic")
)

func usage() {
	fmt.Fprintf(os.Stderr, "Usage: go-api-demo <api-demo-name> [api name args]\n\nPossible APIs:\n\n")
	for n, _ := range demoFunc {
		fmt.Fprintf(os.Stderr, "  * %s\n", n)
	}
	os.Exit(2)
}

func main() {
	flag.Parse()
	if flag.NArg() == 0 {
		usage()
	}

	name := flag.Arg(0)
	demo, ok := demoFunc[name]
	if !ok {
		usage()
	}

	config.Scope = demoScope[name]
	config.ClientId = valueOrFileContents(*clientId, *clientIdFile)
	config.ClientSecret = valueOrFileContents(*secret, *secretFile)

	c := getOAuthClient(config)
	demo(c, flag.Args()[1:])
}

var (
	demoFunc  = make(map[string]func(*http.Client, []string))
	demoScope = make(map[string]string)
)

func registerDemo(name, scope string, main func(c *http.Client, argv []string)) {
	if demoFunc[name] != nil {
		panic(name + " already registered")
	}
	demoFunc[name] = main
	demoScope[name] = scope
}

func osUserCacheDir() string {
	switch runtime.GOOS {
	case "darwin":
		return filepath.Join(os.Getenv("HOME"), "Library", "Caches")
	case "linux", "freebsd":
		return filepath.Join(os.Getenv("HOME"), ".cache")
	}
	log.Printf("TODO: osUserCacheDir on GOOS %q", runtime.GOOS)
	return "."
}

func tokenCacheFile(config *oauth.Config) string {
	hash := fnv.New32a()
	hash.Write([]byte(config.ClientId))
	hash.Write([]byte(config.ClientSecret))
	hash.Write([]byte(config.Scope))
	fn := fmt.Sprintf("go-api-demo-tok%v", hash.Sum32())
	return filepath.Join(osUserCacheDir(), url.QueryEscape(fn))
}

func tokenFromFile(file string) (*oauth.Token, error) {
	if !*cacheToken {
		return nil, errors.New("--cachetoken is false")
	}
	f, err := os.Open(file)
	if err != nil {
		return nil, err
	}
	t := new(oauth.Token)
	err = gob.NewDecoder(f).Decode(t)
	return t, err
}

func saveToken(file string, token *oauth.Token) {
	f, err := os.Create(file)
	if err != nil {
		log.Printf("Warning: failed to cache oauth token: %v", err)
		return
	}
	defer f.Close()
	gob.NewEncoder(f).Encode(token)
}

func condDebugTransport(rt http.RoundTripper) http.RoundTripper {
	if *debug {
		return &logTransport{rt}
	}
	return rt
}

func getOAuthClient(config *oauth.Config) *http.Client {
	cacheFile := tokenCacheFile(config)
	token, err := tokenFromFile(cacheFile)
	if err != nil {
		token = tokenFromWeb(config)
		saveToken(cacheFile, token)
	} else {
		log.Printf("Using cached token %#v from %q", token, cacheFile)
	}

	t := &oauth.Transport{
		Token:     token,
		Config:    config,
		Transport: condDebugTransport(http.DefaultTransport),
	}
	return t.Client()
}

func tokenFromWeb(config *oauth.Config) *oauth.Token {
	ch := make(chan string)
	randState := fmt.Sprintf("st%d", time.Now().UnixNano())
	ts := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
		if req.URL.Path == "/favicon.ico" {
			http.Error(rw, "", 404)
			return
		}
		if req.FormValue("state") != randState {
			log.Printf("State doesn't match: req = %#v", req)
			http.Error(rw, "", 500)
			return
		}
		if code := req.FormValue("code"); code != "" {
			fmt.Fprintf(rw, "
Success
Authorized.")
			rw.(http.Flusher).Flush()
			ch <- code
			return
		}
		log.Printf("no code")
		http.Error(rw, "", 500)
	}))
	defer ts.Close()

	config.RedirectURL = ts.URL
	authUrl := config.AuthCodeURL(randState)
	go openUrl(authUrl)
	log.Printf("Authorize this app at: %s", authUrl)
	code := <-ch
	log.Printf("Got code: %s", code)

	t := &oauth.Transport{
		Config:    config,
		Transport: condDebugTransport(http.DefaultTransport),
	}
	_, err := t.Exchange(code)
	if err != nil {
		log.Fatalf("Token exchange error: %v", err)
	}
	return t.Token
}

func openUrl(url string) {
	try := []string{"xdg-open", "google-chrome", "open"}
	for _, bin := range try {
		err := exec.Command(bin, url).Run()
		if err == nil {
			return
		}
	}
	log.Printf("Error opening URL in browser.")
}

func valueOrFileContents(value string, filename string) string {
	if value != "" {
		return value
	}
	slurp, err := ioutil.ReadFile(filename)
	if err != nil {
		log.Fatalf("Error reading %q: %v", filename, err)
	}
	return strings.TrimSpace(string(slurp))
}
<commit_msg>google.golang.org/api/examples: use golang.org/x/oauth2<commit_after>// Copyright 2011 Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"encoding/gob"
	"errors"
	"flag"
	"fmt"
	"hash/fnv"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/httptest"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"golang.org/x/net/context"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
)

// Flags
var (
	clientID     = flag.String("clientid", "", "OAuth 2.0 Client ID.  If non-empty, overrides --clientid_file")
	clientIDFile = flag.String("clientid-file", "clientid.dat",
		"Name of a file containing just the project's OAuth 2.0 Client ID from https://developers.google.com/console.")
	secret     = flag.String("secret", "", "OAuth 2.0 Client Secret.  If non-empty, overrides --secret_file")
	secretFile = flag.String("secret-file", "clientsecret.dat",
		"Name of a file containing just the project's OAuth 2.0 Client Secret from https://developers.google.com/console.")
	cacheToken = flag.Bool("cachetoken", true, "cache the OAuth 2.0 token")
	debug      = flag.Bool("debug", false, "show HTTP traffic")
)

func usage() {
	fmt.Fprintf(os.Stderr, "Usage: go-api-demo <api-demo-name> [api name args]\n\nPossible APIs:\n\n")
	for n := range demoFunc {
		fmt.Fprintf(os.Stderr, "  * %s\n", n)
	}
	os.Exit(2)
}

func main() {
	flag.Parse()
	if flag.NArg() == 0 {
		usage()
	}

	name := flag.Arg(0)
	demo, ok := demoFunc[name]
	if !ok {
		usage()
	}

	config := &oauth2.Config{
		ClientID:     valueOrFileContents(*clientID, *clientIDFile),
		ClientSecret: valueOrFileContents(*secret, *secretFile),
		Endpoint:     google.Endpoint,
		Scopes:       []string{demoScope[name]},
	}

	ctx := context.Background()
	if *debug {
		ctx = context.WithValue(ctx, oauth2.HTTPClient, &http.Client{
			Transport: &logTransport{http.DefaultTransport},
		})
	}
	c := newOAuthClient(ctx, config)
	demo(c, flag.Args()[1:])
}

var (
	demoFunc  = make(map[string]func(*http.Client, []string))
	demoScope = make(map[string]string)
)

func registerDemo(name, scope string, main func(c *http.Client, argv []string)) {
	if demoFunc[name] != nil {
		panic(name + " already registered")
	}
	demoFunc[name] = main
	demoScope[name] = scope
}

func osUserCacheDir() string {
	switch runtime.GOOS {
	case "darwin":
		return filepath.Join(os.Getenv("HOME"), "Library", "Caches")
	case "linux", "freebsd":
		return filepath.Join(os.Getenv("HOME"), ".cache")
	}
	log.Printf("TODO: osUserCacheDir on GOOS %q", runtime.GOOS)
	return "."
}

func tokenCacheFile(config *oauth2.Config) string {
	hash := fnv.New32a()
	hash.Write([]byte(config.ClientID))
	hash.Write([]byte(config.ClientSecret))
	hash.Write([]byte(strings.Join(config.Scopes, " ")))
	fn := fmt.Sprintf("go-api-demo-tok%v", hash.Sum32())
	return filepath.Join(osUserCacheDir(), url.QueryEscape(fn))
}

func tokenFromFile(file string) (*oauth2.Token, error) {
	if !*cacheToken {
		return nil, errors.New("--cachetoken is false")
	}
	f, err := os.Open(file)
	if err != nil {
		return nil, err
	}
	t := new(oauth2.Token)
	err = gob.NewDecoder(f).Decode(t)
	return t, err
}

func saveToken(file string, token *oauth2.Token) {
	f, err := os.Create(file)
	if err != nil {
		log.Printf("Warning: failed to cache oauth token: %v", err)
		return
	}
	defer f.Close()
	gob.NewEncoder(f).Encode(token)
}

func newOAuthClient(ctx context.Context, config *oauth2.Config) *http.Client {
	cacheFile := tokenCacheFile(config)
	token, err := tokenFromFile(cacheFile)
	if err != nil {
		token = tokenFromWeb(ctx, config)
		saveToken(cacheFile, token)
	} else {
		log.Printf("Using cached token %#v from %q", token, cacheFile)
	}

	return config.Client(ctx, token)
}

func tokenFromWeb(ctx context.Context, config *oauth2.Config) *oauth2.Token {
	ch := make(chan string)
	randState := fmt.Sprintf("st%d", time.Now().UnixNano())
	ts := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
		if req.URL.Path == "/favicon.ico" {
			http.Error(rw, "", 404)
			return
		}
		if req.FormValue("state") != randState {
			log.Printf("State doesn't match: req = %#v", req)
			http.Error(rw, "", 500)
			return
		}
		if code := req.FormValue("code"); code != "" {
			fmt.Fprintf(rw, "
Success
Authorized.")
			rw.(http.Flusher).Flush()
			ch <- code
			return
		}
		log.Printf("no code")
		http.Error(rw, "", 500)
	}))
	defer ts.Close()

	config.RedirectURL = ts.URL
	authURL := config.AuthCodeURL(randState)
	go openURL(authURL)
	log.Printf("Authorize this app at: %s", authURL)
	code := <-ch
	log.Printf("Got code: %s", code)

	token, err := config.Exchange(ctx, code)
	if err != nil {
		log.Fatalf("Token exchange error: %v", err)
	}
	return token
}

func openURL(url string) {
	try := []string{"xdg-open", "google-chrome", "open"}
	for _, bin := range try {
		err := exec.Command(bin, url).Run()
		if err == nil {
			return
		}
	}
	log.Printf("Error opening URL in browser.")
}

func valueOrFileContents(value string, filename string) string {
	if value != "" {
		return value
	}
	slurp, err := ioutil.ReadFile(filename)
	if err != nil {
		log.Fatalf("Error reading %q: %v", filename, err)
	}
	return strings.TrimSpace(string(slurp))
}
<|endoftext|>


<commit_before>package main

import (
	"time"

	"github.com/philchia/gol"
)

func main() {
	gol.Debug("Hello world")
	gol.Info("Hello world")
	gol.Warn("Hello world")
	gol.Error("Hello world")
	gol.Critical("Hello world")
	time.Sleep(time.Second)
}
<commit_msg>delete examples<commit_after><|endoftext|>


<commit_before>package rtfdoc_test

import (
	"fmt"

	rtfdoc "github.com/therox/rtf-doc"
)

func ExampleDocument() {
	doc := rtfdoc.NewDocument()
	doc.AddParagraph().SetAlignt(rtfdoc.AlignCenter).AddText("Hello, world!", 14, rtfdoc.FontTimesNewRoman, rtfdoc.ColorAqua)
	bs := doc.Export()
	fmt.Println(string(bs))
}
<commit_msg>Fix broken test<commit_after>package rtfdoc_test

import (
	"fmt"

	rtfdoc "github.com/therox/rtf-doc"
)

func ExampleDocument() {
	doc := rtfdoc.NewDocument()
	doc.AddParagraph().SetAlign(rtfdoc.AlignCenter).AddText("Hello, world!", 14, rtfdoc.FontTimesNewRoman, rtfdoc.ColorAqua)
	bs := doc.Export()
	fmt.Println(string(bs))
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"

	"github.com/tamnd/httpclient"
)

func main() {
	content, err := httpclient.String("http://www.example.com")
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println("%s", content)
}
<commit_msg>Update example<commit_after>package main

import (
	"fmt"
	"log"

	"github.com/tamnd/httpclient"
)

func main() {
	content, err := httpclient.String("http://www.google.com")
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println("%s", content)
}
<|endoftext|>


<commit_before>package gorums

import (
	"bytes"
	"errors"
	"fmt"
	"go/ast"
	"go/parser"
	"go/printer"
	"go/token"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"testing"
)

const (
	gorumsBaseImport = "github.com/relab/gorums"
	devImport        = gorumsBaseImport + "/" + devDir
	e2eTestDevImport = gorumsBaseImport + "/" + e2eTestDir + "/" + devDir

	devDir     = "dev"
	e2eTestDir = "e2etest"

	storageProtoFile = "storage.proto"

	devStorageProtoRelPath = devDir + "/" + storageProtoFile

	protoc        = "protoc"
	protocIFlag   = "-I=../../../:."
	protocOutFlag = "--gorums_out=plugins=grpc+gorums:"
)

func run(t *testing.T, name string, args ...string) {
	t.Helper()
	cmd := exec.Command(name, args...)
	cmd.Env = append(cmd.Env, os.Environ()...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

func runAndCaptureOutput(command string, args ...string) ([]byte, error) {
	cmd := exec.Command(command, args...)
	out, err := cmd.CombinedOutput()
	if err != nil {
		return nil, fmt.Errorf("%v\n%s", err, out)
	}
	return bytes.TrimSuffix(out, []byte{'\n'}), nil
}

const (
	protocVersionPrefix  = "libprotoc "
	currentProtocVersion = "3.4.0"
)

func checkProtocVersion(t *testing.T) {
	t.Helper()
	out, err := runAndCaptureOutput("protoc", "--version")
	if err != nil {
		t.Skipf("skipping test due to protoc error: %v", err)
	}
	gotVersion := string(out)
	gotVersion = strings.TrimPrefix(gotVersion, protocVersionPrefix)
	if gotVersion != currentProtocVersion {
		t.Skipf("skipping test due to old protoc version, got %q, required is %q", gotVersion, currentProtocVersion)
	}
}

var devFilesToCopy = []struct {
	name              string
	devPath, testPath string
	rewriteImport     bool
}{
	{
		"config_qc_test.go",
		"", "",
		true,
	},
	{
		"config_qc_secure_test.go",
		"", "",
		true,
	},
	{
		"config_qc_qspecs_test.go",
		"", "",
		true,
	},
	{
		"node_test.go",
		"", "",
		false,
	},
	{
		"mgr_test.go",
		"", "",
		true,
	},
	{
		"storage_server_udef.go",
		"", "",
		false,
	},
}

// copy copies the from file to the to file.
func copy(to, from string) error {
	toFd, err := os.Create(to)
	if err != nil {
		return err
	}
	defer toFd.Close()
	fromFd, err := os.Open(from)
	if err != nil {
		return err
	}
	defer fromFd.Close()
	_, err = io.Copy(toFd, fromFd)
	return err
}

// Based on https://github.com/tools/godep/blob/master/rewrite.go.
// https://github.com/tools/godep/blob/master/License
func rewriteGoFile(name, old, new string) error {
	printerConfig := &printer.Config{Mode: printer.TabIndent | printer.UseSpaces, Tabwidth: 8}
	fset := token.NewFileSet()
	f, err := parser.ParseFile(fset, name, nil, parser.ParseComments)
	if err != nil {
		return err
	}

	var changed bool
	for _, s := range f.Imports {
		iname, ierr := strconv.Unquote(s.Path.Value)
		if ierr != nil {
			return err // can't happen
		}
		if iname == old {
			s.Path.Value = strconv.Quote(new)
			changed = true
		}
	}

	if !changed {
		return errors.New("no import changed for file")
	}

	var buffer bytes.Buffer
	if err = printerConfig.Fprint(&buffer, fset, f); err != nil {
		return err
	}
	fset = token.NewFileSet()
	f, err = parser.ParseFile(fset, name, &buffer, parser.ParseComments)
	if err != nil {
		return err
	}
	ast.SortImports(fset, f)

	tpath := name + ".temp"
	t, err := os.Create(tpath)
	if err != nil {
		return err
	}
	if err = printerConfig.Fprint(t, fset, f); err != nil {
		return err
	}
	if err = t.Close(); err != nil {
		return err
	}
	// This is required before the rename on windows.
	if err = os.Remove(name); err != nil {
		return err
	}
	return os.Rename(tpath, name)
}

// TestEndToEnd runs the test suite from dev on the output generated by the gorums plugin.
func TestEndToEnd(t *testing.T) {
	checkProtocVersion(t)

	// Create temporary test dir.
	err := os.MkdirAll(e2eTestDir, 0755)
	if err != nil {
		t.Errorf("%v", err)
	}
	defer os.RemoveAll(e2eTestDir)

	// Run the proto compiler.
	run(t, protoc, protocIFlag, protocOutFlag+e2eTestDir, devStorageProtoRelPath)

	// Set file paths.
	e2eTestDevDirPath := filepath.Join(e2eTestDir, devDir)
	for i, file := range devFilesToCopy {
		devFilesToCopy[i].devPath = filepath.Join(devDir, file.name)
		devFilesToCopy[i].testPath = filepath.Join(e2eTestDevDirPath, file.name)
	}

	// Copy relevant files from dev dir.
	for _, file := range devFilesToCopy {
		err := copy(file.testPath, file.devPath)
		if err != nil {
			t.Fatalf("error copying file %q: %v", file.devPath, err)
		}
	}

	// Rewrite one import path for some test files.
	for _, file := range devFilesToCopy {
		if !file.rewriteImport {
			continue
		}
		err := rewriteGoFile(file.testPath, devImport, e2eTestDevImport)
		if err != nil {
			t.Fatalf("error rewriting import for file %q: %v", file.testPath, err)
		}
	}

	// Run go test. Use an explicit base for localhost ports to avoid
	// "address already in use" errors with regular 'dev' package tests
	// that may run concurrently.
	run(t, "go", "test", e2eTestDevImport, "-portbase=30000")
}
<commit_msg>endtoend_test: use gogo/protobuf/version to check for protoc version<commit_after>package gorums

import (
	"bytes"
	"errors"
	"go/ast"
	"go/parser"
	"go/printer"
	"go/token"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"testing"

	"github.com/gogo/protobuf/version"
)

const (
	gorumsBaseImport = "github.com/relab/gorums"
	devImport        = gorumsBaseImport + "/" + devDir
	e2eTestDevImport = gorumsBaseImport + "/" + e2eTestDir + "/" + devDir

	devDir     = "dev"
	e2eTestDir = "e2etest"

	storageProtoFile = "storage.proto"

	devStorageProtoRelPath = devDir + "/" + storageProtoFile

	protoc        = "protoc"
	protocIFlag   = "-I=../../../:."
	protocOutFlag = "--gorums_out=plugins=grpc+gorums:"
)

func run(t *testing.T, name string, args ...string) {
	t.Helper()
	cmd := exec.Command(name, args...)
	cmd.Env = append(cmd.Env, os.Environ()...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

const currentProtocVersion = "3.5.0"

var devFilesToCopy = []struct {
	name              string
	devPath, testPath string
	rewriteImport     bool
}{
	{
		"config_qc_test.go",
		"", "",
		true,
	},
	{
		"config_qc_secure_test.go",
		"", "",
		true,
	},
	{
		"config_qc_qspecs_test.go",
		"", "",
		true,
	},
	{
		"node_test.go",
		"", "",
		false,
	},
	{
		"mgr_test.go",
		"", "",
		true,
	},
	{
		"storage_server_udef.go",
		"", "",
		false,
	},
}

// copy copies the from file to the to file.
func copy(to, from string) error {
	toFd, err := os.Create(to)
	if err != nil {
		return err
	}
	defer toFd.Close()
	fromFd, err := os.Open(from)
	if err != nil {
		return err
	}
	defer fromFd.Close()
	_, err = io.Copy(toFd, fromFd)
	return err
}

// Based on https://github.com/tools/godep/blob/master/rewrite.go.
// https://github.com/tools/godep/blob/master/License
func rewriteGoFile(name, old, new string) error {
	printerConfig := &printer.Config{Mode: printer.TabIndent | printer.UseSpaces, Tabwidth: 8}
	fset := token.NewFileSet()
	f, err := parser.ParseFile(fset, name, nil, parser.ParseComments)
	if err != nil {
		return err
	}

	var changed bool
	for _, s := range f.Imports {
		iname, ierr := strconv.Unquote(s.Path.Value)
		if ierr != nil {
			return err // can't happen
		}
		if iname == old {
			s.Path.Value = strconv.Quote(new)
			changed = true
		}
	}

	if !changed {
		return errors.New("no import changed for file")
	}

	var buffer bytes.Buffer
	if err = printerConfig.Fprint(&buffer, fset, f); err != nil {
		return err
	}
	fset = token.NewFileSet()
	f, err = parser.ParseFile(fset, name, &buffer, parser.ParseComments)
	if err != nil {
		return err
	}
	ast.SortImports(fset, f)

	tpath := name + ".temp"
	t, err := os.Create(tpath)
	if err != nil {
		return err
	}
	if err = printerConfig.Fprint(t, fset, f); err != nil {
		return err
	}
	if err = t.Close(); err != nil {
		return err
	}
	// This is required before the rename on windows.
	if err = os.Remove(name); err != nil {
		return err
	}
	return os.Rename(tpath, name)
}

// TestEndToEnd runs the test suite from dev on the output generated by the gorums plugin.
func TestEndToEnd(t *testing.T) {
	if !version.AtLeast(currentProtocVersion) {
		t.Skipf("skipping test due to old protoc version, got %q, required is at least %q", version.Get(), currentProtocVersion)
	}

	// Create temporary test dir.
	err := os.MkdirAll(e2eTestDir, 0755)
	if err != nil {
		t.Errorf("%v", err)
	}
	defer os.RemoveAll(e2eTestDir)

	// Run the proto compiler.
	run(t, protoc, protocIFlag, protocOutFlag+e2eTestDir, devStorageProtoRelPath)

	// Set file paths.
	e2eTestDevDirPath := filepath.Join(e2eTestDir, devDir)
	for i, file := range devFilesToCopy {
		devFilesToCopy[i].devPath = filepath.Join(devDir, file.name)
		devFilesToCopy[i].testPath = filepath.Join(e2eTestDevDirPath, file.name)
	}

	// Copy relevant files from dev dir.
	for _, file := range devFilesToCopy {
		err := copy(file.testPath, file.devPath)
		if err != nil {
			t.Fatalf("error copying file %q: %v", file.devPath, err)
		}
	}

	// Rewrite one import path for some test files.
	for _, file := range devFilesToCopy {
		if !file.rewriteImport {
			continue
		}
		err := rewriteGoFile(file.testPath, devImport, e2eTestDevImport)
		if err != nil {
			t.Fatalf("error rewriting import for file %q: %v", file.testPath, err)
		}
	}

	// Run go test. Use an explicit base for localhost ports to avoid
	// "address already in use" errors with regular 'dev' package tests
	// that may run concurrently.
	run(t, "go", "test", e2eTestDevImport, "-portbase=30000")
}
<|endoftext|>


<commit_before>// the engine package will hold the data structures that meowy uses
// to represent people and things that people want to do
package engine

import "strings"

// TODO better name than 'person'?
type Person struct {
	Things []string
}

// TODO find more replacements? ask amber
var replacements = map[string]string{
	"my": "your",
}

func replaceInput(s string) string {
	for old, new := range replacements {
		s = strings.Replace(s, old, new, -1)
	}
	return s
}

func NewPerson() *Person {
	return &Person{
		Things: make([]string, 0, 1),
	}
}

// Pushes t to the front of Person.Things
func (p *Person) AddThing(t string) {
	p.Things = append([]string{t}, p.Things...)
}

// For template processing, we need to define the following methods:
// TopPriority and Rest. Then can then be used in the template as
// {{ .TopPriority }} and {{ range .Rest }}
func (p *Person) TopPriority() string {
	if len(p.Things) == 0 {
		return "________"
	}
	return p.Things[0]
}

func (p *Person) Rest() []string {
	if len(p.Things) < 2 {
		return nil
	}
	return p.Things[1:]
}

func (p *Person) Delete(i int) {
	if len(p.Things) == 0 || i < 0 || i >= len(p.Things) {
		// this shouldn't happen
		return
	}
	if i == len(p.Things)-1 {
		p.Things = p.Things[:i]
		return
	}
	p.Things = append(p.Things[:i], p.Things[i+1:]...)
}

func (p *Person) Promote(i int) {
	if len(p.Things) == 0 || i < 0 || i >= len(p.Things) {
		// this shouldn't happen
		return
	}
	t := p.Things[i]
	p.Delete(i)
	p.Things = append([]string{t}, p.Things...)
}
<commit_msg>Replaces my -> your<commit_after>// the engine package will hold the data structures that meowy uses
// to represent people and things that people want to do
package engine

import "strings"

// TODO better name than 'person'?
type Person struct {
	Things []string
}

// TODO find more replacements? ask amber
var replacements = map[string]string{
	"my": "your",
}

// TODO name this better
func replaceInput(s string) string {
	for old, new := range replacements {
		s = strings.Replace(s, old, new, -1)
	}
	return s
}

func NewPerson() *Person {
	return &Person{
		Things: make([]string, 0, 1),
	}
}

// Pushes t to the front of Person.Things
func (p *Person) AddThing(t string) {
	p.Things = append([]string{replaceInput(t)}, p.Things...)
}

// For template processing, we need to define the following methods:
// TopPriority and Rest. Then can then be used in the template as
// {{ .TopPriority }} and {{ range .Rest }}
func (p *Person) TopPriority() string {
	if len(p.Things) == 0 {
		return "________"
	}
	return p.Things[0]
}

func (p *Person) Rest() []string {
	if len(p.Things) < 2 {
		return nil
	}
	return p.Things[1:]
}

func (p *Person) Delete(i int) {
	if len(p.Things) == 0 || i < 0 || i >= len(p.Things) {
		// this shouldn't happen
		return
	}
	if i == len(p.Things)-1 {
		p.Things = p.Things[:i]
		return
	}
	p.Things = append(p.Things[:i], p.Things[i+1:]...)
}

func (p *Person) Promote(i int) {
	if len(p.Things) == 0 || i < 0 || i >= len(p.Things) {
		// this shouldn't happen
		return
	}
	t := p.Things[i]
	p.Delete(i)
	p.Things = append([]string{t}, p.Things...)
}
<|endoftext|>


<commit_before>package envplate

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"testing"

	"github.com/stretchr/testify/assert"
)

func _delete(t *testing.T, name string) {

	if err := os.Remove(name); err != nil {
		t.Fatalf("Error while deleting '%s': %v", name, err)
	}

}

func _exists(name string) bool {
	_, err := os.Stat(name)
	return err == nil
}

func _read(t *testing.T, name string) string {

	content, err := ioutil.ReadFile(name)

	if err != nil {
		t.Fatalf("Error while reading '%s': %v", name, err)
	}

	return string(content)

}

func _template(t *testing.T) (string, string) {

	tpl := `Database1=${DATABASE}
  Mode=${MODE}
  Database2=${DATABASE}
  Database4=${DATABASE1:-FOOBAR}
  Database3=$FOOBAR`

	return _write(t, "parse.txt", tpl, 0644), tpl

}

func _write(t *testing.T, name, content string, mode os.FileMode) string {

	file, err := ioutil.TempFile("", name)

	if err != nil {
		t.Fatalf("Error while opening '%s': %v", name, err)
	}

	if _, err := file.WriteString(content); err != nil {
		t.Fatalf("Error while writing to '%s': %v", name, err)
	}

	if err := file.Close(); err != nil {
		t.Fatalf("Error while closing '%s': %v", name, err)
	}

	if err := os.Chmod(file.Name(), mode); err != nil {
		t.Fatalf("Error while chmod '%s': %v", name, err)
	}

	return file.Name()

}

func TestCreateBackup(t *testing.T) {

	assert := assert.New(t)

	file := _write(t, "create-backup.txt", "hello world", 0644)
	defer _delete(t, file)

	backup := fmt.Sprintf("%s.bak", file)

	assert.False(_exists(backup))

	err := createBackup(file)
	defer _delete(t, backup)

	assert.NoError(err)

	content := _read(t, backup)

	assert.Equal("hello world", content)
	assert.Equal(filemode(file).String(), filemode(backup).String())

}

func TestApplyNoGlobs(t *testing.T) {

	var buf bytes.Buffer

	ErrorFunc = func(format string, args ...interface{}) {
		fmt.Fprintf(&buf, format, args...)
	}

	globs := []string{
		"*.not-here",
		"*.not-there",
	}

	Apply(globs)

	assert.Equal(t, "[ ERROR ] Zero files matched passed globs '[*.not-here *.not-there]'", buf.String())

}

func TestFullParse(t *testing.T) {

	Config.Backup = true
	Config.DryRun = false
	Config.Verbose = true

	ErrorFunc = log.Panicf

	assert := assert.New(t)

	file, _ := _template(t)
	defer _delete(t, file)

	backup := fmt.Sprintf("%s.bak", file)

	err := parse(file)

	assert.NoError(err)
	assert.True(_exists(backup))
	assert.Equal(`Database1=db.example.com
  Mode=debug
  Database2=db.example.com
  Database4=FOOBAR
  Database3=$FOOBAR`, _read(t, file))

}

func TestFilemode(t *testing.T) {

	file := _write(t, "filemode.text", "", 0654)
	defer _delete(t, file)

	mode := filemode(file)

	assert.Equal(t, 0654, mode)

}
<commit_msg>Testcase<commit_after>package envplate

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"testing"

	"github.com/stretchr/testify/assert"
)

func _delete(t *testing.T, name string) {

	if err := os.Remove(name); err != nil {
		t.Fatalf("Error while deleting '%s': %v", name, err)
	}

}

func _exists(name string) bool {
	_, err := os.Stat(name)
	return err == nil
}

func _read(t *testing.T, name string) string {

	content, err := ioutil.ReadFile(name)

	if err != nil {
		t.Fatalf("Error while reading '%s': %v", name, err)
	}

	return string(content)

}

func _template(t *testing.T) (string, string) {

	tpl := `Database1=${DATABASE}
  Mode=${MODE}
  Database2=${DATABASE}
  Database4=${DATABASE1:-FOOBAR}
  Database5=${DATABASE:-FOOBAR}
  Database3=$FOOBAR`

	return _write(t, "parse.txt", tpl, 0644), tpl

}

func _write(t *testing.T, name, content string, mode os.FileMode) string {

	file, err := ioutil.TempFile("", name)

	if err != nil {
		t.Fatalf("Error while opening '%s': %v", name, err)
	}

	if _, err := file.WriteString(content); err != nil {
		t.Fatalf("Error while writing to '%s': %v", name, err)
	}

	if err := file.Close(); err != nil {
		t.Fatalf("Error while closing '%s': %v", name, err)
	}

	if err := os.Chmod(file.Name(), mode); err != nil {
		t.Fatalf("Error while chmod '%s': %v", name, err)
	}

	return file.Name()

}

func TestCreateBackup(t *testing.T) {

	assert := assert.New(t)

	file := _write(t, "create-backup.txt", "hello world", 0644)
	defer _delete(t, file)

	backup := fmt.Sprintf("%s.bak", file)

	assert.False(_exists(backup))

	err := createBackup(file)
	defer _delete(t, backup)

	assert.NoError(err)

	content := _read(t, backup)

	assert.Equal("hello world", content)
	assert.Equal(filemode(file).String(), filemode(backup).String())

}

func TestApplyNoGlobs(t *testing.T) {

	var buf bytes.Buffer

	ErrorFunc = func(format string, args ...interface{}) {
		fmt.Fprintf(&buf, format, args...)
	}

	globs := []string{
		"*.not-here",
		"*.not-there",
	}

	Apply(globs)

	assert.Equal(t, "[ ERROR ] Zero files matched passed globs '[*.not-here *.not-there]'", buf.String())

}

func TestFullParse(t *testing.T) {

	Config.Backup = true
	Config.DryRun = false
	Config.Verbose = true

	ErrorFunc = log.Panicf

	assert := assert.New(t)

	file, _ := _template(t)
	defer _delete(t, file)

	backup := fmt.Sprintf("%s.bak", file)

	err := parse(file)

	assert.NoError(err)
	assert.True(_exists(backup))
	assert.Equal(`Database1=db.example.com
  Mode=debug
  Database2=db.example.com
  Database4=FOOBAR
  Database5=db.example.com
  Database3=$FOOBAR`, _read(t, file))

}

func TestFilemode(t *testing.T) {

	file := _write(t, "filemode.text", "", 0654)
	defer _delete(t, file)

	mode := filemode(file)

	assert.Equal(t, 0654, mode)

}
<|endoftext|>


<commit_before>package integration

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"testing"

	"github.com/buildkite/agent/v3/bootstrap/shell"
	"github.com/buildkite/bintest/v3"
)

func TestRunningPlugins(t *testing.T) {
	t.Parallel()

	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	pluginMock := tester.MustMock(t, "my-plugin")

	var p *testPlugin

	if runtime.GOOS == "windows" {
		p = createTestPlugin(t, map[string][]string{
			"environment.bat": {
				"@echo off",
				"set LLAMAS_ROCK=absolutely",
				pluginMock.Path + " testing",
			},
		})
	} else {
		p = createTestPlugin(t, map[string][]string{
			"environment": {
				"#!/bin/bash",
				"export LLAMAS_ROCK=absolutely",
				pluginMock.Path + " testing",
			},
		})
	}

	json, err := p.ToJSON()
	if err != nil {
		t.Fatal(err)
	}

	env := []string{
		`MY_CUSTOM_ENV=1`,
		`BUILDKITE_PLUGINS=` + json,
	}

	pluginMock.Expect("testing").Once().AndCallFunc(func(c *bintest.Call) {
		if err := bintest.ExpectEnv(t, c.Env, `MY_CUSTOM_ENV=1`, `LLAMAS_ROCK=absolutely`); err != nil {
			fmt.Fprintf(c.Stderr, "%v\n", err)
			c.Exit(1)
		} else {
			c.Exit(0)
		}
	})

	tester.ExpectGlobalHook("command").Once().AndExitWith(0).AndCallFunc(func(c *bintest.Call) {
		if err := bintest.ExpectEnv(t, c.Env, `MY_CUSTOM_ENV=1`, `LLAMAS_ROCK=absolutely`); err != nil {
			fmt.Fprintf(c.Stderr, "%v\n", err)
			c.Exit(1)
		} else {
			c.Exit(0)
		}
	})

	tester.RunAndCheck(t, env...)
}

func TestExitCodesPropagateOutFromPlugins(t *testing.T) {
	t.Parallel()

	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	var p *testPlugin

	if runtime.GOOS == "windows" {
		p = createTestPlugin(t, map[string][]string{
			"environment.bat": {
				"@echo off",
				"exit 5",
			},
		})
	} else {
		p = createTestPlugin(t, map[string][]string{
			"environment": {
				"#!/bin/bash",
				"exit 5",
			},
		})
	}

	json, err := p.ToJSON()
	if err != nil {
		t.Fatal(err)
	}

	env := []string{
		`BUILDKITE_PLUGINS=` + json,
	}

	err = tester.Run(t, env...)
	if err == nil {
		t.Fatal("Expected the bootstrap to fail")
	}

	exitCode := shell.GetExitCode(err)

	if exitCode != 5 {
		t.Fatalf("Expected an exit code of %d, got %d", 5, exitCode)
	}

	tester.CheckMocks(t)
}

func TestMalformedPluginNamesDontCrashBootstrap(t *testing.T) {
	t.Parallel()

	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	env := []string{
		`BUILDKITE_PLUGINS=["sdgmdgn.@$!sdf,asdf#llamas"]`,
	}

	if err = tester.Run(t, env...); err == nil {
		t.Fatal("Expected the bootstrap to fail")
	}

	tester.CheckMocks(t)
}

// A job may have multiple plugins that provide multiple hooks of a given type.
// For a while (late 2019 / early 2020) we disallowed duplicate checkout and
// command hooks from plugins; only the first would execute.  We since decided
// to roll that back and permit, for example, multiple checkout plugin hooks.
func TestOverlappingPluginHooks(t *testing.T) {
	t.Parallel()

	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	var testPlugins []*testPlugin
	var pluginMocks []*bintest.Mock
	setupMock := func(name string, hooks ...string) *bintest.Mock {
		pluginMock := tester.MustMock(t, name)
		fakeHooks := make(map[string][]string)
		for _, hook := range hooks {
			if runtime.GOOS == "windows" {
				fakeHooks[hook+".bat"] = []string{
					"@echo off",
					pluginMock.Path + " " + hook,
				}
			} else {
				fakeHooks[hook] = []string{
					"#!/bin/bash",
					pluginMock.Path + " " + hook,
				}
			}
		}
		testPlugins = append(testPlugins, createTestPlugin(t, fakeHooks))
		pluginMocks = append(pluginMocks, pluginMock)
		return pluginMock
	}

	mockA := setupMock("plugin-a", "environment")
	mockA.Expect("environment").Once()

	mockB := setupMock("plugin-b", "environment", "checkout")
	mockB.Expect("environment").Once()
	mockB.Expect("checkout").Once()

	mockC := setupMock("plugin-c", "checkout", "command")
	mockC.Expect("checkout").Once() // even though plugin-b already ran checkout
	mockC.Expect("command").Once()

	mockD := setupMock("plugin-d", "command", "post-command")
	mockD.Expect("command").Once() // even though plugin-c already ran command
	mockD.Expect("post-command").Once()

	pluginsJSON, err := json.Marshal(testPlugins)
	if err != nil {
		t.Fatal(err)
	}

	env := []string{
		`MY_CUSTOM_ENV=1`,
		`BUILDKITE_PLUGINS=` + string(pluginsJSON),
	}

	tester.RunAndCheck(t, env...)
}

func TestPluginCloneRetried(t *testing.T) {
	if runtime.GOOS == `windows` {
		t.Skip("Not passing on windows, needs investigation")
	}

	t.Parallel()

	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	var p *testPlugin
	if runtime.GOOS == "windows" {
		p = createTestPlugin(t, map[string][]string{
			"environment.bat": {
				"@echo off",
				"set LLAMAS_ROCK=absolutely",
			},
		})
	} else {
		p = createTestPlugin(t, map[string][]string{
			"environment": {
				"#!/bin/bash",
				"export LLAMAS_ROCK=absolutely",
			},
		})
	}

	callCount := 0

	realGit, err := exec.LookPath("git")
	if err != nil {
		t.Fatal(err)
	}

	git := tester.MustMock(t, "git")

	git.Expect("clone", "-v", "--", p.Path, bintest.MatchAny()).Exactly(2).AndCallFunc(func(c *bintest.Call) {
		callCount = callCount + 1

		if callCount == 1 {
			c.Exit(1)
		} else {
			c.Passthrough(realGit)
		}
	})

	git.IgnoreUnexpectedInvocations()

	json, err := p.ToJSON()
	if err != nil {
		t.Fatal(err)
	}

	env := []string{
		`BUILDKITE_PLUGINS=` + json,
	}

	tester.RunAndCheck(t, env...)
}

type testPlugin struct {
	*gitRepository

	// What version of this mock plugin do we want?  Defaults to `git rev-parse HEAD`
	versionTag string
}

func createTestPlugin(t *testing.T, hooks map[string][]string) *testPlugin {
	repo, err := newGitRepository()
	if err != nil {
		t.Fatal(err)
	}

	if err := os.MkdirAll(filepath.Join(repo.Path, "hooks"), 0700); err != nil {
		t.Fatal(err)
	}

	for hook, lines := range hooks {
		data := []byte(strings.Join(lines, "\n"))
		if err := ioutil.WriteFile(filepath.Join(repo.Path, "hooks", hook), data, 0600); err != nil {
			t.Fatal(err)
		}
	}

	if err = repo.Add("."); err != nil {
		t.Fatal(err)
	}

	if err = repo.Commit("Initial commit of plugin hooks"); err != nil {
		t.Fatal(err)
	}

	commitHash, err := repo.RevParse("HEAD")
	if err != nil {
		t.Fatal(err)
	}
	return &testPlugin{repo, commitHash}
}

// modifyTestPlugin applies a change to a plugin's contents and makes a commit.  This is useful for
// testing BUILDKITE_PLUGINS_ALWAYS_CLONE_FRESH behaviour.
func modifyTestPlugin(t *testing.T, hooks map[string][]string, testPlugin *testPlugin) {
	repo := testPlugin.gitRepository

	for hook, lines := range hooks {
		data := []byte(strings.Join(lines, "\n"))
		if err := ioutil.WriteFile(filepath.Join(repo.Path, "hooks", hook), data, 0600); err != nil {
			t.Fatal(err)
		}
	}

	if err := repo.Add("."); err != nil {
		t.Fatal(err)
	}

	if err := repo.Commit("Updating content of plugin"); err != nil {
		t.Fatal(err)
	}
}

// ToJSON turns a single testPlugin into a single-item JSON
// array suitable for BUILDKITE_PLUGINS
func (tp *testPlugin) ToJSON() (string, error) {
	data, err := json.Marshal([]*testPlugin{tp})
	if err != nil {
		return "", err
	}
	return string(data), nil
}

// MarshalJSON turns a single testPlugin into a JSON object.
// BUILDKITE_PLUGINS expects an array of these, so it would
// generally be used on a []testPlugin slice.
func (tp *testPlugin) MarshalJSON() ([]byte, error) {
	normalizedPath := strings.TrimPrefix(strings.Replace(tp.Path, "\\", "/", -1), "/")

	p := map[string]interface{}{
		fmt.Sprintf(`file:///%s#%s`, normalizedPath, strings.TrimSpace(tp.versionTag)): map[string]string{
			"settings": "blah",
		},
	}
	b, err := json.Marshal(&p)
	if err != nil {
		return nil, err
	}
	return b, nil
}
<commit_msg>Add helper to replace BUILDKITE_PLUGINS_PATH in an Env slice.<commit_after>package integration

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"testing"

	"github.com/buildkite/agent/v3/bootstrap/shell"
	"github.com/buildkite/bintest/v3"
)

func TestRunningPlugins(t *testing.T) {
	t.Parallel()

	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	pluginMock := tester.MustMock(t, "my-plugin")

	var p *testPlugin

	if runtime.GOOS == "windows" {
		p = createTestPlugin(t, map[string][]string{
			"environment.bat": {
				"@echo off",
				"set LLAMAS_ROCK=absolutely",
				pluginMock.Path + " testing",
			},
		})
	} else {
		p = createTestPlugin(t, map[string][]string{
			"environment": {
				"#!/bin/bash",
				"export LLAMAS_ROCK=absolutely",
				pluginMock.Path + " testing",
			},
		})
	}

	json, err := p.ToJSON()
	if err != nil {
		t.Fatal(err)
	}

	env := []string{
		`MY_CUSTOM_ENV=1`,
		`BUILDKITE_PLUGINS=` + json,
	}

	pluginMock.Expect("testing").Once().AndCallFunc(func(c *bintest.Call) {
		if err := bintest.ExpectEnv(t, c.Env, `MY_CUSTOM_ENV=1`, `LLAMAS_ROCK=absolutely`); err != nil {
			fmt.Fprintf(c.Stderr, "%v\n", err)
			c.Exit(1)
		} else {
			c.Exit(0)
		}
	})

	tester.ExpectGlobalHook("command").Once().AndExitWith(0).AndCallFunc(func(c *bintest.Call) {
		if err := bintest.ExpectEnv(t, c.Env, `MY_CUSTOM_ENV=1`, `LLAMAS_ROCK=absolutely`); err != nil {
			fmt.Fprintf(c.Stderr, "%v\n", err)
			c.Exit(1)
		} else {
			c.Exit(0)
		}
	})

	tester.RunAndCheck(t, env...)
}

func TestExitCodesPropagateOutFromPlugins(t *testing.T) {
	t.Parallel()

	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	var p *testPlugin

	if runtime.GOOS == "windows" {
		p = createTestPlugin(t, map[string][]string{
			"environment.bat": {
				"@echo off",
				"exit 5",
			},
		})
	} else {
		p = createTestPlugin(t, map[string][]string{
			"environment": {
				"#!/bin/bash",
				"exit 5",
			},
		})
	}

	json, err := p.ToJSON()
	if err != nil {
		t.Fatal(err)
	}

	env := []string{
		`BUILDKITE_PLUGINS=` + json,
	}

	err = tester.Run(t, env...)
	if err == nil {
		t.Fatal("Expected the bootstrap to fail")
	}

	exitCode := shell.GetExitCode(err)

	if exitCode != 5 {
		t.Fatalf("Expected an exit code of %d, got %d", 5, exitCode)
	}

	tester.CheckMocks(t)
}

func TestMalformedPluginNamesDontCrashBootstrap(t *testing.T) {
	t.Parallel()

	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	env := []string{
		`BUILDKITE_PLUGINS=["sdgmdgn.@$!sdf,asdf#llamas"]`,
	}

	if err = tester.Run(t, env...); err == nil {
		t.Fatal("Expected the bootstrap to fail")
	}

	tester.CheckMocks(t)
}

// A job may have multiple plugins that provide multiple hooks of a given type.
// For a while (late 2019 / early 2020) we disallowed duplicate checkout and
// command hooks from plugins; only the first would execute.  We since decided
// to roll that back and permit, for example, multiple checkout plugin hooks.
func TestOverlappingPluginHooks(t *testing.T) {
	t.Parallel()

	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	var testPlugins []*testPlugin
	var pluginMocks []*bintest.Mock
	setupMock := func(name string, hooks ...string) *bintest.Mock {
		pluginMock := tester.MustMock(t, name)
		fakeHooks := make(map[string][]string)
		for _, hook := range hooks {
			if runtime.GOOS == "windows" {
				fakeHooks[hook+".bat"] = []string{
					"@echo off",
					pluginMock.Path + " " + hook,
				}
			} else {
				fakeHooks[hook] = []string{
					"#!/bin/bash",
					pluginMock.Path + " " + hook,
				}
			}
		}
		testPlugins = append(testPlugins, createTestPlugin(t, fakeHooks))
		pluginMocks = append(pluginMocks, pluginMock)
		return pluginMock
	}

	mockA := setupMock("plugin-a", "environment")
	mockA.Expect("environment").Once()

	mockB := setupMock("plugin-b", "environment", "checkout")
	mockB.Expect("environment").Once()
	mockB.Expect("checkout").Once()

	mockC := setupMock("plugin-c", "checkout", "command")
	mockC.Expect("checkout").Once() // even though plugin-b already ran checkout
	mockC.Expect("command").Once()

	mockD := setupMock("plugin-d", "command", "post-command")
	mockD.Expect("command").Once() // even though plugin-c already ran command
	mockD.Expect("post-command").Once()

	pluginsJSON, err := json.Marshal(testPlugins)
	if err != nil {
		t.Fatal(err)
	}

	env := []string{
		`MY_CUSTOM_ENV=1`,
		`BUILDKITE_PLUGINS=` + string(pluginsJSON),
	}

	tester.RunAndCheck(t, env...)
}

func TestPluginCloneRetried(t *testing.T) {
	if runtime.GOOS == `windows` {
		t.Skip("Not passing on windows, needs investigation")
	}

	t.Parallel()

	tester, err := NewBootstrapTester()
	if err != nil {
		t.Fatal(err)
	}
	defer tester.Close()

	var p *testPlugin
	if runtime.GOOS == "windows" {
		p = createTestPlugin(t, map[string][]string{
			"environment.bat": {
				"@echo off",
				"set LLAMAS_ROCK=absolutely",
			},
		})
	} else {
		p = createTestPlugin(t, map[string][]string{
			"environment": {
				"#!/bin/bash",
				"export LLAMAS_ROCK=absolutely",
			},
		})
	}

	callCount := 0

	realGit, err := exec.LookPath("git")
	if err != nil {
		t.Fatal(err)
	}

	git := tester.MustMock(t, "git")

	git.Expect("clone", "-v", "--", p.Path, bintest.MatchAny()).Exactly(2).AndCallFunc(func(c *bintest.Call) {
		callCount = callCount + 1

		if callCount == 1 {
			c.Exit(1)
		} else {
			c.Passthrough(realGit)
		}
	})

	git.IgnoreUnexpectedInvocations()

	json, err := p.ToJSON()
	if err != nil {
		t.Fatal(err)
	}

	env := []string{
		`BUILDKITE_PLUGINS=` + json,
	}

	tester.RunAndCheck(t, env...)
}

type testPlugin struct {
	*gitRepository

	// What version of this mock plugin do we want?  Defaults to `git rev-parse HEAD`
	versionTag string
}

func createTestPlugin(t *testing.T, hooks map[string][]string) *testPlugin {
	repo, err := newGitRepository()
	if err != nil {
		t.Fatal(err)
	}

	if err := os.MkdirAll(filepath.Join(repo.Path, "hooks"), 0700); err != nil {
		t.Fatal(err)
	}

	for hook, lines := range hooks {
		data := []byte(strings.Join(lines, "\n"))
		if err := ioutil.WriteFile(filepath.Join(repo.Path, "hooks", hook), data, 0600); err != nil {
			t.Fatal(err)
		}
	}

	if err = repo.Add("."); err != nil {
		t.Fatal(err)
	}

	if err = repo.Commit("Initial commit of plugin hooks"); err != nil {
		t.Fatal(err)
	}

	commitHash, err := repo.RevParse("HEAD")
	if err != nil {
		t.Fatal(err)
	}
	return &testPlugin{repo, commitHash}
}

// modifyTestPlugin applies a change to a plugin's contents and makes a commit.  This is useful for
// testing BUILDKITE_PLUGINS_ALWAYS_CLONE_FRESH behaviour.
func modifyTestPlugin(t *testing.T, hooks map[string][]string, testPlugin *testPlugin) {
	repo := testPlugin.gitRepository

	for hook, lines := range hooks {
		data := []byte(strings.Join(lines, "\n"))
		if err := ioutil.WriteFile(filepath.Join(repo.Path, "hooks", hook), data, 0600); err != nil {
			t.Fatal(err)
		}
	}

	if err := repo.Add("."); err != nil {
		t.Fatal(err)
	}

	if err := repo.Commit("Updating content of plugin"); err != nil {
		t.Fatal(err)
	}
}

// ToJSON turns a single testPlugin into a single-item JSON
// array suitable for BUILDKITE_PLUGINS
func (tp *testPlugin) ToJSON() (string, error) {
	data, err := json.Marshal([]*testPlugin{tp})
	if err != nil {
		return "", err
	}
	return string(data), nil
}

// MarshalJSON turns a single testPlugin into a JSON object.
// BUILDKITE_PLUGINS expects an array of these, so it would
// generally be used on a []testPlugin slice.
func (tp *testPlugin) MarshalJSON() ([]byte, error) {
	normalizedPath := strings.TrimPrefix(strings.Replace(tp.Path, "\\", "/", -1), "/")

	p := map[string]interface{}{
		fmt.Sprintf(`file:///%s#%s`, normalizedPath, strings.TrimSpace(tp.versionTag)): map[string]string{
			"settings": "blah",
		},
	}
	b, err := json.Marshal(&p)
	if err != nil {
		return nil, err
	}
	return b, nil
}

// replacePluginPathInEnv is useful for modifying the Env blob of a tester created with
// NewBootstrapTester().  We need to do that because the tester relies on BUILDKITE_PLUGINS_PATH,
// not on the .PluginsDir field as one might expect.
func replacePluginPathInEnv(originalEnv []string, pluginsDir string) (newEnv []string) {
	newEnv = []string{}
	for _, val := range originalEnv {
		if !strings.HasPrefix(val, "BUILDKITE_PLUGINS_PATH=") {
			newEnv = append(newEnv, val)
		}
	}
	newEnv = append(newEnv, "BUILDKITE_PLUGINS_PATH="+pluginsDir)
	return newEnv
}
<|endoftext|>


<commit_before>/*
Copyright 2015 Steven Labrum

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package CheckIt

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"net/http"
	"path/filepath"
	"strconv"
	"strings"
	"text/template"
)

var interfaces = []Box{}
var configuration = &Config{}

var (
	httpListen = flag.String("http", "127.0.0.1:3999", "host:port to listen on")
	htmlOutput = flag.Bool("html", false, "render program output as HTML")
)

func baseCase(w http.ResponseWriter, r *http.Request, page Page) {

	headTemp.Execute(w, nil)
	openBodyTemp.Execute(w, nil)
	pageStartTemp.Execute(w, page)

	boxes := initBoxes(interfaces)

	for key := range boxes {
		boxTemp.Execute(w, boxes[key])
	}

	pageCloseTemp.Execute(w, nil)
	htmlCloseTemp.Execute(w, nil)

}

/*  FrontPage is an HTTP handler that displays the basecase
unless a stored page is being loaded.
*/
func FrontPage(w http.ResponseWriter, r *http.Request) {
	title := r.URL.Path[len("/"):]

	page, _ := initConfig(configuration)

	boxes := initBoxes(interfaces)

	if len(title) < 1 {
		baseCase(w, r, page)
	} else {
		title := r.URL.Path[len("/"):]
		title = configuration.Path + "/" + title
		fmt.Println(title)
		pageNames, _ := filepath.Glob(title + "/*.config")
		boxNames, _ := filepath.Glob(title + "/*.box")

		fmt.Println(len(pageNames))
		fmt.Println("Loaded shared page")

		if pageNames == nil || boxNames == nil {
			http.Redirect(w, r, "/", http.StatusFound)
		} else {

			pageName := pageNames[0]
			fmt.Println(boxNames)

			headTemp.Execute(w, nil)
			openBodyTemp.Execute(w, nil)

			configuration = ReadConfig(pageName)

			initConfig(configuration)

			pageStartTemp.Execute(w, page)

			boxes = []*BoxStruct{}
			for key := range boxNames {
				boxP := ReadBox(boxNames[key])
				boxes = append(boxes, boxP)
				boxTemp.Execute(w, boxP)
			}

			pageCloseTemp.Execute(w, nil)
			htmlCloseTemp.Execute(w, nil)
		}
	}
}

func AboutPage(w http.ResponseWriter, r *http.Request) {
	var aboutPage = Page{Heading: "About",
		SubHeading: ""}

	_, about := initConfig(configuration)

	headTemp.Execute(w, nil)
	openBodyTemp.Execute(w, nil)
	pageStartTemp.Execute(w, aboutPage)
	aboutTemp.Execute(w, about)
	pageCloseTemp.Execute(w, nil)
	htmlCloseTemp.Execute(w, nil)

}

var outputText = `<pre>{{printf "%s" . |html}}</pre>`
var output = template.Must(template.New("output").Parse(outputText))
var shareText = `{{printf "%s" . |html}}`
var shareOutput = template.Must(template.New("shareOutput").Parse(shareText))

// Compile is an HTTP handler that reads Source code from the request,
// runs the program (returning any errors),
// and sends the program's output as the HTTP response.
func PipeCompile(w http.ResponseWriter, req *http.Request) {

	title := req.URL.Path[len("/pipeile/"):]

	boxes := initBoxes(interfaces)

	fmt.Println(title)
	str := strings.Split(title, "/")
	title = str[0]

	position, _ := strconv.Atoi(str[1])

	body := new(bytes.Buffer)

	if _, err := body.ReadFrom(req.Body); err != nil {
		return
	}

	fmt.Println(position)

	updateBody(boxes, title, body.String())

	out, err := InterfaceRun(interfaces[position-1], boxes, body.Bytes(), title)

	if err != nil {
		w.WriteHeader(404)
		output.Execute(w, out)
	} else if *htmlOutput {
		w.Write(out)
	} else {
		output.Execute(w, out)
	}
}

func sharHandler(w http.ResponseWriter, r *http.Request) {
	page, _ := initConfig(configuration)
	boxes := initBoxes(interfaces)
	out := Share(page, boxes)
	fmt.Println("PATH :/ " + configuration.Path)
	Save(configuration.Path+"/", out, boxes)
	shareOutput.Execute(w, out)
}

func initConfig(config *Config) (Page, AboutStruct) {
	var about = AboutStruct{}
	var page = Page{}

	configuration = config
	page.Heading = config.Heading
	page.SubHeading = config.SubHeading
	about.Text = config.About
	about.SecondaryText = config.AboutSide

	return page, about
}

func initBoxes(boxs []Box) (boxes []*BoxStruct) {

	for key := range boxs {

		var box = BoxStruct{}

		box.Id = strconv.Itoa(key)
		box.Position = strconv.Itoa(key + 1)
		box.Total = len(boxs)
		box.Lang = boxs[key].Syntax()
		box.Body = boxs[key].Default()
		box.Head = "Heading"
		box.SubHead = "Subhead"
		box.Text = boxs[key].Help()

		boxes = append(boxes, &box)

	}
	return boxes
}

func Serve(config *Config, boxs ...Box) (err error) {
	configuration = config
	interfaces = boxs

	fmt.Println("cool beans")
	http.HandleFunc("/share/", sharHandler)
	http.HandleFunc("/about", AboutPage)
	http.HandleFunc("/", FrontPage)
	http.HandleFunc("/compile/", PipeCompile)
	http.ListenAndServe(":"+configuration.Port, nil)

	err = errors.New("Server crashed")
	return err
}
<commit_msg>Fixed loading pages<commit_after>/*
Copyright 2015 Steven Labrum

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package CheckIt

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"net/http"
	"path/filepath"
	"strconv"
	"strings"
	"text/template"
)

var interfaces = []Box{}
var configuration = &Config{}

var (
	httpListen = flag.String("http", "127.0.0.1:3999", "host:port to listen on")
	htmlOutput = flag.Bool("html", false, "render program output as HTML")
)

func baseCase(w http.ResponseWriter, r *http.Request, page Page) {

	headTemp.Execute(w, nil)
	openBodyTemp.Execute(w, nil)
	pageStartTemp.Execute(w, page)

	boxes := initBoxes(interfaces)

	for key := range boxes {
		boxTemp.Execute(w, boxes[key])
	}

	pageCloseTemp.Execute(w, nil)
	htmlCloseTemp.Execute(w, nil)

}

/*  FrontPage is an HTTP handler that displays the basecase
unless a stored page is being loaded.
*/
func FrontPage(w http.ResponseWriter, r *http.Request) {
	title := r.URL.Path[len("/"):]

	page, _ := initConfig(configuration)

	if len(title) < 1 {
		baseCase(w, r, page)
	} else {
		title := r.URL.Path[len("/"):]
		title = configuration.Path + "/" + title
		fmt.Println(title)
		pageNames, _ := filepath.Glob(title + "/*.config")
		boxNames, _ := filepath.Glob(title + "/*.box")

		fmt.Println(len(pageNames))
		fmt.Println("Loaded shared page")

		if pageNames == nil || boxNames == nil {
			http.Redirect(w, r, "/", http.StatusFound)
		} else {

			pageName := pageNames[0]
			fmt.Println(boxNames)

			headTemp.Execute(w, nil)
			openBodyTemp.Execute(w, nil)

			configuration = ReadConfig(pageName)

			initConfig(configuration)

			pageStartTemp.Execute(w, page)

			var boxes = []*BoxStruct{}

			for key := range boxNames {
				boxP := ReadBox(boxNames[key])
				boxes = append(boxes, boxP)
				boxTemp.Execute(w, boxP)
			}

			pageCloseTemp.Execute(w, nil)
			htmlCloseTemp.Execute(w, nil)
		}
	}
}

func AboutPage(w http.ResponseWriter, r *http.Request) {
	var aboutPage = Page{Heading: "About",
		SubHeading: ""}

	_, about := initConfig(configuration)

	headTemp.Execute(w, nil)
	openBodyTemp.Execute(w, nil)
	pageStartTemp.Execute(w, aboutPage)
	aboutTemp.Execute(w, about)
	pageCloseTemp.Execute(w, nil)
	htmlCloseTemp.Execute(w, nil)

}

var outputText = `<pre>{{printf "%s" . |html}}</pre>`
var output = template.Must(template.New("output").Parse(outputText))
var shareText = `{{printf "%s" . |html}}`
var shareOutput = template.Must(template.New("shareOutput").Parse(shareText))

// Compile is an HTTP handler that reads Source code from the request,
// runs the program (returning any errors),
// and sends the program's output as the HTTP response.
func PipeCompile(w http.ResponseWriter, req *http.Request) {

	title := req.URL.Path[len("/pipeile/"):]

	boxes := initBoxes(interfaces)

	fmt.Println(title)
	str := strings.Split(title, "/")
	title = str[0]

	position, _ := strconv.Atoi(str[1])

	body := new(bytes.Buffer)

	if _, err := body.ReadFrom(req.Body); err != nil {
		return
	}

	fmt.Println(position)

	updateBody(boxes, title, body.String())

	out, err := InterfaceRun(interfaces[position-1], boxes, body.Bytes(), title)

	if err != nil {
		w.WriteHeader(404)
		output.Execute(w, out)
	} else if *htmlOutput {
		w.Write(out)
	} else {
		output.Execute(w, out)
	}
}

func sharHandler(w http.ResponseWriter, r *http.Request) {
	page, _ := initConfig(configuration)
	boxes := initBoxes(interfaces)
	out := Share(page, boxes)
	fmt.Println("PATH :/ " + configuration.Path)
	Save(configuration.Path+"/", out, boxes)
	shareOutput.Execute(w, out)
}

func initConfig(config *Config) (Page, AboutStruct) {
	var about = AboutStruct{}
	var page = Page{}

	configuration = config
	page.Heading = config.Heading
	page.SubHeading = config.SubHeading
	about.Text = config.About
	about.SecondaryText = config.AboutSide

	return page, about
}

func initBoxes(boxs []Box) (boxes []*BoxStruct) {

	for key := range boxs {

		var box = BoxStruct{}

		box.Id = strconv.Itoa(key)
		box.Position = strconv.Itoa(key + 1)
		box.Total = len(boxs)
		box.Lang = boxs[key].Syntax()
		box.Body = boxs[key].Default()
		box.Head = "Heading"
		box.SubHead = "Subhead"
		box.Text = boxs[key].Help()

		boxes = append(boxes, &box)

	}
	return boxes
}

func Serve(config *Config, boxs ...Box) (err error) {
	configuration = config
	interfaces = boxs

	fmt.Println("cool beans")
	http.HandleFunc("/share/", sharHandler)
	http.HandleFunc("/about", AboutPage)
	http.HandleFunc("/", FrontPage)
	http.HandleFunc("/compile/", PipeCompile)
	http.ListenAndServe(":"+configuration.Port, nil)

	err = errors.New("Server crashed")
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/unixpickle/gocube"
	"github.com/unixpickle/gocube/args"
	"os"
)

func main() {
	// Input the cube.
	a := args.NewArgs(flag.CommandLine)
	flag.Parse()
	scramble, err := a.Scramble()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	// Setup search variables.
	moves, _ := gocube.ParseMoves("R U L D R' U' L' D' R2 U2 L2 D2 F B F' B' " +
		"F2 B2")
	goal := gocube.EOLineGoal{}
	start := scramble.Edges
	fmt.Println("Generating heuristic...")
	pruner := gocube.MakeEOPruner(moves)

	for i := 0; i <= 9; i++ {
		fmt.Println("Exploring depth", i, "...")
		search := start.Search(goal, pruner, moves, i, 1)
		solution, ok := <-search.Solutions()
		if ok {
			fmt.Println("Found solution:", solution)
			search.Cancel()
			break
		}
	}
}
<commit_msg>eo solver uses new args<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/unixpickle/gocube"
	"github.com/unixpickle/gocube/args"
	"os"
)

func main() {
	// Input the cube.
	a := args.NewArgs(flag.CommandLine)
	flag.Parse()
	if err := a.Parse(); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	scramble := a.Scramble()

	// Setup search variables.
	moves := a.Moves()
	goal := gocube.EOLineGoal{}
	start := scramble.Edges
	fmt.Println("Generating heuristic...")
	pruner := gocube.MakeEOPruner(moves)

	for i := a.MinDepth(); i <= a.MaxDepth(); i++ {
		fmt.Println("Exploring depth", i, "...")
		search := start.Search(goal, pruner, moves, i, a.Branch())
		for {
			solution, ok := <-search.Solutions()
			if !ok {
				break
			}
			fmt.Println("Found solution:", solution)
			if !a.Multiple() {
				search.Cancel()
				return
			}
		}
	}
}
<|endoftext|>


<commit_before>package etcd

import (
	"fmt"
	"testing"
	"time"
)

func TestGet(t *testing.T) {
	Set("foo", "bar", 100)

	// wait for commit
	time.Sleep(100 * time.Millisecond)

	results, err := Get("foo")

	if err != nil || results[0].Key != "/foo" || results[0].Value != "bar" {
		if err != nil {
			t.Fatal(err)
		}
		t.Fatalf("Get failed with %s %s %v", results[0].Key, results[0].Value, results[0].TTL)
	}

	results, err = Get("goo")

	if err == nil {
		t.Fatalf("should not be able to get non-exist key")
	}

	results, err = GetFrom("foo", "0.0.0.0:4001")

	if err != nil || results[0].Key != "/foo" || results[0].Value != "bar" {
		if err != nil {
			t.Fatal(err)
		}
		t.Fatalf("Get failed with %s %s %v", results[0].Key, results[0].Value, results[0].TTL)
	}

	results, err = GetFrom("foo", "0.0.0.0:4009")

	if err == nil {
		t.Fatal("should not get from port 4009")
	}
}
<commit_msg>gofmt<commit_after>package etcd

import (
	"testing"
	"time"
)

func TestGet(t *testing.T) {
	Set("foo", "bar", 100)

	// wait for commit
	time.Sleep(100 * time.Millisecond)

	results, err := Get("foo")

	if err != nil || results[0].Key != "/foo" || results[0].Value != "bar" {
		if err != nil {
			t.Fatal(err)
		}
		t.Fatalf("Get failed with %s %s %v", results[0].Key, results[0].Value, results[0].TTL)
	}

	results, err = Get("goo")

	if err == nil {
		t.Fatalf("should not be able to get non-exist key")
	}

	results, err = GetFrom("foo", "0.0.0.0:4001")

	if err != nil || results[0].Key != "/foo" || results[0].Value != "bar" {
		if err != nil {
			t.Fatal(err)
		}
		t.Fatalf("Get failed with %s %s %v", results[0].Key, results[0].Value, results[0].TTL)
	}

	results, err = GetFrom("foo", "0.0.0.0:4009")

	if err == nil {
		t.Fatal("should not get from port 4009")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"

	"github.com/kyokomi/emoji"
)

func main() {
	fmt.Println("Hello Wolrd Emoji!")

	emoji.Println(":beer: Beer!!!")

	pizzaMessage := emoji.Sprint("I like a :pizza: and :sushi:!!")
	fmt.Println(pizzaMessage)
}
<commit_msg>Fix typo in example/hello.go<commit_after>package main

import (
	"fmt"

	"github.com/kyokomi/emoji"
)

func main() {
	fmt.Println("Hello World Emoji!")

	emoji.Println(":beer: Beer!!!")

	pizzaMessage := emoji.Sprint("I like a :pizza: and :sushi:!!")
	fmt.Println(pizzaMessage)
}
<|endoftext|>