<commit_before>package main

import (
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/docker/libswarm/backends"
	"github.com/docker/libswarm/beam"
	_ "github.com/dotcloud/docker/api/server"
	"github.com/flynn/go-shlex"
	"io"
	"log"
	"os"
	"strings"
)

func main() {
	app := cli.NewApp()
	app.Name = "swarmd"
	app.Usage = "Compose distributed systems from lightweight services"
	app.Version = "0.0.1"
	app.Flags = []cli.Flag{
	}
	app.Action = cmdDaemon
	app.Run(os.Args)
}

func cmdDaemon(c *cli.Context) {
	app := beam.NewServer()
	app.OnLog(beam.Handler(func(msg *beam.Message) error {
		log.Printf("%s\n", strings.Join(msg.Args, " "))
		return nil
	}))
	app.OnError(beam.Handler(func(msg *beam.Message) error {
		Fatalf("Fatal: %v", strings.Join(msg.Args[:1], ""))
		return nil
	}))
	back := backends.New()
	if len(c.Args()) == 0 {
		names, err := back.Ls()
		if err != nil {
			Fatalf("ls: %v", err)
		}
		fmt.Println(strings.Join(names, "\n"))
		return
	}
	bName, bArgs, err := parseCmd(c.Args()[0])
	if err != nil {
		Fatalf("parse: %v", err)
	}
	fmt.Printf("---> Loading backend '%s'\n", strings.Join(append([]string{bName}, bArgs...), " "))
	_, backend, err := back.Attach(bName)
	if err != nil {
		Fatalf("%s: %v\n", bName, err)
	}
	fmt.Printf("---> Spawning\n")
	instance, err := backend.Spawn(bArgs...)
	if err != nil {
		Fatalf("spawn %s: %v\n", bName, err)
	}
	fmt.Printf("---> Attaching\n")
	instanceIn, _, err := instance.Attach("")
	if err != nil {
		Fatalf("attach: %v", err)
	}
	fmt.Printf("---> Starting\n")
	if err := instance.Start(); err != nil {
		Fatalf("start: %v", err)
	}
	_, err = beam.Copy(app, instanceIn)
	if err != nil {
		Fatalf("copy: %v", err)
	}
}

func parseCmd(txt string) (string, []string, error) {
	l, err := shlex.NewLexer(strings.NewReader(txt))
	if err != nil {
		return "", nil, err
	}
	var cmd []string
	for {
		word, err := l.NextWord()
		if err == io.EOF {
			break
		}
		if err != nil {
			return "", nil, err
		}
		cmd = append(cmd, word)
	}
	if len(cmd) == 0 {
		return "", nil, fmt.Errorf("parse error: empty command")
	}
	return cmd[0], cmd[1:], nil
}

func Fatalf(msg string, args ...interface{}) {
	if !strings.HasSuffix(msg, "\n") {
		msg = msg + "\n"
	}
	fmt.Fprintf(os.Stderr, msg, args...)
	os.Exit(1)
}
<commit_msg>Add support for multiple backends piped together<commit_after>package main

import (
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/docker/libswarm/backends"
	"github.com/docker/libswarm/beam"
	_ "github.com/dotcloud/docker/api/server"
	"github.com/flynn/go-shlex"
	"io"
	"log"
	"os"
	"strings"
)

func main() {
	app := cli.NewApp()
	app.Name = "swarmd"
	app.Usage = "Compose distributed systems from lightweight services"
	app.Version = "0.0.1"
	app.Flags = []cli.Flag{
	}
	app.Action = cmdDaemon
	app.Run(os.Args)
}

func cmdDaemon(c *cli.Context) {
	app := beam.NewServer()
	app.OnLog(beam.Handler(func(msg *beam.Message) error {
		log.Printf("%s\n", strings.Join(msg.Args, " "))
		return nil
	}))
	app.OnError(beam.Handler(func(msg *beam.Message) error {
		Fatalf("Fatal: %v", strings.Join(msg.Args[:1], ""))
		return nil
	}))
	back := backends.New()
	if len(c.Args()) == 0 {
		names, err := back.Ls()
		if err != nil {
			Fatalf("ls: %v", err)
		}
		fmt.Println(strings.Join(names, "\n"))
		return
	}
	var previousInstanceIn beam.Receiver
	for _, backendArg := range c.Args() {
		bName, bArgs, err := parseCmd(backendArg)
		if err != nil {
			Fatalf("parse: %v", err)
		}
		fmt.Printf("---> Loading backend '%s'\n", strings.Join(append([]string{bName}, bArgs...), " "))
		_, backend, err := back.Attach(bName)
		if err != nil {
			Fatalf("%s: %v\n", bName, err)
		}
		fmt.Printf("---> Spawning\n")
		instance, err := backend.Spawn(bArgs...)
		if err != nil {
			Fatalf("spawn %s: %v\n", bName, err)
		}
		fmt.Printf("---> Attaching\n")
		instanceIn, instanceOut, err := instance.Attach("")
		if err != nil {
			Fatalf("attach: %v", err)
		}
		fmt.Printf("---> Starting\n")
		if err := instance.Start(); err != nil {
			Fatalf("start: %v", err)
		}
		if previousInstanceIn != nil {
			go beam.Copy(instanceOut, previousInstanceIn)
		}
		previousInstanceIn = instanceIn
	}
	_, err := beam.Copy(app, previousInstanceIn)
	if err != nil {
		Fatalf("copy: %v", err)
	}
}

func parseCmd(txt string) (string, []string, error) {
	l, err := shlex.NewLexer(strings.NewReader(txt))
	if err != nil {
		return "", nil, err
	}
	var cmd []string
	for {
		word, err := l.NextWord()
		if err == io.EOF {
			break
		}
		if err != nil {
			return "", nil, err
		}
		cmd = append(cmd, word)
	}
	if len(cmd) == 0 {
		return "", nil, fmt.Errorf("parse error: empty command")
	}
	return cmd[0], cmd[1:], nil
}

func Fatalf(msg string, args ...interface{}) {
	if !strings.HasSuffix(msg, "\n") {
		msg = msg + "\n"
	}
	fmt.Fprintf(os.Stderr, msg, args...)
	os.Exit(1)
}
<|endoftext|>


<commit_before>package stretcher

import (
	"log"
	"os/exec"
)

type SyncStrategy interface {
	Sync(from, to string) error
}

type RsyncStrategy struct {
	*Manifest
}

var RsyncDefaultOpts = []string{"-av", "--delete"}

func (s *RsyncStrategy) Sync(from, to string) error {
	m := s.Manifest

	args := []string{}
	args = append(args, RsyncDefaultOpts...)
	if m.ExcludeFrom != "" {
		args = append(args, "--exclude-from", from+m.ExcludeFrom)
	}
	if len(m.Excludes) > 0 {
		for _, ex := range m.Excludes {
			args = append(args, "--exclude", ex)
		}
	}
	args = append(args, from, to)

	log.Println("rsync", args)
	out, err := exec.Command("rsync", args...).CombinedOutput()
	if len(out) > 0 {
		log.Println(string(out))
	}
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>Add MvSyncStrategy<commit_after>package stretcher

import (
	"log"
	"os"
	"os/exec"
)

type SyncStrategy interface {
	Sync(from, to string) error
}

type RsyncStrategy struct {
	*Manifest
}

var RsyncDefaultOpts = []string{"-av", "--delete"}

func (s *RsyncStrategy) Sync(from, to string) error {
	m := s.Manifest

	args := []string{}
	args = append(args, RsyncDefaultOpts...)
	if m.ExcludeFrom != "" {
		args = append(args, "--exclude-from", from+m.ExcludeFrom)
	}
	if len(m.Excludes) > 0 {
		for _, ex := range m.Excludes {
			args = append(args, "--exclude", ex)
		}
	}
	args = append(args, from, to)

	log.Println("rsync", args)
	out, err := exec.Command("rsync", args...).CombinedOutput()
	if len(out) > 0 {
		log.Println(string(out))
	}
	if err != nil {
		return err
	}

	return nil
}

type MvSyncStrategy struct {
}

func (s *MvSyncStrategy) Sync(from, to string) error {
	log.Printf("Rename srcdir %s to dest %s\n", from, to)
	return os.Rename(from, to)
}
<|endoftext|>


<commit_before>package main

import (
	// "fmt"
	"os"
	"strings"
	"testing"
)

func TestGeneral(t *testing.T) {
	defer os.RemoveAll("data")
	p := Open("testpage")
	err := p.Update("**bold**")
	if err != nil {
		t.Error(err)
	}
	if strings.TrimSpace(p.RenderedPage) != "
bold
" {
		t.Errorf("Did not render: '%s'", p.RenderedPage)
	}
	err = p.Update("**bold** and *italic*")
	if err != nil {
		t.Error(err)
	}
	p.Save()

	p2 := Open("testpage")
	if strings.TrimSpace(p2.RenderedPage) != "
bold and italic
" {
		t.Errorf("Did not render: '%s'", p2.RenderedPage)
	}

}
<commit_msg>Create special folder for tests<commit_after>package main

import (
	// "fmt"
	"os"
	"strings"
	"testing"
)

func TestGeneral(t *testing.T) {
	pathToData = "testdata"
	os.MkdirAll(pathToData, 0755)
	defer os.RemoveAll(pathToData)
	p := Open("testpage")
	err := p.Update("**bold**")
	if err != nil {
		t.Error(err)
	}
	if strings.TrimSpace(p.RenderedPage) != "
bold
" {
		t.Errorf("Did not render: '%s'", p.RenderedPage)
	}
	err = p.Update("**bold** and *italic*")
	if err != nil {
		t.Error(err)
	}
	p.Save()

	p2 := Open("testpage")
	if strings.TrimSpace(p2.RenderedPage) != "
bold and italic
" {
		t.Errorf("Did not render: '%s'", p2.RenderedPage)
	}

}
<|endoftext|>


<commit_before>package command

import (
	"errors"
	"time"
	//log "github.com/funkygao/log4go"
)

var (
	ErrUnsupMetricsType      = errors.New("unsupported metrics type")
	ErrUnsupGaugeMetrics     = errors.New("unsupported gauge metrics")
	ErrUnsupMeterMetrics     = errors.New("unsupported meter metrics")
	ErrMetricsColumnNotFound = errors.New("metrics column not found")
)

type haproxyMetrics struct {
	ctx *Start

	interval time.Duration
	uri      string

	colMapSetted    bool
	colMap          map[string]int           //rate:33, scur:4, ...
	proxyMetricsMap map[string]*proxyMetrics //pub.frontend:, pub.backend:, ...

}

type proxyMetrics struct {
	proxyName   string                     //pub, sub
	proxyType   string                     //FRONTEND, BACKEND
	metricsDefs []metricsDefine            //session.rate, session.pctg, ...
	metricsMap  map[string]*metricsWrapper //session.rate:gauge, ..
}

type metricsDefine struct {
	metricsName string //session.rate, session.pctg,...
	metricsType string //Gauge, Meter, ...
}

type metricsWrapper struct {
	meterBase        int64
	metricsCollector interface{} //Gauge, Meter, ...
}

func (this *haproxyMetrics) start() {
	if this.ctx == nil {
		panic("nil ctx not allowed")
	}

	tick := time.NewTicker(this.interval)
	defer tick.Stop()

	//init haproxy metrics settings
	err := this.init()
	if err != nil {
		log.Error("haproxyMetrics init, %v", err)
		return
	}
	log.Info("haproxyMetrics init succ")

	for {
		select {
		case <-this.ctx.quitCh:
			return

		case <-tick.C:
			this.reportStats()
		}
	}
}

func (this *haproxyMetrics) reportStats() {

}
<commit_msg>fix merge conflict bug<commit_after>package command

import (
	"errors"
	"time"
	//log "github.com/funkygao/log4go"
)

var (
	ErrUnsupMetricsType      = errors.New("unsupported metrics type")
	ErrUnsupGaugeMetrics     = errors.New("unsupported gauge metrics")
	ErrUnsupMeterMetrics     = errors.New("unsupported meter metrics")
	ErrMetricsColumnNotFound = errors.New("metrics column not found")
)

type haproxyMetrics struct {
	ctx *Start

	interval time.Duration
	uri      string

	colMapSetted    bool
	colMap          map[string]int           //rate:33, scur:4, ...
	proxyMetricsMap map[string]*proxyMetrics //pub.frontend:, pub.backend:, ...

}

type proxyMetrics struct {
	proxyName   string                     //pub, sub
	proxyType   string                     //FRONTEND, BACKEND
	metricsDefs []metricsDefine            //session.rate, session.pctg, ...
	metricsMap  map[string]*metricsWrapper //session.rate:gauge, ..
}

type metricsDefine struct {
	metricsName string //session.rate, session.pctg,...
	metricsType string //Gauge, Meter, ...
}

type metricsWrapper struct {
	meterBase        int64
	metricsCollector interface{} //Gauge, Meter, ...
}

func (this *haproxyMetrics) start() {
	if this.ctx == nil {
		panic("nil ctx not allowed")
	}

	tick := time.NewTicker(this.interval)
	defer tick.Stop()

	//init haproxy metrics settings
	// err := this.init()
	// if err != nil {
	// 	log.Error("haproxyMetrics init, %v", err)
	// 	return
	// }
	// log.Info("haproxyMetrics init succ")

	for {
		select {
		case <-this.ctx.quitCh:
			return

		case <-tick.C:
			this.reportStats()
		}
	}
}

func (this *haproxyMetrics) reportStats() {

}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"math/rand"
	"os"
	"time"

	"github.com/spf13/pflag"

	cliflag "k8s.io/component-base/cli/flag"
	"k8s.io/component-base/logs"
	_ "k8s.io/component-base/metrics/prometheus/clientgo"
	_ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration
	"k8s.io/kubernetes/cmd/kube-scheduler/app"
)

func main() {
	rand.Seed(time.Now().UnixNano())

	pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc)

	command := app.NewSchedulerCommand()

	logs.InitLogs()
	defer logs.FlushLogs()

	if err := command.Execute(); err != nil {
		os.Exit(1)
	}
}
<commit_msg>JSON log format registration for kube-scheduler<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"math/rand"
	"os"
	"time"

	"github.com/spf13/pflag"

	cliflag "k8s.io/component-base/cli/flag"
	"k8s.io/component-base/logs"
	_ "k8s.io/component-base/logs/json/register" // for JSON log format registration
	_ "k8s.io/component-base/metrics/prometheus/clientgo"
	_ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration
	"k8s.io/kubernetes/cmd/kube-scheduler/app"
)

func main() {
	rand.Seed(time.Now().UnixNano())

	pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc)

	command := app.NewSchedulerCommand()

	logs.InitLogs()
	defer logs.FlushLogs()

	if err := command.Execute(); err != nil {
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package gopkg

import (
	"bytes"
	"fmt"
	"go/importer"
	"go/types"
	"io"
	"io/ioutil"
	"os"
	"testing"
)

// -----------------------------------------------------------------------------

type goFunc struct {
	Name string
	This interface{}
}

const expected = `// Package strings provide Go+ "strings" package, as "strings" package in Go.
package strings

import (
	gop "github.com/qiniu/goplus/gop"
	strings "strings"
)

func execNewReplacer(arity int, p *gop.Context) {
	args := p.GetArgs(arity)
	ret0 := strings.NewReplacer(gop.ToStrings(args)...)
	p.Ret(arity, ret0)
}

func execReplacerReplace(_ int, p *gop.Context) {
	args := p.GetArgs(2)
	ret0 := args[0].(*strings.Replacer).Replace(args[1].(string))
	p.Ret(2, ret0)
}

// I is a Go package instance.
var I = gop.NewGoPackage("strings")

func init() {
	I.RegisterFuncs(
		I.Func("(*Replacer).Replace", (*strings.Replacer).Replace, execReplacerReplace),
	)
	I.RegisterFuncvs(
		I.Funcv("NewReplacer", strings.NewReplacer, execNewReplacer),
	)
}
`

func getMethod(o *types.Named, name string) *types.Func {
	n := o.NumMethods()
	for i := 0; i < n; i++ {
		m := o.Method(i)
		if m.Name() == name {
			return m
		}
	}
	return nil
}

func TestBasic(t *testing.T) {
	pkg, err := importer.Default().Import("strings")
	if err != nil {
		t.Fatal("Import failed:", err)
	}
	gbl := pkg.Scope()
	newReplacer := gbl.Lookup("NewReplacer").(*types.Func)
	replacer := gbl.Lookup("Replacer").(*types.TypeName).Type().(*types.Named)
	b := bytes.NewBuffer(nil)
	e := NewExporter(b, pkg)
	e.ExportFunc(newReplacer)
	e.ExportFunc(getMethod(replacer, "Replace"))
	e.Close()
	if real := b.String(); real != expected {
		fmt.Println(real)
		t.Fatal("Test failed")
	}
}

// -----------------------------------------------------------------------------

type nopCloser struct {
	io.Writer
}

func (nopCloser) Close() error { return nil }

func nilExportFile(pkgDir string) (f io.WriteCloser, err error) {
	return &nopCloser{ioutil.Discard}, nil
}

func stdoutExportFile(pkgDir string) (f io.WriteCloser, err error) {
	return &nopCloser{os.Stdout}, nil
}

func TestExportStrings(t *testing.T) {
	err := Export("strings", nilExportFile)
	if err != nil {
		t.Fatal("TestExport failed:", err)
	}
}

func TestExportStrconv(t *testing.T) {
	err := Export("strconv", nilExportFile)
	if err != nil {
		t.Fatal("TestExport failed:", err)
	}
}

func TestExportReflect(t *testing.T) {
	err := Export("reflect", nilExportFile)
	if err != nil {
		t.Fatal("TestExport failed:", err)
	}
}

func TestExportSyncAtomic(t *testing.T) {
	err := Export("sync/atomic", nilExportFile)
	if err != nil {
		t.Fatal("TestExport failed:", err)
	}
}

// -----------------------------------------------------------------------------
<commit_msg>TestExportGoTypes: todo<commit_after>package gopkg

import (
	"bytes"
	"fmt"
	"go/importer"
	"go/types"
	"io"
	"io/ioutil"
	"os"
	"testing"
)

// -----------------------------------------------------------------------------

type goFunc struct {
	Name string
	This interface{}
}

const expected = `// Package strings provide Go+ "strings" package, as "strings" package in Go.
package strings

import (
	gop "github.com/qiniu/goplus/gop"
	strings "strings"
)

func execNewReplacer(arity int, p *gop.Context) {
	args := p.GetArgs(arity)
	ret0 := strings.NewReplacer(gop.ToStrings(args)...)
	p.Ret(arity, ret0)
}

func execReplacerReplace(_ int, p *gop.Context) {
	args := p.GetArgs(2)
	ret0 := args[0].(*strings.Replacer).Replace(args[1].(string))
	p.Ret(2, ret0)
}

// I is a Go package instance.
var I = gop.NewGoPackage("strings")

func init() {
	I.RegisterFuncs(
		I.Func("(*Replacer).Replace", (*strings.Replacer).Replace, execReplacerReplace),
	)
	I.RegisterFuncvs(
		I.Funcv("NewReplacer", strings.NewReplacer, execNewReplacer),
	)
}
`

func getMethod(o *types.Named, name string) *types.Func {
	n := o.NumMethods()
	for i := 0; i < n; i++ {
		m := o.Method(i)
		if m.Name() == name {
			return m
		}
	}
	return nil
}

func TestBasic(t *testing.T) {
	pkg, err := importer.Default().Import("strings")
	if err != nil {
		t.Fatal("Import failed:", err)
	}
	gbl := pkg.Scope()
	newReplacer := gbl.Lookup("NewReplacer").(*types.Func)
	replacer := gbl.Lookup("Replacer").(*types.TypeName).Type().(*types.Named)
	b := bytes.NewBuffer(nil)
	e := NewExporter(b, pkg)
	e.ExportFunc(newReplacer)
	e.ExportFunc(getMethod(replacer, "Replace"))
	e.Close()
	if real := b.String(); real != expected {
		fmt.Println(real)
		t.Fatal("Test failed")
	}
}

// -----------------------------------------------------------------------------

type nopCloser struct {
	io.Writer
}

func (nopCloser) Close() error { return nil }

func nilExportFile(pkgDir string) (f io.WriteCloser, err error) {
	return &nopCloser{ioutil.Discard}, nil
}

func stdoutExportFile(pkgDir string) (f io.WriteCloser, err error) {
	return &nopCloser{os.Stdout}, nil
}

func TestExportStrings(t *testing.T) {
	err := Export("strings", nilExportFile)
	if err != nil {
		t.Fatal("TestExport failed:", err)
	}
}

func TestExportStrconv(t *testing.T) {
	err := Export("strconv", nilExportFile)
	if err != nil {
		t.Fatal("TestExport failed:", err)
	}
}

func TestExportReflect(t *testing.T) {
	err := Export("reflect", nilExportFile)
	if err != nil {
		t.Fatal("TestExport failed:", err)
	}
}

func TestExportGoTypes(t *testing.T) {
	err := Export("go/types", stdoutExportFile)
	if err != nil {
		t.Fatal("TestExport failed:", err)
	}
}

// -----------------------------------------------------------------------------
<|endoftext|>


<commit_before>package cluster

import (
	"errors"
	"fmt"
	"os"
	"text/tabwriter"

	"github.com/docker/swarmkit/api"
	"github.com/docker/swarmkit/cmd/swarmctl/common"
	"github.com/docker/swarmkit/protobuf/ptypes"
	"github.com/spf13/cobra"
)

func printClusterSummary(cluster *api.Cluster) {
	w := tabwriter.NewWriter(os.Stdout, 8, 8, 8, ' ', 0)
	defer w.Flush()

	common.FprintfIfNotEmpty(w, "ID\t: %s\n", cluster.ID)
	common.FprintfIfNotEmpty(w, "Name\t: %s\n", cluster.Spec.Annotations.Name)
	if len(cluster.Spec.AcceptancePolicy.Policies) > 0 {
		fmt.Fprintf(w, "Acceptance Policies:\n")
		for _, policy := range cluster.Spec.AcceptancePolicy.Policies {
			fmt.Fprintf(w, "  Role\t: %v\n", policy.Role)
			fmt.Fprintf(w, "    Autoaccept\t: %v\n", policy.Autoaccept)
			if policy.Secret != nil {
				fmt.Fprintf(w, "    Secret\t: %v\n", string(policy.Secret.Data))
			}
		}
	}
	fmt.Fprintf(w, "Orchestration settings:\n")
	fmt.Fprintf(w, "  Task history entries: %d\n", cluster.Spec.Orchestration.TaskHistoryRetentionLimit)

	heartbeatPeriod, err := ptypes.Duration(cluster.Spec.Dispatcher.HeartbeatPeriod)
	if err == nil {
		fmt.Fprintf(w, "Dispatcher settings:\n")
		fmt.Fprintf(w, "  Dispatcher heartbeat period: %s\n", heartbeatPeriod.String())
	}

	fmt.Fprintf(w, "Certificate Authority settings:\n")
	if cluster.Spec.CAConfig.NodeCertExpiry != nil {
		clusterDuration, err := ptypes.Duration(cluster.Spec.CAConfig.NodeCertExpiry)
		if err != nil {
			fmt.Fprintf(w, "  Certificate Validity Duration: [ERROR PARSING DURATION]\n")
		} else {
			fmt.Fprintf(w, "  Certificate Validity Duration: %s\n", clusterDuration.String())
		}
	}
	if len(cluster.Spec.CAConfig.ExternalCAs) > 0 {
		fmt.Fprintf(w, "  External CAs:\n")
		for _, ca := range cluster.Spec.CAConfig.ExternalCAs {
			fmt.Fprintf(w, "    %s: %s\n", ca.Protocol, ca.URL)
		}
	}
}

var (
	inspectCmd = &cobra.Command{
		Use:   "inspect <cluster name>",
		Short: "Inspect a cluster",
		RunE: func(cmd *cobra.Command, args []string) error {
			if len(args) == 0 {
				return errors.New("cluster name missing")
			}

			c, err := common.Dial(cmd)
			if err != nil {
				return err
			}

			cluster, err := getCluster(common.Context(cmd), c, args[0])
			if err != nil {
				return err
			}

			printClusterSummary(cluster)

			return nil
		},
	}
)
<commit_msg>swarmctl: Don't print out hashed secret<commit_after>package cluster

import (
	"errors"
	"fmt"
	"os"
	"text/tabwriter"

	"github.com/docker/swarmkit/api"
	"github.com/docker/swarmkit/cmd/swarmctl/common"
	"github.com/docker/swarmkit/protobuf/ptypes"
	"github.com/spf13/cobra"
)

func printClusterSummary(cluster *api.Cluster) {
	w := tabwriter.NewWriter(os.Stdout, 8, 8, 8, ' ', 0)
	defer w.Flush()

	common.FprintfIfNotEmpty(w, "ID\t: %s\n", cluster.ID)
	common.FprintfIfNotEmpty(w, "Name\t: %s\n", cluster.Spec.Annotations.Name)
	if len(cluster.Spec.AcceptancePolicy.Policies) > 0 {
		fmt.Fprintf(w, "Acceptance Policies:\n")
		for _, policy := range cluster.Spec.AcceptancePolicy.Policies {
			fmt.Fprintf(w, "  Role\t: %v\n", policy.Role)
			fmt.Fprintf(w, "    Autoaccept\t: %v\n", policy.Autoaccept)
			if policy.Secret != nil {
				fmt.Fprintln(w, "    Secret\t: yes")
			} else {
				fmt.Fprintln(w, "    Secret\t: no")
			}
		}
	}
	fmt.Fprintf(w, "Orchestration settings:\n")
	fmt.Fprintf(w, "  Task history entries: %d\n", cluster.Spec.Orchestration.TaskHistoryRetentionLimit)

	heartbeatPeriod, err := ptypes.Duration(cluster.Spec.Dispatcher.HeartbeatPeriod)
	if err == nil {
		fmt.Fprintf(w, "Dispatcher settings:\n")
		fmt.Fprintf(w, "  Dispatcher heartbeat period: %s\n", heartbeatPeriod.String())
	}

	fmt.Fprintf(w, "Certificate Authority settings:\n")
	if cluster.Spec.CAConfig.NodeCertExpiry != nil {
		clusterDuration, err := ptypes.Duration(cluster.Spec.CAConfig.NodeCertExpiry)
		if err != nil {
			fmt.Fprintf(w, "  Certificate Validity Duration: [ERROR PARSING DURATION]\n")
		} else {
			fmt.Fprintf(w, "  Certificate Validity Duration: %s\n", clusterDuration.String())
		}
	}
	if len(cluster.Spec.CAConfig.ExternalCAs) > 0 {
		fmt.Fprintf(w, "  External CAs:\n")
		for _, ca := range cluster.Spec.CAConfig.ExternalCAs {
			fmt.Fprintf(w, "    %s: %s\n", ca.Protocol, ca.URL)
		}
	}
}

var (
	inspectCmd = &cobra.Command{
		Use:   "inspect <cluster name>",
		Short: "Inspect a cluster",
		RunE: func(cmd *cobra.Command, args []string) error {
			if len(args) == 0 {
				return errors.New("cluster name missing")
			}

			c, err := common.Dial(cmd)
			if err != nil {
				return err
			}

			cluster, err := getCluster(common.Context(cmd), c, args[0])
			if err != nil {
				return err
			}

			printClusterSummary(cluster)

			return nil
		},
	}
)
<|endoftext|>


<commit_before>/* Callisto - Yet another Solar System simulator
 *
 * Copyright (c) 2016, Valerian Saliou <valerian@valeriansaliou.name>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 *   * Redistributions of source code must retain the above copyright notice,
 *     this list of conditions and the following disclaimer.
 *   * Redistributions in binary form must reproduce the above copyright
 *     notice, this list of conditions and the following disclaimer in the
 *     documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

package main

import (
  "math"
  "github.com/go-gl/mathgl/mgl32"
)

// Math
const (
  ConfigMathDegreeToRadian float64 = math.Pi / 180
)

// Time
const (
  ConfigTimeSecondToMilliseconds int = 1000
  ConfigTimeMinuteToMilliseconds int = 60 * ConfigTimeSecondToMilliseconds
  ConfigTimeHourToMilliseconds int = 60 * ConfigTimeMinuteToMilliseconds
  ConfigTimeDayToMilliseconds int = 24 * ConfigTimeHourToMilliseconds
  ConfigTimeYearToMilliseconds int = 365 * ConfigTimeDayToMilliseconds

  ConfigTimeStartFromMilliseconds int = ConfigTimeYearToMilliseconds

  ConfigTimeNormalizeFactor float32 = 64.0
)

// Window
const (
  ConfigWindowTitle string = "Callisto - Solar System Simulator"
  ConfigWindowFullScreen bool = false
)

// Speed
const (
  ConfigSpeedFramerateDefault float64 = 60
  ConfigSpeedFramerateFactor float64 = 1.5
)

// Projection
var (
  ConfigProjectionFieldNear float32 = 0.1
  ConfigProjectionFieldFar float32 = 9999999999999999999.0
)

// Camera
var (
  ConfigCameraDefaultEye = mgl32.Vec3{1600, -1800, -5657}
  ConfigCameraDefaultTarget = mgl32.Vec3{0.255, 0.650, 0.000}

  ConfigCameraMoveCelerityCruise = 5.0
  ConfigCameraMoveCelerityTurbo = 50.0

  ConfigCameraInertiaProduceForward = 0.05
  ConfigCameraInertiaProduceBackward = -0.05
  ConfigCameraInertiaConsumeForward = -0.04
  ConfigCameraInertiaConsumeBackward = 0.04

  ConfigCameraTargetAmortizeFactor float32 = 0.0025
)

// Object
const (
  ConfigObjectTexturePhiMax int = 90
  ConfigObjectTextureThetaMax int = 360
  ConfigObjectTextureStepLatitude int = 3
  ConfigObjectTextureStepLongitude int = 6

  ConfigObjectFullAngle float64 = 4.0 * math.Pi

  ConfigObjectFactorSize float64 = 0.25
  ConfigObjectFactorSpeedScene float64 = 1.0
  ConfigObjectFactorSpeedMaximum float64 = 200000.0
  ConfigObjectFactorSpeedChangeFactor float64 = 100.0
)
<commit_msg>Adjust factors<commit_after>/* Callisto - Yet another Solar System simulator
 *
 * Copyright (c) 2016, Valerian Saliou <valerian@valeriansaliou.name>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 *   * Redistributions of source code must retain the above copyright notice,
 *     this list of conditions and the following disclaimer.
 *   * Redistributions in binary form must reproduce the above copyright
 *     notice, this list of conditions and the following disclaimer in the
 *     documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

package main

import (
  "math"
  "github.com/go-gl/mathgl/mgl32"
)

// Math
const (
  ConfigMathDegreeToRadian float64 = math.Pi / 180
)

// Time
const (
  ConfigTimeSecondToMilliseconds int = 1000
  ConfigTimeMinuteToMilliseconds int = 60 * ConfigTimeSecondToMilliseconds
  ConfigTimeHourToMilliseconds int = 60 * ConfigTimeMinuteToMilliseconds
  ConfigTimeDayToMilliseconds int = 24 * ConfigTimeHourToMilliseconds
  ConfigTimeYearToMilliseconds int = 365 * ConfigTimeDayToMilliseconds

  ConfigTimeStartFromMilliseconds int = ConfigTimeYearToMilliseconds

  ConfigTimeNormalizeFactor float32 = 64.0
)

// Window
const (
  ConfigWindowTitle string = "Callisto - Solar System Simulator"
  ConfigWindowFullScreen bool = false
)

// Speed
const (
  ConfigSpeedFramerateDefault float64 = 60
  ConfigSpeedFramerateFactor float64 = 1.5
)

// Projection
var (
  ConfigProjectionFieldNear float32 = 0.1
  ConfigProjectionFieldFar float32 = 9999999999999999999.0
)

// Camera
var (
  ConfigCameraDefaultEye = mgl32.Vec3{300, -350, -800}
  ConfigCameraDefaultTarget = mgl32.Vec3{0.255, 0.650, 0.000}

  ConfigCameraMoveCelerityCruise = 1.0
  ConfigCameraMoveCelerityTurbo = 10.0

  ConfigCameraInertiaProduceForward = 0.0075
  ConfigCameraInertiaProduceBackward = -0.0075
  ConfigCameraInertiaConsumeForward = -0.005
  ConfigCameraInertiaConsumeBackward = 0.005

  ConfigCameraTargetAmortizeFactor float32 = 0.002
)

// Object
const (
  ConfigObjectTexturePhiMax int = 90
  ConfigObjectTextureThetaMax int = 360
  ConfigObjectTextureStepLatitude int = 3
  ConfigObjectTextureStepLongitude int = 6

  ConfigObjectFullAngle float64 = 4.0 * math.Pi

  ConfigObjectFactorSize float64 = 0.05
  ConfigObjectFactorSpeedScene float64 = 1.0
  ConfigObjectFactorSpeedMaximum float64 = 200000.0
  ConfigObjectFactorSpeedChangeFactor float64 = 100.0
)
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"sort"
	"strings"

	"github.com/hashicorp/packer/command"
	"github.com/hashicorp/packer/packer"
	"github.com/hashicorp/packer/packer/plugin"
	"github.com/kardianos/osext"
)

// PACKERSPACE is used to represent the spaces that separate args for a command
// without being confused with spaces in the path to the command itself.
const PACKERSPACE = "-PACKERSPACE-"

type config struct {
	DisableCheckpoint          bool `json:"disable_checkpoint"`
	DisableCheckpointSignature bool `json:"disable_checkpoint_signature"`
	PluginMinPort              int
	PluginMaxPort              int

	Builders       packer.MapOfBuilder
	Provisioners   packer.MapOfProvisioner
	PostProcessors packer.MapOfPostProcessor `json:"post-processors"`
}

// Decodes configuration in JSON format from the given io.Reader into
// the config object pointed to.
func decodeConfig(r io.Reader, c *config) error {
	decoder := json.NewDecoder(r)
	return decoder.Decode(c)
}

// Discover discovers plugins.
//
// Search the directory of the executable, then the plugins directory, and
// finally the CWD, in that order. Any conflicts will overwrite previously
// found plugins, in that order.
// Hence, the priority order is the reverse of the search order - i.e., the
// CWD has the highest priority.
func (c *config) Discover() error {
	// If we are already inside a plugin process we should not need to
	// discover anything.
	if os.Getenv(plugin.MagicCookieKey) == plugin.MagicCookieValue {
		return nil
	}

	// First, look in the same directory as the executable.
	exePath, err := osext.Executable()
	if err != nil {
		log.Printf("[ERR] Error loading exe directory: %s", err)
	} else {
		if err := c.discoverExternalComponents(filepath.Dir(exePath)); err != nil {
			return err
		}
	}

	// Next, look in the plugins directory.
	dir, err := packer.ConfigDir()
	if err != nil {
		log.Printf("[ERR] Error loading config directory: %s", err)
	} else {
		if err := c.discoverExternalComponents(filepath.Join(dir, "plugins")); err != nil {
			return err
		}
	}

	// Next, look in the CWD.
	if err := c.discoverExternalComponents("."); err != nil {
		return err
	}

	// Finally, try to use an internal plugin. Note that this will not override
	// any previously-loaded plugins.
	if err := c.discoverInternalComponents(); err != nil {
		return err
	}

	return nil
}

// This is a proper packer.BuilderFunc that can be used to load packer.Builder
// implementations from the defined plugins.
func (c *config) StartBuilder(name string) (packer.Builder, error) {
	log.Printf("Loading builder: %s\n", name)
	return c.Builders.Start(name)
}

// This is a proper implementation of packer.HookFunc that can be used
// to load packer.Hook implementations from the defined plugins.
func (c *config) StarHook(name string) (packer.Hook, error) {
	log.Printf("Loading hook: %s\n", name)
	return c.pluginClient(name).Hook()
}

// This is a proper packer.PostProcessorFunc that can be used to load
// packer.PostProcessor implementations from defined plugins.
func (c *config) StartPostProcessor(name string) (packer.PostProcessor, error) {
	log.Printf("Loading post-processor: %s", name)
	return c.PostProcessors.Start(name)
}

// This is a proper packer.ProvisionerFunc that can be used to load
// packer.Provisioner implementations from defined plugins.
func (c *config) StartProvisioner(name string) (packer.Provisioner, error) {
	log.Printf("Loading provisioner: %s\n", name)
	return c.Provisioners.Start(name)
}

func (c *config) discoverExternalComponents(path string) error {
	var err error

	if !filepath.IsAbs(path) {
		path, err = filepath.Abs(path)
		if err != nil {
			return err
		}
	}
	externallyUsed := []string{}

	pluginPaths, err := c.discoverSingle(filepath.Join(path, "packer-builder-*"))
	if err != nil {
		return err
	}
	for plugin, path := range pluginPaths {
		newPath := path // this needs to be stored in a new variable for the func below
		c.Builders[plugin] = func() (packer.Builder, error) {
			return c.pluginClient(newPath).Builder()
		}
		externallyUsed = append(externallyUsed, plugin)
	}
	if len(externallyUsed) > 0 {
		sort.Strings(externallyUsed)
		log.Printf("using external builders %v", externallyUsed)
		externallyUsed = nil
	}

	pluginPaths, err = c.discoverSingle(filepath.Join(path, "packer-post-processor-*"))
	if err != nil {
		return err
	}
	for plugin, path := range pluginPaths {
		newPath := path // this needs to be stored in a new variable for the func below
		c.PostProcessors[plugin] = func() (packer.PostProcessor, error) {
			return c.pluginClient(newPath).PostProcessor()
		}
		externallyUsed = append(externallyUsed, plugin)
	}
	if len(externallyUsed) > 0 {
		sort.Strings(externallyUsed)
		log.Printf("using external post-processors %v", externallyUsed)
		externallyUsed = nil
	}

	pluginPaths, err = c.discoverSingle(filepath.Join(path, "packer-provisioner-*"))
	if err != nil {
		return err
	}
	for plugin, path := range pluginPaths {
		newPath := path // this needs to be stored in a new variable for the func below
		c.Provisioners[plugin] = func() (packer.Provisioner, error) {
			return c.pluginClient(newPath).Provisioner()
		}
		externallyUsed = append(externallyUsed, plugin)
	}
	if len(externallyUsed) > 0 {
		sort.Strings(externallyUsed)
		log.Printf("using external provisioners %v", externallyUsed)
		externallyUsed = nil
	}

	return nil
}

func (c *config) discoverSingle(glob string) (map[string]string, error) {
	matches, err := filepath.Glob(glob)
	if err != nil {
		return nil, err
	}

	res := make(map[string]string)

	prefix := filepath.Base(glob)
	prefix = prefix[:strings.Index(prefix, "*")]
	for _, match := range matches {
		file := filepath.Base(match)

		// One Windows, ignore any plugins that don't end in .exe.
		// We could do a full PATHEXT parse, but this is probably good enough.
		if runtime.GOOS == "windows" && strings.ToLower(filepath.Ext(file)) != ".exe" {
			log.Printf(
				"[DEBUG] Ignoring plugin match %s, no exe extension",
				match)
			continue
		}

		// If the filename has a ".", trim up to there
		if idx := strings.Index(file, "."); idx >= 0 {
			file = file[:idx]
		}

		// Look for foo-bar-baz. The plugin name is "baz"
		plugin := file[len(prefix):]
		log.Printf("[DEBUG] Discovered plugin: %s = %s", plugin, match)
		res[plugin] = match
	}

	return res, nil
}

func (c *config) discoverInternalComponents() error {
	// Get the packer binary path
	packerPath, err := osext.Executable()
	if err != nil {
		log.Printf("[ERR] Error loading exe directory: %s", err)
		return err
	}

	for builder := range command.Builders {
		builder := builder
		_, found := (c.Builders)[builder]
		if !found {
			c.Builders[builder] = func() (packer.Builder, error) {
				bin := fmt.Sprintf("%s%splugin%spacker-builder-%s",
					packerPath, PACKERSPACE, PACKERSPACE, builder)
				return c.pluginClient(bin).Builder()
			}
		}
	}

	for provisioner := range command.Provisioners {
		provisioner := provisioner
		_, found := (c.Provisioners)[provisioner]
		if !found {
			c.Provisioners[provisioner] = func() (packer.Provisioner, error) {
				bin := fmt.Sprintf("%s%splugin%spacker-provisioner-%s",
					packerPath, PACKERSPACE, PACKERSPACE, provisioner)
				return c.pluginClient(bin).Provisioner()
			}
		}
	}

	for postProcessor := range command.PostProcessors {
		postProcessor := postProcessor
		_, found := (c.PostProcessors)[postProcessor]
		if !found {
			c.PostProcessors[postProcessor] = func() (packer.PostProcessor, error) {
				bin := fmt.Sprintf("%s%splugin%spacker-post-processor-%s",
					packerPath, PACKERSPACE, PACKERSPACE, postProcessor)
				return c.pluginClient(bin).PostProcessor()
			}
		}
	}

	return nil
}

func (c *config) pluginClient(path string) *plugin.Client {
	originalPath := path

	// First attempt to find the executable by consulting the PATH.
	path, err := exec.LookPath(path)
	if err != nil {
		// If that doesn't work, look for it in the same directory
		// as the `packer` executable (us).
		log.Printf("Plugin could not be found at %s (%v). Checking same directory as executable.", originalPath, err)
		exePath, err := osext.Executable()
		if err != nil {
			log.Printf("Couldn't get current exe path: %s", err)
		} else {
			log.Printf("Current exe path: %s", exePath)
			path = filepath.Join(filepath.Dir(exePath), filepath.Base(originalPath))
		}
	}

	// Check for special case using `packer plugin PLUGIN`
	args := []string{}
	if strings.Contains(path, PACKERSPACE) {
		parts := strings.Split(path, PACKERSPACE)
		path = parts[0]
		args = parts[1:]
	}

	// If everything failed, just use the original path and let the error
	// bubble through.
	if path == "" {
		path = originalPath
	}

	log.Printf("Creating plugin client for path: %s", path)
	var config plugin.ClientConfig
	config.Cmd = exec.Command(path, args...)
	config.Managed = true
	config.MinPort = c.PluginMinPort
	config.MaxPort = c.PluginMaxPort
	return plugin.NewClient(&config)
}
<commit_msg>Fix names of range variables to reduce confusion and remove overriden plugin name<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"sort"
	"strings"

	"github.com/hashicorp/packer/command"
	"github.com/hashicorp/packer/packer"
	"github.com/hashicorp/packer/packer/plugin"
	"github.com/kardianos/osext"
)

// PACKERSPACE is used to represent the spaces that separate args for a command
// without being confused with spaces in the path to the command itself.
const PACKERSPACE = "-PACKERSPACE-"

type config struct {
	DisableCheckpoint          bool `json:"disable_checkpoint"`
	DisableCheckpointSignature bool `json:"disable_checkpoint_signature"`
	PluginMinPort              int
	PluginMaxPort              int

	Builders       packer.MapOfBuilder
	Provisioners   packer.MapOfProvisioner
	PostProcessors packer.MapOfPostProcessor `json:"post-processors"`
}

// Decodes configuration in JSON format from the given io.Reader into
// the config object pointed to.
func decodeConfig(r io.Reader, c *config) error {
	decoder := json.NewDecoder(r)
	return decoder.Decode(c)
}

// Discover discovers plugins.
//
// Search the directory of the executable, then the plugins directory, and
// finally the CWD, in that order. Any conflicts will overwrite previously
// found plugins, in that order.
// Hence, the priority order is the reverse of the search order - i.e., the
// CWD has the highest priority.
func (c *config) Discover() error {
	// If we are already inside a plugin process we should not need to
	// discover anything.
	if os.Getenv(plugin.MagicCookieKey) == plugin.MagicCookieValue {
		return nil
	}

	// First, look in the same directory as the executable.
	exePath, err := osext.Executable()
	if err != nil {
		log.Printf("[ERR] Error loading exe directory: %s", err)
	} else {
		if err := c.discoverExternalComponents(filepath.Dir(exePath)); err != nil {
			return err
		}
	}

	// Next, look in the plugins directory.
	dir, err := packer.ConfigDir()
	if err != nil {
		log.Printf("[ERR] Error loading config directory: %s", err)
	} else {
		if err := c.discoverExternalComponents(filepath.Join(dir, "plugins")); err != nil {
			return err
		}
	}

	// Next, look in the CWD.
	if err := c.discoverExternalComponents("."); err != nil {
		return err
	}

	// Finally, try to use an internal plugin. Note that this will not override
	// any previously-loaded plugins.
	if err := c.discoverInternalComponents(); err != nil {
		return err
	}

	return nil
}

// This is a proper packer.BuilderFunc that can be used to load packer.Builder
// implementations from the defined plugins.
func (c *config) StartBuilder(name string) (packer.Builder, error) {
	log.Printf("Loading builder: %s\n", name)
	return c.Builders.Start(name)
}

// This is a proper implementation of packer.HookFunc that can be used
// to load packer.Hook implementations from the defined plugins.
func (c *config) StarHook(name string) (packer.Hook, error) {
	log.Printf("Loading hook: %s\n", name)
	return c.pluginClient(name).Hook()
}

// This is a proper packer.PostProcessorFunc that can be used to load
// packer.PostProcessor implementations from defined plugins.
func (c *config) StartPostProcessor(name string) (packer.PostProcessor, error) {
	log.Printf("Loading post-processor: %s", name)
	return c.PostProcessors.Start(name)
}

// This is a proper packer.ProvisionerFunc that can be used to load
// packer.Provisioner implementations from defined plugins.
func (c *config) StartProvisioner(name string) (packer.Provisioner, error) {
	log.Printf("Loading provisioner: %s\n", name)
	return c.Provisioners.Start(name)
}

func (c *config) discoverExternalComponents(path string) error {
	var err error

	if !filepath.IsAbs(path) {
		path, err = filepath.Abs(path)
		if err != nil {
			return err
		}
	}
	externallyUsed := []string{}

	pluginPaths, err := c.discoverSingle(filepath.Join(path, "packer-builder-*"))
	if err != nil {
		return err
	}
	for pluginName, pluginPath := range pluginPaths {
		newPath := pluginPath // this needs to be stored in a new variable for the func below
		c.Builders[pluginName] = func() (packer.Builder, error) {
			return c.pluginClient(newPath).Builder()
		}
		externallyUsed = append(externallyUsed, pluginName)
	}
	if len(externallyUsed) > 0 {
		sort.Strings(externallyUsed)
		log.Printf("using external builders %v", externallyUsed)
		externallyUsed = nil
	}

	pluginPaths, err = c.discoverSingle(filepath.Join(path, "packer-post-processor-*"))
	if err != nil {
		return err
	}
	for pluginName, pluginPath := range pluginPaths {
		newPath := pluginPath // this needs to be stored in a new variable for the func below
		c.PostProcessors[pluginName] = func() (packer.PostProcessor, error) {
			return c.pluginClient(newPath).PostProcessor()
		}
		externallyUsed = append(externallyUsed, pluginName)
	}
	if len(externallyUsed) > 0 {
		sort.Strings(externallyUsed)
		log.Printf("using external post-processors %v", externallyUsed)
		externallyUsed = nil
	}

	pluginPaths, err = c.discoverSingle(filepath.Join(path, "packer-provisioner-*"))
	if err != nil {
		return err
	}
	for pluginName, pluginPath := range pluginPaths {
		newPath := pluginPath // this needs to be stored in a new variable for the func below
		c.Provisioners[pluginName] = func() (packer.Provisioner, error) {
			return c.pluginClient(newPath).Provisioner()
		}
		externallyUsed = append(externallyUsed, pluginName)
	}
	if len(externallyUsed) > 0 {
		sort.Strings(externallyUsed)
		log.Printf("using external provisioners %v", externallyUsed)
		externallyUsed = nil
	}

	return nil
}

func (c *config) discoverSingle(glob string) (map[string]string, error) {
	matches, err := filepath.Glob(glob)
	if err != nil {
		return nil, err
	}

	res := make(map[string]string)

	prefix := filepath.Base(glob)
	prefix = prefix[:strings.Index(prefix, "*")]
	for _, match := range matches {
		file := filepath.Base(match)

		// One Windows, ignore any plugins that don't end in .exe.
		// We could do a full PATHEXT parse, but this is probably good enough.
		if runtime.GOOS == "windows" && strings.ToLower(filepath.Ext(file)) != ".exe" {
			log.Printf(
				"[DEBUG] Ignoring plugin match %s, no exe extension",
				match)
			continue
		}

		// If the filename has a ".", trim up to there
		if idx := strings.Index(file, "."); idx >= 0 {
			file = file[:idx]
		}

		// Look for foo-bar-baz. The plugin name is "baz"
		pluginName := file[len(prefix):]
		log.Printf("[DEBUG] Discovered plugin: %s = %s", pluginName, match)
		res[pluginName] = match
	}

	return res, nil
}

func (c *config) discoverInternalComponents() error {
	// Get the packer binary path
	packerPath, err := osext.Executable()
	if err != nil {
		log.Printf("[ERR] Error loading exe directory: %s", err)
		return err
	}

	for builder := range command.Builders {
		builder := builder
		_, found := (c.Builders)[builder]
		if !found {
			c.Builders[builder] = func() (packer.Builder, error) {
				bin := fmt.Sprintf("%s%splugin%spacker-builder-%s",
					packerPath, PACKERSPACE, PACKERSPACE, builder)
				return c.pluginClient(bin).Builder()
			}
		}
	}

	for provisioner := range command.Provisioners {
		provisioner := provisioner
		_, found := (c.Provisioners)[provisioner]
		if !found {
			c.Provisioners[provisioner] = func() (packer.Provisioner, error) {
				bin := fmt.Sprintf("%s%splugin%spacker-provisioner-%s",
					packerPath, PACKERSPACE, PACKERSPACE, provisioner)
				return c.pluginClient(bin).Provisioner()
			}
		}
	}

	for postProcessor := range command.PostProcessors {
		postProcessor := postProcessor
		_, found := (c.PostProcessors)[postProcessor]
		if !found {
			c.PostProcessors[postProcessor] = func() (packer.PostProcessor, error) {
				bin := fmt.Sprintf("%s%splugin%spacker-post-processor-%s",
					packerPath, PACKERSPACE, PACKERSPACE, postProcessor)
				return c.pluginClient(bin).PostProcessor()
			}
		}
	}

	return nil
}

func (c *config) pluginClient(path string) *plugin.Client {
	originalPath := path

	// First attempt to find the executable by consulting the PATH.
	path, err := exec.LookPath(path)
	if err != nil {
		// If that doesn't work, look for it in the same directory
		// as the `packer` executable (us).
		log.Printf("Plugin could not be found at %s (%v). Checking same directory as executable.", originalPath, err)
		exePath, err := osext.Executable()
		if err != nil {
			log.Printf("Couldn't get current exe path: %s", err)
		} else {
			log.Printf("Current exe path: %s", exePath)
			path = filepath.Join(filepath.Dir(exePath), filepath.Base(originalPath))
		}
	}

	// Check for special case using `packer plugin PLUGIN`
	args := []string{}
	if strings.Contains(path, PACKERSPACE) {
		parts := strings.Split(path, PACKERSPACE)
		path = parts[0]
		args = parts[1:]
	}

	// If everything failed, just use the original path and let the error
	// bubble through.
	if path == "" {
		path = originalPath
	}

	log.Printf("Creating plugin client for path: %s", path)
	var config plugin.ClientConfig
	config.Cmd = exec.Command(path, args...)
	config.Managed = true
	config.MinPort = c.PluginMinPort
	config.MaxPort = c.PluginMaxPort
	return plugin.NewClient(&config)
}
<|endoftext|>


<commit_before>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package test

import (
	"fmt"
	"os"
	"path/filepath"

	"v.io/jiri/collect"
	"v.io/jiri/project"
	"v.io/jiri/tool"
	"v.io/x/devtools/internal/test"
	"v.io/x/devtools/internal/xunit"
)

var (
	mirrors = []Mirror{
		Mirror{
			name:         "browser",
			googlesource: "https://vanadium.googlesource.com/release.projects.browser",
			github:       "git@github.com:vanadium/browser.git",
		},
		Mirror{
			name:         "chat",
			googlesource: "https://vanadium.googlesource.com/release.projects.chat",
			github:       "git@github.com:vanadium/chat.git",
		},
		Mirror{
			name:         "croupier",
			googlesource: "https://vanadium.googlesource.com/release.projects.croupier",
			github:       "git@github.com:vanadium/croupier.git",
		},
		Mirror{
			name:         "docs",
			googlesource: "https://vanadium.googlesource.com/docs",
			github:       "git@github.com:vanadium/docs.git",
		},
		Mirror{
			name:         "go.devtools",
			googlesource: "https://vanadium.googlesource.com/release.go.x.devtools",
			github:       "git@github.com:vanadium/go.devtools.git",
		},
		Mirror{
			name:         "go.jiri",
			googlesource: "https://vanadium.googlesource.com/release.go.jiri",
			github:       "git@github.com:vanadium/go.jiri.git",
		},
		Mirror{
			name:         "go.jni",
			googlesource: "https://vanadium.googlesource.com/release.go.jni",
			github:       "git@github.com:vanadium/go.jni.git",
		},
		Mirror{
			name:         "go.lib",
			googlesource: "https://vanadium.googlesource.com/release.go.x.lib",
			github:       "git@github.com:vanadium/go.lib.git",
		},
		Mirror{
			name:         "go.ref",
			googlesource: "https://vanadium.googlesource.com/release.go.x.ref",
			github:       "git@github.com:vanadium/go.ref.git",
		},
		Mirror{
			name:         "go.v23",
			googlesource: "https://vanadium.googlesource.com/release.go.v23",
			github:       "git@github.com:vanadium/go.v23.git",
		},
		Mirror{
			name:         "java",
			googlesource: "https://vanadium.googlesource.com/release.java",
			github:       "git@github.com:vanadium/java.git",
		},
		Mirror{
			name:         "js",
			googlesource: "https://vanadium.googlesource.com/release.js.core",
			github:       "git@github.com:vanadium/js.git",
		},
		Mirror{
			name:         "js.syncbase",
			googlesource: "https://vanadium.googlesource.com/release.js.syncbase",
			github:       "git@github.com:vanadium/js.syncbase.git",
		},
		Mirror{
			name:         "media-sharing",
			googlesource: "https://vanadium.googlesource.com/release.projects.media-sharing",
			github:       "git@github.com:vanadium/media-sharing.git",
		},
		Mirror{
			name:         "mojo.discovery",
			googlesource: "https://vanadium.googlesource.com/release.mojo.discovery",
			github:       "git@github.com:vanadium/mojo.discovery.git",
		},
		Mirror{
			name:         "mojo.shared",
			googlesource: "https://vanadium.googlesource.com/release.mojo.shared",
			github:       "git@github.com:vanadium/mojo.shared.git",
		},
		Mirror{
			name:         "mojo.syncbase",
			googlesource: "https://vanadium.googlesource.com/release.mojo.syncbase",
			github:       "git@github.com:vanadium/mojo.syncbase.git",
		},
		Mirror{
			name:         "physical-lock",
			googlesource: "https://vanadium.googlesource.com/release.projects.physical-lock",
			github:       "git@github.com:vanadium/physical-lock.git",
		},
		Mirror{
			name:         "pipe2browser",
			googlesource: "https://vanadium.googlesource.com/release.projects.pipe2browser",
			github:       "git@github.com:vanadium/pipe2browser.git",
		},
		Mirror{
			name:         "playground",
			googlesource: "https://vanadium.googlesource.com/release.projects.playground",
			github:       "git@github.com:vanadium/playground.git",
		},
		Mirror{
			name:         "reader",
			googlesource: "https://vanadium.googlesource.com/release.projects.reader",
			github:       "git@github.com:vanadium/reader.git",
		},
		Mirror{
			name:         "third_party",
			googlesource: "https://vanadium.googlesource.com/third_party",
			github:       "git@github.com:vanadium/third_party.git",
		},
		Mirror{
			name:         "todos",
			googlesource: "https://vanadium.googlesource.com/release.projects.todos",
			github:       "git@github.com:vanadium/todos.git",
		},
		Mirror{
			name:         "travel",
			googlesource: "https://vanadium.googlesource.com/release.projects.travel",
			github:       "git@github.com:vanadium/travel.git",
		},
		Mirror{
			name:         "www",
			googlesource: "https://vanadium.googlesource.com/www",
			github:       "git@github.com:vanadium/www.git",
		},
	}
)

type Mirror struct {
	name, googlesource, github string
}

// vanadiumGitHubMirror mirrors googlesource.com vanadium projects to
// github.com.
func vanadiumGitHubMirror(ctx *tool.Context, testName string, _ ...Opt) (_ *test.Result, e error) {
	// Initialize the test/task.
	cleanup, err := initTest(ctx, testName, nil)
	if err != nil {
		return nil, internalTestError{err, "Init"}
	}
	defer collect.Error(func() error { return cleanup() }, &e)

	root, err := project.JiriRoot()
	if err != nil {
		return nil, internalTestError{err, "JiriRoot"}
	}

	projects := filepath.Join(root, "projects")
	mode := os.FileMode(0755)
	if err := ctx.Run().MkdirAll(projects, mode); err != nil {
		return nil, internalTestError{err, "MkdirAll"}
	}

	allPassed := true
	suites := []xunit.TestSuite{}
	for _, mirror := range mirrors {
		suite, err := sync(ctx, mirror, projects)
		if err != nil {
			return nil, internalTestError{err, "sync"}
		}

		allPassed = allPassed && (suite.Failures == 0)
		suites = append(suites, *suite)
	}

	if err := xunit.CreateReport(ctx, testName, suites); err != nil {
		return nil, err
	}

	if !allPassed {
		return &test.Result{Status: test.Failed}, nil
	}

	return &test.Result{Status: test.Passed}, nil
}

func sync(ctx *tool.Context, mirror Mirror, projects string) (*xunit.TestSuite, error) {
	suite := xunit.TestSuite{Name: mirror.name}
	dirname := filepath.Join(projects, mirror.name)

	// If dirname does not exist `git clone` otherwise `git pull`.
	if _, err := ctx.Run().Stat(dirname); err != nil {
		if !os.IsNotExist(err) {
			return nil, internalTestError{err, "stat"}
		}

		err := clone(ctx, mirror, projects)
		testCase := makeTestCase("clone", err)
		if err != nil {
			suite.Failures++
		}
		suite.Cases = append(suite.Cases, *testCase)
	} else {
		err := pull(ctx, mirror, projects)
		testCase := makeTestCase("pull", err)
		if err != nil {
			suite.Failures++
		}
		suite.Cases = append(suite.Cases, *testCase)
	}

	err := push(ctx, mirror, projects)
	testCase := makeTestCase("push", err)
	if err != nil {
		suite.Failures++
	}
	suite.Cases = append(suite.Cases, *testCase)

	return &suite, nil
}

func makeTestCase(action string, err error) *xunit.TestCase {
	c := xunit.TestCase{
		Classname: "git",
		Name:      action,
	}

	if err != nil {
		f := xunit.Failure{
			Message: "git error",
			Data:    fmt.Sprintf("%v", err),
		}
		c.Failures = append(c.Failures, f)
	}

	return &c
}

func clone(ctx *tool.Context, mirror Mirror, projects string) error {
	dirname := filepath.Join(projects, mirror.name)
	return ctx.Git().Clone(mirror.googlesource, dirname)
}

func pull(ctx *tool.Context, mirror Mirror, projects string) error {
	dirname := filepath.Join(projects, mirror.name)
	opts := tool.RootDirOpt(dirname)
	return ctx.Git(opts).Pull("origin", "master")
}

func push(ctx *tool.Context, mirror Mirror, projects string) error {
	dirname := filepath.Join(projects, mirror.name)
	opts := tool.RootDirOpt(dirname)
	return ctx.Git(opts).Push(mirror.github, "master")
}
<commit_msg>devtools: add manifest to github mirror targets.<commit_after>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package test

import (
	"fmt"
	"os"
	"path/filepath"

	"v.io/jiri/collect"
	"v.io/jiri/project"
	"v.io/jiri/tool"
	"v.io/x/devtools/internal/test"
	"v.io/x/devtools/internal/xunit"
)

var (
	mirrors = []Mirror{
		Mirror{
			name:         "browser",
			googlesource: "https://vanadium.googlesource.com/release.projects.browser",
			github:       "git@github.com:vanadium/browser.git",
		},
		Mirror{
			name:         "chat",
			googlesource: "https://vanadium.googlesource.com/release.projects.chat",
			github:       "git@github.com:vanadium/chat.git",
		},
		Mirror{
			name:         "croupier",
			googlesource: "https://vanadium.googlesource.com/release.projects.croupier",
			github:       "git@github.com:vanadium/croupier.git",
		},
		Mirror{
			name:         "docs",
			googlesource: "https://vanadium.googlesource.com/docs",
			github:       "git@github.com:vanadium/docs.git",
		},
		Mirror{
			name:         "go.devtools",
			googlesource: "https://vanadium.googlesource.com/release.go.x.devtools",
			github:       "git@github.com:vanadium/go.devtools.git",
		},
		Mirror{
			name:         "go.jiri",
			googlesource: "https://vanadium.googlesource.com/release.go.jiri",
			github:       "git@github.com:vanadium/go.jiri.git",
		},
		Mirror{
			name:         "go.jni",
			googlesource: "https://vanadium.googlesource.com/release.go.jni",
			github:       "git@github.com:vanadium/go.jni.git",
		},
		Mirror{
			name:         "go.lib",
			googlesource: "https://vanadium.googlesource.com/release.go.x.lib",
			github:       "git@github.com:vanadium/go.lib.git",
		},
		Mirror{
			name:         "go.ref",
			googlesource: "https://vanadium.googlesource.com/release.go.x.ref",
			github:       "git@github.com:vanadium/go.ref.git",
		},
		Mirror{
			name:         "go.v23",
			googlesource: "https://vanadium.googlesource.com/release.go.v23",
			github:       "git@github.com:vanadium/go.v23.git",
		},
		Mirror{
			name:         "java",
			googlesource: "https://vanadium.googlesource.com/release.java",
			github:       "git@github.com:vanadium/java.git",
		},
		Mirror{
			name:         "js",
			googlesource: "https://vanadium.googlesource.com/release.js.core",
			github:       "git@github.com:vanadium/js.git",
		},
		Mirror{
			name:         "js.syncbase",
			googlesource: "https://vanadium.googlesource.com/release.js.syncbase",
			github:       "git@github.com:vanadium/js.syncbase.git",
		},
		Mirror{
			name:         "manifest",
			googlesource: "https://vanadium.googlesource.com/manifest",
			github:       "git@github.com:vanadium/manifest.git",
		},
		Mirror{
			name:         "media-sharing",
			googlesource: "https://vanadium.googlesource.com/release.projects.media-sharing",
			github:       "git@github.com:vanadium/media-sharing.git",
		},
		Mirror{
			name:         "mojo.discovery",
			googlesource: "https://vanadium.googlesource.com/release.mojo.discovery",
			github:       "git@github.com:vanadium/mojo.discovery.git",
		},
		Mirror{
			name:         "mojo.shared",
			googlesource: "https://vanadium.googlesource.com/release.mojo.shared",
			github:       "git@github.com:vanadium/mojo.shared.git",
		},
		Mirror{
			name:         "mojo.syncbase",
			googlesource: "https://vanadium.googlesource.com/release.mojo.syncbase",
			github:       "git@github.com:vanadium/mojo.syncbase.git",
		},
		Mirror{
			name:         "physical-lock",
			googlesource: "https://vanadium.googlesource.com/release.projects.physical-lock",
			github:       "git@github.com:vanadium/physical-lock.git",
		},
		Mirror{
			name:         "pipe2browser",
			googlesource: "https://vanadium.googlesource.com/release.projects.pipe2browser",
			github:       "git@github.com:vanadium/pipe2browser.git",
		},
		Mirror{
			name:         "playground",
			googlesource: "https://vanadium.googlesource.com/release.projects.playground",
			github:       "git@github.com:vanadium/playground.git",
		},
		Mirror{
			name:         "reader",
			googlesource: "https://vanadium.googlesource.com/release.projects.reader",
			github:       "git@github.com:vanadium/reader.git",
		},
		Mirror{
			name:         "third_party",
			googlesource: "https://vanadium.googlesource.com/third_party",
			github:       "git@github.com:vanadium/third_party.git",
		},
		Mirror{
			name:         "todos",
			googlesource: "https://vanadium.googlesource.com/release.projects.todos",
			github:       "git@github.com:vanadium/todos.git",
		},
		Mirror{
			name:         "travel",
			googlesource: "https://vanadium.googlesource.com/release.projects.travel",
			github:       "git@github.com:vanadium/travel.git",
		},
		Mirror{
			name:         "www",
			googlesource: "https://vanadium.googlesource.com/www",
			github:       "git@github.com:vanadium/www.git",
		},
	}
)

type Mirror struct {
	name, googlesource, github string
}

// vanadiumGitHubMirror mirrors googlesource.com vanadium projects to
// github.com.
func vanadiumGitHubMirror(ctx *tool.Context, testName string, _ ...Opt) (_ *test.Result, e error) {
	// Initialize the test/task.
	cleanup, err := initTest(ctx, testName, nil)
	if err != nil {
		return nil, internalTestError{err, "Init"}
	}
	defer collect.Error(func() error { return cleanup() }, &e)

	root, err := project.JiriRoot()
	if err != nil {
		return nil, internalTestError{err, "JiriRoot"}
	}

	projects := filepath.Join(root, "projects")
	mode := os.FileMode(0755)
	if err := ctx.Run().MkdirAll(projects, mode); err != nil {
		return nil, internalTestError{err, "MkdirAll"}
	}

	allPassed := true
	suites := []xunit.TestSuite{}
	for _, mirror := range mirrors {
		suite, err := sync(ctx, mirror, projects)
		if err != nil {
			return nil, internalTestError{err, "sync"}
		}

		allPassed = allPassed && (suite.Failures == 0)
		suites = append(suites, *suite)
	}

	if err := xunit.CreateReport(ctx, testName, suites); err != nil {
		return nil, err
	}

	if !allPassed {
		return &test.Result{Status: test.Failed}, nil
	}

	return &test.Result{Status: test.Passed}, nil
}

func sync(ctx *tool.Context, mirror Mirror, projects string) (*xunit.TestSuite, error) {
	suite := xunit.TestSuite{Name: mirror.name}
	dirname := filepath.Join(projects, mirror.name)

	// If dirname does not exist `git clone` otherwise `git pull`.
	if _, err := ctx.Run().Stat(dirname); err != nil {
		if !os.IsNotExist(err) {
			return nil, internalTestError{err, "stat"}
		}

		err := clone(ctx, mirror, projects)
		testCase := makeTestCase("clone", err)
		if err != nil {
			suite.Failures++
		}
		suite.Cases = append(suite.Cases, *testCase)
	} else {
		err := pull(ctx, mirror, projects)
		testCase := makeTestCase("pull", err)
		if err != nil {
			suite.Failures++
		}
		suite.Cases = append(suite.Cases, *testCase)
	}

	err := push(ctx, mirror, projects)
	testCase := makeTestCase("push", err)
	if err != nil {
		suite.Failures++
	}
	suite.Cases = append(suite.Cases, *testCase)

	return &suite, nil
}

func makeTestCase(action string, err error) *xunit.TestCase {
	c := xunit.TestCase{
		Classname: "git",
		Name:      action,
	}

	if err != nil {
		f := xunit.Failure{
			Message: "git error",
			Data:    fmt.Sprintf("%v", err),
		}
		c.Failures = append(c.Failures, f)
	}

	return &c
}

func clone(ctx *tool.Context, mirror Mirror, projects string) error {
	dirname := filepath.Join(projects, mirror.name)
	return ctx.Git().Clone(mirror.googlesource, dirname)
}

func pull(ctx *tool.Context, mirror Mirror, projects string) error {
	dirname := filepath.Join(projects, mirror.name)
	opts := tool.RootDirOpt(dirname)
	return ctx.Git(opts).Pull("origin", "master")
}

func push(ctx *tool.Context, mirror Mirror, projects string) error {
	dirname := filepath.Join(projects, mirror.name)
	opts := tool.RootDirOpt(dirname)
	return ctx.Git(opts).Push(mirror.github, "master")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"strings"

	"github.com/aerogo/flow"
	"github.com/animenotifier/arn"
	"github.com/fatih/color"
)

func main() {
	color.Yellow("Updating search index")
	defer arn.Node.Close()

	flow.Parallel(
		updateAnimeIndex,
		updateUserIndex,
		updatePostIndex,
		updateThreadIndex,
	)

	color.Green("Finished.")
}

func updateAnimeIndex() {
	animeSearchIndex := arn.NewSearchIndex()

	// Anime
	for anime := range arn.StreamAnime() {
		if anime.Title.Canonical != "" {
			animeSearchIndex.TextToID[strings.ToLower(anime.Title.Canonical)] = anime.ID
		}

		if anime.Title.Romaji != "" {
			animeSearchIndex.TextToID[strings.ToLower(anime.Title.Romaji)] = anime.ID
		}

		// Make sure we only include Japanese titles that
		// don't overlap with the English titles.
		if anime.Title.Japanese != "" && animeSearchIndex.TextToID[strings.ToLower(anime.Title.Japanese)] == "" {
			animeSearchIndex.TextToID[strings.ToLower(anime.Title.Japanese)] = anime.ID
		}

		// Same with English titles, don't overwrite other stuff.
		if anime.Title.English != "" && animeSearchIndex.TextToID[strings.ToLower(anime.Title.English)] == "" {
			animeSearchIndex.TextToID[strings.ToLower(anime.Title.English)] = anime.ID
		}

		for _, synonym := range anime.Title.Synonyms {
			synonym = strings.ToLower(synonym)

			if synonym != "" && len(synonym) <= 10 {
				animeSearchIndex.TextToID[synonym] = anime.ID
			}
		}
	}

	fmt.Println(len(animeSearchIndex.TextToID), "anime titles")

	// Save in database
	arn.DB.Set("SearchIndex", "Anime", animeSearchIndex)
}

func updateUserIndex() {
	userSearchIndex := arn.NewSearchIndex()

	// Users
	for user := range arn.StreamUsers() {
		if user.HasNick() {
			userSearchIndex.TextToID[strings.ToLower(user.Nick)] = user.ID
		}
	}

	fmt.Println(len(userSearchIndex.TextToID), "user names")

	// Save in database
	arn.DB.Set("SearchIndex", "User", userSearchIndex)
}

func updatePostIndex() {
	postSearchIndex := arn.NewSearchIndex()

	// Users
	for post := range arn.StreamPosts() {
		postSearchIndex.TextToID[strings.ToLower(post.Text)] = post.ID
	}

	fmt.Println(len(postSearchIndex.TextToID), "posts")

	// Save in database
	arn.DB.Set("SearchIndex", "Post", postSearchIndex)
}

func updateThreadIndex() {
	threadSearchIndex := arn.NewSearchIndex()

	// Users
	for thread := range arn.StreamThreads() {
		threadSearchIndex.TextToID[strings.ToLower(thread.Title)] = thread.ID
		threadSearchIndex.TextToID[strings.ToLower(thread.Text)] = thread.ID
	}

	fmt.Println(len(threadSearchIndex.TextToID)/2, "threads")

	// Save in database
	arn.DB.Set("SearchIndex", "Thread", threadSearchIndex)
}
<commit_msg>Deleted search index job<commit_after><|endoftext|>


<commit_before>// Package gocontainer is a dependency injection and service container library for go.
// This library allow you to register service objects using string id, inject them
// into other service objects using an `inject:"some_service_id"` struct tags and
// get the service from the container by its id
package gocontainer

import (
	"fmt"
	"github.com/facebookgo/inject"
	"os"
	"os/signal"
)

// Service interface
type Service interface {
	// Start up routine, will be executed on container Ready
	StartUp() error
	// Shutdown routine, will be executed on container Shutdown
	Shutdown() error
}

// ContainerAware is an interface to make a service to have reference to the container
type ContainerAware interface {
	// Inject the service container
	SetContainer(c ServiceContainer)
}

// ServiceContainer is an interface for custom service container
type ServiceContainer interface {
	// Lock-up the container for further service registration and populate current
	// dependency trees
	Ready() error
	// Find a service by specified id in the container and return them
	// Should return false if the service is not found
	GetService(id string) (interface{}, bool)
	// Register a service to the container
	RegisterService(id string, svc interface{})
	// Register multiple services into the container
	// services is a map of service id and service object
	RegisterServices(services map[string]interface{})
	// Shutdown all service objects
	Shutdown() error
}

// ServiceRegistry is default implementation of ServiceContainer interface
type ServiceRegistry struct {
	graph    inject.Graph
	services map[string]interface{}
	order    map[int]string
}

// GetService find a service by its id in ServiceRegistry
func (reg *ServiceRegistry) GetService(id string) (svc interface{}, ok bool) {
	svc, ok = reg.services[id]
	return
}

// RegisterService adds a service object with specified id
func (reg *ServiceRegistry) RegisterService(id string, svc interface{}) {
	err := reg.graph.Provide(&inject.Object{Name: id, Value: svc, Complete: false})
	if err != nil {
		panic(err.Error())
	}
	reg.order[len(reg.order)] = id
	reg.services[id] = svc
}

// RegisterServices adds multiple services into the container
// services is a map of service id and service object
func (reg *ServiceRegistry) RegisterServices(services map[string]interface{}) {
	for id, svc := range reg.services {
		reg.RegisterService(id, svc)
	}
}

// Ready locks up the container to prevent futher service registration and
// populate current dependency tree
func (reg *ServiceRegistry) Ready() (err error) {
	err = reg.graph.Populate()
	if err != nil {
		return
	}
	defer reg.shutdownHandler()
	for i := 0; i < len(reg.order); i++ {
		k := reg.order[i]
		obj := reg.services[k]
		var startErr error
		switch s := obj.(type) {
		case Service:
			containerAware, ok := s.(ContainerAware)
			if ok {
				containerAware.SetContainer(reg)
			}
			startErr = s.StartUp()
		}
		if startErr != nil {
			fmt.Println("Error starting up service [%s] %v", k, startErr)
		}
	}
	return
}

func (reg *ServiceRegistry) Shutdown() (err error) {
	fmt.Println("Gracefully shutting down the service container")
	for i := 0; i < len(reg.order); i++ {
		k := reg.order[i]
		obj := reg.services[k]
		var shutdownErr error
		switch s := obj.(type) {
		case Service:
			shutdownErr = s.Shutdown()
		}
		if shutdownErr != nil {
			fmt.Printf("Error while shutting down service [%s] %v", k, shutdownErr)
		}
	}
	return
}

func (reg *ServiceRegistry) shutdownHandler() {
	go func() {
		sigchan := make(chan os.Signal, 15)
		signal.Notify(sigchan, os.Interrupt)
		<-sigchan
		reg.Shutdown()
	}()
}

// NewContainer creates a new empty ServiceRegistry
func NewContainer() (reg ServiceContainer) {
	reg = &ServiceRegistry{services: make(map[string]interface{}), order: make(map[int]string)}
	return
}
<commit_msg>Add `EnableGracefulShutdown` method to handle graceful shutdown<commit_after>// Package gocontainer is a dependency injection and service container library for go.
// This library allow you to register service objects using string id, inject them
// into other service objects using an `inject:"some_service_id"` struct tags and
// get the service from the container by its id
package gocontainer

import (
	"fmt"
	"os"
	"os/signal"
	"time"

	"github.com/facebookgo/inject"
)

// Service interface
type Service interface {
	// Start up routine, will be executed on container Ready
	StartUp() error
	// Shutdown routine, will be executed on container Shutdown
	Shutdown() error
}

// ContainerAware is an interface to make a service to have reference to the container
type ContainerAware interface {
	// Inject the service container
	SetContainer(c ServiceContainer)
}

// ServiceContainer is an interface for custom service container
type ServiceContainer interface {
	// Lock-up the container for further service registration and populate current
	// dependency trees
	Ready() error
	// Find a service by specified id in the container and return them
	// Should return false if the service is not found
	GetService(id string) (interface{}, bool)
	// Register a service to the container
	RegisterService(id string, svc interface{})
	// Register multiple services into the container
	// services is a map of service id and service object
	RegisterServices(services map[string]interface{})
	// Shutdown all service objects
	Shutdown() error
	// Set shutdown duration
	HandleGracefulShutdown(time.Duration)
}

// ServiceRegistry is default implementation of ServiceContainer interface
type ServiceRegistry struct {
	graph          inject.Graph
	services       map[string]interface{}
	order          map[int]string
	exitOnShutdown bool
	gracefulPeriod time.Duration
}

// GetService find a service by its id in ServiceRegistry
func (reg *ServiceRegistry) GetService(id string) (svc interface{}, ok bool) {
	svc, ok = reg.services[id]
	return
}

// RegisterService adds a service object with specified id
func (reg *ServiceRegistry) RegisterService(id string, svc interface{}) {
	err := reg.graph.Provide(&inject.Object{Name: id, Value: svc, Complete: false})
	if err != nil {
		panic(err.Error())
	}
	reg.order[len(reg.order)] = id
	reg.services[id] = svc
}

// RegisterServices adds multiple services into the container
// services is a map of service id and service object
func (reg *ServiceRegistry) RegisterServices(services map[string]interface{}) {
	for id, svc := range reg.services {
		reg.RegisterService(id, svc)
	}
}

// Ready locks up the container to prevent futher service registration and
// populate current dependency tree
func (reg *ServiceRegistry) Ready() (err error) {
	err = reg.graph.Populate()
	if err != nil {
		return
	}
	if reg.exitOnShutdown {
		defer reg.shutdownHandler()
	}
	for i := 0; i < len(reg.order); i++ {
		k := reg.order[i]
		obj := reg.services[k]
		var startErr error
		switch s := obj.(type) {
		case Service:
			containerAware, ok := s.(ContainerAware)
			if ok {
				containerAware.SetContainer(reg)
			}
			startErr = s.StartUp()
		}
		if startErr != nil {
			fmt.Println("Error starting up service [%s] %v", k, startErr)
		}
	}
	return
}

func (reg *ServiceRegistry) Shutdown() (err error) {
	if reg.exitOnShutdown {
		fmt.Println("Gracefully shutting down the service container")
	}
	for i := 0; i < len(reg.order); i++ {
		k := reg.order[i]
		obj := reg.services[k]
		var shutdownErr error
		switch s := obj.(type) {
		case Service:
			shutdownErr = s.Shutdown()
		}
		if shutdownErr != nil {
			fmt.Printf("Error while shutting down service [%s] %v", k, shutdownErr)
		}
	}
	if reg.exitOnShutdown {
		<-time.After(reg.gracefulPeriod)
		os.Exit(0)
	}
	return
}

func (reg *ServiceRegistry) shutdownHandler() {
	go func() {
		sigchan := make(chan os.Signal, 15)
		signal.Notify(sigchan, os.Interrupt)
		<-sigchan
		reg.Shutdown()
	}()
}

func (reg *ServiceRegistry) HandleGracefulShutdown(d time.Duration) {
	reg.exitOnShutdown = true
	reg.gracefulPeriod = d
}

// NewContainer creates a new empty ServiceRegistry
func NewContainer() (reg ServiceContainer) {
	reg = &ServiceRegistry{services: make(map[string]interface{}), order: make(map[int]string)}
	return
}
<|endoftext|>


<commit_before>package hawk

import (
	"crypto/hmac"
	"crypto/sha256"
	"crypto/sha512"
	"encoding/base64"
	"hash"
	"net"
	"net/url"
	"strconv"
	"strings"
)

const headerVersion = 1

type AuthType int

const (
	AuthHeader AuthType = iota
	AuthResponse
	AuthBewit
)

type Mac struct {
	Type       AuthType
	Credential Credential
	Uri        string
	Method     string
	Option     Option
}

type TsMac struct {
	TimeStamp  int64
	Credential Credential
}

func (m *Mac) String() (string, error) {
	digest, err := m.digest()
	if err != nil {
		return "", err
	}
	return base64.StdEncoding.EncodeToString(digest), nil
}

func (m *Mac) digest() ([]byte, error) {
	h := getHash(m.Credential.Alg)

	mac := hmac.New(h, []byte(m.Credential.Key))
	ns, err := m.nomarized()
	if err != nil {
		return nil, err
	}
	mac.Write([]byte(ns))

	return mac.Sum(nil), nil
}

func (m *Mac) nomarized() (string, error) {
	return nomarized(m.Type, m.Uri, m.Method, m.Option)
}

func (tm *TsMac) String() string {
	digest := tm.digest()

	return base64.StdEncoding.EncodeToString(digest)
}

func (tm *TsMac) digest() []byte {
	h := getHash(tm.Credential.Alg)

	mac := hmac.New(h, []byte(tm.Credential.Key))
	ns := "hawk." + strconv.Itoa(headerVersion) + ".ts" + "\n" + strconv.FormatInt(tm.TimeStamp, 10) + "\n"
	mac.Write([]byte(ns))

	return mac.Sum(nil)
}

func nomarized(authType AuthType, uri string, method string, option Option) (string, error) {
	u, err := url.Parse(uri)
	if err != nil {
		return "", err
	}

	host := u.Host
	_, port, _ := net.SplitHostPort(u.Host)
	if port == "" {
		switch u.Scheme {
		case "http":
			port = "80"
		case "https":
			port = "443"
		}
	}
	path := u.Path

	header := "hawk" + "." + strconv.Itoa(headerVersion) + "." + authType.String()

	ns := header + "\n" +
		strconv.FormatInt(option.TimeStamp, 10) + "\n" +
		option.Nonce + "\n" +
		strings.ToUpper(method) + "\n" +
		strings.ToLower(path) + "\n" +
		host + "\n" +
		port + "\n" +
		option.Hash + "\n" +
		option.Ext + "\n"

	return ns, nil
}

func getHash(alg Alg) func() hash.Hash {
	switch alg {
	case SHA256:
		return sha256.New
	case SHA512:
		return sha512.New
	default:
		return sha256.New
	}
}
<commit_msg>Fix typo<commit_after>package hawk

import (
	"crypto/hmac"
	"crypto/sha256"
	"crypto/sha512"
	"encoding/base64"
	"hash"
	"net"
	"net/url"
	"strconv"
	"strings"
)

const headerVersion = 1

type AuthType int

const (
	AuthHeader AuthType = iota
	AuthResponse
	AuthBewit
)

type Mac struct {
	Type       AuthType
	Credential Credential
	Uri        string
	Method     string
	Option     Option
}

type TsMac struct {
	TimeStamp  int64
	Credential Credential
}

func (m *Mac) String() (string, error) {
	digest, err := m.digest()
	if err != nil {
		return "", err
	}
	return base64.StdEncoding.EncodeToString(digest), nil
}

func (m *Mac) digest() ([]byte, error) {
	h := getHash(m.Credential.Alg)

	mac := hmac.New(h, []byte(m.Credential.Key))
	ns, err := m.normalized()
	if err != nil {
		return nil, err
	}
	mac.Write([]byte(ns))

	return mac.Sum(nil), nil
}

func (m *Mac) normalized() (string, error) {
	return normalized(m.Type, m.Uri, m.Method, m.Option)
}

func (tm *TsMac) String() string {
	digest := tm.digest()

	return base64.StdEncoding.EncodeToString(digest)
}

func (tm *TsMac) digest() []byte {
	h := getHash(tm.Credential.Alg)

	mac := hmac.New(h, []byte(tm.Credential.Key))
	ns := "hawk." + strconv.Itoa(headerVersion) + ".ts" + "\n" + strconv.FormatInt(tm.TimeStamp, 10) + "\n"
	mac.Write([]byte(ns))

	return mac.Sum(nil)
}

func normalized(authType AuthType, uri string, method string, option Option) (string, error) {
	u, err := url.Parse(uri)
	if err != nil {
		return "", err
	}

	host := u.Host
	_, port, _ := net.SplitHostPort(u.Host)
	if port == "" {
		switch u.Scheme {
		case "http":
			port = "80"
		case "https":
			port = "443"
		}
	}
	path := u.Path

	header := "hawk" + "." + strconv.Itoa(headerVersion) + "." + authType.String()

	ns := header + "\n" +
		strconv.FormatInt(option.TimeStamp, 10) + "\n" +
		option.Nonce + "\n" +
		strings.ToUpper(method) + "\n" +
		strings.ToLower(path) + "\n" +
		host + "\n" +
		port + "\n" +
		option.Hash + "\n" +
		option.Ext + "\n"

	return ns, nil
}

func getHash(alg Alg) func() hash.Hash {
	switch alg {
	case SHA256:
		return sha256.New
	case SHA512:
		return sha512.New
	default:
		return sha256.New
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net/http"
	"net/url"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/fsouza/go-dockerclient"
	"github.com/pwaller/barrier"

	"github.com/scraperwiki/hanoverd/pkg/source"
)

type Container struct {
	Name      string
	ImageName string
	Args, Env []string
	Volumes   []string
	StatusURI string

	client    *docker.Client
	container *docker.Container

	Failed, Superceded, Obtained, Ready, Closing barrier.Barrier

	wg *sync.WaitGroup

	Errors  <-chan error
	errorsW chan<- error
}

// Construct a *Container. When the `wg` WaitGroup is zero, there is nothing
// outstanding (such as firewall rules which need garbage collecting).
func NewContainer(client *docker.Client, name string, wg *sync.WaitGroup) *Container {

	errors := make(chan error)

	c := &Container{
		Name:    name,
		client:  client,
		wg:      wg,
		Errors:  errors,
		errorsW: errors,
	}

	// If the container fails we should assume it should be torn down.
	c.Failed.Forward(&c.Closing)

	return c
}

func makeVolumeSet(in []string) map[string]struct{} {
	volumes := map[string]struct{}{}
	for _, v := range in {
		if strings.Contains(v, ":") {
			continue
		}
		volumes[v] = struct{}{}
	}
	return volumes
}

func makeBinds(in []string) []string {
	binds := []string{}
	for _, v := range in {
		if !strings.Contains(v, ":") {
			continue
		}
		binds = append(binds, v)
	}
	return binds
}

// `docker create` the container.
func (c *Container) Create(imageName string) error {
	opts := docker.CreateContainerOptions{
		Config: &docker.Config{
			Hostname:     c.Name,
			AttachStdout: true,
			AttachStderr: true,
			Env:          c.Env,
			Cmd:          c.Args,
			Image:        imageName,
			Volumes:      makeVolumeSet(c.Volumes),
			Labels: map[string]string{
				"orchestrator":  "hanoverd",
				"hanoverd-name": c.Name,
			},
		},
	}

	var err error
	c.container, err = c.client.CreateContainer(opts)
	return err
}

// CopyOutput copies the output of the container to `w` and blocks until
// completion
func (c *Container) CopyOutput() error {

	// TODO(pwaller): at some point move this on to 'c' for configurability?
	w := os.Stderr
	// Blocks until stream closed
	return c.client.AttachToContainer(docker.AttachToContainerOptions{
		Container:    c.container.ID,
		OutputStream: w,
		ErrorStream:  w,
		Logs:         true,
		Stdout:       true,
		Stderr:       true,
		Stream:       true,
	})
}

// Poll for the program inside the container being ready to accept connections
// Returns `true` for success and `false` for failure.
func (c *Container) AwaitListening() error {

	if len(c.container.NetworkSettings.PortMappingAPI()) == 0 {
		return fmt.Errorf("no ports are exposed (specify EXPOSE in Dockerfile)")
	}

	const (
		DefaultTimeout = 5 * time.Minute
		PollFrequency  = 5 // approx. times per second.
	)

	success := make(chan struct{}, len(c.container.NetworkSettings.PortMappingAPI()))
	finished := make(chan struct{})
	defer close(finished)

	// Poll the statusURL once.
	// Returns true if polling should continue and false otherwise.
	poll := func(statusURL string) bool {
		req, err := http.NewRequest("GET", statusURL, nil)
		if err != nil {
			log.Printf("Warning, malformed URL: %q: %v", statusURL, err)
			return false
		}
		req.Cancel = finished

		resp, err := http.DefaultClient.Do(req)
		if resp != nil && resp.Body != nil {
			// Don't care about the body, make sure we close it.
			resp.Body.Close()
		}

		if urlErr, ok := err.(*url.Error); ok {
			errStr := urlErr.Err.Error()
			if strings.Contains(errStr, "malformed HTTP response") {
				// Seen in case endpoint doesn't speak HTTP. Give up.
				return false
			}
		}

		if resp == nil {
			// Keep going, connection probably failed.
			return true
		}
		switch resp.StatusCode {
		case http.StatusOK:
			success <- struct{}{}
			return false

		default:
			log.Printf("Status poller got non-200 status: %q returned %v",
				statusURL, resp.Status)
			return false
		}

		return true
	}

	var pollers sync.WaitGroup

	// Start one poller per exposed port.
	for _, port := range c.container.NetworkSettings.PortMappingAPI() {
		statusURL := fmt.Sprint("http://", port.IP, ":", port.PublicPort, c.StatusURI)

		c.wg.Add(1)
		pollers.Add(1)
		go func() {
			defer c.wg.Done()
			defer pollers.Done()

			// Poll until:
			// * 200 status code
			// * malformed response
			// * teardown
			for poll(statusURL) {
				select {
				case <-finished:
					return
				case <-time.After(time.Second / PollFrequency):
				}
			}
		}()
	}

	noPollersRemain := make(chan struct{})
	go func() {
		defer close(noPollersRemain)
		pollers.Wait()
	}()

	select {
	case <-success:
		return nil

	case <-noPollersRemain:
		return fmt.Errorf("no status checks succeeded")

	case <-c.Closing.Barrier():
		return fmt.Errorf("shutting down")

	case <-time.After(DefaultTimeout):
		return fmt.Errorf("took longer than %v to start, giving up", DefaultTimeout)
	}
}

// Given an internal port, return the port mapped by docker, if there is one.
func (c *Container) MappedPort(internal int) (int, bool) {
	for _, m := range c.container.NetworkSettings.PortMappingAPI() {
		if int(m.PrivatePort) == internal {
			return int(m.PublicPort), true
		}
	}
	return -1, false
}

// Start the container (and notify it if c.Closing falls)
func (c *Container) Start() error {
	hc := &docker.HostConfig{
		PublishAllPorts: true,
		Binds:           makeBinds(c.Volumes),
	}
	err := c.client.StartContainer(c.container.ID, hc)
	if err != nil {
		return err
	}

	// Load container.NetworkSettings
	c.container, err = c.client.InspectContainer(c.container.ID)
	if err != nil {
		return err
	}

	// Listen on the Closing barrier and send a kill to the container if it
	// falls.
	c.wg.Add(1)
	go func() {
		defer c.wg.Done()

		<-c.Closing.Barrier()
		// If the container is signaled to close, send a kill signal
		err := c.client.KillContainer(docker.KillContainerOptions{
			ID: c.container.ID,
		})
		if err == nil {
			return
		}
		switch err := err.(type) {
		case *docker.NoSuchContainer:
			// The container already went away, who cares.
			return
		default:
			log.Println("Killing container failed:", c.container.ID, err)
		}
	}()
	return nil
}

// Wait until container exits
func (c *Container) Wait() (int, error) {
	return c.client.WaitContainer(c.container.ID)
}

// Internal function for raising an error.
func (c *Container) err(err error) {
	c.errorsW <- err
	c.Closing.Fall()
}

// Manage the whole lifecycle of the container in response to a request to
// start it.
func (c *Container) Run(imageSource source.ImageSource, payload []byte) (int, error) {

	defer c.Closing.Fall()
	defer close(c.errorsW)

	go func() {
		for err := range c.Errors {
			log.Println("BUG: Async container error:", err)
			// TODO(pwaller): If this case is hit we might not want to
			// tear the container down really.
			c.Failed.Fall()
		}
	}()

	imageName, err := imageSource.Obtain(c.client, payload)
	c.Obtained.Fall()
	if err != nil {
		c.Failed.Fall()
		return -2, err
	}

	err = c.Create(imageName)
	if err != nil {
		c.Failed.Fall()
		return -1, err
	}
	defer c.Delete()

	err = c.Start()
	if err != nil {
		c.Failed.Fall()
		return -1, err
	}

	// Must come after container start has succeeded, otherwise we end up
	// perpetually attached if it fails to succeed, which blocks program exit.
	// Program exit must be blocked ordinarily until this completes so that
	// if we are quitting we see all of the messages sent by the container
	// until it quit.
	c.wg.Add(1)
	go func() {
		defer c.wg.Done()

		err := c.CopyOutput()
		if err != nil {
			c.err(err)
		}
	}()

	go func() {
		if err := c.AwaitListening(); err != nil {
			log.Printf("AwaitListening failed: %v", err)
			c.Failed.Fall()
			return
		}
		c.Ready.Fall()
	}()

	return c.Wait()
}

func (c *Container) Delete() {
	err := c.client.RemoveContainer(docker.RemoveContainerOptions{
		ID:            c.container.ID,
		RemoveVolumes: true,
		Force:         true,
	})
	if err != nil {
		log.Println("Warn: failed to delete container:", err)
	}
}
<commit_msg>Acknowledge success<commit_after>package main

import (
	"fmt"
	"log"
	"net/http"
	"net/url"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/fsouza/go-dockerclient"
	"github.com/pwaller/barrier"

	"github.com/scraperwiki/hanoverd/pkg/source"
)

type Container struct {
	Name      string
	ImageName string
	Args, Env []string
	Volumes   []string
	StatusURI string

	client    *docker.Client
	container *docker.Container

	Failed, Superceded, Obtained, Ready, Closing barrier.Barrier

	wg *sync.WaitGroup

	Errors  <-chan error
	errorsW chan<- error
}

// Construct a *Container. When the `wg` WaitGroup is zero, there is nothing
// outstanding (such as firewall rules which need garbage collecting).
func NewContainer(client *docker.Client, name string, wg *sync.WaitGroup) *Container {

	errors := make(chan error)

	c := &Container{
		Name:    name,
		client:  client,
		wg:      wg,
		Errors:  errors,
		errorsW: errors,
	}

	// If the container fails we should assume it should be torn down.
	c.Failed.Forward(&c.Closing)

	return c
}

func makeVolumeSet(in []string) map[string]struct{} {
	volumes := map[string]struct{}{}
	for _, v := range in {
		if strings.Contains(v, ":") {
			continue
		}
		volumes[v] = struct{}{}
	}
	return volumes
}

func makeBinds(in []string) []string {
	binds := []string{}
	for _, v := range in {
		if !strings.Contains(v, ":") {
			continue
		}
		binds = append(binds, v)
	}
	return binds
}

// `docker create` the container.
func (c *Container) Create(imageName string) error {
	opts := docker.CreateContainerOptions{
		Config: &docker.Config{
			Hostname:     c.Name,
			AttachStdout: true,
			AttachStderr: true,
			Env:          c.Env,
			Cmd:          c.Args,
			Image:        imageName,
			Volumes:      makeVolumeSet(c.Volumes),
			Labels: map[string]string{
				"orchestrator":  "hanoverd",
				"hanoverd-name": c.Name,
			},
		},
	}

	var err error
	c.container, err = c.client.CreateContainer(opts)
	return err
}

// CopyOutput copies the output of the container to `w` and blocks until
// completion
func (c *Container) CopyOutput() error {

	// TODO(pwaller): at some point move this on to 'c' for configurability?
	w := os.Stderr
	// Blocks until stream closed
	return c.client.AttachToContainer(docker.AttachToContainerOptions{
		Container:    c.container.ID,
		OutputStream: w,
		ErrorStream:  w,
		Logs:         true,
		Stdout:       true,
		Stderr:       true,
		Stream:       true,
	})
}

// Poll for the program inside the container being ready to accept connections
// Returns `true` for success and `false` for failure.
func (c *Container) AwaitListening() error {

	if len(c.container.NetworkSettings.PortMappingAPI()) == 0 {
		return fmt.Errorf("no ports are exposed (specify EXPOSE in Dockerfile)")
	}

	const (
		DefaultTimeout = 5 * time.Minute
		PollFrequency  = 5 // approx. times per second.
	)

	success := make(chan chan struct{}, len(c.container.NetworkSettings.PortMappingAPI()))
	finished := make(chan struct{})
	defer close(finished)

	// Poll the statusURL once.
	// Returns true if polling should continue and false otherwise.
	poll := func(statusURL string) bool {
		req, err := http.NewRequest("GET", statusURL, nil)
		if err != nil {
			log.Printf("Warning, malformed URL: %q: %v", statusURL, err)
			return false
		}
		req.Cancel = finished

		resp, err := http.DefaultClient.Do(req)
		if resp != nil && resp.Body != nil {
			// Don't care about the body, make sure we close it.
			resp.Body.Close()
		}

		if urlErr, ok := err.(*url.Error); ok {
			errStr := urlErr.Err.Error()
			if strings.Contains(errStr, "malformed HTTP response") {
				// Seen in case endpoint doesn't speak HTTP. Give up.
				return false
			}
		}

		if resp == nil {
			// Keep going, connection probably failed.
			return true
		}
		switch resp.StatusCode {
		case http.StatusOK:
			// Protocol: poll() must not return before success
			// has been acknowledged, otherwise we may hit
			// noPollersRemain.
			response := make(chan struct{})
			success <- response
			<-response
			return false

		default:
			log.Printf("Status poller got non-200 status: %q returned %v",
				statusURL, resp.Status)
			return false
		}

		return true
	}

	var pollers sync.WaitGroup

	// Start one poller per exposed port.
	for _, port := range c.container.NetworkSettings.PortMappingAPI() {
		statusURL := fmt.Sprint("http://", port.IP, ":", port.PublicPort, c.StatusURI)

		c.wg.Add(1)
		pollers.Add(1)
		go func() {
			defer c.wg.Done()
			defer pollers.Done()

			// Poll until:
			// * 200 status code
			// * malformed response
			// * teardown
			for poll(statusURL) {
				select {
				case <-finished:
					return
				case <-time.After(time.Second / PollFrequency):
				}
			}
		}()
	}

	noPollersRemain := make(chan struct{})
	go func() {
		defer close(noPollersRemain)
		pollers.Wait()
	}()

	select {
	case ack := <-success:
		ack <- struct{}{}
		return nil

	case <-noPollersRemain:
		return fmt.Errorf("no status checks succeeded")

	case <-c.Closing.Barrier():
		return fmt.Errorf("shutting down")

	case <-time.After(DefaultTimeout):
		return fmt.Errorf("took longer than %v to start, giving up", DefaultTimeout)
	}
}

// Given an internal port, return the port mapped by docker, if there is one.
func (c *Container) MappedPort(internal int) (int, bool) {
	for _, m := range c.container.NetworkSettings.PortMappingAPI() {
		if int(m.PrivatePort) == internal {
			return int(m.PublicPort), true
		}
	}
	return -1, false
}

// Start the container (and notify it if c.Closing falls)
func (c *Container) Start() error {
	hc := &docker.HostConfig{
		PublishAllPorts: true,
		Binds:           makeBinds(c.Volumes),
	}
	err := c.client.StartContainer(c.container.ID, hc)
	if err != nil {
		return err
	}

	// Load container.NetworkSettings
	c.container, err = c.client.InspectContainer(c.container.ID)
	if err != nil {
		return err
	}

	// Listen on the Closing barrier and send a kill to the container if it
	// falls.
	c.wg.Add(1)
	go func() {
		defer c.wg.Done()

		<-c.Closing.Barrier()
		// If the container is signaled to close, send a kill signal
		err := c.client.KillContainer(docker.KillContainerOptions{
			ID: c.container.ID,
		})
		if err == nil {
			return
		}
		switch err := err.(type) {
		case *docker.NoSuchContainer:
			// The container already went away, who cares.
			return
		default:
			log.Println("Killing container failed:", c.container.ID, err)
		}
	}()
	return nil
}

// Wait until container exits
func (c *Container) Wait() (int, error) {
	return c.client.WaitContainer(c.container.ID)
}

// Internal function for raising an error.
func (c *Container) err(err error) {
	c.errorsW <- err
	c.Closing.Fall()
}

// Manage the whole lifecycle of the container in response to a request to
// start it.
func (c *Container) Run(imageSource source.ImageSource, payload []byte) (int, error) {

	defer c.Closing.Fall()
	defer close(c.errorsW)

	go func() {
		for err := range c.Errors {
			log.Println("BUG: Async container error:", err)
			// TODO(pwaller): If this case is hit we might not want to
			// tear the container down really.
			c.Failed.Fall()
		}
	}()

	imageName, err := imageSource.Obtain(c.client, payload)
	c.Obtained.Fall()
	if err != nil {
		c.Failed.Fall()
		return -2, err
	}

	err = c.Create(imageName)
	if err != nil {
		c.Failed.Fall()
		return -1, err
	}
	defer c.Delete()

	err = c.Start()
	if err != nil {
		c.Failed.Fall()
		return -1, err
	}

	// Must come after container start has succeeded, otherwise we end up
	// perpetually attached if it fails to succeed, which blocks program exit.
	// Program exit must be blocked ordinarily until this completes so that
	// if we are quitting we see all of the messages sent by the container
	// until it quit.
	c.wg.Add(1)
	go func() {
		defer c.wg.Done()

		err := c.CopyOutput()
		if err != nil {
			c.err(err)
		}
	}()

	go func() {
		if err := c.AwaitListening(); err != nil {
			log.Printf("AwaitListening failed: %v", err)
			c.Failed.Fall()
			return
		}
		c.Ready.Fall()
	}()

	return c.Wait()
}

func (c *Container) Delete() {
	err := c.client.RemoveContainer(docker.RemoveContainerOptions{
		ID:            c.container.ID,
		RemoveVolumes: true,
		Force:         true,
	})
	if err != nil {
		log.Println("Warn: failed to delete container:", err)
	}
}
<|endoftext|>


<commit_before>package engi

type Entity struct {
	id         string
	components []Component
	requires   []string
}

func NewEntity(requires []string) *Entity {
	return &Entity{requires: requires}
}

func (e *Entity) DoesRequire(name string) bool {
	for _, requirement := range e.requires {
		if requirement == name {
			return true
		}
	}

	return false
}

func (e *Entity) AddComponent(component Component) {
	e.components = append(e.components, component)
}

func (e *Entity) GetComponent(name string) Component {
	for _, component := range e.components {
		if component.Name() == name {
			return component
		}
	}
	return nil
}

func (e *Entity) ID() string {
	return e.id
}
<commit_msg>Change Entity.requires into a map for faster lookups.<commit_after>package engi

type Entity struct {
	id         string
	components []Component
	requires   map[string]bool
}

func NewEntity(requires []string) *Entity {
	e := &Entity{requires: make(map[string]bool)}
	for _, req := range requires {
		e.requires[req] = true
	}
	return e
}

func (e *Entity) DoesRequire(name string) bool { return e.requires[name] }

func (e *Entity) AddComponent(component Component) {
	e.components = append(e.components, component)
}

func (e *Entity) GetComponent(name string) Component {
	for _, component := range e.components {
		if component.Name() == name {
			return component
		}
	}
	return nil
}

func (e *Entity) ID() string {
	return e.id
}
<|endoftext|>