<commit_before>package geom

import (
	"reflect"
	"testing"
)

type testLinearRing struct {
	layout     Layout
	stride     int
	coords     []Coord
	flatCoords []float64
	bounds     *Bounds
}

func testLinearRingEquals(t *testing.T, lr *LinearRing, tlr *testLinearRing) {
	if err := lr.verify(); err != nil {
		t.Error(err)
	}
	if lr.Layout() != tlr.layout {
		t.Errorf("lr.Layout() == %v, want %v", lr.Layout(), tlr.layout)
	}
	if lr.Stride() != tlr.stride {
		t.Errorf("lr.Stride() == %v, want %v", lr.Stride(), tlr.stride)
	}
	if !reflect.DeepEqual(lr.Coords(), tlr.coords) {
		t.Errorf("lr.Coords() == %v, want %v", lr.Coords(), tlr.coords)
	}
	if !reflect.DeepEqual(lr.FlatCoords(), tlr.flatCoords) {
		t.Errorf("lr.FlatCoords() == %v, want %v", lr.FlatCoords(), tlr.flatCoords)
	}
	if !reflect.DeepEqual(lr.Bounds(), tlr.bounds) {
		t.Errorf("lr.Bounds() == %v, want %v", lr.Bounds(), tlr.bounds)
	}
	if got := lr.NumCoords(); got != len(tlr.coords) {
		t.Errorf("lr.NumCoords() == %v, want %v", got, len(tlr.coords))
	}
	for i, c := range tlr.coords {
		if !reflect.DeepEqual(lr.Coord(i), c) {
			t.Errorf("lr.Coord(%v) == %v, want %v", i, lr.Coord(i), c)
		}
	}
}

func TestLinearRing(t *testing.T) {
	for _, c := range []struct {
		lr  *LinearRing
		tlr *testLinearRing
	}{
		{
			lr: NewLinearRing(XY).MustSetCoords([]Coord{{1, 2}, {3, 4}, {5, 6}}),
			tlr: &testLinearRing{
				layout:     XY,
				stride:     2,
				coords:     []Coord{{1, 2}, {3, 4}, {5, 6}},
				flatCoords: []float64{1, 2, 3, 4, 5, 6},
				bounds:     NewBounds(XY).Set(1, 2, 5, 6),
			},
		},
		{
			lr: NewLinearRing(XYZ).MustSetCoords([]Coord{{1, 2, 3}, {4, 5, 6}, {7, 8, 9}}),
			tlr: &testLinearRing{
				layout:     XYZ,
				stride:     3,
				coords:     []Coord{{1, 2, 3}, {4, 5, 6}, {7, 8, 9}},
				flatCoords: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9},
				bounds:     NewBounds(XYZ).Set(1, 2, 3, 7, 8, 9),
			},
		},
		{
			lr: NewLinearRing(XYM).MustSetCoords([]Coord{{1, 2, 3}, {4, 5, 6}, {7, 8, 9}}),
			tlr: &testLinearRing{
				layout:     XYM,
				stride:     3,
				coords:     []Coord{{1, 2, 3}, {4, 5, 6}, {7, 8, 9}},
				flatCoords: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9},
				bounds:     NewBounds(XYM).Set(1, 2, 3, 7, 8, 9),
			},
		},
		{
			lr: NewLinearRing(XYZM).MustSetCoords([]Coord{{1, 2, 3, 4}, {5, 6, 7, 8}, {9, 10, 11, 12}}),
			tlr: &testLinearRing{
				layout:     XYZM,
				stride:     4,
				coords:     []Coord{{1, 2, 3, 4}, {5, 6, 7, 8}, {9, 10, 11, 12}},
				flatCoords: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12},
				bounds:     NewBounds(XYZM).Set(1, 2, 3, 4, 9, 10, 11, 12),
			},
		},
	} {

		testLinearRingEquals(t, c.lr, c.tlr)
	}
}

func TestLinearRingClone(t *testing.T) {
	p1 := NewLinearRing(XY).MustSetCoords([]Coord{{1, 2}, {3, 4}, {5, 6}})
	if p2 := p1.Clone(); aliases(p1.FlatCoords(), p2.FlatCoords()) {
		t.Error("Clone() should not alias flatCoords")
	}
}

func TestLinearRingStrideMismatch(t *testing.T) {
	for _, c := range []struct {
		layout Layout
		coords []Coord
		err    error
	}{
		{
			layout: XY,
			coords: nil,
			err:    nil,
		},
		{
			layout: XY,
			coords: []Coord{},
			err:    nil,
		},
		{
			layout: XY,
			coords: []Coord{{1, 2}, {}},
			err:    ErrStrideMismatch{Got: 0, Want: 2},
		},
		{
			layout: XY,
			coords: []Coord{{1, 2}, {1}},
			err:    ErrStrideMismatch{Got: 1, Want: 2},
		},
		{
			layout: XY,
			coords: []Coord{{1, 2}, {3, 4}},
			err:    nil,
		},
		{
			layout: XY,
			coords: []Coord{{1, 2}, {3, 4, 5}},
			err:    ErrStrideMismatch{Got: 3, Want: 2},
		},
	} {
		p := NewLinearRing(c.layout)
		if _, err := p.SetCoords(c.coords); err != c.err {
			t.Errorf("p.SetCoords(%v) == %v, want %v", c.coords, err, c.err)
		}
	}
}
<commit_msg>Remove stray empty line<commit_after>package geom

import (
	"reflect"
	"testing"
)

type testLinearRing struct {
	layout     Layout
	stride     int
	coords     []Coord
	flatCoords []float64
	bounds     *Bounds
}

func testLinearRingEquals(t *testing.T, lr *LinearRing, tlr *testLinearRing) {
	if err := lr.verify(); err != nil {
		t.Error(err)
	}
	if lr.Layout() != tlr.layout {
		t.Errorf("lr.Layout() == %v, want %v", lr.Layout(), tlr.layout)
	}
	if lr.Stride() != tlr.stride {
		t.Errorf("lr.Stride() == %v, want %v", lr.Stride(), tlr.stride)
	}
	if !reflect.DeepEqual(lr.Coords(), tlr.coords) {
		t.Errorf("lr.Coords() == %v, want %v", lr.Coords(), tlr.coords)
	}
	if !reflect.DeepEqual(lr.FlatCoords(), tlr.flatCoords) {
		t.Errorf("lr.FlatCoords() == %v, want %v", lr.FlatCoords(), tlr.flatCoords)
	}
	if !reflect.DeepEqual(lr.Bounds(), tlr.bounds) {
		t.Errorf("lr.Bounds() == %v, want %v", lr.Bounds(), tlr.bounds)
	}
	if got := lr.NumCoords(); got != len(tlr.coords) {
		t.Errorf("lr.NumCoords() == %v, want %v", got, len(tlr.coords))
	}
	for i, c := range tlr.coords {
		if !reflect.DeepEqual(lr.Coord(i), c) {
			t.Errorf("lr.Coord(%v) == %v, want %v", i, lr.Coord(i), c)
		}
	}
}

func TestLinearRing(t *testing.T) {
	for _, c := range []struct {
		lr  *LinearRing
		tlr *testLinearRing
	}{
		{
			lr: NewLinearRing(XY).MustSetCoords([]Coord{{1, 2}, {3, 4}, {5, 6}}),
			tlr: &testLinearRing{
				layout:     XY,
				stride:     2,
				coords:     []Coord{{1, 2}, {3, 4}, {5, 6}},
				flatCoords: []float64{1, 2, 3, 4, 5, 6},
				bounds:     NewBounds(XY).Set(1, 2, 5, 6),
			},
		},
		{
			lr: NewLinearRing(XYZ).MustSetCoords([]Coord{{1, 2, 3}, {4, 5, 6}, {7, 8, 9}}),
			tlr: &testLinearRing{
				layout:     XYZ,
				stride:     3,
				coords:     []Coord{{1, 2, 3}, {4, 5, 6}, {7, 8, 9}},
				flatCoords: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9},
				bounds:     NewBounds(XYZ).Set(1, 2, 3, 7, 8, 9),
			},
		},
		{
			lr: NewLinearRing(XYM).MustSetCoords([]Coord{{1, 2, 3}, {4, 5, 6}, {7, 8, 9}}),
			tlr: &testLinearRing{
				layout:     XYM,
				stride:     3,
				coords:     []Coord{{1, 2, 3}, {4, 5, 6}, {7, 8, 9}},
				flatCoords: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9},
				bounds:     NewBounds(XYM).Set(1, 2, 3, 7, 8, 9),
			},
		},
		{
			lr: NewLinearRing(XYZM).MustSetCoords([]Coord{{1, 2, 3, 4}, {5, 6, 7, 8}, {9, 10, 11, 12}}),
			tlr: &testLinearRing{
				layout:     XYZM,
				stride:     4,
				coords:     []Coord{{1, 2, 3, 4}, {5, 6, 7, 8}, {9, 10, 11, 12}},
				flatCoords: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12},
				bounds:     NewBounds(XYZM).Set(1, 2, 3, 4, 9, 10, 11, 12),
			},
		},
	} {
		testLinearRingEquals(t, c.lr, c.tlr)
	}
}

func TestLinearRingClone(t *testing.T) {
	p1 := NewLinearRing(XY).MustSetCoords([]Coord{{1, 2}, {3, 4}, {5, 6}})
	if p2 := p1.Clone(); aliases(p1.FlatCoords(), p2.FlatCoords()) {
		t.Error("Clone() should not alias flatCoords")
	}
}

func TestLinearRingStrideMismatch(t *testing.T) {
	for _, c := range []struct {
		layout Layout
		coords []Coord
		err    error
	}{
		{
			layout: XY,
			coords: nil,
			err:    nil,
		},
		{
			layout: XY,
			coords: []Coord{},
			err:    nil,
		},
		{
			layout: XY,
			coords: []Coord{{1, 2}, {}},
			err:    ErrStrideMismatch{Got: 0, Want: 2},
		},
		{
			layout: XY,
			coords: []Coord{{1, 2}, {1}},
			err:    ErrStrideMismatch{Got: 1, Want: 2},
		},
		{
			layout: XY,
			coords: []Coord{{1, 2}, {3, 4}},
			err:    nil,
		},
		{
			layout: XY,
			coords: []Coord{{1, 2}, {3, 4, 5}},
			err:    ErrStrideMismatch{Got: 3, Want: 2},
		},
	} {
		p := NewLinearRing(c.layout)
		if _, err := p.SetCoords(c.coords); err != c.err {
			t.Errorf("p.SetCoords(%v) == %v, want %v", c.coords, err, c.err)
		}
	}
}
<|endoftext|>


<commit_before>// Package octets contains consts and methods to simplify dealing with
// file sizes
package octets

// List of all different supported type
const (
	Byte = 1.0 << (10 * iota)
	KiloByte
	MegaByte
	GigaByte
	TeraByte
	PetaByte
	ExaByte

	B  = Byte
	KB = KiloByte
	MB = MegaByte
	GB = GigaByte
	TB = TeraByte
	PB = PetaByte
	EB = ExaByte

	Octet     = Byte
	KiloOctet = KiloByte
	MegaOctet = MegaByte
	GigaOctet = GigaByte
	TeraOctet = GigaByte
	PetaOctet = PetaByte
	ExaOctet  = ExaByte

	O  = Octet
	Ko = KiloOctet
	Mo = MegaOctet
	Go = GigaOctet
	To = TeraOctet
	Po = PetaOctet
	Eo = ExaOctet
)
<commit_msg>refactor(octets):use int64 instead of the default float type<commit_after>// Package octets contains consts and methods to simplify dealing with
// file sizes
package octets

// List of all different supported type
const (
	Byte int64 = 1.0 << (10 * iota)
	KiloByte
	MegaByte
	GigaByte
	TeraByte
	PetaByte
	ExaByte

	B  int64 = Byte
	KB int64 = KiloByte
	MB int64 = MegaByte
	GB int64 = GigaByte
	TB int64 = TeraByte
	PB int64 = PetaByte
	EB int64 = ExaByte

	Octet     int64 = Byte
	KiloOctet int64 = KiloByte
	MegaOctet int64 = MegaByte
	GigaOctet int64 = GigaByte
	TeraOctet int64 = GigaByte
	PetaOctet int64 = PetaByte
	ExaOctet  int64 = ExaByte

	O  int64 = Octet
	Ko int64 = KiloOctet
	Mo int64 = MegaOctet
	Go int64 = GigaOctet
	To int64 = TeraOctet
	Po int64 = PetaOctet
	Eo int64 = ExaOctet
)
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"strconv"

	"github.com/bjyoungblood/gozw/zwave/application"
	"github.com/bjyoungblood/gozw/zwave/frame"
	"github.com/bjyoungblood/gozw/zwave/serial-api"
	"github.com/bjyoungblood/gozw/zwave/session"
	"github.com/bjyoungblood/gozw/zwave/transport"
	"github.com/davecgh/go-spew/spew"
	"github.com/peterh/liner"
)

func main() {
	transport, err := transport.NewSerialTransportLayer("/tmp/usbmodem", 115200)
	if err != nil {
		panic(err)
	}

	frameLayer := frame.NewFrameLayer(transport)
	sessionLayer := session.NewSessionLayer(frameLayer)
	apiLayer := serialapi.NewSerialAPILayer(sessionLayer)
	appLayer, err := application.NewApplicationLayer(apiLayer)
	if err != nil {
		panic(err)
	}

	// spew.Dump(applicationLayer.Nodes)

	// apiLayer.SoftReset()

	// spew.Dump(apiLayer.GetVersion())
	// nodeList, err := apiLayer.GetNodeList()
	// fmt.Println(nodeList.GetNodeIds())
	// spew.Dump(apiLayer.MemoryGetId())
	// spew.Dump(apiLayer.GetNodeProtocolInfo(27))
	// spew.Dump(apiLayer.GetSerialApiCapabilities())

	// txTime, err := apiLayer.SendData(27, commandclass.NewVersionGet())
	// fmt.Println("TX: ", txTime)
	// if err != nil {
	// 	fmt.Println(err)
	// }

	// if apiLayer.AddNode() != nil {
	// 	apiLayer.AddNode()
	// }

	// sessionLayer := zwave.NewSessionLayer(frameLayer)
	// manager := zwave.NewManager(sessionLayer)

	// defer manager.Close()

	fmt.Printf("Home ID: 0x%x; Node ID: %d\n", appLayer.HomeId, appLayer.NodeId)
	fmt.Println("API Version:", appLayer.ApiVersion)
	fmt.Println("Library:", appLayer.ApiLibraryType)
	fmt.Println("Version:", appLayer.Version)
	fmt.Println("API Type:", appLayer.ApiType)
	fmt.Println("Is Primary Controller:", appLayer.IsPrimaryController)
	fmt.Println("Node count:", len(appLayer.Nodes()))
	//
	// appLayer.SendDataSecure(42, []byte{
	// 	commandclass.CommandClassDoorLock,
	// 	0x01, // door lock operation set
	// 	0xFF, // unsecured
	// })

	// manager.SetApplicationNodeInformation()
	// manager.FactoryReset()

	for _, node := range appLayer.Nodes() {
		fmt.Println(node.String())
	}

	// manager.SendData(3, cc.NewSwitchMultilevelCommand(0))

	line := liner.NewLiner()
	defer line.Close()

	for {
		cmd, _ := line.Prompt("(a)dd node\n(r)emove node\n(g)et nonce\n(q)uit\n> ")
		switch cmd {
		case "a":
			spew.Dump(appLayer.AddNode())
		case "r":
			spew.Dump(appLayer.RemoveNode())
		case "L":
			input, _ := line.Prompt("node id: ")
			nodeId, _ := strconv.Atoi(input)
			node, err := appLayer.Node(byte(nodeId))
			if err != nil {
				spew.Dump(err)
				continue
			}

			spew.Dump(node.LoadAllUserCodes())
		case "F":
			input, _ := line.Prompt("node id: ")
			nodeId, _ := strconv.Atoi(input)
			spew.Dump(appLayer.RemoveFailedNode(byte(nodeId)))
		// case "s":
		// 	input, _ := line.Prompt("node id: ")
		// 	nodeId, _ := strconv.Atoi(input)
		// 	manager.SendData(uint8(nodeId), commandclass.NewSecuritySchemeGet())
		// case "g":
		// 	input, _ := line.Prompt("node id: ")
		// 	nodeId, _ := strconv.Atoi(input)
		// 	manager.SendData(uint8(nodeId), commandclass.NewSecurityNonceGet())
		// case "v":
		// 	input, _ := line.Prompt("node id: ")
		// 	nodeId, _ := strconv.Atoi(input)
		// 	manager.SendData(uint8(nodeId), commandclass.NewVersionGet())
		case "q":
			appLayer.Shutdown()
			return
		default:
			fmt.Println("invalid selection")
		}
	}

}
<commit_msg>Clean up test cli "repl"<commit_after>package main

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/bjyoungblood/gozw/zwave/application"
	"github.com/bjyoungblood/gozw/zwave/frame"
	"github.com/bjyoungblood/gozw/zwave/serial-api"
	"github.com/bjyoungblood/gozw/zwave/session"
	"github.com/bjyoungblood/gozw/zwave/transport"
	"github.com/davecgh/go-spew/spew"
	"github.com/peterh/liner"
)

func main() {
	transport, err := transport.NewSerialTransportLayer("/tmp/usbmodem", 115200)
	if err != nil {
		panic(err)
	}

	frameLayer := frame.NewFrameLayer(transport)
	sessionLayer := session.NewSessionLayer(frameLayer)
	apiLayer := serialapi.NewSerialAPILayer(sessionLayer)
	appLayer, err := application.NewApplicationLayer(apiLayer)
	if err != nil {
		panic(err)
	}

	defer appLayer.Shutdown()

	line := liner.NewLiner()
	defer line.Close()

	commands := strings.Join([]string{
		"(a)dd node",
		"(r)emove node",
		"(V) load command class versions for node",
		"(L) load all user codes for node",
		"(NIF) request node information frame from node",
		"(F)ailed node removal",
		"(p)rint network info",
		"(q)uit",
	}, "\n")

	fmt.Println(commands)

	for {
		cmd, _ := line.Prompt("> ")
		switch cmd {
		case "a":
			spew.Dump(appLayer.AddNode())
		case "r":
			spew.Dump(appLayer.RemoveNode())
		case "V":
			input, _ := line.Prompt("node id: ")
			nodeId, _ := strconv.Atoi(input)
			node, err := appLayer.Node(byte(nodeId))
			if err != nil {
				spew.Dump(err)
				continue
			}

			spew.Dump(node.LoadCommandClassVersions())
		case "L":
			input, _ := line.Prompt("node id: ")
			nodeId, _ := strconv.Atoi(input)
			node, err := appLayer.Node(byte(nodeId))
			if err != nil {
				spew.Dump(err)
				continue
			}

			spew.Dump(node.LoadAllUserCodes())
		case "NIF":
			input, _ := line.Prompt("node id: ")
			nodeId, _ := strconv.Atoi(input)
			node, _ := appLayer.Node(byte(nodeId))
			spew.Dump(node.RequestNodeInformationFrame())
		case "F":
			input, _ := line.Prompt("node id: ")
			nodeId, _ := strconv.Atoi(input)
			spew.Dump(appLayer.RemoveFailedNode(byte(nodeId)))
		case "p":
			fmt.Printf("Home ID: 0x%x; Node ID: %d\n", appLayer.HomeId, appLayer.NodeId)
			fmt.Println("API Version:", appLayer.ApiVersion)
			fmt.Println("Library:", appLayer.ApiLibraryType)
			fmt.Println("Version:", appLayer.Version)
			fmt.Println("API Type:", appLayer.ApiType)
			fmt.Println("Is Primary Controller:", appLayer.IsPrimaryController)
			fmt.Println("Node count:", len(appLayer.Nodes()))

			for _, node := range appLayer.Nodes() {
				fmt.Println(node.String())
			}
		case "q":
			return
		default:
			fmt.Println("invalid selection\n")
			fmt.Println(commands)
		}
	}

}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"net/http"

	dockercontext "github.com/docker/distribution/context"
	docker "github.com/docker/distribution/registry"

	"code.uber.internal/infra/kraken/agent/agentserver"
	"code.uber.internal/infra/kraken/core"
	"code.uber.internal/infra/kraken/lib/backend"
	"code.uber.internal/infra/kraken/lib/dockerregistry"
	"code.uber.internal/infra/kraken/lib/dockerregistry/transfer"
	"code.uber.internal/infra/kraken/lib/serverset"
	"code.uber.internal/infra/kraken/lib/store"
	"code.uber.internal/infra/kraken/lib/torrent"
	"code.uber.internal/infra/kraken/lib/torrent/announcequeue"
	torrentstorage "code.uber.internal/infra/kraken/lib/torrent/storage"
	"code.uber.internal/infra/kraken/metrics"
	"code.uber.internal/infra/kraken/tracker/announceclient"
	"code.uber.internal/infra/kraken/tracker/metainfoclient"
	"code.uber.internal/infra/kraken/utils/configutil"
	"code.uber.internal/infra/kraken/utils/log"
)

func main() {
	peerIP := flag.String("peer_ip", "", "ip which peer will announce itself as")
	peerPort := flag.Int("peer_port", 0, "port which peer will announce itself as")
	agentServerPort := flag.Int("agent_server_port", 0, "port which agent server will listen on")
	configFile := flag.String("config", "", "Configuration file that has to be loaded from one of UBER_CONFIG_DIR locations")
	zone := flag.String("zone", "", "zone/datacenter name")
	cluster := flag.String("cluster", "", "cluster name (e.g. prod01-sjc1)")

	flag.Parse()

	if agentServerPort == nil || *agentServerPort == 0 {
		panic("must specify non-zero agent server port")
	}

	var config Config
	if err := configutil.Load(*configFile, &config); err != nil {
		panic(err)
	}

	zlog := log.ConfigureLogger(config.ZapLogging)
	defer zlog.Sync()

	pctx, err := core.NewPeerContext(
		core.PeerIDFactory(config.Torrent.PeerIDFactory), *zone, *peerIP, *peerPort, false)
	if err != nil {
		log.Fatalf("Failed to create peer context: %s", err)
	}

	stats, closer, err := metrics.New(config.Metrics, *cluster)
	if err != nil {
		log.Fatalf("Failed to init metrics: %s", err)
	}
	defer closer.Close()

	trackers, err := serverset.NewRoundRobin(config.Tracker.RoundRobin)
	if err != nil {
		log.Fatalf("Error creating tracker round robin: %s", err)
	}

	fs, err := store.NewLocalFileStore(config.Store, stats, config.Registry.TagDeletion.Enable)
	if err != nil {
		log.Fatalf("Failed to create local store: %s", err)
	}
	archive := torrentstorage.NewAgentTorrentArchive(
		config.AgentTorrentArchive, stats, fs, metainfoclient.Default(trackers))

	torrentClient, err := torrent.NewSchedulerClient(
		config.Torrent,
		stats,
		pctx,
		announceclient.New(pctx, trackers),
		announcequeue.New(),
		archive)
	if err != nil {
		log.Fatalf("Failed to create scheduler client: %s", err)
		panic(err)
	}
	defer torrentClient.Close()

	backendManager, err := backend.NewManager(config.Registry.Namespaces, config.AuthNamespaces)
	if err != nil {
		log.Fatalf("Error creating backend manager: %s", err)
	}
	tagClient, err := backendManager.GetClient(config.Registry.TagNamespace)
	if err != nil {
		log.Fatalf("Error creating backend tag client: %s", err)
	}
	transferer := transfer.NewAgentTransferer(
		fs, tagClient, config.Registry.BlobNamespace, torrentClient)

	dockerConfig := config.Registry.CreateDockerConfig(dockerregistry.Name, transferer, fs, stats)
	registry, err := docker.NewRegistry(dockercontext.Background(), dockerConfig)
	if err != nil {
		log.Fatalf("Failed to init registry: %s", err)
	}

	agentServer := agentserver.New(config.AgentServer, stats, fs, torrentClient)
	addr := fmt.Sprintf(":%d", *agentServerPort)
	log.Infof("Starting agent server on %s", addr)
	go func() {
		log.Fatal(http.ListenAndServe(addr, agentServer.Handler()))
	}()

	log.Info("Starting registry...")
	go func() {
		log.Fatal(registry.ListenAndServe())
	}()

	select {}
}
<commit_msg>Add agent heartbeat<commit_after>package main

import (
	"flag"
	"fmt"
	"net/http"
	"time"

	dockercontext "github.com/docker/distribution/context"
	docker "github.com/docker/distribution/registry"
	"github.com/uber-go/tally"

	"code.uber.internal/infra/kraken/agent/agentserver"
	"code.uber.internal/infra/kraken/core"
	"code.uber.internal/infra/kraken/lib/backend"
	"code.uber.internal/infra/kraken/lib/dockerregistry"
	"code.uber.internal/infra/kraken/lib/dockerregistry/transfer"
	"code.uber.internal/infra/kraken/lib/serverset"
	"code.uber.internal/infra/kraken/lib/store"
	"code.uber.internal/infra/kraken/lib/torrent"
	"code.uber.internal/infra/kraken/lib/torrent/announcequeue"
	torrentstorage "code.uber.internal/infra/kraken/lib/torrent/storage"
	"code.uber.internal/infra/kraken/metrics"
	"code.uber.internal/infra/kraken/tracker/announceclient"
	"code.uber.internal/infra/kraken/tracker/metainfoclient"
	"code.uber.internal/infra/kraken/utils/configutil"
	"code.uber.internal/infra/kraken/utils/log"
)

// heartbeat periodically emits a counter metric which allows us to monitor the
// number of active agents.
func heartbeat(stats tally.Scope) {
	for {
		stats.Counter("heartbeat").Inc(1)
		time.Sleep(10 * time.Second)
	}
}

func main() {
	peerIP := flag.String("peer_ip", "", "ip which peer will announce itself as")
	peerPort := flag.Int("peer_port", 0, "port which peer will announce itself as")
	agentServerPort := flag.Int("agent_server_port", 0, "port which agent server will listen on")
	configFile := flag.String("config", "", "Configuration file that has to be loaded from one of UBER_CONFIG_DIR locations")
	zone := flag.String("zone", "", "zone/datacenter name")
	cluster := flag.String("cluster", "", "cluster name (e.g. prod01-sjc1)")

	flag.Parse()

	if agentServerPort == nil || *agentServerPort == 0 {
		panic("must specify non-zero agent server port")
	}

	var config Config
	if err := configutil.Load(*configFile, &config); err != nil {
		panic(err)
	}

	zlog := log.ConfigureLogger(config.ZapLogging)
	defer zlog.Sync()

	pctx, err := core.NewPeerContext(
		core.PeerIDFactory(config.Torrent.PeerIDFactory), *zone, *peerIP, *peerPort, false)
	if err != nil {
		log.Fatalf("Failed to create peer context: %s", err)
	}

	stats, closer, err := metrics.New(config.Metrics, *cluster)
	if err != nil {
		log.Fatalf("Failed to init metrics: %s", err)
	}
	defer closer.Close()

	trackers, err := serverset.NewRoundRobin(config.Tracker.RoundRobin)
	if err != nil {
		log.Fatalf("Error creating tracker round robin: %s", err)
	}

	fs, err := store.NewLocalFileStore(config.Store, stats, config.Registry.TagDeletion.Enable)
	if err != nil {
		log.Fatalf("Failed to create local store: %s", err)
	}
	archive := torrentstorage.NewAgentTorrentArchive(
		config.AgentTorrentArchive, stats, fs, metainfoclient.Default(trackers))

	torrentClient, err := torrent.NewSchedulerClient(
		config.Torrent,
		stats,
		pctx,
		announceclient.New(pctx, trackers),
		announcequeue.New(),
		archive)
	if err != nil {
		log.Fatalf("Failed to create scheduler client: %s", err)
		panic(err)
	}
	defer torrentClient.Close()

	backendManager, err := backend.NewManager(config.Registry.Namespaces, config.AuthNamespaces)
	if err != nil {
		log.Fatalf("Error creating backend manager: %s", err)
	}
	tagClient, err := backendManager.GetClient(config.Registry.TagNamespace)
	if err != nil {
		log.Fatalf("Error creating backend tag client: %s", err)
	}
	transferer := transfer.NewAgentTransferer(
		fs, tagClient, config.Registry.BlobNamespace, torrentClient)

	dockerConfig := config.Registry.CreateDockerConfig(dockerregistry.Name, transferer, fs, stats)
	registry, err := docker.NewRegistry(dockercontext.Background(), dockerConfig)
	if err != nil {
		log.Fatalf("Failed to init registry: %s", err)
	}

	agentServer := agentserver.New(config.AgentServer, stats, fs, torrentClient)
	addr := fmt.Sprintf(":%d", *agentServerPort)
	log.Infof("Starting agent server on %s", addr)
	go func() {
		log.Fatal(http.ListenAndServe(addr, agentServer.Handler()))
	}()

	log.Info("Starting registry...")
	go func() {
		log.Fatal(registry.ListenAndServe())
	}()

	go heartbeat(stats)

	select {}
}
<|endoftext|>


<commit_before>package migmem

import (
	"bufio"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
)

// HasLibrary checks if a process with a given pid has a certain library.
// To do this we use the /proc/<pid>/maps to know which files are mapped.
// the file format is described in `man proc`
func HasLibrary(pid int, r *regexp.Regexp) (bool, error) {
	path := filepath.Join("/proc", strconv.Itoa(pid), "maps")
	f, err := os.Open(path)
	if err != nil {
		return false, err
	}
	defer f.Close()

	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		line := scanner.Text()

		// Just keep the last part of the mapped filename
		// TODO(mvanotti): Probably now that we are using regexp,
		// we may want to do the regexp over the whole filename.
		fields := strings.Split(line, "/")
		if len(fields) <= 1 {
			continue
		}
		library := fields[len(fields)-1]

		if r.MatchString(library) {
			return true, nil
		}
	}

	if err := scanner.Err(); err != nil {
		return false, err
	}

	return false, nil
}

// FindProcWithLib returns a list of process ids that have the given library loaded in memory.
// It works looking at all the pids listed in /proc folder, and for each of them, checking its maps file.
func FindProcWithLib(r *regexp.Regexp) ([]int, error) {
	files, _ := ioutil.ReadDir("/proc/")
	res := make([]int, 0)

	for _, f := range files {
		pid, err := strconv.Atoi(f.Name())
		if err != nil {
			continue
		}

		if has, err := HasLibrary(pid, r); err != nil {
			//TODO(mvanotti): How should we report errors for multiple files? maybe a map[filepath]error ?
			log.Println(err)
			continue
		} else if has {
			res = append(res, pid)
		}
	}

	return res, nil
}
<commit_msg>pids are now uint for all the os's<commit_after>package migmem

import (
	"bufio"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
)

// HasLibrary checks if a process with a given pid has a certain library.
// To do this we use the /proc/<pid>/maps to know which files are mapped.
// the file format is described in `man proc`
func HasLibrary(pid uint, r *regexp.Regexp) (bool, error) {
	path := filepath.Join("/proc", strconv.Itoa(pid), "maps")
	f, err := os.Open(path)
	if err != nil {
		return false, err
	}
	defer f.Close()

	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		line := scanner.Text()

		// Just keep the last part of the mapped filename
		// TODO(mvanotti): Probably now that we are using regexp,
		// we may want to do the regexp over the whole filename.
		fields := strings.Split(line, "/")
		if len(fields) <= 1 {
			continue
		}
		library := fields[len(fields)-1]

		if r.MatchString(library) {
			return true, nil
		}
	}

	if err := scanner.Err(); err != nil {
		return false, err
	}

	return false, nil
}

// FindProcWithLib returns a list of process ids that have the given library loaded in memory.
// It works looking at all the pids listed in /proc folder, and for each of them, checking its maps file.
func FindProcWithLib(r *regexp.Regexp) ([]uint, error) {
	files, _ := ioutil.ReadDir("/proc/")
	res := make([]int, 0)

	for _, f := range files {
		pid, err := strconv.Atoi(f.Name())
		if err != nil {
			continue
		}

		if has, err := HasLibrary(uint(pid), r); err != nil {
			//TODO(mvanotti): How should we report errors for multiple files? maybe a map[filepath]error ?
			log.Println(err)
			continue
		} else if has {
			res = append(res, uint(pid))
		}
	}

	return res, nil
}
<|endoftext|>


<commit_before>package ai

import (
	"bytes"
	"log"

	"nelhage.com/tak/ptn"
	"nelhage.com/tak/tak"
)

const (
	maxEval int64 = 1 << 30
	minEval       = -maxEval
)

type MinimaxAI struct {
	depth int

	Debug bool
}

func formatpv(ms []tak.Move) string {
	var out bytes.Buffer
	out.WriteString("[")
	for i, m := range ms {
		if i != 0 {
			out.WriteString(" ")
		}
		out.WriteString(ptn.FormatMove(&m))
	}
	out.WriteString("]")
	return out.String()
}

func (m *MinimaxAI) GetMove(p *tak.Position) tak.Move {
	ms, _ := m.Analyze(p)
	return ms[0]
}

func (m *MinimaxAI) Analyze(p *tak.Position) ([]tak.Move, int64) {
	var ms []tak.Move
	var v int64
	for i := 1; i <= m.depth; i++ {
		ms, v = m.minimax(p, i, ms, minEval-1, maxEval+1)
		if m.Debug {
			log.Printf("[minimax] depth=%d val=%d pv=%s",
				i, v, formatpv(ms))
		}
	}
	return ms, v
}

func (ai *MinimaxAI) minimax(
	p *tak.Position,
	depth int,
	pv []tak.Move,
	α, β int64) ([]tak.Move, int64) {
	over, _ := p.GameOver()
	if depth == 0 || over {
		return nil, ai.evaluate(p)
	}
	moves := p.AllMoves()
	if len(pv) > 0 {
		for i, m := range moves {
			if m.Equal(&pv[0]) {
				moves[0], moves[i] = moves[i], moves[0]
				break
			}
		}
	}

	best := make([]tak.Move, 1, depth)
	max := minEval - 1
	for _, m := range moves {
		child, e := p.Move(&m)
		if e != nil {
			continue
		}
		ms, v := ai.minimax(child, depth-1, nil, -β, -α)
		v = -v
		if v > max {
			max = v
			best[0] = m
			best = append(best[:1], ms...)
		}
		if v > α {
			α = v
			if α > β {
				break
			}
		}
	}
	return best, max
}

func imin(a, b int) int {
	if a < b {
		return a
	}
	return b
}

func (m *MinimaxAI) evaluate(p *tak.Position) int64 {
	if over, winner := p.GameOver(); over {
		switch winner {
		case tak.NoColor:
			return 0
		case p.ToMove():
			return maxEval - int64(p.MoveNumber())
		default:
			return minEval + int64(p.MoveNumber())
		}
	}
	me, them := 0, 0
	for x := 0; x < p.Size(); x++ {
		for y := 0; y < p.Size(); y++ {
			sq := p.At(x, y)
			if len(sq) == 0 {
				continue
			}
			val := 0
			val += imin(x, p.Size()-x-1)
			val += imin(y, p.Size()-y-1)
			if sq[0].Kind() == tak.Flat {
				if sq[0].Color() == p.ToMove() {
					me += val
				} else {
					them += val
				}
			}
		}
	}
	return int64(me - them)
}

func NewMinimax(depth int) *MinimaxAI {
	return &MinimaxAI{depth: depth}
}
<commit_msg>tweaked eval function<commit_after>package ai

import (
	"bytes"
	"log"

	"nelhage.com/tak/ptn"
	"nelhage.com/tak/tak"
)

const (
	maxEval int64 = 1 << 30
	minEval       = -maxEval
)

type MinimaxAI struct {
	depth int

	Debug bool
}

func formatpv(ms []tak.Move) string {
	var out bytes.Buffer
	out.WriteString("[")
	for i, m := range ms {
		if i != 0 {
			out.WriteString(" ")
		}
		out.WriteString(ptn.FormatMove(&m))
	}
	out.WriteString("]")
	return out.String()
}

func (m *MinimaxAI) GetMove(p *tak.Position) tak.Move {
	ms, _ := m.Analyze(p)
	return ms[0]
}

func (m *MinimaxAI) Analyze(p *tak.Position) ([]tak.Move, int64) {
	var ms []tak.Move
	var v int64
	for i := 1; i <= m.depth; i++ {
		ms, v = m.minimax(p, i, ms, minEval-1, maxEval+1)
		if m.Debug {
			log.Printf("[minimax] depth=%d val=%d pv=%s",
				i, v, formatpv(ms))
		}
	}
	return ms, v
}

func (ai *MinimaxAI) minimax(
	p *tak.Position,
	depth int,
	pv []tak.Move,
	α, β int64) ([]tak.Move, int64) {
	over, _ := p.GameOver()
	if depth == 0 || over {
		return nil, ai.evaluate(p)
	}
	moves := p.AllMoves()
	if len(pv) > 0 {
		for i, m := range moves {
			if m.Equal(&pv[0]) {
				moves[0], moves[i] = moves[i], moves[0]
				break
			}
		}
	}

	best := make([]tak.Move, 1, depth)
	max := minEval - 1
	for _, m := range moves {
		child, e := p.Move(&m)
		if e != nil {
			continue
		}
		ms, v := ai.minimax(child, depth-1, nil, -β, -α)
		v = -v
		if v > max {
			max = v
			best[0] = m
			best = append(best[:1], ms...)
		}
		if v > α {
			α = v
			if α > β {
				break
			}
		}
	}
	return best, max
}

func imin(a, b int) int {
	if a < b {
		return a
	}
	return b
}

const (
	weightFlat       = 1
	weightCaptured   = 1
	weightControlled = 5
)

func (m *MinimaxAI) evaluate(p *tak.Position) int64 {
	if over, winner := p.GameOver(); over {
		switch winner {
		case tak.NoColor:
			return 0
		case p.ToMove():
			return maxEval - int64(p.MoveNumber())
		default:
			return minEval + int64(p.MoveNumber())
		}
	}
	mine, theirs := 0, 0
	me := p.ToMove()
	addw := func(c tak.Color, w int) {
		if c == me {
			mine += w
		} else {
			theirs += w
		}
	}
	for x := 0; x < p.Size(); x++ {
		for y := 0; y < p.Size(); y++ {
			sq := p.At(x, y)
			if len(sq) == 0 {
				continue
			}
			addw(sq[0].Color(), weightControlled)
			for i, stone := range sq {
				if i > 0 && i < p.Size() {
					addw(sq[0].Color(), weightCaptured)
				}
				if stone.Kind() == tak.Flat {
					addw(stone.Color(), weightFlat)
				}
			}
		}
	}

	return int64(mine - theirs)
}

func NewMinimax(depth int) *MinimaxAI {
	return &MinimaxAI{depth: depth}
}
<|endoftext|>


<commit_before>package keytab

import (
	"encoding/hex"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"gopkg.in/jcmturner/gokrb5.v7/test/testdata"
)

func TestUnmarshal(t *testing.T) {
	t.Parallel()
	b, _ := hex.DecodeString(testdata.TESTUSER1_KEYTAB)
	kt := New()
	err := kt.Unmarshal(b)
	if err != nil {
		t.Fatalf("Error parsing keytab data: %v\n", err)
	}
	assert.Equal(t, uint8(2), kt.version, "Keytab version not as expected")
	assert.Equal(t, uint32(1), kt.Entries[0].KVNO, "KVNO not as expected")
	assert.Equal(t, uint8(1), kt.Entries[0].KVNO8, "KVNO8 not as expected")
	assert.Equal(t, time.Unix(1505669592, 0), kt.Entries[0].Timestamp, "Timestamp not as expected")
	assert.Equal(t, int32(17), kt.Entries[0].Key.KeyType, "Key's EType not as expected")
	assert.Equal(t, "698c4df8e9f60e7eea5a21bf4526ad25", hex.EncodeToString(kt.Entries[0].Key.KeyValue), "Key material not as expected")
	assert.Equal(t, int16(1), kt.Entries[0].Principal.NumComponents, "Number of components in principal not as expected")
	assert.Equal(t, int32(1), kt.Entries[0].Principal.NameType, "Name type of principal not as expected")
	assert.Equal(t, "TEST.GOKRB5", kt.Entries[0].Principal.Realm, "Realm of principal not as expected")
	assert.Equal(t, "testuser1", kt.Entries[0].Principal.Components[0], "Component in principal not as expected")
}

func TestMarshal(t *testing.T) {
	t.Parallel()
	b, _ := hex.DecodeString(testdata.TESTUSER1_KEYTAB)
	kt := New()
	err := kt.Unmarshal(b)
	if err != nil {
		t.Fatalf("Error parsing keytab data: %v\n", err)
	}
	mb, err := kt.Marshal()
	if err != nil {
		t.Fatalf("Error marshaling: %v", err)
	}
	assert.Equal(t, b, mb, "Marshaled bytes not the same as input bytes")
	err = kt.Unmarshal(mb)
	if err != nil {
		t.Fatalf("Error parsing marshaled bytes: %v", err)
	}
}

func TestLoad(t *testing.T) {
	t.Parallel()
	kt, err := Load("test/testdata/testuser1.testtab")
	if err != nil {
		t.Fatalf("could not load keytab: %v", err)
	}
	assert.Equal(t, uint8(2), kt.version, "keytab version not as expected")
	assert.Equal(t, 12, len(kt.Entries), "keytab entry count not as expected: %+v", *kt)
	for _, e := range kt.Entries {
		if e.Principal.Realm != "TEST.GOKRB5" {
			t.Error("principal realm not as expected")
		}
		if e.Principal.NameType != int32(1) {
			t.Error("name type not as expected")
		}
		if e.Principal.NumComponents != int16(1) {
			t.Error("number of component not as expected")
		}
		if len(e.Principal.Components) != 1 {
			t.Error("number of component not as expected")
		}
		if e.Principal.Components[0] != "testuser1" {
			t.Error("principal components not as expected")
		}
		if e.Timestamp.IsZero() {
			t.Error("entry timestamp incorrect")
		}
		if e.KVNO == uint32(0) {
			t.Error("entry kvno not as expected")
		}
		if e.KVNO8 == uint8(0) {
			t.Error("entry kvno8 not as expected")
		}
	}
}
<commit_msg>travis build fix<commit_after>package keytab

import (
	"encoding/hex"
	"os"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"gopkg.in/jcmturner/gokrb5.v7/test/testdata"
)

func TestUnmarshal(t *testing.T) {
	t.Parallel()
	b, _ := hex.DecodeString(testdata.TESTUSER1_KEYTAB)
	kt := New()
	err := kt.Unmarshal(b)
	if err != nil {
		t.Fatalf("Error parsing keytab data: %v\n", err)
	}
	assert.Equal(t, uint8(2), kt.version, "Keytab version not as expected")
	assert.Equal(t, uint32(1), kt.Entries[0].KVNO, "KVNO not as expected")
	assert.Equal(t, uint8(1), kt.Entries[0].KVNO8, "KVNO8 not as expected")
	assert.Equal(t, time.Unix(1505669592, 0), kt.Entries[0].Timestamp, "Timestamp not as expected")
	assert.Equal(t, int32(17), kt.Entries[0].Key.KeyType, "Key's EType not as expected")
	assert.Equal(t, "698c4df8e9f60e7eea5a21bf4526ad25", hex.EncodeToString(kt.Entries[0].Key.KeyValue), "Key material not as expected")
	assert.Equal(t, int16(1), kt.Entries[0].Principal.NumComponents, "Number of components in principal not as expected")
	assert.Equal(t, int32(1), kt.Entries[0].Principal.NameType, "Name type of principal not as expected")
	assert.Equal(t, "TEST.GOKRB5", kt.Entries[0].Principal.Realm, "Realm of principal not as expected")
	assert.Equal(t, "testuser1", kt.Entries[0].Principal.Components[0], "Component in principal not as expected")
}

func TestMarshal(t *testing.T) {
	t.Parallel()
	b, _ := hex.DecodeString(testdata.TESTUSER1_KEYTAB)
	kt := New()
	err := kt.Unmarshal(b)
	if err != nil {
		t.Fatalf("Error parsing keytab data: %v\n", err)
	}
	mb, err := kt.Marshal()
	if err != nil {
		t.Fatalf("Error marshaling: %v", err)
	}
	assert.Equal(t, b, mb, "Marshaled bytes not the same as input bytes")
	err = kt.Unmarshal(mb)
	if err != nil {
		t.Fatalf("Error parsing marshaled bytes: %v", err)
	}
}

func TestLoad(t *testing.T) {
	t.Parallel()
	f := "test/testdata/testuser1.testtab"
	dir := os.Getenv("TRAVIS_BUILD_DIR")
	if dir != "" {
		f = dir + "/" + f
	}
	kt, err := Load(f)
	if err != nil {
		t.Fatalf("could not load keytab: %v", err)
	}
	assert.Equal(t, uint8(2), kt.version, "keytab version not as expected")
	assert.Equal(t, 12, len(kt.Entries), "keytab entry count not as expected: %+v", *kt)
	for _, e := range kt.Entries {
		if e.Principal.Realm != "TEST.GOKRB5" {
			t.Error("principal realm not as expected")
		}
		if e.Principal.NameType != int32(1) {
			t.Error("name type not as expected")
		}
		if e.Principal.NumComponents != int16(1) {
			t.Error("number of component not as expected")
		}
		if len(e.Principal.Components) != 1 {
			t.Error("number of component not as expected")
		}
		if e.Principal.Components[0] != "testuser1" {
			t.Error("principal components not as expected")
		}
		if e.Timestamp.IsZero() {
			t.Error("entry timestamp incorrect")
		}
		if e.KVNO == uint32(0) {
			t.Error("entry kvno not as expected")
		}
		if e.KVNO8 == uint8(0) {
			t.Error("entry kvno8 not as expected")
		}
	}
}
<|endoftext|>


<commit_before>package specs

/*
	Helper functions for client repos.
*/

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"reflect"
	"strings"

	"go.skia.org/infra/go/sklog"

	"go.skia.org/infra/go/common"
)

var (
	// Flags.
	test = flag.Bool("test", false, "Run in test mode: verify that the output hasn't changed.")
)

// getCheckoutRoot returns the path of the root of the checkout.
func getCheckoutRoot() (string, error) {
	cwd, err := os.Getwd()
	if err != nil {
		return "", err
	}
	for {
		if _, err := os.Stat(cwd); err != nil {
			return "", err
		}
		// TODO(borenet): Should we verify that this is the
		// correct checkout and not something else?

		// Check for infra/bots dir.
		s, err := os.Stat(path.Join(cwd, "infra", "bots"))
		if err == nil && s.IsDir() {
			return cwd, nil
		}
		// Check for .git dir.
		s, err = os.Stat(path.Join(cwd, ".git"))
		if err == nil && s.IsDir() {
			return cwd, nil
		}

		// Stop if we're at the filesystem root.
		if cwd == string(filepath.Separator) {
			return "", fmt.Errorf("Unable to find repository root.")
		}
		cwd = filepath.Clean(path.Join(cwd, ".."))
	}
}

// TasksCfgBuilder is a helper struct used for building a TasksCfg.
type TasksCfgBuilder struct {
	cfg          *TasksCfg
	cipdPackages map[string]*CipdPackage
	root         string
}

// NewTasksCfgBuilder returns a TasksCfgBuilder instance.
func NewTasksCfgBuilder() (*TasksCfgBuilder, error) {
	common.Init()

	// Create the config.
	cfg := &TasksCfg{
		Jobs:  map[string]*JobSpec{},
		Tasks: map[string]*TaskSpec{},
	}

	root, err := getCheckoutRoot()
	if err != nil {
		return nil, err
	}

	return &TasksCfgBuilder{
		cfg:          cfg,
		cipdPackages: map[string]*CipdPackage{},
		root:         root,
	}, nil
}

// MustNewTasksCfgBuilder returns a TasksCfgBuilder instance. Panics on error.
func MustNewTasksCfgBuilder() *TasksCfgBuilder {
	b, err := NewTasksCfgBuilder()
	if err != nil {
		sklog.Fatal(err)
	}
	return b
}

// CheckoutRoot returns the path to the root of the client checkout.
func (b *TasksCfgBuilder) CheckoutRoot() string {
	return b.root
}

// AddTask adds a TaskSpec to the TasksCfgBuilder. Returns an error if the
// config already contains a Task with the same name and a different
// implementation.
func (b *TasksCfgBuilder) AddTask(name string, t *TaskSpec) error {
	if old, ok := b.cfg.Tasks[name]; ok {
		if !reflect.DeepEqual(old, t) {
			return fmt.Errorf("Config already contains a Task named %q with a different implementation!\nHave:\n%v\n\nGot:\n%v", name, old, t)
		}
		return nil
	}
	b.cfg.Tasks[name] = t
	return nil
}

// MustAddTask adds a TaskSpec to the TasksCfgBuilder and panics on failure.
func (b *TasksCfgBuilder) MustAddTask(name string, t *TaskSpec) {
	if err := b.AddTask(name, t); err != nil {
		sklog.Fatal(err)
	}
}

// AddJob adds a JobSpec to the TasksCfgBuilder.
func (b *TasksCfgBuilder) AddJob(name string, j *JobSpec) error {
	if _, ok := b.cfg.Jobs[name]; ok {
		return fmt.Errorf("Config already contains a Job named %q", name)
	}
	b.cfg.Jobs[name] = j
	return nil
}

// MustAddJob adds a JobSpec to the TasksCfgBuilder and panics on failure.
func (b *TasksCfgBuilder) MustAddJob(name string, j *JobSpec) {
	if err := b.AddJob(name, j); err != nil {
		sklog.Fatal(err)
	}
}

// GetCipdPackageFromAsset reads the version information for the given asset
// and returns a CipdPackage instance.
func (b *TasksCfgBuilder) GetCipdPackageFromAsset(assetName string) (*CipdPackage, error) {
	if pkg, ok := b.cipdPackages[assetName]; ok {
		return pkg, nil
	}
	versionFile := path.Join(b.root, "infra", "bots", "assets", assetName, "VERSION")
	contents, err := ioutil.ReadFile(versionFile)
	if err != nil {
		return nil, err
	}
	version := strings.TrimSpace(string(contents))
	pkg := &CipdPackage{
		Name:    fmt.Sprintf("skia/bots/%s", assetName),
		Path:    assetName,
		Version: fmt.Sprintf("version:%s", version),
	}
	if assetName == "win_toolchain" {
		pkg.Path = "t" // Workaround for path length limit on Windows.
	}
	b.cipdPackages[assetName] = pkg
	return pkg, nil
}

// MustGetCipdPackageFromAsset reads the version information for the given asset
// and returns a CipdPackage instance. Panics on failure.
func (b *TasksCfgBuilder) MustGetCipdPackageFromAsset(assetName string) *CipdPackage {
	pkg, err := b.GetCipdPackageFromAsset(assetName)
	if err != nil {
		sklog.Fatal(err)
	}
	return pkg
}

// Finish validates and writes out the TasksCfg, or, if the --test flag is
// provided, verifies that the contents have not changed.
func (b *TasksCfgBuilder) Finish() error {
	// Validate the config.
	if err := b.cfg.Validate(); err != nil {
		return err
	}

	// Encode the JSON config.
	enc, err := json.MarshalIndent(b.cfg, "", "  ")
	if err != nil {
		return err
	}
	// The json package escapes HTML characters, which makes our output
	// much less readable. Replace the escape characters with the real
	// character.
	enc = bytes.Replace(enc, []byte("\\u003c"), []byte("<"), -1)

	// Add a newline to the end of the file. Most text editors add one, so
	// adding one here enables manual editing of the file, even though we'd
	// rather that not happen.
	enc = append(enc, []byte("\n")...)

	// Write the tasks.json file.
	outFile := path.Join(b.root, TASKS_CFG_FILE)
	if *test {
		// Don't write the file; read it and compare.
		expect, err := ioutil.ReadFile(outFile)
		if err != nil {
			return err
		}
		if !bytes.Equal(expect, enc) {
			return fmt.Errorf("Expected no changes, but changes were found!")
		}
	} else {
		if err := ioutil.WriteFile(outFile, enc, os.ModePerm); err != nil {
			return err
		}
	}
	return nil
}

// MustFinish validates and writes out the TasksCfg, or, if the --test flag is
// provided, verifies that the contents have not changed. Panics on failure.
func (b *TasksCfgBuilder) MustFinish() {
	if err := b.Finish(); err != nil {
		sklog.Fatal(err)
	}
}
<commit_msg>[task scheduler] Specs helpers for gen_tasks.go changes<commit_after>package specs

/*
	Helper functions for client repos.
*/

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"reflect"
	"strings"

	"go.skia.org/infra/go/sklog"

	"go.skia.org/infra/go/common"
)

var (
	// Flags.
	test = flag.Bool("test", false, "Run in test mode: verify that the output hasn't changed.")
)

// GetCheckoutRoot returns the path of the root of the checkout.
func GetCheckoutRoot() (string, error) {
	cwd, err := os.Getwd()
	if err != nil {
		return "", err
	}
	for {
		if _, err := os.Stat(cwd); err != nil {
			return "", err
		}
		// TODO(borenet): Should we verify that this is the
		// correct checkout and not something else?

		// Check for infra/bots dir.
		s, err := os.Stat(path.Join(cwd, "infra", "bots"))
		if err == nil && s.IsDir() {
			return cwd, nil
		}
		// Check for .git dir.
		s, err = os.Stat(path.Join(cwd, ".git"))
		if err == nil && s.IsDir() {
			return cwd, nil
		}

		// Stop if we're at the filesystem root.
		if cwd == string(filepath.Separator) {
			return "", fmt.Errorf("Unable to find repository root.")
		}
		cwd = filepath.Clean(path.Join(cwd, ".."))
	}
}

// TasksCfgBuilder is a helper struct used for building a TasksCfg.
type TasksCfgBuilder struct {
	assetsDir    string
	cfg          *TasksCfg
	cipdPackages map[string]*CipdPackage
	root         string
}

// NewTasksCfgBuilder returns a TasksCfgBuilder instance.
func NewTasksCfgBuilder() (*TasksCfgBuilder, error) {
	common.Init()

	// Create the config.
	cfg := &TasksCfg{
		Jobs:  map[string]*JobSpec{},
		Tasks: map[string]*TaskSpec{},
	}

	root, err := GetCheckoutRoot()
	if err != nil {
		return nil, err
	}

	return &TasksCfgBuilder{
		cfg:          cfg,
		cipdPackages: map[string]*CipdPackage{},
		root:         root,
	}, nil
}

// MustNewTasksCfgBuilder returns a TasksCfgBuilder instance. Panics on error.
func MustNewTasksCfgBuilder() *TasksCfgBuilder {
	b, err := NewTasksCfgBuilder()
	if err != nil {
		sklog.Fatal(err)
	}
	return b
}

// CheckoutRoot returns the path to the root of the client checkout.
func (b *TasksCfgBuilder) CheckoutRoot() string {
	return b.root
}

// SetAssetsDir sets the directory path used for assets.
func (b *TasksCfgBuilder) SetAssetsDir(assetsDir string) {
	b.assetsDir = assetsDir
}

// AddTask adds a TaskSpec to the TasksCfgBuilder. Returns an error if the
// config already contains a Task with the same name and a different
// implementation.
func (b *TasksCfgBuilder) AddTask(name string, t *TaskSpec) error {
	if old, ok := b.cfg.Tasks[name]; ok {
		if !reflect.DeepEqual(old, t) {
			return fmt.Errorf("Config already contains a Task named %q with a different implementation!\nHave:\n%v\n\nGot:\n%v", name, old, t)
		}
		return nil
	}
	b.cfg.Tasks[name] = t
	return nil
}

// MustAddTask adds a TaskSpec to the TasksCfgBuilder and panics on failure.
func (b *TasksCfgBuilder) MustAddTask(name string, t *TaskSpec) {
	if err := b.AddTask(name, t); err != nil {
		sklog.Fatal(err)
	}
}

// AddJob adds a JobSpec to the TasksCfgBuilder.
func (b *TasksCfgBuilder) AddJob(name string, j *JobSpec) error {
	if _, ok := b.cfg.Jobs[name]; ok {
		return fmt.Errorf("Config already contains a Job named %q", name)
	}
	b.cfg.Jobs[name] = j
	return nil
}

// MustAddJob adds a JobSpec to the TasksCfgBuilder and panics on failure.
func (b *TasksCfgBuilder) MustAddJob(name string, j *JobSpec) {
	if err := b.AddJob(name, j); err != nil {
		sklog.Fatal(err)
	}
}

// GetCipdPackageFromAsset reads the version information for the given asset
// and returns a CipdPackage instance.
func (b *TasksCfgBuilder) GetCipdPackageFromAsset(assetName string) (*CipdPackage, error) {
	if pkg, ok := b.cipdPackages[assetName]; ok {
		return pkg, nil
	}
	assetsDir := b.assetsDir
	if assetsDir == "" {
		assetsDir = path.Join(b.root, "infra", "bots", "assets")
	}
	versionFile := path.Join(assetsDir, assetName, "VERSION")
	contents, err := ioutil.ReadFile(versionFile)
	if err != nil {
		return nil, err
	}
	version := strings.TrimSpace(string(contents))
	pkg := &CipdPackage{
		Name:    fmt.Sprintf("skia/bots/%s", assetName),
		Path:    assetName,
		Version: fmt.Sprintf("version:%s", version),
	}
	if assetName == "win_toolchain" {
		pkg.Path = "t" // Workaround for path length limit on Windows.
	}
	b.cipdPackages[assetName] = pkg
	return pkg, nil
}

// MustGetCipdPackageFromAsset reads the version information for the given asset
// and returns a CipdPackage instance. Panics on failure.
func (b *TasksCfgBuilder) MustGetCipdPackageFromAsset(assetName string) *CipdPackage {
	pkg, err := b.GetCipdPackageFromAsset(assetName)
	if err != nil {
		sklog.Fatal(err)
	}
	return pkg
}

// Finish validates and writes out the TasksCfg, or, if the --test flag is
// provided, verifies that the contents have not changed.
func (b *TasksCfgBuilder) Finish() error {
	// Validate the config.
	if err := b.cfg.Validate(); err != nil {
		return err
	}

	// Encode the JSON config.
	enc, err := json.MarshalIndent(b.cfg, "", "  ")
	if err != nil {
		return err
	}
	// The json package escapes HTML characters, which makes our output
	// much less readable. Replace the escape characters with the real
	// character.
	enc = bytes.Replace(enc, []byte("\\u003c"), []byte("<"), -1)

	// Add a newline to the end of the file. Most text editors add one, so
	// adding one here enables manual editing of the file, even though we'd
	// rather that not happen.
	enc = append(enc, []byte("\n")...)

	// Write the tasks.json file.
	outFile := path.Join(b.root, TASKS_CFG_FILE)
	if *test {
		// Don't write the file; read it and compare.
		expect, err := ioutil.ReadFile(outFile)
		if err != nil {
			return err
		}
		if !bytes.Equal(expect, enc) {
			return fmt.Errorf("Expected no changes, but changes were found!")
		}
	} else {
		if err := ioutil.WriteFile(outFile, enc, os.ModePerm); err != nil {
			return err
		}
	}
	return nil
}

// MustFinish validates and writes out the TasksCfg, or, if the --test flag is
// provided, verifies that the contents have not changed. Panics on failure.
func (b *TasksCfgBuilder) MustFinish() {
	if err := b.Finish(); err != nil {
		sklog.Fatal(err)
	}
}
<|endoftext|>


<commit_before><commit_msg>Avoid PID file collisions that prevent restarting the Sauce proxy<commit_after><|endoftext|>


<commit_before>package amald
<commit_msg>Removed amald test<commit_after><|endoftext|>


<commit_before>package septa

var stations = []string{
	"Narberth",
	"Suburban Station",
}

// IsValidStation returns whether the station is a valid station.
func IsValidStation(station string) bool {
	for _, v := range stations {
		if station == v {
			return true
		}
	}
	return false
}
<commit_msg>Added station names<commit_after>package septa

var stations = []string{
	"9th St",
	"30th Street Station",
	"49th St",
	"Airport Terminal A",
	"Airport Terminal B",
	"Airport Terminal C-D",
	"Airport Terminal E-F",
	"Allegheny",
	"Allen Lane",
	"Ambler",
	"Angora",
	"Ardmore",
	"Ardsley",
	"Bala",
	"Berwyn",
	"Bethayres",
	"Bridesburg",
	"Bristol",
	"Bryn Mawr",
	"Carpenter",
	"Chalfont",
	"Chelten Avenue",
	"Cheltenham",
	"Chester TC",
	"Chestnut Hill East",
	"Chestnut Hill West",
	"Churchmans Crossing",
	"Claymont",
	"Clifton-Aldan",
	"Colmar",
	"Conshohocken",
	"Cornwells Heights",
	"Crestmont",
	"Croydon",
	"Crum Lynne",
	"Curtis Park",
	"Cynwyd",
	"Daylesford",
	"Darby",
	"Delaware Valley College",
	"Devon",
	"Downingtown",
	"Doylestown",
	"East Falls",
	"Eastwick Station",
	"Eddington",
	"Eddystone",
	"Elkins Park",
	"Elm St",
	"Elwyn Station",
	"Exton",
	"Fern Rock TC",
	"Fernwood",
	"Folcroft",
	"Forest Hills",
	"Ft Washington",
	"Fortuna",
	"Fox Chase",
	"Germantown",
	"Gladstone",
	"Glenolden",
	"Glenside",
	"Gravers",
	"Gwynedd Valley",
	"Hatboro",
	"Haverford",
	"Highland Ave",
	"Highland",
	"Holmesburg Jct",
	"Ivy Ridge",
	"Market East",
	"Jenkintown-Wyncote",
	"Langhorne",
	"Lansdale",
	"Lansdowne",
	"Lawndale",
	"Levittown",
	"Link Belt",
	"Main St",
	"Malvern",
	"Manayunk",
	"Marcus Hook",
	"Market East",
	"Meadowbrook",
	"Media",
	"Melrose Park",
	"Merion",
	"Miquon",
	"Morton",
	"Moylan-Rose Valley",
	"Mt Airy",
	"Narberth",
	"Neshaminy Falls",
	"New Britain",
	"Newark",
	"Noble",
	"Norristown TC",
	"North Broad St",
	"North Hills",
	"North Philadelphia",
	"North Wales",
	"Norwood",
	"Olney",
	"Oreland",
	"Overbrook",
	"Paoli",
	"Penllyn",
	"Pennbrook",
	"Philmont",
	"Primos",
	"Prospect Park",
	"Queen Lane",
	"Radnor",
	"Ridley Park",
	"Rosemont",
	"Roslyn",
	"Rydal",
	"Ryers",
	"Secane",
	"Sedgwick",
	"Sharon Hill",
	"Somerton",
	"Spring Mill",
	"St. Davids",
	"St. Martins",
	"Stenton",
	"Strafford",
	"Suburban Station",
	"Swarthmore",
	"Tacony",
	"Temple U",
	"Thorndale",
	"Torresdale",
	"Trenton",
	"Trevose",
	"Tulpehocken",
	"University City",
	"Upsal",
	"Villanova",
	"Wallingford",
	"Warminster",
	"Washington Lane",
	"Wayne Jct",
	"Wayne Station",
	"West Trenton",
	"Whitford",
	"Willow Grove",
	"Wilmington",
	"Wissahickon",
	"Wister",
	"Woodbourne",
	"Wyndmoor",
	"Wynnefield Avenue",
	"Wynnewood",
	"Yardley",
}

// IsValidStation returns whether the station is a valid station.
func IsValidStation(station string) bool {
	for _, v := range stations {
		if station == v {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>/*
 * Minio Go Library for Amazon S3 Compatible Cloud Storage
 * Copyright 2015-2017 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package minio

import (
	"bytes"
	"context"
	"encoding/xml"
	"io"
	"net/http"
	"net/url"

	"github.com/minio/minio-go/pkg/s3utils"
)

// RemoveBucket deletes the bucket name.
//
//  All objects (including all object versions and delete markers).
//  in the bucket must be deleted before successfully attempting this request.
func (c Client) RemoveBucket(bucketName string) error {
	// Input validation.
	if err := s3utils.CheckValidBucketName(bucketName); err != nil {
		return err
	}
	// Execute DELETE on bucket.
	resp, err := c.executeMethod(context.Background(), "DELETE", requestMetadata{
		bucketName:       bucketName,
		contentSHA256Hex: emptySHA256Hex,
	})
	defer closeResponse(resp)
	if err != nil {
		return err
	}
	if resp != nil {
		if resp.StatusCode != http.StatusNoContent {
			return httpRespToErrorResponse(resp, bucketName, "")
		}
	}

	// Remove the location from cache on a successful delete.
	c.bucketLocCache.Delete(bucketName)

	return nil
}

// RemoveObject remove an object from a bucket.
func (c Client) RemoveObject(bucketName, objectName string) error {
	// Input validation.
	if err := s3utils.CheckValidBucketName(bucketName); err != nil {
		return err
	}
	if err := s3utils.CheckValidObjectName(objectName); err != nil {
		return err
	}
	// Execute DELETE on objectName.
	resp, err := c.executeMethod(context.Background(), "DELETE", requestMetadata{
		bucketName:       bucketName,
		objectName:       objectName,
		contentSHA256Hex: emptySHA256Hex,
	})
	defer closeResponse(resp)
	if err != nil {
		return err
	}
	if resp != nil {
		// if some unexpected error happened and max retry is reached, we want to let client know
		if resp.StatusCode != http.StatusNoContent {
			return httpRespToErrorResponse(resp, bucketName, objectName)
		}
	}

	// DeleteObject always responds with http '204' even for
	// objects which do not exist. So no need to handle them
	// specifically.
	return nil
}

// RemoveObjectError - container of Multi Delete S3 API error
type RemoveObjectError struct {
	ObjectName string
	Err        error
}

// generateRemoveMultiObjects - generate the XML request for remove multi objects request
func generateRemoveMultiObjectsRequest(objects []string) []byte {
	rmObjects := []deleteObject{}
	for _, obj := range objects {
		rmObjects = append(rmObjects, deleteObject{Key: obj})
	}
	xmlBytes, _ := xml.Marshal(deleteMultiObjects{Objects: rmObjects, Quiet: true})
	return xmlBytes
}

// processRemoveMultiObjectsResponse - parse the remove multi objects web service
// and return the success/failure result status for each object
func processRemoveMultiObjectsResponse(body io.Reader, objects []string, errorCh chan<- RemoveObjectError) {
	// Parse multi delete XML response
	rmResult := &deleteMultiObjectsResult{}
	err := xmlDecoder(body, rmResult)
	if err != nil {
		errorCh <- RemoveObjectError{ObjectName: "", Err: err}
		return
	}

	// Fill deletion that returned an error.
	for _, obj := range rmResult.UnDeletedObjects {
		errorCh <- RemoveObjectError{
			ObjectName: obj.Key,
			Err: ErrorResponse{
				Code:    obj.Code,
				Message: obj.Message,
			},
		}
	}
}

// RemoveObjectsWithContext - Identical to RemoveObjects call, but accepts context to facilitate request cancellation.
func (c Client) RemoveObjectsWithContext(ctx context.Context, bucketName string, objectsCh <-chan string) <-chan RemoveObjectError {
	errorCh := make(chan RemoveObjectError, 1)

	// Validate if bucket name is valid.
	if err := s3utils.CheckValidBucketName(bucketName); err != nil {
		defer close(errorCh)
		errorCh <- RemoveObjectError{
			Err: err,
		}
		return errorCh
	}
	// Validate objects channel to be properly allocated.
	if objectsCh == nil {
		defer close(errorCh)
		errorCh <- RemoveObjectError{
			Err: ErrInvalidArgument("Objects channel cannot be nil"),
		}
		return errorCh
	}

	// Generate and call MultiDelete S3 requests based on entries received from objectsCh
	go func(errorCh chan<- RemoveObjectError) {
		maxEntries := 1000
		finish := false
		urlValues := make(url.Values)
		urlValues.Set("delete", "")

		// Close error channel when Multi delete finishes.
		defer close(errorCh)

		// Loop over entries by 1000 and call MultiDelete requests
		for {
			if finish {
				break
			}
			count := 0
			var batch []string

			// Try to gather 1000 entries
			for object := range objectsCh {
				batch = append(batch, object)
				if count++; count >= maxEntries {
					break
				}
			}
			if count == 0 {
				// Multi Objects Delete API doesn't accept empty object list, quit immediately
				break
			}
			if count < maxEntries {
				// We didn't have 1000 entries, so this is the last batch
				finish = true
			}

			// Generate remove multi objects XML request
			removeBytes := generateRemoveMultiObjectsRequest(batch)
			// Execute GET on bucket to list objects.
			resp, err := c.executeMethod(ctx, "POST", requestMetadata{
				bucketName:       bucketName,
				queryValues:      urlValues,
				contentBody:      bytes.NewReader(removeBytes),
				contentLength:    int64(len(removeBytes)),
				contentMD5Base64: sumMD5Base64(removeBytes),
				contentSHA256Hex: sum256Hex(removeBytes),
			})
			if err != nil {
				for _, b := range batch {
					errorCh <- RemoveObjectError{ObjectName: b, Err: err}
				}
				continue
			}

			// Process multiobjects remove xml response
			processRemoveMultiObjectsResponse(resp.Body, batch, errorCh)

			closeResponse(resp)
		}
	}(errorCh)
	return errorCh
}

// RemoveObjects removes multiple objects from a bucket.
// The list of objects to remove are received from objectsCh.
// Remove failures are sent back via error channel.
func (c Client) RemoveObjects(bucketName string, objectsCh <-chan string) <-chan RemoveObjectError {
	return c.RemoveObjectsWithContext(context.Background(), bucketName, objectsCh)
}

// RemoveIncompleteUpload aborts an partially uploaded object.
func (c Client) RemoveIncompleteUpload(bucketName, objectName string) error {
	// Input validation.
	if err := s3utils.CheckValidBucketName(bucketName); err != nil {
		return err
	}
	if err := s3utils.CheckValidObjectName(objectName); err != nil {
		return err
	}
	// Find multipart upload id of the object to be aborted.
	uploadID, err := c.findUploadID(bucketName, objectName)
	if err != nil {
		return err
	}
	if uploadID != "" {
		// Upload id found, abort the incomplete multipart upload.
		err := c.abortMultipartUpload(context.Background(), bucketName, objectName, uploadID)
		if err != nil {
			return err
		}
	}
	return nil
}

// abortMultipartUpload aborts a multipart upload for the given
// uploadID, all previously uploaded parts are deleted.
func (c Client) abortMultipartUpload(ctx context.Context, bucketName, objectName, uploadID string) error {
	// Input validation.
	if err := s3utils.CheckValidBucketName(bucketName); err != nil {
		return err
	}
	if err := s3utils.CheckValidObjectName(objectName); err != nil {
		return err
	}

	// Initialize url queries.
	urlValues := make(url.Values)
	urlValues.Set("uploadId", uploadID)

	// Execute DELETE on multipart upload.
	resp, err := c.executeMethod(ctx, "DELETE", requestMetadata{
		bucketName:       bucketName,
		objectName:       objectName,
		queryValues:      urlValues,
		contentSHA256Hex: emptySHA256Hex,
	})
	defer closeResponse(resp)
	if err != nil {
		return err
	}
	if resp != nil {
		if resp.StatusCode != http.StatusNoContent {
			// Abort has no response body, handle it for any errors.
			var errorResponse ErrorResponse
			switch resp.StatusCode {
			case http.StatusNotFound:
				// This is needed specifically for abort and it cannot
				// be converged into default case.
				errorResponse = ErrorResponse{
					Code:       "NoSuchUpload",
					Message:    "The specified multipart upload does not exist.",
					BucketName: bucketName,
					Key:        objectName,
					RequestID:  resp.Header.Get("x-amz-request-id"),
					HostID:     resp.Header.Get("x-amz-id-2"),
					Region:     resp.Header.Get("x-amz-bucket-region"),
				}
			default:
				return httpRespToErrorResponse(resp, bucketName, objectName)
			}
			return errorResponse
		}
	}
	return nil
}
<commit_msg>Fix xml parsing error for RemoveObjects API (#949)<commit_after>/*
 * Minio Go Library for Amazon S3 Compatible Cloud Storage
 * Copyright 2015-2017 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package minio

import (
	"bytes"
	"context"
	"encoding/xml"
	"io"
	"net/http"
	"net/url"

	"github.com/minio/minio-go/pkg/s3utils"
)

// RemoveBucket deletes the bucket name.
//
//  All objects (including all object versions and delete markers).
//  in the bucket must be deleted before successfully attempting this request.
func (c Client) RemoveBucket(bucketName string) error {
	// Input validation.
	if err := s3utils.CheckValidBucketName(bucketName); err != nil {
		return err
	}
	// Execute DELETE on bucket.
	resp, err := c.executeMethod(context.Background(), "DELETE", requestMetadata{
		bucketName:       bucketName,
		contentSHA256Hex: emptySHA256Hex,
	})
	defer closeResponse(resp)
	if err != nil {
		return err
	}
	if resp != nil {
		if resp.StatusCode != http.StatusNoContent {
			return httpRespToErrorResponse(resp, bucketName, "")
		}
	}

	// Remove the location from cache on a successful delete.
	c.bucketLocCache.Delete(bucketName)

	return nil
}

// RemoveObject remove an object from a bucket.
func (c Client) RemoveObject(bucketName, objectName string) error {
	// Input validation.
	if err := s3utils.CheckValidBucketName(bucketName); err != nil {
		return err
	}
	if err := s3utils.CheckValidObjectName(objectName); err != nil {
		return err
	}
	// Execute DELETE on objectName.
	resp, err := c.executeMethod(context.Background(), "DELETE", requestMetadata{
		bucketName:       bucketName,
		objectName:       objectName,
		contentSHA256Hex: emptySHA256Hex,
	})
	defer closeResponse(resp)
	if err != nil {
		return err
	}
	if resp != nil {
		// if some unexpected error happened and max retry is reached, we want to let client know
		if resp.StatusCode != http.StatusNoContent {
			return httpRespToErrorResponse(resp, bucketName, objectName)
		}
	}

	// DeleteObject always responds with http '204' even for
	// objects which do not exist. So no need to handle them
	// specifically.
	return nil
}

// RemoveObjectError - container of Multi Delete S3 API error
type RemoveObjectError struct {
	ObjectName string
	Err        error
}

// generateRemoveMultiObjects - generate the XML request for remove multi objects request
func generateRemoveMultiObjectsRequest(objects []string) []byte {
	rmObjects := []deleteObject{}
	for _, obj := range objects {
		rmObjects = append(rmObjects, deleteObject{Key: obj})
	}
	xmlBytes, _ := xml.Marshal(deleteMultiObjects{Objects: rmObjects, Quiet: true})
	return xmlBytes
}

// processRemoveMultiObjectsResponse - parse the remove multi objects web service
// and return the success/failure result status for each object
func processRemoveMultiObjectsResponse(body io.Reader, objects []string, errorCh chan<- RemoveObjectError) {
	// Parse multi delete XML response
	rmResult := &deleteMultiObjectsResult{}
	err := xmlDecoder(body, rmResult)
	if err != nil {
		errorCh <- RemoveObjectError{ObjectName: "", Err: err}
		return
	}

	// Fill deletion that returned an error.
	for _, obj := range rmResult.UnDeletedObjects {
		errorCh <- RemoveObjectError{
			ObjectName: obj.Key,
			Err: ErrorResponse{
				Code:    obj.Code,
				Message: obj.Message,
			},
		}
	}
}

// RemoveObjectsWithContext - Identical to RemoveObjects call, but accepts context to facilitate request cancellation.
func (c Client) RemoveObjectsWithContext(ctx context.Context, bucketName string, objectsCh <-chan string) <-chan RemoveObjectError {
	errorCh := make(chan RemoveObjectError, 1)

	// Validate if bucket name is valid.
	if err := s3utils.CheckValidBucketName(bucketName); err != nil {
		defer close(errorCh)
		errorCh <- RemoveObjectError{
			Err: err,
		}
		return errorCh
	}
	// Validate objects channel to be properly allocated.
	if objectsCh == nil {
		defer close(errorCh)
		errorCh <- RemoveObjectError{
			Err: ErrInvalidArgument("Objects channel cannot be nil"),
		}
		return errorCh
	}

	// Generate and call MultiDelete S3 requests based on entries received from objectsCh
	go func(errorCh chan<- RemoveObjectError) {
		maxEntries := 1000
		finish := false
		urlValues := make(url.Values)
		urlValues.Set("delete", "")

		// Close error channel when Multi delete finishes.
		defer close(errorCh)

		// Loop over entries by 1000 and call MultiDelete requests
		for {
			if finish {
				break
			}
			count := 0
			var batch []string

			// Try to gather 1000 entries
			for object := range objectsCh {
				batch = append(batch, object)
				if count++; count >= maxEntries {
					break
				}
			}
			if count == 0 {
				// Multi Objects Delete API doesn't accept empty object list, quit immediately
				break
			}
			if count < maxEntries {
				// We didn't have 1000 entries, so this is the last batch
				finish = true
			}

			// Generate remove multi objects XML request
			removeBytes := generateRemoveMultiObjectsRequest(batch)
			// Execute GET on bucket to list objects.
			resp, err := c.executeMethod(ctx, "POST", requestMetadata{
				bucketName:       bucketName,
				queryValues:      urlValues,
				contentBody:      bytes.NewReader(removeBytes),
				contentLength:    int64(len(removeBytes)),
				contentMD5Base64: sumMD5Base64(removeBytes),
				contentSHA256Hex: sum256Hex(removeBytes),
			})
			if resp != nil {
				if resp.StatusCode != http.StatusOK {
					e := httpRespToErrorResponse(resp, bucketName, "")
					errorCh <- RemoveObjectError{ObjectName: "", Err: e}
				}
			}
			if err != nil {
				for _, b := range batch {
					errorCh <- RemoveObjectError{ObjectName: b, Err: err}
				}
				continue
			}

			// Process multiobjects remove xml response
			processRemoveMultiObjectsResponse(resp.Body, batch, errorCh)

			closeResponse(resp)
		}
	}(errorCh)
	return errorCh
}

// RemoveObjects removes multiple objects from a bucket.
// The list of objects to remove are received from objectsCh.
// Remove failures are sent back via error channel.
func (c Client) RemoveObjects(bucketName string, objectsCh <-chan string) <-chan RemoveObjectError {
	return c.RemoveObjectsWithContext(context.Background(), bucketName, objectsCh)
}

// RemoveIncompleteUpload aborts an partially uploaded object.
func (c Client) RemoveIncompleteUpload(bucketName, objectName string) error {
	// Input validation.
	if err := s3utils.CheckValidBucketName(bucketName); err != nil {
		return err
	}
	if err := s3utils.CheckValidObjectName(objectName); err != nil {
		return err
	}
	// Find multipart upload id of the object to be aborted.
	uploadID, err := c.findUploadID(bucketName, objectName)
	if err != nil {
		return err
	}
	if uploadID != "" {
		// Upload id found, abort the incomplete multipart upload.
		err := c.abortMultipartUpload(context.Background(), bucketName, objectName, uploadID)
		if err != nil {
			return err
		}
	}
	return nil
}

// abortMultipartUpload aborts a multipart upload for the given
// uploadID, all previously uploaded parts are deleted.
func (c Client) abortMultipartUpload(ctx context.Context, bucketName, objectName, uploadID string) error {
	// Input validation.
	if err := s3utils.CheckValidBucketName(bucketName); err != nil {
		return err
	}
	if err := s3utils.CheckValidObjectName(objectName); err != nil {
		return err
	}

	// Initialize url queries.
	urlValues := make(url.Values)
	urlValues.Set("uploadId", uploadID)

	// Execute DELETE on multipart upload.
	resp, err := c.executeMethod(ctx, "DELETE", requestMetadata{
		bucketName:       bucketName,
		objectName:       objectName,
		queryValues:      urlValues,
		contentSHA256Hex: emptySHA256Hex,
	})
	defer closeResponse(resp)
	if err != nil {
		return err
	}
	if resp != nil {
		if resp.StatusCode != http.StatusNoContent {
			// Abort has no response body, handle it for any errors.
			var errorResponse ErrorResponse
			switch resp.StatusCode {
			case http.StatusNotFound:
				// This is needed specifically for abort and it cannot
				// be converged into default case.
				errorResponse = ErrorResponse{
					Code:       "NoSuchUpload",
					Message:    "The specified multipart upload does not exist.",
					BucketName: bucketName,
					Key:        objectName,
					RequestID:  resp.Header.Get("x-amz-request-id"),
					HostID:     resp.Header.Get("x-amz-id-2"),
					Region:     resp.Header.Get("x-amz-bucket-region"),
				}
			default:
				return httpRespToErrorResponse(resp, bucketName, objectName)
			}
			return errorResponse
		}
	}
	return nil
}
<|endoftext|>


<commit_before><commit_msg>auth: fail if workload identity fails (#57)<commit_after><|endoftext|>


<commit_before><commit_msg>remove duplicate case<commit_after><|endoftext|>


<commit_before>package main

import (
	"os"
	"os/signal"

	"github.com/iamthemuffinman/cli"
	"github.com/iamthemuffinman/overseer/cmd"
)

var Commands map[string]cli.CommandFactory
var PlumbingCommands map[string]struct{}
var UI cli.Ui

func init() {
	UI = &cli.BasicUI{
		Reader:      os.Stdin,
		Writer:      os.Stdout,
		ErrorWriter: os.Stderr,
	}

	PlumbingCommands = map[string]struct{}{
		"provision": {}, // includes all subcommands
	}

	Commands = map[string]cli.CommandFactory{
		"init": func() (cli.Command, error) {
			return &cmd.InitCommand{
				UI: UI,
			}, nil
		},

		"version": func() (cli.Command, error) {
			return &cmd.VersionCommand{
				UI:       UI,
				Revision: GitCommit,
				Version:  Version,
			}, nil
		},

		"provision": func() (cli.Command, error) {
			return &cmd.ProvisionCommand{
				UI: UI,
			}, nil
		},

		"provision virtual": func() (cli.Command, error) {
			return &cmd.ProvisionVirtualCommand{
				UI:         UI,
				ShutdownCh: makeShutdownCh(),
			}, nil
		},

		"provision physical": func() (cli.Command, error) {
			return &cmd.ProvisionPhysicalCommand{
				UI:         UI,
				ShutdownCh: makeShutdownCh(),
			}, nil
		},
	}
}

func makeShutdownCh() <-chan struct{} {
	resultCh := make(chan struct{})

	signalCh := make(chan os.Signal, 4)
	signal.Notify(signalCh, os.Interrupt)
	go func() {
		for {
			<-signalCh
			resultCh <- struct{}{}

		}
	}()

	return resultCh
}
<commit_msg>Accidentally changed this when fixing suggestions via golint<commit_after>package main

import (
	"os"
	"os/signal"

	"github.com/iamthemuffinman/cli"
	"github.com/iamthemuffinman/overseer/cmd"
)

var Commands map[string]cli.CommandFactory
var PlumbingCommands map[string]struct{}
var UI cli.Ui

func init() {
	UI = &cli.BasicUi{
		Reader:      os.Stdin,
		Writer:      os.Stdout,
		ErrorWriter: os.Stderr,
	}

	PlumbingCommands = map[string]struct{}{
		"provision": {}, // includes all subcommands
	}

	Commands = map[string]cli.CommandFactory{
		"init": func() (cli.Command, error) {
			return &cmd.InitCommand{
				UI: UI,
			}, nil
		},

		"version": func() (cli.Command, error) {
			return &cmd.VersionCommand{
				UI:       UI,
				Revision: GitCommit,
				Version:  Version,
			}, nil
		},

		"provision": func() (cli.Command, error) {
			return &cmd.ProvisionCommand{
				UI: UI,
			}, nil
		},

		"provision virtual": func() (cli.Command, error) {
			return &cmd.ProvisionVirtualCommand{
				UI:         UI,
				ShutdownCh: makeShutdownCh(),
			}, nil
		},

		"provision physical": func() (cli.Command, error) {
			return &cmd.ProvisionPhysicalCommand{
				UI:         UI,
				ShutdownCh: makeShutdownCh(),
			}, nil
		},
	}
}

func makeShutdownCh() <-chan struct{} {
	resultCh := make(chan struct{})

	signalCh := make(chan os.Signal, 4)
	signal.Notify(signalCh, os.Interrupt)
	go func() {
		for {
			<-signalCh
			resultCh <- struct{}{}

		}
	}()

	return resultCh
}
<|endoftext|>


<commit_before>// Basic commands for Cookoo.
package cookoo

import (
	"log"
)

// Print a message to the log.
//
// Params:
// - msg: The message to print
func LogMessage(cxt Context, params *Params) (interface{}, Interrupt) {
	msg := params.Get("msg", "tick")
	log.Print(msg)
	return nil, nil
}
<commit_msg>Added AddToContext() command.<commit_after>// Basic commands for Cookoo.
package cookoo

import (
	"log"
)

// Print a message to the log.
//
// Params:
// - msg: The message to print
func LogMessage(cxt Context, params *Params) (interface{}, Interrupt) {
	msg := params.Get("msg", "tick")
	log.Print(msg)
	return nil, nil
}

func AddToContext(cxt Context, params *Params) (interface{}, Interrupt) {
	p := params.AsMap()
	for k, v := range p {
		cxt.Add(k, v)
	}
	return true, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"strings"

	"github.com/urfave/cli/v2"
)

var commands = []*cli.Command{
	commandGet,
	commandList,
	commandLook,
	commandImport,
	commandRoot,
	commandCreate,
}

// cloneFlags are comman flags of `get` and `import` subcommands
var cloneFlags = []cli.Flag{
	&cli.BoolFlag{Name: "update, u", Usage: "Update local repository if cloned already"},
	&cli.BoolFlag{Name: "p", Usage: "Clone with SSH"},
	&cli.BoolFlag{Name: "shallow", Usage: "Do a shallow clone"},
	&cli.BoolFlag{Name: "look, l", Usage: "Look after get"},
	&cli.StringFlag{Name: "vcs", Usage: "Specify VCS backend for cloning"},
	&cli.BoolFlag{Name: "silent, s", Usage: "clone or update silently"},
	&cli.BoolFlag{Name: "no-recursive", Usage: "prevent recursive fetching"},
}

var commandGet = &cli.Command{
	Name:  "get",
	Usage: "Clone/sync with a remote repository",
	Description: `
    Clone a GitHub repository under ghq root directory. If the repository is
    already cloned to local, nothing will happen unless '-u' ('--update')
    flag is supplied, in which case 'git remote update' is executed.
    When you use '-p' option, the repository is cloned via SSH.
`,
	Action: doGet,
	Flags: append(cloneFlags,
		&cli.StringFlag{Name: "branch, b", Usage: "Specify branch name. This flag implies --single-branch on Git"}),
}

var commandList = &cli.Command{
	Name:  "list",
	Usage: "List local repositories",
	Description: `
    List locally cloned repositories. If a query argument is given, only
    repositories whose names contain that query text are listed. '-e'
    ('--exact') forces the match to be an exact one (i.e. the query equals to
    _project_ or _user_/_project_) If '-p' ('--full-path') is given, the full paths
    to the repository root are printed instead of relative ones.
`,
	Action: doList,
	Flags: []cli.Flag{
		&cli.BoolFlag{Name: "exact, e", Usage: "Perform an exact match"},
		&cli.StringFlag{Name: "vcs", Usage: "Specify VCS backend for matching"},
		&cli.BoolFlag{Name: "full-path, p", Usage: "Print full paths"},
		&cli.BoolFlag{Name: "unique", Usage: "Print unique subpaths"},
	},
}

var commandLook = &cli.Command{
	Name:  "look",
	Usage: "Look into a local repository",
	Description: `
    Look into a locally cloned repository with the shell.
`,
	Action: doLook,
}

var commandImport = &cli.Command{
	Name:   "import",
	Usage:  "Bulk get repositories from stdin",
	Action: doImport,
	Flags: append(cloneFlags,
		&cli.BoolFlag{Name: "parallel, P", Usage: "[Experimental] Import parallely"}),
}

var commandRoot = &cli.Command{
	Name:   "root",
	Usage:  "Show repositories' root",
	Action: doRoot,
	Flags: []cli.Flag{
		&cli.BoolFlag{Name: "all", Usage: "Show all roots"},
	},
}

var commandCreate = &cli.Command{
	Name:   "create",
	Usage:  "Create repository",
	Action: doCreate,
	Flags: []cli.Flag{
		&cli.StringFlag{Name: "vcs", Usage: "Specify VCS backend explicitly"},
	},
}

type commandDoc struct {
	Parent    string
	Arguments string
}

var commandDocs = map[string]commandDoc{
	"get":    {"", "[-u] [--vcs <vcs>] <repository URL> | [-u] [-p] <user>/<project>"},
	"list":   {"", "[-p] [-e] [<query>]"},
	"look":   {"", "<project> | <user>/<project> | <host>/<user>/<project>"},
	"import": {"", "< file"},
	"root":   {"", ""},
	"create": {"", "<project> | <user>/<project> | <host>/<user>/<project>"},
}

// Makes template conditionals to generate per-command documents.
func mkCommandsTemplate(genTemplate func(commandDoc) string) string {
	template := "{{if false}}"
	for _, command := range append(commands) {
		template = template + fmt.Sprintf("{{else if (eq .Name %q)}}%s", command.Name, genTemplate(commandDocs[command.Name]))
	}
	return template + "{{end}}"
}

func init() {
	argsTemplate := mkCommandsTemplate(func(doc commandDoc) string { return doc.Arguments })
	parentTemplate := mkCommandsTemplate(func(doc commandDoc) string { return string(strings.TrimLeft(doc.Parent+" ", " ")) })

	cli.CommandHelpTemplate = `NAME:
    {{.Name}} - {{.Usage}}

USAGE:
    ghq ` + parentTemplate + `{{.Name}} ` + argsTemplate + `
{{if (len .Description)}}
DESCRIPTION: {{.Description}}
{{end}}{{if (len .Flags)}}
OPTIONS:
    {{range .Flags}}{{.}}
    {{end}}
{{end}}`
}
<commit_msg>define flag aliases<commit_after>package main

import (
	"fmt"
	"strings"

	"github.com/urfave/cli/v2"
)

var commands = []*cli.Command{
	commandGet,
	commandList,
	commandLook,
	commandImport,
	commandRoot,
	commandCreate,
}

// cloneFlags are comman flags of `get` and `import` subcommands
var cloneFlags = []cli.Flag{
	&cli.BoolFlag{Name: "update", Aliases: []string{"u"},
		Usage: "Update local repository if cloned already"},
	&cli.BoolFlag{Name: "p", Usage: "Clone with SSH"},
	&cli.BoolFlag{Name: "shallow", Usage: "Do a shallow clone"},
	&cli.BoolFlag{Name: "look", Aliases: []string{"l"}, Usage: "Look after get"},
	&cli.StringFlag{Name: "vcs", Usage: "Specify VCS backend for cloning"},
	&cli.BoolFlag{Name: "silent", Aliases: []string{"s"}, Usage: "clone or update silently"},
	&cli.BoolFlag{Name: "no-recursive", Usage: "prevent recursive fetching"},
}

var commandGet = &cli.Command{
	Name:  "get",
	Usage: "Clone/sync with a remote repository",
	Description: `
    Clone a GitHub repository under ghq root directory. If the repository is
    already cloned to local, nothing will happen unless '-u' ('--update')
    flag is supplied, in which case 'git remote update' is executed.
    When you use '-p' option, the repository is cloned via SSH.
`,
	Action: doGet,
	Flags: append(cloneFlags,
		&cli.StringFlag{Name: "branch", Aliases: []string{"b"},
			Usage: "Specify branch name. This flag implies --single-branch on Git"}),
}

var commandList = &cli.Command{
	Name:  "list",
	Usage: "List local repositories",
	Description: `
    List locally cloned repositories. If a query argument is given, only
    repositories whose names contain that query text are listed. '-e'
    ('--exact') forces the match to be an exact one (i.e. the query equals to
    _project_ or _user_/_project_) If '-p' ('--full-path') is given, the full paths
    to the repository root are printed instead of relative ones.
`,
	Action: doList,
	Flags: []cli.Flag{
		&cli.BoolFlag{Name: "exact", Aliases: []string{"e"}, Usage: "Perform an exact match"},
		&cli.StringFlag{Name: "vcs", Usage: "Specify VCS backend for matching"},
		&cli.BoolFlag{Name: "full-path", Aliases: []string{"p"}, Usage: "Print full paths"},
		&cli.BoolFlag{Name: "unique", Usage: "Print unique subpaths"},
	},
}

var commandLook = &cli.Command{
	Name:  "look",
	Usage: "Look into a local repository",
	Description: `
    Look into a locally cloned repository with the shell.
`,
	Action: doLook,
}

var commandImport = &cli.Command{
	Name:   "import",
	Usage:  "Bulk get repositories from stdin",
	Action: doImport,
	Flags: append(cloneFlags,
		&cli.BoolFlag{Name: "parallel", Aliases: []string{"P"},
			Usage: "[Experimental] Import parallely"}),
}

var commandRoot = &cli.Command{
	Name:   "root",
	Usage:  "Show repositories' root",
	Action: doRoot,
	Flags: []cli.Flag{
		&cli.BoolFlag{Name: "all", Usage: "Show all roots"},
	},
}

var commandCreate = &cli.Command{
	Name:   "create",
	Usage:  "Create repository",
	Action: doCreate,
	Flags: []cli.Flag{
		&cli.StringFlag{Name: "vcs", Usage: "Specify VCS backend explicitly"},
	},
}

type commandDoc struct {
	Parent    string
	Arguments string
}

var commandDocs = map[string]commandDoc{
	"get":    {"", "[-u] [--vcs <vcs>] <repository URL> | [-u] [-p] <user>/<project>"},
	"list":   {"", "[-p] [-e] [<query>]"},
	"look":   {"", "<project> | <user>/<project> | <host>/<user>/<project>"},
	"import": {"", "< file"},
	"root":   {"", ""},
	"create": {"", "<project> | <user>/<project> | <host>/<user>/<project>"},
}

// Makes template conditionals to generate per-command documents.
func mkCommandsTemplate(genTemplate func(commandDoc) string) string {
	template := "{{if false}}"
	for _, command := range append(commands) {
		template = template + fmt.Sprintf("{{else if (eq .Name %q)}}%s", command.Name, genTemplate(commandDocs[command.Name]))
	}
	return template + "{{end}}"
}

func init() {
	argsTemplate := mkCommandsTemplate(func(doc commandDoc) string { return doc.Arguments })
	parentTemplate := mkCommandsTemplate(func(doc commandDoc) string { return string(strings.TrimLeft(doc.Parent+" ", " ")) })

	cli.CommandHelpTemplate = `NAME:
    {{.Name}} - {{.Usage}}

USAGE:
    ghq ` + parentTemplate + `{{.Name}} ` + argsTemplate + `
{{if (len .Description)}}
DESCRIPTION: {{.Description}}
{{end}}{{if (len .Flags)}}
OPTIONS:
    {{range .Flags}}{{.}}
    {{end}}
{{end}}`
}
<|endoftext|>


<commit_before><commit_msg>option to show g and h (constraints) in simpleproblem added<commit_after><|endoftext|>


<commit_before>// package others contains a functions helping in unclassified scenarios.
package others

import (
	"math/rand"
	"time"
)

const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
const (
	letterIdxBits = 6                    // 6 bits to represent a letter index
	letterIdxMask = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits
	letterIdxMax  = 63 / letterIdxBits   // # of letter indices fitting in 63 bits
)

var src = rand.NewSource(time.Now().UnixNano())

// RandString produce a random string
// comes from: http://stackoverflow.com/questions/22892120/how-to-generate-a-random-string-of-a-fixed-length-in-golang
func RandString(n int) string {
	b := make([]byte, n)
	// A src.Int63() generates 63 random bits, enough for letterIdxMax characters!
	for i, cache, remain := n-1, src.Int63(), letterIdxMax; i >= 0; {
		if remain == 0 {
			cache, remain = src.Int63(), letterIdxMax
		}
		if idx := int(cache & letterIdxMask); idx < len(letterBytes) {
			b[i] = letterBytes[idx]
			i--
		}
		cache >>= letterIdxBits
		remain--
	}

	return string(b)
}

// TODO, it would be even coler to just byte64 encode a random byte sequence.

<commit_msg>Fix unsafe concurrent access to source.<commit_after>// package others contains a functions helping in unclassified scenarios.
package others

import (
	"math/rand"
	"time"
	"sync"
)

const (
	letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"

	letterIdxBits = 6                    // 6 bits to represent a letter index
	letterIdxMask = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits
	letterIdxMax  = 63 / letterIdxBits   // # of letter indices fitting in 63 bits
)

var (
	src = rand.NewSource(time.Now().UnixNano())
	m sync.Mutex
)

func int63() int64 {
	m.Lock()
	x := src.Int63()
	m.Unlock()
	return x
}

// RandString produce a random string
// comes from: http://stackoverflow.com/questions/22892120/how-to-generate-a-random-string-of-a-fixed-length-in-golang
func RandString(n int) string {
	b := make([]byte, n)
	// A src.Int63() generates 63 random bits, enough for letterIdxMax characters!
	for i, cache, remain := n-1, int63(), letterIdxMax; i >= 0; {
		if remain == 0 {
			cache, remain = int63(), letterIdxMax
		}
		if idx := int(cache & letterIdxMask); idx < len(letterBytes) {
			b[i] = letterBytes[idx]
			i--
		}
		cache >>= letterIdxBits
		remain--
	}

	return string(b)
}

// TODO it would be even cooler to just byte64 encode a random byte sequence.

<|endoftext|>


<commit_before>// Copyright 2012 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package app

import (
	"bytes"
	"crypto/rand"
	"fmt"
	"github.com/fsouza/go-iam"
	"github.com/globocom/config"
	"io"
	"launchpad.net/goamz/aws"
	"launchpad.net/goamz/s3"
	"strings"
	"text/template"
)

type s3Env struct {
	aws.Auth
	bucket             string
	endpoint           string
	locationConstraint bool
}

func (s *s3Env) empty() bool {
	return s.bucket == "" || s.AccessKey == "" || s.SecretKey == ""
}

const (
	randBytes      = 32
	s3InstanceName = "tsurus3"
)

var (
	rReader = rand.Reader
	policy  = template.Must(template.New("policy").Parse(`{
  "Statement": [
    {
      "Action": [
        "s3:CreateBucket",
        "s3:DeleteBucket",
        "s3:DeleteBucketPolicy",
        "s3:DeleteBucketWebsite",
        "s3:PutBucketLogging",
        "s3:PutBucketPolicy",
        "s3:PutBucketRequestPayment",
        "s3:PutBucketVersioning",
        "s3:PutBucketWebsite"
      ],
      "Effect": "Deny",
      "Resource": [
        "arn:aws:s3:::{{.}}/*",
        "arn:aws:s3:::{{.}}"
      ]
    },
    {
      "Action": [
        "s3:*"
      ],
      "Effect": "Allow",
      "Resource": [
        "arn:aws:s3:::{{.}}/*",
        "arn:aws:s3:::{{.}}"
      ]
    }
  ]
}`))
)

func getAWSAuth() aws.Auth {
	access, err := config.GetString("aws:access-key-id")
	if err != nil {
		panic("FATAL: aws:access-key-id must be defined in configuration file.")
	}
	secret, err := config.GetString("aws:secret-access-key")
	if err != nil {
		panic("FATAL: aws:secret-access-key must be defined in configuration file.")
	}
	return aws.Auth{
		AccessKey: access,
		SecretKey: secret,
	}
}

func getS3Endpoint() *s3.S3 {
	regionName, _ := config.GetString("aws:s3:region-name")
	endpoint, err := config.GetString("aws:s3:endpoint")
	if err != nil {
		panic("FATAL: aws:s3:endpoint must be defined in configuration file.")
	}
	bucketEndpoint, _ := config.GetString("aws:s3:bucketEndpoint")
	locationConstraint, err := config.GetBool("aws:s3:location-constraint")
	if err != nil {
		panic("FATAL: aws:s3:location-constraint must be defined in configuration file.")
	}
	lowercaseBucket, err := config.GetBool("aws:s3:lowercase-bucket")
	if err != nil {
		panic("FATAL: aws:s3:lowercase-bucket must be defined in configuration file.")
	}
	region := aws.Region{
		Name:                 regionName,
		S3Endpoint:           endpoint,
		S3BucketEndpoint:     bucketEndpoint,
		S3LocationConstraint: locationConstraint,
		S3LowercaseBucket:    lowercaseBucket,
	}
	return s3.New(getAWSAuth(), region)
}

func getIAMEndpoint() *iam.IAM {
	endpoint, err := config.GetString("aws:iam:endpoint")
	if err != nil {
		panic("FATAL: aws:iam:endpoint must be defined in configuration file.")
	}
	region := aws.Region{IAMEndpoint: endpoint}
	return iam.New(getAWSAuth(), region)
}

func putBucket(appName string, bucketChan chan s3.Bucket, errChan chan error) {
	randPart := make([]byte, randBytes)
	n, err := rReader.Read(randPart)
	if err != nil {
		errChan <- err
		return
	}
	if n != randBytes {
		errChan <- io.ErrShortBuffer
		return
	}
	name := fmt.Sprintf("%s%x", appName, randPart)
	bucket := getS3Endpoint().Bucket(name)
	if err := bucket.PutBucket(s3.BucketOwnerFull); err != nil {
		errChan <- err
		return
	}
	bucketChan <- *bucket
}

func createIAMCredentials(appName string, keyChan chan iam.AccessKey, errChan chan error) {
	iamEndpoint := getIAMEndpoint()
	uResp, err := iamEndpoint.CreateUser(appName, fmt.Sprintf("/%s/", appName))
	if err != nil {
		errChan <- err
		return
	}
	kResp, err := iamEndpoint.CreateAccessKey(uResp.User.Name)
	if err != nil {
		errChan <- err
		return
	}
	keyChan <- kResp.AccessKey
}

func createBucket(app *App) (*s3Env, error) {
	var env s3Env
	appName := strings.ToLower(app.Name)
	errChan := make(chan error)
	bChan := make(chan s3.Bucket, 1)
	kChan := make(chan iam.AccessKey, 1)
	s := getS3Endpoint()
	go putBucket(appName, bChan, errChan)
	go createIAMCredentials(appName, kChan, errChan)
	iamEndpoint := getIAMEndpoint()
	var userName string
	for env.empty() {
		select {
		case k := <-kChan:
			env.AccessKey = k.Id
			env.SecretKey = k.Secret
			userName = k.UserName
		case bucket := <-bChan:
			env.bucket = bucket.Name
			env.locationConstraint = bucket.S3LocationConstraint
			env.endpoint = bucket.S3Endpoint
		case err := <-errChan:
			switch err.(type) {
			case *iam.Error:
				if env.bucket != "" {
					s.Bucket(env.bucket).DelBucket()
				}
			case *s3.Error:
				if userName != "" {
					iamEndpoint.DeleteUser(userName)
				}
			}
			return nil, err
		}
	}
	policyName := fmt.Sprintf("app-%s-bucket", appName)
	var buf bytes.Buffer
	policy.Execute(&buf, env.bucket)
	if _, err := iamEndpoint.PutUserPolicy(userName, policyName, buf.String()); err != nil {
		return nil, err
	}
	return &env, nil
}

func destroyBucket(app *App) error {
	appName := strings.ToLower(app.Name)
	env := app.InstanceEnv(s3InstanceName)
	accessKeyId := env["TSURU_S3_ACCESS_KEY_ID"].Value
	bucketName := env["TSURU_S3_BUCKET"].Value
	policyName := fmt.Sprintf("app-%s-bucket", appName)
	s3Endpoint := getS3Endpoint()
	iamEndpoint := getIAMEndpoint()
	if _, err := iamEndpoint.DeleteUserPolicy(appName, policyName); err != nil {
		return err
	}
	bucket := s3Endpoint.Bucket(bucketName)
	if err := bucket.DelBucket(); err != nil {
		return err
	}
	if _, err := iamEndpoint.DeleteAccessKey(accessKeyId, appName); err != nil {
		return err
	}
	_, err := iamEndpoint.DeleteUser(appName)
	return err
}
<commit_msg>api/app: removed unnecessary type switching<commit_after>// Copyright 2012 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package app

import (
	"bytes"
	"crypto/rand"
	"fmt"
	"github.com/fsouza/go-iam"
	"github.com/globocom/config"
	"io"
	"launchpad.net/goamz/aws"
	"launchpad.net/goamz/s3"
	"strings"
	"text/template"
)

type s3Env struct {
	aws.Auth
	bucket             string
	endpoint           string
	locationConstraint bool
}

func (s *s3Env) empty() bool {
	return s.bucket == "" || s.AccessKey == "" || s.SecretKey == ""
}

const (
	randBytes      = 32
	s3InstanceName = "tsurus3"
)

var (
	rReader = rand.Reader
	policy  = template.Must(template.New("policy").Parse(`{
  "Statement": [
    {
      "Action": [
        "s3:CreateBucket",
        "s3:DeleteBucket",
        "s3:DeleteBucketPolicy",
        "s3:DeleteBucketWebsite",
        "s3:PutBucketLogging",
        "s3:PutBucketPolicy",
        "s3:PutBucketRequestPayment",
        "s3:PutBucketVersioning",
        "s3:PutBucketWebsite"
      ],
      "Effect": "Deny",
      "Resource": [
        "arn:aws:s3:::{{.}}/*",
        "arn:aws:s3:::{{.}}"
      ]
    },
    {
      "Action": [
        "s3:*"
      ],
      "Effect": "Allow",
      "Resource": [
        "arn:aws:s3:::{{.}}/*",
        "arn:aws:s3:::{{.}}"
      ]
    }
  ]
}`))
)

func getAWSAuth() aws.Auth {
	access, err := config.GetString("aws:access-key-id")
	if err != nil {
		panic("FATAL: aws:access-key-id must be defined in configuration file.")
	}
	secret, err := config.GetString("aws:secret-access-key")
	if err != nil {
		panic("FATAL: aws:secret-access-key must be defined in configuration file.")
	}
	return aws.Auth{
		AccessKey: access,
		SecretKey: secret,
	}
}

func getS3Endpoint() *s3.S3 {
	regionName, _ := config.GetString("aws:s3:region-name")
	endpoint, err := config.GetString("aws:s3:endpoint")
	if err != nil {
		panic("FATAL: aws:s3:endpoint must be defined in configuration file.")
	}
	bucketEndpoint, _ := config.GetString("aws:s3:bucketEndpoint")
	locationConstraint, err := config.GetBool("aws:s3:location-constraint")
	if err != nil {
		panic("FATAL: aws:s3:location-constraint must be defined in configuration file.")
	}
	lowercaseBucket, err := config.GetBool("aws:s3:lowercase-bucket")
	if err != nil {
		panic("FATAL: aws:s3:lowercase-bucket must be defined in configuration file.")
	}
	region := aws.Region{
		Name:                 regionName,
		S3Endpoint:           endpoint,
		S3BucketEndpoint:     bucketEndpoint,
		S3LocationConstraint: locationConstraint,
		S3LowercaseBucket:    lowercaseBucket,
	}
	return s3.New(getAWSAuth(), region)
}

func getIAMEndpoint() *iam.IAM {
	endpoint, err := config.GetString("aws:iam:endpoint")
	if err != nil {
		panic("FATAL: aws:iam:endpoint must be defined in configuration file.")
	}
	region := aws.Region{IAMEndpoint: endpoint}
	return iam.New(getAWSAuth(), region)
}

func putBucket(appName string, bucketChan chan s3.Bucket, errChan chan error) {
	randPart := make([]byte, randBytes)
	n, err := rReader.Read(randPart)
	if err != nil {
		errChan <- err
		return
	}
	if n != randBytes {
		errChan <- io.ErrShortBuffer
		return
	}
	name := fmt.Sprintf("%s%x", appName, randPart)
	bucket := getS3Endpoint().Bucket(name)
	if err := bucket.PutBucket(s3.BucketOwnerFull); err != nil {
		errChan <- err
		return
	}
	bucketChan <- *bucket
}

func createIAMCredentials(appName string, keyChan chan iam.AccessKey, errChan chan error) {
	iamEndpoint := getIAMEndpoint()
	uResp, err := iamEndpoint.CreateUser(appName, fmt.Sprintf("/%s/", appName))
	if err != nil {
		errChan <- err
		return
	}
	kResp, err := iamEndpoint.CreateAccessKey(uResp.User.Name)
	if err != nil {
		errChan <- err
		return
	}
	keyChan <- kResp.AccessKey
}

func createBucket(app *App) (*s3Env, error) {
	var env s3Env
	appName := strings.ToLower(app.Name)
	errChan := make(chan error)
	bChan := make(chan s3.Bucket, 1)
	kChan := make(chan iam.AccessKey, 1)
	s := getS3Endpoint()
	go putBucket(appName, bChan, errChan)
	go createIAMCredentials(appName, kChan, errChan)
	iamEndpoint := getIAMEndpoint()
	var userName string
	for env.empty() {
		select {
		case k := <-kChan:
			env.AccessKey = k.Id
			env.SecretKey = k.Secret
			userName = k.UserName
		case bucket := <-bChan:
			env.bucket = bucket.Name
			env.locationConstraint = bucket.S3LocationConstraint
			env.endpoint = bucket.S3Endpoint
		case err := <-errChan:
			if env.bucket != "" {
				s.Bucket(env.bucket).DelBucket()
			}
			if userName != "" {
				iamEndpoint.DeleteUser(userName)
			}
			return nil, err
		}
	}
	policyName := fmt.Sprintf("app-%s-bucket", appName)
	var buf bytes.Buffer
	policy.Execute(&buf, env.bucket)
	if _, err := iamEndpoint.PutUserPolicy(userName, policyName, buf.String()); err != nil {
		return nil, err
	}
	return &env, nil
}

func destroyBucket(app *App) error {
	appName := strings.ToLower(app.Name)
	env := app.InstanceEnv(s3InstanceName)
	accessKeyId := env["TSURU_S3_ACCESS_KEY_ID"].Value
	bucketName := env["TSURU_S3_BUCKET"].Value
	policyName := fmt.Sprintf("app-%s-bucket", appName)
	s3Endpoint := getS3Endpoint()
	iamEndpoint := getIAMEndpoint()
	if _, err := iamEndpoint.DeleteUserPolicy(appName, policyName); err != nil {
		return err
	}
	bucket := s3Endpoint.Bucket(bucketName)
	if err := bucket.DelBucket(); err != nil {
		return err
	}
	if _, err := iamEndpoint.DeleteAccessKey(accessKeyId, appName); err != nil {
		return err
	}
	_, err := iamEndpoint.DeleteUser(appName)
	return err
}
<|endoftext|>


<commit_before>package cobe

import (
	"regexp"
	"strings"

	"bitbucket.org/tebeka/snowball"
)

type stemmer interface {
	Stem(word string) string
}

// Wrap a snowball stemmer in one that also stems smileys.
type cobeStemmer struct {
	sub    stemmer
	words  *regexp.Regexp
	smiley *regexp.Regexp
	frowny *regexp.Regexp
}

func newCobeStemmer(s *snowball.Stemmer) *cobeStemmer {
	cs := cobeStemmer{sub: s}
	cs.words = regexp.MustCompile(`\w`)
	cs.smiley = regexp.MustCompile(`:-?[ \)]*\)|☺|☺`)
	cs.frowny = regexp.MustCompile(`:-?[' \(]*\(`)

	return &cs
}

func (s *cobeStemmer) Stem(token string) string {
	// Tokens with a word character go through the snowball stemmer.
	if s.words.FindString(token) != "" {
		return s.sub.Stem(strings.ToLower(token))
	}

	if s.smiley.FindString(token) != "" {
		return ":)"
	}

	if s.frowny.FindString(token) != "" {
		return ":("
	}

	return ""
}
<commit_msg>Frowning face recognition symmetry<commit_after>package cobe

import (
	"regexp"
	"strings"

	"bitbucket.org/tebeka/snowball"
)

type stemmer interface {
	Stem(word string) string
}

// Wrap a snowball stemmer in one that also stems smileys.
type cobeStemmer struct {
	sub    stemmer
	words  *regexp.Regexp
	smiley *regexp.Regexp
	frowny *regexp.Regexp
}

func newCobeStemmer(s *snowball.Stemmer) *cobeStemmer {
	cs := cobeStemmer{sub: s}
	cs.words = regexp.MustCompile(`\w`)
	cs.smiley = regexp.MustCompile(`:-?[ \)]*\)|☺|☺`)
	cs.frowny = regexp.MustCompile(`:-?[' \(]*\(|☹|😦`)

	return &cs
}

func (s *cobeStemmer) Stem(token string) string {
	// Tokens with a word character go through the snowball stemmer.
	if s.words.FindString(token) != "" {
		return s.sub.Stem(strings.ToLower(token))
	}

	if s.smiley.FindString(token) != "" {
		return ":)"
	}

	if s.frowny.FindString(token) != "" {
		return ":("
	}

	return ""
}
<|endoftext|>