<commit_before>package main

import (
	"flag"
	"io/ioutil"
	"log"
	"strconv"
	"time"

	"github.com/brotherlogic/goserver"
	"golang.org/x/net/context"
	"google.golang.org/grpc"

	pbd "github.com/brotherlogic/discovery/proto"
	pb "github.com/brotherlogic/gobuildmaster/proto"
	pbs "github.com/brotherlogic/gobuildslave/proto"
	pbg "github.com/brotherlogic/goserver/proto"
	"github.com/brotherlogic/goserver/utils"
)

const (
	intentWait = time.Second
)

// Server the main server type
type Server struct {
	*goserver.GoServer
	config     *pb.Config
	serving    bool
	LastIntent time.Time
}

type mainChecker struct {
	prev []string
}

func getIP(servertype, servername string) (string, int) {
	conn, err := grpc.Dial(utils.RegistryIP+":"+strconv.Itoa(utils.RegistryPort), grpc.WithInsecure())
	log.Printf("Error? %v", err)
	defer conn.Close()

	registry := pbd.NewDiscoveryServiceClient(conn)
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	r, err := registry.ListAllServices(ctx, &pbd.Empty{}, grpc.FailFast(false))
	if err != nil {
		return "", -1
	}
	for _, s := range r.Services {
		if s.Name == servertype && s.Identifier == servername {
			return s.Ip, int(s.Port)
		}
	}

	return "", -1
}

func (t *mainChecker) getprev() []string {
	return t.prev
}
func (t *mainChecker) setprev(v []string) {
	t.prev = v
}

func (t *mainChecker) assess(server string) (*pbs.JobList, *pbs.Config) {
	list := &pbs.JobList{}
	conf := &pbs.Config{}

	ip, port := getIP("gobuildslave", server)
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	conn, err := grpc.DialContext(ctx, ip+":"+strconv.Itoa(port), grpc.WithInsecure())
	if err != nil {
		return list, conf
	}
	defer conn.Close()

	slave := pbs.NewGoBuildSlaveClient(conn)
	r, err := slave.List(ctx, &pbs.Empty{}, grpc.FailFast(false))
	if err != nil {
		return list, conf
	}

	r2, err := slave.GetConfig(ctx, &pbs.Empty{}, grpc.FailFast(false))
	if err != nil {
		return list, conf
	}

	return r, r2
}

func (t *mainChecker) master(entry *pbd.RegistryEntry, master bool) bool {
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	conn, _ := grpc.DialContext(ctx, entry.GetIp()+":"+strconv.Itoa(int(entry.GetPort())), grpc.WithInsecure())
	defer conn.Close()

	server := pbg.NewGoserverServiceClient(conn)
	log.Printf("SETTING MASTER: %v", entry)
	_, err := server.Mote(ctx, &pbg.MoteRequest{Master: master}, grpc.FailFast(false))
	if err != nil {
		log.Printf("RESPONSE: %v", err)
	}

	log.Printf("SET MASTER on %v", entry)
	return err == nil
}

func runJob(job *pbs.JobSpec, server string) {
	log.Printf("Run %v on %v", job.Name, server)
	if server != "" {
		ip, port := getIP("gobuildslave", server)
		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
		defer cancel()
		conn, _ := grpc.DialContext(ctx, ip+":"+strconv.Itoa(port), grpc.WithInsecure())
		defer conn.Close()

		slave := pbs.NewGoBuildSlaveClient(conn)
		job.Server = server
		slave.Run(ctx, job, grpc.FailFast(false))
	}
}

func (t *mainChecker) discover() *pbd.ServiceList {
	ret := &pbd.ServiceList{}

	conn, _ := grpc.Dial(utils.RegistryIP+":"+strconv.Itoa(utils.RegistryPort), grpc.WithInsecure())
	defer conn.Close()

	registry := pbd.NewDiscoveryServiceClient(conn)
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	r, err := registry.ListAllServices(ctx, &pbd.Empty{}, grpc.FailFast(false))
	if err == nil {
		for _, s := range r.Services {
			ret.Services = append(ret.Services, s)
		}
	}

	return ret
}

// DoRegister Registers this server
func (s Server) DoRegister(server *grpc.Server) {
	pb.RegisterGoBuildMasterServer(server, &s)
}

// ReportHealth determines if the server is healthy
func (s Server) ReportHealth() bool {
	return true
}

// Mote promotes/demotes this server
func (s Server) Mote(master bool) error {
	return nil
}

//GetState gets the state of the server
func (s Server) GetState() []*pbg.State {
	log.Printf("GETTING STATE: %p", &s)
	return []*pbg.State{&pbg.State{Key: "last_intent", TimeValue: s.LastIntent.Unix()}}
}

//Compare compares current state to desired state
func (s Server) Compare(ctx context.Context, in *pb.Empty) (*pb.CompareResponse, error) {
	resp := &pb.CompareResponse{}
	list, _ := getFleetStatus(&mainChecker{})
	cc := &pb.Config{}
	for _, jlist := range list {
		for _, job := range jlist.GetDetails() {
			cc.Intents = append(cc.Intents, &pb.Intent{Spec: job.GetSpec()})
		}
	}
	resp.Current = cc
	resp.Desired = s.config

	return resp, nil
}

func getConfig(c checker) *pb.Config {
	list, _ := getFleetStatus(c)
	config := &pb.Config{}

	for _, jlist := range list {
		for _, job := range jlist.Details {
			found := false
			for _, ij := range config.Intents {
				if job.Spec.Name == ij.Spec.Name {
					ij.Count++
					found = true
				}
			}

			if !found {
				config.Intents = append(config.Intents, &pb.Intent{Spec: &pbs.JobSpec{Name: job.Spec.Name}, Count: 1})
			}
		}
	}

	return config
}

// MatchIntent tries to match the intent with the state of production
func (s *Server) MatchIntent() {
	checker := &mainChecker{}
	log.Printf("SERVING: %v", s.serving)
	for s.serving {
		time.Sleep(intentWait)
		s.LastIntent = time.Now()
		log.Printf("SETTING %p", s)

		state := getConfig(checker)
		diff := configDiff(s.config, state)
		joblist := runJobs(diff)
		for _, job := range joblist {
			runJob(job, chooseServer(job, checker))
		}
	}
}

// SetMaster sets up the master settings
func (s *Server) SetMaster() {
	checker := &mainChecker{}
	for s.serving {
		time.Sleep(intentWait)

		log.Printf("Running SetMaster on %p", s)

		fleet := checker.discover()
		matcher := make(map[string][]*pbd.RegistryEntry)
		hasMaster := make(map[string]int)
		for _, entry := range fleet.GetServices() {
			if _, ok := matcher[entry.GetName()]; !ok {
				if entry.GetMaster() {
					hasMaster[entry.GetName()]++
				}
				matcher[entry.GetName()] = []*pbd.RegistryEntry{entry}
			} else {
				if entry.GetMaster() {
					hasMaster[entry.GetName()] = 1
					matcher[entry.GetName()] = append(matcher[entry.GetName()], entry)
				}
			}
		}

		for key, entries := range matcher {
			if hasMaster[key] > 1 {
				hasMaster[key] = 1
				seen := false
				for _, entry := range entries {
					if seen && entry.GetMaster() {
						checker.master(entry, false)
					} else if entry.GetMaster() {
						seen = true
					}
				}
			}

			if hasMaster[key] == 0 {
				log.Printf("%v HAS NO MASTER", key)
				for _, entry := range entries {
					log.Printf("CHECKING %v for %v", entry, key)
					if checker.master(entry, true) {
						entry.Master = true
						log.Printf("AND NOW: %v", entry)
						break
					}
				}
			}
		}
	}
}

func main() {
	config, err := loadConfig("config.pb")
	if err != nil {
		log.Fatalf("Fatal loading of config: %v", err)
	}

	var sync = flag.Bool("once", false, "One pass intent match")
	s := &Server{&goserver.GoServer{}, config, true, time.Now()}

	var quiet = flag.Bool("quiet", true, "Show all output")
	flag.Parse()

	if *quiet {
		log.SetFlags(0)
		log.SetOutput(ioutil.Discard)
	}

	if *sync {
		s.MatchIntent()
	} else {
		s.Register = s
		s.PrepServer()
		s.GoServer.Killme = false
		s.RegisterServer("gobuildmaster", false)
		s.RegisterServingTask(s.MatchIntent)
		s.RegisterServingTask(s.SetMaster)
		s.Serve()
	}
}
<commit_msg>Adding logging of master fails<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"strconv"
	"time"

	"github.com/brotherlogic/goserver"
	"golang.org/x/net/context"
	"google.golang.org/grpc"

	pbd "github.com/brotherlogic/discovery/proto"
	pb "github.com/brotherlogic/gobuildmaster/proto"
	pbs "github.com/brotherlogic/gobuildslave/proto"
	pbg "github.com/brotherlogic/goserver/proto"
	"github.com/brotherlogic/goserver/utils"
)

const (
	intentWait = time.Second
)

// Server the main server type
type Server struct {
	*goserver.GoServer
	config     *pb.Config
	serving    bool
	LastIntent time.Time
}

type mainChecker struct {
	prev   []string
	logger func(string)
}

func getIP(servertype, servername string) (string, int) {
	conn, err := grpc.Dial(utils.RegistryIP+":"+strconv.Itoa(utils.RegistryPort), grpc.WithInsecure())
	log.Printf("Error? %v", err)
	defer conn.Close()

	registry := pbd.NewDiscoveryServiceClient(conn)
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	r, err := registry.ListAllServices(ctx, &pbd.Empty{}, grpc.FailFast(false))
	if err != nil {
		return "", -1
	}
	for _, s := range r.Services {
		if s.Name == servertype && s.Identifier == servername {
			return s.Ip, int(s.Port)
		}
	}

	return "", -1
}

func (t *mainChecker) getprev() []string {
	return t.prev
}
func (t *mainChecker) setprev(v []string) {
	t.prev = v
}

func (t *mainChecker) assess(server string) (*pbs.JobList, *pbs.Config) {
	list := &pbs.JobList{}
	conf := &pbs.Config{}

	ip, port := getIP("gobuildslave", server)
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	conn, err := grpc.DialContext(ctx, ip+":"+strconv.Itoa(port), grpc.WithInsecure())
	if err != nil {
		return list, conf
	}
	defer conn.Close()

	slave := pbs.NewGoBuildSlaveClient(conn)
	r, err := slave.List(ctx, &pbs.Empty{}, grpc.FailFast(false))
	if err != nil {
		return list, conf
	}

	r2, err := slave.GetConfig(ctx, &pbs.Empty{}, grpc.FailFast(false))
	if err != nil {
		return list, conf
	}

	return r, r2
}

func (t *mainChecker) master(entry *pbd.RegistryEntry, master bool) bool {
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	conn, _ := grpc.DialContext(ctx, entry.GetIp()+":"+strconv.Itoa(int(entry.GetPort())), grpc.WithInsecure())
	defer conn.Close()

	server := pbg.NewGoserverServiceClient(conn)
	log.Printf("SETTING MASTER: %v", entry)
	_, err := server.Mote(ctx, &pbg.MoteRequest{Master: master}, grpc.FailFast(false))
	if err != nil {
		log.Printf("RESPONSE: %v", err)
		t.logger(fmt.Sprintf("Master REJECT: %v", err))
	}

	log.Printf("SET MASTER on %v", entry)
	return err == nil
}

func runJob(job *pbs.JobSpec, server string) {
	log.Printf("Run %v on %v", job.Name, server)
	if server != "" {
		ip, port := getIP("gobuildslave", server)
		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
		defer cancel()
		conn, _ := grpc.DialContext(ctx, ip+":"+strconv.Itoa(port), grpc.WithInsecure())
		defer conn.Close()

		slave := pbs.NewGoBuildSlaveClient(conn)
		job.Server = server
		slave.Run(ctx, job, grpc.FailFast(false))
	}
}

func (t *mainChecker) discover() *pbd.ServiceList {
	ret := &pbd.ServiceList{}

	conn, _ := grpc.Dial(utils.RegistryIP+":"+strconv.Itoa(utils.RegistryPort), grpc.WithInsecure())
	defer conn.Close()

	registry := pbd.NewDiscoveryServiceClient(conn)
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	r, err := registry.ListAllServices(ctx, &pbd.Empty{}, grpc.FailFast(false))
	if err == nil {
		for _, s := range r.Services {
			ret.Services = append(ret.Services, s)
		}
	}

	return ret
}

// DoRegister Registers this server
func (s Server) DoRegister(server *grpc.Server) {
	pb.RegisterGoBuildMasterServer(server, &s)
}

// ReportHealth determines if the server is healthy
func (s Server) ReportHealth() bool {
	return true
}

// Mote promotes/demotes this server
func (s Server) Mote(master bool) error {
	return nil
}

//GetState gets the state of the server
func (s Server) GetState() []*pbg.State {
	log.Printf("GETTING STATE: %p", &s)
	return []*pbg.State{&pbg.State{Key: "last_intent", TimeValue: s.LastIntent.Unix()}}
}

//Compare compares current state to desired state
func (s Server) Compare(ctx context.Context, in *pb.Empty) (*pb.CompareResponse, error) {
	resp := &pb.CompareResponse{}
	list, _ := getFleetStatus(&mainChecker{logger: s.Log})
	cc := &pb.Config{}
	for _, jlist := range list {
		for _, job := range jlist.GetDetails() {
			cc.Intents = append(cc.Intents, &pb.Intent{Spec: job.GetSpec()})
		}
	}
	resp.Current = cc
	resp.Desired = s.config

	return resp, nil
}

func getConfig(c checker) *pb.Config {
	list, _ := getFleetStatus(c)
	config := &pb.Config{}

	for _, jlist := range list {
		for _, job := range jlist.Details {
			found := false
			for _, ij := range config.Intents {
				if job.Spec.Name == ij.Spec.Name {
					ij.Count++
					found = true
				}
			}

			if !found {
				config.Intents = append(config.Intents, &pb.Intent{Spec: &pbs.JobSpec{Name: job.Spec.Name}, Count: 1})
			}
		}
	}

	return config
}

// MatchIntent tries to match the intent with the state of production
func (s *Server) MatchIntent() {
	checker := &mainChecker{logger: s.Log}
	log.Printf("SERVING: %v", s.serving)
	for s.serving {
		time.Sleep(intentWait)
		s.LastIntent = time.Now()
		log.Printf("SETTING %p", s)

		state := getConfig(checker)
		diff := configDiff(s.config, state)
		joblist := runJobs(diff)
		for _, job := range joblist {
			runJob(job, chooseServer(job, checker))
		}
	}
}

// SetMaster sets up the master settings
func (s *Server) SetMaster() {
	checker := &mainChecker{logger: s.Log}
	for s.serving {
		time.Sleep(intentWait)

		log.Printf("Running SetMaster on %p", s)

		fleet := checker.discover()
		matcher := make(map[string][]*pbd.RegistryEntry)
		hasMaster := make(map[string]int)
		for _, entry := range fleet.GetServices() {
			if _, ok := matcher[entry.GetName()]; !ok {
				if entry.GetMaster() {
					hasMaster[entry.GetName()]++
				}
				matcher[entry.GetName()] = []*pbd.RegistryEntry{entry}
			} else {
				if entry.GetMaster() {
					hasMaster[entry.GetName()] = 1
					matcher[entry.GetName()] = append(matcher[entry.GetName()], entry)
				}
			}
		}

		for key, entries := range matcher {
			if hasMaster[key] > 1 {
				hasMaster[key] = 1
				seen := false
				for _, entry := range entries {
					if seen && entry.GetMaster() {
						checker.master(entry, false)
					} else if entry.GetMaster() {
						seen = true
					}
				}
			}

			if hasMaster[key] == 0 {
				log.Printf("%v HAS NO MASTER", key)
				for _, entry := range entries {
					log.Printf("CHECKING %v for %v", entry, key)
					if checker.master(entry, true) {
						entry.Master = true
						log.Printf("AND NOW: %v", entry)
						break
					}
				}
			}
		}
	}
}

func main() {
	config, err := loadConfig("config.pb")
	if err != nil {
		log.Fatalf("Fatal loading of config: %v", err)
	}

	var sync = flag.Bool("once", false, "One pass intent match")
	s := &Server{&goserver.GoServer{}, config, true, time.Now()}

	var quiet = flag.Bool("quiet", true, "Show all output")
	flag.Parse()

	if *quiet {
		log.SetFlags(0)
		log.SetOutput(ioutil.Discard)
	}

	if *sync {
		s.MatchIntent()
	} else {
		s.Register = s
		s.PrepServer()
		s.GoServer.Killme = false
		s.RegisterServer("gobuildmaster", false)
		s.RegisterServingTask(s.MatchIntent)
		s.RegisterServingTask(s.SetMaster)
		s.Serve()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/hex"
	"os"
	"strings"

	"golang.org/x/crypto/ssh/terminal"

	"github.com/rfjakob/gocryptfs/internal/cryptocore"
	"github.com/rfjakob/gocryptfs/internal/tlog"
)

// printMasterKey - remind the user that he should store the master key in
// a safe place
func printMasterKey(key []byte) {
	if !terminal.IsTerminal(int(os.Stdout.Fd())) {
		// We don't want the master key to end up in a log file
		tlog.Info.Printf("Not running on a terminal, suppressing master key display\n")
		return
	}
	h := hex.EncodeToString(key)
	var hChunked string
	// Try to make it less scary by splitting it up in chunks
	for i := 0; i < len(h); i += 8 {
		hChunked += h[i : i+8]
		if i < 52 {
			hChunked += "-"
		}
		if i == 24 {
			hChunked += "\n    "
		}
	}
	tlog.Info.Printf(`
Your master key is:

    %s

If the gocryptfs.conf file becomes corrupted or you ever forget your password,
there is only one hope for recovery: The master key. Print it to a piece of
paper and store it in a drawer.

`, tlog.ColorGrey+hChunked+tlog.ColorReset)
}

// parseMasterKey - Parse a hex-encoded master key that was passed on the command line
// Calls os.Exit on failure
func parseMasterKey(masterkey string) []byte {
	masterkey = strings.Replace(masterkey, "-", "", -1)
	key, err := hex.DecodeString(masterkey)
	if err != nil {
		tlog.Fatal.Printf("Could not parse master key: %v", err)
		os.Exit(1)
	}
	if len(key) != cryptocore.KeyLen {
		tlog.Fatal.Printf("Master key has length %d but we require length %d", len(key), cryptocore.KeyLen)
		os.Exit(1)
	}
	tlog.Info.Printf("Using explicit master key.")
	tlog.Info.Printf(tlog.ColorYellow +
		"THE MASTER KEY IS VISIBLE VIA \"ps ax\" AND MAY BE STORED IN YOUR SHELL HISTORY!\n" +
		"ONLY USE THIS MODE FOR EMERGENCIES." + tlog.ColorReset)
	return key
}
<commit_msg>main: suggest "-q" to suppress master key message<commit_after>package main

import (
	"encoding/hex"
	"os"
	"strings"

	"golang.org/x/crypto/ssh/terminal"

	"github.com/rfjakob/gocryptfs/internal/cryptocore"
	"github.com/rfjakob/gocryptfs/internal/tlog"
)

// printMasterKey - remind the user that he should store the master key in
// a safe place
func printMasterKey(key []byte) {
	if !terminal.IsTerminal(int(os.Stdout.Fd())) {
		// We don't want the master key to end up in a log file
		tlog.Info.Printf("Not running on a terminal, suppressing master key display\n")
		return
	}
	h := hex.EncodeToString(key)
	var hChunked string
	// Try to make it less scary by splitting it up in chunks
	for i := 0; i < len(h); i += 8 {
		hChunked += h[i : i+8]
		if i < 52 {
			hChunked += "-"
		}
		if i == 24 {
			hChunked += "\n    "
		}
	}
	tlog.Info.Printf(`
Your master key is:

    %s

If the gocryptfs.conf file becomes corrupted or you ever forget your password,
there is only one hope for recovery: The master key. Print it to a piece of
paper and store it in a drawer. Use "-q" to suppress this message.

`, tlog.ColorGrey+hChunked+tlog.ColorReset)
}

// parseMasterKey - Parse a hex-encoded master key that was passed on the command line
// Calls os.Exit on failure
func parseMasterKey(masterkey string) []byte {
	masterkey = strings.Replace(masterkey, "-", "", -1)
	key, err := hex.DecodeString(masterkey)
	if err != nil {
		tlog.Fatal.Printf("Could not parse master key: %v", err)
		os.Exit(1)
	}
	if len(key) != cryptocore.KeyLen {
		tlog.Fatal.Printf("Master key has length %d but we require length %d", len(key), cryptocore.KeyLen)
		os.Exit(1)
	}
	tlog.Info.Printf("Using explicit master key.")
	tlog.Info.Printf(tlog.ColorYellow +
		"THE MASTER KEY IS VISIBLE VIA \"ps ax\" AND MAY BE STORED IN YOUR SHELL HISTORY!\n" +
		"ONLY USE THIS MODE FOR EMERGENCIES." + tlog.ColorReset)
	return key
}
<|endoftext|>


<commit_before>package neurgo

import (
	"github.com/couchbaselabs/go.assert"
	"testing"
)

func fakeLayerToNeuronMap() (LayerToNeuronMap, *Neuron, *Neuron, *Neuron) {
	layerToNeuronMap := make(LayerToNeuronMap)

	neuron1 := &Neuron{
		ActivationFunction: EncodableSigmoid(),
		NodeId:             NewNeuronId("neuron1", 0.25),
		Bias:               -30,
	}
	neurons25 := []*Neuron{neuron1}
	layerToNeuronMap[0.25] = neurons25

	neuron2 := &Neuron{
		ActivationFunction: EncodableSigmoid(),
		NodeId:             NewNeuronId("neuron2", 0.35),
		Bias:               -30,
	}
	neurons35 := []*Neuron{neuron2}
	layerToNeuronMap[0.35] = neurons35

	neuron3 := &Neuron{
		ActivationFunction: EncodableSigmoid(),
		NodeId:             NewNeuronId("neuron3", 0.45),
		Bias:               -30,
	}
	neurons45 := []*Neuron{neuron3}
	layerToNeuronMap[0.45] = neurons45
	return layerToNeuronMap, neuron1, neuron2, neuron3

}

func TestChooseRandomLayer(t *testing.T) {
	layerToNeuronMap := make(LayerToNeuronMap)
	neurons := make([]*Neuron, 0)
	layerToNeuronMap[0.0] = neurons
	layerToNeuronMap[0.25] = neurons

	foundFirstLayer := false
	foundSecondLayer := false

	for i := 0; i < 20; i++ {
		layerIndex := layerToNeuronMap.ChooseRandomLayer()
		if layerIndex == 0.0 {
			foundFirstLayer = true
		}
		if layerIndex == 0.25 {
			foundSecondLayer = true
		}
	}

	assert.True(t, foundFirstLayer)
	assert.True(t, foundSecondLayer)

}

func TestChooseNeuronPrecedingLayer(t *testing.T) {

	layerToNeuronMap, neuron1, neuron2, neuron3 := fakeLayerToNeuronMap()

	foundNeuron1 := false
	foundNeuron2 := false
	foundNeuron3 := false

	for i := 0; i < 20; i++ {
		chosenNeuron := layerToNeuronMap.ChooseNeuronPrecedingLayer(0.45)
		switch chosenNeuron.NodeId.UUID {
		case neuron1.NodeId.UUID:
			foundNeuron1 = true
		case neuron2.NodeId.UUID:
			foundNeuron2 = true
		case neuron3.NodeId.UUID:
			foundNeuron3 = true
		}

	}

	assert.True(t, foundNeuron1)
	assert.True(t, foundNeuron2)
	assert.False(t, foundNeuron3)

}

func TestChooseNeuronFollowingLayer(t *testing.T) {

	layerToNeuronMap, neuron1, neuron2, neuron3 := fakeLayerToNeuronMap()

	foundNeuron1 := false
	foundNeuron2 := false
	foundNeuron3 := false

	for i := 0; i < 20; i++ {
		chosenNeuron := layerToNeuronMap.ChooseNeuronFollowingLayer(0.25)
		switch chosenNeuron.NodeId.UUID {
		case neuron1.NodeId.UUID:
			foundNeuron1 = true
		case neuron2.NodeId.UUID:
			foundNeuron2 = true
		case neuron3.NodeId.UUID:
			foundNeuron3 = true
		}

	}

	assert.False(t, foundNeuron1)
	assert.True(t, foundNeuron2)
	assert.True(t, foundNeuron3)

}
<commit_msg>change tests to use nodeid<commit_after>package neurgo

import (
	"github.com/couchbaselabs/go.assert"
	"testing"
)

func fakeLayerToNodeIdMap() (LayerToNodeIdMap, *NodeId, *NodeId, *NodeId) {
	layerToNodeIdMap := make(LayerToNodeIdMap)

	nodeId1 := NewNeuronId("nodeId1", 0.25)
	nodeIds25 := []*NodeId{nodeId1}
	layerToNodeIdMap[0.25] = nodeIds25

	nodeId2 := NewNeuronId("nodeId2", 0.35)
	nodeIds35 := []*NodeId{nodeId2}
	layerToNodeIdMap[0.35] = nodeIds35

	nodeId3 := NewNeuronId("nodeId3", 0.45)
	nodeIds45 := []*NodeId{nodeId3}
	layerToNodeIdMap[0.45] = nodeIds45
	return layerToNodeIdMap, nodeId1, nodeId2, nodeId3

}

func TestChooseRandomLayer(t *testing.T) {
	layerToNodeIdMap := make(LayerToNodeIdMap)
	nodeIds := make([]*NodeId, 0)
	layerToNodeIdMap[0.0] = nodeIds
	layerToNodeIdMap[0.25] = nodeIds

	foundFirstLayer := false
	foundSecondLayer := false

	for i := 0; i < 20; i++ {
		layerIndex := layerToNodeIdMap.ChooseRandomLayer()
		if layerIndex == 0.0 {
			foundFirstLayer = true
		}
		if layerIndex == 0.25 {
			foundSecondLayer = true
		}
	}

	assert.True(t, foundFirstLayer)
	assert.True(t, foundSecondLayer)

}

func TestChooseNodeIdPrecedingLayer(t *testing.T) {

	layerToNodeIdMap, nodeId1, nodeId2, nodeId3 := fakeLayerToNodeIdMap()

	foundNodeId1 := false
	foundNodeId2 := false
	foundNodeId3 := false

	for i := 0; i < 20; i++ {
		chosenNodeId := layerToNodeIdMap.ChooseNodeIdPrecedingLayer(0.45)
		switch chosenNodeId.UUID {
		case nodeId1.UUID:
			foundNodeId1 = true
		case nodeId2.UUID:
			foundNodeId2 = true
		case nodeId3.UUID:
			foundNodeId3 = true
		}

	}

	assert.True(t, foundNodeId1)
	assert.True(t, foundNodeId2)
	assert.False(t, foundNodeId3)

}

func TestChooseNodeIdFollowingLayer(t *testing.T) {

	layerToNodeIdMap, nodeId1, nodeId2, nodeId3 := fakeLayerToNodeIdMap()

	foundNodeId1 := false
	foundNodeId2 := false
	foundNodeId3 := false

	for i := 0; i < 20; i++ {
		chosenNodeId := layerToNodeIdMap.ChooseNodeIdFollowingLayer(0.25)
		switch chosenNodeId.UUID {
		case nodeId1.UUID:
			foundNodeId1 = true
		case nodeId2.UUID:
			foundNodeId2 = true
		case nodeId3.UUID:
			foundNodeId3 = true
		}

	}

	assert.False(t, foundNodeId1)
	assert.True(t, foundNodeId2)
	assert.True(t, foundNodeId3)

}
<|endoftext|>


<commit_before>package lbx

import (
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
)

type datapkgType struct {
	Path    string
	Version string
}

func (i datapkgType) less(j datapkgType) bool {
	ii := i.version()
	jj := j.version()
	nn := len(ii)
	if nn > len(jj) {
		nn = len(jj)
	}
	for idx := 0; idx < nn; idx++ {
		if ii[idx] != jj[idx] {
			return ii[idx] < jj[idx]
		}
	}
	return len(ii) < len(jj)
}

func (dp datapkgType) version() []int {
	re := regexp.MustCompile(`\d+`)
	slice := re.Split(dp.Version, -1)
	v := make([]int, 0, len(slice))
	for _, str := range slice {
		vv, err := strconv.Atoi(str)
		if err != nil {
			panic(fmt.Errorf("lbx.datapkgType.version: %v (string=%q)", err, str))
		}
		v = append(v, vv)
	}
	return v
}

type datapkgTypes []datapkgType

func (p datapkgTypes) Len() int           { return len(p) }
func (p datapkgTypes) Less(i, j int) bool { return p[i].Version < p[j].Version }
func (p datapkgTypes) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

// FindDataPackage finds a data package in the directories specified in paths,
// using optionally the standard suffixes "DBASE", "PARAM" and "EXTRAPACKAGES".
func FindDataPackage(paths []string, name, version string) (string, error) {
	suffixes := []string{"", "EXTRAPACKAGES", "DBASE", "PARAM"}
	versions := make([]datapkgType, 0, 1)

	for _, path := range paths {
		for _, suffix := range suffixes {
			p := filepath.Join(path, suffix, name)
			if _, err := os.Stat(p); err != nil {
				continue
			}
			list, err := filepath.Glob(p + "/*")
			if err != nil {
				return "", err
			}
			for _, v := range list {
				if v == version {
					// stop searching if we've found an exact match
					return filepath.Join(p, v), nil
				}
				if ok, err := filepath.Match(version, v); ok && err == nil {
					versions = append(versions, datapkgType{Path: p, Version: v})
				}
			}
		}
	}

	if len(versions) <= 0 {
		return "", fmt.Errorf("lbx: could not find data package %[1]q %[2]q in %[3]v",
			name, version, paths,
		)
	}

	sort.Reverse(datapkgTypes(versions))
	sel := versions[0]
	return filepath.Join(sel.Path, sel.Version), nil
}
<commit_msg>lookup: attach FindDataPackage to Context<commit_after>package lbx

import (
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
)

type datapkgType struct {
	Path    string
	Version string
}

func (i datapkgType) less(j datapkgType) bool {
	ii := i.version()
	jj := j.version()
	nn := len(ii)
	if nn > len(jj) {
		nn = len(jj)
	}
	for idx := 0; idx < nn; idx++ {
		if ii[idx] != jj[idx] {
			return ii[idx] < jj[idx]
		}
	}
	return len(ii) < len(jj)
}

func (dp datapkgType) version() []int {
	re := regexp.MustCompile(`\d+`)
	slice := re.Split(dp.Version, -1)
	v := make([]int, 0, len(slice))
	for _, str := range slice {
		vv, err := strconv.Atoi(str)
		if err != nil {
			panic(fmt.Errorf("lbx.datapkgType.version: %v (string=%q)", err, str))
		}
		v = append(v, vv)
	}
	return v
}

type datapkgTypes []datapkgType

func (p datapkgTypes) Len() int           { return len(p) }
func (p datapkgTypes) Less(i, j int) bool { return p[i].Version < p[j].Version }
func (p datapkgTypes) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

// FindDataPackage finds a data package among the Context.ProjectsPath,
// using optionally the standard suffixes "DBASE", "PARAM" and "EXTRAPACKAGES".
func (ctx *Context) FindDataPackage(name, version string) (string, error) {
	suffixes := []string{"", "EXTRAPACKAGES", "DBASE", "PARAM"}
	versions := make([]datapkgType, 0, 1)

	for _, path := range ctx.ProjectsPath {
		for _, suffix := range suffixes {
			p := filepath.Join(path, suffix, name)
			if _, err := os.Stat(p); err != nil {
				continue
			}
			list, err := filepath.Glob(p + "/*")
			if err != nil {
				return "", err
			}
			for _, v := range list {
				if v == version {
					// stop searching if we've found an exact match
					return filepath.Join(p, v), nil
				}
				if ok, err := filepath.Match(version, v); ok && err == nil {
					versions = append(versions, datapkgType{Path: p, Version: v})
				}
			}
		}
	}

	if len(versions) <= 0 {
		return "", fmt.Errorf("lbx: could not find data package %[1]q %[2]q in %[3]v",
			name, version, ctx.ProjectsPath,
		)
	}

	sort.Reverse(datapkgTypes(versions))
	sel := versions[0]
	return filepath.Join(sel.Path, sel.Version), nil
}
<|endoftext|>


<commit_before>package vegeta

import (
	"crypto/tls"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"strings"
	"sync"
	"time"
)

// Attacker is an attack executor which wraps an http.Client
type Attacker struct {
	dialer    *net.Dialer
	client    http.Client
	stop      chan struct{}
	workers   uint64
	redirects int
}

const (
	// DefaultRedirects is the default number of times an Attacker follows
	// redirects.
	DefaultRedirects = 10
	// DefaultTimeout is the default amount of time an Attacker waits for a request
	// before it times out.
	DefaultTimeout = 30 * time.Second
	// NoFollow is the value when redirects are not followed but marked successful
	NoFollow = -1
)

var (
	// DefaultLocalAddr is the default local IP address an Attacker uses.
	DefaultLocalAddr = net.IPAddr{IP: net.IPv4zero}
	// DefaultTLSConfig is the default tls.Config an Attacker uses.
	DefaultTLSConfig = &tls.Config{InsecureSkipVerify: true}
)

// NewAttacker returns a new Attacker with default options which are overridden
// by the optionally provided opts.
func NewAttacker(opts ...func(*Attacker)) *Attacker {
	a := &Attacker{stop: make(chan struct{})}
	a.dialer = &net.Dialer{
		LocalAddr: &net.TCPAddr{IP: DefaultLocalAddr.IP, Zone: DefaultLocalAddr.Zone},
		KeepAlive: 30 * time.Second,
		Timeout:   DefaultTimeout,
	}
	a.client = http.Client{
		Transport: &http.Transport{
			Proxy: http.ProxyFromEnvironment,
			Dial:  a.dialer.Dial,
			ResponseHeaderTimeout: DefaultTimeout,
			TLSClientConfig:       DefaultTLSConfig,
			TLSHandshakeTimeout:   10 * time.Second,
		},
	}
	for _, opt := range opts {
		opt(a)
	}
	return a
}

// Workers returns a functional option which sets the number of workers
// an Attacker uses to hit its targets.
// If zero or greater than the total number of hits, workers will be capped
// to that maximum.
func Workers(n uint64) func(*Attacker) {
	return func(a *Attacker) { a.workers = n }
}

// Redirects returns a functional option which sets the maximum
// number of redirects an Attacker will follow.
func Redirects(n int) func(*Attacker) {
	return func(a *Attacker) {
		a.redirects = n
		a.client.CheckRedirect = func(_ *http.Request, via []*http.Request) error {
			if len(via) > n {
				return fmt.Errorf("stopped after %d redirects", n)
			}
			return nil
		}
	}
}

// Timeout returns a functional option which sets the maximum amount of time
// an Attacker will wait for a request to be responded to.
func Timeout(d time.Duration) func(*Attacker) {
	return func(a *Attacker) {
		tr := a.client.Transport.(*http.Transport)
		tr.ResponseHeaderTimeout = d
		a.dialer.Timeout = d
		tr.Dial = a.dialer.Dial
	}
}

// LocalAddr returns a functional option which sets the local address
// an Attacker will use with its requests.
func LocalAddr(addr net.IPAddr) func(*Attacker) {
	return func(a *Attacker) {
		tr := a.client.Transport.(*http.Transport)
		a.dialer.LocalAddr = &net.TCPAddr{IP: addr.IP, Zone: addr.Zone}
		tr.Dial = a.dialer.Dial
	}
}

// KeepAlive returns a functional option which toggles KeepAlive
// connections on the dialer and transport.
func KeepAlive(keepalive bool) func(*Attacker) {
	return func(a *Attacker) {
		tr := a.client.Transport.(*http.Transport)
		tr.DisableKeepAlives = !keepalive
		if !keepalive {
			a.dialer.KeepAlive = 0
			tr.Dial = a.dialer.Dial
		}
	}
}

// TLSConfig returns a functional option which sets the *tls.Config for a
// Attacker to use with its requests.
func TLSConfig(c *tls.Config) func(*Attacker) {
	return func(a *Attacker) {
		tr := a.client.Transport.(*http.Transport)
		tr.TLSClientConfig = c
	}
}

// Attack reads its Targets from the passed Targeter and attacks them at
// the rate specified for duration time. Results are put into the returned channel
// as soon as they arrive.
func (a *Attacker) Attack(tr Targeter, rate uint64, du time.Duration) chan *Result {
	resc := make(chan *Result)
	throttle := time.NewTicker(time.Duration(1e9 / rate))
	hits := rate * uint64(du.Seconds())
	wrk := a.workers
	if wrk == 0 || wrk > hits {
		wrk = hits
	}
	share := hits / wrk

	var wg sync.WaitGroup
	for i := uint64(0); i < wrk; i++ {
		wg.Add(1)
		go func(share uint64) {
			defer wg.Done()
			for j := uint64(0); j < share; j++ {
				select {
				case tm := <-throttle.C:
					resc <- a.hit(tr, tm)
				case <-a.stop:
					return
				}
			}
		}(share)
	}

	go func() {
		wg.Wait()
		close(resc)
		throttle.Stop()
	}()

	return resc
}

// Stop stops the current attack.
func (a *Attacker) Stop() { close(a.stop) }

func (a *Attacker) hit(tr Targeter, tm time.Time) *Result {
	var (
		res = Result{Timestamp: tm}
		err error
	)

	defer func() {
		res.Latency = time.Since(tm)
		if err != nil {
			res.Error = err.Error()
		}
	}()

	tgt, err := tr()
	if err != nil {
		return &res
	}

	req, err := tgt.Request()
	if err != nil {
		return &res
	}

	r, err := a.client.Do(req)
	if err != nil {
		// ignore redirect errors when the user set --redirects=NoFollow
		if a.redirects == NoFollow && strings.Contains(err.Error(), "stopped after") {
			err = nil
		}
		return &res
	}
	defer r.Body.Close()

	in, err := io.Copy(ioutil.Discard, r.Body)
	if err != nil {
		return &res
	}
	res.BytesIn = uint64(in)

	if req.ContentLength != -1 {
		res.BytesOut = uint64(req.ContentLength)
	}

	if res.Code = uint16(r.StatusCode); res.Code < 200 || res.Code >= 400 {
		res.Error = r.Status
	}

	return &res
}
<commit_msg>Improve core attack rate capacity<commit_after>package vegeta

import (
	"crypto/tls"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"strings"
	"sync"
	"time"
)

// Attacker is an attack executor which wraps an http.Client
type Attacker struct {
	dialer    *net.Dialer
	client    http.Client
	stopch    chan struct{}
	workers   uint64
	redirects int
}

const (
	// DefaultRedirects is the default number of times an Attacker follows
	// redirects.
	DefaultRedirects = 10
	// DefaultTimeout is the default amount of time an Attacker waits for a request
	// before it times out.
	DefaultTimeout = 30 * time.Second
	// NoFollow is the value when redirects are not followed but marked successful
	NoFollow = -1
)

var (
	// DefaultLocalAddr is the default local IP address an Attacker uses.
	DefaultLocalAddr = net.IPAddr{IP: net.IPv4zero}
	// DefaultTLSConfig is the default tls.Config an Attacker uses.
	DefaultTLSConfig = &tls.Config{InsecureSkipVerify: true}
)

// NewAttacker returns a new Attacker with default options which are overridden
// by the optionally provided opts.
func NewAttacker(opts ...func(*Attacker)) *Attacker {
	a := &Attacker{stopch: make(chan struct{})}
	a.dialer = &net.Dialer{
		LocalAddr: &net.TCPAddr{IP: DefaultLocalAddr.IP, Zone: DefaultLocalAddr.Zone},
		KeepAlive: 30 * time.Second,
		Timeout:   DefaultTimeout,
	}
	a.client = http.Client{
		Transport: &http.Transport{
			Proxy: http.ProxyFromEnvironment,
			Dial:  a.dialer.Dial,
			ResponseHeaderTimeout: DefaultTimeout,
			TLSClientConfig:       DefaultTLSConfig,
			TLSHandshakeTimeout:   10 * time.Second,
			MaxIdleConnsPerHost:   10000,
		},
	}
	for _, opt := range opts {
		opt(a)
	}
	return a
}

// Workers returns a functional option which sets the number of workers
// an Attacker uses to hit its targets.
// If zero or greater than the total number of hits, workers will be capped
// to that maximum.
func Workers(n uint64) func(*Attacker) {
	return func(a *Attacker) { a.workers = n }
}

// Redirects returns a functional option which sets the maximum
// number of redirects an Attacker will follow.
func Redirects(n int) func(*Attacker) {
	return func(a *Attacker) {
		a.redirects = n
		a.client.CheckRedirect = func(_ *http.Request, via []*http.Request) error {
			if len(via) > n {
				return fmt.Errorf("stopped after %d redirects", n)
			}
			return nil
		}
	}
}

// Timeout returns a functional option which sets the maximum amount of time
// an Attacker will wait for a request to be responded to.
func Timeout(d time.Duration) func(*Attacker) {
	return func(a *Attacker) {
		tr := a.client.Transport.(*http.Transport)
		tr.ResponseHeaderTimeout = d
		a.dialer.Timeout = d
		tr.Dial = a.dialer.Dial
	}
}

// LocalAddr returns a functional option which sets the local address
// an Attacker will use with its requests.
func LocalAddr(addr net.IPAddr) func(*Attacker) {
	return func(a *Attacker) {
		tr := a.client.Transport.(*http.Transport)
		a.dialer.LocalAddr = &net.TCPAddr{IP: addr.IP, Zone: addr.Zone}
		tr.Dial = a.dialer.Dial
	}
}

// KeepAlive returns a functional option which toggles KeepAlive
// connections on the dialer and transport.
func KeepAlive(keepalive bool) func(*Attacker) {
	return func(a *Attacker) {
		tr := a.client.Transport.(*http.Transport)
		tr.DisableKeepAlives = !keepalive
		if !keepalive {
			a.dialer.KeepAlive = 0
			tr.Dial = a.dialer.Dial
		}
	}
}

// TLSConfig returns a functional option which sets the *tls.Config for a
// Attacker to use with its requests.
func TLSConfig(c *tls.Config) func(*Attacker) {
	return func(a *Attacker) {
		tr := a.client.Transport.(*http.Transport)
		tr.TLSClientConfig = c
	}
}

// Attack reads its Targets from the passed Targeter and attacks them at
// the rate specified for duration time. Results are put into the returned channel
// as soon as they arrive.
func (a *Attacker) Attack(tr Targeter, rate uint64, du time.Duration) chan *Result {
	hits := rate * uint64(du.Seconds())
	wrk := a.workers
	if wrk == 0 || wrk > rate {
		wrk = rate
	}

	workers := &sync.WaitGroup{}
	results := make(chan *Result)
	ticks := make(chan time.Time)
	for i := uint64(0); i < wrk; i++ {
		go a.attack(tr, workers, ticks, results)
	}

	go func() {
		defer close(results)
		defer workers.Wait()
		defer close(ticks)
		interval := 1e9 / rate
		for began, done := time.Now(), uint64(0); done < hits; done++ {
			next := began.Add(time.Duration(done * interval))
			time.Sleep(next.Sub(time.Now()))
			select {
			case ticks <- next:
			case <-a.stopch:
				return
			default: // all workers are blocked. start one more and try again
				go a.attack(tr, workers, ticks, results)
				done--
			}
		}
	}()

	return results
}

// Stop stops the current attack.
func (a *Attacker) Stop() { close(a.stopch) }

func (a *Attacker) attack(tr Targeter, workers *sync.WaitGroup, ticks <-chan time.Time, results chan<- *Result) {
	workers.Add(1)
	defer workers.Done()
	for tm := range ticks {
		results <- a.hit(tr, tm)
	}
}

func (a *Attacker) hit(tr Targeter, tm time.Time) *Result {
	var (
		res = Result{Timestamp: tm}
		err error
	)

	defer func() {
		res.Latency = time.Since(tm)
		if err != nil {
			res.Error = err.Error()
		}
	}()

	tgt, err := tr()
	if err != nil {
		return &res
	}

	req, err := tgt.Request()
	if err != nil {
		return &res
	}

	r, err := a.client.Do(req)
	if err != nil {
		// ignore redirect errors when the user set --redirects=NoFollow
		if a.redirects == NoFollow && strings.Contains(err.Error(), "stopped after") {
			err = nil
		}
		return &res
	}
	defer r.Body.Close()

	in, err := io.Copy(ioutil.Discard, r.Body)
	if err != nil {
		return &res
	}
	res.BytesIn = uint64(in)

	if req.ContentLength != -1 {
		res.BytesOut = uint64(req.ContentLength)
	}

	if res.Code = uint16(r.StatusCode); res.Code < 200 || res.Code >= 400 {
		res.Error = r.Status
	}

	return &res
}
<|endoftext|>


<commit_before>package lib

import (
	"net/url"
	"sort"
	"strings"
)

// BackupSchedule represents a scheduled backup on a server
// see: server/backup_set_schedule, server/backup_get_schedule
type BackupSchedule struct {
	Enabled              bool   `json:"enabled"`
	CronType             string `json:"cron_type"`
	NextScheduledTimeUtc string `json:"next_scheduled_time_utc"`
	Hour                 int    `json:"hour"`
	Dow                  int    `json:"dow"`
	Dom                  int    `json:"dom"`
}

// Backup of a virtual machine
type Backup struct {
	ID          string `json:"BACKUPID"`
	Created     string `json:"date_created"`
	Description string `json:"description"`
	Size        string `json:"size"`
	Status      string `json:"status"`
}

type backups []Backup

func (s backups) Len() int      { return len(s) }
func (s backups) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s backups) Less(i, j int) bool {
	return strings.ToLower(s[i].Name) < strings.ToLower(s[j].Name)
}

// GetBackups retrieves a list of all backups on Vultr account
func (c *Client) GetBackups(id string, backupid string) (backupList []BackupSchedule, err error) {
	var backupMap map[string]Backup
	values := url.Values{
		"SUBID":    {id},
		"BACKUPID": {backupid},
	}

	if err := c.post(`backup/list`, values, &backupMap); err != nil {
		return nil, err
	}

	for _, backup := range backupMap {
		backups = append(backups, backup)
	}
	sort.Sort(backups(backups))
	return backups, nil
}
<commit_msg>remove backupschedule struct, change return type of GetBackups<commit_after>package lib

import (
	"net/url"
	"sort"
	"strings"
)

// Backup of a virtual machine
type Backup struct {
	ID          string `json:"BACKUPID"`
	Created     string `json:"date_created"`
	Description string `json:"description"`
	Size        string `json:"size"`
	Status      string `json:"status"`
}

type backups []Backup

func (s backups) Len() int      { return len(s) }
func (s backups) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s backups) Less(i, j int) bool {
	return strings.ToLower(s[i].Name) < strings.ToLower(s[j].Name)
}

// GetBackups retrieves a list of all backups on Vultr account
func (c *Client) GetBackups(id string, backupid string) (backups []Backup, err error) {
	var backupMap map[string]Backup
	values := url.Values{
		"SUBID":    {id},
		"BACKUPID": {backupid},
	}

	if err := c.post(`backup/list`, values, &backupMap); err != nil {
		return nil, err
	}

	for _, backup := range backupMap {
		backups = append(backups, backup)
	}
	sort.Sort(backups(backups))
	return backups, nil
}
<|endoftext|>


<commit_before>package external

import (
	"fmt"
)

// Surrogate is an interpolant for a function.
type Surrogate struct {
	Inputs    uint      // Number of inputs
	Outputs   uint      // Number of outputs
	Nodes     uint      // Number of nodes
	Indices   []uint64  // Indices of the nodes
	Surpluses []float64 // Hierarchical surpluses
	Integral  []float64 // Integral over the whole domain
}

// NewSurrogate returns an empty surrogate.
func NewSurrogate(ni, no uint) *Surrogate {
	return &Surrogate{
		Inputs:    ni,
		Outputs:   no,
		Indices:   make([]uint64, 0),
		Surpluses: make([]float64, 0),
		Integral:  make([]float64, no),
	}
}

// Push takes into account new indices and surpluses.
func (self *Surrogate) Push(integrator Integrator, indices []uint64, surpluses []float64) {
	na := uint(len(indices)) / self.Inputs
	self.Nodes += na
	self.Indices = append(self.Indices, indices...)
	self.Surpluses = append(self.Surpluses, surpluses...)
	cumulate(integrator, indices, surpluses, self.Inputs, self.Outputs, self.Integral)
}

// String returns a summary.
func (self *Surrogate) String() string {
	phantom := struct {
		inputs  uint
		outputs uint
		nodes   uint
	}{
		inputs:  self.Inputs,
		outputs: self.Outputs,
		nodes:   self.Nodes,
	}
	return fmt.Sprintf("%+v", phantom)
}

func cumulate(integrator Integrator, indices []uint64, surpluses []float64, ni, no uint,
	integral []float64) {

	nn := uint(len(indices)) / ni
	for i := uint(0); i < nn; i++ {
		volume := integrator.Integrate(indices[i*ni : (i+1)*ni])
		for j := uint(0); j < no; j++ {
			integral[j] += surpluses[i*no+j] * volume
		}
	}
}
<commit_msg>a/external: make a cosmetic adjustment<commit_after>package external

import (
	"fmt"
)

// Surrogate is an interpolant for a function.
type Surrogate struct {
	Inputs  uint // Number of inputs
	Outputs uint // Number of outputs
	Nodes   uint // Number of nodes

	Indices   []uint64  // Indices of the nodes
	Surpluses []float64 // Hierarchical surpluses
	Integral  []float64 // Integral over the whole domain
}

// NewSurrogate returns an empty surrogate.
func NewSurrogate(ni, no uint) *Surrogate {
	return &Surrogate{
		Inputs:  ni,
		Outputs: no,

		Indices:   make([]uint64, 0),
		Surpluses: make([]float64, 0),
		Integral:  make([]float64, no),
	}
}

// Push takes into account new indices and surpluses.
func (self *Surrogate) Push(integrator Integrator, indices []uint64, surpluses []float64) {
	self.Nodes += uint(len(indices)) / self.Inputs
	self.Indices = append(self.Indices, indices...)
	self.Surpluses = append(self.Surpluses, surpluses...)
	cumulate(integrator, indices, surpluses, self.Inputs, self.Outputs, self.Integral)
}

// String returns a summary.
func (self *Surrogate) String() string {
	phantom := struct {
		inputs  uint
		outputs uint
		nodes   uint
	}{
		inputs:  self.Inputs,
		outputs: self.Outputs,
		nodes:   self.Nodes,
	}
	return fmt.Sprintf("%+v", phantom)
}

func cumulate(integrator Integrator, indices []uint64, surpluses []float64, ni, no uint,
	integral []float64) {

	nn := uint(len(indices)) / ni
	for i := uint(0); i < nn; i++ {
		volume := integrator.Integrate(indices[i*ni : (i+1)*ni])
		for j := uint(0); j < no; j++ {
			integral[j] += surpluses[i*no+j] * volume
		}
	}
}
<|endoftext|>


<commit_before>package fluentd_forwarder

import (
	"bytes"
	"compress/gzip"
	"errors"
	logging "github.com/op/go-logging"
	td_client "github.com/treasure-data/td-client-go"
	"github.com/ugorji/go/codec"
	"net"
	"os"
	"reflect"
	"strings"
	"sync"
	"sync/atomic"
	"time"
	"unsafe"
)

type tdOutputSpooler struct {
	daemon         *tdOutputSpoolerDaemon
	ticker         *time.Ticker
	tag            string
	databaseName   string
	tableName      string
	key            string
	journal        Journal
	client         *td_client.TDClient
	shutdownChan   chan struct{}
	isShuttingDown unsafe.Pointer
}

type tdOutputSpoolerDaemon struct {
	output       *TDOutput
	shutdownChan chan struct{}
	spoolersMtx  sync.Mutex
	spoolers     map[string]*tdOutputSpooler
	tempFactory  TempFileRandomAccessStoreFactory
	wg           sync.WaitGroup
	endNotify  func(*tdOutputSpoolerDaemon)
}

type TDOutput struct {
	logger         *logging.Logger
	codec          *codec.MsgpackHandle
	retryInterval  time.Duration
	databaseName   string
	tableName      string
	tempDir        string
	enc            *codec.Encoder
	conn           net.Conn
	flushInterval  time.Duration
	wg             sync.WaitGroup
	journalGroup   JournalGroup
	emitterChan    chan FluentRecordSet
	spoolerDaemon  *tdOutputSpoolerDaemon
	isShuttingDown unsafe.Pointer
	client         *td_client.TDClient
	gcChan         chan *os.File
	completion     sync.Cond
}

func encodeRecords(encoder *codec.Encoder, records []TinyFluentRecord) error {
	for _, record := range records {
		e := map[string]interface{}{"time": record.Timestamp}
		for k, v := range record.Data {
			e[k] = v
		}
		err := encoder.Encode(e)
		if err != nil {
			return err
		}
	}
	return nil
}

func (spooler *tdOutputSpooler) cleanup() {
	spooler.ticker.Stop()
	spooler.journal.Dispose()
	spooler.daemon.wg.Done()
}

func (spooler *tdOutputSpooler) handle() {
	defer spooler.cleanup()
	spooler.daemon.output.logger.Notice("Spooler started")
outer:
	for {
		select {
		case <-spooler.ticker.C:
			spooler.daemon.output.logger.Notice("Flushing...")
			err := spooler.journal.Flush(func(chunk JournalChunk) error {
				defer chunk.Dispose()
				spooler.daemon.output.logger.Info("Flushing chunk %s", chunk.String())
				_, err := spooler.client.Import(
					spooler.databaseName,
					spooler.tableName,
					"msgpack.gz",
					NewCompressingBlob(
						chunk,
						16777216,
						gzip.BestSpeed,
						&spooler.daemon.tempFactory,
					),
					chunk.Id(),
				)
				return err
			})
			if err != nil {
				spooler.daemon.output.logger.Error("Error during reading from the journal: %s", err.Error())
			}
		case <-spooler.shutdownChan:
			break outer
		}
	}
	spooler.daemon.output.logger.Notice("Spooler ended")
}

func normalizeDatabaseName(name string) (string, error) {
	name_ := ([]byte)(name)
	if len(name_) == 0 {
		return "", errors.New("Empty name is not allowed")
	}
	if len(name_) < 3 {
		name_ = append(name_, ("___"[0 : 3-len(name)])...)
	}
	if 255 < len(name_) {
		name_ = append(name_[0:253], "__"...)
	}
	name_ = bytes.ToLower(name_)
	for i, c := range name_ {
		if !((c >= 'a' || c <= 'z') || (c >= '0' && c <= '9') || c == '_') {
			c = '_'
		}
		name_[i] = c
	}
	return (string)(name_), nil
}

func normalizeTableName(name string) (string, error) {
	return normalizeDatabaseName(name)
}

func newTDOutputSpooler(daemon *tdOutputSpoolerDaemon, databaseName, tableName, key string) *tdOutputSpooler {
	journal := daemon.output.journalGroup.GetJournal(key)
	return &tdOutputSpooler{
		daemon:       daemon,
		ticker:       time.NewTicker(daemon.output.flushInterval),
		databaseName: databaseName,
		tableName:    tableName,
		key:          key,
		journal:      journal,
		shutdownChan: make(chan struct{}, 1),
		client:       daemon.output.client,
	}
}

func (daemon *tdOutputSpoolerDaemon) spawnSpooler(databaseName, tableName, key string) *tdOutputSpooler {
	daemon.spoolersMtx.Lock()
	defer daemon.spoolersMtx.Unlock()
	spooler, exists := daemon.spoolers[key]
	if exists {
		return spooler
	}
	spooler = newTDOutputSpooler(daemon, databaseName, tableName, key)
	daemon.output.logger.Notice("Spawning spooler " + spooler.key)
	daemon.spoolers[spooler.key] = spooler
	daemon.wg.Add(1)
	go spooler.handle()
	return spooler
}

func (daemon *tdOutputSpoolerDaemon) cleanup() {
	func() {
		daemon.spoolersMtx.Lock()
		defer daemon.spoolersMtx.Unlock()
		for _, spooler := range daemon.spoolers {
			if atomic.CompareAndSwapPointer(&spooler.isShuttingDown, unsafe.Pointer(uintptr(0)), unsafe.Pointer(uintptr(1))) {
				spooler.shutdownChan <- struct{}{}
			}
		}
	}()
	daemon.wg.Wait()
	daemon.output.wg.Done()
}

func (daemon *tdOutputSpoolerDaemon) handle() {
	defer daemon.cleanup()
	daemon.output.logger.Notice("Spooler daemon started")
outer:
	for {
		select {
		case <-daemon.shutdownChan:
			break outer
		}
	}
	if daemon.endNotify != nil {
		daemon.endNotify(daemon)
	}
	daemon.output.logger.Notice("Spooler daemon ended")
}

func newTDOutputSpoolerDaemon(output *TDOutput) *tdOutputSpoolerDaemon {
	return &tdOutputSpoolerDaemon{
		output:       output,
		shutdownChan: make(chan struct{}, 1),
		spoolers:     make(map[string]*tdOutputSpooler),
		tempFactory:  TempFileRandomAccessStoreFactory{output.tempDir, "", output.gcChan},
		wg:           sync.WaitGroup{},
		endNotify:    func(*tdOutputSpoolerDaemon) {
			close(output.gcChan)
		},
	}
}

func (output *TDOutput) spawnSpoolerDaemon() {
	output.logger.Notice("Spawning spooler daemon")
	output.spoolerDaemon = newTDOutputSpoolerDaemon(output)
	output.wg.Add(1)
	go output.spoolerDaemon.handle()
}

func (daemon *tdOutputSpoolerDaemon) getSpooler(tag string) (*tdOutputSpooler, error) {
	databaseName := daemon.output.databaseName
	tableName := daemon.output.tableName
	if databaseName == "*" {
		if tableName == "*" {
			c := strings.SplitN(tag, ".", 2)
			if len(c) == 1 {
				tableName = c[0]
			} else if len(c) == 2 {
				databaseName = c[0]
				tableName = c[1]
			}
		} else {
			databaseName = tag
		}
	} else {
		if tableName == "*" {
			tableName = tag
		}
	}
	databaseName, err := normalizeDatabaseName(databaseName)
	if err != nil {
		return nil, err
	}
	tableName, err = normalizeTableName(tableName)
	if err != nil {
		return nil, err
	}
	key := databaseName + "." + tableName
	return daemon.spawnSpooler(databaseName, tableName, key), nil
}

func (output *TDOutput) spawnEmitter() {
	output.logger.Notice("Spawning emitter")
	output.wg.Add(1)
	go func() {
		defer func() {
			output.spoolerDaemon.shutdownChan <- struct{}{}
			output.wg.Done()
		}()
		output.logger.Notice("Emitter started")
		buffer := bytes.Buffer{}
		for recordSet := range output.emitterChan {
			buffer.Reset()
			encoder := codec.NewEncoder(&buffer, output.codec)
			err := func() error {
				spooler, err := output.spoolerDaemon.getSpooler(recordSet.Tag)
				if err != nil {
					return err
				}
				err = encodeRecords(encoder, recordSet.Records)
				if err != nil {
					return err
				}
				output.logger.Debug("Emitter processed %d entries", len(recordSet.Records))
				return spooler.journal.Write(buffer.Bytes())
			}()
			if err != nil {
				output.logger.Error("%s", err.Error())
				continue
			}
		}
		output.logger.Notice("Emitter ended")
	}()
}

func (output *TDOutput) spawnTempFileCollector() {
	output.logger.Notice("Spawning temporary file collector")
	output.wg.Add(1)
	go func() {
		defer func() {
			output.wg.Done()
		}()
		for f := range output.gcChan {
			output.logger.Debug("Deleting %s...", f.Name())
			err := os.Remove(f.Name())
			if err != nil {
				output.logger.Warning("Failed to delete %s: %s", f.Name(), err.Error())
			}
		}
		output.logger.Debug("temporary file collector ended")
	}()
}

func (output *TDOutput) Emit(recordSets []FluentRecordSet) error {
	defer func() {
		recover()
	}()
	for _, recordSet := range recordSets {
		output.emitterChan <- recordSet
	}
	return nil
}

func (output *TDOutput) String() string {
	return "output"
}

func (output *TDOutput) Stop() {
	if atomic.CompareAndSwapPointer(&output.isShuttingDown, unsafe.Pointer(uintptr(0)), unsafe.Pointer(uintptr(1))) {
		close(output.emitterChan)
	}
}

func (output *TDOutput) WaitForShutdown() {
	output.completion.L.Lock()
	output.completion.Wait()
	output.completion.L.Unlock()
}

func (output *TDOutput) Start() {
	syncCh := make(chan struct{})
	go func () {
		<-syncCh
		output.wg.Wait()
		err := output.journalGroup.Dispose()
		if err != nil {
			output.logger.Error("%s", err.Error())
		}
		output.completion.Broadcast()
	}()
	output.spawnTempFileCollector()
	output.spawnEmitter()
	output.spawnSpoolerDaemon()
	syncCh <- struct{}{}
}

func NewTDOutput(
	logger *logging.Logger,
	endpoint string,
	retryInterval time.Duration,
	connectionTimeout time.Duration,
	writeTimeout time.Duration,
	flushInterval time.Duration,
	journalGroupPath string,
	maxJournalChunkSize int64,
	apiKey string,
	databaseName string,
	tableName string,
	tempDir string,
	useSsl bool,
	httpProxy string,
) (*TDOutput, error) {
	_codec := codec.MsgpackHandle{}
	_codec.MapType = reflect.TypeOf(map[string]interface{}(nil))
	_codec.RawToString = false
	_codec.StructToArray = true

	journalFactory := NewFileJournalGroupFactory(
		logger,
		randSource,
		time.Now,
		".log",
		os.FileMode(0600),
		maxJournalChunkSize,
	)
	router := (td_client.EndpointRouter)(nil)
	if endpoint != "" {
		router = &td_client.FixedEndpointRouter{endpoint}
	}
	httpProxy_ := (interface{})(nil)
	if httpProxy != "" {
		httpProxy_ = httpProxy
	}
	client, err := td_client.NewTDClient(td_client.Settings{
		ApiKey:            apiKey,
		Router:            router,
		ConnectionTimeout: connectionTimeout,
		// ReadTimeout: readTimeout, // TODO
		SendTimeout: writeTimeout,
		Ssl:         useSsl,
		Proxy:       httpProxy_,
	})
	if err != nil {
		return nil, err
	}
	output := &TDOutput{
		logger:         logger,
		codec:          &_codec,
		retryInterval:  retryInterval,
		wg:             sync.WaitGroup{},
		flushInterval:  flushInterval,
		emitterChan:    make(chan FluentRecordSet),
		isShuttingDown: unsafe.Pointer(uintptr(0)),
		client:         client,
		databaseName:   databaseName,
		tableName:      tableName,
		tempDir:        tempDir,
		gcChan:         make(chan *os.File, 10),
		completion:     sync.Cond { L: &sync.Mutex {} },
	}
	journalGroup, err := journalFactory.GetJournalGroup(journalGroupPath, output)
	if err != nil {
		return nil, err
	}
	output.journalGroup = journalGroup
	return output, nil
}
<commit_msg>Spawn the spooler for each unflushed journal on startup.<commit_after>package fluentd_forwarder

import (
	"bytes"
	"compress/gzip"
	"errors"
	logging "github.com/op/go-logging"
	td_client "github.com/treasure-data/td-client-go"
	"github.com/ugorji/go/codec"
	"net"
	"os"
	"reflect"
	"strings"
	"sync"
	"sync/atomic"
	"time"
	"unsafe"
)

type tdOutputSpooler struct {
	daemon         *tdOutputSpoolerDaemon
	ticker         *time.Ticker
	tag            string
	databaseName   string
	tableName      string
	key            string
	journal        Journal
	client         *td_client.TDClient
	shutdownChan   chan struct{}
	isShuttingDown unsafe.Pointer
}

type tdOutputSpoolerDaemon struct {
	output       *TDOutput
	shutdownChan chan struct{}
	spoolersMtx  sync.Mutex
	spoolers     map[string]*tdOutputSpooler
	tempFactory  TempFileRandomAccessStoreFactory
	wg           sync.WaitGroup
	endNotify  func(*tdOutputSpoolerDaemon)
}

type TDOutput struct {
	logger         *logging.Logger
	codec          *codec.MsgpackHandle
	retryInterval  time.Duration
	databaseName   string
	tableName      string
	tempDir        string
	enc            *codec.Encoder
	conn           net.Conn
	flushInterval  time.Duration
	wg             sync.WaitGroup
	journalGroup   JournalGroup
	emitterChan    chan FluentRecordSet
	spoolerDaemon  *tdOutputSpoolerDaemon
	isShuttingDown unsafe.Pointer
	client         *td_client.TDClient
	gcChan         chan *os.File
	completion     sync.Cond
}

func encodeRecords(encoder *codec.Encoder, records []TinyFluentRecord) error {
	for _, record := range records {
		e := map[string]interface{}{"time": record.Timestamp}
		for k, v := range record.Data {
			e[k] = v
		}
		err := encoder.Encode(e)
		if err != nil {
			return err
		}
	}
	return nil
}

func (spooler *tdOutputSpooler) cleanup() {
	spooler.ticker.Stop()
	spooler.journal.Dispose()
	spooler.daemon.wg.Done()
}

func (spooler *tdOutputSpooler) handle() {
	defer spooler.cleanup()
	spooler.daemon.output.logger.Notice("Spooler started")
outer:
	for {
		select {
		case <-spooler.ticker.C:
			spooler.daemon.output.logger.Notice("Flushing...")
			err := spooler.journal.Flush(func(chunk JournalChunk) error {
				defer chunk.Dispose()
				spooler.daemon.output.logger.Info("Flushing chunk %s", chunk.String())
				_, err := spooler.client.Import(
					spooler.databaseName,
					spooler.tableName,
					"msgpack.gz",
					NewCompressingBlob(
						chunk,
						16777216,
						gzip.BestSpeed,
						&spooler.daemon.tempFactory,
					),
					chunk.Id(),
				)
				return err
			})
			if err != nil {
				spooler.daemon.output.logger.Error("Error during reading from the journal: %s", err.Error())
			}
		case <-spooler.shutdownChan:
			break outer
		}
	}
	spooler.daemon.output.logger.Notice("Spooler ended")
}

func normalizeDatabaseName(name string) (string, error) {
	name_ := ([]byte)(name)
	if len(name_) == 0 {
		return "", errors.New("Empty name is not allowed")
	}
	if len(name_) < 3 {
		name_ = append(name_, ("___"[0 : 3-len(name)])...)
	}
	if 255 < len(name_) {
		name_ = append(name_[0:253], "__"...)
	}
	name_ = bytes.ToLower(name_)
	for i, c := range name_ {
		if !((c >= 'a' || c <= 'z') || (c >= '0' && c <= '9') || c == '_') {
			c = '_'
		}
		name_[i] = c
	}
	return (string)(name_), nil
}

func normalizeTableName(name string) (string, error) {
	return normalizeDatabaseName(name)
}

func newTDOutputSpooler(daemon *tdOutputSpoolerDaemon, databaseName, tableName, key string) *tdOutputSpooler {
	journal := daemon.output.journalGroup.GetJournal(key)
	return &tdOutputSpooler{
		daemon:       daemon,
		ticker:       time.NewTicker(daemon.output.flushInterval),
		databaseName: databaseName,
		tableName:    tableName,
		key:          key,
		journal:      journal,
		shutdownChan: make(chan struct{}, 1),
		client:       daemon.output.client,
	}
}

func (daemon *tdOutputSpoolerDaemon) spawnSpooler(databaseName, tableName, key string) *tdOutputSpooler {
	daemon.spoolersMtx.Lock()
	defer daemon.spoolersMtx.Unlock()
	spooler, exists := daemon.spoolers[key]
	if exists {
		return spooler
	}
	spooler = newTDOutputSpooler(daemon, databaseName, tableName, key)
	daemon.output.logger.Notice("Spawning spooler " + spooler.key)
	daemon.spoolers[spooler.key] = spooler
	daemon.wg.Add(1)
	go spooler.handle()
	return spooler
}

func (daemon *tdOutputSpoolerDaemon) cleanup() {
	func() {
		daemon.spoolersMtx.Lock()
		defer daemon.spoolersMtx.Unlock()
		for _, spooler := range daemon.spoolers {
			if atomic.CompareAndSwapPointer(&spooler.isShuttingDown, unsafe.Pointer(uintptr(0)), unsafe.Pointer(uintptr(1))) {
				spooler.shutdownChan <- struct{}{}
			}
		}
	}()
	daemon.wg.Wait()
	daemon.output.wg.Done()
}

func (daemon *tdOutputSpoolerDaemon) handle() {
	defer daemon.cleanup()
	daemon.output.logger.Notice("Spooler daemon started")
	// spawn Spooler according to the existing journals
	for _, key := range daemon.output.journalGroup.GetJournalKeys() {
		pair := strings.SplitN(key, ".", 2)
		if len(pair) != 2 {
			daemon.output.logger.Warning("Journal %s ignored", key)
			continue
		}
		daemon.spawnSpooler(pair[0], pair[1], key)
	}
outer:
	for {
		select {
		case <-daemon.shutdownChan:
			break outer
		}
	}
	if daemon.endNotify != nil {
		daemon.endNotify(daemon)
	}
	daemon.output.logger.Notice("Spooler daemon ended")
}

func newTDOutputSpoolerDaemon(output *TDOutput) *tdOutputSpoolerDaemon {
	return &tdOutputSpoolerDaemon{
		output:       output,
		shutdownChan: make(chan struct{}, 1),
		spoolers:     make(map[string]*tdOutputSpooler),
		tempFactory:  TempFileRandomAccessStoreFactory{output.tempDir, "", output.gcChan},
		wg:           sync.WaitGroup{},
		endNotify:    func(*tdOutputSpoolerDaemon) {
			close(output.gcChan)
		},
	}
}

func (output *TDOutput) spawnSpoolerDaemon() {
	output.logger.Notice("Spawning spooler daemon")
	output.spoolerDaemon = newTDOutputSpoolerDaemon(output)
	output.wg.Add(1)
	go output.spoolerDaemon.handle()
}

func (daemon *tdOutputSpoolerDaemon) getSpooler(tag string) (*tdOutputSpooler, error) {
	databaseName := daemon.output.databaseName
	tableName := daemon.output.tableName
	if databaseName == "*" {
		if tableName == "*" {
			c := strings.SplitN(tag, ".", 2)
			if len(c) == 1 {
				tableName = c[0]
			} else if len(c) == 2 {
				databaseName = c[0]
				tableName = c[1]
			}
		} else {
			databaseName = tag
		}
	} else {
		if tableName == "*" {
			tableName = tag
		}
	}
	databaseName, err := normalizeDatabaseName(databaseName)
	if err != nil {
		return nil, err
	}
	tableName, err = normalizeTableName(tableName)
	if err != nil {
		return nil, err
	}
	key := databaseName + "." + tableName
	return daemon.spawnSpooler(databaseName, tableName, key), nil
}

func (output *TDOutput) spawnEmitter() {
	output.logger.Notice("Spawning emitter")
	output.wg.Add(1)
	go func() {
		defer func() {
			output.spoolerDaemon.shutdownChan <- struct{}{}
			output.wg.Done()
		}()
		output.logger.Notice("Emitter started")
		buffer := bytes.Buffer{}
		for recordSet := range output.emitterChan {
			buffer.Reset()
			encoder := codec.NewEncoder(&buffer, output.codec)
			err := func() error {
				spooler, err := output.spoolerDaemon.getSpooler(recordSet.Tag)
				if err != nil {
					return err
				}
				err = encodeRecords(encoder, recordSet.Records)
				if err != nil {
					return err
				}
				output.logger.Debug("Emitter processed %d entries", len(recordSet.Records))
				return spooler.journal.Write(buffer.Bytes())
			}()
			if err != nil {
				output.logger.Error("%s", err.Error())
				continue
			}
		}
		output.logger.Notice("Emitter ended")
	}()
}

func (output *TDOutput) spawnTempFileCollector() {
	output.logger.Notice("Spawning temporary file collector")
	output.wg.Add(1)
	go func() {
		defer func() {
			output.wg.Done()
		}()
		for f := range output.gcChan {
			output.logger.Debug("Deleting %s...", f.Name())
			err := os.Remove(f.Name())
			if err != nil {
				output.logger.Warning("Failed to delete %s: %s", f.Name(), err.Error())
			}
		}
		output.logger.Debug("temporary file collector ended")
	}()
}

func (output *TDOutput) Emit(recordSets []FluentRecordSet) error {
	defer func() {
		recover()
	}()
	for _, recordSet := range recordSets {
		output.emitterChan <- recordSet
	}
	return nil
}

func (output *TDOutput) String() string {
	return "output"
}

func (output *TDOutput) Stop() {
	if atomic.CompareAndSwapPointer(&output.isShuttingDown, unsafe.Pointer(uintptr(0)), unsafe.Pointer(uintptr(1))) {
		close(output.emitterChan)
	}
}

func (output *TDOutput) WaitForShutdown() {
	output.completion.L.Lock()
	output.completion.Wait()
	output.completion.L.Unlock()
}

func (output *TDOutput) Start() {
	syncCh := make(chan struct{})
	go func () {
		<-syncCh
		output.wg.Wait()
		err := output.journalGroup.Dispose()
		if err != nil {
			output.logger.Error("%s", err.Error())
		}
		output.completion.Broadcast()
	}()
	output.spawnTempFileCollector()
	output.spawnEmitter()
	output.spawnSpoolerDaemon()
	syncCh <- struct{}{}
}

func NewTDOutput(
	logger *logging.Logger,
	endpoint string,
	retryInterval time.Duration,
	connectionTimeout time.Duration,
	writeTimeout time.Duration,
	flushInterval time.Duration,
	journalGroupPath string,
	maxJournalChunkSize int64,
	apiKey string,
	databaseName string,
	tableName string,
	tempDir string,
	useSsl bool,
	httpProxy string,
) (*TDOutput, error) {
	_codec := codec.MsgpackHandle{}
	_codec.MapType = reflect.TypeOf(map[string]interface{}(nil))
	_codec.RawToString = false
	_codec.StructToArray = true

	journalFactory := NewFileJournalGroupFactory(
		logger,
		randSource,
		time.Now,
		".log",
		os.FileMode(0600),
		maxJournalChunkSize,
	)
	router := (td_client.EndpointRouter)(nil)
	if endpoint != "" {
		router = &td_client.FixedEndpointRouter{endpoint}
	}
	httpProxy_ := (interface{})(nil)
	if httpProxy != "" {
		httpProxy_ = httpProxy
	}
	client, err := td_client.NewTDClient(td_client.Settings{
		ApiKey:            apiKey,
		Router:            router,
		ConnectionTimeout: connectionTimeout,
		// ReadTimeout: readTimeout, // TODO
		SendTimeout: writeTimeout,
		Ssl:         useSsl,
		Proxy:       httpProxy_,
	})
	if err != nil {
		return nil, err
	}
	output := &TDOutput{
		logger:         logger,
		codec:          &_codec,
		retryInterval:  retryInterval,
		wg:             sync.WaitGroup{},
		flushInterval:  flushInterval,
		emitterChan:    make(chan FluentRecordSet),
		isShuttingDown: unsafe.Pointer(uintptr(0)),
		client:         client,
		databaseName:   databaseName,
		tableName:      tableName,
		tempDir:        tempDir,
		gcChan:         make(chan *os.File, 10),
		completion:     sync.Cond { L: &sync.Mutex {} },
	}
	journalGroup, err := journalFactory.GetJournalGroup(journalGroupPath, output)
	if err != nil {
		return nil, err
	}
	output.journalGroup = journalGroup
	return output, nil
}
<|endoftext|>


<commit_before>package p

/*
#include "Python.h"

PyObject* idOrNone(PyObject* o)
{
  return o ? o : Py_BuildValue("");
}

PyObject* fetchPythonError()
{
  PyObject *type, *value, *traceback;
  PyObject* ret;

  PyErr_Fetch(&type, &value, &traceback);
  ret = PyTuple_New(3);
  PyTuple_SetItem(ret, 0, idOrNone(type));
  PyTuple_SetItem(ret, 1, idOrNone(value));
  PyTuple_SetItem(ret, 2, idOrNone(traceback));
  return ret;
}
*/
import "C"
import (
	"errors"
	"fmt"
	"runtime"
	"strings"
	"unicode"
	"unsafe"
)

func init() {
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	releaseGIL, err := initAndLockPython()
	if err != nil {
		panic(err)
	}
	defer releaseGIL()

	traceback, err := loadModule("traceback")
	if err != nil {
		panic(err)
	}
	defer traceback.decRef()
	if formatException, err := getPyFunc(traceback.p, "format_exception"); err != nil {
		panic(err)
	} else {
		tracebackFormatExceptionFunc = formatException
	}

	exceptions, err := loadModule("exceptions")
	if err != nil {
		panic(err)
	}
	defer exceptions.decRef()
	syntaxErrorCString := C.CString("SyntaxError")
	defer C.free(unsafe.Pointer(syntaxErrorCString))
	if syntaxError := C.PyObject_GetAttrString(exceptions.p, syntaxErrorCString); syntaxError == nil {
		panic("cannot load exceptions.SyntaxError")
	} else {
		syntaxErrorType.p = syntaxError
	}
}

func loadModule(name string) (mod ObjectModule, err error) {
	cName := C.CString(name)
	defer C.free(unsafe.Pointer(cName))
	if m := C.PyImport_ImportModule(cName); m == nil {
		err = fmt.Errorf("failed to import '%v'", name)
	} else {
		mod.p = m
	}
	return
}

var tracebackFormatExceptionFunc ObjectFunc
var syntaxErrorType Object

// tracebackFormatException calls traceback.format_exception().
func tracebackFormatException(excInfo Object) (Object, error) {
	formatted := C.PyObject_CallObject(tracebackFormatExceptionFunc.p, excInfo.p)
	if formatted == nil {
		return Object{p: nil}, errors.New("failed to extract error info from Python")
	}
	return Object{p: formatted}, nil
}

func isSyntaxError(p *C.PyObject) bool {
	return C.PyType_IsSubtype(pyObjectToPyTypeObject(p), pyObjectToPyTypeObject(syntaxErrorType.p)) != 0
}

func pyObjectToPyTypeObject(p *C.PyObject) *C.PyTypeObject {
	return (*C.PyTypeObject)(unsafe.Pointer(p))
}

func getPyErr() error {
	excInfo := Object{p: C.fetchPythonError()}
	defer excInfo.decRef()

	formatted, err := tracebackFormatException(excInfo)
	if err != nil {
		return err
	}
	defer formatted.decRef()

	ln := C.PyList_Size(formatted.p)
	mainMsg := strings.TrimSpace(extractLineFromFormattedErrorMessage(formatted, ln-1))
	syntaxErr := ""
	nTracebackLines := ln - 1
	if isSyntaxError(C.PyTuple_GetItem(excInfo.p, 0)) {
		firstLine := extractLineFromFormattedErrorMessage(formatted, ln-3)
		secondLine := extractLineFromFormattedErrorMessage(formatted, ln-2)
		syntaxErr = firstLine + secondLine
		nTracebackLines = ln - 3
	}
	stackTrace := ""
	for i := C.Py_ssize_t(0); i < nTracebackLines; i++ {
		stackTrace += extractLineFromFormattedErrorMessage(formatted, i)
	}
	strings.TrimRightFunc(stackTrace, unicode.IsSpace)
	return &pyErr{
		mainMsg:      mainMsg,
		syntaxErrMsg: syntaxErr,
		stackTrace:   stackTrace,
	}
}

func extractLineFromFormattedErrorMessage(formatted Object, n C.Py_ssize_t) string {
	line := C.PyList_GetItem(formatted.p, n)
	return C.GoString(C.PyString_AsString(line))
}

type pyErr struct {
	mainMsg      string
	syntaxErrMsg string
	stackTrace   string
}

// Error returns an error message string for pyErr.
// This string contains multiple lines for stacktrace.
func (e *pyErr) Error() string {
	return e.mainMsg + "\n" + e.syntaxErrMsg + e.stackTrace
}
<commit_msg>Trim properly<commit_after>package p

/*
#include "Python.h"

PyObject* idOrNone(PyObject* o)
{
  return o ? o : Py_BuildValue("");
}

PyObject* fetchPythonError()
{
  PyObject *type, *value, *traceback;
  PyObject* ret;

  PyErr_Fetch(&type, &value, &traceback);
  ret = PyTuple_New(3);
  PyTuple_SetItem(ret, 0, idOrNone(type));
  PyTuple_SetItem(ret, 1, idOrNone(value));
  PyTuple_SetItem(ret, 2, idOrNone(traceback));
  return ret;
}
*/
import "C"
import (
	"errors"
	"fmt"
	"runtime"
	"strings"
	"unicode"
	"unsafe"
)

func init() {
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	releaseGIL, err := initAndLockPython()
	if err != nil {
		panic(err)
	}
	defer releaseGIL()

	traceback, err := loadModule("traceback")
	if err != nil {
		panic(err)
	}
	defer traceback.decRef()
	if formatException, err := getPyFunc(traceback.p, "format_exception"); err != nil {
		panic(err)
	} else {
		tracebackFormatExceptionFunc = formatException
	}

	exceptions, err := loadModule("exceptions")
	if err != nil {
		panic(err)
	}
	defer exceptions.decRef()
	syntaxErrorCString := C.CString("SyntaxError")
	defer C.free(unsafe.Pointer(syntaxErrorCString))
	if syntaxError := C.PyObject_GetAttrString(exceptions.p, syntaxErrorCString); syntaxError == nil {
		panic("cannot load exceptions.SyntaxError")
	} else {
		syntaxErrorType.p = syntaxError
	}
}

func loadModule(name string) (mod ObjectModule, err error) {
	cName := C.CString(name)
	defer C.free(unsafe.Pointer(cName))
	if m := C.PyImport_ImportModule(cName); m == nil {
		err = fmt.Errorf("failed to import '%v'", name)
	} else {
		mod.p = m
	}
	return
}

var tracebackFormatExceptionFunc ObjectFunc
var syntaxErrorType Object

// tracebackFormatException calls traceback.format_exception().
func tracebackFormatException(excInfo Object) (Object, error) {
	formatted := C.PyObject_CallObject(tracebackFormatExceptionFunc.p, excInfo.p)
	if formatted == nil {
		return Object{p: nil}, errors.New("failed to extract error info from Python")
	}
	return Object{p: formatted}, nil
}

func isSyntaxError(p *C.PyObject) bool {
	return C.PyType_IsSubtype(pyObjectToPyTypeObject(p), pyObjectToPyTypeObject(syntaxErrorType.p)) != 0
}

func pyObjectToPyTypeObject(p *C.PyObject) *C.PyTypeObject {
	return (*C.PyTypeObject)(unsafe.Pointer(p))
}

func getPyErr() error {
	excInfo := Object{p: C.fetchPythonError()}
	defer excInfo.decRef()

	formatted, err := tracebackFormatException(excInfo)
	if err != nil {
		return err
	}
	defer formatted.decRef()

	ln := C.PyList_Size(formatted.p)
	mainMsg := strings.TrimSpace(extractLineFromFormattedErrorMessage(formatted, ln-1))
	syntaxErr := ""
	nTracebackLines := ln - 1
	if isSyntaxError(C.PyTuple_GetItem(excInfo.p, 0)) {
		firstLine := extractLineFromFormattedErrorMessage(formatted, ln-3)
		secondLine := extractLineFromFormattedErrorMessage(formatted, ln-2)
		syntaxErr = firstLine + secondLine
		nTracebackLines = ln - 3
	}
	stackTrace := ""
	for i := C.Py_ssize_t(0); i < nTracebackLines; i++ {
		stackTrace += extractLineFromFormattedErrorMessage(formatted, i)
	}
	stackTrace = strings.TrimRightFunc(stackTrace, unicode.IsSpace)
	return &pyErr{
		mainMsg:      mainMsg,
		syntaxErrMsg: syntaxErr,
		stackTrace:   stackTrace,
	}
}

func extractLineFromFormattedErrorMessage(formatted Object, n C.Py_ssize_t) string {
	line := C.PyList_GetItem(formatted.p, n)
	return C.GoString(C.PyString_AsString(line))
}

type pyErr struct {
	mainMsg      string
	syntaxErrMsg string
	stackTrace   string
}

// Error returns an error message string for pyErr.
// This string contains multiple lines for stacktrace.
func (e *pyErr) Error() string {
	return e.mainMsg + "\n" + e.syntaxErrMsg + e.stackTrace
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"os"
	"os/exec"
	"runtime"
	"strings"

	"github.com/hashicorp/go-version"
	"github.com/phase2/rig/cli/util"
	"github.com/urfave/cli"
	"strconv"
)

type Doctor struct {
	BaseCommand
}

func (cmd *Doctor) Commands() []cli.Command {
	return []cli.Command{
		{
			Name:   "doctor",
			Usage:  "Troubleshoot the Rig environment",
			Before: cmd.Before,
			Action: cmd.Run,
		},
	}
}

func (cmd *Doctor) Run(c *cli.Context) error {
	// 1. Ensure the configured docker-machine matches the set environment.
	if cmd.machine.Exists() {
		if _, isset := os.LookupEnv("DOCKER_MACHINE_NAME"); isset == false {
			cmd.out.Error.Fatalf("Docker configuration is not set. Please run 'eval \"$(rig config)\"'.")
		} else if cmd.machine.Name != os.Getenv("DOCKER_MACHINE_NAME") {
			cmd.out.Error.Fatalf("Your environment configuration specifies a different machine. Please re-run as 'rig --name=\"%s\" doctor'.", cmd.machine.Name)
		} else {
			cmd.out.Info.Printf("Docker Machine (%s) name matches your environment configuration.", cmd.machine.Name)
		}
		if output, err := exec.Command("docker-machine", "url", cmd.machine.Name).Output(); err == nil {
			hostUrl := strings.TrimSpace(string(output))
			if hostUrl != os.Getenv("DOCKER_HOST") {
				cmd.out.Error.Fatalf("Docker Host configuration should be '%s' but got '%s'. Please re-run 'eval \"$(rig config)\"'.", os.Getenv("DOCKER_HOST"), hostUrl)
			} else {
				cmd.out.Info.Printf("Docker Machine (%s) URL (%s) matches your environment configuration.", cmd.machine.Name, hostUrl)
			}
		}
	} else {
		cmd.out.Error.Fatalf("No machine named '%s' exists. Did you run 'rig start --name=\"%s\"'?", cmd.machine.Name, cmd.machine.Name)
	}

	// 2. Check Docker API Version compatibility
	clientApiVersion := util.GetDockerClientApiVersion()
	serverApiVersion, err := util.GetDockerServerApiVersion(cmd.machine.Name)
	serverMinApiVersion, _ := util.GetDockerServerMinApiVersion(cmd.machine.Name)

	// Older clients can talk to newer servers, and when you ask a newer server
	// it's version in the presence of an older server it will downgrade it's
	// compatability as far as possible. So as long as the client API is not greater
	// than the servers current version or less than the servers minimum api version
	// then we are compatible
	constraintString := fmt.Sprintf("<= %s", serverApiVersion)
	if serverMinApiVersion != nil {
		constraintString = fmt.Sprintf(">= %s", serverMinApiVersion)
	}
	apiConstraint, _ := version.NewConstraint(constraintString)

	if err != nil {
		cmd.out.Error.Println("Could not determine Docker Machine Docker versions: ", err)
	} else if clientApiVersion.Equal(serverApiVersion) {
		cmd.out.Info.Printf("Docker Client (%s) and Server (%s) have equal API Versions", clientApiVersion, serverApiVersion)
	} else if apiConstraint.Check(clientApiVersion) {
		cmd.out.Info.Printf("Docker Client (%s) has Server compatible API version (%s). Server current (%s), Server min compat (%s)", clientApiVersion, constraintString, serverApiVersion, serverMinApiVersion)
	} else {
		cmd.out.Error.Printf("Docker Client (%s) is incompatible with Server. Server current (%s), Server min compat (%s). Use `rig upgrade` to fix this.", clientApiVersion, serverApiVersion, serverMinApiVersion)
	}

	// 3. Pull down the data from DNSDock. This will confirm we can resolve names as well
	//    as route to the appropriate IP addresses via the added route commands
	if cmd.machine.IsRunning() {
		dnsRecords := DnsRecords{BaseCommand{machine: cmd.machine, out: cmd.out}}
		if records, err := dnsRecords.LoadRecords(); err == nil {
			resolved := false
			for _, record := range records {
				if record["Name"] == "dnsdock" {
					resolved = true
					cmd.out.Info.Printf("DNS and routing services are working. DNSDock resolves to %s", record["IPs"])
					break
				}
			}

			if !resolved {
				cmd.out.Error.Println("Unable to verify DNS services are working.")
			}
		} else {
			cmd.out.Error.Println("Unable to verify DNS services and routing are working.")
			cmd.out.Error.Println(err)
		}
	} else {
		cmd.out.Warning.Printf("Docker Machine `%s` is not running. Cannot determine if DNS resolution is working correctly.", cmd.machine.Name)
	}

	// 4. Ensure that docker-machine-nfs script is available for our NFS mounts (Mac ONLY)
	if runtime.GOOS == "darwin" {
		if err := exec.Command("which", "docker-machine-nfs").Run(); err != nil {
			cmd.out.Error.Println("Docker Machine NFS is not installed.")
		} else {
			cmd.out.Info.Println("Docker Machine NFS is installed.")
		}
	}

	// 5. Check for storage on VM volume
	output, err := exec.Command("docker-machine", "ssh", cmd.machine.Name, "df -h 2> /dev/null | grep /dev/sda1 | head -1 | awk '{print $5}' | sed 's/%//'").Output()
	dataUsage := strings.TrimSpace(string(output))
	if i, err := strconv.Atoi(dataUsage); err == nil {
		if i >= 85 && i < 95 {
			cmd.out.Warning.Printf("Data volume (/data) is %d%% used. Please free up space soon.", i)
		} else if i >= 95 {
			cmd.out.Error.Printf("Data volume (/data) is %d%% used. Please free up space. Try 'docker system prune' or removing old projects / databases from /data.", i)
		} else {
			cmd.out.Info.Printf("Data volume (/data) is %d%% used.", i)
		}
	} else {
		cmd.out.Warning.Printf("Unable to determine usage level of /data volume. Failed to parse '%s'", dataUsage)
	}

	// 6. Check for storage on /Users
	output, err = exec.Command("docker-machine", "ssh", cmd.machine.Name, "df -h 2> /dev/null | grep /Users | head -1 | awk '{print $5}' | sed 's/%//'").Output()
	userUsage := strings.TrimSpace(string(output))
	if i, err := strconv.Atoi(userUsage); err == nil {
		if i >= 85 && i < 95 {
			cmd.out.Warning.Printf("Root drive (/Users) is %d%% used. Please free up space soon.", i)
		} else if i >= 95 {
			cmd.out.Error.Printf("Root drive (/Users) is %d%% used. Please free up space.", i)
		} else {
			cmd.out.Info.Printf("Root drive (/Users) is %d%% used.", i)
		}
	} else {
		cmd.out.Warning.Printf("Unable to determine usage level of root drive (/Users). Failed to parse '%s'", userUsage)
	}

	return nil
}
<commit_msg>Add checks for rig dependencies and harden environment and error handling of disk usage checks.<commit_after>package commands

import (
	"fmt"
	"os"
	"os/exec"
	"runtime"
	"strings"

	"github.com/hashicorp/go-version"
	"github.com/phase2/rig/cli/util"
	"github.com/urfave/cli"
	"strconv"
)

type Doctor struct {
	BaseCommand
}

func (cmd *Doctor) Commands() []cli.Command {
	return []cli.Command{
		{
			Name:   "doctor",
			Usage:  "Troubleshoot the Rig environment",
			Before: cmd.Before,
			Action: cmd.Run,
		},
	}
}

func (cmd *Doctor) Run(c *cli.Context) error {
	// 0. Ensure all of rig's dependencies are available in the PATH.
	if err := exec.Command("docker", "-h").Start(); err == nil {
		cmd.out.Info.Println("Docker is installed.");
	} else {
		cmd.out.Error.Fatal("Docker (docker) is not installed.");
	}
	if runtime.GOOS != "linux" {
		if err := exec.Command("docker-machine", "-h").Start(); err == nil {
			cmd.out.Info.Println("Docker Machine is installed.");
		} else {
			cmd.out.Error.Fatal("Docker Machine (docker-machine) is not installed.");
		}
	}
	if err := exec.Command("docker-compose", "-h").Start(); err == nil {
		cmd.out.Info.Println("Docker Compose is installed.");
	} else {
		cmd.out.Warning.Printf("Docker Compose (docker-compose) is not installed.");
	}

	// 1. Ensure the configured docker-machine matches the set environment.
	if cmd.machine.Exists() {
		if _, isset := os.LookupEnv("DOCKER_MACHINE_NAME"); isset == false {
			cmd.out.Error.Fatalf("Docker configuration is not set. Please run 'eval \"$(rig config)\"'.")
		} else if cmd.machine.Name != os.Getenv("DOCKER_MACHINE_NAME") {
			cmd.out.Error.Fatalf("Your environment configuration specifies a different machine. Please re-run as 'rig --name=\"%s\" doctor'.", cmd.machine.Name)
		} else {
			cmd.out.Info.Printf("Docker Machine (%s) name matches your environment configuration.", cmd.machine.Name)
		}
		if output, err := exec.Command("docker-machine", "url", cmd.machine.Name).Output(); err == nil {
			hostUrl := strings.TrimSpace(string(output))
			if hostUrl != os.Getenv("DOCKER_HOST") {
				cmd.out.Error.Fatalf("Docker Host configuration should be '%s' but got '%s'. Please re-run 'eval \"$(rig config)\"'.", os.Getenv("DOCKER_HOST"), hostUrl)
			} else {
				cmd.out.Info.Printf("Docker Machine (%s) URL (%s) matches your environment configuration.", cmd.machine.Name, hostUrl)
			}
		}
	} else {
		cmd.out.Error.Fatalf("No machine named '%s' exists. Did you run 'rig start --name=\"%s\"'?", cmd.machine.Name, cmd.machine.Name)
	}

	// 2. Check Docker API Version compatibility
	clientApiVersion := util.GetDockerClientApiVersion()
	serverApiVersion, err := util.GetDockerServerApiVersion(cmd.machine.Name)
	serverMinApiVersion, _ := util.GetDockerServerMinApiVersion(cmd.machine.Name)

	// Older clients can talk to newer servers, and when you ask a newer server
	// it's version in the presence of an older server it will downgrade it's
	// compatability as far as possible. So as long as the client API is not greater
	// than the servers current version or less than the servers minimum api version
	// then we are compatible
	constraintString := fmt.Sprintf("<= %s", serverApiVersion)
	if serverMinApiVersion != nil {
		constraintString = fmt.Sprintf(">= %s", serverMinApiVersion)
	}
	apiConstraint, _ := version.NewConstraint(constraintString)

	if err != nil {
		cmd.out.Error.Println("Could not determine Docker Machine Docker versions: ", err)
	} else if clientApiVersion.Equal(serverApiVersion) {
		cmd.out.Info.Printf("Docker Client (%s) and Server (%s) have equal API Versions", clientApiVersion, serverApiVersion)
	} else if apiConstraint.Check(clientApiVersion) {
		cmd.out.Info.Printf("Docker Client (%s) has Server compatible API version (%s). Server current (%s), Server min compat (%s)", clientApiVersion, constraintString, serverApiVersion, serverMinApiVersion)
	} else {
		cmd.out.Error.Printf("Docker Client (%s) is incompatible with Server. Server current (%s), Server min compat (%s). Use `rig upgrade` to fix this.", clientApiVersion, serverApiVersion, serverMinApiVersion)
	}

	// 3. Pull down the data from DNSDock. This will confirm we can resolve names as well
	//    as route to the appropriate IP addresses via the added route commands
	if cmd.machine.IsRunning() {
		dnsRecords := DnsRecords{BaseCommand{machine: cmd.machine, out: cmd.out}}
		if records, err := dnsRecords.LoadRecords(); err == nil {
			resolved := false
			for _, record := range records {
				if record["Name"] == "dnsdock" {
					resolved = true
					cmd.out.Info.Printf("DNS and routing services are working. DNSDock resolves to %s", record["IPs"])
					break
				}
			}

			if !resolved {
				cmd.out.Error.Println("Unable to verify DNS services are working.")
			}
		} else {
			cmd.out.Error.Println("Unable to verify DNS services and routing are working.")
			cmd.out.Error.Println(err)
		}
	} else {
		cmd.out.Warning.Printf("Docker Machine `%s` is not running. Cannot determine if DNS resolution is working correctly.", cmd.machine.Name)
	}

	// 4. Ensure that docker-machine-nfs script is available for our NFS mounts (Mac ONLY)
	if runtime.GOOS == "darwin" {
		if err := exec.Command("which", "docker-machine-nfs").Run(); err != nil {
			cmd.out.Error.Println("Docker Machine NFS is not installed.")
		} else {
			cmd.out.Info.Println("Docker Machine NFS is installed.")
		}
	}

	// 5. Check for storage on VM volume
        if runtime.GOOS != "linux" {
		output, err := exec.Command("docker-machine", "ssh", cmd.machine.Name, "df -h 2> /dev/null | grep /dev/sda1 | head -1 | awk '{print $5}' | sed 's/%//'").Output()
		if err == nil {
			dataUsage := strings.TrimSpace(string(output))
			if i, err := strconv.Atoi(dataUsage); err == nil {
				if i >= 85 && i < 95 {
					cmd.out.Warning.Printf("Data volume (/data) is %d%% used. Please free up space soon.", i)
				} else if i >= 95 {
					cmd.out.Error.Printf("Data volume (/data) is %d%% used. Please free up space. Try 'docker system prune' or removing old projects / databases from /data.", i)
				} else {
					cmd.out.Info.Printf("Data volume (/data) is %d%% used.", i)
				}
			} else {
				cmd.out.Warning.Printf("Unable to determine usage level of /data volume. Failed to parse '%s'", dataUsage)
			}
		} else {
			cmd.out.Warning.Printf("Unable to determine usage level of /data volume. Failed to execute 'df': %v", err);
		}
	}

	// 6. Check for storage on /Users
	if runtime.GOOS != "linux" {
		output, err := exec.Command("docker-machine", "ssh", cmd.machine.Name, "df -h 2> /dev/null | grep /Users | head -1 | awk '{print $5}' | sed 's/%//'").Output()
		if err == nil {
			userUsage := strings.TrimSpace(string(output))
			if i, err := strconv.Atoi(userUsage); err == nil {
				if i >= 85 && i < 95 {
					cmd.out.Warning.Printf("Root drive (/Users) is %d%% used. Please free up space soon.", i)
				} else if i >= 95 {
					cmd.out.Error.Printf("Root drive (/Users) is %d%% used. Please free up space.", i)
				} else {
					cmd.out.Info.Printf("Root drive (/Users) is %d%% used.", i)
				}
			} else {
				cmd.out.Warning.Printf("Unable to determine usage level of root drive (/Users). Failed to parse '%s'", userUsage)
			}
		} else {
			cmd.out.Warning.Printf("Unable to determine usage level of root drive (/Users). Failed to execute 'df': %v", err);
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package address

import "googlemaps.github.io/maps"

func FromGoogleAddressComponents(components []maps.AddressComponent) (address Address) {
	type routeparts struct {
		name, number string
	}
	var street routeparts
	for _, component := range components {
		val := component.ShortName
		for _, label := range component.Types {
			switch label {
			case "street_number":
				street.name = val
			case "route":
				street.number = val

			case "administrative_area_level_1":
				address.Region = val
			case "country":
				address.Country = val
			case "postal_code":
				address.PostalCode = val
			}
			//note: PO box doesn't seem to be handled.
		}
	}
	address.Street = street.number + " " + street.name
	return address
}
<commit_msg>account for extensions in google addresses<commit_after>package address

import "googlemaps.github.io/maps"

func FromGoogleAddressComponents(components []maps.AddressComponent) (address Address) {
	type routeparts struct {
		name, number string
	}
	var street routeparts
	for _, component := range components {
		val := component.ShortName
		for _, label := range component.Types {
			switch label {
			case "street_number":
				street.name = val
			case "route":
				street.number = val
			case "subpremise":
				address.Extension = val
			case "administrative_area_level_1":
				address.Region = val
			case "country":
				address.Country = val
			case "postal_code":
				address.PostalCode = val
			}
			//note: PO box doesn't seem to be handled.
		}
	}
	address.Street = street.number + " " + street.name
	return address
}
<|endoftext|>


<commit_before>package admin

import (
	"context"
	"flag"
	"fmt"
	log "github.com/golang/glog"
	"github.com/jrupac/goliath/fetch"
	"github.com/jrupac/goliath/models"
	"github.com/jrupac/goliath/storage"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/reflection"
	"google.golang.org/grpc/status"
	"net"
)

var (
	adminPort = flag.Int("adminPort", 9997, "Port of gRPC admin server.")
)

type server struct {
	db *storage.Database
}

// AddUser adds a specified user into the database.
// NOTE: This method is currently unimplemented.
func (s *server) AddUser(_ context.Context, _ *AddUserRequest) (*AddUserResponse, error) {
	return nil, status.Error(codes.Unimplemented, "not yet implemented")
}

// AddFeed adds the specified feed into the database.
// During the operation of adding a feed, fetching is paused and restarted.
func (s *server) AddFeed(_ context.Context, req *AddFeedRequest) (*AddFeedResponse, error) {
	resp := &AddFeedResponse{}
	// Zero indicates the root folder.
	parentID := int64(0)

	if req.Title == "" {
		return nil, status.Errorf(codes.InvalidArgument, "must specify Title")
	}
	if req.URL == "" {
		return nil, status.Errorf(codes.InvalidArgument, "must specify URL")
	}
	if req.Link == "" {
		return nil, status.Errorf(codes.InvalidArgument, "must specify Link")
	}
	if req.Username == "" {
		return nil, status.Errorf(codes.InvalidArgument, "must specify Username")
	}

	feed := models.Feed{
		Title:       req.Title,
		Description: req.Description,
		URL:         req.URL,
		Link:        req.Link,
	}

	user, err := s.db.GetUserByUsername(req.Username)
	if err != nil {
		return nil, status.Errorf(codes.NotFound, "could not find user")
	}

	if req.Folder != "" {
		folders, err := s.db.GetAllFoldersForUser(user)
		if err != nil {
			return nil, status.Error(codes.Internal, "internal error")
		}

		for _, f := range folders {
			if f.Name == req.Folder {
				parentID = f.ID
				break
			}
		}

		if parentID == 0 {
			return nil, status.Error(codes.InvalidArgument, "could not find folder")
		}
	}

	fetch.Pause()
	defer fetch.Resume()

	feedID, err := s.db.InsertFeedForUser(user, feed, parentID)
	if err != nil {
		return resp, status.Error(codes.DataLoss, "failed to persist feed")
	}

	resp.Id = feedID

	// TODO: Indicate if feed already existed.
	return resp, nil
}

// GetFeeds lists all the feeds belonging to the requested user.
func (s *server) GetFeeds(_ context.Context, req *GetFeedsRequest) (*GetFeedsResponse, error) {
	resp := &GetFeedsResponse{}

	if req.Username == "" {
		return nil, status.Error(codes.InvalidArgument, "must specify Username")
	}

	user, err := s.db.GetUserByUsername(req.Username)
	if err != nil {
		return nil, status.Error(codes.NotFound, "could not find user")
	}

	feeds, err := s.db.GetAllFeedsForUser(user)
	if err != nil {
		return nil, status.Error(codes.Internal, "internal error")
	}

	for _, f := range feeds {
		resp.Feeds = append(resp.Feeds, &GetFeedsResponse_Feed{Id: f.ID, Title: f.Title})
	}

	return resp, nil
}

// RemoveFeed removes the requested feed for the requested user.
// During the operation of adding a feed, fetching is paused and restarted.
func (s *server) RemoveFeed(_ context.Context, req *RemoveFeedRequest) (*RemoveFeedResponse, error) {
	resp := &RemoveFeedResponse{}

	if req.Username == "" {
		return nil, status.Error(codes.InvalidArgument, "must specify Username")
	}
	if req.Id == 0 {
		return nil, status.Error(codes.InvalidArgument, "must specify non-zero Id")
	}

	user, err := s.db.GetUserByUsername(req.Username)
	if err != nil {
		return nil, status.Error(codes.NotFound, "could not find user")
	}

	feeds, err := s.db.GetAllFeedsForUser(user)
	if err != nil {
		return nil, status.Error(codes.Internal, "internal error")
	}

	var folderId int64 = -1
	for _, f := range feeds {
		if f.ID == req.Id {
			folderId = f.FolderID
			break
		}
	}
	if folderId == -1 {
		return nil, status.Error(codes.InvalidArgument, "could not find feed")
	}

	fetch.Pause()
	defer fetch.Resume()

	err = s.db.DeleteFeedForUser(user, req.Id, folderId)
	if err != nil {
		return nil, status.Error(codes.Internal, "internal error")
	}

	return resp, nil
}

func newServer(d *storage.Database) AdminServiceServer {
	s := &server{db: d}
	return s
}

// Start starts the gRPC admin server.
func Start(ctx context.Context, d *storage.Database) {
	log.Infof("Starting gRPC admin server.")

	s := grpc.NewServer()

	RegisterAdminServiceServer(s, newServer(d))
	reflection.Register(s)

	go func(s *grpc.Server) {
		select {
		case <-ctx.Done():
			log.Infof("Shutting down admin server.")
			s.GracefulStop()
		}
	}(s)

	lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *adminPort))
	if err != nil {
		log.Warningf("failed to listen on admin port: %s", err)
		return
	}

	if err := s.Serve(lis); err != nil {
		log.Infof("%s", err)
	}
}
<commit_msg>core/admin: Support creating a new folder when inserting a feed.<commit_after>package admin

import (
	"context"
	"flag"
	"fmt"
	log "github.com/golang/glog"
	"github.com/jrupac/goliath/fetch"
	"github.com/jrupac/goliath/models"
	"github.com/jrupac/goliath/storage"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/reflection"
	"google.golang.org/grpc/status"
	"net"
)

var (
	adminPort = flag.Int("adminPort", 9997, "Port of gRPC admin server.")
)

type server struct {
	db *storage.Database
}

// AddUser adds a specified user into the database.
// NOTE: This method is currently unimplemented.
func (s *server) AddUser(_ context.Context, _ *AddUserRequest) (*AddUserResponse, error) {
	return nil, status.Error(codes.Unimplemented, "not yet implemented")
}

// AddFeed adds the specified feed into the database.
// During the operation of adding a feed, fetching is paused and restarted.
func (s *server) AddFeed(_ context.Context, req *AddFeedRequest) (*AddFeedResponse, error) {
	resp := &AddFeedResponse{}
	folderID := int64(-1)

	if req.Title == "" {
		return nil, status.Errorf(codes.InvalidArgument, "must specify Title")
	}
	if req.URL == "" {
		return nil, status.Errorf(codes.InvalidArgument, "must specify URL")
	}
	if req.Link == "" {
		return nil, status.Errorf(codes.InvalidArgument, "must specify Link")
	}
	if req.Username == "" {
		return nil, status.Errorf(codes.InvalidArgument, "must specify Username")
	}

	feed := models.Feed{
		Title:       req.Title,
		Description: req.Description,
		URL:         req.URL,
		Link:        req.Link,
	}

	user, err := s.db.GetUserByUsername(req.Username)
	if err != nil {
		return nil, status.Errorf(codes.NotFound, "could not find user")
	}

	if req.Folder != "" {
		folders, err := s.db.GetAllFoldersForUser(user)
		if err != nil {
			return nil, status.Error(codes.Internal, "internal error")
		}

		for _, f := range folders {
			if f.Name == req.Folder {
				folderID = f.ID
				break
			}
		}
	}

	fetch.Pause()
	defer fetch.Resume()

	// Folder ID not found, so create a new one
	if folderID == -1 {
		newFolder := models.Folder{Name: req.Folder}
		folderID, err = s.db.InsertFolderForUser(user, newFolder, 0)
		if err != nil {
			return nil, status.Error(codes.DataLoss, "could not create new folder")
		}
	}

	feedID, err := s.db.InsertFeedForUser(user, feed, folderID)
	if err != nil {
		return resp, status.Error(codes.DataLoss, "failed to persist feed")
	}

	resp.Id = feedID

	// TODO: Indicate if feed already existed.
	return resp, nil
}

// GetFeeds lists all the feeds belonging to the requested user.
func (s *server) GetFeeds(_ context.Context, req *GetFeedsRequest) (*GetFeedsResponse, error) {
	resp := &GetFeedsResponse{}

	if req.Username == "" {
		return nil, status.Error(codes.InvalidArgument, "must specify Username")
	}

	user, err := s.db.GetUserByUsername(req.Username)
	if err != nil {
		return nil, status.Error(codes.NotFound, "could not find user")
	}

	feeds, err := s.db.GetAllFeedsForUser(user)
	if err != nil {
		return nil, status.Error(codes.Internal, "internal error")
	}

	for _, f := range feeds {
		resp.Feeds = append(resp.Feeds, &GetFeedsResponse_Feed{Id: f.ID, Title: f.Title})
	}

	return resp, nil
}

// RemoveFeed removes the requested feed for the requested user.
// During the operation of adding a feed, fetching is paused and restarted.
func (s *server) RemoveFeed(_ context.Context, req *RemoveFeedRequest) (*RemoveFeedResponse, error) {
	resp := &RemoveFeedResponse{}

	if req.Username == "" {
		return nil, status.Error(codes.InvalidArgument, "must specify Username")
	}
	if req.Id == 0 {
		return nil, status.Error(codes.InvalidArgument, "must specify non-zero Id")
	}

	user, err := s.db.GetUserByUsername(req.Username)
	if err != nil {
		return nil, status.Error(codes.NotFound, "could not find user")
	}

	feeds, err := s.db.GetAllFeedsForUser(user)
	if err != nil {
		return nil, status.Error(codes.Internal, "internal error")
	}

	var folderId int64 = -1
	for _, f := range feeds {
		if f.ID == req.Id {
			folderId = f.FolderID
			break
		}
	}
	if folderId == -1 {
		return nil, status.Error(codes.InvalidArgument, "could not find feed")
	}

	fetch.Pause()
	defer fetch.Resume()

	err = s.db.DeleteFeedForUser(user, req.Id, folderId)
	if err != nil {
		return nil, status.Error(codes.Internal, "internal error")
	}

	return resp, nil
}

func newServer(d *storage.Database) AdminServiceServer {
	s := &server{db: d}
	return s
}

// Start starts the gRPC admin server.
func Start(ctx context.Context, d *storage.Database) {
	log.Infof("Starting gRPC admin server.")

	s := grpc.NewServer()

	RegisterAdminServiceServer(s, newServer(d))
	reflection.Register(s)

	go func(s *grpc.Server) {
		select {
		case <-ctx.Done():
			log.Infof("Shutting down admin server.")
			s.GracefulStop()
		}
	}(s)

	lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *adminPort))
	if err != nil {
		log.Warningf("failed to listen on admin port: %s", err)
		return
	}

	if err := s.Serve(lis); err != nil {
		log.Infof("%s", err)
	}
}
<|endoftext|>