<commit_before>package vsolver

import "strings"

type selection struct {
	projects []ProjectID
	deps     map[ProjectIdentifier][]Dependency
}

func (s *selection) getDependenciesOn(id ProjectIdentifier) []Dependency {
	if deps, exists := s.deps[id]; exists {
		return deps
	}

	return nil
}

func (s *selection) setDependenciesOn(id ProjectIdentifier, deps []Dependency) {
	s.deps[id] = deps
}

func (s *selection) getConstraint(id ProjectIdentifier) Constraint {
	deps, exists := s.deps[id]
	if !exists {
		return anyConstraint{}
	}

	// TODO recomputing this sucks and is quite wasteful. Precompute/cache it
	// on changes to the constraint set, instead.

	// The solver itself is expected to maintain the invariant that all the
	// constraints kept here collectively admit a non-empty set of versions. We
	// assume this is the case here while assembling a composite constraint.
	//
	// TODO verify that this invariant is maintained; also verify that the slice
	// can't be empty

	// If the first constraint requires an exact match, then we know all the
	// others must be identical, so just return the first one
	if deps[0].Dep.Constraint.Type()&C_ExactMatch != 0 {
		return deps[0].Dep.Constraint
	}

	// Otherwise, we're dealing with semver ranges, so we have to compute the
	// constraint intersection
	var cs []string
	for _, dep := range deps {
		cs = append(cs, dep.Dep.Constraint.Body())
	}

	c, err := NewConstraint(C_SemverRange, strings.Join(cs, ", "))
	if err != nil {
		panic("canary - something wrong with constraint computation")
	}

	return c
}

func (s *selection) selected(id ProjectIdentifier) (ProjectID, bool) {
	for _, pi := range s.projects {
		if pi.ID == id {
			return pi, true
		}
	}

	return ProjectID{}, false
}

type unselected struct {
	sl  []ProjectIdentifier
	cmp func(i, j int) bool
}

// TODO should these be pointer receivers? container/heap examples aren't
func (u unselected) Len() int {
	return len(u.sl)
}

func (u unselected) Less(i, j int) bool {
	return u.cmp(i, j)
}

func (u unselected) Swap(i, j int) {
	u.sl[i], u.sl[j] = u.sl[j], u.sl[i]
}

func (u *unselected) Push(x interface{}) {
	//*u.sl = append(*u.sl, x.(ProjectIdentifier))
	u.sl = append(u.sl, x.(ProjectIdentifier))
}

func (u *unselected) Pop() (v interface{}) {
	//old := *u.sl
	//v := old[len(old)-1]
	//*u = old[:len(old)-1]
	v, u.sl = u.sl[len(u.sl)-1], u.sl[:len(u.sl)-1]
	return v
}

// remove takes an ProjectIdentifier out of the priority queue (if it was
// present), then reapplies the heap invariants.
func (u *unselected) remove(id ProjectIdentifier) {
	for k, pi := range u.sl {
		if pi == id {
			u.sl = append(u.sl[:k], u.sl[k+1:]...)
			// TODO need to heap.Fix()? shouldn't have to...
		}
	}
}
<commit_msg>On heap removal, only cut if not last elem<commit_after>package vsolver

import "strings"

type selection struct {
	projects []ProjectID
	deps     map[ProjectIdentifier][]Dependency
}

func (s *selection) getDependenciesOn(id ProjectIdentifier) []Dependency {
	if deps, exists := s.deps[id]; exists {
		return deps
	}

	return nil
}

func (s *selection) setDependenciesOn(id ProjectIdentifier, deps []Dependency) {
	s.deps[id] = deps
}

func (s *selection) getConstraint(id ProjectIdentifier) Constraint {
	deps, exists := s.deps[id]
	if !exists {
		return anyConstraint{}
	}

	// TODO recomputing this sucks and is quite wasteful. Precompute/cache it
	// on changes to the constraint set, instead.

	// The solver itself is expected to maintain the invariant that all the
	// constraints kept here collectively admit a non-empty set of versions. We
	// assume this is the case here while assembling a composite constraint.
	//
	// TODO verify that this invariant is maintained; also verify that the slice
	// can't be empty

	// If the first constraint requires an exact match, then we know all the
	// others must be identical, so just return the first one
	if deps[0].Dep.Constraint.Type()&C_ExactMatch != 0 {
		return deps[0].Dep.Constraint
	}

	// Otherwise, we're dealing with semver ranges, so we have to compute the
	// constraint intersection
	var cs []string
	for _, dep := range deps {
		cs = append(cs, dep.Dep.Constraint.Body())
	}

	c, err := NewConstraint(C_SemverRange, strings.Join(cs, ", "))
	if err != nil {
		panic("canary - something wrong with constraint computation")
	}

	return c
}

func (s *selection) selected(id ProjectIdentifier) (ProjectID, bool) {
	for _, pi := range s.projects {
		if pi.ID == id {
			return pi, true
		}
	}

	return ProjectID{}, false
}

type unselected struct {
	sl  []ProjectIdentifier
	cmp func(i, j int) bool
}

// TODO should these be pointer receivers? container/heap examples aren't
func (u unselected) Len() int {
	return len(u.sl)
}

func (u unselected) Less(i, j int) bool {
	return u.cmp(i, j)
}

func (u unselected) Swap(i, j int) {
	u.sl[i], u.sl[j] = u.sl[j], u.sl[i]
}

func (u *unselected) Push(x interface{}) {
	//*u.sl = append(*u.sl, x.(ProjectIdentifier))
	u.sl = append(u.sl, x.(ProjectIdentifier))
}

func (u *unselected) Pop() (v interface{}) {
	//old := *u.sl
	//v := old[len(old)-1]
	//*u = old[:len(old)-1]
	v, u.sl = u.sl[len(u.sl)-1], u.sl[:len(u.sl)-1]
	return v
}

// remove takes a ProjectIdentifier out of the priority queue (if it was
// present), then reapplies the heap invariants.
func (u *unselected) remove(id ProjectIdentifier) {
	for k, pi := range u.sl {
		if pi == id {
			if k == len(u.sl)-1 {
				// if we're on the last element, just pop, no splice
				u.sl = u.sl[:len(u.sl)-1]
			} else {
				u.sl = append(u.sl[:k], u.sl[k+1:]...)
			}
			break
			// TODO need to heap.Fix()? shouldn't have to...
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"flag"
	"log"
	"strconv"
	"time"

	"github.com/brotherlogic/goserver"
	"google.golang.org/grpc"

	pbd "github.com/brotherlogic/discovery/proto"
	pb "github.com/brotherlogic/gobuildmaster/proto"
	pbs "github.com/brotherlogic/gobuildslave/proto"
)

const (
	intentWait = 1000
)

// Server the main server type
type Server struct {
	*goserver.GoServer
	config  *pb.Config
	serving bool
}

type mainChecker struct{}

func getIP(servertype, servername string) (string, int) {
	conn, _ := grpc.Dial("192.168.86.34:50055", grpc.WithInsecure())
	defer conn.Close()

	registry := pbd.NewDiscoveryServiceClient(conn)
	r, err := registry.ListAllServices(context.Background(), &pbd.Empty{})
	if err != nil {
		return "", -1
	}
	for _, s := range r.Services {
		log.Printf("Does %v = %v and %v = %v?", s.Name, servertype, s.Identifier, servername)
		if s.Name == servertype && s.Identifier == servername {
			return s.Ip, int(s.Port)
		}
	}

	return "", -1
}

func (t *mainChecker) assess(server string) *pbs.JobList {
	list := &pbs.JobList{}

	log.Printf("Assessing server %v", server)
	ip, port := getIP("gobuildslave", server)
	conn, _ := grpc.Dial(ip+":"+strconv.Itoa(port), grpc.WithInsecure())
	defer conn.Close()

	slave := pbs.NewGoBuildSlaveClient(conn)
	r, err := slave.List(context.Background(), &pbs.Empty{})
	if err == nil {
		log.Printf("FOUND %v", r)
		return r
	}

	log.Printf("Error in listing: %v", err)

	return list
}

// Find the first available server
func chooseServer(job *pbs.JobSpec, c checker) string {
	for _, service := range c.discover().Services {
		if service.Name == "gobuildslave" {
			return service.Identifier
		}
	}
	return ""
}

func runJob(job *pbs.JobSpec, server string) {
	log.Printf("RUNNING: %v on %v", job, server)
	ip, port := getIP("gobuildslave", server)
	conn, _ := grpc.Dial(ip+":"+strconv.Itoa(port), grpc.WithInsecure())
	defer conn.Close()

	slave := pbs.NewGoBuildSlaveClient(conn)
	slave.Run(context.Background(), job)
	log.Printf("RUN COMMAND SENT %v", job)
}

func (t *mainChecker) discover() *pbd.ServiceList {
	ret := &pbd.ServiceList{}

	conn, _ := grpc.Dial("192.168.86.34:50055", grpc.WithInsecure())
	defer conn.Close()

	registry := pbd.NewDiscoveryServiceClient(conn)
	r, err := registry.ListAllServices(context.Background(), &pbd.Empty{})
	log.Printf("DISCOVERED: %v", r)
	if err == nil {
		for _, s := range r.Services {
			ret.Services = append(ret.Services, s)
		}
	}

	return ret
}

// DoRegister Registers this server
func (s Server) DoRegister(server *grpc.Server) {
	// Do nothing
}

// ReportHealth determines if the server is healthy
func (s Server) ReportHealth() bool {
	return true
}

func getConfig(c checker) *pb.Config {
	list := getFleetStatus(c)
	config := &pb.Config{}

	for _, jlist := range list {
		for _, job := range jlist.Details {
			found := false
			for _, ij := range config.Intents {
				if job.Spec.Name == ij.Spec.Name {
					ij.Masters++
					found = true
				}
			}

			if !found {
				config.Intents = append(config.Intents, &pb.Intent{Spec: &pbs.JobSpec{Name: job.Spec.Name}, Masters: 1})
			}
		}
	}

	return config
}

// MatchIntent tries to match the intent with the state of production
func (s Server) MatchIntent() {
	//for s.serving {
	time.Sleep(intentWait)

	state := getConfig(&mainChecker{})
	diff := configDiff(s.config, state)
	joblist := runJobs(diff)
	log.Printf("FOUND %v from %v and %v", joblist, state, s.config)
	for _, job := range joblist {
		runJob(job, chooseServer(job, &mainChecker{}))
	}
	//}
}

func main() {
	config, err := loadConfig("config.pb")
	if err != nil {
		log.Fatalf("Fatal loading of config: %v", err)
	}
	log.Printf("READ: %v", config)

	var sync = flag.Bool("once", true, "One pass intent match")
	s := Server{&goserver.GoServer{}, config, true}
	if *sync {
		s.MatchIntent()
	} else {
		s.Register = s
		s.PrepServer()
		s.RegisterServer("gobuildmaster", false)
		s.Serve()
	}
}
<commit_msg>Added flag parse step<commit_after>package main

import (
	"context"
	"flag"
	"log"
	"strconv"
	"time"

	"github.com/brotherlogic/goserver"
	"google.golang.org/grpc"

	pbd "github.com/brotherlogic/discovery/proto"
	pb "github.com/brotherlogic/gobuildmaster/proto"
	pbs "github.com/brotherlogic/gobuildslave/proto"
)

const (
	intentWait = 1000
)

// Server the main server type
type Server struct {
	*goserver.GoServer
	config  *pb.Config
	serving bool
}

type mainChecker struct{}

func getIP(servertype, servername string) (string, int) {
	conn, _ := grpc.Dial("192.168.86.34:50055", grpc.WithInsecure())
	defer conn.Close()

	registry := pbd.NewDiscoveryServiceClient(conn)
	r, err := registry.ListAllServices(context.Background(), &pbd.Empty{})
	if err != nil {
		return "", -1
	}
	for _, s := range r.Services {
		log.Printf("Does %v = %v and %v = %v?", s.Name, servertype, s.Identifier, servername)
		if s.Name == servertype && s.Identifier == servername {
			return s.Ip, int(s.Port)
		}
	}

	return "", -1
}

func (t *mainChecker) assess(server string) *pbs.JobList {
	list := &pbs.JobList{}

	log.Printf("Assessing server %v", server)
	ip, port := getIP("gobuildslave", server)
	conn, _ := grpc.Dial(ip+":"+strconv.Itoa(port), grpc.WithInsecure())
	defer conn.Close()

	slave := pbs.NewGoBuildSlaveClient(conn)
	r, err := slave.List(context.Background(), &pbs.Empty{})
	if err == nil {
		log.Printf("FOUND %v", r)
		return r
	}

	log.Printf("Error in listing: %v", err)

	return list
}

// Find the first available server
func chooseServer(job *pbs.JobSpec, c checker) string {
	for _, service := range c.discover().Services {
		if service.Name == "gobuildslave" {
			return service.Identifier
		}
	}
	return ""
}

func runJob(job *pbs.JobSpec, server string) {
	log.Printf("RUNNING: %v on %v", job, server)
	ip, port := getIP("gobuildslave", server)
	conn, _ := grpc.Dial(ip+":"+strconv.Itoa(port), grpc.WithInsecure())
	defer conn.Close()

	slave := pbs.NewGoBuildSlaveClient(conn)
	slave.Run(context.Background(), job)
	log.Printf("RUN COMMAND SENT %v", job)
}

func (t *mainChecker) discover() *pbd.ServiceList {
	ret := &pbd.ServiceList{}

	conn, _ := grpc.Dial("192.168.86.34:50055", grpc.WithInsecure())
	defer conn.Close()

	registry := pbd.NewDiscoveryServiceClient(conn)
	r, err := registry.ListAllServices(context.Background(), &pbd.Empty{})
	log.Printf("DISCOVERED: %v", r)
	if err == nil {
		for _, s := range r.Services {
			ret.Services = append(ret.Services, s)
		}
	}

	return ret
}

// DoRegister Registers this server
func (s Server) DoRegister(server *grpc.Server) {
	// Do nothing
}

// ReportHealth determines if the server is healthy
func (s Server) ReportHealth() bool {
	return true
}

func getConfig(c checker) *pb.Config {
	list := getFleetStatus(c)
	config := &pb.Config{}

	for _, jlist := range list {
		for _, job := range jlist.Details {
			found := false
			for _, ij := range config.Intents {
				if job.Spec.Name == ij.Spec.Name {
					ij.Masters++
					found = true
				}
			}

			if !found {
				config.Intents = append(config.Intents, &pb.Intent{Spec: &pbs.JobSpec{Name: job.Spec.Name}, Masters: 1})
			}
		}
	}

	return config
}

// MatchIntent tries to match the intent with the state of production
func (s Server) MatchIntent() {
	//for s.serving {
	time.Sleep(intentWait)

	state := getConfig(&mainChecker{})
	diff := configDiff(s.config, state)
	joblist := runJobs(diff)
	log.Printf("FOUND %v from %v and %v", joblist, state, s.config)
	for _, job := range joblist {
		runJob(job, chooseServer(job, &mainChecker{}))
	}
	//}
}

func main() {
	config, err := loadConfig("config.pb")
	if err != nil {
		log.Fatalf("Fatal loading of config: %v", err)
	}
	log.Printf("READ: %v", config)

	var sync = flag.Bool("once", true, "One pass intent match")
	s := Server{&goserver.GoServer{}, config, true}

	flag.Parse()
	if *sync {
		s.MatchIntent()
	} else {
		s.Register = s
		s.PrepServer()
		s.RegisterServer("gobuildmaster", false)
		s.Serve()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"strconv"
	"sync"
	"time"

	"github.com/brotherlogic/goserver"
	"golang.org/x/net/context"
	"google.golang.org/grpc"

	pbd "github.com/brotherlogic/discovery/proto"
	pb "github.com/brotherlogic/gobuildmaster/proto"
	pbs "github.com/brotherlogic/gobuildslave/proto"
	pbg "github.com/brotherlogic/goserver/proto"
	"github.com/brotherlogic/goserver/utils"
)

const (
	intentWait = time.Second
)

// Server the main server type
type Server struct {
	*goserver.GoServer
	config     *pb.Config
	serving    bool
	LastIntent time.Time
	LastMaster time.Time
	worldMutex *sync.Mutex
	world      map[string]map[string]struct{}
	getter     getter
}

type prodGetter struct{}

func (g *prodGetter) getJobs(server *pbd.RegistryEntry) (*pbs.JobList, error) {
	list := &pbs.JobList{}

	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	conn, err := grpc.Dial(server.GetIp()+":"+strconv.Itoa(int(server.GetPort())), grpc.WithInsecure())
	defer conn.Close()
	if err != nil {
		return list, err
	}

	slave := pbs.NewGoBuildSlaveClient(conn)
	r, err := slave.List(ctx, &pbs.Empty{}, grpc.FailFast(false))
	return r, err
}

func (s *Server) checkerThread(i *pb.Intent) {
	for true {
		time.Sleep(time.Minute)

		if len(s.world[i.GetSpec().GetName()]) != int(i.Count) {
			s.Log(fmt.Sprintf("MISMATCH: %v, %v", i, s.world[i.GetSpec().GetName()]))
			if len(s.world[i.GetSpec().GetName()]) < int(i.Count) {
				s.Log(fmt.Sprintf("UP : %v", i.GetSpec().GetName()))
			}
		}
	}
}

func (g *prodGetter) getSlaves() (*pbd.ServiceList, error) {
	ret := &pbd.ServiceList{}

	conn, err := grpc.Dial(utils.RegistryIP+":"+strconv.Itoa(utils.RegistryPort), grpc.WithInsecure())
	if err != nil {
		return ret, err
	}
	defer conn.Close()

	registry := pbd.NewDiscoveryServiceClient(conn)
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	r, err := registry.ListAllServices(ctx, &pbd.Empty{}, grpc.FailFast(false))
	if err != nil {
		return ret, err
	}

	for _, s := range r.Services {
		if s.GetName() == "gobuildslave" {
			ret.Services = append(ret.Services, s)
		}
	}

	return ret, nil
}

type mainChecker struct {
	prev   []string
	logger func(string)
}

func getIP(servertype, servername string) (string, int) {
	conn, _ := grpc.Dial(utils.RegistryIP+":"+strconv.Itoa(utils.RegistryPort), grpc.WithInsecure())
	defer conn.Close()

	registry := pbd.NewDiscoveryServiceClient(conn)
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	r, err := registry.ListAllServices(ctx, &pbd.Empty{}, grpc.FailFast(false))
	if err != nil {
		return "", -1
	}
	for _, s := range r.Services {
		if s.Name == servertype && s.Identifier == servername {
			return s.Ip, int(s.Port)
		}
	}

	return "", -1
}

func (t *mainChecker) getprev() []string {
	return t.prev
}
func (t *mainChecker) setprev(v []string) {
	t.prev = v
}

func (t *mainChecker) assess(server string) (*pbs.JobList, *pbs.Config) {
	list := &pbs.JobList{}
	conf := &pbs.Config{}

	ip, port := getIP("gobuildslave", server)
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	conn, err := grpc.Dial(ip+":"+strconv.Itoa(port), grpc.WithInsecure())
	defer conn.Close()
	if err != nil {
		return list, conf
	}

	slave := pbs.NewGoBuildSlaveClient(conn)
	r, err := slave.List(ctx, &pbs.Empty{}, grpc.FailFast(false))
	if err != nil {
		return list, conf
	}

	r2, err := slave.GetConfig(ctx, &pbs.Empty{}, grpc.FailFast(false))
	if err != nil {
		return list, conf
	}

	return r, r2
}

func (t *mainChecker) master(entry *pbd.RegistryEntry, master bool) bool {
	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
	defer cancel()
	conn, _ := grpc.Dial(entry.GetIp()+":"+strconv.Itoa(int(entry.GetPort())), grpc.WithInsecure())
	defer conn.Close()

	server := pbg.NewGoserverServiceClient(conn)
	_, err := server.Mote(ctx, &pbg.MoteRequest{Master: master}, grpc.FailFast(false))
	if err != nil {
		t.logger(fmt.Sprintf("Master REJECT(%v): %v", entry, err))
	}

	return err == nil
}

func runJob(job *pbs.JobSpec, server string) {
	if server != "" {
		ip, port := getIP("gobuildslave", server)
		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
		defer cancel()
		conn, _ := grpc.Dial(ip+":"+strconv.Itoa(port), grpc.WithInsecure())
		defer conn.Close()

		slave := pbs.NewGoBuildSlaveClient(conn)
		job.Server = server
		slave.Run(ctx, job, grpc.FailFast(false))
	}
}

func (t *mainChecker) discover() *pbd.ServiceList {
	ret := &pbd.ServiceList{}

	conn, _ := grpc.Dial(utils.RegistryIP+":"+strconv.Itoa(utils.RegistryPort), grpc.WithInsecure())
	defer conn.Close()

	registry := pbd.NewDiscoveryServiceClient(conn)
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	r, err := registry.ListAllServices(ctx, &pbd.Empty{}, grpc.FailFast(false))
	if err == nil {
		for _, s := range r.Services {
			ret.Services = append(ret.Services, s)
		}
	}

	return ret
}

// DoRegister Registers this server
func (s Server) DoRegister(server *grpc.Server) {
	pb.RegisterGoBuildMasterServer(server, &s)
}

// ReportHealth determines if the server is healthy
func (s Server) ReportHealth() bool {
	return true
}

// Mote promotes/demotes this server
func (s Server) Mote(master bool) error {
	return nil
}

//GetState gets the state of the server
func (s Server) GetState() []*pbg.State {
	return []*pbg.State{&pbg.State{Key: "last_intent", TimeValue: s.LastIntent.Unix()},
		&pbg.State{Key: "last_master", TimeValue: s.LastMaster.Unix()},
		&pbg.State{Key: "world", Text: fmt.Sprintf("%v", s.world)}}
}

//Compare compares current state to desired state
func (s Server) Compare(ctx context.Context, in *pb.Empty) (*pb.CompareResponse, error) {
	resp := &pb.CompareResponse{}
	list, _ := getFleetStatus(&mainChecker{logger: s.Log})
	cc := &pb.Config{}
	for _, jlist := range list {
		for _, job := range jlist.GetDetails() {
			cc.Intents = append(cc.Intents, &pb.Intent{Spec: job.GetSpec()})
		}
	}
	resp.Current = cc
	resp.Desired = s.config

	return resp, nil
}

func getConfig(c checker) *pb.Config {
	list, _ := getFleetStatus(c)
	config := &pb.Config{}

	for _, jlist := range list {
		for _, job := range jlist.Details {
			found := false
			for _, ij := range config.Intents {
				if job.Spec.Name == ij.Spec.Name {
					ij.Count++
					found = true
				}
			}

			if !found {
				config.Intents = append(config.Intents, &pb.Intent{Spec: &pbs.JobSpec{Name: job.Spec.Name}, Count: 1})
			}
		}
	}

	return config
}

// MatchIntent tries to match the intent with the state of production
func (s *Server) MatchIntent() {
	checker := &mainChecker{logger: s.Log}
	for s.serving {
		time.Sleep(intentWait)
		s.LastIntent = time.Now()

		state := getConfig(checker)
		diff := configDiff(s.config, state)
		joblist := runJobs(diff)
		for _, job := range joblist {
			runJob(job, chooseServer(job, checker))
		}
	}
}

// SetMaster sets up the master settings
func (s *Server) SetMaster() {
	t := time.Now()
	checker := &mainChecker{logger: s.Log}
	s.LastMaster = time.Now()

	fleet := checker.discover()
	matcher := make(map[string][]*pbd.RegistryEntry)
	hasMaster := make(map[string]int)
	for _, entry := range fleet.GetServices() {
		if !entry.GetIgnoresMaster() {
			if _, ok := matcher[entry.GetName()]; !ok {
				if entry.GetMaster() {
					hasMaster[entry.GetName()]++
				}
				matcher[entry.GetName()] = []*pbd.RegistryEntry{entry}
			} else {
				if entry.GetMaster() {
					hasMaster[entry.GetName()] = 1
					matcher[entry.GetName()] = append(matcher[entry.GetName()], entry)
				}
			}
		}
	}

	for key, entries := range matcher {
		if hasMaster[key] > 1 {
			hasMaster[key] = 1
			seen := false
			for _, entry := range entries {
				if seen && entry.GetMaster() {
					checker.master(entry, false)
				} else if entry.GetMaster() {
					seen = true
				}
			}
		}

		if hasMaster[key] == 0 {
			for _, entry := range entries {
				if checker.master(entry, true) {
					entry.Master = true
					break
				}
			}
		}
	}
	s.LogFunction("SetMasterRun", t)
}

//Init builds up the server
func Init(config *pb.Config) *Server {
	s := &Server{
		&goserver.GoServer{},
		config,
		true,
		time.Now(),
		time.Now(),
		&sync.Mutex{},
		make(map[string]map[string]struct{}),
		&prodGetter{},
	}
	return s
}

func (s *Server) becomeMaster() {
	for true {
		time.Sleep(time.Second * 5)
		_, _, err := utils.Resolve("gobuildmaster")
		if err != nil {
			s.Registry.Master = true
		}
	}
}

func main() {
	config, err := loadConfig("config.pb")
	if err != nil {
		log.Fatalf("Fatal loading of config: %v", err)
	}

	s := Init(config)

	var quiet = flag.Bool("quiet", false, "Show all output")
	flag.Parse()

	if *quiet {
		log.SetFlags(0)
		log.SetOutput(ioutil.Discard)
	}

	s.Register = s
	s.PrepServer()
	s.GoServer.Killme = false
	s.RegisterServer("gobuildmaster", false)
	s.RegisterServingTask(s.MatchIntent)
	s.RegisterServingTask(s.becomeMaster)
	s.RegisterRepeatingTask(s.SetMaster, time.Second)
	s.RegisterRepeatingTask(s.buildWorld, time.Minute)

	for i := 0; i < len(s.config.GetIntents()); i++ {
		go s.checkerThread(s.config.GetIntents()[i])
	}

	err = s.Serve()
	if err != nil {
		log.Fatalf("Serve error: %v", err)
	}
}
<commit_msg>Added return of master map<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"strconv"
	"sync"
	"time"

	"github.com/brotherlogic/goserver"
	"golang.org/x/net/context"
	"google.golang.org/grpc"

	pbd "github.com/brotherlogic/discovery/proto"
	pb "github.com/brotherlogic/gobuildmaster/proto"
	pbs "github.com/brotherlogic/gobuildslave/proto"
	pbg "github.com/brotherlogic/goserver/proto"
	"github.com/brotherlogic/goserver/utils"
)

const (
	intentWait = time.Second
)

// Server the main server type
type Server struct {
	*goserver.GoServer
	config     *pb.Config
	serving    bool
	LastIntent time.Time
	LastMaster time.Time
	worldMutex *sync.Mutex
	world      map[string]map[string]struct{}
	getter     getter
	mapString  string
}

type prodGetter struct{}

func (g *prodGetter) getJobs(server *pbd.RegistryEntry) (*pbs.JobList, error) {
	list := &pbs.JobList{}

	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	conn, err := grpc.Dial(server.GetIp()+":"+strconv.Itoa(int(server.GetPort())), grpc.WithInsecure())
	defer conn.Close()
	if err != nil {
		return list, err
	}

	slave := pbs.NewGoBuildSlaveClient(conn)
	r, err := slave.List(ctx, &pbs.Empty{}, grpc.FailFast(false))
	return r, err
}

func (s *Server) checkerThread(i *pb.Intent) {
	for true {
		time.Sleep(time.Minute)

		if len(s.world[i.GetSpec().GetName()]) != int(i.Count) {
			s.Log(fmt.Sprintf("MISMATCH: %v, %v", i, s.world[i.GetSpec().GetName()]))
			if len(s.world[i.GetSpec().GetName()]) < int(i.Count) {
				s.Log(fmt.Sprintf("UP : %v", i.GetSpec().GetName()))
			}
		}
	}
}

func (g *prodGetter) getSlaves() (*pbd.ServiceList, error) {
	ret := &pbd.ServiceList{}

	conn, err := grpc.Dial(utils.RegistryIP+":"+strconv.Itoa(utils.RegistryPort), grpc.WithInsecure())
	if err != nil {
		return ret, err
	}
	defer conn.Close()

	registry := pbd.NewDiscoveryServiceClient(conn)
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	r, err := registry.ListAllServices(ctx, &pbd.Empty{}, grpc.FailFast(false))
	if err != nil {
		return ret, err
	}

	for _, s := range r.Services {
		if s.GetName() == "gobuildslave" {
			ret.Services = append(ret.Services, s)
		}
	}

	return ret, nil
}

type mainChecker struct {
	prev   []string
	logger func(string)
}

func getIP(servertype, servername string) (string, int) {
	conn, _ := grpc.Dial(utils.RegistryIP+":"+strconv.Itoa(utils.RegistryPort), grpc.WithInsecure())
	defer conn.Close()

	registry := pbd.NewDiscoveryServiceClient(conn)
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	r, err := registry.ListAllServices(ctx, &pbd.Empty{}, grpc.FailFast(false))
	if err != nil {
		return "", -1
	}
	for _, s := range r.Services {
		if s.Name == servertype && s.Identifier == servername {
			return s.Ip, int(s.Port)
		}
	}

	return "", -1
}

func (t *mainChecker) getprev() []string {
	return t.prev
}
func (t *mainChecker) setprev(v []string) {
	t.prev = v
}

func (t *mainChecker) assess(server string) (*pbs.JobList, *pbs.Config) {
	list := &pbs.JobList{}
	conf := &pbs.Config{}

	ip, port := getIP("gobuildslave", server)
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	conn, err := grpc.Dial(ip+":"+strconv.Itoa(port), grpc.WithInsecure())
	defer conn.Close()
	if err != nil {
		return list, conf
	}

	slave := pbs.NewGoBuildSlaveClient(conn)
	r, err := slave.List(ctx, &pbs.Empty{}, grpc.FailFast(false))
	if err != nil {
		return list, conf
	}

	r2, err := slave.GetConfig(ctx, &pbs.Empty{}, grpc.FailFast(false))
	if err != nil {
		return list, conf
	}

	return r, r2
}

func (t *mainChecker) master(entry *pbd.RegistryEntry, master bool) bool {
	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
	defer cancel()
	conn, _ := grpc.Dial(entry.GetIp()+":"+strconv.Itoa(int(entry.GetPort())), grpc.WithInsecure())
	defer conn.Close()

	server := pbg.NewGoserverServiceClient(conn)
	_, err := server.Mote(ctx, &pbg.MoteRequest{Master: master}, grpc.FailFast(false))
	if err != nil {
		t.logger(fmt.Sprintf("Master REJECT(%v): %v", entry, err))
	}

	return err == nil
}

func runJob(job *pbs.JobSpec, server string) {
	if server != "" {
		ip, port := getIP("gobuildslave", server)
		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
		defer cancel()
		conn, _ := grpc.Dial(ip+":"+strconv.Itoa(port), grpc.WithInsecure())
		defer conn.Close()

		slave := pbs.NewGoBuildSlaveClient(conn)
		job.Server = server
		slave.Run(ctx, job, grpc.FailFast(false))
	}
}

func (t *mainChecker) discover() *pbd.ServiceList {
	ret := &pbd.ServiceList{}

	conn, _ := grpc.Dial(utils.RegistryIP+":"+strconv.Itoa(utils.RegistryPort), grpc.WithInsecure())
	defer conn.Close()

	registry := pbd.NewDiscoveryServiceClient(conn)
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	r, err := registry.ListAllServices(ctx, &pbd.Empty{}, grpc.FailFast(false))
	if err == nil {
		for _, s := range r.Services {
			ret.Services = append(ret.Services, s)
		}
	}

	return ret
}

// DoRegister Registers this server
func (s Server) DoRegister(server *grpc.Server) {
	pb.RegisterGoBuildMasterServer(server, &s)
}

// ReportHealth determines if the server is healthy
func (s Server) ReportHealth() bool {
	return true
}

// Mote promotes/demotes this server
func (s Server) Mote(master bool) error {
	return nil
}

//GetState gets the state of the server
func (s Server) GetState() []*pbg.State {
	return []*pbg.State{&pbg.State{Key: "last_intent", TimeValue: s.LastIntent.Unix()},
		&pbg.State{Key: "last_master", TimeValue: s.LastMaster.Unix()},
		&pbg.State{Key: "world", Text: fmt.Sprintf("%v", s.world)},
		&pbg.State{Key: "master", Text: s.mapString}}
}

//Compare compares current state to desired state
func (s Server) Compare(ctx context.Context, in *pb.Empty) (*pb.CompareResponse, error) {
	resp := &pb.CompareResponse{}
	list, _ := getFleetStatus(&mainChecker{logger: s.Log})
	cc := &pb.Config{}
	for _, jlist := range list {
		for _, job := range jlist.GetDetails() {
			cc.Intents = append(cc.Intents, &pb.Intent{Spec: job.GetSpec()})
		}
	}
	resp.Current = cc
	resp.Desired = s.config

	return resp, nil
}

func getConfig(c checker) *pb.Config {
	list, _ := getFleetStatus(c)
	config := &pb.Config{}

	for _, jlist := range list {
		for _, job := range jlist.Details {
			found := false
			for _, ij := range config.Intents {
				if job.Spec.Name == ij.Spec.Name {
					ij.Count++
					found = true
				}
			}

			if !found {
				config.Intents = append(config.Intents, &pb.Intent{Spec: &pbs.JobSpec{Name: job.Spec.Name}, Count: 1})
			}
		}
	}

	return config
}

// MatchIntent tries to match the intent with the state of production
func (s *Server) MatchIntent() {
	checker := &mainChecker{logger: s.Log}
	for s.serving {
		time.Sleep(intentWait)
		s.LastIntent = time.Now()

		state := getConfig(checker)
		diff := configDiff(s.config, state)
		joblist := runJobs(diff)
		for _, job := range joblist {
			runJob(job, chooseServer(job, checker))
		}
	}
}

// SetMaster sets up the master settings
func (s *Server) SetMaster() {
	t := time.Now()
	checker := &mainChecker{logger: s.Log}
	s.LastMaster = time.Now()
	masterMap := make(map[string]string)

	fleet := checker.discover()
	matcher := make(map[string][]*pbd.RegistryEntry)
	hasMaster := make(map[string]int)
	for _, entry := range fleet.GetServices() {
		if !entry.GetIgnoresMaster() {
			if _, ok := matcher[entry.GetName()]; !ok {
				if entry.GetMaster() {
					hasMaster[entry.GetName()]++
					masterMap[entry.GetName()] = entry.GetIdentifier()
				}
				matcher[entry.GetName()] = []*pbd.RegistryEntry{entry}
			} else {
				if entry.GetMaster() {
					hasMaster[entry.GetName()] = 1
					matcher[entry.GetName()] = append(matcher[entry.GetName()], entry)
					masterMap[entry.GetName()] = entry.GetIdentifier()
				}
			}
		}
	}

	for key, entries := range matcher {
		if hasMaster[key] > 1 {
			hasMaster[key] = 1
			seen := false
			for _, entry := range entries {
				if seen && entry.GetMaster() {
					checker.master(entry, false)
				} else if entry.GetMaster() {
					seen = true
				}
			}
		}

		if hasMaster[key] == 0 {
			for _, entry := range entries {
				if checker.master(entry, true) {
					masterMap[entry.GetName()] = entry.GetIdentifier()
					entry.Master = true
					break
				}
			}
		}
	}
	s.mapString = fmt.Sprintf("%v", masterMap)
	s.LogFunction("SetMasterRun", t)
}

//Init builds up the server
func Init(config *pb.Config) *Server {
	s := &Server{
		&goserver.GoServer{},
		config,
		true,
		time.Now(),
		time.Now(),
		&sync.Mutex{},
		make(map[string]map[string]struct{}),
		&prodGetter{},
		"",
	}
	return s
}

func (s *Server) becomeMaster() {
	for true {
		time.Sleep(time.Second * 5)
		_, _, err := utils.Resolve("gobuildmaster")
		if err != nil {
			s.Registry.Master = true
		}
	}
}

func main() {
	config, err := loadConfig("config.pb")
	if err != nil {
		log.Fatalf("Fatal loading of config: %v", err)
	}

	s := Init(config)

	var quiet = flag.Bool("quiet", false, "Show all output")
	flag.Parse()

	if *quiet {
		log.SetFlags(0)
		log.SetOutput(ioutil.Discard)
	}

	s.Register = s
	s.PrepServer()
	s.GoServer.Killme = false
	s.RegisterServer("gobuildmaster", false)
	s.RegisterServingTask(s.MatchIntent)
	s.RegisterServingTask(s.becomeMaster)
	s.RegisterRepeatingTask(s.SetMaster, time.Second)
	s.RegisterRepeatingTask(s.buildWorld, time.Minute)

	for i := 0; i < len(s.config.GetIntents()); i++ {
		go s.checkerThread(s.config.GetIntents()[i])
	}

	err = s.Serve()
	if err != nil {
		log.Fatalf("Serve error: %v", err)
	}
}
<|endoftext|>


<commit_before>package ccv2_test

import (
	"net/http"
	"time"

	"code.cloudfoundry.org/cli/api/cloudcontroller/ccerror"
	. "code.cloudfoundry.org/cli/api/cloudcontroller/ccv2"
	"code.cloudfoundry.org/cli/api/cloudcontroller/ccv2/constant"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/ghttp"
)

var _ = Describe("Event", func() {
	var client *Client

	BeforeEach(func() {
		client = NewTestClient()
	})

	Describe("GetEvents", func() {
		var (
			events     []Event
			warnings   Warnings
			executeErr error
		)

		JustBeforeEach(func() {
			events, warnings, executeErr = client.GetEvents(
				Filter{
					Type:     constant.TimestampFilter,
					Operator: constant.GreaterThanOperator,
					Values:   []string{"2015-03-10T23:11:54Z"},
				},
				Filter{
					Type:     constant.TypeFilter,
					Operator: constant.InOperator,
					Values:   []string{"audit.app.create", "audit.app.delete-request"},
				},
			)
		})
		Context("when getting events errors", func() {
			BeforeEach(func() {
				response := `{
					"code": 1,
					"description": "some error description",
					"error_code": "CF-SomeError"
				}`
				server.AppendHandlers(
					CombineHandlers(
						VerifyRequest(http.MethodGet, "/v2/events", "q=timestamp>2015-03-10T23:11:54Z&q=type+IN+audit.app.create,audit.app.delete-request"),
						RespondWith(http.StatusTeapot, response, http.Header{"X-Cf-Warnings": {"warning-1, warning-2"}}),
					),
				)
			})

			It("returns the error and all warnings", func() {
				Expect(executeErr).To(MatchError(ccerror.V2UnexpectedResponseError{
					V2ErrorResponse: ccerror.V2ErrorResponse{
						Code:        1,
						Description: "some error description",
						ErrorCode:   "CF-SomeError",
					},
					ResponseCode: http.StatusTeapot,
				}))

				Expect(warnings).To(ConsistOf("warning-1", "warning-2"))
			})
		})

		Context("when getting events succeeds", func() {
			BeforeEach(func() {
				response1 := `{
				"next_url": "/v2/events?q=timestamp>2015-03-10T23:11:54Z&q=type+IN+audit.app.create,audit.app.delete-request&page=2",
				"resources": [
					{
						"metadata": {
							"guid": "some-event-guid-1",
							"updated_at": null
						},
						"entity": {
							"type": "audit.app.create",
							"actor": "some-actor-guid",
							"actor_type": "some-actor-type",
							"actor_name": "some-actor-name",
							"actee": "some-actee-guid",
							"actee_type": "some-actee-type",
							"actee_name": "some-actee-name",
							"timestamp": "2015-03-10T23:11:54Z",
							"metadata": {
								"route_mapping_guid": "some-route-mapping-guid"
							}
						}
					},
					{
						"metadata": {
							"guid": "some-event-guid-2",
							"updated_at": null
						},
						"entity": {
							"type": "audit.app.delete-request",
							"actor": "some-actor-guid",
							"actor_type": "some-actor-type",
							"actor_name": "some-actor-name",
							"actee": "some-actee-guid",
							"actee_type": "some-actee-type",
							"actee_name": "some-actee-name",
							"timestamp": "2015-03-10T23:11:54Z"
						}
					}
				]
			}`
				response2 := `{
				"next_url": null,
				"resources": [
					{
						"metadata": {
							"guid": "some-event-guid-3",
							"updated_at": null
						},
						"entity": {
							"type": "audit.app.create",
							"actor": "some-actor-guid",
							"actor_type": "some-actor-type",
							"actor_name": "some-actor-name",
							"actee": "some-actee-guid",
							"actee_type": "some-actee-type",
							"actee_name": "some-actee-name",
							"timestamp": "2015-03-10T23:11:54Z"
						}
					},
					{
						"metadata": {
							"guid": "some-event-guid-4",
							"updated_at": null
						},
						"entity": {
							"type": "audit.app.delete-request",
							"actor": "some-actor-guid",
							"actor_type": "some-actor-type",
							"actor_name": "some-actor-name",
							"actee": "some-actee-guid",
							"actee_type": "some-actee-type",
							"actee_name": "some-actee-name",
							"timestamp": "2015-03-10T23:11:54Z"
						}
					}
				]
			}`
				server.AppendHandlers(
					CombineHandlers(
						VerifyRequest(http.MethodGet, "/v2/events", "q=timestamp>2015-03-10T23:11:54Z&q=type+IN+audit.app.create,audit.app.delete-request"),
						RespondWith(http.StatusOK, response1, http.Header{"X-Cf-Warnings": {"warning-1, warning-2"}}),
					),
				)
				server.AppendHandlers(
					CombineHandlers(
						VerifyRequest(http.MethodGet, "/v2/events", "q=timestamp>2015-03-10T23:11:54Z&q=type+IN+audit.app.create,audit.app.delete-request&page=2"),
						RespondWith(http.StatusOK, response2, http.Header{"X-Cf-Warnings": {"warning-3, warning-4"}}),
					),
				)
			})

			It("returns all the events", func() {
				expectedTimestamp, err := time.Parse(time.RFC3339, "2015-03-10T23:11:54Z")
				Expect(err).ToNot(HaveOccurred())

				Expect(executeErr).ToNot(HaveOccurred())
				Expect(warnings).To(ConsistOf("warning-1", "warning-2", "warning-3", "warning-4"))
				Expect(events).To(Equal([]Event{
					{
						GUID:      "some-event-guid-1",
						Type:      constant.EventTypeAuditApplicationCreate,
						ActorGUID: "some-actor-guid",
						ActorType: "some-actor-type",
						ActorName: "some-actor-name",
						ActeeGUID: "some-actee-guid",
						ActeeType: "some-actee-type",
						ActeeName: "some-actee-name",
						Timestamp: expectedTimestamp,
						Metadata: map[string]interface{}{
							"route_mapping_guid": "some-route-mapping-guid",
						},
					},
					{
						GUID:      "some-event-guid-2",
						Type:      constant.EventTypeAuditApplicationDelete,
						ActorGUID: "some-actor-guid",
						ActorType: "some-actor-type",
						ActorName: "some-actor-name",
						ActeeGUID: "some-actee-guid",
						ActeeType: "some-actee-type",
						ActeeName: "some-actee-name",
						Timestamp: expectedTimestamp,
						Metadata:  nil,
					},
					{
						GUID:      "some-event-guid-3",
						Type:      constant.EventTypeAuditApplicationCreate,
						ActorGUID: "some-actor-guid",
						ActorType: "some-actor-type",
						ActorName: "some-actor-name",
						ActeeGUID: "some-actee-guid",
						ActeeType: "some-actee-type",
						ActeeName: "some-actee-name",
						Timestamp: expectedTimestamp,
						Metadata:  nil,
					},
					{
						GUID:      "some-event-guid-4",
						Type:      constant.EventTypeAuditApplicationDelete,
						ActorGUID: "some-actor-guid",
						ActorType: "some-actor-type",
						ActorName: "some-actor-name",
						ActeeGUID: "some-actee-guid",
						ActeeType: "some-actee-type",
						ActeeName: "some-actee-name",
						Timestamp: expectedTimestamp,
						Metadata:  nil,
					},
				}))
			})
		})
	})
})
<commit_msg>modify unit test to reflect renaming of events constant<commit_after>package ccv2_test

import (
	"net/http"
	"time"

	"code.cloudfoundry.org/cli/api/cloudcontroller/ccerror"
	. "code.cloudfoundry.org/cli/api/cloudcontroller/ccv2"
	"code.cloudfoundry.org/cli/api/cloudcontroller/ccv2/constant"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/ghttp"
)

var _ = Describe("Event", func() {
	var client *Client

	BeforeEach(func() {
		client = NewTestClient()
	})

	Describe("GetEvents", func() {
		var (
			events     []Event
			warnings   Warnings
			executeErr error
		)

		JustBeforeEach(func() {
			events, warnings, executeErr = client.GetEvents(
				Filter{
					Type:     constant.TimestampFilter,
					Operator: constant.GreaterThanOperator,
					Values:   []string{"2015-03-10T23:11:54Z"},
				},
				Filter{
					Type:     constant.TypeFilter,
					Operator: constant.InOperator,
					Values:   []string{"audit.app.create", "audit.app.delete-request"},
				},
			)
		})
		Context("when getting events errors", func() {
			BeforeEach(func() {
				response := `{
					"code": 1,
					"description": "some error description",
					"error_code": "CF-SomeError"
				}`
				server.AppendHandlers(
					CombineHandlers(
						VerifyRequest(http.MethodGet, "/v2/events", "q=timestamp>2015-03-10T23:11:54Z&q=type+IN+audit.app.create,audit.app.delete-request"),
						RespondWith(http.StatusTeapot, response, http.Header{"X-Cf-Warnings": {"warning-1, warning-2"}}),
					),
				)
			})

			It("returns the error and all warnings", func() {
				Expect(executeErr).To(MatchError(ccerror.V2UnexpectedResponseError{
					V2ErrorResponse: ccerror.V2ErrorResponse{
						Code:        1,
						Description: "some error description",
						ErrorCode:   "CF-SomeError",
					},
					ResponseCode: http.StatusTeapot,
				}))

				Expect(warnings).To(ConsistOf("warning-1", "warning-2"))
			})
		})

		Context("when getting events succeeds", func() {
			BeforeEach(func() {
				response1 := `{
				"next_url": "/v2/events?q=timestamp>2015-03-10T23:11:54Z&q=type+IN+audit.app.create,audit.app.delete-request&page=2",
				"resources": [
					{
						"metadata": {
							"guid": "some-event-guid-1",
							"updated_at": null
						},
						"entity": {
							"type": "audit.app.create",
							"actor": "some-actor-guid",
							"actor_type": "some-actor-type",
							"actor_name": "some-actor-name",
							"actee": "some-actee-guid",
							"actee_type": "some-actee-type",
							"actee_name": "some-actee-name",
							"timestamp": "2015-03-10T23:11:54Z",
							"metadata": {
								"route_mapping_guid": "some-route-mapping-guid"
							}
						}
					},
					{
						"metadata": {
							"guid": "some-event-guid-2",
							"updated_at": null
						},
						"entity": {
							"type": "audit.app.delete-request",
							"actor": "some-actor-guid",
							"actor_type": "some-actor-type",
							"actor_name": "some-actor-name",
							"actee": "some-actee-guid",
							"actee_type": "some-actee-type",
							"actee_name": "some-actee-name",
							"timestamp": "2015-03-10T23:11:54Z"
						}
					}
				]
			}`
				response2 := `{
				"next_url": null,
				"resources": [
					{
						"metadata": {
							"guid": "some-event-guid-3",
							"updated_at": null
						},
						"entity": {
							"type": "audit.app.create",
							"actor": "some-actor-guid",
							"actor_type": "some-actor-type",
							"actor_name": "some-actor-name",
							"actee": "some-actee-guid",
							"actee_type": "some-actee-type",
							"actee_name": "some-actee-name",
							"timestamp": "2015-03-10T23:11:54Z"
						}
					},
					{
						"metadata": {
							"guid": "some-event-guid-4",
							"updated_at": null
						},
						"entity": {
							"type": "audit.app.delete-request",
							"actor": "some-actor-guid",
							"actor_type": "some-actor-type",
							"actor_name": "some-actor-name",
							"actee": "some-actee-guid",
							"actee_type": "some-actee-type",
							"actee_name": "some-actee-name",
							"timestamp": "2015-03-10T23:11:54Z"
						}
					}
				]
			}`
				server.AppendHandlers(
					CombineHandlers(
						VerifyRequest(http.MethodGet, "/v2/events", "q=timestamp>2015-03-10T23:11:54Z&q=type+IN+audit.app.create,audit.app.delete-request"),
						RespondWith(http.StatusOK, response1, http.Header{"X-Cf-Warnings": {"warning-1, warning-2"}}),
					),
				)
				server.AppendHandlers(
					CombineHandlers(
						VerifyRequest(http.MethodGet, "/v2/events", "q=timestamp>2015-03-10T23:11:54Z&q=type+IN+audit.app.create,audit.app.delete-request&page=2"),
						RespondWith(http.StatusOK, response2, http.Header{"X-Cf-Warnings": {"warning-3, warning-4"}}),
					),
				)
			})

			It("returns all the events", func() {
				expectedTimestamp, err := time.Parse(time.RFC3339, "2015-03-10T23:11:54Z")
				Expect(err).ToNot(HaveOccurred())

				Expect(executeErr).ToNot(HaveOccurred())
				Expect(warnings).To(ConsistOf("warning-1", "warning-2", "warning-3", "warning-4"))
				Expect(events).To(Equal([]Event{
					{
						GUID:      "some-event-guid-1",
						Type:      constant.EventTypeAuditApplicationCreate,
						ActorGUID: "some-actor-guid",
						ActorType: "some-actor-type",
						ActorName: "some-actor-name",
						ActeeGUID: "some-actee-guid",
						ActeeType: "some-actee-type",
						ActeeName: "some-actee-name",
						Timestamp: expectedTimestamp,
						Metadata: map[string]interface{}{
							"route_mapping_guid": "some-route-mapping-guid",
						},
					},
					{
						GUID:      "some-event-guid-2",
						Type:      constant.EventTypeAuditApplicationDeleteRequest,
						ActorGUID: "some-actor-guid",
						ActorType: "some-actor-type",
						ActorName: "some-actor-name",
						ActeeGUID: "some-actee-guid",
						ActeeType: "some-actee-type",
						ActeeName: "some-actee-name",
						Timestamp: expectedTimestamp,
						Metadata:  nil,
					},
					{
						GUID:      "some-event-guid-3",
						Type:      constant.EventTypeAuditApplicationCreate,
						ActorGUID: "some-actor-guid",
						ActorType: "some-actor-type",
						ActorName: "some-actor-name",
						ActeeGUID: "some-actee-guid",
						ActeeType: "some-actee-type",
						ActeeName: "some-actee-name",
						Timestamp: expectedTimestamp,
						Metadata:  nil,
					},
					{
						GUID:      "some-event-guid-4",
						Type:      constant.EventTypeAuditApplicationDeleteRequest,
						ActorGUID: "some-actor-guid",
						ActorType: "some-actor-type",
						ActorName: "some-actor-name",
						ActeeGUID: "some-actee-guid",
						ActeeType: "some-actee-type",
						ActeeName: "some-actee-name",
						Timestamp: expectedTimestamp,
						Metadata:  nil,
					},
				}))
			})
		})
	})
})
<|endoftext|>


<commit_before>// Copyright © 2015 Clement 'cmc' Rey <cr.rey.clement@gmail.com>.
//
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

package seq

import "sort"

// -----------------------------------------------------------------------------

// ID is a 64bits wide unsigned identifier.
type ID uint64

// IDSlice implements the `sort.Interface` for slices of `ID`s.`
//
// It is mainly used for testing purposes.
type IDSlice []ID

func (ids IDSlice) Len() int           { return len(ids) }
func (ids IDSlice) Less(i, j int) bool { return ids[i] < ids[j] }
func (ids IDSlice) Swap(i, j int)      { ids[i], ids[j] = ids[j], ids[i] }
func (ids IDSlice) Sort() IDSlice      { sort.Sort(ids); return ids }

// IDStream is a read-only channel of `ID`s.
//
// An `IDStream` should always starts at `ID(1)`.
type IDStream <-chan ID

// Next is a simple helper to get the next `ID` from an `IDStream`.
//
// It can be helpful if you cannot simply `range` on the stream for
// some reason.
//
// `Next` blocks if no `ID` is available, and returns `ID(0)` if the stream is
// already closed.
func (ids IDStream) Next() ID {
	select {
	case id := <-ids:
		return id
	}
}

// -----------------------------------------------------------------------------

// A Sequencer exposes methods to retrieve streams of sequential `ID`s.
type Sequencer interface {
	// Stream returns the `IDStream` associated with this `Sequencer`.
	Stream() IDStream
	// Close stops this `Sequencer` and cleans the associated resources.
	Close() error
}
<commit_msg>seq - package level doc<commit_after>// Copyright © 2015 Clement 'cmc' Rey <cr.rey.clement@gmail.com>.
//
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

// Package seq offers a gentle overview of the possible design solutions to the
// common problem of generating sequential / monotonically increasing IDs in a
// distributed system.
//
// Specifically, it focuses on maximizing performances and guaranteeing a fair
// distribution of the workload between the nodes as the size of the cluster
// increases.
package seq

import "sort"

// -----------------------------------------------------------------------------

// ID is a 64bits wide unsigned identifier.
type ID uint64

// IDSlice implements the `sort.Interface` for slices of `ID`s.`
//
// It is mainly used for testing purposes.
type IDSlice []ID

func (ids IDSlice) Len() int           { return len(ids) }
func (ids IDSlice) Less(i, j int) bool { return ids[i] < ids[j] }
func (ids IDSlice) Swap(i, j int)      { ids[i], ids[j] = ids[j], ids[i] }
func (ids IDSlice) Sort() IDSlice      { sort.Sort(ids); return ids }

// IDStream is a read-only channel of `ID`s.
//
// An `IDStream` should always starts at `ID(1)`.
type IDStream <-chan ID

// Next is a simple helper to get the next `ID` from an `IDStream`.
//
// It can be helpful if you cannot simply `range` on the stream for
// some reason.
//
// `Next` blocks if no `ID` is available, and returns `ID(0)` if the stream is
// already closed.
func (ids IDStream) Next() ID {
	select {
	case id := <-ids:
		return id
	}
}

// -----------------------------------------------------------------------------

// A Sequencer exposes methods to retrieve streams of sequential `ID`s.
type Sequencer interface {
	// Stream returns the `IDStream` associated with this `Sequencer`.
	Stream() IDStream
	// Close stops this `Sequencer` and cleans the associated resources.
	Close() error
}
<|endoftext|>


<commit_before>//go:generate msgp
//msgp:ignore LinkResultMap
package db

import (
	"encoding/csv"
	"encoding/json"
	"fmt"
	"io"
	"sync"
	"time"

	turtle "github.com/gtfierro/hod/goraptor"
	"github.com/gtfierro/hod/query"

	"github.com/mitghi/btree"
)

var emptyResultMapList = []ResultMap{}
var emptyLinkResultmapList = []LinkResultMap{}

type QueryResult struct {
	selectVars []query.SelectVar
	Rows       []ResultMap
	Links      []LinkResultMap `msg:"-"`
	Count      int
	Elapsed    time.Duration `msg:"-"`
}

func newQueryResult() QueryResult {
	return QueryResult{
		Rows:  emptyResultMapList,
		Links: emptyLinkResultmapList,
	}
}

func (qr QueryResult) Dump() {
	if len(qr.Rows) > 0 {
		for _, row := range qr.Rows {
			fmt.Println(row)
		}
		return
	}
	if len(qr.Links) > 0 {
		for _, link := range qr.Links {
			fmt.Println(link)
		}
		return
	}
	fmt.Println(qr.Count)
}

func (qr QueryResult) DumpToCSV(usePrefixes bool, db *DB, w io.Writer) error {
	csvwriter := csv.NewWriter(w)
	if len(qr.Rows) > 0 {
		for _, row := range qr.Rows {
			var line = make([]string, len(qr.selectVars))
			for idx, varname := range qr.selectVars {
				if usePrefixes {
					line[idx] = db.Abbreviate(row[varname.Var.Value])
				} else {
					line[idx] = row[varname.Var.Value].String()
				}
			}
			if err := csvwriter.Write(line); err != nil {
				return err
			}
			csvwriter.Flush()
			if err := csvwriter.Error(); err != nil {
				return err
			}
		}
		return nil
	}
	if len(qr.Links) > 0 {
		for _, link := range qr.Links {
			fmt.Println(link)
		}
		return nil
	}
	return nil
}

type ResultMap map[string]turtle.URI
type LinkResultMap map[turtle.URI]map[string]string

func (m LinkResultMap) MarshalJSON() ([]byte, error) {
	var n = make(map[string]map[string]string)
	for k, v := range m {
		n[k.String()] = v
	}
	return json.Marshal(n)
}

type ResultRow struct {
	row   []turtle.URI
	count int
}

func (rr ResultRow) Less(than btree.Item, ctx interface{}) bool {
	row := than.(*ResultRow)
	if rr.count < row.count {
		return true
	} else if row.count < rr.count {
		return false
	}
	before := false
	for idx, item := range rr.row[:rr.count] {
		before = before || item.Value < row.row[idx].Value || item.Namespace < row.row[idx].Namespace
	}
	return before
}

var _emptyResultRow = make([]turtle.URI, 16)
var _RESULTROWPOOL = sync.Pool{
	New: func() interface{} {
		return &ResultRow{
			row:   make([]turtle.URI, 16),
			count: 0,
		}
	},
}

func getResultRow(num int) *ResultRow {
	r := _RESULTROWPOOL.Get().(*ResultRow)
	r.count = num
	return r
}

func finishResultRow(r *ResultRow) {
	r.count = 0
	_RESULTROWPOOL.Put(r)
}

func cleanResultRows(b *btree.BTree) {
	i := b.DeleteMax()
	for i != nil {
		row := i.(*ResultRow)
		finishResultRow(row)
		i = b.DeleteMax()
	}
}
<commit_msg>remove Links from result reporting<commit_after>//go:generate msgp
//msgp:ignore LinkResultMap
package db

import (
	"encoding/csv"
	"encoding/json"
	"fmt"
	"io"
	"sync"
	"time"

	turtle "github.com/gtfierro/hod/goraptor"
	"github.com/gtfierro/hod/query"

	"github.com/mitghi/btree"
)

var emptyResultMapList = []ResultMap{}
var emptyLinkResultmapList = []LinkResultMap{}

type QueryResult struct {
	selectVars []query.SelectVar
	Rows       []ResultMap
	Count      int
	Elapsed    time.Duration `msg:"-"`
}

func newQueryResult() QueryResult {
	return QueryResult{
		Rows: emptyResultMapList,
	}
}

func (qr QueryResult) Dump() {
	if len(qr.Rows) > 0 {
		for _, row := range qr.Rows {
			fmt.Println(row)
		}
		return
	}
	fmt.Println(qr.Count)
}

func (qr QueryResult) DumpToCSV(usePrefixes bool, db *DB, w io.Writer) error {
	csvwriter := csv.NewWriter(w)
	if len(qr.Rows) > 0 {
		for _, row := range qr.Rows {
			var line = make([]string, len(qr.selectVars))
			for idx, varname := range qr.selectVars {
				if usePrefixes {
					line[idx] = db.Abbreviate(row[varname.Var.Value])
				} else {
					line[idx] = row[varname.Var.Value].String()
				}
			}
			if err := csvwriter.Write(line); err != nil {
				return err
			}
			csvwriter.Flush()
			if err := csvwriter.Error(); err != nil {
				return err
			}
		}
		return nil
	}
	return nil
}

type ResultMap map[string]turtle.URI
type LinkResultMap map[turtle.URI]map[string]string

func (m LinkResultMap) MarshalJSON() ([]byte, error) {
	var n = make(map[string]map[string]string)
	for k, v := range m {
		n[k.String()] = v
	}
	return json.Marshal(n)
}

type ResultRow struct {
	row   []turtle.URI
	count int
}

func (rr ResultRow) Less(than btree.Item, ctx interface{}) bool {
	row := than.(*ResultRow)
	if rr.count < row.count {
		return true
	} else if row.count < rr.count {
		return false
	}
	before := false
	for idx, item := range rr.row[:rr.count] {
		before = before || item.Value < row.row[idx].Value || item.Namespace < row.row[idx].Namespace
	}
	return before
}

var _emptyResultRow = make([]turtle.URI, 16)
var _RESULTROWPOOL = sync.Pool{
	New: func() interface{} {
		return &ResultRow{
			row:   make([]turtle.URI, 16),
			count: 0,
		}
	},
}

func getResultRow(num int) *ResultRow {
	r := _RESULTROWPOOL.Get().(*ResultRow)
	r.count = num
	return r
}

func finishResultRow(r *ResultRow) {
	r.count = 0
	_RESULTROWPOOL.Put(r)
}

func cleanResultRows(b *btree.BTree) {
	i := b.DeleteMax()
	for i != nil {
		row := i.(*ResultRow)
		finishResultRow(row)
		i = b.DeleteMax()
	}
}
<|endoftext|>


<commit_before>package dnode

import (
	"fmt"
	"reflect"
	"strconv"
	"strings"
)

func (s *Scrubber) Unscrub(arguments interface{}, callbacks map[string]Path, f func(uint64) functionReceived) error {
	v := reflect.ValueOf(arguments)
	if v.Kind() != reflect.Ptr {
		panic("arguments must be a pointer")
	}

	// Parse callbacks field and create callback functions.
	for sid, path := range callbacks {
		id, err := strconv.ParseUint(sid, 10, 64)
		if err != nil {
			return err
		}

		if err = setCallback(v, path, f(id)); err != nil {
			return err
		}
	}

	return nil
}

func setCallback(value reflect.Value, path Path, cb functionReceived) error {
	i := 0
	for {
		switch value.Kind() {
		case reflect.Slice:
			if i == len(path) {
				return fmt.Errorf("callback path too short: %v", path)
			}

			// Path component may be a string or an integer.
			var index int
			var err error
			switch v := path[i].(type) {
			case string:
				index, err = strconv.Atoi(v)
				if err != nil {
					return fmt.Errorf("integer expected in callback path, got '%v'.", path[i])
				}
			case float64:
				index = int(v)
			default:
				panic(fmt.Errorf("unknown type: %#v", path[i]))
			}

			value = value.Index(index)
			i++
		case reflect.Map:
			if i == len(path) {
				return fmt.Errorf("callback path too short: %v", path)
			}
			if i == len(path)-1 && value.Type().Elem().Kind() == reflect.Interface {
				value.SetMapIndex(reflect.ValueOf(path[i]), reflect.ValueOf(cb))
				return nil
			}
			value = value.MapIndex(reflect.ValueOf(path[i]))
			i++
		case reflect.Ptr:
			value = value.Elem()
		case reflect.Interface:
			if i == len(path) {
				value.Set(reflect.ValueOf(cb))
				return nil
			}
			value = value.Elem()
		case reflect.Struct:
			if value.Type().Implements(callerType) {
				caller := value.FieldByName("Caller")
				caller.Set(reflect.ValueOf(cb))
				return nil
			}

			if innerPartial, ok := value.Addr().Interface().(*Partial); ok {
				spec := CallbackSpec{path[i:], Function{cb}}
				innerPartial.CallbackSpecs = append(innerPartial.CallbackSpecs, spec)
				return nil
			}

			// Path component may be a string or an integer.
			name, ok := path[i].(string)
			if !ok {
				return fmt.Errorf("Invalid path: %#v", path[i])
			}

			value = value.FieldByName(strings.ToUpper(name[0:1]) + name[1:])
			i++
		case reflect.Func:
			// plain func is not supported, use Function type
			return nil
		case reflect.Invalid:
			// callback path does not exist, skip
			return nil
		default:
			return fmt.Errorf("Unhandled value of kind '%v' in callback path: %s", value.Kind(), value.Interface())
		}
	}
	return nil
}
<commit_msg>Revert "check for interface instead of concrete type"<commit_after>package dnode

import (
	"fmt"
	"reflect"
	"strconv"
	"strings"
)

func (s *Scrubber) Unscrub(arguments interface{}, callbacks map[string]Path, f func(uint64) functionReceived) error {
	v := reflect.ValueOf(arguments)
	if v.Kind() != reflect.Ptr {
		panic("arguments must be a pointer")
	}

	// Parse callbacks field and create callback functions.
	for sid, path := range callbacks {
		id, err := strconv.ParseUint(sid, 10, 64)
		if err != nil {
			return err
		}

		if err = setCallback(v, path, f(id)); err != nil {
			return err
		}
	}

	return nil
}

func setCallback(value reflect.Value, path Path, cb functionReceived) error {
	i := 0
	for {
		switch value.Kind() {
		case reflect.Slice:
			if i == len(path) {
				return fmt.Errorf("callback path too short: %v", path)
			}

			// Path component may be a string or an integer.
			var index int
			var err error
			switch v := path[i].(type) {
			case string:
				index, err = strconv.Atoi(v)
				if err != nil {
					return fmt.Errorf("integer expected in callback path, got '%v'.", path[i])
				}
			case float64:
				index = int(v)
			default:
				panic(fmt.Errorf("unknown type: %#v", path[i]))
			}

			value = value.Index(index)
			i++
		case reflect.Map:
			if i == len(path) {
				return fmt.Errorf("callback path too short: %v", path)
			}
			if i == len(path)-1 && value.Type().Elem().Kind() == reflect.Interface {
				value.SetMapIndex(reflect.ValueOf(path[i]), reflect.ValueOf(cb))
				return nil
			}
			value = value.MapIndex(reflect.ValueOf(path[i]))
			i++
		case reflect.Ptr:
			value = value.Elem()
		case reflect.Interface:
			if i == len(path) {
				value.Set(reflect.ValueOf(cb))
				return nil
			}
			value = value.Elem()
		case reflect.Struct:
			if value.Type() == reflect.TypeOf(Function{}) {
				caller := value.FieldByName("Caller")
				caller.Set(reflect.ValueOf(cb))
				return nil
			}

			if innerPartial, ok := value.Addr().Interface().(*Partial); ok {
				spec := CallbackSpec{path[i:], Function{cb}}
				innerPartial.CallbackSpecs = append(innerPartial.CallbackSpecs, spec)
				return nil
			}

			// Path component may be a string or an integer.
			name, ok := path[i].(string)
			if !ok {
				return fmt.Errorf("Invalid path: %#v", path[i])
			}

			value = value.FieldByName(strings.ToUpper(name[0:1]) + name[1:])
			i++
		case reflect.Func:
			// plain func is not supported, use Function type
			return nil
		case reflect.Invalid:
			// callback path does not exist, skip
			return nil
		default:
			return fmt.Errorf("Unhandled value of kind '%v' in callback path: %s", value.Kind(), value.Interface())
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package dbmigrate

import (
	"database/sql"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/gocql/gocql"
)

// Database interface needs to be inmplemented to migrate a new type of database

type Database interface {
	CreateMigrationsTable() error
	HasMigrated(filename string) (bool, error)
	Migrate(filename string, migration string) error
}

// CassandraDatabase migrates Cassandra databases

type CassandraDatabase struct {
	readerSession *gocql.Session
	writerSession *gocql.Session
}

func (cassandra *CassandraDatabase) CreateMigrationsTable() error {
	err := cassandra.writerSession.Query(`
		create table migrations (
			name text,
			created_at timeuuid,
			primary key (name)
		);
	`).Exec()
	return err
}

func (cassandra *CassandraDatabase) HasMigrated(filename string) (bool, error) {
	var count int
	/*
		err := postgres.database.QueryRow("select count(1) from migrations where name = $1", filename).Scan(&count)
		if err != nil {
			return false, err
		}
	*/
	return count > 0, nil
}

func (cassandra *CassandraDatabase) Migrate(filename string, migration string) error {
	/*
		_, err := postgres.database.Exec(migration)
		if err != nil {
			return err
		}
		_, err = postgres.database.Exec("insert into migrations(name, created_at) values($1, current_timestamp)", filename)
		return err
	*/
	return nil
}

func NewCassandraDatabase(readerSession *gocql.Session, writerSession *gocql.Session) *CassandraDatabase {
	return &CassandraDatabase{
		readerSession: readerSession,
		writerSession: writerSession,
	}
}

// PostgresDatabase migrates Postgresql databases

type PostgresDatabase struct {
	database *sql.DB
}

func (postgres *PostgresDatabase) CreateMigrationsTable() error {
	_, err := postgres.database.Exec(`
		create table if not exists migrations(id serial, name text not null, created_at timestamp with time zone not null)
	`)
	if err != nil {
		return err
	}
	_, err = postgres.database.Exec("create unique index idx_migrations_name on migrations(name)")
	if err != nil {
		if !strings.Contains(err.Error(), "already exists") {
			return err
		}
	}
	return nil
}

func (postgres *PostgresDatabase) HasMigrated(filename string) (bool, error) {
	var count int
	err := postgres.database.QueryRow("select count(1) from migrations where name = $1", filename).Scan(&count)
	if err != nil {
		return false, err
	}
	return count > 0, nil
}

func (postgres *PostgresDatabase) Migrate(filename string, migration string) error {
	_, err := postgres.database.Exec(migration)
	if err != nil {
		return err
	}
	_, err = postgres.database.Exec("insert into migrations(name, created_at) values($1, current_timestamp)", filename)
	return err
}

func NewPostgresDatabase(db *sql.DB) *PostgresDatabase {
	return &PostgresDatabase{database: db}
}

// By default, apply Postgresql migrations, as in older versions
func Run(db *sql.DB, migrationsFolder string) error {
	postgres := NewPostgresDatabase(db)
	return ApplyMigrations(postgres, migrationsFolder)
}

// Run applies migrations from migrationsFolder to database.
func ApplyMigrations(database Database, migrationsFolder string) error {
	// Initialize migrations table, if it does not exist yet
	if err := database.CreateMigrationsTable(); err != nil {
		return err
	}

	// Scan migration file names in migrations folder
	d, err := os.Open(migrationsFolder)
	if err != nil {
		return err
	}
	dir, err := d.Readdir(-1)
	if err != nil {
		return err
	}

	// Run migrations
	sqlFiles := make([]string, 0)
	for _, f := range dir {
		if filepath.Ext(f.Name()) == ".sql" {
			sqlFiles = append(sqlFiles, f.Name())
		}
	}
	sort.Strings(sqlFiles)
	for _, filename := range sqlFiles {
		// if exists in migrations table, leave it
		// else execute sql
		migrated, err := database.HasMigrated(filename)
		if err != nil {
			return err
		}
		if migrated {
			continue
		}
		b, err := ioutil.ReadFile(filepath.Join(migrationsFolder, filename))
		if err != nil {
			return err
		}
		migration := string(b)
		if len(migration) == 0 {
			continue // empty file
		}
		err = database.Migrate(filename, migration)
		if err != nil {
			return err
		}
		fmt.Println("Migrated", filename)
	}

	return nil
}
<commit_msg>allow cql files<commit_after>package dbmigrate

import (
	"database/sql"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/gocql/gocql"
)

// Database interface needs to be inmplemented to migrate a new type of database

type Database interface {
	CreateMigrationsTable() error
	HasMigrated(filename string) (bool, error)
	Migrate(filename string, migration string) error
}

// CassandraDatabase migrates Cassandra databases

type CassandraDatabase struct {
	readerSession *gocql.Session
	writerSession *gocql.Session
}

func (cassandra *CassandraDatabase) CreateMigrationsTable() error {
	err := cassandra.writerSession.Query(`
		create table migrations (
			name text,
			created_at timeuuid,
			primary key (name)
		);
	`).Exec()
	return err
}

func (cassandra *CassandraDatabase) HasMigrated(filename string) (bool, error) {
	var count int
	/*
		err := postgres.database.QueryRow("select count(1) from migrations where name = $1", filename).Scan(&count)
		if err != nil {
			return false, err
		}
	*/
	return count > 0, nil
}

func (cassandra *CassandraDatabase) Migrate(filename string, migration string) error {
	/*
		_, err := postgres.database.Exec(migration)
		if err != nil {
			return err
		}
		_, err = postgres.database.Exec("insert into migrations(name, created_at) values($1, current_timestamp)", filename)
		return err
	*/
	return nil
}

func NewCassandraDatabase(readerSession *gocql.Session, writerSession *gocql.Session) *CassandraDatabase {
	return &CassandraDatabase{
		readerSession: readerSession,
		writerSession: writerSession,
	}
}

// PostgresDatabase migrates Postgresql databases

type PostgresDatabase struct {
	database *sql.DB
}

func (postgres *PostgresDatabase) CreateMigrationsTable() error {
	_, err := postgres.database.Exec(`
		create table if not exists migrations(id serial, name text not null, created_at timestamp with time zone not null)
	`)
	if err != nil {
		return err
	}
	_, err = postgres.database.Exec("create unique index idx_migrations_name on migrations(name)")
	if err != nil {
		if !strings.Contains(err.Error(), "already exists") {
			return err
		}
	}
	return nil
}

func (postgres *PostgresDatabase) HasMigrated(filename string) (bool, error) {
	var count int
	err := postgres.database.QueryRow("select count(1) from migrations where name = $1", filename).Scan(&count)
	if err != nil {
		return false, err
	}
	return count > 0, nil
}

func (postgres *PostgresDatabase) Migrate(filename string, migration string) error {
	_, err := postgres.database.Exec(migration)
	if err != nil {
		return err
	}
	_, err = postgres.database.Exec("insert into migrations(name, created_at) values($1, current_timestamp)", filename)
	return err
}

func NewPostgresDatabase(db *sql.DB) *PostgresDatabase {
	return &PostgresDatabase{database: db}
}

// By default, apply Postgresql migrations, as in older versions
func Run(db *sql.DB, migrationsFolder string) error {
	postgres := NewPostgresDatabase(db)
	return ApplyMigrations(postgres, migrationsFolder)
}

// Run applies migrations from migrationsFolder to database.
func ApplyMigrations(database Database, migrationsFolder string) error {
	// Initialize migrations table, if it does not exist yet
	if err := database.CreateMigrationsTable(); err != nil {
		return err
	}

	// Scan migration file names in migrations folder
	d, err := os.Open(migrationsFolder)
	if err != nil {
		return err
	}
	dir, err := d.Readdir(-1)
	if err != nil {
		return err
	}

	// Run migrations
	sqlFiles := make([]string, 0)
	for _, f := range dir {
		ext := filepath.Ext(f.Name())
		if ".sql" == ext || ".cql" == ext {
			sqlFiles = append(sqlFiles, f.Name())
		}
	}
	sort.Strings(sqlFiles)
	for _, filename := range sqlFiles {
		// if exists in migrations table, leave it
		// else execute sql
		migrated, err := database.HasMigrated(filename)
		if err != nil {
			return err
		}
		if migrated {
			continue
		}
		b, err := ioutil.ReadFile(filepath.Join(migrationsFolder, filename))
		if err != nil {
			return err
		}
		migration := string(b)
		if len(migration) == 0 {
			continue // empty file
		}
		err = database.Migrate(filename, migration)
		if err != nil {
			return err
		}
		fmt.Println("Migrated", filename)
	}

	return nil
}
<|endoftext|>


<commit_before>

// Copyright 2012 Daniel Connelly.  All rights reserved.  Use of
// this source code is governed by a BSD-style license that can be
// found in the `LICENSE` file.

package litebrite

import (
	"bytes"
	"fmt"
	"go/scanner"
	"go/token"
	"html/template"
	"strings"
)

// ## Source tokenizing

// sep represents a split in source code due to a token occurrence.
type sep struct {
	pos int
	tok token.Token
}

// tokenize scans src and emits a sep on every token occurrence.
func tokenize(src string, tc chan *sep) {
	var s scanner.Scanner
	fset := token.NewFileSet() // boilerplate stuff for scanner...
	file := fset.AddFile("", fset.Base(), len(src))
	s.Init(file, []byte(src), nil, 1)
	for {
		filePos, tok, _ := s.Scan()
		tc <- &sep{int(filePos) - file.Base(), tok}
		if tok == token.EOF {
			close(tc)
			break
		}
	}
}

// tokens returns a channel that emits seps from tokenizing src.
func tokens(src string) <-chan *sep {
	tc := make(chan *sep)
	go tokenize(src, tc)
	return tc
}

// trim splits a source chunk into three pieces: the leading whitespace, the
// source code, and the trailing whitespace.
func trim(chunk string) (string, string, string) {
	code := strings.TrimSpace(chunk)
	wsl := chunk[:strings.Index(chunk, code)]
	wsr := chunk[len(wsl)+len(code):]
	return wsl, code, wsr
}

// ## HTML templating

type div struct {
	Code  string
	Class string
}

const ELEM = `{{with .Class}}<div class="{{.}}">{{end}}{{.Code}}{{with .Class}}</div>{{end}}`

var elemT = template.Must(template.New("golang-elem").Parse(ELEM))

// ## Highlighting

// Highlighter contains the CSS class names that are applied to the
// corresponding source code token types and the desired width of tabs.
type Highlighter struct {
	OperatorClass string
	IdentClass    string
	LiteralClass  string
	KeywordClass  string
	CommentClass  string
}

// getClass returns the CSS class name associated with tok.
func (h *Highlighter) getClass(tok token.Token) string {
	switch {
	case tok.IsKeyword():
		return h.KeywordClass
	case tok.IsLiteral():
		if tok == token.IDENT {
			return h.IdentClass
		} else {
			return h.LiteralClass
		}
	case tok.IsOperator():
		return h.OperatorClass
	case tok == token.COMMENT:
		return h.CommentClass
	case tok == token.ILLEGAL:
		break
	default:
		panic(fmt.Sprintf("unknown token type: %v", tok))
	}
	return ""
}

// Highlight returns an HTML fragment containing elements for all Go tokens in
// src.  The elements will be of the form `<div class="TYPE_CLASS">CODE</div>`
// where `TYPE_CLASS` is the CSS class name provided in `h` corresponding to
// the token type of `CODE`.  For instance, if `CODE` is a keyword, then
// `TYPE_CLASS` will be `h.keywordClass`.
func (h *Highlighter) Highlight(src string) string {
	var b bytes.Buffer
	tc := tokens(src)
	prev := <-tc
	for cur := <-tc; cur != nil; prev, cur = cur, <-tc {
		wsl, code, wsr := trim(src[prev.pos:cur.pos])
		b.WriteString(wsl)
		elemT.Execute(&b, &div{code, h.getClass(prev.tok)})
		b.WriteString(wsr)
	}
	return b.String()
}
<commit_msg>added docstring<commit_after>// litebrite is a library for generating syntax-highlighted HTML from your
// Go source code.  Use is simple: create a Highlighter, add some CSS styles,
// and pass in some source.
//
// `h := new(litebrite.Highlighter)`

// `h.CommentClass = "commentz"`

// `h.OperatorClass = "opz"`

// `html := h.Highlight(myCodez)`
//
// This will return a string of HTML where all comments are surrounded by
// `<div class="commentz">` tags and all operators are surrounded by
// `<div class="opz">` tags.
//
// The following fields are exposed on Highlighter for customizing the
// div tags: CommentClass, OperatorClass, IdentClass, LiteralClass, and
// KeywordClass.

// Copyright 2012 Daniel Connelly.  All rights reserved.  Use of
// this source code is governed by a BSD-style license that can be
// found in the `LICENSE` file.

package litebrite

import (
	"bytes"
	"fmt"
	"go/scanner"
	"go/token"
	"html/template"
	"strings"
)

// ## Source tokenizing

// sep represents a split in source code due to a token occurrence.
type sep struct {
	pos int
	tok token.Token
}

// tokenize scans src and emits a sep on every token occurrence.
func tokenize(src string, tc chan *sep) {
	var s scanner.Scanner
	fset := token.NewFileSet() // boilerplate stuff for scanner...
	file := fset.AddFile("", fset.Base(), len(src))
	s.Init(file, []byte(src), nil, 1)
	for {
		filePos, tok, _ := s.Scan()
		tc <- &sep{int(filePos) - file.Base(), tok}
		if tok == token.EOF {
			close(tc)
			break
		}
	}
}

// tokens returns a channel that emits seps from tokenizing src.
func tokens(src string) <-chan *sep {
	tc := make(chan *sep)
	go tokenize(src, tc)
	return tc
}

// trim splits a source chunk into three pieces: the leading whitespace, the
// source code, and the trailing whitespace.
func trim(chunk string) (string, string, string) {
	code := strings.TrimSpace(chunk)
	wsl := chunk[:strings.Index(chunk, code)]
	wsr := chunk[len(wsl)+len(code):]
	return wsl, code, wsr
}

// ## HTML templating

type div struct {
	Code  string
	Class string
}

const ELEM = `{{with .Class}}<div class="{{.}}">{{end}}{{.Code}}{{with .Class}}</div>{{end}}`

var elemT = template.Must(template.New("golang-elem").Parse(ELEM))

// ## Highlighting

// Highlighter contains the CSS class names that are applied to the
// corresponding source code token types and the desired width of tabs.
type Highlighter struct {
	OperatorClass string
	IdentClass    string
	LiteralClass  string
	KeywordClass  string
	CommentClass  string
}

// getClass returns the CSS class name associated with tok.
func (h *Highlighter) getClass(tok token.Token) string {
	switch {
	case tok.IsKeyword():
		return h.KeywordClass
	case tok.IsLiteral():
		if tok == token.IDENT {
			return h.IdentClass
		} else {
			return h.LiteralClass
		}
	case tok.IsOperator():
		return h.OperatorClass
	case tok == token.COMMENT:
		return h.CommentClass
	case tok == token.ILLEGAL:
		break
	default:
		panic(fmt.Sprintf("unknown token type: %v", tok))
	}
	return ""
}

// Highlight returns an HTML fragment containing elements for all Go tokens in
// src.  The elements will be of the form `<div class="TYPE_CLASS">CODE</div>`
// where `TYPE_CLASS` is the CSS class name provided in `h` corresponding to
// the token type of `CODE`.  For instance, if `CODE` is a keyword, then
// `TYPE_CLASS` will be `h.keywordClass`.
func (h *Highlighter) Highlight(src string) string {
	var b bytes.Buffer
	tc := tokens(src)
	prev := <-tc
	for cur := <-tc; cur != nil; prev, cur = cur, <-tc {
		wsl, code, wsr := trim(src[prev.pos:cur.pos])
		b.WriteString(wsl)
		elemT.Execute(&b, &div{code, h.getClass(prev.tok)})
		b.WriteString(wsr)
	}
	return b.String()
}
<|endoftext|>


<commit_before>// Copyright 2016 Stratumn SAS. All rights reserved.
// Use of this source code is governed by the license
// that can be found in the LICENSE file.

package main

import (
	"crypto/sha256"
	"encoding/json"
	"fmt"
	"log"
	"os"
	"strconv"

	"github.com/stratumn/go/types"
	"github.com/stratumn/goprivate/merkle"
)

func main() {
	if len(os.Args) < 3 {
		log.Fatal("Fatal: unexpected number of arguments")
	}

	var (
		a = os.Args[1:]
		l = a[:len(a)-1]
		i = a[len(a)-1]
		s = len(l)
	)

	leaves := make([]types.Bytes32, s)
	for j, v := range l {
		leaves[j] = sha256.Sum256([]byte(v))
	}

	index, err := strconv.Atoi(i)
	if err != nil {
		log.Fatalf("Fatal: %s", err)
	}

	tree, err := merkle.NewStaticTree(leaves)
	if err != nil {
		log.Fatalf("Fatal: %s", err)
	}

	b, err := json.MarshalIndent(tree.Path(index), "", "  ")
	if err != nil {
		log.Fatalf("Fatal: %s", err)
	}

	fmt.Println(string(b))
}
<commit_msg>merkle: Use a better logger (#20) merkle:  * Add vendor dependencies<commit_after><|endoftext|>


<commit_before>package drivers

import (
	"fmt"
	"io"
	"io/ioutil"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/pborman/uuid"

	"github.com/lxc/lxd/lxd/migration"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/ioprogress"
)

func (d *zfs) dataset(vol Volume, deleted bool) string {
	name, snapName, _ := shared.InstanceGetParentAndSnapshotName(vol.name)
	if (vol.volType == VolumeTypeVM || vol.volType == VolumeTypeImage) && vol.contentType == ContentTypeBlock {
		name = fmt.Sprintf("%s.block", name)
	}

	if snapName != "" {
		if deleted {
			name = fmt.Sprintf("%s@deleted-%s", name, uuid.NewRandom().String())
		} else {
			name = fmt.Sprintf("%s@snapshot-%s", name, snapName)
		}
	} else if deleted {
		if vol.volType != VolumeTypeImage {
			name = uuid.NewRandom().String()
		}

		return filepath.Join(d.config["zfs.pool_name"], "deleted", string(vol.volType), name)
	}

	return filepath.Join(d.config["zfs.pool_name"], string(vol.volType), name)
}

func (d *zfs) createDataset(dataset string, options ...string) error {
	args := []string{"create"}
	for _, option := range options {
		args = append(args, "-o")
		args = append(args, option)
	}
	args = append(args, dataset)

	_, err := shared.RunCommand("zfs", args...)
	if err != nil {
		return err
	}

	return nil
}

func (d *zfs) createVolume(dataset string, size int64, options ...string) error {
	size = (size / minBlockBoundary) * minBlockBoundary

	args := []string{"create", "-s", "-V", fmt.Sprintf("%d", size)}
	for _, option := range options {
		args = append(args, "-o")
		args = append(args, option)
	}
	args = append(args, dataset)

	_, err := shared.RunCommand("zfs", args...)
	if err != nil {
		return err
	}

	return nil
}

func (d *zfs) checkDataset(dataset string) bool {
	out, err := shared.RunCommand("zfs", "get", "-H", "-o", "name", "name", dataset)
	if err != nil {
		return false
	}

	return strings.TrimSpace(out) == dataset
}

func (d *zfs) getClones(dataset string) ([]string, error) {
	out, err := shared.RunCommand("zfs", "get", "-H", "-p", "-r", "-o", "value", "clones", dataset)
	if err != nil {
		return nil, err
	}

	clones := []string{}
	for _, line := range strings.Split(out, "\n") {
		line = strings.TrimSpace(line)
		if line == dataset || line == "" || line == "-" {
			continue
		}

		line = strings.TrimPrefix(line, fmt.Sprintf("%s/", dataset))
		clones = append(clones, line)
	}

	return clones, nil
}

func (d *zfs) getDatasets(dataset string) ([]string, error) {
	out, err := shared.RunCommand("zfs", "get", "-H", "-r", "-o", "name", "name", dataset)
	if err != nil {
		return nil, err
	}

	children := []string{}
	for _, line := range strings.Split(out, "\n") {
		line = strings.TrimSpace(line)
		if line == dataset || line == "" {
			continue
		}

		line = strings.TrimPrefix(line, dataset)
		line = strings.TrimPrefix(line, "/")
		children = append(children, line)
	}

	return children, nil
}

func (d *zfs) setDatasetProperties(dataset string, options ...string) error {
	if len(zfsVersion) >= 3 && zfsVersion[0:3] == "0.6" {
		// Slow path for ZFS 0.6
		for _, option := range options {
			_, err := shared.RunCommand("zfs", "set", option, dataset)
			if err != nil {
				return err
			}
		}

		return nil
	}

	args := []string{"set"}
	args = append(args, options...)
	args = append(args, dataset)

	_, err := shared.RunCommand("zfs", args...)
	if err != nil {
		return err
	}

	return nil
}

func (d *zfs) getDatasetProperty(dataset string, key string) (string, error) {
	output, err := shared.RunCommand("zfs", "get", "-H", "-p", "-o", "value", key, dataset)
	if err != nil {
		return "", err
	}

	return strings.TrimSpace(output), nil
}

// version returns the ZFS version based on package or kernel module version.
func (d *zfs) version() (string, error) {
	// This function is only really ever relevant on Ubuntu as the only
	// distro that ships out of sync tools and kernel modules
	out, err := shared.RunCommand("dpkg-query", "--showformat=${Version}", "--show", "zfsutils-linux")
	if err == nil {
		return strings.TrimSpace(string(out)), nil
	}

	// Loaded kernel module version
	if shared.PathExists("/sys/module/zfs/version") {
		out, err := ioutil.ReadFile("/sys/module/zfs/version")
		if err == nil {
			return strings.TrimSpace(string(out)), nil
		}
	}

	// Module information version
	out, err = shared.RunCommand("modinfo", "-F", "version", "zfs")
	if err == nil {
		return strings.TrimSpace(string(out)), nil
	}

	return "", fmt.Errorf("Could not determine ZFS module version")
}

// initialDatasets returns the list of all expected datasets.
func (d *zfs) initialDatasets() []string {
	entries := []string{"deleted"}

	// Iterate over the listed supported volume types.
	for _, volType := range d.Info().VolumeTypes {
		entries = append(entries, BaseDirectories[volType][0])
		entries = append(entries, filepath.Join("deleted", BaseDirectories[volType][0]))
	}

	return entries
}

func (d *zfs) sendDataset(dataset string, parent string, volSrcArgs *migration.VolumeSourceArgs, conn io.ReadWriteCloser, tracker *ioprogress.ProgressTracker) error {
	// Assemble zfs send command.
	args := []string{"send"}
	if shared.StringInSlice("compress", volSrcArgs.MigrationType.Features) {
		args = append(args, "-c")
		args = append(args, "-L")
	}
	if parent != "" {
		args = append(args, "-i", parent)
	}
	args = append(args, dataset)
	cmd := exec.Command("zfs", args...)

	// Prepare stdout/stderr.
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}

	stderr, err := cmd.StderrPipe()
	if err != nil {
		return err
	}

	// Setup progress tracker.
	stdoutPipe := stdout
	if tracker != nil {
		stdoutPipe = &ioprogress.ProgressReader{
			ReadCloser: stdout,
			Tracker:    tracker,
		}
	}

	// Forward any output on stdout.
	chStdoutPipe := make(chan error, 1)
	go func() {
		_, err := io.Copy(conn, stdoutPipe)
		chStdoutPipe <- err
		conn.Close()
	}()

	// Run the command.
	err = cmd.Start()
	if err != nil {
		return err
	}

	// Read any error.
	output, _ := ioutil.ReadAll(stderr)

	// Handle errors.
	errs := []error{}
	chStdoutPipeErr := <-chStdoutPipe

	err = cmd.Wait()
	if err != nil {
		errs = append(errs, err)

		if chStdoutPipeErr != nil {
			errs = append(errs, chStdoutPipeErr)
		}
	}

	if len(errs) > 0 {
		return fmt.Errorf("zfs send failed: %v (%s)", errs, string(output))
	}

	return nil
}

func (d *zfs) receiveDataset(dataset string, conn io.ReadWriteCloser, writeWrapper func(io.WriteCloser) io.WriteCloser) error {
	// Assemble zfs send command.
	cmd := exec.Command("zfs", "receive", "-F", "-u", dataset)

	// Prepare stdin/stderr.
	stdin, err := cmd.StdinPipe()
	if err != nil {
		return err
	}

	stderr, err := cmd.StderrPipe()
	if err != nil {
		return err
	}

	// Forward input through stdin.
	chCopyConn := make(chan error, 1)
	go func() {
		_, err = io.Copy(stdin, conn)
		stdin.Close()
		chCopyConn <- err
	}()

	// Run the command.
	err = cmd.Start()
	if err != nil {
		return err
	}

	// Read any error.
	output, _ := ioutil.ReadAll(stderr)

	// Handle errors.
	errs := []error{}
	chCopyConnErr := <-chCopyConn

	err = cmd.Wait()
	if err != nil {
		errs = append(errs, err)

		if chCopyConnErr != nil {
			errs = append(errs, chCopyConnErr)
		}
	}

	if len(errs) > 0 {
		return fmt.Errorf("Problem with zfs receive: (%v) %s", errs, string(output))
	}

	return nil
}
<commit_msg>lxd/storage/drivers/driver/zfs/utils: Adds volumeSize function<commit_after>package drivers

import (
	"fmt"
	"io"
	"io/ioutil"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/pborman/uuid"

	"github.com/lxc/lxd/lxd/migration"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/ioprogress"
)

func (d *zfs) dataset(vol Volume, deleted bool) string {
	name, snapName, _ := shared.InstanceGetParentAndSnapshotName(vol.name)
	if (vol.volType == VolumeTypeVM || vol.volType == VolumeTypeImage) && vol.contentType == ContentTypeBlock {
		name = fmt.Sprintf("%s.block", name)
	}

	if snapName != "" {
		if deleted {
			name = fmt.Sprintf("%s@deleted-%s", name, uuid.NewRandom().String())
		} else {
			name = fmt.Sprintf("%s@snapshot-%s", name, snapName)
		}
	} else if deleted {
		if vol.volType != VolumeTypeImage {
			name = uuid.NewRandom().String()
		}

		return filepath.Join(d.config["zfs.pool_name"], "deleted", string(vol.volType), name)
	}

	return filepath.Join(d.config["zfs.pool_name"], string(vol.volType), name)
}

func (d *zfs) createDataset(dataset string, options ...string) error {
	args := []string{"create"}
	for _, option := range options {
		args = append(args, "-o")
		args = append(args, option)
	}
	args = append(args, dataset)

	_, err := shared.RunCommand("zfs", args...)
	if err != nil {
		return err
	}

	return nil
}

func (d *zfs) createVolume(dataset string, size int64, options ...string) error {
	size = (size / minBlockBoundary) * minBlockBoundary

	args := []string{"create", "-s", "-V", fmt.Sprintf("%d", size)}
	for _, option := range options {
		args = append(args, "-o")
		args = append(args, option)
	}
	args = append(args, dataset)

	_, err := shared.RunCommand("zfs", args...)
	if err != nil {
		return err
	}

	return nil
}

func (d *zfs) checkDataset(dataset string) bool {
	out, err := shared.RunCommand("zfs", "get", "-H", "-o", "name", "name", dataset)
	if err != nil {
		return false
	}

	return strings.TrimSpace(out) == dataset
}

func (d *zfs) getClones(dataset string) ([]string, error) {
	out, err := shared.RunCommand("zfs", "get", "-H", "-p", "-r", "-o", "value", "clones", dataset)
	if err != nil {
		return nil, err
	}

	clones := []string{}
	for _, line := range strings.Split(out, "\n") {
		line = strings.TrimSpace(line)
		if line == dataset || line == "" || line == "-" {
			continue
		}

		line = strings.TrimPrefix(line, fmt.Sprintf("%s/", dataset))
		clones = append(clones, line)
	}

	return clones, nil
}

func (d *zfs) getDatasets(dataset string) ([]string, error) {
	out, err := shared.RunCommand("zfs", "get", "-H", "-r", "-o", "name", "name", dataset)
	if err != nil {
		return nil, err
	}

	children := []string{}
	for _, line := range strings.Split(out, "\n") {
		line = strings.TrimSpace(line)
		if line == dataset || line == "" {
			continue
		}

		line = strings.TrimPrefix(line, dataset)
		line = strings.TrimPrefix(line, "/")
		children = append(children, line)
	}

	return children, nil
}

func (d *zfs) setDatasetProperties(dataset string, options ...string) error {
	if len(zfsVersion) >= 3 && zfsVersion[0:3] == "0.6" {
		// Slow path for ZFS 0.6
		for _, option := range options {
			_, err := shared.RunCommand("zfs", "set", option, dataset)
			if err != nil {
				return err
			}
		}

		return nil
	}

	args := []string{"set"}
	args = append(args, options...)
	args = append(args, dataset)

	_, err := shared.RunCommand("zfs", args...)
	if err != nil {
		return err
	}

	return nil
}

func (d *zfs) getDatasetProperty(dataset string, key string) (string, error) {
	output, err := shared.RunCommand("zfs", "get", "-H", "-p", "-o", "value", key, dataset)
	if err != nil {
		return "", err
	}

	return strings.TrimSpace(output), nil
}

// version returns the ZFS version based on package or kernel module version.
func (d *zfs) version() (string, error) {
	// This function is only really ever relevant on Ubuntu as the only
	// distro that ships out of sync tools and kernel modules
	out, err := shared.RunCommand("dpkg-query", "--showformat=${Version}", "--show", "zfsutils-linux")
	if err == nil {
		return strings.TrimSpace(string(out)), nil
	}

	// Loaded kernel module version
	if shared.PathExists("/sys/module/zfs/version") {
		out, err := ioutil.ReadFile("/sys/module/zfs/version")
		if err == nil {
			return strings.TrimSpace(string(out)), nil
		}
	}

	// Module information version
	out, err = shared.RunCommand("modinfo", "-F", "version", "zfs")
	if err == nil {
		return strings.TrimSpace(string(out)), nil
	}

	return "", fmt.Errorf("Could not determine ZFS module version")
}

// initialDatasets returns the list of all expected datasets.
func (d *zfs) initialDatasets() []string {
	entries := []string{"deleted"}

	// Iterate over the listed supported volume types.
	for _, volType := range d.Info().VolumeTypes {
		entries = append(entries, BaseDirectories[volType][0])
		entries = append(entries, filepath.Join("deleted", BaseDirectories[volType][0]))
	}

	return entries
}

func (d *zfs) sendDataset(dataset string, parent string, volSrcArgs *migration.VolumeSourceArgs, conn io.ReadWriteCloser, tracker *ioprogress.ProgressTracker) error {
	// Assemble zfs send command.
	args := []string{"send"}
	if shared.StringInSlice("compress", volSrcArgs.MigrationType.Features) {
		args = append(args, "-c")
		args = append(args, "-L")
	}
	if parent != "" {
		args = append(args, "-i", parent)
	}
	args = append(args, dataset)
	cmd := exec.Command("zfs", args...)

	// Prepare stdout/stderr.
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}

	stderr, err := cmd.StderrPipe()
	if err != nil {
		return err
	}

	// Setup progress tracker.
	stdoutPipe := stdout
	if tracker != nil {
		stdoutPipe = &ioprogress.ProgressReader{
			ReadCloser: stdout,
			Tracker:    tracker,
		}
	}

	// Forward any output on stdout.
	chStdoutPipe := make(chan error, 1)
	go func() {
		_, err := io.Copy(conn, stdoutPipe)
		chStdoutPipe <- err
		conn.Close()
	}()

	// Run the command.
	err = cmd.Start()
	if err != nil {
		return err
	}

	// Read any error.
	output, _ := ioutil.ReadAll(stderr)

	// Handle errors.
	errs := []error{}
	chStdoutPipeErr := <-chStdoutPipe

	err = cmd.Wait()
	if err != nil {
		errs = append(errs, err)

		if chStdoutPipeErr != nil {
			errs = append(errs, chStdoutPipeErr)
		}
	}

	if len(errs) > 0 {
		return fmt.Errorf("zfs send failed: %v (%s)", errs, string(output))
	}

	return nil
}

func (d *zfs) receiveDataset(dataset string, conn io.ReadWriteCloser, writeWrapper func(io.WriteCloser) io.WriteCloser) error {
	// Assemble zfs send command.
	cmd := exec.Command("zfs", "receive", "-F", "-u", dataset)

	// Prepare stdin/stderr.
	stdin, err := cmd.StdinPipe()
	if err != nil {
		return err
	}

	stderr, err := cmd.StderrPipe()
	if err != nil {
		return err
	}

	// Forward input through stdin.
	chCopyConn := make(chan error, 1)
	go func() {
		_, err = io.Copy(stdin, conn)
		stdin.Close()
		chCopyConn <- err
	}()

	// Run the command.
	err = cmd.Start()
	if err != nil {
		return err
	}

	// Read any error.
	output, _ := ioutil.ReadAll(stderr)

	// Handle errors.
	errs := []error{}
	chCopyConnErr := <-chCopyConn

	err = cmd.Wait()
	if err != nil {
		errs = append(errs, err)

		if chCopyConnErr != nil {
			errs = append(errs, chCopyConnErr)
		}
	}

	if len(errs) > 0 {
		return fmt.Errorf("Problem with zfs receive: (%v) %s", errs, string(output))
	}

	return nil
}

// volumeSize returns the size to use when creating new a volume.
func (d *zfs) volumeSize(vol Volume) string {
	size := vol.ExpandedConfig("size")

	// Block images always need a size.
	if vol.contentType == ContentTypeBlock && (size == "" || size == "0") {
		return defaultBlockSize
	}

	return size
}
<|endoftext|>


<commit_before>package main

import (
	ampl "github.com/alanctgardner/ampl-go"
	"fmt"
)

/* Test commands to submit to the AMPL CLI - in this case
   use the diet model from the AMPL book and write an MPS file */
var amplCommands = []string {
	"model src/github.com/alanctgardner/ampl-go/demo/diet1.mod;",
	"data src/github.com/alanctgardner/ampl-go/demo/diet1.dat",
	"write mdiet1;",
}

func main() {
	/* Try to get the cached location of the AMPL binary from a file on disk */
	amplLoc, err := ampl.GetAMPLLoc()
	if err != nil {
		fmt.Printf("Error getting cached AMPL executable location: %v\n", err)
		return
	}

	/* If there was no cached location, prompt the user and cache what they enter - NB: if the user's entry is wrong, delete the cached file to be prompted again. */
	if amplLoc == "" {
		amplLoc, err = ampl.PromptAMPLLoc()
		if err != nil {
			fmt.Printf("Error getting AMPL executable location on stdin: %v\n", err)
			return
		}
	}

	/* Run the AMPL binary */
	amplRunner, err := ampl.NewRunner(amplLoc)
	if err != nil {
		fmt.Printf("Error starting AMPL executable %q: %v\n", amplLoc, err)
		return
	}

	/* Run each of the commands in order. If there's an error, print it out and quit. Continuing after an error will not work. */
	for _, cmd := range amplCommands {
		fmt.Printf("Running command: %q\n", cmd)
		err := amplRunner.RunCommand(cmd)
		if err != nil {
			fmt.Printf("%v\n", err)
			return
		}
	}
}
<commit_msg>Add demo to codebase<commit_after>package main

import (
	runner "github.com/alanctgardner/ampl-go/runner"
	"fmt"
)

/* Test commands to submit to the AMPL CLI - in this case
   use the diet model from the AMPL book and write an MPS file */
var amplCommands = []string {
	"model src/github.com/alanctgardner/ampl-go/demo/diet1.mod;",
	"data src/github.com/alanctgardner/ampl-go/demo/diet1.dat",
	"write gdiet1;",
}

func main() {
	/* Try to get the cached location of the AMPL binary from a file on disk */
	amplLoc, err := runner.GetAMPLLoc()
	if err != nil {
		fmt.Printf("Error getting cached AMPL executable location: %v\n", err)
		return
	}

	/* If there was no cached location, prompt the user and cache what they enter - NB: if the user's entry is wrong, delete the cached file to be prompted again. */
	if amplLoc == "" {
		amplLoc, err = runner.PromptAMPLLoc()
		if err != nil {
			fmt.Printf("Error getting AMPL executable location on stdin: %v\n", err)
			return
		}
	}

	/* Run the AMPL binary */
	amplRunner, err := runner.NewRunner(amplLoc)
	if err != nil {
		fmt.Printf("Error starting AMPL executable %q: %v\n", amplLoc, err)
		return
	}

	/* Run each of the commands in order. If there's an error, print it out and quit. Continuing after an error will not work. */
	for _, cmd := range amplCommands {
		fmt.Printf("Running command: %q\n", cmd)
		err := amplRunner.RunCommand(cmd)
		if err != nil {
			fmt.Printf("%v\n", err)
			return
		}
	}
}
<|endoftext|>


<commit_before>package template

import (
	"fmt"
	"strings"
	"text/template"
)

// FuncRegistry registry func for templates
type FuncRegistry struct {
	TemplateData map[string]interface{}
}

// RegistryAllFuncs for template
func (fr *FuncRegistry) RegistryAllFuncs() (funcs template.FuncMap) {
	funcs = template.FuncMap{
		"isSet":          fr.isSet,
		"defaultOrValue": fr.defaultOrValue,
		"inFormat":       fr.inFormat,
	}
	return
}

func (fr *FuncRegistry) isSet(key string) (ok bool) {
	_, ok = fr.TemplateData[key]
	return
}

func (fr *FuncRegistry) defaultOrValue(key, defaultValue string) (value interface{}) {
	if ok := fr.isSet(key); !ok {
		fr.TemplateData[key] = defaultValue
	}
	value = fr.TemplateData[key]
	return
}

func (fr *FuncRegistry) inFormat(key string) (query string) {
	items, ok := fr.TemplateData[key].([]string)
	if !ok {
		query = fmt.Sprintf("('%v')", fr.TemplateData[key])
		return
	}
	query = fmt.Sprintf("('%s')", strings.Join(items, "', '"))
	return
}
<commit_msg>unescape url before to send it to database (#4)<commit_after>package template

import (
	"fmt"
	"net/url"
	"strings"
	"text/template"
)

// FuncRegistry registry func for templates
type FuncRegistry struct {
	TemplateData map[string]interface{}
}

// RegistryAllFuncs for template
func (fr *FuncRegistry) RegistryAllFuncs() (funcs template.FuncMap) {
	funcs = template.FuncMap{
		"isSet":          fr.isSet,
		"defaultOrValue": fr.defaultOrValue,
		"inFormat":       fr.inFormat,
		"unEscape":       fr.unEscape,
	}
	return
}

func (fr *FuncRegistry) isSet(key string) (ok bool) {
	_, ok = fr.TemplateData[key]
	return
}

func (fr *FuncRegistry) defaultOrValue(key, defaultValue string) (value interface{}) {
	if ok := fr.isSet(key); !ok {
		fr.TemplateData[key] = defaultValue
	}
	value = fr.TemplateData[key]
	return
}

func (fr *FuncRegistry) inFormat(key string) (query string) {
	items, ok := fr.TemplateData[key].([]string)
	if !ok {
		query = fmt.Sprintf("('%v')", fr.TemplateData[key])
		return
	}
	query = fmt.Sprintf("('%s')", strings.Join(items, "', '"))
	return
}

func (fr *FuncRegistry) unEscape(key string) (value string) {
	value, _ = url.QueryUnescape(key)
	return
}
<|endoftext|>


<commit_before>package fw

const (
	ChainInput   = "INPUT"
	ChainOutput  = "OUTPUT"
	ChainForward = "FORWARD"
	ProtocolTcp  = "tcp"
	ProtocolUdp  = "udp"

	PortHttp  = 80
	PortHttps = 443
)
<commit_msg>add VPN port to constants<commit_after>package fw

const (
	ChainInput   = "INPUT"
	ChainOutput  = "OUTPUT"
	ChainForward = "FORWARD"
	ProtocolTcp  = "tcp"
	ProtocolUdp  = "udp"

	PortHttp    = 80
	PortHttps   = 443
	PortOpenVPN = 1194
)
<|endoftext|>