<commit_before>package project

import (
	"fmt"
	"strconv"

	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/units"
	"github.com/pkg/errors"
)

// CheckLimitsUponInstanceCreation returns an error if any project-specific
// limit is violated when creating a new instance.
func CheckLimitsUponInstanceCreation(tx *db.ClusterTx, projectName string, req api.InstancesPost) error {
	project, profiles, instances, err := fetchProject(tx, projectName)
	if err != nil {
		return err
	}

	err = checkInstanceCountLimit(project, len(instances), req.Type)
	if err != nil {
		return err
	}

	// Add the instance being created.
	instances = append(instances, db.Instance{
		Profiles: req.Profiles,
		Config:   req.Config,
	})

	err = checkAggregateInstanceLimits(tx, project, instances, profiles)
	if err != nil {
		return err
	}

	return nil
}

// Check that we have not reached the maximum number of instances for
// this type.
func checkInstanceCountLimit(project *api.Project, instanceCount int, instanceType api.InstanceType) error {
	var key string
	switch instanceType {
	case api.InstanceTypeContainer:
		key = "limits.containers"
	case api.InstanceTypeVM:
		key = "limits.virtual-machines"
	default:
		return fmt.Errorf("Unexpected instance type '%s'", instanceType)
	}
	value, ok := project.Config[key]
	if ok {
		limit, err := strconv.Atoi(value)
		if err != nil || limit < 0 {
			return fmt.Errorf("Unexpected '%s' value: '%s'", key, value)
		}
		if instanceCount >= limit {
			return fmt.Errorf(
				"Reached maximum number of instances of type %s in project %s",
				instanceType, project.Name)
		}
	}

	return nil
}

// Check that we would not violate the project limits if we were to commit the
// given instances and profiles.
func checkAggregateInstanceLimits(tx *db.ClusterTx, project *api.Project, instances []db.Instance, profiles []db.Profile) error {
	// List of config keys for which we need to check aggregate values
	// across all project instances.
	aggregateKeys := []string{}
	for key := range project.Config {
		if shared.StringInSlice(key, []string{"limits.memory"}) {
			aggregateKeys = append(aggregateKeys, key)
		}
	}
	if len(aggregateKeys) == 0 {
		return nil
	}

	instances = expandInstancesConfig(instances, profiles)

	totals, err := getTotalsAcrossInstances(instances, aggregateKeys)
	if err != nil {
		return err
	}

	for _, key := range aggregateKeys {
		parser := aggregateLimitConfigValueParsers[key]
		max, err := parser(project.Config[key])
		if err != nil {
			return err
		}
		if totals[key] > max {
			return fmt.Errorf(
				"Reached maximum aggregate value %s for %q in project %s",
				project.Config[key], key, project.Name)
		}
	}

	return nil
}

// CheckLimitsUponInstanceUpdate returns an error if any project-specific limit
// is violated when updating an existing instance.
func CheckLimitsUponInstanceUpdate(tx *db.ClusterTx, projectName, instanceName string, req api.InstancePut) error {
	project, profiles, instances, err := fetchProject(tx, projectName)
	if err != nil {
		return err
	}

	// Change the instance being updated.
	for i, instance := range instances {
		if instance.Name != instanceName {
			continue
		}
		instances[i].Profiles = req.Profiles
		instances[i].Config = req.Config
	}

	err = checkAggregateInstanceLimits(tx, project, instances, profiles)
	if err != nil {
		return err
	}

	return nil
}

// Fetch the given project from the database along with its profiles and instances.
func fetchProject(tx *db.ClusterTx, projectName string) (*api.Project, []db.Profile, []db.Instance, error) {
	project, err := tx.ProjectGet(projectName)
	if err != nil {
		return nil, nil, nil, errors.Wrap(err, "Fetch project database object")
	}

	profilesFilter := db.ProfileFilter{}

	// If the project has the profiles feature enabled, we use its own
	// profiles to expand the instances configs, otherwise we use the
	// profiles from the default project.
	if projectName == "default" || project.Config["features.profiles"] == "true" {
		profilesFilter.Project = projectName
	} else {
		profilesFilter.Project = "default"
	}

	profiles, err := tx.ProfileList(profilesFilter)
	if err != nil {
		return nil, nil, nil, errors.Wrap(err, "Fetch profiles from database")
	}

	instances, err := tx.InstanceList(db.InstanceFilter{Project: projectName})
	if err != nil {
		return nil, nil, nil, errors.Wrap(err, "Fetch project instances from database")
	}

	return project, profiles, instances, nil
}

// Expand the configuration of the given instances, taking the give project
// profiles into account.
func expandInstancesConfig(instances []db.Instance, profiles []db.Profile) []db.Instance {
	expandedInstances := make([]db.Instance, len(instances))

	// Index of all profiles by name.
	profilesByName := map[string]db.Profile{}
	for _, profile := range profiles {
		profilesByName[profile.Name] = profile
	}

	for i, instance := range instances {
		profiles := make([]api.Profile, len(instance.Profiles))

		for j, name := range instance.Profiles {
			profile := profilesByName[name]
			profiles[j] = *db.ProfileToAPI(&profile)
		}

		expandedInstances[i] = instance
		expandedInstances[i].Config = db.ProfilesExpandConfig(instance.Config, profiles)
	}

	return expandedInstances
}

// Sum of the effective instance-level value for the given limits across all
// project instances. If excludeInstance is not the empty string, exclude the
// instance with that name.
func getTotalsAcrossInstances(instances []db.Instance, keys []string) (map[string]int64, error) {
	totals := map[string]int64{}

	for _, key := range keys {
		totals[key] = 0
	}

	for _, instance := range instances {
		limits, err := getInstanceLimits(instance, keys)
		if err != nil {
			return nil, err
		}
		for _, key := range keys {
			totals[key] += limits[key]
		}
	}

	return totals, nil
}

// Return the effective instance-level values for the limits with the given
// keys.
func getInstanceLimits(instance db.Instance, keys []string) (map[string]int64, error) {
	limits := map[string]int64{}

	for _, key := range keys {
		value, ok := instance.Config[key]
		if !ok || value == "" {
			return nil, fmt.Errorf(
				"Instance %s in project %s has no '%s' config, either directly or via a profile",
				instance.Name, instance.Project, key)
		}
		parser := aggregateLimitConfigValueParsers[key]
		limit, err := parser(value)
		if err != nil {
			return nil, errors.Wrapf(
				err, "Parse '%s' for instance %s in project %s",
				key, instance.Name, instance.Project)
		}
		limits[key] = limit
	}

	return limits, nil
}

var aggregateLimitConfigValueParsers = map[string]func(string) (int64, error){
	"limits.memory": units.ParseByteSizeString,
}
<commit_msg>lxd/project: Add initial ValidateLimitsUponProjectUpdate<commit_after>package project

import (
	"fmt"
	"strconv"

	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/instance/instancetype"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/units"
	"github.com/pkg/errors"
)

// CheckLimitsUponInstanceCreation returns an error if any project-specific
// limit is violated when creating a new instance.
func CheckLimitsUponInstanceCreation(tx *db.ClusterTx, projectName string, req api.InstancesPost) error {
	project, profiles, instances, err := fetchProject(tx, projectName)
	if err != nil {
		return err
	}

	err = checkInstanceCountLimit(project, len(instances), req.Type)
	if err != nil {
		return err
	}

	// Add the instance being created.
	instances = append(instances, db.Instance{
		Profiles: req.Profiles,
		Config:   req.Config,
	})

	err = checkAggregateInstanceLimits(tx, project, instances, profiles)
	if err != nil {
		return err
	}

	return nil
}

// Check that we have not reached the maximum number of instances for
// this type.
func checkInstanceCountLimit(project *api.Project, instanceCount int, instanceType api.InstanceType) error {
	var key string
	switch instanceType {
	case api.InstanceTypeContainer:
		key = "limits.containers"
	case api.InstanceTypeVM:
		key = "limits.virtual-machines"
	default:
		return fmt.Errorf("Unexpected instance type '%s'", instanceType)
	}
	value, ok := project.Config[key]
	if ok {
		limit, err := strconv.Atoi(value)
		if err != nil || limit < 0 {
			return fmt.Errorf("Unexpected '%s' value: '%s'", key, value)
		}
		if instanceCount >= limit {
			return fmt.Errorf(
				"Reached maximum number of instances of type %s in project %s",
				instanceType, project.Name)
		}
	}

	return nil
}

// Check that we would not violate the project limits if we were to commit the
// given instances and profiles.
func checkAggregateInstanceLimits(tx *db.ClusterTx, project *api.Project, instances []db.Instance, profiles []db.Profile) error {
	// List of config keys for which we need to check aggregate values
	// across all project instances.
	aggregateKeys := []string{}
	for key := range project.Config {
		if shared.StringInSlice(key, []string{"limits.memory"}) {
			aggregateKeys = append(aggregateKeys, key)
		}
	}
	if len(aggregateKeys) == 0 {
		return nil
	}

	instances = expandInstancesConfig(instances, profiles)

	totals, err := getTotalsAcrossInstances(instances, aggregateKeys)
	if err != nil {
		return err
	}

	for _, key := range aggregateKeys {
		parser := aggregateLimitConfigValueParsers[key]
		max, err := parser(project.Config[key])
		if err != nil {
			return err
		}
		if totals[key] > max {
			return fmt.Errorf(
				"Reached maximum aggregate value %s for %q in project %s",
				project.Config[key], key, project.Name)
		}
	}

	return nil
}

// CheckLimitsUponInstanceUpdate returns an error if any project-specific limit
// is violated when updating an existing instance.
func CheckLimitsUponInstanceUpdate(tx *db.ClusterTx, projectName, instanceName string, req api.InstancePut) error {
	project, profiles, instances, err := fetchProject(tx, projectName)
	if err != nil {
		return err
	}

	// Change the instance being updated.
	for i, instance := range instances {
		if instance.Name != instanceName {
			continue
		}
		instances[i].Profiles = req.Profiles
		instances[i].Config = req.Config
	}

	err = checkAggregateInstanceLimits(tx, project, instances, profiles)
	if err != nil {
		return err
	}

	return nil
}

// ValidateLimitsUponProjectUpdate checks the new limits to be set on a project
// are valid.
func ValidateLimitsUponProjectUpdate(tx *db.ClusterTx, projectName string, config map[string]string, changed []string) error {
	_, profiles, instances, err := fetchProject(tx, projectName)
	if err != nil {
		return err
	}

	instances = expandInstancesConfig(instances, profiles)

	for _, key := range changed {
		switch key {
		case "limits.containers":
			fallthrough
		case "limits.virtual-machines":
			err := validateInstanceCountLimit(instances, key, config[key], projectName)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

// Check that limits.containers or limits.virtual-machines is equal or above
// the current count.
func validateInstanceCountLimit(instances []db.Instance, key, value, project string) error {
	instanceType := countConfigInstanceType[key]
	limit, err := strconv.Atoi(value)
	if err != nil {
		return err
	}
	dbType, err := instancetype.New(string(instanceType))
	if err != nil {
		return err
	}
	count := 0
	for _, instance := range instances {
		if instance.Type == dbType {
			count++
		}
	}
	if limit < count {
		return fmt.Errorf(
			"'%s' is too low: there currently are %d instances of type %s in project %s",
			key, count, instanceType, project)
	}
	return nil
}

var countConfigInstanceType = map[string]api.InstanceType{
	"limits.containers":       api.InstanceTypeContainer,
	"limits.virtual-machines": api.InstanceTypeVM,
}

// Fetch the given project from the database along with its profiles and instances.
func fetchProject(tx *db.ClusterTx, projectName string) (*api.Project, []db.Profile, []db.Instance, error) {
	project, err := tx.ProjectGet(projectName)
	if err != nil {
		return nil, nil, nil, errors.Wrap(err, "Fetch project database object")
	}

	profilesFilter := db.ProfileFilter{}

	// If the project has the profiles feature enabled, we use its own
	// profiles to expand the instances configs, otherwise we use the
	// profiles from the default project.
	if projectName == "default" || project.Config["features.profiles"] == "true" {
		profilesFilter.Project = projectName
	} else {
		profilesFilter.Project = "default"
	}

	profiles, err := tx.ProfileList(profilesFilter)
	if err != nil {
		return nil, nil, nil, errors.Wrap(err, "Fetch profiles from database")
	}

	instances, err := tx.InstanceList(db.InstanceFilter{Project: projectName})
	if err != nil {
		return nil, nil, nil, errors.Wrap(err, "Fetch project instances from database")
	}

	return project, profiles, instances, nil
}

// Expand the configuration of the given instances, taking the give project
// profiles into account.
func expandInstancesConfig(instances []db.Instance, profiles []db.Profile) []db.Instance {
	expandedInstances := make([]db.Instance, len(instances))

	// Index of all profiles by name.
	profilesByName := map[string]db.Profile{}
	for _, profile := range profiles {
		profilesByName[profile.Name] = profile
	}

	for i, instance := range instances {
		profiles := make([]api.Profile, len(instance.Profiles))

		for j, name := range instance.Profiles {
			profile := profilesByName[name]
			profiles[j] = *db.ProfileToAPI(&profile)
		}

		expandedInstances[i] = instance
		expandedInstances[i].Config = db.ProfilesExpandConfig(instance.Config, profiles)
	}

	return expandedInstances
}

// Sum of the effective instance-level value for the given limits across all
// project instances. If excludeInstance is not the empty string, exclude the
// instance with that name.
func getTotalsAcrossInstances(instances []db.Instance, keys []string) (map[string]int64, error) {
	totals := map[string]int64{}

	for _, key := range keys {
		totals[key] = 0
	}

	for _, instance := range instances {
		limits, err := getInstanceLimits(instance, keys)
		if err != nil {
			return nil, err
		}
		for _, key := range keys {
			totals[key] += limits[key]
		}
	}

	return totals, nil
}

// Return the effective instance-level values for the limits with the given
// keys.
func getInstanceLimits(instance db.Instance, keys []string) (map[string]int64, error) {
	limits := map[string]int64{}

	for _, key := range keys {
		value, ok := instance.Config[key]
		if !ok || value == "" {
			return nil, fmt.Errorf(
				"Instance %s in project %s has no '%s' config, either directly or via a profile",
				instance.Name, instance.Project, key)
		}
		parser := aggregateLimitConfigValueParsers[key]
		limit, err := parser(value)
		if err != nil {
			return nil, errors.Wrapf(
				err, "Parse '%s' for instance %s in project %s",
				key, instance.Name, instance.Project)
		}
		limits[key] = limit
	}

	return limits, nil
}

var aggregateLimitConfigValueParsers = map[string]func(string) (int64, error){
	"limits.memory": units.ParseByteSizeString,
}
<|endoftext|>


<commit_before>package pool

import (
	"errors"
	"net"
	"sync"
	"time"
)

// NextConn returns a new net.Conn
type NextConn func() (net.Conn, error)

type limitConn struct {
	net.Conn
	*sync.Cond
	limit *uint64
}

func (c *limitConn) Close() error {
	go func() {
		c.Cond.L.Lock()
		defer c.Cond.L.Unlock()
		(*c.limit)++
		c.Cond.Signal()
	}()
	return c.Conn.Close()
}

// Limiter limits the number of concurrent connections returned by next to limit.
// The number of connections is calculated as # of connections returned - # closed.
func Limiter(next NextConn, limit uint64) NextConn {
	var mu sync.Mutex
	cond := sync.NewCond(&mu)

	return func() (net.Conn, error) {
		mu.Lock()
		defer mu.Unlock()
		for limit == 0 {
			cond.Wait()
		}
		c, err := next()
		if err == nil {
			limit--
			c = &limitConn{
				Conn:  c,
				Cond:  cond,
				limit: &limit,
			}
		}
		return c, err
	}
}

// Pool manages connections
type Pool struct {
	next NextConn
	reqs *requests
	dead chan struct{}
}

// NewPool creates a new Pool object for managing connections
func NewPool(next NextConn) *Pool {
	p := &Pool{
		next: next,
		reqs: newRequests(),
		dead: make(chan struct{}),
	}
	go p.manage()
	return p
}

var ErrPoolClosed = errors.New("pool closed")

// Get will return a net.Conn, it can be called concurrently
// and calls to Get() will be returned in the order they were called.
func (p *Pool) Get() (net.Conn, error) {
	if c, ok := <-p.reqs.submit(); ok {
		return c, nil
	}
	return nil, ErrPoolClosed
}

func (p *Pool) manage() {
	for {
		req, ok := p.reqs.next()
		if !ok {
			return
		}

		var c net.Conn
		err := errors.New("temp error")

		for err != nil {
			c, err = p.next()
			if err != nil {
				select {
				case <-time.After(5 * time.Second):
				case <-p.dead:
					close(req)
					return
				}
			}
		}

		req <- c
	}
}

// Close closes the pool and cancels calls to Get()
func (p *Pool) Close() error {
	p.reqs.close()
	close(p.dead)
	return nil
}

type requests struct {
	mu   sync.Mutex
	cond *sync.Cond
	dead bool
	reqs []chan net.Conn
}

func newRequests() *requests {
	r := &requests{reqs: make([]chan net.Conn, 0)}
	r.cond = sync.NewCond(&r.mu)
	return r
}

func (r *requests) submit() <-chan net.Conn {
	r.mu.Lock()
	defer r.mu.Unlock()
	req := make(chan net.Conn)
	if r.dead {
		close(req)
	} else {
		r.reqs = append(r.reqs, req)
		r.cond.Signal()
	}
	return req
}

func (r *requests) next() (req chan<- net.Conn, ok bool) {
	r.mu.Lock()
	defer r.mu.Unlock()
	for len(r.reqs) == 0 && !r.dead {
		r.cond.Wait()
	}
	if r.dead {
		return nil, false
	} else {
		req, r.reqs = r.reqs[0], r.reqs[1:]
		return req, true
	}
}

func (r *requests) close() {
	r.mu.Lock()
	defer r.mu.Unlock()
	r.dead = true
	for _, req := range r.reqs {
		close(req)
	}
	r.reqs = nil
	r.cond.Broadcast()
}
<commit_msg>fixing linting suggestions<commit_after>package pool

import (
	"errors"
	"net"
	"sync"
	"time"
)

// NextConn returns a new net.Conn
type NextConn func() (net.Conn, error)

type limitConn struct {
	net.Conn
	*sync.Cond
	limit *uint64
}

func (c *limitConn) Close() error {
	go func() {
		c.Cond.L.Lock()
		defer c.Cond.L.Unlock()
		(*c.limit)++
		c.Cond.Signal()
	}()
	return c.Conn.Close()
}

// Limiter limits the number of concurrent connections returned by next to limit.
// The number of connections is calculated as # of connections returned - # closed.
func Limiter(next NextConn, limit uint64) NextConn {
	var mu sync.Mutex
	cond := sync.NewCond(&mu)

	return func() (net.Conn, error) {
		mu.Lock()
		defer mu.Unlock()
		for limit == 0 {
			cond.Wait()
		}
		c, err := next()
		if err == nil {
			limit--
			c = &limitConn{
				Conn:  c,
				Cond:  cond,
				limit: &limit,
			}
		}
		return c, err
	}
}

// Pool manages connections
type Pool struct {
	next NextConn
	reqs *requests
	dead chan struct{}
}

// NewPool creates a new Pool object for managing connections
func NewPool(next NextConn) *Pool {
	p := &Pool{
		next: next,
		reqs: newRequests(),
		dead: make(chan struct{}),
	}
	go p.manage()
	return p
}

// ErrPoolClosed is returned by ops called when the pool is closed
var ErrPoolClosed = errors.New("pool closed")

// Get will return a net.Conn, it can be called concurrently
// and calls to Get() will be returned in the order they were called.
func (p *Pool) Get() (net.Conn, error) {
	if c, ok := <-p.reqs.submit(); ok {
		return c, nil
	}
	return nil, ErrPoolClosed
}

func (p *Pool) manage() {
	for {
		req, ok := p.reqs.next()
		if !ok {
			return
		}

		var c net.Conn
		err := errors.New("temp error")

		for err != nil {
			c, err = p.next()
			if err != nil {
				select {
				case <-time.After(5 * time.Second):
				case <-p.dead:
					close(req)
					return
				}
			}
		}

		req <- c
	}
}

// Close closes the pool and cancels calls to Get()
func (p *Pool) Close() error {
	p.reqs.close()
	close(p.dead)
	return nil
}

type requests struct {
	mu   sync.Mutex
	cond *sync.Cond
	dead bool
	reqs []chan net.Conn
}

func newRequests() *requests {
	r := &requests{reqs: make([]chan net.Conn, 0)}
	r.cond = sync.NewCond(&r.mu)
	return r
}

func (r *requests) submit() <-chan net.Conn {
	r.mu.Lock()
	defer r.mu.Unlock()
	req := make(chan net.Conn)
	if r.dead {
		close(req)
	} else {
		r.reqs = append(r.reqs, req)
		r.cond.Signal()
	}
	return req
}

func (r *requests) next() (req chan<- net.Conn, ok bool) {
	r.mu.Lock()
	defer r.mu.Unlock()

	for len(r.reqs) == 0 && !r.dead {
		r.cond.Wait()
	}

	if r.dead {
		return nil, false
	}

	req, r.reqs = r.reqs[0], r.reqs[1:]
	return req, true
}

func (r *requests) close() {
	r.mu.Lock()
	defer r.mu.Unlock()
	r.dead = true
	for _, req := range r.reqs {
		close(req)
	}
	r.reqs = nil
	r.cond.Broadcast()
}
<|endoftext|>


<commit_before>/* Copyright (c) 2013, Stefan Talpalaru <stefan.talpalaru@od-eon.com>, Odeon Consulting Group Pte Ltd <od-eon.com>
 * All rights reserved. */

/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

// Package pool provides a worker pool.
package pool

import (
	"fmt"
	"log"
	"sync"
	"time"
)

// Job holds all the data related to a worker's instance.
type Job struct {
	F      func(...interface{}) interface{}
	Args   []interface{}
	Result interface{}
	Err    error
}

// stats is a structure holding statistical data about the pool.
type stats struct {
	Submitted int
	Running   int
	Completed int
}

// Pool is the main data structure.
type Pool struct {
	started              bool
	num_workers          int
	job_pipe             chan *Job
	done_pipe            chan *Job
	add_pipe             chan *Job
	result_pipe          chan *Job
	jobs_ready_to_run    []*Job
	num_jobs_submitted   int
	num_jobs_running     int
	num_jobs_completed   int
	jobs_completed       []*Job
	interval             time.Duration // for sleeping, in ms
	working_pipe         chan bool
	stats_pipe           chan stats
	worker_kill_pipe     chan bool
	supervisor_kill_pipe chan bool
	worker_wg            sync.WaitGroup
	supervisor_wg        sync.WaitGroup
}

// subworker catches any panic while running the job.
func (pool *Pool) subworker(job *Job) {
	defer func() {
		if err := recover(); err != nil {
			log.Println("panic while running job:", err)
			job.Result = nil
			job.Err = fmt.Errorf(err.(string))
		}
	}()
	job.Result = job.F(job.Args...)
}

// worker gets a job from the job_pipe, passes it to a
// subworker and puts the job in the done_pipe when finished.
func (pool *Pool) worker(num int) {
WORKER_LOOP:
	for {
		select {
		case <-pool.worker_kill_pipe:
			// worker suicide
			break WORKER_LOOP
		case job := <-pool.job_pipe:
			pool.subworker(job)
			pool.done_pipe <- job
		}
	}
	pool.worker_wg.Done()
}

// NewPool creates a new Pool.
func NewPool(workers int) (pool *Pool) {
	pool = new(Pool)
	pool.num_workers = workers
	pool.job_pipe = make(chan *Job)
	pool.done_pipe = make(chan *Job)
	pool.add_pipe = make(chan *Job)
	pool.result_pipe = make(chan *Job)
	pool.jobs_ready_to_run = make([]*Job, 0)
	pool.jobs_completed = make([]*Job, 0)
	pool.working_pipe = make(chan bool)
	pool.stats_pipe = make(chan stats)
	pool.worker_kill_pipe = make(chan bool)
	pool.supervisor_kill_pipe = make(chan bool)
	pool.interval = 1
	return
}

// supervisor feeds jobs to workers and keeps track of them.
func (pool *Pool) supervisor() {
SUPERVISOR_LOOP:
	for {
		select {
		case job := <-pool.add_pipe:
			pool.jobs_ready_to_run = append(pool.jobs_ready_to_run, job)
			pool.num_jobs_submitted++
		default:
		}

		num_ready_jobs := len(pool.jobs_ready_to_run)
		if num_ready_jobs > 0 {
			select {
			case pool.job_pipe <- pool.jobs_ready_to_run[num_ready_jobs-1]:
				pool.num_jobs_running++
				pool.jobs_ready_to_run = pool.jobs_ready_to_run[:num_ready_jobs-1]
			default:
			}
		}

		if pool.num_jobs_running > 0 {
			select {
			case job := <-pool.done_pipe:
				pool.num_jobs_running--
				pool.jobs_completed = append(pool.jobs_completed, job)
				pool.num_jobs_completed++
			default:
			}
		}

		working := true
		if len(pool.jobs_ready_to_run) == 0 && pool.num_jobs_running == 0 {
			working = false
		}
		select {
		case pool.working_pipe <- working:
		default:
		}

		res := (*Job)(nil)
		if len(pool.jobs_completed) > 0 {
			res = pool.jobs_completed[0]
		}
		select {
		case pool.result_pipe <- res:
			if len(pool.jobs_completed) > 0 {
				pool.jobs_completed = pool.jobs_completed[1:]
			}
		default:
		}

		pool_stats := stats{pool.num_jobs_submitted, pool.num_jobs_running, pool.num_jobs_completed}
		select {
		case pool.stats_pipe <- pool_stats:
		default:
		}

		// stopping
		select {
		case <-pool.supervisor_kill_pipe:
			break SUPERVISOR_LOOP
		default:
		}

		time.Sleep(pool.interval * time.Millisecond)
	}
	pool.supervisor_wg.Done()
}

// Run starts the Pool by launching the workers and a supervisor goroutine.
// It's OK to start an empty Pool. The jobs will be fed to the workers as soon
// as they become available.
func (pool *Pool) Run() {
	if pool.started {
		panic("trying to start a pool that's already running")
	}
	for i := 0; i < pool.num_workers; i++ {
		pool.worker_wg.Add(1)
		go pool.worker(i)
	}
	pool.supervisor_wg.Add(1)
	go pool.supervisor()
	pool.started = true
}

// Stop will signal the workers and supervisor to exit and wait for them to actually do that.
func (pool *Pool) Stop() {
	if !pool.started {
		panic("trying to stop a pool that's already stopped")
	}
	// stop the workers
	for i := 0; i < pool.num_workers; i++ {
		pool.worker_kill_pipe <- true
	}
	pool.worker_wg.Wait()
	// stop the supervisor
	pool.supervisor_kill_pipe <- true
	pool.supervisor_wg.Wait()
	// set the flag
	pool.started = false
}

// Add creates a Job from the given function and args and
// adds it to the Pool.
func (pool *Pool) Add(f func(...interface{}) interface{}, args ...interface{}) {
	pool.add_pipe <- &Job{f, args, nil, nil}
}

// Wait blocks until all the jobs in the Pool are done.
func (pool *Pool) Wait() {
	for <-pool.working_pipe {
		time.Sleep(pool.interval * time.Millisecond)
	}
}

// Results retrieves the completed jobs.
func (pool *Pool) Results() (res []*Job) {
	res = make([]*Job, len(pool.jobs_completed))
	for i, job := range pool.jobs_completed {
		res[i] = job
	}
	pool.jobs_completed = pool.jobs_completed[0:0]
	return
}

// WaitForJob blocks until a completed job is available and returns it.
// If there are no jobs running, it returns nil.
func (pool *Pool) WaitForJob() *Job {
	for {
		working := <-pool.working_pipe
		r := <-pool.result_pipe
		if r == (*Job)(nil) {
			if !working {
				break
			}
		} else {
			return r
		}
	}
	return nil
}

// Status returns a "stats" instance.
func (pool *Pool) Status() stats {
	if pool.started {
		return <-pool.stats_pipe
	}
	// the pool wasn't started so we return a zeroed structure
	return stats{}
}
<commit_msg>change the supervisor starting/stopping logic<commit_after>/* Copyright (c) 2013, Stefan Talpalaru <stefan.talpalaru@od-eon.com>, Odeon Consulting Group Pte Ltd <od-eon.com>
 * All rights reserved. */

/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

// Package pool provides a worker pool.
package pool

import (
	"fmt"
	"log"
	"sync"
	"time"
)

// Job holds all the data related to a worker's instance.
type Job struct {
	F      func(...interface{}) interface{}
	Args   []interface{}
	Result interface{}
	Err    error
}

// stats is a structure holding statistical data about the pool.
type stats struct {
	Submitted int
	Running   int
	Completed int
}

// Pool is the main data structure.
type Pool struct {
	workers_started              bool
	supervisor_started              bool
	num_workers          int
	job_pipe             chan *Job
	done_pipe            chan *Job
	add_pipe             chan *Job
	result_pipe          chan *Job
	jobs_ready_to_run    []*Job
	num_jobs_submitted   int
	num_jobs_running     int
	num_jobs_completed   int
	jobs_completed       []*Job
	interval             time.Duration // for sleeping, in ms
	working_pipe         chan bool
	stats_pipe           chan stats
	worker_kill_pipe     chan bool
	supervisor_kill_pipe chan bool
	worker_wg            sync.WaitGroup
	supervisor_wg        sync.WaitGroup
}

// subworker catches any panic while running the job.
func (pool *Pool) subworker(job *Job) {
	defer func() {
		if err := recover(); err != nil {
			log.Println("panic while running job:", err)
			job.Result = nil
			job.Err = fmt.Errorf(err.(string))
		}
	}()
	job.Result = job.F(job.Args...)
}

// worker gets a job from the job_pipe, passes it to a
// subworker and puts the job in the done_pipe when finished.
func (pool *Pool) worker(num int) {
WORKER_LOOP:
	for {
		select {
		case <-pool.worker_kill_pipe:
			// worker suicide
			break WORKER_LOOP
		case job := <-pool.job_pipe:
			pool.subworker(job)
			pool.done_pipe <- job
		}
	}
	pool.worker_wg.Done()
}

// NewPool creates a new Pool.
func NewPool(workers int) (pool *Pool) {
	pool = new(Pool)
	pool.num_workers = workers
	pool.job_pipe = make(chan *Job)
	pool.done_pipe = make(chan *Job)
	pool.add_pipe = make(chan *Job)
	pool.result_pipe = make(chan *Job)
	pool.jobs_ready_to_run = make([]*Job, 0)
	pool.jobs_completed = make([]*Job, 0)
	pool.working_pipe = make(chan bool)
	pool.stats_pipe = make(chan stats)
	pool.worker_kill_pipe = make(chan bool)
	pool.supervisor_kill_pipe = make(chan bool)
	pool.interval = 1
	// start the supervisor here so we can accept jobs before a Run call
	pool.startSupervisor()
	return
}

// supervisor feeds jobs to workers and keeps track of them.
func (pool *Pool) supervisor() {
SUPERVISOR_LOOP:
	for {
		select {
		case job := <-pool.add_pipe:
			pool.jobs_ready_to_run = append(pool.jobs_ready_to_run, job)
			pool.num_jobs_submitted++
		default:
		}

		num_ready_jobs := len(pool.jobs_ready_to_run)
		if num_ready_jobs > 0 {
			select {
			case pool.job_pipe <- pool.jobs_ready_to_run[num_ready_jobs-1]:
				pool.num_jobs_running++
				pool.jobs_ready_to_run = pool.jobs_ready_to_run[:num_ready_jobs-1]
			default:
			}
		}

		if pool.num_jobs_running > 0 {
			select {
			case job := <-pool.done_pipe:
				pool.num_jobs_running--
				pool.jobs_completed = append(pool.jobs_completed, job)
				pool.num_jobs_completed++
			default:
			}
		}

		working := true
		if len(pool.jobs_ready_to_run) == 0 && pool.num_jobs_running == 0 {
			working = false
		}
		select {
		case pool.working_pipe <- working:
		default:
		}

		res := (*Job)(nil)
		if len(pool.jobs_completed) > 0 {
			res = pool.jobs_completed[0]
		}
		select {
		case pool.result_pipe <- res:
			if len(pool.jobs_completed) > 0 {
				pool.jobs_completed = pool.jobs_completed[1:]
			}
		default:
		}

		pool_stats := stats{pool.num_jobs_submitted, pool.num_jobs_running, pool.num_jobs_completed}
		select {
		case pool.stats_pipe <- pool_stats:
		default:
		}

		// stopping
		select {
		case <-pool.supervisor_kill_pipe:
			break SUPERVISOR_LOOP
		default:
		}

		time.Sleep(pool.interval * time.Millisecond)
	}
	pool.supervisor_wg.Done()
}

// Run starts the Pool by launching the workers and a supervisor goroutine.
// It's OK to start an empty Pool. The jobs will be fed to the workers as soon
// as they become available.
func (pool *Pool) Run() {
	if pool.workers_started {
		panic("trying to start a pool that's already running")
	}
	for i := 0; i < pool.num_workers; i++ {
		pool.worker_wg.Add(1)
		go pool.worker(i)
	}
	pool.workers_started = true
	// handle the supervisor
	if !pool.supervisor_started {
		pool.startSupervisor()
	}
}

// Stop will signal the workers to exit and wait for them to actually do that.
func (pool *Pool) Stop() {
	if !pool.workers_started {
		panic("trying to stop a pool that's already stopped")
	}
	// stop the workers
	for i := 0; i < pool.num_workers; i++ {
		pool.worker_kill_pipe <- true
	}
	pool.worker_wg.Wait()
	// set the flag
	pool.workers_started = false
	// handle the supervisor
	if pool.supervisor_started {
		pool.stopSupervisor()
	}
}

func (pool *Pool) startSupervisor() {
	pool.supervisor_wg.Add(1)
	go pool.supervisor()
	pool.supervisor_started = true
}

func (pool *Pool) stopSupervisor() {
	pool.supervisor_kill_pipe <- true
	pool.supervisor_wg.Wait()
	pool.supervisor_started = false
}

// Add creates a Job from the given function and args and
// adds it to the Pool.
func (pool *Pool) Add(f func(...interface{}) interface{}, args ...interface{}) {
	pool.add_pipe <- &Job{f, args, nil, nil}
}

// Wait blocks until all the jobs in the Pool are done.
func (pool *Pool) Wait() {
	for <-pool.working_pipe {
		time.Sleep(pool.interval * time.Millisecond)
	}
}

// Results retrieves the completed jobs.
func (pool *Pool) Results() (res []*Job) {
	res = make([]*Job, len(pool.jobs_completed))
	for i, job := range pool.jobs_completed {
		res[i] = job
	}
	pool.jobs_completed = pool.jobs_completed[0:0]
	return
}

// WaitForJob blocks until a completed job is available and returns it.
// If there are no jobs running, it returns nil.
func (pool *Pool) WaitForJob() *Job {
	for {
		working := <-pool.working_pipe
		r := <-pool.result_pipe
		if r == (*Job)(nil) {
			if !working {
				break
			}
		} else {
			return r
		}
	}
	return nil
}

// Status returns a "stats" instance.
func (pool *Pool) Status() stats {
	if pool.supervisor_started {
		return <-pool.stats_pipe
	}
	// the supervisor wasn't started so we return a zeroed structure
	return stats{}
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"errors"
	mysql "github.com/go-sql-driver/mysql"
	"time"
)

type Post struct {
	id                int
	slug, title, body string
	author            User
	date              time.Time
}

// Fetches data from database by ID or slug (whichever one is available)
// and updates structure
func (p *Post) Fetch() error {
	var data *sql.Row

	switch {
	case p.id != 0:
		data = p.byId(p.id)
	case p.slug != "":
		data = p.bySlug(p.slug)
	default:
		return errors.New("Must provide ID or slug for fetching")
	}

	if err := p.update(data); err != nil {
		return errors.New("Error scanning row")
	}
	return nil
}

// Query DB by ID
func (p *Post) byId(id int) *sql.Row {
	return db.QueryRow("SELECT title, body, idUser, date FROM posts WHERE idPost=?", id)
}

// Query DB by slug
func (p *Post) bySlug(slug string) *sql.Row {
	return db.QueryRow("SELECT title, body, idUser, date FROM posts WHERE slug=?", slug)
}

// Scans a fetched row and updates the structure
func (p *Post) update(data *sql.Row) error {
	date := new(mysql.NullTime)
	author := new(User)
	err := data.Scan(&p.title, &p.body, &author.id, date)

	if err == sql.ErrNoRows || err != nil {
		return errors.New("Post not found")
	}

	if err := author.Fetch(); err != nil {
		return err
	}

	if date.Valid {
		p.date = date.Time
	}

	p.author = *author
	return nil
}
<commit_msg>Beautiful<commit_after>package main

import (
	"database/sql"
	"errors"
	mysql "github.com/go-sql-driver/mysql"
	"time"
)

type Post struct {
	id                int
	slug, title, body string
	author            User
	date              time.Time
}

// Fetches data from database by ID or slug (whichever one is available)
// and updates structure
func (p *Post) Fetch() error {
	var data *sql.Row

	switch {
	case p.id != 0:
		data = p.byId(p.id)
	case p.slug != "":
		data = p.bySlug(p.slug)
	default:
		return errors.New("Must provide ID or slug for fetching")
	}

	if err := p.update(data); err != nil {
		return errors.New("Error scanning row")
	}
	return nil
}

// Query DB by ID
func (p *Post) byId(id int) *sql.Row {
	return db.QueryRow("SELECT title, body, idUser, date FROM posts WHERE idPost=?", id)
}

// Query DB by slug
func (p *Post) bySlug(slug string) *sql.Row {
	return db.QueryRow("SELECT title, body, idUser, date FROM posts WHERE slug=?", slug)
}

// Scans a fetched row and updates the structure
func (p *Post) update(data *sql.Row) error {
	date := new(mysql.NullTime)
	author := new(User)

	err := data.Scan(&p.title, &p.body, &author.id, date)
	if err == sql.ErrNoRows || err != nil {
		return errors.New("Post not found")
	}

	if date.Valid {
		p.date = date.Time
	}

	err = author.Fetch()
	if err != nil {
		return err
	}

	p.author = *author
	return nil
}
<|endoftext|>


<commit_before>package main

// qotd.txt source: ftp://ftp.mrynet.com/USENIX/85.1/langston/sun/TODAY/qotd.txt

import (
    "io/ioutil"
    "flag"
    "log"
    "net"
    "strings"
    "time"
)

type Quotes map[string]string

func (file *Quotes) Parse(qfile string) {
    bytes, err := ioutil.ReadFile(qfile)
    if err != nil {
        log.Fatal("Error:", err.Error())
    }

    content := strings.Replace(string(bytes), "}\n", "}", -1)
    fields := strings.FieldsFunc(content, brace)

    for i, field := range fields {
        if len(field) == 4 {
            (*file)[field] = fields[i+1]
        }
    }
}

func brace(r rune) bool {
    return r == '{' || r == '}'
}

// --------------------------------------------------------------------------------

func qotd(conn net.Conn) {
    today := time.Now().Format("0102")
    buf := []byte(quotes[today] + "\n")

    // RFC 865 (https://tools.ietf.org/html/rfc865) states that the quote should
    // be less than 512 characters
    if len(buf) > 512 {
        buf = buf[:512]
    }

    _, err := conn.Write(buf)
    if err != nil {
        log.Println("Error send reply:", err.Error())
    }

    defer conn.Close()
}

// --------------------------------------------------------------------------------

var quotes Quotes
var qfile = flag.String("file", "qotd.txt", "The QOTD file")

func main() {
    flag.Parse()

    quotes = make(Quotes)
    quotes.Parse(*qfile)

    listener, err := net.Listen("tcp", ":17")
    if err != nil {
		log.Fatal("Error listening:", err.Error())
	}

    for {
		conn, err := listener.Accept()
		if err != nil {
			log.Println("Error accept:", err.Error())
            continue
		}
		go qotd(conn)
	}
}
<commit_msg>Ran qotd.go through go fmt<commit_after>package main

// qotd.txt source: ftp://ftp.mrynet.com/USENIX/85.1/langston/sun/TODAY/qotd.txt

import (
	"flag"
	"io/ioutil"
	"log"
	"net"
	"strings"
	"time"
)

type Quotes map[string]string

func (file *Quotes) Parse(qfile string) {
	bytes, err := ioutil.ReadFile(qfile)
	if err != nil {
		log.Fatal("Error:", err.Error())
	}

	content := strings.Replace(string(bytes), "}\n", "}", -1)
	fields := strings.FieldsFunc(content, brace)

	for i, field := range fields {
		if len(field) == 4 {
			(*file)[field] = fields[i+1]
		}
	}
}

func brace(r rune) bool {
	return r == '{' || r == '}'
}

// --------------------------------------------------------------------------------

func qotd(conn net.Conn) {
	today := time.Now().Format("0102")
	buf := []byte(quotes[today] + "\n")

	// RFC 865 (https://tools.ietf.org/html/rfc865) states that the quote should
	// be less than 512 characters
	if len(buf) > 512 {
		buf = buf[:512]
	}

	_, err := conn.Write(buf)
	if err != nil {
		log.Println("Error send reply:", err.Error())
	}

	defer conn.Close()
}

// --------------------------------------------------------------------------------

var quotes Quotes
var qfile = flag.String("file", "qotd.txt", "The QOTD file")

func main() {
	flag.Parse()

	quotes = make(Quotes)
	quotes.Parse(*qfile)

	listener, err := net.Listen("tcp", ":17")
	if err != nil {
		log.Fatal("Error listening:", err.Error())
	}

	for {
		conn, err := listener.Accept()
		if err != nil {
			log.Println("Error accept:", err.Error())
			continue
		}
		go qotd(conn)
	}
}
<|endoftext|>


<commit_before>package qset

import (
	"errors"
	"strconv"
	"sync"
	"time"

	"github.com/garyburd/redigo/redis"
	"github.com/kavehmz/crdt"
)

/*QSet is a race free implementation of what LWW can use as udnerlying set.
This implementation uses redis ZSET.
ZSET in redis uses scores to sort the elements. Score is a IEEE 754 floating point number,
that is able to represent precisely integer numbers between -(2^53) and +(2^53) included.
That is between -9007199254740992 and 9007199254740992.
This will limit this sets precision to save element's action timestamp to 1 milli-seconds.
Notice that time.Time precision is 1 nano-seconds by defaults. For this lack of precision all
timestamps are rounded to nearest microsecond.
Using redis can also cause latency cause by network or socket communication.
*/
type QSet struct {
	// Conn is the redis connection to be used.
	Conn redis.Conn
	// AddSet sets which key will be used in redis for the set.
	SetKey string
	// Marshal function needs to convert the lww.Element to string. Redis can only store and retrieve string values.
	Marshal func(lww.Element) string
	// UnMarshal function needs to be able to convert a Marshalled string back to a readable structure for consumer of library.
	UnMarshal func(string) lww.Element
	// LastState is an error type that will return the error state of last executed redis command. Add redis connection are not shareable this can be used after each command to know the last state.
	LastState error

	set lww.Set
	sync.RWMutex

	setChannel chan setData
}

type setData struct {
	element lww.Element
	ts      time.Time
}

func roundToMicro(t time.Time) int64 {
	return t.Round(time.Microsecond).UnixNano() / 1000
}

func (s *QSet) checkErr(err error) {
	if err != nil {
		s.LastState = err
		return
	}
	s.LastState = nil
}

//Init will do a one time setup for underlying set. It will be called from WLL.Init
func (s *QSet) Init() {
	if s.Conn == nil {
		s.checkErr(errors.New("Conn must be set"))
		return
	}
	if s.Marshal == nil {
		s.checkErr(errors.New("Marshal must be set"))
		return
	}
	if s.UnMarshal == nil {
		s.checkErr(errors.New("UnMarshal must be set"))
		return
	}
	if s.SetKey == "" {
		s.checkErr(errors.New("SetKey must be set"))
		return
	}
	_, err := s.Conn.Do("DEL", s.SetKey)
	s.checkErr(err)

	s.set.Init()
	s.readMembers()
	s.setChannel = make(chan setData, 10000)
	go func() {
		for {
			select {
			case d := <-s.setChannel:
				s.Lock()
				defer s.Unlock()
				_, err := s.Conn.Do("ZADD", s.SetKey, roundToMicro(d.ts), s.Marshal(d.element))
				s.checkErr(err)
			}
		}

	}()
}

func (s *QSet) readMembers() {
	zs, err := redis.Strings(s.Conn.Do("ZRANGE", s.SetKey, 0, -1, "WITHSCORES"))
	s.checkErr(err)
	for i := 0; i < len(zs); i += 2 {
		n, _ := strconv.Atoi(zs[i+1])
		s.set.Set(zs[i], time.Unix(0, 0).Add(time.Duration(n)*time.Microsecond))
	}
}

//Set adds an element to the set if it does not exists. It it exists Set will update the provided timestamp.
func (s *QSet) Set(e lww.Element, t time.Time) {
	s.set.Set(s.Marshal(e), t.Round(time.Microsecond))
	// s.setChannel <- setData{ts: t.Round(time.Microsecond), element: e}
	go func() {
		s.Lock()
		defer s.Unlock()
		_, err := s.Conn.Do("ZADD", s.SetKey, roundToMicro(t), s.Marshal(e))
		s.checkErr(err)
	}()
}

//Len must return the number of members in the set
func (s *QSet) Len() int {
	return s.set.Len()
}

//Get returns timestmap of the element in the set if it exists and true. Otherwise it will return an empty timestamp and false.
func (s *QSet) Get(e lww.Element) (time.Time, bool) {
	return s.set.Get(e)
}

//List returns list of all elements in the set
func (s *QSet) List() []lww.Element {
	var l []lww.Element
	for _, v := range s.set.List() {

		l = append(l, s.UnMarshal(v.(string)))
	}
	return l
}
<commit_msg>Using goroutine to add elements to Redis and removing the long running grouting concept.<commit_after>package qset

import (
	"errors"
	"strconv"
	"sync"
	"time"

	"github.com/garyburd/redigo/redis"
	"github.com/kavehmz/crdt"
)

/*QSet is a race free implementation of what LWW can use as udnerlying set.
This implementation uses redis ZSET.
ZSET in redis uses scores to sort the elements. Score is a IEEE 754 floating point number,
that is able to represent precisely integer numbers between -(2^53) and +(2^53) included.
That is between -9007199254740992 and 9007199254740992.
This will limit this sets precision to save element's action timestamp to 1 milli-seconds.
Notice that time.Time precision is 1 nano-seconds by defaults. For this lack of precision all
timestamps are rounded to nearest microsecond.
Using redis can also cause latency cause by network or socket communication.
*/
type QSet struct {
	// Conn is the redis connection to be used.
	Conn redis.Conn
	// AddSet sets which key will be used in redis for the set.
	SetKey string
	// Marshal function needs to convert the lww.Element to string. Redis can only store and retrieve string values.
	Marshal func(lww.Element) string
	// UnMarshal function needs to be able to convert a Marshalled string back to a readable structure for consumer of library.
	UnMarshal func(string) lww.Element
	// LastState is an error type that will return the error state of last executed redis command. Add redis connection are not shareable this can be used after each command to know the last state.
	LastState error

	set lww.Set
	sync.RWMutex
}

type setData struct {
	element lww.Element
	ts      time.Time
}

func roundToMicro(t time.Time) int64 {
	return t.Round(time.Microsecond).UnixNano() / 1000
}

func (s *QSet) checkErr(err error) {
	if err != nil {
		s.LastState = err
		return
	}
	s.LastState = nil
}

//Init will do a one time setup for underlying set. It will be called from WLL.Init
func (s *QSet) Init() {
	if s.Conn == nil {
		s.checkErr(errors.New("Conn must be set"))
		return
	}
	if s.Marshal == nil {
		s.checkErr(errors.New("Marshal must be set"))
		return
	}
	if s.UnMarshal == nil {
		s.checkErr(errors.New("UnMarshal must be set"))
		return
	}
	if s.SetKey == "" {
		s.checkErr(errors.New("SetKey must be set"))
		return
	}
	_, err := s.Conn.Do("DEL", s.SetKey)
	s.checkErr(err)

	s.set.Init()
	s.readMembers()
}

func (s *QSet) readMembers() {
	zs, err := redis.Strings(s.Conn.Do("ZRANGE", s.SetKey, 0, -1, "WITHSCORES"))
	s.checkErr(err)
	for i := 0; i < len(zs); i += 2 {
		n, _ := strconv.Atoi(zs[i+1])
		s.set.Set(zs[i], time.Unix(0, 0).Add(time.Duration(n)*time.Microsecond))
	}
}

//Set adds an element to the set if it does not exists. It it exists Set will update the provided timestamp.
func (s *QSet) Set(e lww.Element, t time.Time) {
	s.set.Set(s.Marshal(e), t.Round(time.Microsecond))

	go func() {
		s.Lock()
		defer s.Unlock()
		_, err := s.Conn.Do("ZADD", s.SetKey, roundToMicro(t), s.Marshal(e))
		s.checkErr(err)
	}()
}

//Len must return the number of members in the set
func (s *QSet) Len() int {
	return s.set.Len()
}

//Get returns timestmap of the element in the set if it exists and true. Otherwise it will return an empty timestamp and false.
func (s *QSet) Get(e lww.Element) (time.Time, bool) {
	return s.set.Get(e)
}

//List returns list of all elements in the set
func (s *QSet) List() []lww.Element {
	var l []lww.Element
	for _, v := range s.set.List() {

		l = append(l, s.UnMarshal(v.(string)))
	}
	return l
}
<|endoftext|>


<commit_before>// Copyright 2014 SteelSeries ApS.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package implements a basic LISP interpretor for embedding in a go program for scripting.
// This file provides a REPL.

package golisp

import (
    "fmt"
)

func Repl() {
    fmt.Printf("Welcome to GoLisp\n")
    fmt.Printf("Copyright 2013 SteelSeries\n")
    fmt.Printf("Evaluate '(quit)' to exit.\n\n")
    prompt := "> "
    LoadHistoryFromFile(".golisp_history")
    lastInput := ""
    for true {
        input := *ReadLine(&prompt)
        if input != "" {
            if input != lastInput {
                AddHistory(input)
            }
            lastInput = input
            code, err := Parse(input)
            if err != nil {
                fmt.Printf("Error: %s\n", err)
            } else {
                d, err := Eval(code, Global)
                if err != nil {
                    fmt.Printf("Error in evaluation: %s\n", err)
                } else {
                    fmt.Printf("==> %s\n", String(d))
                }
            }
        }
    }
}
<commit_msg>Cleanup formatting<commit_after>// Copyright 2014 SteelSeries ApS.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package implements a basic LISP interpretor for embedding in a go program for scripting.
// This file provides a REPL.

package golisp

import (
	"fmt"
)

func Repl() {
	fmt.Printf("Welcome to GoLisp\n")
	fmt.Printf("Copyright 2013 SteelSeries\n")
	fmt.Printf("Evaluate '(quit)' to exit.\n\n")
	prompt := "> "
	LoadHistoryFromFile(".golisp_history")
	lastInput := ""
	for true {
		input := *ReadLine(&prompt)
		if input != "" {
			if input != lastInput {
				AddHistory(input)
			}
			lastInput = input
			code, err := Parse(input)
			if err != nil {
				fmt.Printf("Error: %s\n", err)
			} else {
				d, err := Eval(code, Global)
				if err != nil {
					fmt.Printf("Error in evaluation: %s\n", err)
				} else {
					fmt.Printf("==> %s\n", String(d))
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/fatih/color"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
)

// Repo represents a repository of information yaml files.
type Repo struct {
	Key      string `yaml:"key"`
	Summary  string `yaml:"summary"`
	Alias    string `yaml:"alias"`
	Info     map[string]Info
	Control  map[string]Info
	Subrepos map[string]Repo
	Parent   *Repo
	root     string
	wg       sync.WaitGroup
}

func (r Repo) String() string {
	return fmt.Sprintf("R: %s (%d articles)", r.Key, len(r.Info))
}

// LoadRepos loads multiple repositories and stores them
func LoadRepos(p string) (repos map[string]Repo) {
	repos = make(map[string]Repo)
	p = getPath(p)
	wg := sync.WaitGroup{}

	files, _ := ioutil.ReadDir(p)
	for _, file := range files {
		fn := filepath.Join(p, file.Name())

		if _, err := os.Stat(filepath.Join(fn, "_repo.yaml")); os.IsNotExist(err) {
			log.Println(fmt.Sprintf("Skipping repo %s: no _repo.yaml found.", file.Name()))
			continue
		}

		wg.Add(1)
		go func() {
			defer wg.Done()
			repos[asKey(fn)] = NewRepo(fn)
		}()
	}

	wg.Wait()
	return
}

// UpdateRepos will run git pull on the repos
func UpdateRepos(repos map[string]Repo) {
	for key, repo := range repos {
		log.Printf("Updating %s...", key)
		repo.git("pull", "origin", "master")
	}
}

// AddRepo clones a new repository
func AddRepo(root, name, url string) {
	dir := filepath.Join(root, name)
	git("", "clone", url, dir)
	log.Print("Repository added!")
}

// NewRepo loads a repository on a path
func NewRepo(p string) (r Repo) {
	p = getPath(p)
	r = Repo{Key: asKey(p), root: p}

	// Check if this is a root repo. If it is, load the data from the _repo.yaml file into
	// the newly created repo.
	rfile := filepath.Join(p, "_repo.yaml")
	if _, err := os.Stat(rfile); !os.IsNotExist(err) {
		data, err := ioutil.ReadFile(rfile)

		if err != nil {
			log.Fatal("Reading repo file failed: ", p)
		}
		yaml.Unmarshal(data, &r)
	}

	r.Info = make(map[string]Info)
	r.Control = make(map[string]Info)
	r.Subrepos = make(map[string]Repo)

	filepath.Walk(r.root, r.walk)
	r.wg.Wait()

	return
}

// ListRepos prints a sorted list of available repostiories.
func ListRepos(repos map[string]Repo) {
	keys := make([]string, 0, len(repos))
	for key := range repos {
		keys = append(keys, key)
	}

	sort.Strings(keys)
	for _, key := range keys {
		fmt.Println(key)
	}
}

// Keys returns a sorted list of the info keys in the repository
func (r *Repo) Keys() []string {
	keys := make([]string, 0, len(r.Info))
	for _, info := range r.Info {
		keys = append(keys, info.ID)
	}

	sort.Strings(keys)

	return keys
}

// SubrepoKeys returns a sorted list of the subrepo keys in the repository
func (r *Repo) SubrepoKeys() []string {
	keys := make([]string, 0, len(r.Subrepos))
	for _, sub := range r.Subrepos {
		keys = append(keys, sub.Key)
	}

	sort.Strings(keys)

	return keys
}

// Execute determine what to do:
//
// If the last argument in the command line from c.Args() given points to a
// repo, the index of the loop will be printed.
//
// If the last argument is an Info item, it will be executed.
func (r *Repo) Execute(c *cli.Context) {
	var repo *Repo
	var remaining []string
	repo = r

	// The first argument is not needed since it was used to determine the
	// location to this very repo.
	args := c.Args()[1:]

	if len(args) != 0 {
		// No Info was found. Figure out if there is a subrepo in the query somewhere
		// TODO(thiderman): Handle if there are remaining left.
		repo, remaining, _ = r.GetSubrepo(args)

		if len(remaining) > 0 {
			// Try to find the Info. If it is found, execute it and don't do anything else.
			if info, ok := repo.Info[remaining[0]]; ok {
				info.Execute(repo, remaining[1:])
				return
			}
		}
	}

	// Nothing matched, or the end node is a repo. Print the contents,
	// beginning with a blue listing of subrepos
	blue := color.New(color.FgBlue, color.Bold).SprintfFunc()
	for _, key := range repo.SubrepoKeys() {
		fmt.Println(blue(key))
	}
	for _, key := range repo.Keys() {
		fmt.Println(key)
	}
}

// GetHost will return a Host as defined by the list of arguments
//
// `args` is to be a string containing space separated identifiers to find a
// host category.
func (r *Repo) GetHost(def string) (h *Host) {
	args := strings.Split(def, " ")
	if len(args) < 2 {
		log.Fatal("Too few identifiers in host string. Need at least 2.")
	}

	args = append([]string{"hosts"}, args...)

	info, remaining, err := r.GetInfo(args)
	if err != nil {
		log.Print(err)
		log.Fatal("No host could be found")
	}

	cat := info.Hosts[remaining[0]]
	return cat.PrimaryHost()
}

// GetInfo will return an Info as defined by the list of arguments
//
// If successful, a *Info is returned along with the remaining unparsed arguments.
func (r *Repo) GetInfo(args []string) (*Info, []string, error) {
	var info Info
	var ok bool

	repo, remaining, err := r.GetSubrepo(args)
	if err != nil {
		log.Print(repo)
		return nil, []string{}, errors.New(
			"No matching Info found because no subrepo matched the query.",
		)
	}

	if info, ok = repo.Info[remaining[0]]; ok {
		return &info, remaining[1:], nil
	}

	return nil, []string{}, errors.New("No matching Info found.")
}

// GetSubrepo will return an Info as defined by the list of arguments
//
// If successful, a *Repo is returned along with the remaining unparsed arguments.
func (r *Repo) GetSubrepo(args []string) (*Repo, []string, error) {
	var err error

	if len(args) == 0 {
		return r, args, nil
	}

	arg := args[0]
	if repo, ok := r.Subrepos[arg]; ok {
		return repo.GetSubrepo(args[1:])
	}

	if _, ok := r.Info[arg]; !ok {
		err = fmt.Errorf("Subrepo did not exist: %s", arg)
	}
	return r, args, err
}

// ParentRepo parses the repo tree upwards until it finds the root repository
//
// This is used by things like command execution, where the current repository would be
// `commands` or a subrepository, but the root is needed for host discovery.
func (r *Repo) ParentRepo() *Repo {
	if &r.Parent == nil {
		return r
	}
	return r.Parent
}

// MakeCLI generates a cli.Command chain based on the repository structure
func (r *Repo) MakeCLI() (c cli.Command) {
	c = cli.Command{
		Name:     r.Key,
		Usage:    r.Summary,
		HideHelp: true,
	}

	// Make a list of subcommands to add into the Command.
	subcommands := make([]cli.Command, 0, len(r.Info)+len(r.Subrepos))

	// Loop over the subrepositories first, making sure that they are on top.
	for _, key := range r.SubrepoKeys() {
		subrepo := r.Subrepos[key]
		subcommands = append(subcommands, subrepo.MakeCLI())
	}

	// Then loop the info files.
	for _, key := range r.Keys() {
		info := r.Info[key]

		sc := cli.Command{
			Name:     info.ID,
			Usage:    info.Summary,
			HideHelp: true,
			Action: func(c *cli.Context) {
				info.Execute(r, c.Args())
			},
		}

		if info.Type == "host" {
			sc.Subcommands = append(sc.Subcommands, MakeHostCLI(&info)...)
		} else if info.Type == "command" {
			sc.Subcommands = append(sc.Subcommands, MakeCommandCLI(&info)...)
		}

		subcommands = append(subcommands, sc)
	}

	c.Subcommands = subcommands

	return
}

func (r *Repo) walk(path string, info os.FileInfo, err error) error {
	if err != nil {
		log.Println("walk error: ", err)
		return err
	}

	// Dotfile, like .git or whatever. Skip.
	if strings.HasPrefix(filepath.Base(path), ".") {
		return filepath.SkipDir
	}

	if info.IsDir() && r.isSubrepo(path) {
		r.wg.Add(1)
		go r.loadSubrepo(path)

		// Return SkipDir since the directory will be parsed by the
		// NewRepo call inside of loadSubrepo()
		return filepath.SkipDir

	} else if strings.HasSuffix(path, ".yaml") {
		r.wg.Add(1)
		go r.loadInfo(path)
	}

	return nil
}

func (r *Repo) loadInfo(path string) {
	defer r.wg.Done()

	info, err := LoadInfo(r, path)
	if err != nil {
		log.Println("Failed to load info: ", err)
	}

	// Control files start with an underscore and should not be stored as
	// normal Info documents.
	if strings.HasPrefix(asKey(path), "_") {
		r.Control[info.ID] = info
	} else {
		r.Info[info.ID] = info
	}
}

func (r *Repo) loadSubrepo(path string) {
	defer r.wg.Done()
	nr := NewRepo(path)
	nr.Parent = r
	r.Subrepos[nr.Key] = nr
}

func (r *Repo) isSubrepo(path string) bool {
	// This is the root...
	if r.root == path {
		return false
	}

	return true
}

// Helper to run git commands inside of a repository
func (r *Repo) git(args ...string) {
	git(r.root, args...)
}
<commit_msg>Remove unusued execution function<commit_after>package main

import (
	"errors"
	"fmt"
	"github.com/codegangsta/cli"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
)

// Repo represents a repository of information yaml files.
type Repo struct {
	Key      string `yaml:"key"`
	Summary  string `yaml:"summary"`
	Alias    string `yaml:"alias"`
	Info     map[string]Info
	Control  map[string]Info
	Subrepos map[string]Repo
	Parent   *Repo
	root     string
	wg       sync.WaitGroup
}

func (r Repo) String() string {
	return fmt.Sprintf("R: %s (%d articles)", r.Key, len(r.Info))
}

// LoadRepos loads multiple repositories and stores them
func LoadRepos(p string) (repos map[string]Repo) {
	repos = make(map[string]Repo)
	p = getPath(p)
	wg := sync.WaitGroup{}

	files, _ := ioutil.ReadDir(p)
	for _, file := range files {
		fn := filepath.Join(p, file.Name())

		if _, err := os.Stat(filepath.Join(fn, "_repo.yaml")); os.IsNotExist(err) {
			log.Println(fmt.Sprintf("Skipping repo %s: no _repo.yaml found.", file.Name()))
			continue
		}

		wg.Add(1)
		go func() {
			defer wg.Done()
			repos[asKey(fn)] = NewRepo(fn)
		}()
	}

	wg.Wait()
	return
}

// UpdateRepos will run git pull on the repos
func UpdateRepos(repos map[string]Repo) {
	for key, repo := range repos {
		log.Printf("Updating %s...", key)
		repo.git("pull", "origin", "master")
	}
}

// AddRepo clones a new repository
func AddRepo(root, name, url string) {
	dir := filepath.Join(root, name)
	git("", "clone", url, dir)
	log.Print("Repository added!")
}

// NewRepo loads a repository on a path
func NewRepo(p string) (r Repo) {
	p = getPath(p)
	r = Repo{Key: asKey(p), root: p}

	// Check if this is a root repo. If it is, load the data from the _repo.yaml file into
	// the newly created repo.
	rfile := filepath.Join(p, "_repo.yaml")
	if _, err := os.Stat(rfile); !os.IsNotExist(err) {
		data, err := ioutil.ReadFile(rfile)

		if err != nil {
			log.Fatal("Reading repo file failed: ", p)
		}
		yaml.Unmarshal(data, &r)
	}

	r.Info = make(map[string]Info)
	r.Control = make(map[string]Info)
	r.Subrepos = make(map[string]Repo)

	filepath.Walk(r.root, r.walk)
	r.wg.Wait()

	return
}

// ListRepos prints a sorted list of available repostiories.
func ListRepos(repos map[string]Repo) {
	keys := make([]string, 0, len(repos))
	for key := range repos {
		keys = append(keys, key)
	}

	sort.Strings(keys)
	for _, key := range keys {
		fmt.Println(key)
	}
}

// Keys returns a sorted list of the info keys in the repository
func (r *Repo) Keys() []string {
	keys := make([]string, 0, len(r.Info))
	for _, info := range r.Info {
		keys = append(keys, info.ID)
	}

	sort.Strings(keys)

	return keys
}

// SubrepoKeys returns a sorted list of the subrepo keys in the repository
func (r *Repo) SubrepoKeys() []string {
	keys := make([]string, 0, len(r.Subrepos))
	for _, sub := range r.Subrepos {
		keys = append(keys, sub.Key)
	}

	sort.Strings(keys)

	return keys
}

// GetHost will return a Host as defined by the list of arguments
//
// `args` is to be a string containing space separated identifiers to find a
// host category.
func (r *Repo) GetHost(def string) (h *Host) {
	args := strings.Split(def, " ")
	if len(args) < 2 {
		log.Fatal("Too few identifiers in host string. Need at least 2.")
	}

	args = append([]string{"hosts"}, args...)

	info, remaining, err := r.GetInfo(args)
	if err != nil {
		log.Print(err)
		log.Fatal("No host could be found")
	}

	cat := info.Hosts[remaining[0]]
	return cat.PrimaryHost()
}

// GetInfo will return an Info as defined by the list of arguments
//
// If successful, a *Info is returned along with the remaining unparsed arguments.
func (r *Repo) GetInfo(args []string) (*Info, []string, error) {
	var info Info
	var ok bool

	repo, remaining, err := r.GetSubrepo(args)
	if err != nil {
		log.Print(repo)
		return nil, []string{}, errors.New(
			"No matching Info found because no subrepo matched the query.",
		)
	}

	if info, ok = repo.Info[remaining[0]]; ok {
		return &info, remaining[1:], nil
	}

	return nil, []string{}, errors.New("No matching Info found.")
}

// GetSubrepo will return an Info as defined by the list of arguments
//
// If successful, a *Repo is returned along with the remaining unparsed arguments.
func (r *Repo) GetSubrepo(args []string) (*Repo, []string, error) {
	var err error

	if len(args) == 0 {
		return r, args, nil
	}

	arg := args[0]
	if repo, ok := r.Subrepos[arg]; ok {
		return repo.GetSubrepo(args[1:])
	}

	if _, ok := r.Info[arg]; !ok {
		err = fmt.Errorf("Subrepo did not exist: %s", arg)
	}
	return r, args, err
}

// ParentRepo parses the repo tree upwards until it finds the root repository
//
// This is used by things like command execution, where the current repository would be
// `commands` or a subrepository, but the root is needed for host discovery.
func (r *Repo) ParentRepo() *Repo {
	if &r.Parent == nil {
		return r
	}
	return r.Parent
}

// MakeCLI generates a cli.Command chain based on the repository structure
func (r *Repo) MakeCLI() (c cli.Command) {
	c = cli.Command{
		Name:     r.Key,
		Usage:    r.Summary,
		HideHelp: true,
	}

	// Make a list of subcommands to add into the Command.
	subcommands := make([]cli.Command, 0, len(r.Info)+len(r.Subrepos))

	// Loop over the subrepositories first, making sure that they are on top.
	for _, key := range r.SubrepoKeys() {
		subrepo := r.Subrepos[key]
		subcommands = append(subcommands, subrepo.MakeCLI())
	}

	// Then loop the info files.
	for _, key := range r.Keys() {
		info := r.Info[key]

		sc := cli.Command{
			Name:     info.ID,
			Usage:    info.Summary,
			HideHelp: true,
			Action: func(c *cli.Context) {
				info.Execute(r, c.Args())
			},
		}

		if info.Type == "host" {
			sc.Subcommands = append(sc.Subcommands, MakeHostCLI(&info)...)
		} else if info.Type == "command" {
			sc.Subcommands = append(sc.Subcommands, MakeCommandCLI(&info)...)
		}

		subcommands = append(subcommands, sc)
	}

	c.Subcommands = subcommands

	return
}

func (r *Repo) walk(path string, info os.FileInfo, err error) error {
	if err != nil {
		log.Println("walk error: ", err)
		return err
	}

	// Dotfile, like .git or whatever. Skip.
	if strings.HasPrefix(filepath.Base(path), ".") {
		return filepath.SkipDir
	}

	if info.IsDir() && r.isSubrepo(path) {
		r.wg.Add(1)
		go r.loadSubrepo(path)

		// Return SkipDir since the directory will be parsed by the
		// NewRepo call inside of loadSubrepo()
		return filepath.SkipDir

	} else if strings.HasSuffix(path, ".yaml") {
		r.wg.Add(1)
		go r.loadInfo(path)
	}

	return nil
}

func (r *Repo) loadInfo(path string) {
	defer r.wg.Done()

	info, err := LoadInfo(r, path)
	if err != nil {
		log.Println("Failed to load info: ", err)
	}

	// Control files start with an underscore and should not be stored as
	// normal Info documents.
	if strings.HasPrefix(asKey(path), "_") {
		r.Control[info.ID] = info
	} else {
		r.Info[info.ID] = info
	}
}

func (r *Repo) loadSubrepo(path string) {
	defer r.wg.Done()
	nr := NewRepo(path)
	nr.Parent = r
	r.Subrepos[nr.Key] = nr
}

func (r *Repo) isSubrepo(path string) bool {
	// This is the root...
	if r.root == path {
		return false
	}

	return true
}

// Helper to run git commands inside of a repository
func (r *Repo) git(args ...string) {
	git(r.root, args...)
}
<|endoftext|>


<commit_before>package main

import (
    "database/sql"
    _ "github.com/go-sql-driver/mysql"
)

func DatabaseThing() bool {
    dbUser := ""
    _, err := sql.Open("mysql", dbUser)

    return err != nil
}

//
//var currentId int
//
//var todos Todos
//
//// Give us some seed data
//func init() {
//    RepoCreateTodo(Todo{Name: "Write presentation"})
//    RepoCreateTodo(Todo{Name: "Host meetup"})
//}
//
//func RepoFindTodo(id int) Todo {
//    for _, t := range todos {
//        if t.Id == id {
//            return t
//        }
//    }
//    // return empty Todo if not found
//    return Todo{}
//}
//
//func RepoCreateTodo(t Todo) Todo {
//    currentId += 1
//    t.Id = currentId
//    todos = append(todos, t)
//    return t
//}
//
//func RepoDestroyTodo(id int) error {
//    for i, t := range todos {
//        if t.Id == id {
//            todos = append(todos[:i], todos[i+1:]...)
//            return nil
//        }
//    }
//    return fmt.Errorf("Could not find Todo with id of %d to delete", id)
//}<commit_msg>Actually connect to the db correctly<commit_after>package main

import (
    "database/sql"
    "os"

    _ "github.com/go-sql-driver/mysql"
)

func DatabaseThing() bool {
    _, err := sql.Open("mysql", os.Getenv("CLEARDB_DATABASE_URL"))

    return err != nil
}

//
//var currentId int
//
//var todos Todos
//
//// Give us some seed data
//func init() {
//    RepoCreateTodo(Todo{Name: "Write presentation"})
//    RepoCreateTodo(Todo{Name: "Host meetup"})
//}
//
//func RepoFindTodo(id int) Todo {
//    for _, t := range todos {
//        if t.Id == id {
//            return t
//        }
//    }
//    // return empty Todo if not found
//    return Todo{}
//}
//
//func RepoCreateTodo(t Todo) Todo {
//    currentId += 1
//    t.Id = currentId
//    todos = append(todos, t)
//    return t
//}
//
//func RepoDestroyTodo(id int) error {
//    for i, t := range todos {
//        if t.Id == id {
//            todos = append(todos[:i], todos[i+1:]...)
//            return nil
//        }
//    }
//    return fmt.Errorf("Could not find Todo with id of %d to delete", id)
//}<|endoftext|>


<commit_before>package toystore

import (
	"bytes"
	"crypto/sha256"
	"errors"
	"fmt"
)

var Zero []byte = make([]byte, 256)

var Hash func([]byte) []byte = func(bytes []byte) []byte {
	hash := sha256.New()
	hash.Write(bytes)
	// fmt.Println(hash.Sum(nil))
	return hash.Sum(nil)
}

type Circle struct {
	address []byte
	hash    []byte
	next    *Circle
}

func (c *Circle) String() string {
	var buffer bytes.Buffer
	for current, first := c, true; len(current.hash) != 0 ||
		first; current, first = current.next, false {
		if !first {
			buffer.WriteString(" -> ")
		}
		buffer.Write(current.address)
		buffer.WriteString("/")
		buffer.Write(current.hash)
	}
	return buffer.String()
}

func (c *Circle) AddressList() []string {
	output := make([]string, 0)
	for current, first := c, true; len(current.hash) != 0 ||
		first; current, first = current.next, false {

		output = append(output, string(current.address))
	}
	return output
}

var (
	ReplicationDepth int = 1
)

func NewCircleHead() *Circle {
	circle := new(Circle)
	circle.hash = []byte{} // empty is head.
	// circle.address is undefined
	circle.next = circle
	return circle
}

func NewCircle(address []byte) *Circle {
	circle := new(Circle)
	circle.address = address
	circle.hash = Hash(address)
	return circle
}

func NewCircleString(address string) *Circle {
	return NewCircle([]byte(address))
}

func (c *Circle) Add(incoming *Circle) *Circle {
	var current *Circle
	for current = c; bytes.Compare(current.next.hash, incoming.hash) == -1; current = current.next {
		if bytes.Compare(current.next.hash, incoming.hash) == 0 {
			return nil // Don't do anything if there's already the circle.
		}
		if bytes.Compare(current.next.hash, nil) == 0 {
			break
		}
	}
	incoming.next = current.next
	current.next = incoming
	return incoming
}

func (c *Circle) AddString(address string) *Circle {
	return c.Add(NewCircleString(address))
}

func (c *Circle) RemoveString(address string) error {
	return c.Remove([]byte(address))
}

func (c *Circle) Remove(address []byte) error {
	var current *Circle
	var last *Circle
	for current, last = c.next, c; bytes.Compare(current.address, address) != 0; current, last = current.next, current {
		if string(current.hash) == "" {
			return errors.New(fmt.Sprintf("No such node in circle: %s\n", address))
		}
	}
	// log.Printf("Remove %s, %s -> %s", string(current.address), string(last.address), string(current.next.address))
	last.next = current.next // I think this will be gc'd
	return nil
}

func CircleFromList(strs []string) *Circle {
	circle := NewCircleHead()
	for _, str := range strs {
		circle.AddString(str)
	}
	return circle
}

// Will loop forever with an empty node...
func (c *Circle) KeyAddress(key []byte) func() ([]byte, error) {
	hashed := Hash(key)

	current := c.find(hashed)

	if bytes.Compare(current.hash, nil) == 0 {
		// If we reached the end, just go one step further to loop around.
		current = current.next
	}

	i := 0
	return func() ([]byte, error) {
		output := current.address
		i++

		if i > ReplicationDepth {
			return []byte{}, errors.New("No more replications.")
		}

		current = current.next
		if bytes.Compare(current.hash, nil) == 0 {
			current = current.next
		}

		return output, nil
	}
}

func (c *Circle) find(address []byte) *Circle {
	var current *Circle
	for current = c.next; bytes.Compare(current.hash, nil) != 0 &&
		bytes.Compare(current.address, address) == -1; current = current.next {
	}
	return current
}

func (c *Circle) Adjacent(first []byte, second []byte) bool {
	next := c.find(first).next
	if next.address == nil {
		next = next.next // ignore the head.
	}
	return bytes.Compare(next.address, second) == 0
}
<commit_msg>Remove superflous comments<commit_after>package toystore

import (
	"bytes"
	"crypto/sha256"
	"errors"
	"fmt"
)

var Zero []byte = make([]byte, 256)

var Hash func([]byte) []byte = func(bytes []byte) []byte {
	hash := sha256.New()
	hash.Write(bytes)
	return hash.Sum(nil)
}

type Circle struct {
	address []byte
	hash    []byte
	next    *Circle
}

func (c *Circle) String() string {
	var buffer bytes.Buffer
	for current, first := c, true; len(current.hash) != 0 ||
		first; current, first = current.next, false {
		if !first {
			buffer.WriteString(" -> ")
		}
		buffer.Write(current.address)
		buffer.WriteString("/")
		buffer.Write(current.hash)
	}
	return buffer.String()
}

func (c *Circle) AddressList() []string {
	output := make([]string, 0)
	for current, first := c, true; len(current.hash) != 0 ||
		first; current, first = current.next, false {

		output = append(output, string(current.address))
	}
	return output
}

var (
	ReplicationDepth int = 1
)

func NewCircleHead() *Circle {
	circle := new(Circle)
	circle.hash = []byte{} // empty is head.
	circle.next = circle
	return circle
}

func NewCircle(address []byte) *Circle {
	circle := new(Circle)
	circle.address = address
	circle.hash = Hash(address)
	return circle
}

func NewCircleString(address string) *Circle {
	return NewCircle([]byte(address))
}

func (c *Circle) Add(incoming *Circle) *Circle {
	var current *Circle
	for current = c; bytes.Compare(current.next.hash, incoming.hash) == -1; current = current.next {
		if bytes.Compare(current.next.hash, incoming.hash) == 0 {
			return nil
		}
		if bytes.Compare(current.next.hash, nil) == 0 {
			break
		}
	}
	incoming.next = current.next
	current.next = incoming
	return incoming
}

func (c *Circle) AddString(address string) *Circle {
	return c.Add(NewCircleString(address))
}

func (c *Circle) RemoveString(address string) error {
	return c.Remove([]byte(address))
}

func (c *Circle) Remove(address []byte) error {
	var current *Circle
	var last *Circle
	for current, last = c.next, c; bytes.Compare(current.address, address) != 0; current, last = current.next, current {
		if string(current.hash) == "" {
			return errors.New(fmt.Sprintf("No such node in circle: %s\n", address))
		}
	}
	last.next = current.next
	return nil
}

func CircleFromList(strs []string) *Circle {
	circle := NewCircleHead()
	for _, str := range strs {
		circle.AddString(str)
	}
	return circle
}

func (c *Circle) KeyAddress(key []byte) func() ([]byte, error) {
	hashed := Hash(key)

	current := c.find(hashed)

	if bytes.Compare(current.hash, nil) == 0 {
		current = current.next
	}

	i := 0
	return func() ([]byte, error) {
		output := current.address
		i++

		if i > ReplicationDepth {
			return []byte{}, errors.New("No more replications.")
		}

		current = current.next
		if bytes.Compare(current.hash, nil) == 0 {
			current = current.next
		}

		return output, nil
	}
}

func (c *Circle) find(address []byte) *Circle {
	var current *Circle
	for current = c.next; bytes.Compare(current.hash, nil) != 0 &&
		bytes.Compare(current.address, address) == -1; current = current.next {
	}
	return current
}

func (c *Circle) Adjacent(first []byte, second []byte) bool {
	next := c.find(first).next
	if next.address == nil {
		next = next.next
	}
	return bytes.Compare(next.address, second) == 0
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/hex"
	"fmt"
	"github.com/andrewtj/dnssd"
	"log"
	"net"
	"time"
)

type ConnState int

const (
	StateNew ConnState = iota
	StateActive
	StateIdle
	StateHijacked
	StateClosed
)

//starts the ROAP service
func startROAP(hardwareAddr net.HardwareAddr, hostName string) {

	port := 5000
	name := fmt.Sprintf("%s@%s", hex.EncodeToString(hardwareAddr), hostName)
	op := dnssd.NewRegisterOp(name, "_raop._tcp", port, RegisterROAPCallbackFunc)

	op.SetTXTPair("txtvers", "1")
	op.SetTXTPair("ch", "2")
	op.SetTXTPair("cn", "0,1")
	op.SetTXTPair("et", "0,1")
	op.SetTXTPair("sv", "false")
	op.SetTXTPair("da", "true")
	op.SetTXTPair("sr", "44100")
	op.SetTXTPair("ss", "16")
	op.SetTXTPair("pw", "false")
	op.SetTXTPair("vn", "3")
	op.SetTXTPair("tp", "TCP,UDP")
	op.SetTXTPair("md", "0,1,2")
	op.SetTXTPair("vs", "130.14")
	op.SetTXTPair("sm", "false")
	op.SetTXTPair("ek", "1")
	err := op.Start()
	if err != nil {
		log.Printf("Failed to register ROAP service: %s", err)
		return
	}
	log.Println("started ROAP service")
	go startROAPWebServer(port)
	// later...
	//op.Stop()
}

//helper method for the ROAP service
func RegisterROAPCallbackFunc(op *dnssd.RegisterOp, err error, add bool, name, serviceType, domain string) {
	if err != nil {
		// op is now inactive
		log.Printf("ROAP Service registration failed: %s", err)
		return
	}
	if add {
		log.Printf("ROAP Service registered as “%s“ in %s", name, domain)
	} else {
		log.Printf("ROAP Service “%s” removed from %s", name, domain)
	}
}

func startROAPWebServer(port int) error {
	ln, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
	if err != nil {
		log.Println("error starting ROAP server:", err)
		return err
	}
	defer ln.Close()
	var tempDelay time.Duration // how long to sleep on accept failure
	for {
		rw, e := ln.Accept()
		if e != nil {
			if ne, ok := e.(net.Error); ok && ne.Temporary() {
				if tempDelay == 0 {
					tempDelay = 5 * time.Millisecond
				} else {
					tempDelay *= 2
				}
				if max := 1 * time.Second; tempDelay > max {
					tempDelay = max
				}
				log.Printf("ROAP: Accept error: %v; retrying in %v", e, tempDelay)
				time.Sleep(tempDelay)
				continue
			}
			return e
		}
		tempDelay = 0
		log.Println("got a connection from: ", rw.RemoteAddr())
		//need to setup a connection object that handles the connection
		//then figure out how to handle the RTSP protocol from the data returned.

		// c, err := newConn(rw)
		// if err != nil {
		// 	continue
		// }
		// c.setState(c.rwc, StateNew) // before Serve can return
		// go c.serve()
	}
}

// func newConn(rwc net.Conn) (c *conn) {
// 	c = new(conn)
// 	c.remoteAddr = rwc.RemoteAddr().String()
// 	c.server = srv
// 	c.rwc = rwc
// 	c.sr = liveSwitchReader{r: c.rwc}
// 	c.lr = io.LimitReader(&c.sr, noLimit).(*io.LimitedReader)
// 	br := newBufioReader(c.lr)
// 	bw := newBufioWriterSize(c.rwc, 4<<10)
// 	c.buf = bufio.NewReadWriter(br, bw)
// 	return c
// }
<commit_msg>starting on RTSP<commit_after>package main

import (
	"encoding/hex"
	"fmt"
	"github.com/andrewtj/dnssd"
	"log"
	"net"
	"time"
)

//starts the ROAP service
func startROAP(hardwareAddr net.HardwareAddr, hostName string) {

	port := 5000
	name := fmt.Sprintf("%s@%s", hex.EncodeToString(hardwareAddr), hostName)
	op := dnssd.NewRegisterOp(name, "_raop._tcp", port, RegisterROAPCallbackFunc)

	op.SetTXTPair("txtvers", "1")
	op.SetTXTPair("ch", "2")
	op.SetTXTPair("cn", "0,1")
	op.SetTXTPair("et", "0,1")
	op.SetTXTPair("sv", "false")
	op.SetTXTPair("da", "true")
	op.SetTXTPair("sr", "44100")
	op.SetTXTPair("ss", "16")
	op.SetTXTPair("pw", "false")
	op.SetTXTPair("vn", "3")
	op.SetTXTPair("tp", "TCP,UDP")
	op.SetTXTPair("md", "0,1,2")
	op.SetTXTPair("vs", "130.14")
	op.SetTXTPair("sm", "false")
	op.SetTXTPair("ek", "1")
	err := op.Start()
	if err != nil {
		log.Printf("Failed to register ROAP service: %s", err)
		return
	}
	log.Println("started ROAP service")
	go startROAPWebServer(port)
	// later...
	//op.Stop()
}

//helper method for the ROAP service
func RegisterROAPCallbackFunc(op *dnssd.RegisterOp, err error, add bool, name, serviceType, domain string) {
	if err != nil {
		// op is now inactive
		log.Printf("ROAP Service registration failed: %s", err)
		return
	}
	if add {
		log.Printf("ROAP Service registered as “%s“ in %s", name, domain)
	} else {
		log.Printf("ROAP Service “%s” removed from %s", name, domain)
	}
}

func startROAPWebServer(port int) error {
	ln, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
	if err != nil {
		log.Println("error starting ROAP server:", err)
		return err
	}
	defer ln.Close()
	var tempDelay time.Duration // how long to sleep on accept failure
	for {
		rw, e := ln.Accept()
		if e != nil {
			if ne, ok := e.(net.Error); ok && ne.Temporary() {
				if tempDelay == 0 {
					tempDelay = 5 * time.Millisecond
				} else {
					tempDelay *= 2
				}
				if max := 1 * time.Second; tempDelay > max {
					tempDelay = max
				}
				log.Printf("ROAP: Accept error: %v; retrying in %v", e, tempDelay)
				time.Sleep(tempDelay)
				continue
			}
			return e
		}
		tempDelay = 0
		//setup a connection object that handles the connection
		//this handles the RTSP protocol from interaction from here.
		c := newConn(rw)
		//c.setState(c.rwc, StateNew) // before Serve can return
		go c.serve()
	}
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"net/http"
	"os"

	"github.com/auth0/go-jwt-middleware"
	"github.com/dgrijalva/jwt-go"
	"github.com/gorilla/mux"
	// postgres driver for migrate
	_ "github.com/mattes/migrate/driver/postgres"
	"github.com/nuveo/prest/config"
	"github.com/nuveo/prest/controllers"
	"github.com/spf13/cobra"
	"github.com/urfave/negroni"
)

var cfgFile string
var prestConfig config.Prest

// RootCmd represents the base command when called without any subcommands
var RootCmd = &cobra.Command{
	Use:   "prest",
	Short: "Serve a RESTful API from any PostgreSQL database",
	Long:  `Serve a RESTful API from any PostgreSQL database, start HTTP server`,
	Run: func(cmd *cobra.Command, args []string) {
		app()
	},
}

// Execute adds all child commands to the root command sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}
}

func init() {
	prestConfig = config.Prest{}
	config.Parse(&prestConfig)
}

func app() {
	cfg := config.Prest{}
	config.Parse(&cfg)

	n := negroni.Classic()
	n.Use(negroni.HandlerFunc(handlerSet))
	if cfg.JWTKey != "" {
		n.Use(jwtMiddleware(cfg.JWTKey))
	}
	r := mux.NewRouter()
	r.HandleFunc("/databases", controllers.GetDatabases).Methods("GET")
	r.HandleFunc("/schemas", controllers.GetSchemas).Methods("GET")
	r.HandleFunc("/tables", controllers.GetTables).Methods("GET")
	r.HandleFunc("/{database}/{schema}", controllers.GetTablesByDatabaseAndSchema).Methods("GET")
	r.HandleFunc("/{database}/{schema}/{table}", controllers.SelectFromTables).Methods("GET")
	r.HandleFunc("/{database}/{schema}/{table}", controllers.InsertInTables).Methods("POST")
	r.HandleFunc("/{database}/{schema}/{table}", controllers.DeleteFromTable).Methods("DELETE")
	r.HandleFunc("/{database}/{schema}/{table}", controllers.UpdateTable).Methods("PUT", "PATCH")

	n.UseHandler(r)
	n.Run(fmt.Sprintf(":%v", cfg.HTTPPort))
}

func handlerSet(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
	w.Header().Set("Content-Type", "application/json")
	next(w, r)
}

func jwtMiddleware(key string) negroni.Handler {
	jwtMiddleware := jwtmiddleware.New(jwtmiddleware.Options{
		ValidationKeyGetter: func(token *jwt.Token) (interface{}, error) {
			return []byte(key), nil
		},
		SigningMethod: jwt.SigningMethodHS256,
	})
	return negroni.HandlerFunc(jwtMiddleware.HandlerWithNext)
}
<commit_msg>initial access permissions - WIP<commit_after>package cmd

import (
	"fmt"
	"net/http"
	"os"

	"github.com/auth0/go-jwt-middleware"
	"github.com/dgrijalva/jwt-go"
	"github.com/gorilla/mux"
	// postgres driver for migrate
	_ "github.com/mattes/migrate/driver/postgres"
	"github.com/nuveo/prest/config"
	"github.com/nuveo/prest/controllers"
	"github.com/spf13/cobra"
	"github.com/urfave/negroni"
)

var cfgFile string
var prestConfig config.Prest

// RootCmd represents the base command when called without any subcommands
var RootCmd = &cobra.Command{
	Use:   "prest",
	Short: "Serve a RESTful API from any PostgreSQL database",
	Long:  `Serve a RESTful API from any PostgreSQL database, start HTTP server`,
	Run: func(cmd *cobra.Command, args []string) {
		app()
	},
}

// Execute adds all child commands to the root command sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}
}

func init() {
	prestConfig = config.Prest{}
	config.Parse(&prestConfig)
	config.PREST_CONF = &prestConfig

	if !prestConfig.AccessConf.Restrict {
		fmt.Println("You are running pREST in public mode.")
	}
}

func app() {
	cfg := config.Prest{}
	config.Parse(&cfg)

	n := negroni.Classic()
	n.Use(negroni.HandlerFunc(handlerSet))
	if cfg.JWTKey != "" {
		n.Use(jwtMiddleware(cfg.JWTKey))
	}
	r := mux.NewRouter()
	r.HandleFunc("/databases", controllers.GetDatabases).Methods("GET")
	r.HandleFunc("/schemas", controllers.GetSchemas).Methods("GET")
	r.HandleFunc("/tables", controllers.GetTables).Methods("GET")
	r.HandleFunc("/{database}/{schema}", controllers.GetTablesByDatabaseAndSchema).Methods("GET")
	r.HandleFunc("/{database}/{schema}/{table}", controllers.SelectFromTables).Methods("GET")
	r.HandleFunc("/{database}/{schema}/{table}", controllers.InsertInTables).Methods("POST")
	r.HandleFunc("/{database}/{schema}/{table}", controllers.DeleteFromTable).Methods("DELETE")
	r.HandleFunc("/{database}/{schema}/{table}", controllers.UpdateTable).Methods("PUT", "PATCH")

	n.UseHandler(r)
	n.Run(fmt.Sprintf(":%v", cfg.HTTPPort))
}

func handlerSet(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
	w.Header().Set("Content-Type", "application/json")
	next(w, r)
}

func jwtMiddleware(key string) negroni.Handler {
	jwtMiddleware := jwtmiddleware.New(jwtmiddleware.Options{
		ValidationKeyGetter: func(token *jwt.Token) (interface{}, error) {
			return []byte(key), nil
		},
		SigningMethod: jwt.SigningMethodHS256,
	})
	return negroni.HandlerFunc(jwtMiddleware.HandlerWithNext)
}
<|endoftext|>


<commit_before>package main

import (
    "fmt"
    "os"
    "os/exec"
    "io/ioutil"
    "path"
    "strings"
    "flag"
    "strconv"
)

/*
TODO:
- add commandline args
- convert ag's output to sack_shortcuts format
- add -e arg for exec'ing vim w/ appropriate sack_shortcuts format args
*/

var home string               = os.Getenv("HOME")
var searchPath string         = path.Join(home, ".zsh.d/")
var flagEdit = flag.Bool("edit", false, "zoom to edit this file")

const agCmd string            = "ag"
const flags string            = "-i"
const searchTerm string       = "ruby"
const shortcutFilename string = ".sack_shortcuts"

func check(e error){
    if e != nil {
        panic(e)
    }
}

func content() []string {
    filePath   := path.Join(home, shortcutFilename)
    dat, err   := ioutil.ReadFile(filePath)
    check(err)
    lines      := strings.Split(string(dat), "\n")
    return lines
}

func splitLine(s string) []string{
    arr := strings.Split(s, ":")
    return arr
}

func executeCmd() []string {
    cmd, err := exec.Command(agCmd, flags, searchTerm, searchPath).Output()
    check(err)
    lines := strings.Split(string(cmd), "\n")
    return lines
}

func search() {
    lines     := executeCmd()
    // firstLine := lines[0]
    // lineArr   := splitLine(firstLine)
    // fmt.Println(strings.Join(lineArr, "---"))
    fmt.Println(strings.Join(lines, "\n"))
}

func edit(s string){
    lines     := content()
    fmt.Println("Index entry: ", s)

    ind, err := strconv.Atoi(s)
    check(err)

    selectedLine := lines[ind]
    lineArr   := strings.Split(selectedLine, " ")
    fmt.Println(strings.Join(lineArr, "---"))
}

func setup() []string {
    flag.Parse()
    var args []string = flag.Args()
    return args
}

func checkState() { }

func main() {
    checkState()
    args := setup()
    if *flagEdit {
        edit(args[0])
    } else {
        search()
    }
}
<commit_msg>Gofmt'd<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"strconv"
	"strings"
)

/*
TODO:
- add commandline args
- convert ag's output to sack_shortcuts format
- add -e arg for exec'ing vim w/ appropriate sack_shortcuts format args
*/

var home string = os.Getenv("HOME")
var searchPath string = path.Join(home, ".zsh.d/")
var flagEdit = flag.Bool("edit", false, "zoom to edit this file")

const agCmd string = "ag"
const flags string = "-i"
const searchTerm string = "ruby"
const shortcutFilename string = ".sack_shortcuts"

func check(e error) {
	if e != nil {
		panic(e)
	}
}

func content() []string {
	filePath := path.Join(home, shortcutFilename)
	dat, err := ioutil.ReadFile(filePath)
	check(err)
	lines := strings.Split(string(dat), "\n")
	return lines
}

func splitLine(s string) []string {
	arr := strings.Split(s, ":")
	return arr
}

func executeCmd() []string {
	cmd, err := exec.Command(agCmd, flags, searchTerm, searchPath).Output()
	check(err)
	lines := strings.Split(string(cmd), "\n")
	return lines
}

func search() {
	lines := executeCmd()
	// firstLine := lines[0]
	// lineArr   := splitLine(firstLine)
	// fmt.Println(strings.Join(lineArr, "---"))
	fmt.Println(strings.Join(lines, "\n"))
}

func edit(s string) {
	lines := content()
	fmt.Println("Index entry: ", s)

	ind, err := strconv.Atoi(s)
	check(err)

	selectedLine := lines[ind]
	lineArr := strings.Split(selectedLine, " ")
	fmt.Println(strings.Join(lineArr, "---"))
}

func setup() []string {
	flag.Parse()
	var args []string = flag.Args()
	return args
}

func checkState() {}

func main() {
	checkState()
	args := setup()
	if *flagEdit {
		edit(args[0])
	} else {
		search()
	}
}
<|endoftext|>