<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"regexp"
)

func GetPage(c string) []byte {
	url := fmt.Sprintf("http://www.nciku.com/search/all/%v", c)
	client := &http.Client{}
	req, err := http.NewRequest("GET", url, nil)
	req.Header.Add("User-agent", "Mozilla/5.0")
	resp, err := client.Do(req)
	if err != nil {
		log.Fatal(err)
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal(err)
	}
	return body
}

func SearchForID(c string) string {
	search := fmt.Sprintf("(\\d+)\">%v", c)
	r, err := regexp.Compile(search)
	if err != nil {
		log.Fatal(err)
	}
	page := GetPage(c)
	match := r.Find(page)
	// taking off the last 5 chars, ie ">好 is 5 chars long
	// Chinese characters take up 3 bytes
	match = match[:(len(match) - 5)]
	return string(match)
}

func StrokeURL(c string) string {
	URL := fmt.Sprintf("http://images.nciku.com/stroke_order/%v.swf", SearchForID(c))
	return URL
}

func main() {
	// GetPage("好")
	fmt.Println(StrokeURL("好"))
}
<commit_msg>use constants for partial urls<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"regexp"
)

const searchurl string = "http://www.nciku.com/search/all/%v"
const swfurl string = "http://images.nciku.com/stroke_order/%v.swf"

func GetPage(c string) []byte {
	url := fmt.Sprintf(searchurl, c)
	client := &http.Client{}
	req, err := http.NewRequest("GET", url, nil)
	req.Header.Add("User-agent", "Mozilla/5.0")
	resp, err := client.Do(req)
	if err != nil {
		log.Fatal(err)
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal(err)
	}
	return body
}

func SearchForID(c string) string {
	search := fmt.Sprintf("(\\d+)\">%v", c)
	r, err := regexp.Compile(search)
	if err != nil {
		log.Fatal(err)
	}
	page := GetPage(c)
	match := r.Find(page)
	// taking off the last 5 chars, ie ">好 is 5 chars long
	// Chinese characters take up 3 bytes
	match = match[:(len(match) - 5)]
	return string(match)
}

func StrokeURL(c string) string {
	URL := fmt.Sprintf(swfurl, SearchForID(c))
	return URL
}

func main() {
	// GetPage("好")
	fmt.Println(StrokeURL("好"))
}
<|endoftext|>


<commit_before>// Copyright 2014 The intelengine Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"sync"

	"github.com/jroimartin/rpcmq"
)

const (
	sep = '|'
)

type worker struct {
	cfg    config
	server *rpcmq.Server

	mu       sync.RWMutex
	commands map[string]*command
}

func newWorker(cfg config) *worker {
	return &worker{cfg: cfg}
}

func (w *worker) start() error {
	if w.cfg.Worker.CmdDir == "" || w.cfg.Broker.URI == "" || w.cfg.Broker.Queue == "" {
		return errors.New("missing configuration parameters")
	}

	var tlsConfig *tls.Config
	if w.cfg.Broker.CAFile != "" && w.cfg.Broker.CertFile != "" &&
		w.cfg.Broker.KeyFile != "" {
		cert, err := tls.LoadX509KeyPair(w.cfg.Broker.CertFile, w.cfg.Broker.KeyFile)
		if err != nil {
			return fmt.Errorf("LoadX509KeyPair: %v", err)
		}
		caCert, err := ioutil.ReadFile(w.cfg.Broker.CAFile)
		if err != nil {
			return fmt.Errorf("ReadFile: %v", err)
		}
		caCertPool := x509.NewCertPool()
		caCertPool.AppendCertsFromPEM(caCert)
		tlsConfig = &tls.Config{
			Certificates: []tls.Certificate{cert},
			RootCAs:      caCertPool,
		}
	}
	w.server = rpcmq.NewServer(w.cfg.Broker.URI, w.cfg.Broker.Queue,
		w.cfg.Broker.Exchange, "direct")
	w.server.TLSConfig = tlsConfig
	if err := w.server.Init(); err != nil {
		return fmt.Errorf("Init: %v", err)
	}
	defer w.server.Shutdown()

	w.refreshCommands()
	if err := w.server.Register("listCommands", w.listCommands); err != nil {
		return err
	}
	if err := w.server.Register("execCommand", w.execCommand); err != nil {
		return err
	}

	select {}
}

func (w *worker) listCommands(id string, data []byte) ([]byte, error) {
	if err := w.refreshCommands(); err != nil {
		return nil, fmt.Errorf("cannot refresh commands: %v", err)
	}

	w.mu.RLock()
	defer w.mu.RUnlock()

	b, err := json.Marshal(w.commands)
	if err != nil {
		return nil, fmt.Errorf("cannot marshal commands: %v", err)
	}
	return b, nil
}

func (w *worker) refreshCommands() error {
	w.mu.Lock()
	defer w.mu.Unlock()

	w.commands = map[string]*command{}
	if err := filepath.Walk(w.cfg.Worker.CmdDir, w.handleFile); err != nil {
		return err
	}
	return nil
}

func (w *worker) handleFile(filepath string, info os.FileInfo, err error) error {
	if err != nil {
		return err
	}

	if info.IsDir() || path.Ext(info.Name()) != cmdExt {
		return nil
	}

	cmd, err := newCommand(filepath)
	if err != nil {
		log.Printf("handleFile warning (%v): %v\n", info.Name(), err)
		return nil
	}

	w.commands[cmd.Name] = cmd
	log.Println("command registered:", cmd.Name)
	return nil
}

func (w *worker) execCommand(id string, data []byte) ([]byte, error) {
	w.mu.RLock()
	defer w.mu.RUnlock()

	sepIdx := bytes.IndexByte(data, sep)
	if sepIdx < 0 {
		return nil, errors.New("separator not found")
	}
	name := string(data[:sepIdx])
	br := bytes.NewReader(data[sepIdx+1:])

	cmd := w.command(name)
	if cmd == nil {
		return nil, fmt.Errorf("command not found: %v", name)
	}

	out, err := cmd.exec(br)
	if err != nil {
		return nil, fmt.Errorf("command execution error: %v", err)
	}

	return out, nil
}

func (w *worker) command(name string) *command {
	w.mu.RLock()
	defer w.mu.RUnlock()

	if cmd, ok := w.commands[name]; ok {
		return cmd
	}
	return nil
}
<commit_msg>intelworker: Register methods before rpcmq init<commit_after>// Copyright 2014 The intelengine Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"sync"

	"github.com/jroimartin/rpcmq"
)

const (
	sep = '|'
)

type worker struct {
	cfg    config
	server *rpcmq.Server

	mu       sync.RWMutex
	commands map[string]*command
}

func newWorker(cfg config) *worker {
	return &worker{cfg: cfg}
}

func (w *worker) start() error {
	if w.cfg.Worker.CmdDir == "" || w.cfg.Broker.URI == "" || w.cfg.Broker.Queue == "" {
		return errors.New("missing configuration parameters")
	}

	var tlsConfig *tls.Config
	if w.cfg.Broker.CAFile != "" && w.cfg.Broker.CertFile != "" &&
		w.cfg.Broker.KeyFile != "" {
		cert, err := tls.LoadX509KeyPair(w.cfg.Broker.CertFile, w.cfg.Broker.KeyFile)
		if err != nil {
			return fmt.Errorf("LoadX509KeyPair: %v", err)
		}
		caCert, err := ioutil.ReadFile(w.cfg.Broker.CAFile)
		if err != nil {
			return fmt.Errorf("ReadFile: %v", err)
		}
		caCertPool := x509.NewCertPool()
		caCertPool.AppendCertsFromPEM(caCert)
		tlsConfig = &tls.Config{
			Certificates: []tls.Certificate{cert},
			RootCAs:      caCertPool,
		}
	}
	w.server = rpcmq.NewServer(w.cfg.Broker.URI, w.cfg.Broker.Queue,
		w.cfg.Broker.Exchange, "direct")
	w.server.TLSConfig = tlsConfig
	if err := w.server.Register("listCommands", w.listCommands); err != nil {
		return err
	}
	if err := w.server.Register("execCommand", w.execCommand); err != nil {
		return err
	}
	if err := w.server.Init(); err != nil {
		return fmt.Errorf("Init: %v", err)
	}
	defer w.server.Shutdown()

	w.refreshCommands()
	select {}
}

func (w *worker) listCommands(id string, data []byte) ([]byte, error) {
	if err := w.refreshCommands(); err != nil {
		return nil, fmt.Errorf("cannot refresh commands: %v", err)
	}

	w.mu.RLock()
	defer w.mu.RUnlock()

	b, err := json.Marshal(w.commands)
	if err != nil {
		return nil, fmt.Errorf("cannot marshal commands: %v", err)
	}
	return b, nil
}

func (w *worker) refreshCommands() error {
	w.mu.Lock()
	defer w.mu.Unlock()

	w.commands = map[string]*command{}
	if err := filepath.Walk(w.cfg.Worker.CmdDir, w.handleFile); err != nil {
		return err
	}
	return nil
}

func (w *worker) handleFile(filepath string, info os.FileInfo, err error) error {
	if err != nil {
		return err
	}

	if info.IsDir() || path.Ext(info.Name()) != cmdExt {
		return nil
	}

	cmd, err := newCommand(filepath)
	if err != nil {
		log.Printf("handleFile warning (%v): %v\n", info.Name(), err)
		return nil
	}

	w.commands[cmd.Name] = cmd
	log.Println("command registered:", cmd.Name)
	return nil
}

func (w *worker) execCommand(id string, data []byte) ([]byte, error) {
	w.mu.RLock()
	defer w.mu.RUnlock()

	sepIdx := bytes.IndexByte(data, sep)
	if sepIdx < 0 {
		return nil, errors.New("separator not found")
	}
	name := string(data[:sepIdx])
	br := bytes.NewReader(data[sepIdx+1:])

	cmd := w.command(name)
	if cmd == nil {
		return nil, fmt.Errorf("command not found: %v", name)
	}

	out, err := cmd.exec(br)
	if err != nil {
		return nil, fmt.Errorf("command execution error: %v", err)
	}

	return out, nil
}

func (w *worker) command(name string) *command {
	w.mu.RLock()
	defer w.mu.RUnlock()

	if cmd, ok := w.commands[name]; ok {
		return cmd
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"

	"launchpad.net/gnuflag"
	"launchpad.net/juju-core/juju/cmd"
	"launchpad.net/juju-core/juju/environs"
	"launchpad.net/juju-core/juju/log"
	"launchpad.net/juju-core/juju/state"
	"launchpad.net/tomb"

	// register providers
	_ "launchpad.net/juju-core/juju/environs/dummy"
	_ "launchpad.net/juju-core/juju/environs/ec2"
)

// ProvisioningAgent is a cmd.Command responsible for running a provisioning agent.
type ProvisioningAgent struct {
	Conf AgentConf
}

// Info returns usage information for the command.
func (a *ProvisioningAgent) Info() *cmd.Info {
	return &cmd.Info{"provisioning", "", "run a juju provisioning agent", ""}
}

// Init initializes the command for running.
func (a *ProvisioningAgent) Init(f *gnuflag.FlagSet, args []string) error {
	a.Conf.addFlags(f)
	if err := f.Parse(true, args); err != nil {
		return err
	}
	return a.Conf.checkArgs(f.Args())
}

// Run runs a provisioning agent.
func (a *ProvisioningAgent) Run(_ *cmd.Context) error {
	// TODO(dfc) place the logic in a loop with a suitable delay
	p, err := NewProvisioner(&a.Conf.StateInfo)
	if err != nil {
		return err
	}
	return p.Wait()
}

type Provisioner struct {
	st      *state.State
	info    *state.Info
	environ environs.Environ
	tomb    tomb.Tomb

	environWatcher  *state.ConfigWatcher
	machinesWatcher *state.MachinesWatcher

	// TODO(dfc) machineId should be a uint
	machineIdToInstance map[int]environs.Instance
}

// NewProvisioner returns a Provisioner.
func NewProvisioner(info *state.Info) (*Provisioner, error) {
	st, err := state.Open(info)
	if err != nil {
		return nil, err
	}
	p := &Provisioner{
		st:                  st,
		info:                info,
		machineIdToInstance: make(map[int]environs.Instance),
	}
	go p.loop()
	return p, nil
}

func (p *Provisioner) loop() {
	defer p.tomb.Done()
	defer p.st.Close()
	p.environWatcher = p.st.WatchEnvironConfig()
	for {
		select {
		case <-p.tomb.Dying():
			return
		case config, ok := <-p.environWatcher.Changes():
			if !ok {
				err := p.environWatcher.Stop()
				if err != nil {
					p.tomb.Kill(err)
				}
				return
			}
			var err error
			p.environ, err = environs.NewEnviron(config.Map())
			if err != nil {
				log.Printf("provisioner loaded invalid environment configuration: %v", err)
				continue
			}
			log.Printf("provisioner loaded new environment configuration")
			p.innerLoop()
		}
	}
}

func (p *Provisioner) innerLoop() {
	p.machinesWatcher = p.st.WatchMachines()
	for {
		select {
		case <-p.tomb.Dying():
			return
		case change, ok := <-p.environWatcher.Changes():
			if !ok {
				err := p.environWatcher.Stop()
				if err != nil {
					p.tomb.Kill(err)
				}
				return
			}
			config, err := environs.NewConfig(change.Map())
			if err != nil {
				log.Printf("provisioner loaded invalid environment configuration: %v", err)
				continue
			}
			p.environ.SetConfig(config)
			log.Printf("provisioner loaded new environment configuration")
		case machines, ok := <-p.machinesWatcher.Changes():
			if !ok {
				err := p.machinesWatcher.Stop()
				if err != nil {
					p.tomb.Kill(err)
				}
				return
			}
			if err := p.processMachines(machines); err != nil {
				p.tomb.Kill(err)
			}
		}
	}
}

// Wait waits for the Provisioner to exit.
func (p *Provisioner) Wait() error {
	return p.tomb.Wait()
}

// Stop stops the Provisioner and returns any error encountered while
// provisioning.
func (p *Provisioner) Stop() error {
	p.tomb.Kill(nil)
	return p.tomb.Wait()
}

func (p *Provisioner) processMachines(changes *state.MachinesChange) error {
	// step 1. find which of the added machines have not
	// yet been allocated a running instance.
	notrunning, err := p.findNotRunning(changes.Added)
	if err != nil {
		return err
	}

	// step 2. start an instance for any machines we found.
	if _, err := p.startMachines(notrunning); err != nil {
		return err
	}

	// step 3. stop all unknown machines and the machines that were removed
	// from the state
	stopping, err := p.instancesForMachines(changes.Deleted)
	if err != nil {
		return err
	}

	// TODO(dfc) obtain a list of running instances from the Environ and compare that
	// with the known instances stored in the machine.InstanceId() config.

	// although calling StopInstance with an empty slice should produce no change in the
	// provider, environs like dummy do not consider this a noop.
	if len(stopping) > 0 {
		return p.environ.StopInstances(stopping)
	}
	return nil
}

// findNotRunning fins machines without an InstanceId set, these are defined as not running.
func (p *Provisioner) findNotRunning(machines []*state.Machine) ([]*state.Machine, error) {
	var notrunning []*state.Machine
	for _, m := range machines {
		id, err := m.InstanceId()
		if err != nil {
			return nil, err
		}
		if id == "" {
			notrunning = append(notrunning, m)
		} else {
			log.Printf("machine %s already running as instance %q", m, id)
		}
	}
	return notrunning, nil
}

func (p *Provisioner) startMachines(machines []*state.Machine) ([]*state.Machine, error) {
	var started []*state.Machine
	for _, m := range machines {
		if err := p.startMachine(m); err != nil {
			return nil, err
		}
		log.Printf("starting machine %v", m)
		started = append(started, m)
	}
	return started, nil
}

func (p *Provisioner) startMachine(m *state.Machine) error {
	// TODO(dfc) the state.Info passed to environ.StartInstance remains contentious
	// however as the PA only knows one state.Info, and that info is used by MAs and
	// UAs to locate the ZK for this environment, it is logical to use the same
	// state.Info as the PA.
	inst, err := p.environ.StartInstance(m.Id(), p.info)
	if err != nil {
		return err
	}

	// assign the instance id to the machine
	if err := m.SetInstanceId(inst.Id()); err != nil {
		return fmt.Errorf("unable to store instance id for machine %v: %v", m, err)
	}

	// populate the local caches
	p.machineIdToInstance[m.Id()] = inst
	return nil
}

// instanceForMachine returns the environs.Instance that represents this machines' running
// instance.
func (p *Provisioner) instanceForMachine(m *state.Machine) (environs.Instance, error) {
	inst, ok := p.machineIdToInstance[m.Id()]
	if !ok {
		// not cached locally, ask the environ.
		id, err := m.InstanceId()
		if err != nil {
			return nil, err
		}
		if id == "" {
			// nobody knows about this machine, give up.
			return nil, fmt.Errorf("machine %s not found", m)
		}
		insts, err := p.environ.Instances([]string{id})
		if err != nil {
			// the provider doesn't know about this instance, give up.
			return nil, err
		}
		inst = insts[0]
	}
	return inst, nil
}

// instancesForMachines returns a list of environs.Instance that represent the list of machines running
// in the provider.
func (p *Provisioner) instancesForMachines(machines []*state.Machine) ([]environs.Instance, error) {
	var insts []environs.Instance
	for _, m := range machines {
		inst, err := p.instanceForMachine(m)
		if err != nil {
			return nil, err
		}
		insts = append(insts, inst)
	}
	return insts, nil
}
<commit_msg>fix comment<commit_after>package main

import (
	"fmt"

	"launchpad.net/gnuflag"
	"launchpad.net/juju-core/juju/cmd"
	"launchpad.net/juju-core/juju/environs"
	"launchpad.net/juju-core/juju/log"
	"launchpad.net/juju-core/juju/state"
	"launchpad.net/tomb"

	// register providers
	_ "launchpad.net/juju-core/juju/environs/dummy"
	_ "launchpad.net/juju-core/juju/environs/ec2"
)

// ProvisioningAgent is a cmd.Command responsible for running a provisioning agent.
type ProvisioningAgent struct {
	Conf AgentConf
}

// Info returns usage information for the command.
func (a *ProvisioningAgent) Info() *cmd.Info {
	return &cmd.Info{"provisioning", "", "run a juju provisioning agent", ""}
}

// Init initializes the command for running.
func (a *ProvisioningAgent) Init(f *gnuflag.FlagSet, args []string) error {
	a.Conf.addFlags(f)
	if err := f.Parse(true, args); err != nil {
		return err
	}
	return a.Conf.checkArgs(f.Args())
}

// Run runs a provisioning agent.
func (a *ProvisioningAgent) Run(_ *cmd.Context) error {
	// TODO(dfc) place the logic in a loop with a suitable delay
	p, err := NewProvisioner(&a.Conf.StateInfo)
	if err != nil {
		return err
	}
	return p.Wait()
}

type Provisioner struct {
	st      *state.State
	info    *state.Info
	environ environs.Environ
	tomb    tomb.Tomb

	environWatcher  *state.ConfigWatcher
	machinesWatcher *state.MachinesWatcher

	// TODO(dfc) machineId should be a uint
	machineIdToInstance map[int]environs.Instance
}

// NewProvisioner returns a Provisioner.
func NewProvisioner(info *state.Info) (*Provisioner, error) {
	st, err := state.Open(info)
	if err != nil {
		return nil, err
	}
	p := &Provisioner{
		st:                  st,
		info:                info,
		machineIdToInstance: make(map[int]environs.Instance),
	}
	go p.loop()
	return p, nil
}

func (p *Provisioner) loop() {
	defer p.tomb.Done()
	defer p.st.Close()
	p.environWatcher = p.st.WatchEnvironConfig()
	for {
		select {
		case <-p.tomb.Dying():
			return
		case config, ok := <-p.environWatcher.Changes():
			if !ok {
				err := p.environWatcher.Stop()
				if err != nil {
					p.tomb.Kill(err)
				}
				return
			}
			var err error
			p.environ, err = environs.NewEnviron(config.Map())
			if err != nil {
				log.Printf("provisioner loaded invalid environment configuration: %v", err)
				continue
			}
			log.Printf("provisioner loaded new environment configuration")
			p.innerLoop()
		}
	}
}

func (p *Provisioner) innerLoop() {
	p.machinesWatcher = p.st.WatchMachines()
	for {
		select {
		case <-p.tomb.Dying():
			return
		case change, ok := <-p.environWatcher.Changes():
			if !ok {
				err := p.environWatcher.Stop()
				if err != nil {
					p.tomb.Kill(err)
				}
				return
			}
			config, err := environs.NewConfig(change.Map())
			if err != nil {
				log.Printf("provisioner loaded invalid environment configuration: %v", err)
				continue
			}
			p.environ.SetConfig(config)
			log.Printf("provisioner loaded new environment configuration")
		case machines, ok := <-p.machinesWatcher.Changes():
			if !ok {
				err := p.machinesWatcher.Stop()
				if err != nil {
					p.tomb.Kill(err)
				}
				return
			}
			if err := p.processMachines(machines); err != nil {
				p.tomb.Kill(err)
			}
		}
	}
}

// Wait waits for the Provisioner to exit.
func (p *Provisioner) Wait() error {
	return p.tomb.Wait()
}

// Stop stops the Provisioner and returns any error encountered while
// provisioning.
func (p *Provisioner) Stop() error {
	p.tomb.Kill(nil)
	return p.tomb.Wait()
}

func (p *Provisioner) processMachines(changes *state.MachinesChange) error {
	// step 1. find which of the added machines have not
	// yet been allocated a running instance.
	notrunning, err := p.findNotRunning(changes.Added)
	if err != nil {
		return err
	}

	// step 2. start an instance for any machines we found.
	if _, err := p.startMachines(notrunning); err != nil {
		return err
	}

	// step 3. stop all machines that were removed from the state.
	stopping, err := p.instancesForMachines(changes.Deleted)
	if err != nil {
		return err
	}

	// TODO(dfc) obtain a list of running instances from the Environ and compare that
	// with the known instances stored in the machine.InstanceId() config.

	// although calling StopInstance with an empty slice should produce no change in the
	// provider, environs like dummy do not consider this a noop.
	if len(stopping) > 0 {
		return p.environ.StopInstances(stopping)
	}
	return nil
}

// findNotRunning fins machines without an InstanceId set, these are defined as not running.
func (p *Provisioner) findNotRunning(machines []*state.Machine) ([]*state.Machine, error) {
	var notrunning []*state.Machine
	for _, m := range machines {
		id, err := m.InstanceId()
		if err != nil {
			return nil, err
		}
		if id == "" {
			notrunning = append(notrunning, m)
		} else {
			log.Printf("machine %s already running as instance %q", m, id)
		}
	}
	return notrunning, nil
}

func (p *Provisioner) startMachines(machines []*state.Machine) ([]*state.Machine, error) {
	var started []*state.Machine
	for _, m := range machines {
		if err := p.startMachine(m); err != nil {
			return nil, err
		}
		log.Printf("starting machine %v", m)
		started = append(started, m)
	}
	return started, nil
}

func (p *Provisioner) startMachine(m *state.Machine) error {
	// TODO(dfc) the state.Info passed to environ.StartInstance remains contentious
	// however as the PA only knows one state.Info, and that info is used by MAs and
	// UAs to locate the ZK for this environment, it is logical to use the same
	// state.Info as the PA.
	inst, err := p.environ.StartInstance(m.Id(), p.info)
	if err != nil {
		return err
	}

	// assign the instance id to the machine
	if err := m.SetInstanceId(inst.Id()); err != nil {
		return fmt.Errorf("unable to store instance id for machine %v: %v", m, err)
	}

	// populate the local caches
	p.machineIdToInstance[m.Id()] = inst
	return nil
}

// instanceForMachine returns the environs.Instance that represents this machines' running
// instance.
func (p *Provisioner) instanceForMachine(m *state.Machine) (environs.Instance, error) {
	inst, ok := p.machineIdToInstance[m.Id()]
	if !ok {
		// not cached locally, ask the environ.
		id, err := m.InstanceId()
		if err != nil {
			return nil, err
		}
		if id == "" {
			// nobody knows about this machine, give up.
			return nil, fmt.Errorf("machine %s not found", m)
		}
		insts, err := p.environ.Instances([]string{id})
		if err != nil {
			// the provider doesn't know about this instance, give up.
			return nil, err
		}
		inst = insts[0]
	}
	return inst, nil
}

// instancesForMachines returns a list of environs.Instance that represent the list of machines running
// in the provider.
func (p *Provisioner) instancesForMachines(machines []*state.Machine) ([]environs.Instance, error) {
	var insts []environs.Instance
	for _, m := range machines {
		inst, err := p.instanceForMachine(m)
		if err != nil {
			return nil, err
		}
		insts = append(insts, inst)
	}
	return insts, nil
}
<|endoftext|>


<commit_before>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package main

import (
	//"container/list"
	"fmt"
	"os"
	"os/exec"
	"path"
	"strconv"
	"strings"

	"github.com/codegangsta/cli"
	qlog "github.com/qiniu/log"

	//"github.com/gogits/git"
	"github.com/gogits/gogs/models"
	"github.com/gogits/gogs/modules/base"
)

var (
	COMMANDS_READONLY = map[string]int{
		"git-upload-pack":    models.AU_WRITABLE,
		"git upload-pack":    models.AU_WRITABLE,
		"git-upload-archive": models.AU_WRITABLE,
	}

	COMMANDS_WRITE = map[string]int{
		"git-receive-pack": models.AU_READABLE,
		"git receive-pack": models.AU_READABLE,
	}
)

var CmdServ = cli.Command{
	Name:  "serv",
	Usage: "This command just should be called by ssh shell",
	Description: `
gogs serv provide access auth for repositories`,
	Action: runServ,
	Flags:  []cli.Flag{},
}

func newLogger(execDir string) {
	logPath := execDir + "/log/serv.log"
	os.MkdirAll(path.Dir(logPath), os.ModePerm)

	f, err := os.OpenFile(logPath, os.O_WRONLY|os.O_APPEND|os.O_CREATE, os.ModePerm)
	if err != nil {
		qlog.Fatal(err)
	}

	qlog.SetOutput(f)
	qlog.Info("Start logging serv...")
}

func parseCmd(cmd string) (string, string) {
	ss := strings.SplitN(cmd, " ", 2)
	if len(ss) != 2 {
		return "", ""
	}

	verb, args := ss[0], ss[1]
	if verb == "git" {
		ss = strings.SplitN(args, " ", 2)
		args = ss[1]
		verb = fmt.Sprintf("%s %s", verb, ss[0])
	}
	return verb, args
}

func In(b string, sl map[string]int) bool {
	_, e := sl[b]
	return e
}

func runServ(k *cli.Context) {
	execDir, _ := base.ExecDir()
	newLogger(execDir)

	base.NewConfigContext()
	models.LoadModelsConfig()

	if models.UseSQLite3 {
		os.Chdir(execDir)
	}

	models.SetEngine()

	keys := strings.Split(os.Args[2], "-")
	if len(keys) != 2 {
		println("auth file format error")
		qlog.Fatal("auth file format error")
	}

	keyId, err := strconv.ParseInt(keys[1], 10, 64)
	if err != nil {
		println("auth file format error")
		qlog.Fatal("auth file format error", err)
	}
	user, err := models.GetUserByKeyId(keyId)
	if err != nil {
		println("You have no right to access")
		qlog.Fatalf("SSH visit error: %v", err)
	}

	cmd := os.Getenv("SSH_ORIGINAL_COMMAND")
	if cmd == "" {
		println("Hi", user.Name, "! You've successfully authenticated, but Gogs does not provide shell access.")
		return
	}

	verb, args := parseCmd(cmd)
	repoPath := strings.Trim(args, "'")
	rr := strings.SplitN(repoPath, "/", 2)
	if len(rr) != 2 {
		println("Unavilable repository", args)
		qlog.Fatalf("Unavilable repository %v", args)
	}
	repoUserName := rr[0]
	repoName := strings.TrimSuffix(rr[1], ".git")

	isWrite := In(verb, COMMANDS_WRITE)
	isRead := In(verb, COMMANDS_READONLY)

	repoUser, err := models.GetUserByName(repoUserName)
	if err != nil {
		println("You have no right to access")
		qlog.Fatal("Get user failed", err)
	}

	// access check
	switch {
	case isWrite:
		has, err := models.HasAccess(user.LowerName, path.Join(repoUserName, repoName), models.AU_WRITABLE)
		if err != nil {
			println("Inernel error:", err)
			qlog.Fatal(err)
		} else if !has {
			println("You have no right to write this repository")
			qlog.Fatalf("User %s has no right to write repository %s", user.Name, repoPath)
		}
	case isRead:
		repo, err := models.GetRepositoryByName(repoUser.Id, repoName)
		if err != nil {
			println("Get repository error:", err)
			qlog.Fatal("Get repository error: " + err.Error())
		}

		if !repo.IsPrivate {
			break
		}

		has, err := models.HasAccess(user.Name, path.Join(repoUserName, repoName), models.AU_READABLE)
		if err != nil {
			println("Inernel error")
			qlog.Fatal(err)
		}
		if !has {
			has, err = models.HasAccess(user.Name, repoPath, models.AU_WRITABLE)
			if err != nil {
				println("Inernel error")
				qlog.Fatal(err)
			}
		}
		if !has {
			println("You have no right to access this repository")
			qlog.Fatal("You have no right to access this repository")
		}
	default:
		println("Unknown command")
		qlog.Fatal("Unknown command")
	}

	models.SetRepoEnvs(user.Id, user.Name, repoName)

	gitcmd := exec.Command(verb, repoPath)
	gitcmd.Dir = base.RepoRootPath
	gitcmd.Stdout = os.Stdout
	gitcmd.Stdin = os.Stdin
	gitcmd.Stderr = os.Stderr

	if err = gitcmd.Run(); err != nil {
		println("execute command error:", err.Error())
		qlog.Fatal("execute command error: " + err.Error())
	}

	//refName := os.Getenv("refName")
	//oldCommitId := os.Getenv("oldCommitId")
	//newCommitId := os.Getenv("newCommitId")

	//qlog.Error("get envs:", refName, oldCommitId, newCommitId)

	// update
	//models.Update(refName, oldCommitId, newCommitId, repoUserName, repoName, user.Id)
}
<commit_msg>Typos<commit_after>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package main

import (
	//"container/list"
	"fmt"
	"os"
	"os/exec"
	"path"
	"strconv"
	"strings"

	"github.com/codegangsta/cli"
	qlog "github.com/qiniu/log"

	//"github.com/gogits/git"
	"github.com/gogits/gogs/models"
	"github.com/gogits/gogs/modules/base"
)

var (
	COMMANDS_READONLY = map[string]int{
		"git-upload-pack":    models.AU_WRITABLE,
		"git upload-pack":    models.AU_WRITABLE,
		"git-upload-archive": models.AU_WRITABLE,
	}

	COMMANDS_WRITE = map[string]int{
		"git-receive-pack": models.AU_READABLE,
		"git receive-pack": models.AU_READABLE,
	}
)

var CmdServ = cli.Command{
	Name:  "serv",
	Usage: "This command just should be called by ssh shell",
	Description: `
gogs serv provide access auth for repositories`,
	Action: runServ,
	Flags:  []cli.Flag{},
}

func newLogger(execDir string) {
	logPath := execDir + "/log/serv.log"
	os.MkdirAll(path.Dir(logPath), os.ModePerm)

	f, err := os.OpenFile(logPath, os.O_WRONLY|os.O_APPEND|os.O_CREATE, os.ModePerm)
	if err != nil {
		qlog.Fatal(err)
	}

	qlog.SetOutput(f)
	qlog.Info("Start logging serv...")
}

func parseCmd(cmd string) (string, string) {
	ss := strings.SplitN(cmd, " ", 2)
	if len(ss) != 2 {
		return "", ""
	}

	verb, args := ss[0], ss[1]
	if verb == "git" {
		ss = strings.SplitN(args, " ", 2)
		args = ss[1]
		verb = fmt.Sprintf("%s %s", verb, ss[0])
	}
	return verb, args
}

func In(b string, sl map[string]int) bool {
	_, e := sl[b]
	return e
}

func runServ(k *cli.Context) {
	execDir, _ := base.ExecDir()
	newLogger(execDir)

	base.NewConfigContext()
	models.LoadModelsConfig()

	if models.UseSQLite3 {
		os.Chdir(execDir)
	}

	models.SetEngine()

	keys := strings.Split(os.Args[2], "-")
	if len(keys) != 2 {
		println("auth file format error")
		qlog.Fatal("auth file format error")
	}

	keyId, err := strconv.ParseInt(keys[1], 10, 64)
	if err != nil {
		println("auth file format error")
		qlog.Fatal("auth file format error", err)
	}
	user, err := models.GetUserByKeyId(keyId)
	if err != nil {
		println("You have no right to access")
		qlog.Fatalf("SSH visit error: %v", err)
	}

	cmd := os.Getenv("SSH_ORIGINAL_COMMAND")
	if cmd == "" {
		println("Hi", user.Name, "! You've successfully authenticated, but Gogs does not provide shell access.")
		return
	}

	verb, args := parseCmd(cmd)
	repoPath := strings.Trim(args, "'")
	rr := strings.SplitN(repoPath, "/", 2)
	if len(rr) != 2 {
		println("Unavailable repository", args)
		qlog.Fatalf("Unavailable repository %v", args)
	}
	repoUserName := rr[0]
	repoName := strings.TrimSuffix(rr[1], ".git")

	isWrite := In(verb, COMMANDS_WRITE)
	isRead := In(verb, COMMANDS_READONLY)

	repoUser, err := models.GetUserByName(repoUserName)
	if err != nil {
		println("You have no right to access")
		qlog.Fatal("Get user failed", err)
	}

	// access check
	switch {
	case isWrite:
		has, err := models.HasAccess(user.LowerName, path.Join(repoUserName, repoName), models.AU_WRITABLE)
		if err != nil {
			println("Internal error:", err)
			qlog.Fatal(err)
		} else if !has {
			println("You have no right to write this repository")
			qlog.Fatalf("User %s has no right to write repository %s", user.Name, repoPath)
		}
	case isRead:
		repo, err := models.GetRepositoryByName(repoUser.Id, repoName)
		if err != nil {
			println("Get repository error:", err)
			qlog.Fatal("Get repository error: " + err.Error())
		}

		if !repo.IsPrivate {
			break
		}

		has, err := models.HasAccess(user.Name, path.Join(repoUserName, repoName), models.AU_READABLE)
		if err != nil {
			println("Internal error")
			qlog.Fatal(err)
		}
		if !has {
			has, err = models.HasAccess(user.Name, repoPath, models.AU_WRITABLE)
			if err != nil {
				println("Internal error")
				qlog.Fatal(err)
			}
		}
		if !has {
			println("You have no right to access this repository")
			qlog.Fatal("You have no right to access this repository")
		}
	default:
		println("Unknown command")
		qlog.Fatal("Unknown command")
	}

	models.SetRepoEnvs(user.Id, user.Name, repoName)

	gitcmd := exec.Command(verb, repoPath)
	gitcmd.Dir = base.RepoRootPath
	gitcmd.Stdout = os.Stdout
	gitcmd.Stdin = os.Stdin
	gitcmd.Stderr = os.Stderr

	if err = gitcmd.Run(); err != nil {
		println("execute command error:", err.Error())
		qlog.Fatal("execute command error: " + err.Error())
	}

	//refName := os.Getenv("refName")
	//oldCommitId := os.Getenv("oldCommitId")
	//newCommitId := os.Getenv("newCommitId")

	//qlog.Error("get envs:", refName, oldCommitId, newCommitId)

	// update
	//models.Update(refName, oldCommitId, newCommitId, repoUserName, repoName, user.Id)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"context"
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"net"
	"strconv"
	"strings"

	"github.com/lightningnetwork/lnd/lnrpc"
	"github.com/lightningnetwork/lnd/macaroons"
	"github.com/urfave/cli"
	"gopkg.in/macaroon.v2"
)

var bakeMacaroonCommand = cli.Command{
	Name:     "bakemacaroon",
	Category: "Macaroons",
	Usage: "Bakes a new macaroon with the provided list of permissions " +
		"and restrictions.",
	ArgsUsage: "[--save_to=] [--timeout=] [--ip_address=] permissions...",
	Description: `
	Bake a new macaroon that grants the provided permissions and
	optionally adds restrictions (timeout, IP address) to it.

	The new macaroon can either be shown on command line in hex serialized
	format or it can be saved directly to a file using the --save_to
	argument.

	A permission is a tuple of an entity and an action, separated by a
	colon. Multiple operations can be added as arguments, for example:

	lncli bakemacaroon info:read invoices:write foo:bar
	`,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name: "save_to",
			Usage: "save the created macaroon to this file " +
				"using the default binary format",
		},
		cli.Uint64Flag{
			Name: "timeout",
			Usage: "the number of seconds the macaroon will be " +
				"valid before it times out",
		},
		cli.StringFlag{
			Name:  "ip_address",
			Usage: "the IP address the macaroon will be bound to",
		},
		cli.Uint64Flag{
			Name:  "root_key_id",
			Usage: "the numerical root key ID used to create the macaroon",
		},
	},
	Action: actionDecorator(bakeMacaroon),
}

func bakeMacaroon(ctx *cli.Context) error {
	client, cleanUp := getClient(ctx)
	defer cleanUp()

	// Show command help if no arguments.
	if ctx.NArg() == 0 {
		return cli.ShowCommandHelp(ctx, "bakemacaroon")
	}
	args := ctx.Args()

	var (
		savePath          string
		timeout           int64
		ipAddress         net.IP
		rootKeyID         uint64
		parsedPermissions []*lnrpc.MacaroonPermission
		err               error
	)

	if ctx.String("save_to") != "" {
		savePath = cleanAndExpandPath(ctx.String("save_to"))
	}

	if ctx.IsSet("timeout") {
		timeout = ctx.Int64("timeout")
		if timeout <= 0 {
			return fmt.Errorf("timeout must be greater than 0")
		}
	}

	if ctx.IsSet("ip_address") {
		ipAddress = net.ParseIP(ctx.String("ip_address"))
		if ipAddress == nil {
			return fmt.Errorf("unable to parse ip_address: %s",
				ctx.String("ip_address"))
		}
	}

	if ctx.IsSet("root_key_id") {
		rootKeyID = ctx.Uint64("root_key_id")
	}

	// A command line argument can't be an empty string. So we'll check each
	// entry if it's a valid entity:action tuple. The content itself is
	// validated server side. We just make sure we can parse it correctly.
	for _, permission := range args {
		tuple := strings.Split(permission, ":")
		if len(tuple) != 2 {
			return fmt.Errorf("unable to parse "+
				"permission tuple: %s", permission)
		}
		entity, action := tuple[0], tuple[1]
		if entity == "" {
			return fmt.Errorf("invalid permission [%s]. entity "+
				"cannot be empty", permission)
		}
		if action == "" {
			return fmt.Errorf("invalid permission [%s]. action "+
				"cannot be empty", permission)
		}

		// No we can assume that we have a formally valid entity:action
		// tuple. The rest of the validation happens server side.
		parsedPermissions = append(
			parsedPermissions, &lnrpc.MacaroonPermission{
				Entity: entity,
				Action: action,
			},
		)
	}

	// Now we have gathered all the input we need and can do the actual
	// RPC call.
	req := &lnrpc.BakeMacaroonRequest{
		Permissions: parsedPermissions,
		RootKeyId:   rootKeyID,
	}
	resp, err := client.BakeMacaroon(context.Background(), req)
	if err != nil {
		return err
	}

	// Now we should have gotten a valid macaroon. Unmarshal it so we can
	// add first-party caveats (if necessary) to it.
	macBytes, err := hex.DecodeString(resp.Macaroon)
	if err != nil {
		return err
	}
	unmarshalMac := &macaroon.Macaroon{}
	if err = unmarshalMac.UnmarshalBinary(macBytes); err != nil {
		return err
	}

	// Now apply the desired constraints to the macaroon. This will always
	// create a new macaroon object, even if no constraints are added.
	macConstraints := make([]macaroons.Constraint, 0)
	if timeout > 0 {
		macConstraints = append(
			macConstraints, macaroons.TimeoutConstraint(timeout),
		)
	}
	if ipAddress != nil {
		macConstraints = append(
			macConstraints,
			macaroons.IPLockConstraint(ipAddress.String()),
		)
	}
	constrainedMac, err := macaroons.AddConstraints(
		unmarshalMac, macConstraints...,
	)
	if err != nil {
		return err
	}
	macBytes, err = constrainedMac.MarshalBinary()
	if err != nil {
		return err
	}

	// Now we can output the result. We either write it binary serialized to
	// a file or write to the standard output using hex encoding.
	switch {
	case savePath != "":
		err = ioutil.WriteFile(savePath, macBytes, 0644)
		if err != nil {
			return err
		}
		fmt.Printf("Macaroon saved to %s\n", savePath)

	default:
		fmt.Printf("%s\n", hex.EncodeToString(macBytes))
	}

	return nil
}

var listMacaroonIDsCommand = cli.Command{
	Name:     "listmacaroonids",
	Category: "Macaroons",
	Usage:    "List all macaroons root key IDs in use.",
	Action:   actionDecorator(listMacaroonIDs),
}

func listMacaroonIDs(ctx *cli.Context) error {
	client, cleanUp := getClient(ctx)
	defer cleanUp()

	req := &lnrpc.ListMacaroonIDsRequest{}
	resp, err := client.ListMacaroonIDs(context.Background(), req)
	if err != nil {
		return err
	}

	printRespJSON(resp)
	return nil
}

var deleteMacaroonIDCommand = cli.Command{
	Name:      "deletemacaroonid",
	Category:  "Macaroons",
	Usage:     "Delete a specific macaroon ID.",
	ArgsUsage: "root_key_id",
	Description: `
	Remove a macaroon ID using the specified root key ID. For example:

	lncli deletemacaroonid 1

	WARNING
	When the ID is deleted, all macaroons created from that root key will
	be invalidated.

	Note that the default root key ID 0 cannot be deleted.
	`,
	Action: actionDecorator(deleteMacaroonID),
}

func deleteMacaroonID(ctx *cli.Context) error {
	client, cleanUp := getClient(ctx)
	defer cleanUp()

	// Validate args length. Only one argument is allowed.
	if ctx.NArg() != 1 {
		return cli.ShowCommandHelp(ctx, "deletemacaroonid")
	}

	rootKeyIDString := ctx.Args().First()

	// Convert string into uint64.
	rootKeyID, err := strconv.ParseUint(rootKeyIDString, 10, 64)
	if err != nil {
		return fmt.Errorf("root key ID must be a positive integer")
	}

	// Check that the value is not equal to DefaultRootKeyID. Note that the
	// server also validates the root key ID when removing it. However, we check
	// it here too so that we can give users a nice warning.
	if bytes.Equal([]byte(rootKeyIDString), macaroons.DefaultRootKeyID) {
		return fmt.Errorf("deleting the default root key ID 0 is not allowed")
	}

	// Make the actual RPC call.
	req := &lnrpc.DeleteMacaroonIDRequest{
		RootKeyId: rootKeyID,
	}
	resp, err := client.DeleteMacaroonID(context.Background(), req)
	if err != nil {
		return err
	}

	printRespJSON(resp)
	return nil
}
<commit_msg>lncli: add new URI permissions to bakemacaroon<commit_after>package main

import (
	"bytes"
	"context"
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"net"
	"strconv"
	"strings"

	"github.com/lightningnetwork/lnd/lnrpc"
	"github.com/lightningnetwork/lnd/macaroons"
	"github.com/urfave/cli"
	"gopkg.in/macaroon.v2"
)

var bakeMacaroonCommand = cli.Command{
	Name:     "bakemacaroon",
	Category: "Macaroons",
	Usage: "Bakes a new macaroon with the provided list of permissions " +
		"and restrictions.",
	ArgsUsage: "[--save_to=] [--timeout=] [--ip_address=] permissions...",
	Description: `
	Bake a new macaroon that grants the provided permissions and
	optionally adds restrictions (timeout, IP address) to it.

	The new macaroon can either be shown on command line in hex serialized
	format or it can be saved directly to a file using the --save_to
	argument.

	A permission is a tuple of an entity and an action, separated by a
	colon. Multiple operations can be added as arguments, for example:

	lncli bakemacaroon info:read invoices:write foo:bar

	For even more fine-grained permission control, it is also possible to
	specify single RPC method URIs that are allowed to be accessed by a
	macaroon. This can be achieved by specifying "uri:<methodURI>" pairs,
	for example:

	lncli bakemacaroon uri:/lnrpc.Lightning/GetInfo uri:/verrpc.Versioner/GetVersion

	The macaroon created by this command would only be allowed to use the
	"lncli getinfo" and "lncli version" commands.

	To get a list of all available URIs and permissions, use the
	"lncli listpermissions" command.
	`,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name: "save_to",
			Usage: "save the created macaroon to this file " +
				"using the default binary format",
		},
		cli.Uint64Flag{
			Name: "timeout",
			Usage: "the number of seconds the macaroon will be " +
				"valid before it times out",
		},
		cli.StringFlag{
			Name:  "ip_address",
			Usage: "the IP address the macaroon will be bound to",
		},
		cli.Uint64Flag{
			Name:  "root_key_id",
			Usage: "the numerical root key ID used to create the macaroon",
		},
	},
	Action: actionDecorator(bakeMacaroon),
}

func bakeMacaroon(ctx *cli.Context) error {
	client, cleanUp := getClient(ctx)
	defer cleanUp()

	// Show command help if no arguments.
	if ctx.NArg() == 0 {
		return cli.ShowCommandHelp(ctx, "bakemacaroon")
	}
	args := ctx.Args()

	var (
		savePath          string
		timeout           int64
		ipAddress         net.IP
		rootKeyID         uint64
		parsedPermissions []*lnrpc.MacaroonPermission
		err               error
	)

	if ctx.String("save_to") != "" {
		savePath = cleanAndExpandPath(ctx.String("save_to"))
	}

	if ctx.IsSet("timeout") {
		timeout = ctx.Int64("timeout")
		if timeout <= 0 {
			return fmt.Errorf("timeout must be greater than 0")
		}
	}

	if ctx.IsSet("ip_address") {
		ipAddress = net.ParseIP(ctx.String("ip_address"))
		if ipAddress == nil {
			return fmt.Errorf("unable to parse ip_address: %s",
				ctx.String("ip_address"))
		}
	}

	if ctx.IsSet("root_key_id") {
		rootKeyID = ctx.Uint64("root_key_id")
	}

	// A command line argument can't be an empty string. So we'll check each
	// entry if it's a valid entity:action tuple. The content itself is
	// validated server side. We just make sure we can parse it correctly.
	for _, permission := range args {
		tuple := strings.Split(permission, ":")
		if len(tuple) != 2 {
			return fmt.Errorf("unable to parse "+
				"permission tuple: %s", permission)
		}
		entity, action := tuple[0], tuple[1]
		if entity == "" {
			return fmt.Errorf("invalid permission [%s]. entity "+
				"cannot be empty", permission)
		}
		if action == "" {
			return fmt.Errorf("invalid permission [%s]. action "+
				"cannot be empty", permission)
		}

		// No we can assume that we have a formally valid entity:action
		// tuple. The rest of the validation happens server side.
		parsedPermissions = append(
			parsedPermissions, &lnrpc.MacaroonPermission{
				Entity: entity,
				Action: action,
			},
		)
	}

	// Now we have gathered all the input we need and can do the actual
	// RPC call.
	req := &lnrpc.BakeMacaroonRequest{
		Permissions: parsedPermissions,
		RootKeyId:   rootKeyID,
	}
	resp, err := client.BakeMacaroon(context.Background(), req)
	if err != nil {
		return err
	}

	// Now we should have gotten a valid macaroon. Unmarshal it so we can
	// add first-party caveats (if necessary) to it.
	macBytes, err := hex.DecodeString(resp.Macaroon)
	if err != nil {
		return err
	}
	unmarshalMac := &macaroon.Macaroon{}
	if err = unmarshalMac.UnmarshalBinary(macBytes); err != nil {
		return err
	}

	// Now apply the desired constraints to the macaroon. This will always
	// create a new macaroon object, even if no constraints are added.
	macConstraints := make([]macaroons.Constraint, 0)
	if timeout > 0 {
		macConstraints = append(
			macConstraints, macaroons.TimeoutConstraint(timeout),
		)
	}
	if ipAddress != nil {
		macConstraints = append(
			macConstraints,
			macaroons.IPLockConstraint(ipAddress.String()),
		)
	}
	constrainedMac, err := macaroons.AddConstraints(
		unmarshalMac, macConstraints...,
	)
	if err != nil {
		return err
	}
	macBytes, err = constrainedMac.MarshalBinary()
	if err != nil {
		return err
	}

	// Now we can output the result. We either write it binary serialized to
	// a file or write to the standard output using hex encoding.
	switch {
	case savePath != "":
		err = ioutil.WriteFile(savePath, macBytes, 0644)
		if err != nil {
			return err
		}
		fmt.Printf("Macaroon saved to %s\n", savePath)

	default:
		fmt.Printf("%s\n", hex.EncodeToString(macBytes))
	}

	return nil
}

var listMacaroonIDsCommand = cli.Command{
	Name:     "listmacaroonids",
	Category: "Macaroons",
	Usage:    "List all macaroons root key IDs in use.",
	Action:   actionDecorator(listMacaroonIDs),
}

func listMacaroonIDs(ctx *cli.Context) error {
	client, cleanUp := getClient(ctx)
	defer cleanUp()

	req := &lnrpc.ListMacaroonIDsRequest{}
	resp, err := client.ListMacaroonIDs(context.Background(), req)
	if err != nil {
		return err
	}

	printRespJSON(resp)
	return nil
}

var deleteMacaroonIDCommand = cli.Command{
	Name:      "deletemacaroonid",
	Category:  "Macaroons",
	Usage:     "Delete a specific macaroon ID.",
	ArgsUsage: "root_key_id",
	Description: `
	Remove a macaroon ID using the specified root key ID. For example:

	lncli deletemacaroonid 1

	WARNING
	When the ID is deleted, all macaroons created from that root key will
	be invalidated.

	Note that the default root key ID 0 cannot be deleted.
	`,
	Action: actionDecorator(deleteMacaroonID),
}

func deleteMacaroonID(ctx *cli.Context) error {
	client, cleanUp := getClient(ctx)
	defer cleanUp()

	// Validate args length. Only one argument is allowed.
	if ctx.NArg() != 1 {
		return cli.ShowCommandHelp(ctx, "deletemacaroonid")
	}

	rootKeyIDString := ctx.Args().First()

	// Convert string into uint64.
	rootKeyID, err := strconv.ParseUint(rootKeyIDString, 10, 64)
	if err != nil {
		return fmt.Errorf("root key ID must be a positive integer")
	}

	// Check that the value is not equal to DefaultRootKeyID. Note that the
	// server also validates the root key ID when removing it. However, we check
	// it here too so that we can give users a nice warning.
	if bytes.Equal([]byte(rootKeyIDString), macaroons.DefaultRootKeyID) {
		return fmt.Errorf("deleting the default root key ID 0 is not allowed")
	}

	// Make the actual RPC call.
	req := &lnrpc.DeleteMacaroonIDRequest{
		RootKeyId: rootKeyID,
	}
	resp, err := client.DeleteMacaroonID(context.Background(), req)
	if err != nil {
		return err
	}

	printRespJSON(resp)
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"database/sql"
	"errors"
	_ "expvar"
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"

	"github.com/docker/distribution/health"
	"github.com/docker/notary/cryptoservice"
	"github.com/docker/notary/signer"
	"github.com/docker/notary/signer/api"
	"github.com/docker/notary/tuf/data"
	"github.com/docker/notary/utils"
	"github.com/docker/notary/version"
	_ "github.com/go-sql-driver/mysql"
	"github.com/miekg/pkcs11"
	"github.com/spf13/viper"

	"github.com/Sirupsen/logrus"
	pb "github.com/docker/notary/proto"
)

const (
	debugAddr       = "localhost:8080"
	dbType          = "mysql"
	envPrefix       = "NOTARY_SIGNER"
	defaultAliasEnv = "DEFAULT_ALIAS"
	pinCode         = "PIN"
)

var (
	debug      bool
	configFile string
	mainViper  = viper.New()
)

func init() {
	// set default log level to Error
	mainViper.SetDefault("logging", map[string]interface{}{"level": 2})

	mainViper.SetEnvPrefix(envPrefix)
	mainViper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
	mainViper.AutomaticEnv()

	// Setup flags
	flag.StringVar(&configFile, "config", "", "Path to configuration file")
	flag.BoolVar(&debug, "debug", false, "show the version and exit")
}

func passphraseRetriever(keyName, alias string, createNew bool, attempts int) (passphrase string, giveup bool, err error) {
	passphrase = mainViper.GetString(strings.ToUpper(alias))

	if passphrase == "" {
		return "", false, errors.New("expected env variable to not be empty: " + alias)
	}

	return passphrase, false, nil
}

// parses and sets up the TLS for the signer http + grpc server
func signerTLS(configuration *viper.Viper, printUsage bool) (*tls.Config, error) {
	certFile := configuration.GetString("server.cert_file")
	keyFile := configuration.GetString("server.key_file")
	if certFile == "" || keyFile == "" {
		if printUsage {
			usage()
		}
		return nil, fmt.Errorf("Certificate and key are mandatory")
	}

	clientCAFile := configuration.GetString("server.client_ca_file")
	tlsConfig, err := utils.ConfigureServerTLS(&utils.ServerTLSOpts{
		ServerCertFile:    certFile,
		ServerKeyFile:     keyFile,
		RequireClientAuth: clientCAFile != "",
		ClientCAFile:      clientCAFile,
	})
	if err != nil {
		return nil, fmt.Errorf("Unable to set up TLS: %s", err.Error())
	}
	return tlsConfig, nil
}

func main() {
	flag.Usage = usage
	flag.Parse()

	if debug {
		go debugServer(debugAddr)
	}

	// when the signer starts print the version for debugging and issue logs later
	logrus.Infof("Version: %s, Git commit: %s", version.NotaryVersion, version.GitCommit)

	filename := filepath.Base(configFile)
	ext := filepath.Ext(configFile)
	configPath := filepath.Dir(configFile)

	mainViper.SetConfigType(strings.TrimPrefix(ext, "."))
	mainViper.SetConfigName(strings.TrimSuffix(filename, ext))
	mainViper.AddConfigPath(configPath)
	err := mainViper.ReadInConfig()
	if err != nil {
		logrus.Error("Viper Error: ", err.Error())
		logrus.Error("Could not read config at ", configFile)
		os.Exit(1)
	}

	logrus.SetLevel(logrus.Level(mainViper.GetInt("logging.level")))

	tlsConfig, err := signerTLS(mainViper, true)
	if err != nil {
		logrus.Fatalf(err.Error())
	}

	cryptoServices := make(signer.CryptoServiceIndex)

	pin := mainViper.GetString(pinCode)
	pkcs11Lib := mainViper.GetString("crypto.pkcs11lib")
	if pkcs11Lib != "" {
		if pin == "" {
			log.Fatalf("Using PIN is mandatory with pkcs11")
		}

		ctx, session := SetupHSMEnv(pkcs11Lib, pin)

		defer cleanup(ctx, session)

		cryptoServices[data.RSAKey] = api.NewRSAHardwareCryptoService(ctx, session)
	}

	configDBType := strings.ToLower(mainViper.GetString("storage.backend"))
	dbURL := mainViper.GetString("storage.db_url")
	if configDBType != dbType || dbURL == "" {
		usage()
		log.Fatalf("Currently only a MySQL database backend is supported.")
	}
	dbSQL, err := sql.Open(configDBType, dbURL)
	if err != nil {
		log.Fatalf("failed to open the database: %s, %v", dbURL, err)
	}

	defaultAlias := mainViper.GetString(defaultAliasEnv)
	logrus.Debug("Default Alias: ", defaultAlias)
	keyStore, err := signer.NewKeyDBStore(passphraseRetriever, defaultAlias, configDBType, dbSQL)
	if err != nil {
		log.Fatalf("failed to create a new keydbstore: %v", err)
	}

	health.RegisterPeriodicFunc(
		"DB operational", keyStore.HealthCheck, time.Second*60)

	cryptoService := cryptoservice.NewCryptoService("", keyStore)

	cryptoServices[data.ED25519Key] = cryptoService
	cryptoServices[data.ECDSAKey] = cryptoService

	//RPC server setup
	kms := &api.KeyManagementServer{CryptoServices: cryptoServices,
		HealthChecker: health.CheckStatus}
	ss := &api.SignerServer{CryptoServices: cryptoServices,
		HealthChecker: health.CheckStatus}

	rpcAddr := mainViper.GetString("server.grpc_addr")
	lis, err := net.Listen("tcp", rpcAddr)
	if err != nil {
		log.Fatalf("failed to listen %v", err)
	}
	creds := credentials.NewTLS(tlsConfig)
	opts := []grpc.ServerOption{grpc.Creds(creds)}
	grpcServer := grpc.NewServer(opts...)

	pb.RegisterKeyManagementServer(grpcServer, kms)
	pb.RegisterSignerServer(grpcServer, ss)

	go grpcServer.Serve(lis)

	httpAddr := mainViper.GetString("server.http_addr")
	if httpAddr == "" {
		log.Fatalf("Server address is required")
	}
	//HTTP server setup
	server := http.Server{
		Addr:      httpAddr,
		Handler:   api.Handlers(cryptoServices),
		TLSConfig: tlsConfig,
	}

	if debug {
		log.Println("RPC server listening on", rpcAddr)
		log.Println("HTTP server listening on", httpAddr)
	}

	err = server.ListenAndServe()
	if err != nil {
		log.Fatal("HTTP server failed to start:", err)
	}
}

func usage() {
	log.Println("usage:", os.Args[0], "<config>")
	flag.PrintDefaults()
}

// debugServer starts the debug server with pprof, expvar among other
// endpoints. The addr should not be exposed externally. For most of these to
// work, tls cannot be enabled on the endpoint, so it is generally separate.
func debugServer(addr string) {
	log.Println("Debug server listening on", addr)
	if err := http.ListenAndServe(addr, nil); err != nil {
		log.Fatalf("error listening on debug interface: %v", err)
	}
}

// SetupHSMEnv is a method that depends on the existences
func SetupHSMEnv(libraryPath, pin string) (*pkcs11.Ctx, pkcs11.SessionHandle) {
	p := pkcs11.New(libraryPath)

	if p == nil {
		log.Fatalf("Failed to init library")
	}

	if err := p.Initialize(); err != nil {
		log.Fatalf("Initialize error %s\n", err.Error())
	}

	slots, err := p.GetSlotList(true)
	if err != nil {
		log.Fatalf("Failed to list HSM slots %s", err)
	}
	// Check to see if we got any slots from the HSM.
	if len(slots) < 1 {
		log.Fatalln("No HSM Slots found")
	}

	// CKF_SERIAL_SESSION: TRUE if cryptographic functions are performed in serial with the application; FALSE if the functions may be performed in parallel with the application.
	// CKF_RW_SESSION: TRUE if the session is read/write; FALSE if the session is read-only
	session, err := p.OpenSession(slots[0], pkcs11.CKF_SERIAL_SESSION|pkcs11.CKF_RW_SESSION)
	if err != nil {
		log.Fatalf("Failed to Start Session with HSM %s", err)
	}

	if err = p.Login(session, pkcs11.CKU_USER, pin); err != nil {
		log.Fatalf("User PIN %s\n", err.Error())
	}

	return p, session
}

func cleanup(ctx *pkcs11.Ctx, session pkcs11.SessionHandle) {
	ctx.Destroy()
	ctx.Finalize()
	ctx.CloseSession(session)
	ctx.Logout(session)
}
<commit_msg>Use ListenAndServeTLS with blank args, since ListenAndServe doesn't actually set up TLS<commit_after>package main

import (
	"crypto/tls"
	"database/sql"
	"errors"
	_ "expvar"
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"

	"github.com/docker/distribution/health"
	"github.com/docker/notary/cryptoservice"
	"github.com/docker/notary/signer"
	"github.com/docker/notary/signer/api"
	"github.com/docker/notary/tuf/data"
	"github.com/docker/notary/utils"
	"github.com/docker/notary/version"
	_ "github.com/go-sql-driver/mysql"
	"github.com/miekg/pkcs11"
	"github.com/spf13/viper"

	"github.com/Sirupsen/logrus"
	pb "github.com/docker/notary/proto"
)

const (
	debugAddr       = "localhost:8080"
	dbType          = "mysql"
	envPrefix       = "NOTARY_SIGNER"
	defaultAliasEnv = "DEFAULT_ALIAS"
	pinCode         = "PIN"
)

var (
	debug      bool
	configFile string
	mainViper  = viper.New()
)

func init() {
	// set default log level to Error
	mainViper.SetDefault("logging", map[string]interface{}{"level": 2})

	mainViper.SetEnvPrefix(envPrefix)
	mainViper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
	mainViper.AutomaticEnv()

	// Setup flags
	flag.StringVar(&configFile, "config", "", "Path to configuration file")
	flag.BoolVar(&debug, "debug", false, "show the version and exit")
}

func passphraseRetriever(keyName, alias string, createNew bool, attempts int) (passphrase string, giveup bool, err error) {
	passphrase = mainViper.GetString(strings.ToUpper(alias))

	if passphrase == "" {
		return "", false, errors.New("expected env variable to not be empty: " + alias)
	}

	return passphrase, false, nil
}

// parses and sets up the TLS for the signer http + grpc server
func signerTLS(configuration *viper.Viper, printUsage bool) (*tls.Config, error) {
	certFile := configuration.GetString("server.cert_file")
	keyFile := configuration.GetString("server.key_file")
	if certFile == "" || keyFile == "" {
		if printUsage {
			usage()
		}
		return nil, fmt.Errorf("Certificate and key are mandatory")
	}

	clientCAFile := configuration.GetString("server.client_ca_file")
	tlsConfig, err := utils.ConfigureServerTLS(&utils.ServerTLSOpts{
		ServerCertFile:    certFile,
		ServerKeyFile:     keyFile,
		RequireClientAuth: clientCAFile != "",
		ClientCAFile:      clientCAFile,
	})
	if err != nil {
		return nil, fmt.Errorf("Unable to set up TLS: %s", err.Error())
	}
	return tlsConfig, nil
}

func main() {
	flag.Usage = usage
	flag.Parse()

	if debug {
		go debugServer(debugAddr)
	}

	// when the signer starts print the version for debugging and issue logs later
	logrus.Infof("Version: %s, Git commit: %s", version.NotaryVersion, version.GitCommit)

	filename := filepath.Base(configFile)
	ext := filepath.Ext(configFile)
	configPath := filepath.Dir(configFile)

	mainViper.SetConfigType(strings.TrimPrefix(ext, "."))
	mainViper.SetConfigName(strings.TrimSuffix(filename, ext))
	mainViper.AddConfigPath(configPath)
	err := mainViper.ReadInConfig()
	if err != nil {
		logrus.Error("Viper Error: ", err.Error())
		logrus.Error("Could not read config at ", configFile)
		os.Exit(1)
	}

	logrus.SetLevel(logrus.Level(mainViper.GetInt("logging.level")))

	tlsConfig, err := signerTLS(mainViper, true)
	if err != nil {
		logrus.Fatalf(err.Error())
	}

	cryptoServices := make(signer.CryptoServiceIndex)

	pin := mainViper.GetString(pinCode)
	pkcs11Lib := mainViper.GetString("crypto.pkcs11lib")
	if pkcs11Lib != "" {
		if pin == "" {
			log.Fatalf("Using PIN is mandatory with pkcs11")
		}

		ctx, session := SetupHSMEnv(pkcs11Lib, pin)

		defer cleanup(ctx, session)

		cryptoServices[data.RSAKey] = api.NewRSAHardwareCryptoService(ctx, session)
	}

	configDBType := strings.ToLower(mainViper.GetString("storage.backend"))
	dbURL := mainViper.GetString("storage.db_url")
	if configDBType != dbType || dbURL == "" {
		usage()
		log.Fatalf("Currently only a MySQL database backend is supported.")
	}
	dbSQL, err := sql.Open(configDBType, dbURL)
	if err != nil {
		log.Fatalf("failed to open the database: %s, %v", dbURL, err)
	}

	defaultAlias := mainViper.GetString(defaultAliasEnv)
	logrus.Debug("Default Alias: ", defaultAlias)
	keyStore, err := signer.NewKeyDBStore(passphraseRetriever, defaultAlias, configDBType, dbSQL)
	if err != nil {
		log.Fatalf("failed to create a new keydbstore: %v", err)
	}

	health.RegisterPeriodicFunc(
		"DB operational", keyStore.HealthCheck, time.Second*60)

	cryptoService := cryptoservice.NewCryptoService("", keyStore)

	cryptoServices[data.ED25519Key] = cryptoService
	cryptoServices[data.ECDSAKey] = cryptoService

	//RPC server setup
	kms := &api.KeyManagementServer{CryptoServices: cryptoServices,
		HealthChecker: health.CheckStatus}
	ss := &api.SignerServer{CryptoServices: cryptoServices,
		HealthChecker: health.CheckStatus}

	rpcAddr := mainViper.GetString("server.grpc_addr")
	lis, err := net.Listen("tcp", rpcAddr)
	if err != nil {
		log.Fatalf("failed to listen %v", err)
	}
	creds := credentials.NewTLS(tlsConfig)
	opts := []grpc.ServerOption{grpc.Creds(creds)}
	grpcServer := grpc.NewServer(opts...)

	pb.RegisterKeyManagementServer(grpcServer, kms)
	pb.RegisterSignerServer(grpcServer, ss)

	go grpcServer.Serve(lis)

	httpAddr := mainViper.GetString("server.http_addr")
	if httpAddr == "" {
		log.Fatalf("Server address is required")
	}
	//HTTP server setup
	server := http.Server{
		Addr:      httpAddr,
		Handler:   api.Handlers(cryptoServices),
		TLSConfig: tlsConfig,
	}

	if debug {
		log.Println("RPC server listening on", rpcAddr)
		log.Println("HTTP server listening on", httpAddr)
	}

	err = server.ListenAndServeTLS("", "")
	if err != nil {
		log.Fatal("HTTPS server failed to start:", err)
	}
}

func usage() {
	log.Println("usage:", os.Args[0], "<config>")
	flag.PrintDefaults()
}

// debugServer starts the debug server with pprof, expvar among other
// endpoints. The addr should not be exposed externally. For most of these to
// work, tls cannot be enabled on the endpoint, so it is generally separate.
func debugServer(addr string) {
	log.Println("Debug server listening on", addr)
	if err := http.ListenAndServe(addr, nil); err != nil {
		log.Fatalf("error listening on debug interface: %v", err)
	}
}

// SetupHSMEnv is a method that depends on the existences
func SetupHSMEnv(libraryPath, pin string) (*pkcs11.Ctx, pkcs11.SessionHandle) {
	p := pkcs11.New(libraryPath)

	if p == nil {
		log.Fatalf("Failed to init library")
	}

	if err := p.Initialize(); err != nil {
		log.Fatalf("Initialize error %s\n", err.Error())
	}

	slots, err := p.GetSlotList(true)
	if err != nil {
		log.Fatalf("Failed to list HSM slots %s", err)
	}
	// Check to see if we got any slots from the HSM.
	if len(slots) < 1 {
		log.Fatalln("No HSM Slots found")
	}

	// CKF_SERIAL_SESSION: TRUE if cryptographic functions are performed in serial with the application; FALSE if the functions may be performed in parallel with the application.
	// CKF_RW_SESSION: TRUE if the session is read/write; FALSE if the session is read-only
	session, err := p.OpenSession(slots[0], pkcs11.CKF_SERIAL_SESSION|pkcs11.CKF_RW_SESSION)
	if err != nil {
		log.Fatalf("Failed to Start Session with HSM %s", err)
	}

	if err = p.Login(session, pkcs11.CKU_USER, pin); err != nil {
		log.Fatalf("User PIN %s\n", err.Error())
	}

	return p, session
}

func cleanup(ctx *pkcs11.Ctx, session pkcs11.SessionHandle) {
	ctx.Destroy()
	ctx.Finalize()
	ctx.CloseSession(session)
	ctx.Logout(session)
}
<|endoftext|>


<commit_before>// Copyright 2017 the original author or authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package info

import (
	"log"
	"os"
	"testing"
	"time"

	"github.com/maguro/pbf"
	"github.com/stretchr/testify/assert"
)

func TestRunInfo(t *testing.T) {
	f, err := os.Open("../../../testdata/greater-london.osm.pbf")
	if err != nil {
		log.Fatal(err)
	}
	info := runInfo(f, 2, false)

	bbox := pbf.BoundingBox{-0.511482, 0.335437, 51.69344, 51.28554}
	ts, _ := time.Parse(time.RFC3339, "2014-03-24T21:55:02Z")
	assert.True(t, info.BoundingBox.EqualWithin(bbox, pbf.E6))
	assert.Equal(t, info.RequiredFeatures, []string{"OsmSchema-V0.6", "DenseNodes"})
	assert.Equal(t, info.OptionalFeatures, []string(nil))
	assert.Equal(t, info.WritingProgram, "Osmium (http://wiki.openstreetmap.org/wiki/Osmium)")
	assert.Equal(t, info.Source, "")
	assert.Equal(t, info.OsmosisReplicationTimestamp.UTC(), ts)
	assert.Equal(t, info.OsmosisReplicationSequenceNumber, int64(0))
	assert.Equal(t, info.OsmosisReplicationBaseURL, "")
	assert.Equal(t, info.NodeCount, int64(0))
	assert.Equal(t, info.WayCount, int64(0))
	assert.Equal(t, info.RelationCount, int64(0))
}

func TestRunInfoExtended(t *testing.T) {
	f, err := os.Open("../../../testdata/greater-london.osm.pbf")
	if err != nil {
		log.Fatal(err)
	}
	info := runInfo(f, 2, true)

	bbox := pbf.BoundingBox{-0.511482, 0.335437, 51.69344, 51.28554}
	ts, _ := time.Parse(time.RFC3339, "2014-03-24T21:55:02Z")
	assert.True(t, info.BoundingBox.EqualWithin(bbox, pbf.E6))
	assert.Equal(t, info.RequiredFeatures, []string{"OsmSchema-V0.6", "DenseNodes"})
	assert.Equal(t, info.OptionalFeatures, []string(nil))
	assert.Equal(t, info.WritingProgram, "Osmium (http://wiki.openstreetmap.org/wiki/Osmium)")
	assert.Equal(t, info.Source, "")
	assert.Equal(t, info.OsmosisReplicationTimestamp.UTC(), ts)
	assert.Equal(t, info.OsmosisReplicationSequenceNumber, int64(0))
	assert.Equal(t, info.OsmosisReplicationBaseURL, "")
	assert.Equal(t, info.NodeCount, int64(2729006))
	assert.Equal(t, info.WayCount, int64(459055))
	assert.Equal(t, info.RelationCount, int64(12833))
}
<commit_msg>Add keys to composite literal<commit_after>// Copyright 2017 the original author or authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package info

import (
	"log"
	"os"
	"testing"
	"time"

	"github.com/maguro/pbf"
	"github.com/stretchr/testify/assert"
)

func TestRunInfo(t *testing.T) {
	f, err := os.Open("../../../testdata/greater-london.osm.pbf")
	if err != nil {
		log.Fatal(err)
	}
	info := runInfo(f, 2, false)

	bbox := pbf.BoundingBox{Left: -0.511482, Right: 0.335437, Top: 51.69344, Bottom: 51.28554}
	ts, _ := time.Parse(time.RFC3339, "2014-03-24T21:55:02Z")
	assert.True(t, info.BoundingBox.EqualWithin(bbox, pbf.E6))
	assert.Equal(t, info.RequiredFeatures, []string{"OsmSchema-V0.6", "DenseNodes"})
	assert.Equal(t, info.OptionalFeatures, []string(nil))
	assert.Equal(t, info.WritingProgram, "Osmium (http://wiki.openstreetmap.org/wiki/Osmium)")
	assert.Equal(t, info.Source, "")
	assert.Equal(t, info.OsmosisReplicationTimestamp.UTC(), ts)
	assert.Equal(t, info.OsmosisReplicationSequenceNumber, int64(0))
	assert.Equal(t, info.OsmosisReplicationBaseURL, "")
	assert.Equal(t, info.NodeCount, int64(0))
	assert.Equal(t, info.WayCount, int64(0))
	assert.Equal(t, info.RelationCount, int64(0))
}

func TestRunInfoExtended(t *testing.T) {
	f, err := os.Open("../../../testdata/greater-london.osm.pbf")
	if err != nil {
		log.Fatal(err)
	}
	info := runInfo(f, 2, true)

	bbox := pbf.BoundingBox{Left: -0.511482, Right: 0.335437, Top: 51.69344, Bottom: 51.28554}
	ts, _ := time.Parse(time.RFC3339, "2014-03-24T21:55:02Z")
	assert.True(t, info.BoundingBox.EqualWithin(bbox, pbf.E6))
	assert.Equal(t, info.RequiredFeatures, []string{"OsmSchema-V0.6", "DenseNodes"})
	assert.Equal(t, info.OptionalFeatures, []string(nil))
	assert.Equal(t, info.WritingProgram, "Osmium (http://wiki.openstreetmap.org/wiki/Osmium)")
	assert.Equal(t, info.Source, "")
	assert.Equal(t, info.OsmosisReplicationTimestamp.UTC(), ts)
	assert.Equal(t, info.OsmosisReplicationSequenceNumber, int64(0))
	assert.Equal(t, info.OsmosisReplicationBaseURL, "")
	assert.Equal(t, info.NodeCount, int64(2729006))
	assert.Equal(t, info.WayCount, int64(459055))
	assert.Equal(t, info.RelationCount, int64(12833))
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/gob"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/chrissnell/syslog"
)

type handler struct {
	// To simplify implementation of our handler we embed helper
	// syslog.BaseHandler struct.
	*syslog.BaseHandler
}

type LogentriesHostEntity struct {
	Response  string `json:"response"`
	Host      Host   `json:"host"`
	Host_key  string `json:"host_key"`
	Worker    string `json:"worker"`
	Agent_key string `json:"agent_key"`
}

type Host struct {
	C        float64 `json:"c"`
	Name     string  `json:"name"`
	Distver  string  `json:"distver"`
	Hostname string  `json:"hostname"`
	Object   string  `json:"object"`
	Distname string  `json:"distname"`
	Key      string  `json:"key"`
}

type LogentriesLogEntity struct {
	Response string `json:"response"`
	Log_key  string `json:"log_key"`
	Log      Log    `json:"log"`
}

type Log struct {
	Token     string  `json:"token"`
	Created   float64 `json:"created"`
	Name      string  `json:"name`
	Retention float64 `json:"retention"`
	Filename  string  `json:"filename"`
	Object    string  `json:"object"`
	Type      string  `json:"type"`
	Key       string  `json:"key"`
	Follow    string  `json:"folow"`
}

type LogLine struct {
	Line  syslog.Message
	Token string
}

var (
	logconsumerPtr        *string
	logentriesAPIKeyPtr   *string
	listenAddrPtr         *string
	logentities           = make(map[string]LogentriesLogEntity)
	hostentities          = make(map[string]LogentriesHostEntity)
	tokenchan             = make(chan string)
	logentities_filename  = "logentries-logentities.gob"
	hostentities_filename = "logentries-hostentities.gob"
)

func newHandler() *handler {
	msg := make(chan syslog.Message)
	// Filter function name set to nil to disable filtering
	h := handler{syslog.NewBaseHandler(5, nil, false)}
	go h.mainLoop(msg)
	go ProcessLogMessage(msg)
	return &h
}

func (h *handler) mainLoop(msg chan syslog.Message) {
	for {
		m := h.Get()
		if m == nil {
			break
		}
		msg <- *m
	}
	fmt.Println("Exit handler")
	h.End()
}

func ProcessLogMessage(msg chan syslog.Message) {
	tokenfetchdone := make(chan bool, 1)
	logentrieschan := make(chan LogLine)
	lh := make(chan struct{ host, log string })

	var logline LogLine

	for m := range msg {
		if m.Hostname == "" {
			m.Hostname = "NONE"
		}
		go GetTokenForLog(tokenfetchdone, lh)
		lh <- struct{ host, log string }{m.Hostname, m.Tag}
		token := <-tokenchan
		<-tokenfetchdone

		logline.Token = token
		logline.Line = m

		go SendLogMessages(logentrieschan)
		logentrieschan <- logline

	}
}

func GetTokenForLog(tokenfetchdone chan bool, lh chan struct{ host, log string }) {
	select {
	case lht, msg_ok := <-lh:
		if !msg_ok {
			fmt.Println("msg channel closed")
		} else {

			var hostentity LogentriesHostEntity
			var logentity LogentriesLogEntity

			l := strings.Join([]string{lht.host, lht.log}, "::")

			hostentity = hostentities[lht.host]
			if hostentity.Host.Key == "" {
				hostentity = RegisterNewHost(lht.host)

				// Store our new host token in our map and sync it to disk
				hostentities[lht.host] = hostentity
				err := SyncHostEntitiesToDisk()
				if err != nil {
					log.Fatal(err)
				}
			}

			logentity = logentities[l]
			if logentity.Log.Token == "" {
				logentity := RegisterNewLog(hostentity, l)
				logentities[l] = logentity
				tokenchan <- logentity.Log.Token
				tokenfetchdone <- true
				err := SyncLogEntitiesToDisk()
				if err != nil {
					log.Fatal(err)
				}
			} else {
				tokenchan <- logentity.Log.Token
				tokenfetchdone <- true
			}
		}
	}
}

func DialLogEntries() (err error, conn net.Conn) {
	conn, err = net.Dial("tcp", *logconsumerPtr)
	if err != nil {
		fmt.Println("Could not connect to LogEntries log endpoint ", err.Error())
	}
	return err, conn
}

func SendLogMessages(msg chan LogLine) {
	err, conn := DialLogEntries()
	if err != nil {
		fmt.Println("Could not connect to LogEntries log endpoint ", err.Error())
	}

	select {
	case logline, msg_ok := <-msg:
		if !msg_ok {
			fmt.Println("msg channel closed")
		} else {
			t := logline.Line.Time
			line := fmt.Sprintf("%v %v %v %v\n", logline.Token, t.Format(time.RFC3339), logline.Line.Hostname, logline.Line.Content)
			_, err = conn.Write([]byte(line))
			if err != nil {
				log.Print("Send to Logentries endpoint failed.")
				break
			}
			// fmt.Printf("Sending line: %v", line)
		}
	}
}

func SyncLogEntitiesToDisk() (err error) {
	m := new(bytes.Buffer)
	enc := gob.NewEncoder(m)
	enc.Encode(logentities)
	err = ioutil.WriteFile("logentries-logentities.gob", m.Bytes(), 0600)
	return (err)
}

func SyncHostEntitiesToDisk() (err error) {
	m := new(bytes.Buffer)
	enc := gob.NewEncoder(m)
	enc.Encode(hostentities)
	err = ioutil.WriteFile("logentries-hostentities.gob", m.Bytes(), 0600)
	return (err)
}

func LoadLogEntitiesFromDisk() (err error) {
	n, err := ioutil.ReadFile("logentries-logentities.gob")
	if err != nil {
		return (err)
	}
	p := bytes.NewBuffer(n)
	dec := gob.NewDecoder(p)
	err = dec.Decode(&logentities)
	return (err)
}

func LoadHostEntitiesFromDisk() (err error) {
	n, err := ioutil.ReadFile("logentries-hostentities.gob")
	if err != nil {
		return (err)
	}
	p := bytes.NewBuffer(n)
	dec := gob.NewDecoder(p)
	err = dec.Decode(&hostentities)
	return (err)
}

func RegisterNewHost(h string) (he LogentriesHostEntity) {
	v := url.Values{}
	v.Set("request", "register")
	v.Set("user_key", *logentriesAPIKeyPtr)
	v.Set("name", h)
	v.Set("hostname", h)
	v.Set("distver", "")
	v.Set("system", "")
	v.Set("distname", "")
	res, err := http.PostForm("http://api.logentries.com/", v)
	if err != nil {
		log.Fatal(err)
	}
	body, err := ioutil.ReadAll(res.Body)
	err = json.Unmarshal(body, &he)
	return (he)
}

func RegisterNewLog(e LogentriesHostEntity, n string) (logentity LogentriesLogEntity) {
	v := url.Values{}
	v.Set("request", "new_log")
	v.Set("user_key", *logentriesAPIKeyPtr)
	v.Set("host_key", e.Host.Key)
	v.Set("name", n)
	v.Set("filename", "")
	v.Set("retention", "-1")
	v.Set("source", "token")
	res, err := http.PostForm("http://api.logentries.com/", v)
	if err != nil {
		log.Fatal(err)
	}
	body, err := ioutil.ReadAll(res.Body)
	err = json.Unmarshal(body, &logentity)
	return (logentity)
}

func main() {

	logconsumerPtr = flag.String("consumer", "api.logentries.com:10000", "Logentries log consumer endpoint <host:port> (Default: api.logentries.com:10000)")
	logentriesAPIKeyPtr = flag.String("apikey", "", "Logentries API key")
	listenAddrPtr = flag.String("listen", "0.0.0.0:1987", "Host/port to listen for syslog messages <host:port> (Default: 0.0.0.0:1987)")

	flag.Parse()

	if *logentriesAPIKeyPtr == "" {
		log.Fatal("Must pass a Logentries API key. Use -h for help.")
	}

	if _, err := os.Stat(logentities_filename); err == nil {
		err = LoadLogEntitiesFromDisk()
		if err != nil {
			log.Fatal(err)
		}
	}

	if _, err := os.Stat(hostentities_filename); err == nil {
		err = LoadHostEntitiesFromDisk()
		if err != nil {
			log.Fatal(err)
		}
	}

	// Create a server with one handler and run one listen gorutine
	s := syslog.NewServer()
	s.AddAllowedRunes("-._")
	s.AddHandler(newHandler())
	s.Listen(*listenAddrPtr)

	// Wait for terminating signal
	sc := make(chan os.Signal, 2)
	signal.Notify(sc, syscall.SIGTERM, syscall.SIGINT)
	<-sc

	// Shutdown the server
	fmt.Println("Shutdown the server...")
	s.Shutdown()
	fmt.Println("Server is down")
}
<commit_msg>Rework DialLogEntries() to redial automatically<commit_after>package main

import (
	"bytes"
	"encoding/gob"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/chrissnell/syslog"
)

type handler struct {
	// To simplify implementation of our handler we embed helper
	// syslog.BaseHandler struct.
	*syslog.BaseHandler
}

type LogentriesHostEntity struct {
	Response  string `json:"response"`
	Host      Host   `json:"host"`
	Host_key  string `json:"host_key"`
	Worker    string `json:"worker"`
	Agent_key string `json:"agent_key"`
}

type Host struct {
	C        float64 `json:"c"`
	Name     string  `json:"name"`
	Distver  string  `json:"distver"`
	Hostname string  `json:"hostname"`
	Object   string  `json:"object"`
	Distname string  `json:"distname"`
	Key      string  `json:"key"`
}

type LogentriesLogEntity struct {
	Response string `json:"response"`
	Log_key  string `json:"log_key"`
	Log      Log    `json:"log"`
}

type Log struct {
	Token     string  `json:"token"`
	Created   float64 `json:"created"`
	Name      string  `json:"name`
	Retention float64 `json:"retention"`
	Filename  string  `json:"filename"`
	Object    string  `json:"object"`
	Type      string  `json:"type"`
	Key       string  `json:"key"`
	Follow    string  `json:"folow"`
}

type LogLine struct {
	Line  syslog.Message
	Token string
}

var (
	logconsumerPtr        *string
	logentriesAPIKeyPtr   *string
	listenAddrPtr         *string
	logentities           = make(map[string]LogentriesLogEntity)
	hostentities          = make(map[string]LogentriesHostEntity)
	tokenchan             = make(chan string)
	logentities_filename  = "logentries-logentities.gob"
	hostentities_filename = "logentries-hostentities.gob"
)

func newHandler() *handler {
	msg := make(chan syslog.Message)
	// Filter function name set to nil to disable filtering
	h := handler{syslog.NewBaseHandler(5, nil, false)}
	go h.mainLoop(msg)
	go ProcessLogMessage(msg)
	return &h
}

func (h *handler) mainLoop(msg chan syslog.Message) {
	for {
		m := h.Get()
		if m == nil {
			break
		}
		msg <- *m
	}
	fmt.Println("Exit handler")
	h.End()
}

func ProcessLogMessage(msg chan syslog.Message) {
	tokenfetchdone := make(chan bool, 1)
	logentrieschan := make(chan LogLine)
	lh := make(chan struct{ host, log string })

	var logline LogLine

	for m := range msg {
		if m.Hostname == "" {
			m.Hostname = "NONE"
		}
		go GetTokenForLog(tokenfetchdone, lh)
		lh <- struct{ host, log string }{m.Hostname, m.Tag}
		token := <-tokenchan
		<-tokenfetchdone

		logline.Token = token
		logline.Line = m

		go SendLogMessages(logentrieschan)
		logentrieschan <- logline

	}
}

func GetTokenForLog(tokenfetchdone chan bool, lh chan struct{ host, log string }) {
	select {
	case lht, msg_ok := <-lh:
		if !msg_ok {
			fmt.Println("msg channel closed")
		} else {

			var hostentity LogentriesHostEntity
			var logentity LogentriesLogEntity

			l := strings.Join([]string{lht.host, lht.log}, "::")

			hostentity = hostentities[lht.host]
			if hostentity.Host.Key == "" {
				hostentity = RegisterNewHost(lht.host)

				// Store our new host token in our map and sync it to disk
				hostentities[lht.host] = hostentity
				err := SyncHostEntitiesToDisk()
				if err != nil {
					log.Fatal(err)
				}
			}

			logentity = logentities[l]
			if logentity.Log.Token == "" {
				logentity := RegisterNewLog(hostentity, l)
				logentities[l] = logentity
				tokenchan <- logentity.Log.Token
				tokenfetchdone <- true
				err := SyncLogEntitiesToDisk()
				if err != nil {
					log.Fatal(err)
				}
			} else {
				tokenchan <- logentity.Log.Token
				tokenfetchdone <- true
			}
		}
	}
}

func DialLogEntries() (err error, conn net.Conn) {
	for {
		conn, err = net.Dial("tcp", *logconsumerPtr)
		if err == nil {
			return err, conn
		} else {
			fmt.Println("Could not connect to LogEntries log endpoint...retrying")
			// Wait for 5 seconds before redialing
			timer := time.NewTimer(time.Second * 5)
			<-timer.C
		}
	}
}

func SendLogMessages(msg chan LogLine) {
	err, conn := DialLogEntries()
	if err != nil {
		fmt.Println("Could not connect to LogEntries log endpoint ", err.Error())
	}

	select {
	case logline, msg_ok := <-msg:
		if !msg_ok {
			fmt.Println("msg channel closed")
		} else {
			t := logline.Line.Time
			line := fmt.Sprintf("%v %v %v %v\n", logline.Token, t.Format(time.RFC3339), logline.Line.Hostname, logline.Line.Content)
			_, err = conn.Write([]byte(line))
			if err != nil {
				log.Print("Send to Logentries endpoint failed.")
				break
			}
			// fmt.Printf("Sending line: %v", line)
		}
	}
}

func SyncLogEntitiesToDisk() (err error) {
	m := new(bytes.Buffer)
	enc := gob.NewEncoder(m)
	enc.Encode(logentities)
	err = ioutil.WriteFile("logentries-logentities.gob", m.Bytes(), 0600)
	return (err)
}

func SyncHostEntitiesToDisk() (err error) {
	m := new(bytes.Buffer)
	enc := gob.NewEncoder(m)
	enc.Encode(hostentities)
	err = ioutil.WriteFile("logentries-hostentities.gob", m.Bytes(), 0600)
	return (err)
}

func LoadLogEntitiesFromDisk() (err error) {
	n, err := ioutil.ReadFile("logentries-logentities.gob")
	if err != nil {
		return (err)
	}
	p := bytes.NewBuffer(n)
	dec := gob.NewDecoder(p)
	err = dec.Decode(&logentities)
	return (err)
}

func LoadHostEntitiesFromDisk() (err error) {
	n, err := ioutil.ReadFile("logentries-hostentities.gob")
	if err != nil {
		return (err)
	}
	p := bytes.NewBuffer(n)
	dec := gob.NewDecoder(p)
	err = dec.Decode(&hostentities)
	return (err)
}

func RegisterNewHost(h string) (he LogentriesHostEntity) {
	v := url.Values{}
	v.Set("request", "register")
	v.Set("user_key", *logentriesAPIKeyPtr)
	v.Set("name", h)
	v.Set("hostname", h)
	v.Set("distver", "")
	v.Set("system", "")
	v.Set("distname", "")
	res, err := http.PostForm("http://api.logentries.com/", v)
	if err != nil {
		log.Fatal(err)
	}
	body, err := ioutil.ReadAll(res.Body)
	err = json.Unmarshal(body, &he)
	return (he)
}

func RegisterNewLog(e LogentriesHostEntity, n string) (logentity LogentriesLogEntity) {
	v := url.Values{}
	v.Set("request", "new_log")
	v.Set("user_key", *logentriesAPIKeyPtr)
	v.Set("host_key", e.Host.Key)
	v.Set("name", n)
	v.Set("filename", "")
	v.Set("retention", "-1")
	v.Set("source", "token")
	res, err := http.PostForm("http://api.logentries.com/", v)
	if err != nil {
		log.Fatal(err)
	}
	body, err := ioutil.ReadAll(res.Body)
	err = json.Unmarshal(body, &logentity)
	return (logentity)
}

func main() {

	logconsumerPtr = flag.String("consumer", "api.logentries.com:10000", "Logentries log consumer endpoint <host:port> (Default: api.logentries.com:10000)")
	logentriesAPIKeyPtr = flag.String("apikey", "", "Logentries API key")
	listenAddrPtr = flag.String("listen", "0.0.0.0:1987", "Host/port to listen for syslog messages <host:port> (Default: 0.0.0.0:1987)")

	flag.Parse()

	if *logentriesAPIKeyPtr == "" {
		log.Fatal("Must pass a Logentries API key. Use -h for help.")
	}

	if _, err := os.Stat(logentities_filename); err == nil {
		err = LoadLogEntitiesFromDisk()
		if err != nil {
			log.Fatal(err)
		}
	}

	if _, err := os.Stat(hostentities_filename); err == nil {
		err = LoadHostEntitiesFromDisk()
		if err != nil {
			log.Fatal(err)
		}
	}

	// Create a server with one handler and run one listen gorutine
	s := syslog.NewServer()
	s.AddAllowedRunes("-._")
	s.AddHandler(newHandler())
	s.Listen(*listenAddrPtr)

	// Wait for terminating signal
	sc := make(chan os.Signal, 2)
	signal.Notify(sc, syscall.SIGTERM, syscall.SIGINT)
	<-sc

	// Shutdown the server
	fmt.Println("Shutdown the server...")
	s.Shutdown()
	fmt.Println("Server is down")
}
<|endoftext|>


<commit_before>package new

import (
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"time"

	"github.com/ghthor/journal/entry"
	"github.com/ghthor/journal/git"
	"github.com/ghthor/journal/idea"
	initialize "github.com/ghthor/journal/init"

	"github.com/ghthor/gospec"
	. "github.com/ghthor/gospec"
)

type mockEditor struct {
	start, wait func()
}

func (m mockEditor) Start() error {
	m.start()
	return nil
}

func (m mockEditor) Wait() error {
	m.wait()
	return nil
}

func DescribeNewCmd(c gospec.Context) {
	c.Specify("the `new` command", func() {
		// Create a temporary journal
		journalDir, err := ioutil.TempDir("", "new_cmd_desc_")
		c.Assume(err, IsNil)
		defer func() {
			c.Assume(os.RemoveAll(journalDir), IsNil)
		}()

		commitable, err := initialize.Journal(journalDir)
		c.Assume(err, IsNil)
		c.Assume(git.Commit(commitable), IsNil)

		c.Specify("will include any active ideas in the entries body while editting", func() {
			// Create an active idea
			store, err := idea.NewDirectoryStore(filepath.Join(journalDir, "idea"))
			c.Assume(err, IsNil)

			activeIdea := idea.Idea{
				Status: idea.IS_Active,
				Name:   "test idea",
				Body:   "test idea body\n",
			}

			commitable, err := store.SaveIdea(&activeIdea)
			c.Assume(err, IsNil)
			c.Assume(git.Commit(commitable), IsNil)

			cmd := NewCmd(nil)
			cmd.SetWd(journalDir)

			cmd.Now = func() time.Time {
				return time.Date(2015, 1, 1, 0, 0, 0, 0, time.UTC)
			}

			entryFilename := cmd.Now().Format(entry.FilenameLayout)

			editorProcessHasStarted := make(chan bool)
			expectationsChecked := make(chan bool)
			execCompleted := make(chan bool)

			cmd.EditorProcess = mockEditor{
				start: func() {
					editorProcessHasStarted <- true
				},
				wait: func() {
					<-expectationsChecked
				},
			}

			// Run `journal new` with mocked EditorProcess and Now functions
			go func() {
				c.Assume(cmd.Exec(nil), IsNil)
				execCompleted <- true
			}()

			<-editorProcessHasStarted

			// Entry will have that active idea as it is being editted
			f, err := os.OpenFile(filepath.Join(journalDir, "entry", entryFilename), os.O_RDONLY, 0600)
			c.Assume(err, IsNil)
			defer f.Close()

			ideaScanner := idea.NewIdeaScanner(f)
			ideaScanner.Scan()
			c.Assume(ideaScanner.Err(), IsNil)

			idea := ideaScanner.Idea()
			c.Assume(idea, Not(IsNil))
			c.Expect(*idea, Equals, activeIdea)

			// sync execution back up
			expectationsChecked <- true
			<-execCompleted
		})

		c.Specify("will update the idea store with any modifications made during editting", func() {
			// Create an idea
			store, err := idea.NewDirectoryStore(filepath.Join(journalDir, "idea"))
			c.Assume(err, IsNil)

			activeIdea := idea.Idea{
				Status: idea.IS_Active,
				Name:   "test idea",
				Body:   "test idea body\n",
			}

			commitable, err := store.SaveIdea(&activeIdea)
			c.Assume(err, IsNil)
			c.Assume(git.Commit(commitable), IsNil)

			cmd := NewCmd(nil)
			cmd.SetWd(journalDir)

			cmd.Now = func() time.Time {
				return time.Date(2015, 1, 1, 0, 0, 0, 0, time.UTC)
			}

			entryFilename := cmd.Now().Format(entry.FilenameLayout)

			sedCmd := exec.Command("sed", "-i", "s_active_inactive_", entryFilename)
			sedCmd.Dir = filepath.Join(journalDir, "entry")

			cmd.EditorProcess = sedCmd

			c.Expect(cmd.Exec(nil), IsNil)

			// Modify the status to reflect what happened during the edit
			activeIdea.Status = idea.IS_Inactive

			// Idea in the IdeaStore will be updated if it was editted
			idea, err := store.IdeaById(activeIdea.Id)
			c.Assume(err, IsNil)
			c.Expect(idea, Equals, activeIdea)
		})

		c.Specify("will commit the entry to the git repository", func() {
			// Run `new`
			// Will succeed
			// Entry will be shown in the git repository

			c.Specify("and will commit any modifications to the idea store", func() {
				// Any modifed Ideas will also have commits
			})
		})

		c.Specify("will append the current time after editting is completed", func() {
			// Run `new`
			// Will succeed
			// Entry will have closing time appended
		})

		c.Specify("will fail", func() {
			// Dirty the test journal
			c.Specify("if the journal directory has a dirty git repository", func() {
				// Run `new`
				// Will fail with an error
			})
		})
	})
}
<commit_msg>Spec for appending the ClosedAt timestamp to an entry<commit_after>package new

import (
	"bufio"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"time"

	"github.com/ghthor/journal/entry"
	"github.com/ghthor/journal/git"
	"github.com/ghthor/journal/idea"
	initialize "github.com/ghthor/journal/init"

	"github.com/ghthor/gospec"
	. "github.com/ghthor/gospec"
)

type mockEditor struct {
	start, wait func()
}

func (m mockEditor) Start() error {
	m.start()
	return nil
}

func (m mockEditor) Wait() error {
	m.wait()
	return nil
}

func DescribeNewCmd(c gospec.Context) {
	c.Specify("the `new` command", func() {
		// Create a temporary journal
		journalDir, err := ioutil.TempDir("", "new_cmd_desc_")
		c.Assume(err, IsNil)
		defer func() {
			c.Assume(os.RemoveAll(journalDir), IsNil)
		}()

		commitable, err := initialize.Journal(journalDir)
		c.Assume(err, IsNil)
		c.Assume(git.Commit(commitable), IsNil)

		c.Specify("will include any active ideas in the entries body while editting", func() {
			// Create an active idea
			store, err := idea.NewDirectoryStore(filepath.Join(journalDir, "idea"))
			c.Assume(err, IsNil)

			activeIdea := idea.Idea{
				Status: idea.IS_Active,
				Name:   "test idea",
				Body:   "test idea body\n",
			}

			commitable, err := store.SaveIdea(&activeIdea)
			c.Assume(err, IsNil)
			c.Assume(git.Commit(commitable), IsNil)

			cmd := NewCmd(nil)
			cmd.SetWd(journalDir)

			cmd.Now = func() time.Time {
				return time.Date(2015, 1, 1, 0, 0, 0, 0, time.UTC)
			}

			entryFilename := cmd.Now().Format(entry.FilenameLayout)

			editorProcessHasStarted := make(chan bool)
			expectationsChecked := make(chan bool)
			execCompleted := make(chan bool)

			cmd.EditorProcess = mockEditor{
				start: func() {
					editorProcessHasStarted <- true
				},
				wait: func() {
					<-expectationsChecked
				},
			}

			// Run `journal new` with mocked EditorProcess and Now functions
			go func() {
				c.Assume(cmd.Exec(nil), IsNil)
				execCompleted <- true
			}()

			<-editorProcessHasStarted

			// Entry will have that active idea as it is being editted
			f, err := os.OpenFile(filepath.Join(journalDir, "entry", entryFilename), os.O_RDONLY, 0600)
			c.Assume(err, IsNil)
			defer f.Close()

			ideaScanner := idea.NewIdeaScanner(f)
			ideaScanner.Scan()
			c.Assume(ideaScanner.Err(), IsNil)

			idea := ideaScanner.Idea()
			c.Assume(idea, Not(IsNil))
			c.Expect(*idea, Equals, activeIdea)

			// sync execution back up
			expectationsChecked <- true
			<-execCompleted
		})

		c.Specify("will update the idea store with any modifications made during editting", func() {
			// Create an idea
			store, err := idea.NewDirectoryStore(filepath.Join(journalDir, "idea"))
			c.Assume(err, IsNil)

			activeIdea := idea.Idea{
				Status: idea.IS_Active,
				Name:   "test idea",
				Body:   "test idea body\n",
			}

			commitable, err := store.SaveIdea(&activeIdea)
			c.Assume(err, IsNil)
			c.Assume(git.Commit(commitable), IsNil)

			cmd := NewCmd(nil)
			cmd.SetWd(journalDir)

			cmd.Now = func() time.Time {
				return time.Date(2015, 1, 1, 0, 0, 0, 0, time.UTC)
			}

			entryFilename := cmd.Now().Format(entry.FilenameLayout)

			sedCmd := exec.Command("sed", "-i", "s_active_inactive_", entryFilename)
			sedCmd.Dir = filepath.Join(journalDir, "entry")

			cmd.EditorProcess = sedCmd

			c.Expect(cmd.Exec(nil), IsNil)

			// Modify the status to reflect what happened during the edit
			activeIdea.Status = idea.IS_Inactive

			// Idea in the IdeaStore will be updated if it was editted
			idea, err := store.IdeaById(activeIdea.Id)
			c.Assume(err, IsNil)
			c.Expect(idea, Equals, activeIdea)
		})

		c.Specify("will append the current time after editting is completed", func() {
			cmd := NewCmd(nil)
			cmd.SetWd(journalDir)

			// Mock time to control the filename and openedAt/closedAt times stored in the entry
			openedAt := time.Date(2015, 1, 1, 0, 0, 0, 0, time.UTC)
			closedAt := time.Date(2015, 1, 2, 0, 0, 0, 0, time.UTC)

			// This is a roundabout mock from hell...but it works...
			// TODO figure more elegant mock for this
			var nowFn func() time.Time
			nowFn = func() time.Time {
				// Mutate during first call to return ClosedAt time
				nowFn = func() time.Time { return closedAt }
				// OpenedAt time
				return openedAt
			}
			cmd.Now = func() time.Time { return nowFn() }

			entryFilename := openedAt.Format(entry.FilenameLayout)

			// Mocked editor that does nothing
			cmd.EditorProcess = mockEditor{
				start: func() {},
				wait:  func() {},
			}

			// Run `journal new` with mocked EditorProcess and Now functions
			c.Assume(cmd.Exec(nil), IsNil)

			// Entry will have closing time appended
			f, err := os.OpenFile(filepath.Join(journalDir, "entry", entryFilename), os.O_RDONLY, 0600)
			c.Assume(err, IsNil)
			defer f.Close()

			scanner := bufio.NewScanner(f)
			scanner.Split(bufio.ScanLines)

			var prevLine string
			for scanner.Scan() {
				c.Assume(scanner.Err(), IsNil)
				prevLine = scanner.Text()
			}

			t, err := time.Parse(time.UnixDate, prevLine)
			c.Assume(err, IsNil)
			c.Expect(t, Equals, closedAt)
		})

		c.Specify("will commit the entry to the git repository", func() {
			// Run `new`
			// Will succeed
			// Entry will be shown in the git repository

			c.Specify("and will commit any modifications to the idea store", func() {
				// Any modifed Ideas will also have commits
			})
		})

		c.Specify("will append the current time after editting is completed", func() {
			// Run `new`
			// Will succeed
			// Entry will have closing time appended
		})

		c.Specify("will fail", func() {
			// Dirty the test journal
			c.Specify("if the journal directory has a dirty git repository", func() {
				// Run `new`
				// Will fail with an error
			})
		})
	})
}
<|endoftext|>


<commit_before>package utils

import "os/exec"

// GitRepo type manages a VCS repository with Git
type GitRepo struct {
	// Local path to the repository
	Path string

	// Upstream URL of the Git repository
	Upstream string

	// Path to the Git tool
	git string
}

// NewGitRepo creates a new Git repository
func NewGitRepo(path, upstream string) (*GitRepo, error) {
	path, err := exec.LookPath("git")
	if err != nil {
		return nil, err
	}

	repo := &GitRepo{
		Path:     path,
		Upstream: upstream,
		cmd:      path,
	}

	return repo, nil
}

// Fetch fetches from the given remote
func (gr *GitRepo) Fetch(remote string) ([]byte, error) {
	return exec.Command(gr.git, "fetch", remote).CombinedOutput()
}

// Pull pulls from the given remote and merges changes into the
// local branch
func (gr *GitRepo) Pull(remote, branch string) ([]byte, error) {
	out, err := exec.Command(gr.git, "checkout", branch).CombinedOut()
	if err != nil {
		return out, err
	}

	return exec.Command(gr.git, "pull", remote).CombinedOutput()
}
<commit_msg>utils: implement CheckoutBranch method on GitRepo type<commit_after>package utils

import "os/exec"

// GitRepo type manages a VCS repository with Git
type GitRepo struct {
	// Local path to the repository
	Path string

	// Upstream URL of the Git repository
	Upstream string

	// Path to the Git tool
	git string
}

// NewGitRepo creates a new Git repository
func NewGitRepo(path, upstream string) (*GitRepo, error) {
	path, err := exec.LookPath("git")
	if err != nil {
		return nil, err
	}

	repo := &GitRepo{
		Path:     path,
		Upstream: upstream,
		cmd:      path,
	}

	return repo, nil
}

// Fetch fetches from the given remote
func (gr *GitRepo) Fetch(remote string) ([]byte, error) {
	return exec.Command(gr.git, "fetch", remote).CombinedOutput()
}

// Pull pulls from the given remote and merges changes into the
// local branch
func (gr *GitRepo) Pull(remote, branch string) ([]byte, error) {
	out, err := gr.CheckoutBranch(branch)
	if err != nil {
		return out, err
	}

	return exec.Command(gr.git, "pull", remote).CombinedOutput()
}

// CheckoutBranch checks out a given local branch
func (gr *GitRepo) CheckoutBranch(branch string) ([]byte, error) {
	return exec.Command(gr.git, "checkout", branch).CombinedOutput()
}
<|endoftext|>


<commit_before>package sort

import (
	"fmt"
	"testing"

	"github.com/joshuarubin/gil"
	. "github.com/smartystreets/goconvey/convey"
)

func genList(size int) []gil.Interface {
	if size < 0 {
		return nil
	}

	list := gil.CopyToIntSlice([]int{
		62, 34, 10, 27, 62,
		24, 11, 99, 71, 71,
		45, 83, 71, 18, 29,
		62, 8, 54, 3, 41,
		91, 42, 1, 74, 7,
		81, 14, 73, 56, 47,
		19, 78, 65, 10, 35,
	})

	if size <= len(list) {
		return list[:size]
	}

	return list
}

func TestMergeSort(t *testing.T) {
	for _, size := range []int{-1, 0, 1, 2, 3, 10, 100} {
		list := genList(size)
		sorted, err := Merge(list)
		testSorted(t, "Merge", size, list, sorted, err)
	}
}

func TestQuickSort(t *testing.T) {
	for _, size := range []int{-1, 0, 1, 2, 3, 10, 100} {
		list := genList(size)
		sorted := make([]gil.Interface, len(list))
		copy(sorted, list)
		err := Quick(sorted)
		testSorted(t, "Quick", size, list, sorted, err)
	}
}

func testSorted(t *testing.T, algo string, size int, list, sorted []gil.Interface, err error) {
	Convey(fmt.Sprintf("For the %sSort algorithm (size %d)", algo, size), t, func() {
		Convey("There should be no error", func() {
			So(err, ShouldBeNil)
		})

		Convey("Original list should not be modified", func() {
			So(list, ShouldResemble, genList(size))
		})

		Convey("The length should not change", func() {
			So(len(sorted), ShouldEqual, len(list))
		})

		if size > 0 {
			Convey("The values should be in order", func() {
				prev := sorted[0]
				for _, val := range sorted[1:] {
					less, err := val.Less(prev)

					So(err, ShouldBeNil)
					So(less, ShouldBeFalse)

					prev = val
				}
			})
		}

		Convey("No values should be missing", func() {
			for _, val := range list {
				So(sorted, ShouldContain, val)
			}
		})

		Convey("No values should be added", func() {
			for _, val := range sorted {
				So(list, ShouldContain, val)
			}
		})
	})
}
<commit_msg>add sort benchmarks<commit_after>package sort

import (
	"fmt"
	"math/rand"
	"testing"

	"github.com/joshuarubin/gil"
	. "github.com/smartystreets/goconvey/convey"
)

func genList(size int) []gil.Interface {
	if size < 0 {
		return nil
	}

	list := gil.CopyToIntSlice([]int{
		62, 34, 10, 27, 62,
		24, 11, 99, 71, 71,
		45, 83, 71, 18, 29,
		62, 8, 54, 3, 41,
		91, 42, 1, 74, 7,
		81, 14, 73, 56, 47,
		19, 78, 65, 10, 35,
	})

	if size <= len(list) {
		return list[:size]
	}

	return list
}

func TestMergeSort(t *testing.T) {
	for _, size := range []int{-1, 0, 1, 2, 3, 10, 100} {
		list := genList(size)
		sorted, err := Merge(list)
		testSorted(t, "Merge", size, list, sorted, err)
	}
}

func TestQuickSort(t *testing.T) {
	for _, size := range []int{-1, 0, 1, 2, 3, 10, 100} {
		list := genList(size)
		sorted := make([]gil.Interface, len(list))
		copy(sorted, list)
		err := Quick(sorted)
		testSorted(t, "Quick", size, list, sorted, err)
	}
}

func testSorted(t *testing.T, algo string, size int, list, sorted []gil.Interface, err error) {
	Convey(fmt.Sprintf("For the %sSort algorithm (size %d)", algo, size), t, func() {
		Convey("There should be no error", func() {
			So(err, ShouldBeNil)
		})

		Convey("Original list should not be modified", func() {
			So(list, ShouldResemble, genList(size))
		})

		Convey("The length should not change", func() {
			So(len(sorted), ShouldEqual, len(list))
		})

		if size > 0 {
			Convey("The values should be in order", func() {
				prev := sorted[0]
				for _, val := range sorted[1:] {
					less, err := val.Less(prev)

					So(err, ShouldBeNil)
					So(less, ShouldBeFalse)

					prev = val
				}
			})
		}

		Convey("No values should be missing", func() {
			for _, val := range list {
				So(sorted, ShouldContain, val)
			}
		})

		Convey("No values should be added", func() {
			for _, val := range sorted {
				So(list, ShouldContain, val)
			}
		})
	})
}

func benchmarkList(size int) []gil.Interface {
	list := make([]gil.Interface, size)
	for i := 0; i < size; i++ {
		list[i] = gil.Int(rand.Int())
	}
	return list
}

func BenchmarkMergeSort(b *testing.B) {
	list := benchmarkList(2 ^ 14)
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		b.StopTimer()
		copyOfList := make([]gil.Interface, len(list))
		copy(copyOfList, list)
		b.StartTimer()

		Merge(copyOfList)
	}
}

func BenchmarkQuickSort(b *testing.B) {
	list := benchmarkList(2 ^ 14)
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		b.StopTimer()
		copyOfList := make([]gil.Interface, len(list))
		copy(copyOfList, list)
		b.StartTimer()

		Quick(copyOfList)
	}
}
<|endoftext|>