<commit_before>// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pilosa

import (
	"errors"
	"fmt"
	"regexp"
	"strconv"
	"strings"
)

var schemeRegexp = regexp.MustCompile("^[+a-z]+$")
var hostRegexp = regexp.MustCompile("^[0-9a-z.-]+$|^\\[[:0-9a-fA-F]+\\]$")
var addressRegexp = regexp.MustCompile("^(([+a-z]+):\\/\\/)?([0-9a-z.-]+|\\[[:0-9a-fA-F]+\\])?(:([0-9]+))?$")

// URI represents a Pilosa URI.
// A Pilosa URI consists of three parts:
// 1) Scheme: Protocol of the URI. Default: http.
// 2) Host: Hostname or IP URI. Default: localhost. IPv6 addresses should be written in brackets, e.g., `[fd42:4201:f86b:7e09:216:3eff:fefa:ed80]`.
// 3) Port: Port of the URI. Default: 10101.
//
// All parts of the URI are optional. The following are equivalent:
// 	http://localhost:10101
// 	http://localhost
// 	http://:10101
// 	localhost:10101
// 	localhost
// 	:10101
type URI struct {
	scheme string
	host   string
	port   uint16
}

// DefaultURI creates and returns the default URI.
func DefaultURI() *URI {
	return &URI{
		scheme: "http",
		host:   "localhost",
		port:   10101,
	}
}

// NewURIFromHostPort returns a URI with specified host and port.
func NewURIFromHostPort(host string, port uint16) (*URI, error) {
	uri := DefaultURI()
	err := uri.SetHost(host)
	if err != nil {
		return nil, err
	}
	uri.SetPort(port)
	return uri, nil
}

// NewURIFromAddress parses the passed address and returns a URI.
func NewURIFromAddress(address string) (*URI, error) {
	uri, err := parseAddress(address)
	if err != nil {
		return nil, err
	}
	return uri, err
}

// Scheme returns the scheme of this URI.
func (u *URI) Scheme() string {
	return u.scheme
}

// SetScheme sets the scheme of this URI.
func (u *URI) SetScheme(scheme string) error {
	m := schemeRegexp.FindStringSubmatch(scheme)
	if m == nil {
		return errors.New("invalid scheme")
	}
	u.scheme = scheme
	return nil
}

// Host returns the host of this URI.
func (u *URI) Host() string {
	return u.host
}

// SetHost sets the host of this URI.
func (u *URI) SetHost(host string) error {
	m := hostRegexp.FindStringSubmatch(host)
	if m == nil {
		return errors.New("invalid host")
	}
	u.host = host
	return nil
}

// Port returns the port of this URI.
func (u *URI) Port() uint16 {
	return u.port
}

// SetPort sets the port of this URI.
func (u *URI) SetPort(port uint16) {
	u.port = port
}

// HostPort returns `Host:Port`
func (u *URI) HostPort() string {
	// XXX: The following is just to make TestHandler_Status; remove it
	if u == nil {
		return ""
	}
	s := fmt.Sprintf("%s:%d", u.host, u.port)
	return s
}

// Normalize returns the address in a form usable by a HTTP client.
func (u *URI) Normalize() string {
	scheme := u.scheme
	index := strings.Index(scheme, "+")
	if index >= 0 {
		scheme = scheme[:index]
	}
	return fmt.Sprintf("%s://%s:%d", scheme, u.host, u.port)
}

// Equals returns true if the checked URI is equivalent to this URI.
func (u URI) Equals(other *URI) bool {
	if other == nil {
		return false
	}
	return u.scheme == other.scheme &&
		u.host == other.host &&
		u.port == other.port
}

// Path returns URI with path
func (u *URI) Path(path string) string {
	return fmt.Sprintf("%s%s", u.Normalize(), path)
}

// The following methods are required to implement pflag Value interface.

// Set sets the time quantum value.
func (u *URI) Set(value string) error {
	uri, err := NewURIFromAddress(value)
	if err != nil {
		return err
	}
	*u = *uri
	return nil
}

// Type returns the type of a time quantum value.
func (u URI) Type() string {
	return "URI"
}

func parseAddress(address string) (uri *URI, err error) {
	m := addressRegexp.FindStringSubmatch(address)
	if m == nil {
		return nil, errors.New("invalid address")
	}
	scheme := "http"
	if m[2] != "" {
		scheme = m[2]
	}
	host := "localhost"
	if m[3] != "" {
		host = m[3]
	}
	var port = 10101
	if m[5] != "" {
		port, err = strconv.Atoi(m[5])
		if err != nil {
			return nil, errors.New("converting port string to int")
		}
	}
	uri = &URI{
		scheme: scheme,
		host:   host,
		port:   uint16(port),
	}
	return uri, nil
}
<commit_msg>adding uri to string method<commit_after>// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pilosa

import (
	"errors"
	"fmt"
	"regexp"
	"strconv"
	"strings"
)

var schemeRegexp = regexp.MustCompile("^[+a-z]+$")
var hostRegexp = regexp.MustCompile("^[0-9a-z.-]+$|^\\[[:0-9a-fA-F]+\\]$")
var addressRegexp = regexp.MustCompile("^(([+a-z]+):\\/\\/)?([0-9a-z.-]+|\\[[:0-9a-fA-F]+\\])?(:([0-9]+))?$")

// URI represents a Pilosa URI.
// A Pilosa URI consists of three parts:
// 1) Scheme: Protocol of the URI. Default: http.
// 2) Host: Hostname or IP URI. Default: localhost. IPv6 addresses should be written in brackets, e.g., `[fd42:4201:f86b:7e09:216:3eff:fefa:ed80]`.
// 3) Port: Port of the URI. Default: 10101.
//
// All parts of the URI are optional. The following are equivalent:
// 	http://localhost:10101
// 	http://localhost
// 	http://:10101
// 	localhost:10101
// 	localhost
// 	:10101
type URI struct {
	scheme string
	host   string
	port   uint16
}

// DefaultURI creates and returns the default URI.
func DefaultURI() *URI {
	return &URI{
		scheme: "http",
		host:   "localhost",
		port:   10101,
	}
}

// NewURIFromHostPort returns a URI with specified host and port.
func NewURIFromHostPort(host string, port uint16) (*URI, error) {
	uri := DefaultURI()
	err := uri.SetHost(host)
	if err != nil {
		return nil, err
	}
	uri.SetPort(port)
	return uri, nil
}

// NewURIFromAddress parses the passed address and returns a URI.
func NewURIFromAddress(address string) (*URI, error) {
	uri, err := parseAddress(address)
	if err != nil {
		return nil, err
	}
	return uri, err
}

// Scheme returns the scheme of this URI.
func (u *URI) Scheme() string {
	return u.scheme
}

// SetScheme sets the scheme of this URI.
func (u *URI) SetScheme(scheme string) error {
	m := schemeRegexp.FindStringSubmatch(scheme)
	if m == nil {
		return errors.New("invalid scheme")
	}
	u.scheme = scheme
	return nil
}

// Host returns the host of this URI.
func (u *URI) Host() string {
	return u.host
}

// SetHost sets the host of this URI.
func (u *URI) SetHost(host string) error {
	m := hostRegexp.FindStringSubmatch(host)
	if m == nil {
		return errors.New("invalid host")
	}
	u.host = host
	return nil
}

// Port returns the port of this URI.
func (u *URI) Port() uint16 {
	return u.port
}

// SetPort sets the port of this URI.
func (u *URI) SetPort(port uint16) {
	u.port = port
}

// HostPort returns `Host:Port`
func (u *URI) HostPort() string {
	// XXX: The following is just to make TestHandler_Status; remove it
	if u == nil {
		return ""
	}
	s := fmt.Sprintf("%s:%d", u.host, u.port)
	return s
}

// Normalize returns the address in a form usable by a HTTP client.
func (u *URI) Normalize() string {
	scheme := u.scheme
	index := strings.Index(scheme, "+")
	if index >= 0 {
		scheme = scheme[:index]
	}
	return fmt.Sprintf("%s://%s:%d", scheme, u.host, u.port)
}

// String returns the address as a string.
func (u URI) String() string {
	return fmt.Sprintf("%s://%s:%d", u.scheme, u.host, u.port)
}

// Equals returns true if the checked URI is equivalent to this URI.
func (u URI) Equals(other *URI) bool {
	if other == nil {
		return false
	}
	return u.scheme == other.scheme &&
		u.host == other.host &&
		u.port == other.port
}

// Path returns URI with path
func (u *URI) Path(path string) string {
	return fmt.Sprintf("%s%s", u.Normalize(), path)
}

// The following methods are required to implement pflag Value interface.

// Set sets the time quantum value.
func (u *URI) Set(value string) error {
	uri, err := NewURIFromAddress(value)
	if err != nil {
		return err
	}
	*u = *uri
	return nil
}

// Type returns the type of a time quantum value.
func (u URI) Type() string {
	return "URI"
}

func parseAddress(address string) (uri *URI, err error) {
	m := addressRegexp.FindStringSubmatch(address)
	if m == nil {
		return nil, errors.New("invalid address")
	}
	scheme := "http"
	if m[2] != "" {
		scheme = m[2]
	}
	host := "localhost"
	if m[3] != "" {
		host = m[3]
	}
	var port = 10101
	if m[5] != "" {
		port, err = strconv.Atoi(m[5])
		if err != nil {
			return nil, errors.New("converting port string to int")
		}
	}
	uri = &URI{
		scheme: scheme,
		host:   host,
		port:   uint16(port),
	}
	return uri, nil
}
<|endoftext|>


<commit_before>package dexcom

import (
	"github.com/ecc1/usbserial"
)

const (
	// hex IDs that should match the idVendor and idProduct files
	// in /sys/bus/usb/devices
	dexcomVendor  = "22a3"
	dexcomProduct = "0047"
)

type usbConn struct {
	port usbserial.Port
}

// Open locates the USB device for a Dexcom CGM receiver and opens it.
func OpenUSB() (Connection, error) {
	port, err := usbserial.Open(dexcomVendor, dexcomProduct)
	if err != nil {
		return nil, err
	}
	return &usbConn{port: port}, nil
}

func (conn usbConn) Send(data []byte) error {
	return conn.port.Write(data)
}

func (conn usbConn) Receive(data []byte) error {
	return conn.port.Read(data)
}

func (conn usbConn) Close() {
	conn.port.Close()
}
<commit_msg>Use pointer to usbserial.Port<commit_after>package dexcom

import (
	"github.com/ecc1/usbserial"
)

const (
	// hex IDs that should match the idVendor and idProduct files
	// in /sys/bus/usb/devices
	dexcomVendor  = "22a3"
	dexcomProduct = "0047"
)

type usbConn struct {
	port *usbserial.Port
}

// Open locates the USB device for a Dexcom CGM receiver and opens it.
func OpenUSB() (Connection, error) {
	port, err := usbserial.Open(dexcomVendor, dexcomProduct)
	if err != nil {
		return nil, err
	}
	return &usbConn{port: port}, nil
}

func (conn usbConn) Send(data []byte) error {
	return conn.port.Write(data)
}

func (conn usbConn) Receive(data []byte) error {
	return conn.port.Read(data)
}

func (conn usbConn) Close() {
	conn.port.Close()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
)

// Convenient function to exec a command.
func cmd(name string, args ...string) error {
	cmd := exec.Command(name, args...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

// Convenient function to launch VBoxManage.
func vbm(args ...string) error {
	return cmd(B2D.VBM, args...)
}

// TODO: delete the hostonlyif and dhcpserver when we delete the vm! (need to
// reference count to make sure there are no other vms relying on them)

// Get or create the hostonly network interface
func getHostOnlyNetworkInterface() (string, error) {
	// Check if the interface exists.
	out, err := exec.Command(B2D.VBM, "list", "hostonlyifs").Output()
	if err != nil {
		return "", err
	}
	lists := regexp.MustCompile(`(?m)^(Name|IPAddress|VBoxNetworkName):\s+(.+?)\r?$`).FindAllSubmatch(out, -1)
	var ifname string
	index := 0

	for ifname == "" && len(lists) > index {
		if string(lists[index+1][2]) == B2D.HostIP {
			//test to see that the dhcpserver is the same too
			out, err := exec.Command(B2D.VBM, "list", "dhcpservers").Output()
			if err != nil {
				return "", err
			}
			//WARNING: this relies on the order the virtualbox gives not changing
			dhcp := regexp.MustCompile(`(?m)^(NetworkName|IP|NetworkMask|lowerIPAddress|upperIPAddress|Enabled):\s+(.+?)\r?$`).FindAllSubmatch(out, -1)
			i := 0

			for ifname == "" && len(dhcp) > i {
				if string(dhcp[i][2]) == string(lists[index+2][2]) &&
					string(dhcp[i+1][2]) == B2D.DHCPIP &&
					string(dhcp[i+2][2]) == B2D.NetworkMask &&
					string(dhcp[i+3][2]) == B2D.LowerIPAddress &&
					string(dhcp[i+4][2]) == B2D.UpperIPAddress &&
					string(dhcp[i+5][2]) == B2D.DHCPEnabled {
					ifname = string(lists[index][2])
					fmt.Printf("Reusing hostonly network interface %s\n", ifname)
				}

				i = i + 5
			}

		}
		index = index + 3
	}

	if ifname == "" {
		//create it all fresh
		fmt.Printf("Creating a new hostonly network interface\n")
		out, err = exec.Command(B2D.VBM, "hostonlyif", "create").Output()
		if err != nil {
			return "", err
		}
		groups := regexp.MustCompile(`(?m)^Interface '(\w+)' was successfully created`).FindSubmatch(out)
		if len(groups) < 2 {
			return "", err
		}
		ifname = string(groups[1])
		out, err = exec.Command(B2D.VBM, "dhcpserver", "add",
			"--ifname", ifname,
			"--ip", B2D.DHCPIP,
			"--netmask", B2D.NetworkMask,
			"--lowerip", B2D.LowerIPAddress,
			"--upperip", B2D.UpperIPAddress,
			"--enable",
		).Output()
		if err != nil {
			return "", err
		}
		out, err = exec.Command(B2D.VBM, "hostonlyif", "ipconfig", ifname,
			"--ip", B2D.HostIP,
			"--netmask", B2D.NetworkMask,
		).Output()
		if err != nil {
			return "", err
		}
	}
	return ifname, nil
}

// Get the state of a VM.
func status(vm string) vmState {
	// Check if the VM exists.
	out, err := exec.Command(B2D.VBM, "list", "vms").Output()
	if err != nil {
		if err.(*exec.Error).Err == exec.ErrNotFound {
			return vmVBMNotFound
		}
		return vmUnknown
	}
	found, err := regexp.Match(fmt.Sprintf(`(?m)^"%s"`, regexp.QuoteMeta(vm)), out)
	if err != nil {
		return vmUnknown
	}
	if !found {
		return vmUnregistered
	}

	if out, err = exec.Command(B2D.VBM, "showvminfo", vm, "--machinereadable").Output(); err != nil {
		if err.(*exec.Error).Err == exec.ErrNotFound {
			return vmVBMNotFound
		}
		return vmUnknown
	}
	groups := regexp.MustCompile(`(?m)^VMState="(\w+)"\r?$`).FindSubmatch(out)
	if len(groups) < 2 {
		return vmUnknown
	}
	switch state := vmState(groups[1]); state {
	case vmRunning, vmPaused, vmSaved, vmPoweroff, vmAborted:
		return state
	default:
		return vmUnknown
	}
}

// Get the VirtualBox base folder of the VM.
func basefolder(vm string) string {
	out, err := exec.Command(B2D.VBM, "showvminfo", vm, "--machinereadable").Output()
	if err != nil {
		return ""
	}
	groups := regexp.MustCompile(`(?m)^CfgFile="(.+)"\r?$`).FindSubmatch(out)
	if len(groups) < 2 {
		return ""
	}
	return filepath.Dir(string(groups[1]))
}

// Make a boot2docker VM disk image with the given size (in MB).
func makeDiskImage(dest string, size uint) error {
	// Create the dest dir.
	if err := os.MkdirAll(filepath.Dir(dest), 0755); err != nil {
		return err
	}

	// Convert a raw image from stdin to the dest VMDK image.
	sizeBytes := int64(size) * 1024 * 1024 // usually won't fit in 32-bit int
	cmd := exec.Command(B2D.VBM, "convertfromraw", "stdin", dest,
		fmt.Sprintf("%d", sizeBytes), "--format", "VMDK")
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	stdin, err := cmd.StdinPipe()
	if err != nil {
		return err
	}
	if err := cmd.Start(); err != nil {
		return err
	}

	// Fill in the magic string so boot2docker VM will detect this and format
	// the disk upon first boot.
	magic := []byte("boot2docker, please format-me")
	if _, err := stdin.Write(magic); err != nil {
		return err
	}
	// The total number of bytes written to stdin must match sizeBytes, or
	// VBoxManage.exe on Windows will fail.
	if err := zeroFill(stdin, sizeBytes-int64(len(magic))); err != nil {
		return err
	}
	// cmd won't exit until the stdin is closed.
	if err := stdin.Close(); err != nil {
		return err
	}

	return cmd.Wait()
}

// Write n zero bytes into w.
func zeroFill(w io.Writer, n int64) (err error) {
	const blocksize = 32 * 1024
	zeros := make([]byte, blocksize)
	var k int
	for n > 0 {
		if n > blocksize {
			k, err = w.Write(zeros)
		} else {
			k, err = w.Write(zeros[:n])
		}
		if err != nil {
			return
		}
		n -= int64(k)
	}
	return
}
<commit_msg>Safe error type switch<commit_after>package main

import (
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
)

// Convenient function to exec a command.
func cmd(name string, args ...string) error {
	cmd := exec.Command(name, args...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

// Convenient function to launch VBoxManage.
func vbm(args ...string) error {
	return cmd(B2D.VBM, args...)
}

// TODO: delete the hostonlyif and dhcpserver when we delete the vm! (need to
// reference count to make sure there are no other vms relying on them)

// Get or create the hostonly network interface
func getHostOnlyNetworkInterface() (string, error) {
	// Check if the interface exists.
	out, err := exec.Command(B2D.VBM, "list", "hostonlyifs").Output()
	if err != nil {
		return "", err
	}
	lists := regexp.MustCompile(`(?m)^(Name|IPAddress|VBoxNetworkName):\s+(.+?)\r?$`).FindAllSubmatch(out, -1)
	var ifname string
	index := 0

	for ifname == "" && len(lists) > index {
		if string(lists[index+1][2]) == B2D.HostIP {
			//test to see that the dhcpserver is the same too
			out, err := exec.Command(B2D.VBM, "list", "dhcpservers").Output()
			if err != nil {
				return "", err
			}
			//WARNING: this relies on the order the virtualbox gives not changing
			dhcp := regexp.MustCompile(`(?m)^(NetworkName|IP|NetworkMask|lowerIPAddress|upperIPAddress|Enabled):\s+(.+?)\r?$`).FindAllSubmatch(out, -1)
			i := 0

			for ifname == "" && len(dhcp) > i {
				if string(dhcp[i][2]) == string(lists[index+2][2]) &&
					string(dhcp[i+1][2]) == B2D.DHCPIP &&
					string(dhcp[i+2][2]) == B2D.NetworkMask &&
					string(dhcp[i+3][2]) == B2D.LowerIPAddress &&
					string(dhcp[i+4][2]) == B2D.UpperIPAddress &&
					string(dhcp[i+5][2]) == B2D.DHCPEnabled {
					ifname = string(lists[index][2])
					fmt.Printf("Reusing hostonly network interface %s\n", ifname)
				}

				i = i + 5
			}

		}
		index = index + 3
	}

	if ifname == "" {
		//create it all fresh
		fmt.Printf("Creating a new hostonly network interface\n")
		out, err = exec.Command(B2D.VBM, "hostonlyif", "create").Output()
		if err != nil {
			return "", err
		}
		groups := regexp.MustCompile(`(?m)^Interface '(\w+)' was successfully created`).FindSubmatch(out)
		if len(groups) < 2 {
			return "", err
		}
		ifname = string(groups[1])
		out, err = exec.Command(B2D.VBM, "dhcpserver", "add",
			"--ifname", ifname,
			"--ip", B2D.DHCPIP,
			"--netmask", B2D.NetworkMask,
			"--lowerip", B2D.LowerIPAddress,
			"--upperip", B2D.UpperIPAddress,
			"--enable",
		).Output()
		if err != nil {
			return "", err
		}
		out, err = exec.Command(B2D.VBM, "hostonlyif", "ipconfig", ifname,
			"--ip", B2D.HostIP,
			"--netmask", B2D.NetworkMask,
		).Output()
		if err != nil {
			return "", err
		}
	}
	return ifname, nil
}

// Get the state of a VM.
func status(vm string) vmState {
	// Check if the VM exists.
	out, err := exec.Command(B2D.VBM, "list", "vms").Output()
	if err != nil {
		switch err := err.(type) {
		case *exec.Error:
			if err.Err == exec.ErrNotFound {
				return vmVBMNotFound
			}
		}
		return vmUnknown
	}
	found, err := regexp.Match(fmt.Sprintf(`(?m)^"%s"`, regexp.QuoteMeta(vm)), out)
	if err != nil {
		return vmUnknown
	}
	if !found {
		return vmUnregistered
	}

	if out, err = exec.Command(B2D.VBM, "showvminfo", vm, "--machinereadable").Output(); err != nil {
		switch err := err.(type) {
		case *exec.Error:
			if err.Err == exec.ErrNotFound {
				return vmVBMNotFound
			}
		}
		return vmUnknown
	}
	groups := regexp.MustCompile(`(?m)^VMState="(\w+)"\r?$`).FindSubmatch(out)
	if len(groups) < 2 {
		return vmUnknown
	}
	switch state := vmState(groups[1]); state {
	case vmRunning, vmPaused, vmSaved, vmPoweroff, vmAborted:
		return state
	default:
		return vmUnknown
	}
}

// Get the VirtualBox base folder of the VM.
func basefolder(vm string) string {
	out, err := exec.Command(B2D.VBM, "showvminfo", vm, "--machinereadable").Output()
	if err != nil {
		return ""
	}
	groups := regexp.MustCompile(`(?m)^CfgFile="(.+)"\r?$`).FindSubmatch(out)
	if len(groups) < 2 {
		return ""
	}
	return filepath.Dir(string(groups[1]))
}

// Make a boot2docker VM disk image with the given size (in MB).
func makeDiskImage(dest string, size uint) error {
	// Create the dest dir.
	if err := os.MkdirAll(filepath.Dir(dest), 0755); err != nil {
		return err
	}

	// Convert a raw image from stdin to the dest VMDK image.
	sizeBytes := int64(size) * 1024 * 1024 // usually won't fit in 32-bit int
	cmd := exec.Command(B2D.VBM, "convertfromraw", "stdin", dest,
		fmt.Sprintf("%d", sizeBytes), "--format", "VMDK")
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	stdin, err := cmd.StdinPipe()
	if err != nil {
		return err
	}
	if err := cmd.Start(); err != nil {
		return err
	}

	// Fill in the magic string so boot2docker VM will detect this and format
	// the disk upon first boot.
	magic := []byte("boot2docker, please format-me")
	if _, err := stdin.Write(magic); err != nil {
		return err
	}
	// The total number of bytes written to stdin must match sizeBytes, or
	// VBoxManage.exe on Windows will fail.
	if err := zeroFill(stdin, sizeBytes-int64(len(magic))); err != nil {
		return err
	}
	// cmd won't exit until the stdin is closed.
	if err := stdin.Close(); err != nil {
		return err
	}

	return cmd.Wait()
}

// Write n zero bytes into w.
func zeroFill(w io.Writer, n int64) (err error) {
	const blocksize = 32 * 1024
	zeros := make([]byte, blocksize)
	var k int
	for n > 0 {
		if n > blocksize {
			k, err = w.Write(zeros)
		} else {
			k, err = w.Write(zeros[:n])
		}
		if err != nil {
			return
		}
		n -= int64(k)
	}
	return
}
<|endoftext|>


<commit_before>package main

import (
	"net/http"

	m "github.com/CasualSuperman/Diorite/multiverse"
	gs "github.com/CasualSuperman/GoSocket"
)

type server struct {
	multiverse *m.Multiverse
}

func (s *server) UpdateMultiverse(mlt *m.Multiverse) {
	s.multiverse = mlt
}

func NewServer(mlt *m.Multiverse) *server {
	return &server{mlt}
}

func (s *server) Serve(port string, exit chan exitSignal) {
	socketServer := gs.NewServer()
	http.Handle("/gs/", socketServer)

	socketServer.Handle("nameSearch", s.fuzzyNameSearch)
	socketServer.Handle("card", s.sendCard)
	socketServer.Errored(func(err error) {
		println(err.Error())
	})

	if !*debug {
		socketServer.On(gs.Disconnect, func(c *gs.Conn) {
			exit <- exitSignal{0, "Web interface closed."}
		})
	}

	http.Handle("/", http.FileServer(http.Dir("./web/static")))
	http.ListenAndServe(port, nil)
}

type wsRequest struct {
	Type    string
	Request string
}

type wsResponse struct {
	Type     string
	Response interface{}
}

type webCard struct {
	Name         string
	MultiverseID int
}

func (s *server) fuzzyNameSearch(msg gs.Msg) {
	var searchTerm string
	msg.Receive(&searchTerm)

	results := s.multiverse.FuzzyNameSearch(searchTerm, 15)
	cards := make([]webCard, len(results))
	for i, card := range results {
		cards[i].Name = card.Name
		cards[i].MultiverseID = int(card.Printings[len(card.Printings)-1].ID)
	}

	msg.Respond(cards)
}

func (s *server) sendCard(msg gs.Msg) {
	var id m.MultiverseID
	msg.Receive(&id)
	cards, _ := s.multiverse.Search(id)
	if len(cards) > 0 {
		err := msg.Respond(cards[0])
		if err != nil {
			println(err.Error())
		}
	} else {
		println("no cards")
		msg.Respond(nil)
	}
}
<commit_msg>Forgot I renamed GoSockets again..<commit_after>package main

import (
	"net/http"

	m "github.com/CasualSuperman/Diorite/multiverse"
	gs "github.com/CasualSuperman/gosocket"
)

type server struct {
	multiverse *m.Multiverse
}

func (s *server) UpdateMultiverse(mlt *m.Multiverse) {
	s.multiverse = mlt
}

func NewServer(mlt *m.Multiverse) *server {
	return &server{mlt}
}

func (s *server) Serve(port string, exit chan exitSignal) {
	socketServer := gs.NewServer()
	http.Handle("/gs/", socketServer)

	socketServer.Handle("nameSearch", s.fuzzyNameSearch)
	socketServer.Handle("card", s.sendCard)
	socketServer.Errored(func(err error) {
		println(err.Error())
	})

	if !*debug {
		socketServer.On(gs.Disconnect, func(c *gs.Conn) {
			exit <- exitSignal{0, "Web interface closed."}
		})
	}

	http.Handle("/", http.FileServer(http.Dir("./web/static")))
	http.ListenAndServe(port, nil)
}

type wsRequest struct {
	Type    string
	Request string
}

type wsResponse struct {
	Type     string
	Response interface{}
}

type webCard struct {
	Name         string
	MultiverseID int
}

func (s *server) fuzzyNameSearch(msg gs.Msg) {
	var searchTerm string
	msg.Receive(&searchTerm)

	results := s.multiverse.FuzzyNameSearch(searchTerm, 15)
	cards := make([]webCard, len(results))
	for i, card := range results {
		cards[i].Name = card.Name
		cards[i].MultiverseID = int(card.Printings[len(card.Printings)-1].ID)
	}

	msg.Respond(cards)
}

func (s *server) sendCard(msg gs.Msg) {
	var id m.MultiverseID
	msg.Receive(&id)
	cards, _ := s.multiverse.Search(id)
	if len(cards) > 0 {
		err := msg.Respond(cards[0])
		if err != nil {
			println(err.Error())
		}
	} else {
		println("no cards")
		msg.Respond(nil)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"

	log "github.com/Sirupsen/logrus"
	"github.com/elazarl/go-bindata-assetfs"
	"github.com/gorilla/mux"
)

type WebProvider struct {
	Address           string
	CertFile, KeyFile string
}

type Page struct {
	Configurations configs
}

func (provider *WebProvider) Provide(configurationChan chan<- configMessage) {
	systemRouter := mux.NewRouter()
	systemRouter.Methods("GET").Path("/").Handler(http.HandlerFunc(GetHTMLConfigHandler))
	systemRouter.Methods("GET").Path("/health").Handler(http.HandlerFunc(GetHealthHandler))
	systemRouter.Methods("GET").Path("/api").Handler(http.HandlerFunc(GetConfigHandler))
	systemRouter.Methods("GET").Path("/api/{provider}").Handler(http.HandlerFunc(GetConfigHandler))
	systemRouter.Methods("PUT").Path("/api/{provider}").Handler(http.HandlerFunc(
		func(rw http.ResponseWriter, r *http.Request) {
			vars := mux.Vars(r)
			if vars["provider"] != "web" {
				rw.WriteHeader(http.StatusBadRequest)
				fmt.Fprintf(rw, "Only 'web' provider can be updated through the REST API")
				return
			}

			configuration := new(Configuration)
			b, _ := ioutil.ReadAll(r.Body)
			err := json.Unmarshal(b, configuration)
			if err == nil {
				configurationChan <- configMessage{"web", configuration}
				GetConfigHandler(rw, r)
			} else {
				log.Errorf("Error parsing configuration %+v", err)
				http.Error(rw, fmt.Sprintf("%+v", err), http.StatusBadRequest)
			}
		}))
	systemRouter.Methods("GET").Path("/api/{provider}/backends").Handler(http.HandlerFunc(GetBackendsHandler))
	systemRouter.Methods("GET").Path("/api/{provider}/backends/{backend}").Handler(http.HandlerFunc(GetBackendHandler))
	systemRouter.Methods("GET").Path("/api/{provider}/backends/{backend}/servers").Handler(http.HandlerFunc(GetServersHandler))
	systemRouter.Methods("GET").Path("/api/{provider}/backends/{backend}/servers/{server}").Handler(http.HandlerFunc(GetServerHandler))
	systemRouter.Methods("GET").Path("/api/{provider}/frontends").Handler(http.HandlerFunc(GetFrontendsHandler))
	systemRouter.Methods("GET").Path("/api/{provider}/frontends/{frontend}").Handler(http.HandlerFunc(GetFrontendHandler))
	systemRouter.Methods("GET").PathPrefix("/static/").Handler(http.StripPrefix("/static/", http.FileServer(&assetfs.AssetFS{Asset: Asset, AssetDir: AssetDir, Prefix: "static"})))

	go func() {
		if len(provider.CertFile) > 0 && len(provider.KeyFile) > 0 {
			err := http.ListenAndServeTLS(provider.Address, provider.CertFile, provider.KeyFile, systemRouter)
			if err != nil {
				log.Fatal("Error creating server: ", err)
			}
		} else {
			err := http.ListenAndServe(provider.Address, systemRouter)
			if err != nil {
				log.Fatal("Error creating server: ", err)
			}
		}
	}()
}

func GetConfigHandler(rw http.ResponseWriter, r *http.Request) {
	templatesRenderer.JSON(rw, http.StatusOK, currentConfigurations)
}

func GetHTMLConfigHandler(response http.ResponseWriter, request *http.Request) {
	templatesRenderer.HTML(response, http.StatusOK, "configuration", Page{Configurations: currentConfigurations})
}

func GetHealthHandler(rw http.ResponseWriter, r *http.Request) {
	templatesRenderer.JSON(rw, http.StatusOK, metrics.Data())
}

func GetBackendsHandler(rw http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	providerId := vars["provider"]
	if provider, ok := currentConfigurations[providerId]; ok {
		templatesRenderer.JSON(rw, http.StatusOK, provider.Backends)
	} else {
		http.NotFound(rw, r)
	}
}

func GetBackendHandler(rw http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	providerId := vars["provider"]
	backendId := vars["backend"]
	if provider, ok := currentConfigurations[providerId]; ok {
		if backend, ok := provider.Backends[backendId]; ok {
			templatesRenderer.JSON(rw, http.StatusOK, backend)
			return
		}
	}
	http.NotFound(rw, r)
}

func GetFrontendsHandler(rw http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	providerId := vars["provider"]
	if provider, ok := currentConfigurations[providerId]; ok {
		templatesRenderer.JSON(rw, http.StatusOK, provider.Frontends)
	} else {
		http.NotFound(rw, r)
	}
}

func GetFrontendHandler(rw http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	providerId := vars["provider"]
	frontendId := vars["frontend"]
	if provider, ok := currentConfigurations[providerId]; ok {
		if frontend, ok := provider.Frontends[frontendId]; ok {
			templatesRenderer.JSON(rw, http.StatusOK, frontend)
			return
		}
	}
	http.NotFound(rw, r)
}

func GetServersHandler(rw http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	providerId := vars["provider"]
	backendId := vars["backend"]
	if provider, ok := currentConfigurations[providerId]; ok {
		if backend, ok := provider.Backends[backendId]; ok {
			templatesRenderer.JSON(rw, http.StatusOK, backend.Servers)
			return
		}
	}
	http.NotFound(rw, r)
}

func GetServerHandler(rw http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	providerId := vars["provider"]
	backendId := vars["backend"]
	serverId := vars["server"]
	fmt.Printf("%v %v %v\n", providerId, backendId, serverId)
	if provider, ok := currentConfigurations[providerId]; ok {
		fmt.Printf("provider OK\n")
		if backend, ok := provider.Backends[backendId]; ok {
			fmt.Printf("backend OK\n")
			if server, ok := backend.Servers[serverId]; ok {
				fmt.Printf("server OK\n")
				templatesRenderer.JSON(rw, http.StatusOK, server)
				return
			}
		}
	}
	http.NotFound(rw, r)
}
<commit_msg>Removing debug Printf statements<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"

	log "github.com/Sirupsen/logrus"
	"github.com/elazarl/go-bindata-assetfs"
	"github.com/gorilla/mux"
)

type WebProvider struct {
	Address           string
	CertFile, KeyFile string
}

type Page struct {
	Configurations configs
}

func (provider *WebProvider) Provide(configurationChan chan<- configMessage) {
	systemRouter := mux.NewRouter()
	systemRouter.Methods("GET").Path("/").Handler(http.HandlerFunc(GetHTMLConfigHandler))
	systemRouter.Methods("GET").Path("/health").Handler(http.HandlerFunc(GetHealthHandler))
	systemRouter.Methods("GET").Path("/api").Handler(http.HandlerFunc(GetConfigHandler))
	systemRouter.Methods("GET").Path("/api/{provider}").Handler(http.HandlerFunc(GetConfigHandler))
	systemRouter.Methods("PUT").Path("/api/{provider}").Handler(http.HandlerFunc(
		func(rw http.ResponseWriter, r *http.Request) {
			vars := mux.Vars(r)
			if vars["provider"] != "web" {
				rw.WriteHeader(http.StatusBadRequest)
				fmt.Fprintf(rw, "Only 'web' provider can be updated through the REST API")
				return
			}

			configuration := new(Configuration)
			b, _ := ioutil.ReadAll(r.Body)
			err := json.Unmarshal(b, configuration)
			if err == nil {
				configurationChan <- configMessage{"web", configuration}
				GetConfigHandler(rw, r)
			} else {
				log.Errorf("Error parsing configuration %+v", err)
				http.Error(rw, fmt.Sprintf("%+v", err), http.StatusBadRequest)
			}
		}))
	systemRouter.Methods("GET").Path("/api/{provider}/backends").Handler(http.HandlerFunc(GetBackendsHandler))
	systemRouter.Methods("GET").Path("/api/{provider}/backends/{backend}").Handler(http.HandlerFunc(GetBackendHandler))
	systemRouter.Methods("GET").Path("/api/{provider}/backends/{backend}/servers").Handler(http.HandlerFunc(GetServersHandler))
	systemRouter.Methods("GET").Path("/api/{provider}/backends/{backend}/servers/{server}").Handler(http.HandlerFunc(GetServerHandler))
	systemRouter.Methods("GET").Path("/api/{provider}/frontends").Handler(http.HandlerFunc(GetFrontendsHandler))
	systemRouter.Methods("GET").Path("/api/{provider}/frontends/{frontend}").Handler(http.HandlerFunc(GetFrontendHandler))
	systemRouter.Methods("GET").PathPrefix("/static/").Handler(http.StripPrefix("/static/", http.FileServer(&assetfs.AssetFS{Asset: Asset, AssetDir: AssetDir, Prefix: "static"})))

	go func() {
		if len(provider.CertFile) > 0 && len(provider.KeyFile) > 0 {
			err := http.ListenAndServeTLS(provider.Address, provider.CertFile, provider.KeyFile, systemRouter)
			if err != nil {
				log.Fatal("Error creating server: ", err)
			}
		} else {
			err := http.ListenAndServe(provider.Address, systemRouter)
			if err != nil {
				log.Fatal("Error creating server: ", err)
			}
		}
	}()
}

func GetConfigHandler(rw http.ResponseWriter, r *http.Request) {
	templatesRenderer.JSON(rw, http.StatusOK, currentConfigurations)
}

func GetHTMLConfigHandler(response http.ResponseWriter, request *http.Request) {
	templatesRenderer.HTML(response, http.StatusOK, "configuration", Page{Configurations: currentConfigurations})
}

func GetHealthHandler(rw http.ResponseWriter, r *http.Request) {
	templatesRenderer.JSON(rw, http.StatusOK, metrics.Data())
}

func GetBackendsHandler(rw http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	providerId := vars["provider"]
	if provider, ok := currentConfigurations[providerId]; ok {
		templatesRenderer.JSON(rw, http.StatusOK, provider.Backends)
	} else {
		http.NotFound(rw, r)
	}
}

func GetBackendHandler(rw http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	providerId := vars["provider"]
	backendId := vars["backend"]
	if provider, ok := currentConfigurations[providerId]; ok {
		if backend, ok := provider.Backends[backendId]; ok {
			templatesRenderer.JSON(rw, http.StatusOK, backend)
			return
		}
	}
	http.NotFound(rw, r)
}

func GetFrontendsHandler(rw http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	providerId := vars["provider"]
	if provider, ok := currentConfigurations[providerId]; ok {
		templatesRenderer.JSON(rw, http.StatusOK, provider.Frontends)
	} else {
		http.NotFound(rw, r)
	}
}

func GetFrontendHandler(rw http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	providerId := vars["provider"]
	frontendId := vars["frontend"]
	if provider, ok := currentConfigurations[providerId]; ok {
		if frontend, ok := provider.Frontends[frontendId]; ok {
			templatesRenderer.JSON(rw, http.StatusOK, frontend)
			return
		}
	}
	http.NotFound(rw, r)
}

func GetServersHandler(rw http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	providerId := vars["provider"]
	backendId := vars["backend"]
	if provider, ok := currentConfigurations[providerId]; ok {
		if backend, ok := provider.Backends[backendId]; ok {
			templatesRenderer.JSON(rw, http.StatusOK, backend.Servers)
			return
		}
	}
	http.NotFound(rw, r)
}

func GetServerHandler(rw http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	providerId := vars["provider"]
	backendId := vars["backend"]
	serverId := vars["server"]
	if provider, ok := currentConfigurations[providerId]; ok {
		if backend, ok := provider.Backends[backendId]; ok {
			if server, ok := backend.Servers[serverId]; ok {
				templatesRenderer.JSON(rw, http.StatusOK, server)
				return
			}
		}
	}
	http.NotFound(rw, r)
}
<|endoftext|>


<commit_before>package task_bbs

import (
	"time"

	"github.com/cloudfoundry-incubator/runtime-schema/bbs/shared"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	steno "github.com/cloudfoundry/gosteno"
	"github.com/cloudfoundry/storeadapter"
)

// ConvergeTask is run by *one* executor every X seconds (doesn't really matter what X is.. pick something performant)
// Converge will:
// 1. Kick (by setting) any run-onces that are still pending (and have been for > convergence interval)
// 2. Kick (by setting) any run-onces that are completed (and have been for > convergence interval)
// 3. Demote to pending any claimed run-onces that have been claimed for > 30s
// 4. Demote to completed any resolving run-onces that have been resolving for > 30s
// 5. Mark as failed any run-onces that have been in the pending state for > timeToClaim
// 6. Mark as failed any claimed or running run-onces whose executor has stopped maintaining presence
func (self *TaskBBS) ConvergeTask(timeToClaim time.Duration, convergenceInterval time.Duration) {
	taskState, err := self.store.ListRecursively(shared.TaskSchemaRoot)
	if err != nil {
		return
	}

	executorState, err := self.store.ListRecursively(shared.ExecutorSchemaRoot)
	if err == storeadapter.ErrorKeyNotFound {
		executorState = storeadapter.StoreNode{}
	} else if err != nil {
		return
	}

	logger := steno.NewLogger("bbs")
	logError := func(task models.Task, message string) {
		logger.Errord(map[string]interface{}{
			"task": task,
		}, message)
	}

	keysToDelete := []string{}

	tasksToCAS := []compareAndSwappableTask{}
	scheduleForCASByIndex := func(index uint64, newTask models.Task) {
		tasksToCAS = append(tasksToCAS, compareAndSwappableTask{
			OldIndex: index,
			NewTask:  newTask,
		})
	}

	for _, node := range taskState.ChildNodes {
		task, err := models.NewTaskFromJSON(node.Value)
		if err != nil {
			logger.Errord(map[string]interface{}{
				"key":   node.Key,
				"value": string(node.Value),
			}, "task.converge.json-parse-failure")
			keysToDelete = append(keysToDelete, node.Key)
			continue
		}

		shouldKickTask := self.durationSinceTaskUpdated(task) >= convergenceInterval

		switch task.State {
		case models.TaskStatePending:
			shouldMarkAsFailed := self.durationSinceTaskCreated(task) >= timeToClaim
			if shouldMarkAsFailed {
				logError(task, "task.converge.failed-to-claim")
				scheduleForCASByIndex(node.Index, markTaskFailed(task, "not claimed within time limit"))
			} else if shouldKickTask {
				scheduleForCASByIndex(node.Index, task)
			}
		case models.TaskStateClaimed:
			_, executorIsAlive := executorState.Lookup(task.ExecutorID)

			if !executorIsAlive {
				logError(task, "task.converge.executor-disappeared")
				scheduleForCASByIndex(node.Index, markTaskFailed(task, "executor disappeared before completion"))
			} else if shouldKickTask {
				logError(task, "task.converge.failed-to-start")
				scheduleForCASByIndex(node.Index, demoteToPending(task))
			}
		case models.TaskStateRunning:
			_, executorIsAlive := executorState.Lookup(task.ExecutorID)

			if !executorIsAlive {
				logError(task, "task.converge.executor-disappeared")
				scheduleForCASByIndex(node.Index, markTaskFailed(task, "executor disappeared before completion"))
			}
		case models.TaskStateCompleted:
			if shouldKickTask {
				scheduleForCASByIndex(node.Index, task)
			}
		case models.TaskStateResolving:
			if shouldKickTask {
				logError(task, "task.converge.failed-to-resolve")
				scheduleForCASByIndex(node.Index, demoteToCompleted(task))
			}
		}
	}

	self.batchCompareAndSwapTasks(tasksToCAS, logger)
	self.store.Delete(keysToDelete...)
}

func (self *TaskBBS) durationSinceTaskCreated(task models.Task) time.Duration {
	return self.timeProvider.Time().Sub(time.Unix(0, task.CreatedAt))
}

func (self *TaskBBS) durationSinceTaskUpdated(task models.Task) time.Duration {
	return self.timeProvider.Time().Sub(time.Unix(0, task.UpdatedAt))
}

func markTaskFailed(task models.Task, reason string) models.Task {
	task.State = models.TaskStateCompleted
	task.Failed = true
	task.FailureReason = reason
	return task
}

func (self *TaskBBS) batchCompareAndSwapTasks(tasksToCAS []compareAndSwappableTask, logger *steno.Logger) {
	done := make(chan struct{}, len(tasksToCAS))

	for _, taskToCAS := range tasksToCAS {
		task := taskToCAS.NewTask
		task.UpdatedAt = self.timeProvider.Time().UnixNano()
		newStoreNode := storeadapter.StoreNode{
			Key:   shared.TaskSchemaPath(task),
			Value: task.ToJSON(),
		}

		go func() {
			err := self.store.CompareAndSwapByIndex(taskToCAS.OldIndex, newStoreNode)
			if err != nil {
				logger.Errord(map[string]interface{}{
					"error": err.Error(),
				}, "task.converge.failed-to-compare-and-swap")
			}
			done <- struct{}{}
		}()
	}

	for _ = range tasksToCAS {
		<-done
	}
}

func demoteToPending(task models.Task) models.Task {
	task.State = models.TaskStatePending
	task.ExecutorID = ""
	task.ContainerHandle = ""
	return task
}

func demoteToCompleted(task models.Task) models.Task {
	task.State = models.TaskStateCompleted
	return task
}
<commit_msg>fix data race in convergence<commit_after>package task_bbs

import (
	"sync"
	"time"

	"github.com/cloudfoundry-incubator/runtime-schema/bbs/shared"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	steno "github.com/cloudfoundry/gosteno"
	"github.com/cloudfoundry/storeadapter"
)

// ConvergeTask is run by *one* executor every X seconds (doesn't really matter what X is.. pick something performant)
// Converge will:
// 1. Kick (by setting) any run-onces that are still pending (and have been for > convergence interval)
// 2. Kick (by setting) any run-onces that are completed (and have been for > convergence interval)
// 3. Demote to pending any claimed run-onces that have been claimed for > 30s
// 4. Demote to completed any resolving run-onces that have been resolving for > 30s
// 5. Mark as failed any run-onces that have been in the pending state for > timeToClaim
// 6. Mark as failed any claimed or running run-onces whose executor has stopped maintaining presence
func (self *TaskBBS) ConvergeTask(timeToClaim time.Duration, convergenceInterval time.Duration) {
	taskState, err := self.store.ListRecursively(shared.TaskSchemaRoot)
	if err != nil {
		return
	}

	executorState, err := self.store.ListRecursively(shared.ExecutorSchemaRoot)
	if err == storeadapter.ErrorKeyNotFound {
		executorState = storeadapter.StoreNode{}
	} else if err != nil {
		return
	}

	logger := steno.NewLogger("bbs")
	logError := func(task models.Task, message string) {
		logger.Errord(map[string]interface{}{
			"task": task,
		}, message)
	}

	keysToDelete := []string{}

	tasksToCAS := []compareAndSwappableTask{}
	scheduleForCASByIndex := func(index uint64, newTask models.Task) {
		tasksToCAS = append(tasksToCAS, compareAndSwappableTask{
			OldIndex: index,
			NewTask:  newTask,
		})
	}

	for _, node := range taskState.ChildNodes {
		task, err := models.NewTaskFromJSON(node.Value)
		if err != nil {
			logger.Errord(map[string]interface{}{
				"key":   node.Key,
				"value": string(node.Value),
			}, "task.converge.json-parse-failure")
			keysToDelete = append(keysToDelete, node.Key)
			continue
		}

		shouldKickTask := self.durationSinceTaskUpdated(task) >= convergenceInterval

		switch task.State {
		case models.TaskStatePending:
			shouldMarkAsFailed := self.durationSinceTaskCreated(task) >= timeToClaim
			if shouldMarkAsFailed {
				logError(task, "task.converge.failed-to-claim")
				scheduleForCASByIndex(node.Index, markTaskFailed(task, "not claimed within time limit"))
			} else if shouldKickTask {
				scheduleForCASByIndex(node.Index, task)
			}
		case models.TaskStateClaimed:
			_, executorIsAlive := executorState.Lookup(task.ExecutorID)

			if !executorIsAlive {
				logError(task, "task.converge.executor-disappeared")
				scheduleForCASByIndex(node.Index, markTaskFailed(task, "executor disappeared before completion"))
			} else if shouldKickTask {
				logError(task, "task.converge.failed-to-start")
				scheduleForCASByIndex(node.Index, demoteToPending(task))
			}
		case models.TaskStateRunning:
			_, executorIsAlive := executorState.Lookup(task.ExecutorID)

			if !executorIsAlive {
				logError(task, "task.converge.executor-disappeared")
				scheduleForCASByIndex(node.Index, markTaskFailed(task, "executor disappeared before completion"))
			}
		case models.TaskStateCompleted:
			if shouldKickTask {
				scheduleForCASByIndex(node.Index, task)
			}
		case models.TaskStateResolving:
			if shouldKickTask {
				logError(task, "task.converge.failed-to-resolve")
				scheduleForCASByIndex(node.Index, demoteToCompleted(task))
			}
		}
	}

	self.batchCompareAndSwapTasks(tasksToCAS, logger)
	self.store.Delete(keysToDelete...)
}

func (self *TaskBBS) durationSinceTaskCreated(task models.Task) time.Duration {
	return self.timeProvider.Time().Sub(time.Unix(0, task.CreatedAt))
}

func (self *TaskBBS) durationSinceTaskUpdated(task models.Task) time.Duration {
	return self.timeProvider.Time().Sub(time.Unix(0, task.UpdatedAt))
}

func markTaskFailed(task models.Task, reason string) models.Task {
	task.State = models.TaskStateCompleted
	task.Failed = true
	task.FailureReason = reason
	return task
}

func (self *TaskBBS) batchCompareAndSwapTasks(tasksToCAS []compareAndSwappableTask, logger *steno.Logger) {
	waitGroup := &sync.WaitGroup{}
	waitGroup.Add(len(tasksToCAS))
	for _, taskToCAS := range tasksToCAS {
		task := taskToCAS.NewTask
		task.UpdatedAt = self.timeProvider.Time().UnixNano()
		newStoreNode := storeadapter.StoreNode{
			Key:   shared.TaskSchemaPath(task),
			Value: task.ToJSON(),
		}

		go func(taskToCAS compareAndSwappableTask, newStoreNode storeadapter.StoreNode) {
			err := self.store.CompareAndSwapByIndex(taskToCAS.OldIndex, newStoreNode)
			if err != nil {
				logger.Errord(map[string]interface{}{
					"error": err.Error(),
				}, "task.converge.failed-to-compare-and-swap")
			}
			waitGroup.Done()
		}(taskToCAS, newStoreNode)
	}

	waitGroup.Wait()
}

func demoteToPending(task models.Task) models.Task {
	task.State = models.TaskStatePending
	task.ExecutorID = ""
	task.ContainerHandle = ""
	return task
}

func demoteToCompleted(task models.Task) models.Task {
	task.State = models.TaskStateCompleted
	return task
}
<|endoftext|>


<commit_before>package brats_test

import (
	"os"

	"github.com/cloudfoundry/libbuildpack/bratshelper"
	"github.com/cloudfoundry/libbuildpack/cutlass"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"golang.org/x/crypto/bcrypt"
)

var _ = Describe("Ruby buildpack", func() {
	bratshelper.UnbuiltBuildpack("ruby", CopyBrats)
	bratshelper.DeployingAnAppWithAnUpdatedVersionOfTheSameBuildpack(CopyBrats)
	bratshelper.StagingWithBuildpackThatSetsEOL("ruby", func(_ string) *cutlass.App {
		return CopyBrats("2.2.x")
	})
	//TODO: Old versions of ruby were not available on cflinuxfs3. Bring this test back when possible.
	if os.Getenv("CF_STACK") == "cflinuxfs2" {
		bratshelper.StagingWithADepThatIsNotTheLatest("ruby", CopyBrats)
	}
	bratshelper.StagingWithCustomBuildpackWithCredentialsInDependencies(CopyBrats)
	bratshelper.DeployAppWithExecutableProfileScript("ruby", CopyBrats)
	bratshelper.DeployAnAppWithSensitiveEnvironmentVariables(CopyBrats)
	bratshelper.ForAllSupportedVersions("ruby", CopyBrats, func(rubyVersion string, app *cutlass.App) {
		PushApp(app)

		By("installs the correct version of Ruby", func() {
			Expect(app.Stdout.String()).To(ContainSubstring("Installing ruby " + rubyVersion))
			Expect(app.GetBody("/version")).To(ContainSubstring(rubyVersion))
		})
		By("runs a simple webserver", func() {
			Expect(app.GetBody("/")).To(ContainSubstring("Hello World!"))
		})
		By("parses XML with nokogiri", func() {
			Expect(app.GetBody("/nokogiri")).To(ContainSubstring("Hello, World"))
		})
		By("supports EventMachine", func() {
			Expect(app.GetBody("/em")).To(ContainSubstring("Hello, EventMachine"))
		})
		By("encrypts with bcrypt", func() {
			hashedPassword, err := app.GetBody("/bcrypt")
			Expect(err).ToNot(HaveOccurred())
			Expect(bcrypt.CompareHashAndPassword([]byte(hashedPassword), []byte("Hello, bcrypt"))).ToNot(HaveOccurred())
		})
		By("supports bson", func() {
			Expect(app.GetBody("/bson")).To(ContainSubstring("\x00\x04\x00\x00"))
		})
		By("supports postgres", func() {
			Expect(app.GetBody("/pg")).To(ContainSubstring("could not connect to server: No such file or directory"))
		})
		By("supports mysql2", func() {
			Expect(app.GetBody("/mysql2")).To(ContainSubstring("Unknown MySQL server host 'testing'"))
		})
	})

	bratshelper.ForAllSupportedVersions("jruby", CopyBratsJRuby, func(jrubyVersion string, app *cutlass.App) {
		app.Memory = "2G"
		app.Disk = "1G" //This failed at 300M
		app.StartCommand = "ruby app.rb -p $PORT"
		PushApp(app)

		By("installs the correct version of JRuby", func() {
			Expect(app.Stdout.String()).To(ContainSubstring("Installing jruby " + jrubyVersion))
		})
		By("runs a simple webserver", func() {
			Expect(app.GetBody("/")).To(ContainSubstring("Hello, World"))
		})
		By("parses XML with nokogiri", func() {
			Expect(app.GetBody("/nokogiri")).To(ContainSubstring("Hello, World"))
		})
		By("supports EventMachine", func() {
			Expect(app.GetBody("/em")).To(ContainSubstring("Hello, EventMachine"))
		})
		By("encrypts with bcrypt", func() {
			hashedPassword, err := app.GetBody("/bcrypt")
			Expect(err).ToNot(HaveOccurred())
			Expect(bcrypt.CompareHashAndPassword([]byte(hashedPassword), []byte("Hello, bcrypt"))).ToNot(HaveOccurred())
		})
		By("supports bson", func() {
			Expect(app.GetBody("/bson")).To(ContainSubstring("00040000"))
		})
		By("supports postgres", func() {
			Expect(app.GetBody("/pg")).To(ContainSubstring("The connection attempt failed."))
		})
		By("supports mysql", func() {
			Expect(app.GetBody("/mysql")).To(ContainSubstring("Communications link failure"))
		})
	})
})
<commit_msg>Remove 2.2.x from brats [#165296435]<commit_after>package brats_test

import (
	"os"

	"github.com/cloudfoundry/libbuildpack/bratshelper"
	"github.com/cloudfoundry/libbuildpack/cutlass"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"golang.org/x/crypto/bcrypt"
)

var _ = Describe("Ruby buildpack", func() {
	bratshelper.UnbuiltBuildpack("ruby", CopyBrats)
	bratshelper.DeployingAnAppWithAnUpdatedVersionOfTheSameBuildpack(CopyBrats)
	bratshelper.StagingWithBuildpackThatSetsEOL("ruby", func(_ string) *cutlass.App {
		return CopyBrats("2.4.x")
	})
	//TODO: Old versions of ruby were not available on cflinuxfs3. Bring this test back when possible.
	if os.Getenv("CF_STACK") == "cflinuxfs2" {
		bratshelper.StagingWithADepThatIsNotTheLatest("ruby", CopyBrats)
	}
	bratshelper.StagingWithCustomBuildpackWithCredentialsInDependencies(CopyBrats)
	bratshelper.DeployAppWithExecutableProfileScript("ruby", CopyBrats)
	bratshelper.DeployAnAppWithSensitiveEnvironmentVariables(CopyBrats)
	bratshelper.ForAllSupportedVersions("ruby", CopyBrats, func(rubyVersion string, app *cutlass.App) {
		PushApp(app)

		By("installs the correct version of Ruby", func() {
			Expect(app.Stdout.String()).To(ContainSubstring("Installing ruby " + rubyVersion))
			Expect(app.GetBody("/version")).To(ContainSubstring(rubyVersion))
		})
		By("runs a simple webserver", func() {
			Expect(app.GetBody("/")).To(ContainSubstring("Hello World!"))
		})
		By("parses XML with nokogiri", func() {
			Expect(app.GetBody("/nokogiri")).To(ContainSubstring("Hello, World"))
		})
		By("supports EventMachine", func() {
			Expect(app.GetBody("/em")).To(ContainSubstring("Hello, EventMachine"))
		})
		By("encrypts with bcrypt", func() {
			hashedPassword, err := app.GetBody("/bcrypt")
			Expect(err).ToNot(HaveOccurred())
			Expect(bcrypt.CompareHashAndPassword([]byte(hashedPassword), []byte("Hello, bcrypt"))).ToNot(HaveOccurred())
		})
		By("supports bson", func() {
			Expect(app.GetBody("/bson")).To(ContainSubstring("\x00\x04\x00\x00"))
		})
		By("supports postgres", func() {
			Expect(app.GetBody("/pg")).To(ContainSubstring("could not connect to server: No such file or directory"))
		})
		By("supports mysql2", func() {
			Expect(app.GetBody("/mysql2")).To(ContainSubstring("Unknown MySQL server host 'testing'"))
		})
	})

	bratshelper.ForAllSupportedVersions("jruby", CopyBratsJRuby, func(jrubyVersion string, app *cutlass.App) {
		app.Memory = "2G"
		app.Disk = "1G" //This failed at 300M
		app.StartCommand = "ruby app.rb -p $PORT"
		PushApp(app)

		By("installs the correct version of JRuby", func() {
			Expect(app.Stdout.String()).To(ContainSubstring("Installing jruby " + jrubyVersion))
		})
		By("runs a simple webserver", func() {
			Expect(app.GetBody("/")).To(ContainSubstring("Hello, World"))
		})
		By("parses XML with nokogiri", func() {
			Expect(app.GetBody("/nokogiri")).To(ContainSubstring("Hello, World"))
		})
		By("supports EventMachine", func() {
			Expect(app.GetBody("/em")).To(ContainSubstring("Hello, EventMachine"))
		})
		By("encrypts with bcrypt", func() {
			hashedPassword, err := app.GetBody("/bcrypt")
			Expect(err).ToNot(HaveOccurred())
			Expect(bcrypt.CompareHashAndPassword([]byte(hashedPassword), []byte("Hello, bcrypt"))).ToNot(HaveOccurred())
		})
		By("supports bson", func() {
			Expect(app.GetBody("/bson")).To(ContainSubstring("00040000"))
		})
		By("supports postgres", func() {
			Expect(app.GetBody("/pg")).To(ContainSubstring("The connection attempt failed."))
		})
		By("supports mysql", func() {
			Expect(app.GetBody("/mysql")).To(ContainSubstring("Communications link failure"))
		})
	})
})
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"net/http"
	"os"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/skibish/ddns/conf"
	"github.com/skibish/ddns/do"
	"github.com/skibish/ddns/ipprovider"
)

var (
	digio   *do.DigitalOcean
	cf      *conf.Configuration
	periodC <-chan time.Time
)

var (
	reqTimeouts = flag.Duration("req-timeout", 10*time.Second, "Request timeout to external resources")
	checkPeriod = flag.Duration("check-period", 5*time.Minute, "Check if IP has been changed period")
	confFile    = flag.String("conf-file", "$HOME/.ddns.yml", "Location of the configuration file")
)

// current remembered IP
var currentIP string

func init() {
	log.SetFormatter(&log.TextFormatter{})
	log.SetOutput(os.Stdout)
}

func main() {
	flag.Parse()

	// read configuration
	var errConf error
	cf, errConf = conf.NewConfiguration(*confFile)
	if errConf != nil {
		log.Fatal(errConf.Error())
	}

	// setup http client
	hc := &http.Client{
		Timeout: *reqTimeouts,
	}

	// initialte digital ocean client
	digio = do.NewDigitalOcean(cf.Domain, cf.Token, hc)

	// register providers
	ipprovider.Register(hc)

	// get current IP
	currentIP = ipprovider.GetIP()
	if currentIP == "" {
		log.Fatal("IP can't be empty in the beginning... Do you have internet connection?")
	}
	log.Infof("Current IP is %q", currentIP)

	// do request to the digital ocean API for list of records
	allRecords, errGetDR := digio.GetDomainRecords()
	if errGetDR != nil {
		log.Fatal(errGetDR.Error())
	}

	// do initial sync of records
	errSync := syncRecords(cf, allRecords)
	if errSync != nil {
		log.Fatal(errSync.Error())
	}

	periodC = time.NewTicker(*checkPeriod).C

	// start main proceess
	go checkAndUpdate()

	select {}
}

// syncRecords perform initial sync between what we provided
// in configuration and what already exist in DNS records
func syncRecords(cf *conf.Configuration, allRecords []do.Record) error {

	// search for already defined records in DNS
	for _, cr := range cf.Records {
		for _, rdr := range allRecords {

			// we are only interested in those who have full match
			// by `type AND name`
			if cr.Type == rdr.Type && cr.Name == rdr.Name {
				cr = rdr
				break
			}
		}

		// if there was no match, we should create new DNS record
		// and updatee current configuration
		if cr.ID == 0 {
			log.Debug("creating new record")
			cr.Data = currentIP

			newR, errCreate := digio.CreateRecord(cr)
			if errCreate != nil {
				return errCreate
			}

			cr = *newR
		}

		// if IPs are different, update record
		if cr.Data != currentIP {
			cr.Data = currentIP

			newR, errUpdate := digio.UpdateRecord(cr)
			if errUpdate != nil {
				log.Errorf("Update failed: %s", errUpdate.Error())
			}

			cr = *newR
		}
	}

	return nil
}

// checkAndUpdate check for new IP and if it has been changed,
// trigger the update of the DNS records
func checkAndUpdate() {
	// for defined period of time, perform IP check
	for {
		select {
		case <-periodC:
			log.Info("IP check")
			newIP := ipprovider.GetIP()

			if currentIP != newIP {
				log.Infof("IP has changed from %q to %q", currentIP, newIP)
				currentIP = newIP

				for _, cr := range cf.Records {
					cr.Data = currentIP

					newR, errUpdate := digio.UpdateRecord(cr)
					if errUpdate != nil {
						log.Errorf("Update failed: %s", errUpdate.Error())
					}

					cr = *newR
				}
			}
		}
	}
}
<commit_msg>remove log message<commit_after>package main

import (
	"flag"
	"net/http"
	"os"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/skibish/ddns/conf"
	"github.com/skibish/ddns/do"
	"github.com/skibish/ddns/ipprovider"
)

var (
	digio   *do.DigitalOcean
	cf      *conf.Configuration
	periodC <-chan time.Time
)

var (
	reqTimeouts = flag.Duration("req-timeout", 10*time.Second, "Request timeout to external resources")
	checkPeriod = flag.Duration("check-period", 5*time.Minute, "Check if IP has been changed period")
	confFile    = flag.String("conf-file", "$HOME/.ddns.yml", "Location of the configuration file")
)

// current remembered IP
var currentIP string

func init() {
	log.SetFormatter(&log.TextFormatter{})
	log.SetOutput(os.Stdout)
}

func main() {
	flag.Parse()

	// read configuration
	var errConf error
	cf, errConf = conf.NewConfiguration(*confFile)
	if errConf != nil {
		log.Fatal(errConf.Error())
	}

	// setup http client
	hc := &http.Client{
		Timeout: *reqTimeouts,
	}

	// initialte digital ocean client
	digio = do.NewDigitalOcean(cf.Domain, cf.Token, hc)

	// register providers
	ipprovider.Register(hc)

	// get current IP
	currentIP = ipprovider.GetIP()
	if currentIP == "" {
		log.Fatal("IP can't be empty in the beginning... Do you have internet connection?")
	}
	log.Infof("Current IP is %q", currentIP)

	// do request to the digital ocean API for list of records
	allRecords, errGetDR := digio.GetDomainRecords()
	if errGetDR != nil {
		log.Fatal(errGetDR.Error())
	}

	// do initial sync of records
	errSync := syncRecords(cf, allRecords)
	if errSync != nil {
		log.Fatal(errSync.Error())
	}

	periodC = time.NewTicker(*checkPeriod).C

	// start main proceess
	go checkAndUpdate()

	select {}
}

// syncRecords perform initial sync between what we provided
// in configuration and what already exist in DNS records
func syncRecords(cf *conf.Configuration, allRecords []do.Record) error {

	// search for already defined records in DNS
	for _, cr := range cf.Records {
		for _, rdr := range allRecords {

			// we are only interested in those who have full match
			// by `type AND name`
			if cr.Type == rdr.Type && cr.Name == rdr.Name {
				cr = rdr
				break
			}
		}

		// if there was no match, we should create new DNS record
		// and updatee current configuration
		if cr.ID == 0 {
			cr.Data = currentIP

			newR, errCreate := digio.CreateRecord(cr)
			if errCreate != nil {
				return errCreate
			}

			cr = *newR
		}

		// if IPs are different, update record
		if cr.Data != currentIP {
			cr.Data = currentIP

			newR, errUpdate := digio.UpdateRecord(cr)
			if errUpdate != nil {
				log.Errorf("Update failed: %s", errUpdate.Error())
			}

			cr = *newR
		}
	}

	return nil
}

// checkAndUpdate check for new IP and if it has been changed,
// trigger the update of the DNS records
func checkAndUpdate() {
	// for defined period of time, perform IP check
	for {
		select {
		case <-periodC:
			log.Info("IP check")
			newIP := ipprovider.GetIP()

			if currentIP != newIP {
				log.Infof("IP has changed from %q to %q", currentIP, newIP)
				currentIP = newIP

				for _, cr := range cf.Records {
					cr.Data = currentIP

					newR, errUpdate := digio.UpdateRecord(cr)
					if errUpdate != nil {
						log.Errorf("Update failed: %s", errUpdate.Error())
					}

					cr = *newR
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"io/ioutil"
	"log"
	"os"
	"strconv"
	"strings"
	"time"
)

// receive parameters
var (
	// project path:Must Be Relative path
	project = flag.String("p", "", "path of project.")
	// save path of report
	report = flag.String("r", "", "path of report.")
	// except packages,multiple packages are separated by semicolons
	except = flag.String("e", "", "except packages.")
	// template
	tplpath = flag.String("t", "", "report html template path.")
	// report formate
	formate = flag.String("f", "", "project report formate(json/html).")
)

func main() {
	flag.Parse()
	log.SetPrefix("[Apollo]")
	if *project == "" {
		log.Fatal("The project path is not specified")
	} else {
		_, err := os.Stat(*project)
		if err != nil {
			log.Fatal("project path is invalid")
		}
	}

	if *tplpath == "" {
		log.Println("The template path is not specified,and will use the default template")
	} else {
		if !strings.HasSuffix(*report, ".html") {
			log.Println("The template file is not a html template")
		}
		fileData, err := ioutil.ReadFile(*tplpath)
		if err != nil {
			log.Fatal(err)
		} else {
			tpl = string(fileData)
		}
	}

	if *report == "" {
		log.Println("The report path is not specified, and the current path is used by default")
	} else {
		_, err := os.Stat(*report)
		if err != nil {
			log.Fatal("report path is invalid")
		}
	}

	if *except == "" {
		log.Println("There are no packages that are excepted, review all items of the package")
	}

	startTime := strconv.FormatInt(time.Now().Unix(), 10)
	reporter := NewReporter()
	reporter.Engine(*project, *except)
	htmlData, err := reporter.Json2Html()
	if err != nil {
		log.Println("Json2Html error")
		return
	}
	if *formate == "json" {
		reporter.SaveAsJson(*project, *report, startTime)
	} else {
		reporter.SaveAsHtml(htmlData, *project, *report, startTime)
	}
}
<commit_msg>rewrite log head Apollo into GoReporter<commit_after>package main

import (
	"flag"
	"io/ioutil"
	"log"
	"os"
	"strconv"
	"strings"
	"time"
)

// receive parameters
var (
	// project path:Must Be Relative path
	project = flag.String("p", "", "path of project.")
	// save path of report
	report = flag.String("r", "", "path of report.")
	// except packages,multiple packages are separated by semicolons
	except = flag.String("e", "", "except packages.")
	// template
	tplpath = flag.String("t", "", "report html template path.")
	// report formate
	formate = flag.String("f", "", "project report formate(json/html).")
)

func main() {
	flag.Parse()
	log.SetPrefix("[GoReporter]")
	if *project == "" {
		log.Fatal("The project path is not specified")
	} else {
		_, err := os.Stat(*project)
		if err != nil {
			log.Fatal("project path is invalid")
		}
	}

	if *tplpath == "" {
		log.Println("The template path is not specified,and will use the default template")
	} else {
		if !strings.HasSuffix(*report, ".html") {
			log.Println("The template file is not a html template")
		}
		fileData, err := ioutil.ReadFile(*tplpath)
		if err != nil {
			log.Fatal(err)
		} else {
			tpl = string(fileData)
		}
	}

	if *report == "" {
		log.Println("The report path is not specified, and the current path is used by default")
	} else {
		_, err := os.Stat(*report)
		if err != nil {
			log.Fatal("report path is invalid")
		}
	}

	if *except == "" {
		log.Println("There are no packages that are excepted, review all items of the package")
	}

	startTime := strconv.FormatInt(time.Now().Unix(), 10)
	reporter := NewReporter()
	reporter.Engine(*project, *except)
	htmlData, err := reporter.Json2Html()
	if err != nil {
		log.Println("Json2Html error")
		return
	}
	if *formate == "json" {
		reporter.SaveAsJson(*project, *report, startTime)
	} else {
		reporter.SaveAsHtml(htmlData, *project, *report, startTime)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"os"
	"sort"

	"github.com/fatih/color"
	"github.com/nwidger/jsoncolor"
	"github.com/pkg/errors"
)

// Exit codes
const (
	exitOK = iota
	exitOpenFile
	exitReadInput
	exitFormStatements
	exitFetchURL
	exitParseStatements
	exitJSONEncode
)

// Option bitfields
const (
	optMonochrome = iota + 1
	optNoSort
)

// Output colors
var (
	strColor   = color.New(color.FgYellow)
	braceColor = color.New(color.FgMagenta)
	bareColor  = color.New(color.FgBlue, color.Bold)
	numColor   = color.New(color.FgRed)
	boolColor  = color.New(color.FgCyan)
)

// gronVersion stores the current gron version, set at build
// time with the ldflags -X option
var gronVersion = "dev"

func init() {
	flag.Usage = func() {
		h := "Transform JSON (from a file, URL, or stdin) into discrete assignments to make it greppable\n\n"

		h += "Usage:\n"
		h += "  gron [OPTIONS] [FILE|URL|-]\n\n"

		h += "Options:\n"
		h += "  -u, --ungron     Reverse the operation (turn assignments back into JSON)\n"
		h += "  -c, --colorize   Colorize output (default on tty)\n"
		h += "  -m, --monochrome Monochrome (don't colorize output)\n"
		h += "  -s, --stream     Treat each line of input as a separate JSON object\n"
		h += "  -k, --insecure   Disable certificate validation\n"
		h += "      --no-sort    Don't sort output (faster)\n"
		h += "      --version    Print version information\n\n"

		h += "Exit Codes:\n"
		h += fmt.Sprintf("  %d\t%s\n", exitOK, "OK")
		h += fmt.Sprintf("  %d\t%s\n", exitOpenFile, "Failed to open file")
		h += fmt.Sprintf("  %d\t%s\n", exitReadInput, "Failed to read input")
		h += fmt.Sprintf("  %d\t%s\n", exitFormStatements, "Failed to form statements")
		h += fmt.Sprintf("  %d\t%s\n", exitFetchURL, "Failed to fetch URL")
		h += fmt.Sprintf("  %d\t%s\n", exitParseStatements, "Failed to parse statements")
		h += fmt.Sprintf("  %d\t%s\n", exitJSONEncode, "Failed to encode JSON")
		h += "\n"

		h += "Examples:\n"
		h += "  gron /tmp/apiresponse.json\n"
		h += "  gron http://jsonplaceholder.typicode.com/users/1 \n"
		h += "  curl -s http://jsonplaceholder.typicode.com/users/1 | gron\n"
		h += "  gron http://jsonplaceholder.typicode.com/users/1 | grep company | gron --ungron\n"

		fmt.Fprintf(os.Stderr, h)
	}
}

func main() {
	var (
		ungronFlag     bool
		colorizeFlag   bool
		monochromeFlag bool
		streamFlag     bool
		noSortFlag     bool
		versionFlag    bool
		insecureFlag   bool
	)

	flag.BoolVar(&ungronFlag, "ungron", false, "")
	flag.BoolVar(&ungronFlag, "u", false, "")
	flag.BoolVar(&colorizeFlag, "colorize", false, "")
	flag.BoolVar(&colorizeFlag, "c", false, "")
	flag.BoolVar(&monochromeFlag, "monochrome", false, "")
	flag.BoolVar(&monochromeFlag, "m", false, "")
	flag.BoolVar(&streamFlag, "s", false, "")
	flag.BoolVar(&streamFlag, "stream", false, "")
	flag.BoolVar(&noSortFlag, "no-sort", false, "")
	flag.BoolVar(&versionFlag, "version", false, "")
	flag.BoolVar(&insecureFlag, "k", false, "")
	flag.BoolVar(&insecureFlag, "insecure", false, "")

	flag.Parse()

	// Print version information
	if versionFlag {
		fmt.Printf("gron version %s\n", gronVersion)
		os.Exit(exitOK)
	}

	// Determine what the program's input should be:
	// file, HTTP URL or stdin
	var rawInput io.Reader
	filename := flag.Arg(0)
	if filename == "" || filename == "-" {
		rawInput = os.Stdin
	} else if validURL(filename) {
		r, err := getURL(filename, insecureFlag)
		if err != nil {
			fatal(exitFetchURL, err)
		}
		rawInput = r
	} else {
		r, err := os.Open(filename)
		if err != nil {
			fatal(exitOpenFile, err)
		}
		rawInput = r
	}

	var opts int
	// The monochrome option should be forced if the output isn't a terminal
	// to avoid doing unnecessary work calling the color functions
	switch {
	case colorizeFlag:
		color.NoColor = false
	case monochromeFlag || color.NoColor:
		opts = opts | optMonochrome
	}
	if noSortFlag {
		opts = opts | optNoSort
	}

	// Pick the appropriate action: gron, ungron or gronStream
	var a actionFn = gron
	if ungronFlag {
		a = ungron
	} else if streamFlag {
		a = gronStream
	}
	exitCode, err := a(rawInput, os.Stdout, opts)

	if exitCode != exitOK {
		fatal(exitCode, err)
	}

	os.Exit(exitOK)
}

// an actionFn represents a main action of the program, it accepts
// an input, output and a bitfield of options; returning an exit
// code and any error that occurred
type actionFn func(io.Reader, io.Writer, int) (int, error)

// gron is the default action. Given JSON as the input it returns a list
// of assignment statements. Possible options are optNoSort and optMonochrome
func gron(r io.Reader, w io.Writer, opts int) (int, error) {

	ss, err := statementsFromJSON(r, statement{{"json", typBare}})
	if err != nil {
		return exitFormStatements, fmt.Errorf("failed to form statements: %s", err)
	}

	// Go's maps do not have well-defined ordering, but we want a consistent
	// output for a given input, so we must sort the statements
	if opts&optNoSort == 0 {
		sort.Sort(ss)
	}

	if opts&optMonochrome > 0 {
		for _, s := range ss {
			fmt.Fprintln(w, s.String())
		}
	} else {
		for _, s := range ss {
			fmt.Fprintln(w, s.colorString())
		}
	}

	return exitOK, nil
}

// gronStream is like the gron action, but it treats the input as one
// JSON object per line. There's a bit of code duplication from the
// gron action, but it'd be fairly messy to combine the two actions
func gronStream(r io.Reader, w io.Writer, opts int) (int, error) {

	// Helper function to make the prefix statements for each line
	makePrefix := func(index int) statement {
		return statement{
			{"json", typBare},
			{"[", typLBrace},
			{fmt.Sprintf("%d", index), typNumericKey},
			{"]", typRBrace},
		}
	}

	// The first line of output needs to establish that the top-level
	// thing is actually an array...
	top := statement{
		{"json", typBare},
		{"=", typEquals},
		{"[]", typEmptyArray},
		{";", typSemi},
	}
	if opts&optMonochrome > 0 {
		fmt.Fprintln(w, top.String())
	} else {
		fmt.Fprintln(w, top.colorString())
	}

	// Read the input line by line
	sc := bufio.NewScanner(r)
	i := 0
	for sc.Scan() {

		line := bytes.NewBuffer(sc.Bytes())

		ss, err := statementsFromJSON(line, makePrefix(i))
		i++
		if err != nil {
			return exitFormStatements, fmt.Errorf("failed to form statements: %s", err)
		}

		// Go's maps do not have well-defined ordering, but we want a consistent
		// output for a given input, so we must sort the statements
		if opts&optNoSort == 0 {
			sort.Sort(ss)
		}

		if opts&optMonochrome > 0 {
			for _, s := range ss {
				fmt.Fprintln(w, s.String())
			}
		} else {
			for _, s := range ss {
				fmt.Fprintln(w, s.colorString())
			}
		}
	}
	if err := sc.Err(); err != nil {
		return exitFormStatements, fmt.Errorf("error reading multiline input: %s", err)
	}

	return exitOK, nil

}

// ungron is the reverse of gron. Given assignment statements as input,
// it returns JSON. The only option is optMonochrome
func ungron(r io.Reader, w io.Writer, opts int) (int, error) {
	scanner := bufio.NewScanner(r)

	// Make a list of statements from the input
	var ss statements
	for scanner.Scan() {
		s := statementFromString(scanner.Text())
		ss.add(s)
	}
	if err := scanner.Err(); err != nil {
		return exitReadInput, fmt.Errorf("failed to read input statements")
	}

	// turn the statements into a single merged interface{} type
	merged, err := ss.toInterface()
	if err != nil {
		return exitParseStatements, err
	}

	// If there's only one top level key and it's "json", make that the top level thing
	mergedMap, ok := merged.(map[string]interface{})
	if ok {
		if len(mergedMap) == 1 {
			if _, exists := mergedMap["json"]; exists {
				merged = mergedMap["json"]
			}
		}
	}

	// Marshal the output into JSON to display to the user
	j, err := json.MarshalIndent(merged, "", "  ")
	if err != nil {
		return exitJSONEncode, errors.Wrap(err, "failed to convert statements to JSON")
	}

	// If the output isn't monochrome, add color to the JSON
	if opts&optMonochrome == 0 {
		c, err := colorizeJSON(j)

		// If we failed to colorize the JSON for whatever reason,
		// we'll just fall back to monochrome output, otherwise
		// replace the monochrome JSON with glorious technicolor
		if err == nil {
			j = c
		}
	}

	fmt.Fprintf(w, "%s\n", j)

	return exitOK, nil
}

func colorizeJSON(src []byte) ([]byte, error) {
	out := &bytes.Buffer{}
	f := jsoncolor.NewFormatter()

	f.StringColor = strColor
	f.ObjectColor = braceColor
	f.ArrayColor = braceColor
	f.FieldColor = bareColor
	f.NumberColor = numColor
	f.TrueColor = boolColor
	f.FalseColor = boolColor
	f.NullColor = boolColor

	err := f.Format(out, src)
	if err != nil {
		return out.Bytes(), err
	}
	return out.Bytes(), nil
}

func fatal(code int, err error) {
	fmt.Fprintf(os.Stderr, "%s\n", err)
	os.Exit(code)
}
<commit_msg>Switches from MarshalIndent to json.Encoder with SetEscapeHTML(false); #43<commit_after>package main

import (
	"bufio"
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"os"
	"sort"

	"github.com/fatih/color"
	"github.com/nwidger/jsoncolor"
	"github.com/pkg/errors"
)

// Exit codes
const (
	exitOK = iota
	exitOpenFile
	exitReadInput
	exitFormStatements
	exitFetchURL
	exitParseStatements
	exitJSONEncode
)

// Option bitfields
const (
	optMonochrome = iota + 1
	optNoSort
)

// Output colors
var (
	strColor   = color.New(color.FgYellow)
	braceColor = color.New(color.FgMagenta)
	bareColor  = color.New(color.FgBlue, color.Bold)
	numColor   = color.New(color.FgRed)
	boolColor  = color.New(color.FgCyan)
)

// gronVersion stores the current gron version, set at build
// time with the ldflags -X option
var gronVersion = "dev"

func init() {
	flag.Usage = func() {
		h := "Transform JSON (from a file, URL, or stdin) into discrete assignments to make it greppable\n\n"

		h += "Usage:\n"
		h += "  gron [OPTIONS] [FILE|URL|-]\n\n"

		h += "Options:\n"
		h += "  -u, --ungron     Reverse the operation (turn assignments back into JSON)\n"
		h += "  -c, --colorize   Colorize output (default on tty)\n"
		h += "  -m, --monochrome Monochrome (don't colorize output)\n"
		h += "  -s, --stream     Treat each line of input as a separate JSON object\n"
		h += "  -k, --insecure   Disable certificate validation\n"
		h += "      --no-sort    Don't sort output (faster)\n"
		h += "      --version    Print version information\n\n"

		h += "Exit Codes:\n"
		h += fmt.Sprintf("  %d\t%s\n", exitOK, "OK")
		h += fmt.Sprintf("  %d\t%s\n", exitOpenFile, "Failed to open file")
		h += fmt.Sprintf("  %d\t%s\n", exitReadInput, "Failed to read input")
		h += fmt.Sprintf("  %d\t%s\n", exitFormStatements, "Failed to form statements")
		h += fmt.Sprintf("  %d\t%s\n", exitFetchURL, "Failed to fetch URL")
		h += fmt.Sprintf("  %d\t%s\n", exitParseStatements, "Failed to parse statements")
		h += fmt.Sprintf("  %d\t%s\n", exitJSONEncode, "Failed to encode JSON")
		h += "\n"

		h += "Examples:\n"
		h += "  gron /tmp/apiresponse.json\n"
		h += "  gron http://jsonplaceholder.typicode.com/users/1 \n"
		h += "  curl -s http://jsonplaceholder.typicode.com/users/1 | gron\n"
		h += "  gron http://jsonplaceholder.typicode.com/users/1 | grep company | gron --ungron\n"

		fmt.Fprintf(os.Stderr, h)
	}
}

func main() {
	var (
		ungronFlag     bool
		colorizeFlag   bool
		monochromeFlag bool
		streamFlag     bool
		noSortFlag     bool
		versionFlag    bool
		insecureFlag   bool
	)

	flag.BoolVar(&ungronFlag, "ungron", false, "")
	flag.BoolVar(&ungronFlag, "u", false, "")
	flag.BoolVar(&colorizeFlag, "colorize", false, "")
	flag.BoolVar(&colorizeFlag, "c", false, "")
	flag.BoolVar(&monochromeFlag, "monochrome", false, "")
	flag.BoolVar(&monochromeFlag, "m", false, "")
	flag.BoolVar(&streamFlag, "s", false, "")
	flag.BoolVar(&streamFlag, "stream", false, "")
	flag.BoolVar(&noSortFlag, "no-sort", false, "")
	flag.BoolVar(&versionFlag, "version", false, "")
	flag.BoolVar(&insecureFlag, "k", false, "")
	flag.BoolVar(&insecureFlag, "insecure", false, "")

	flag.Parse()

	// Print version information
	if versionFlag {
		fmt.Printf("gron version %s\n", gronVersion)
		os.Exit(exitOK)
	}

	// Determine what the program's input should be:
	// file, HTTP URL or stdin
	var rawInput io.Reader
	filename := flag.Arg(0)
	if filename == "" || filename == "-" {
		rawInput = os.Stdin
	} else if validURL(filename) {
		r, err := getURL(filename, insecureFlag)
		if err != nil {
			fatal(exitFetchURL, err)
		}
		rawInput = r
	} else {
		r, err := os.Open(filename)
		if err != nil {
			fatal(exitOpenFile, err)
		}
		rawInput = r
	}

	var opts int
	// The monochrome option should be forced if the output isn't a terminal
	// to avoid doing unnecessary work calling the color functions
	switch {
	case colorizeFlag:
		color.NoColor = false
	case monochromeFlag || color.NoColor:
		opts = opts | optMonochrome
	}
	if noSortFlag {
		opts = opts | optNoSort
	}

	// Pick the appropriate action: gron, ungron or gronStream
	var a actionFn = gron
	if ungronFlag {
		a = ungron
	} else if streamFlag {
		a = gronStream
	}
	exitCode, err := a(rawInput, os.Stdout, opts)

	if exitCode != exitOK {
		fatal(exitCode, err)
	}

	os.Exit(exitOK)
}

// an actionFn represents a main action of the program, it accepts
// an input, output and a bitfield of options; returning an exit
// code and any error that occurred
type actionFn func(io.Reader, io.Writer, int) (int, error)

// gron is the default action. Given JSON as the input it returns a list
// of assignment statements. Possible options are optNoSort and optMonochrome
func gron(r io.Reader, w io.Writer, opts int) (int, error) {

	ss, err := statementsFromJSON(r, statement{{"json", typBare}})
	if err != nil {
		return exitFormStatements, fmt.Errorf("failed to form statements: %s", err)
	}

	// Go's maps do not have well-defined ordering, but we want a consistent
	// output for a given input, so we must sort the statements
	if opts&optNoSort == 0 {
		sort.Sort(ss)
	}

	if opts&optMonochrome > 0 {
		for _, s := range ss {
			fmt.Fprintln(w, s.String())
		}
	} else {
		for _, s := range ss {
			fmt.Fprintln(w, s.colorString())
		}
	}

	return exitOK, nil
}

// gronStream is like the gron action, but it treats the input as one
// JSON object per line. There's a bit of code duplication from the
// gron action, but it'd be fairly messy to combine the two actions
func gronStream(r io.Reader, w io.Writer, opts int) (int, error) {

	// Helper function to make the prefix statements for each line
	makePrefix := func(index int) statement {
		return statement{
			{"json", typBare},
			{"[", typLBrace},
			{fmt.Sprintf("%d", index), typNumericKey},
			{"]", typRBrace},
		}
	}

	// The first line of output needs to establish that the top-level
	// thing is actually an array...
	top := statement{
		{"json", typBare},
		{"=", typEquals},
		{"[]", typEmptyArray},
		{";", typSemi},
	}
	if opts&optMonochrome > 0 {
		fmt.Fprintln(w, top.String())
	} else {
		fmt.Fprintln(w, top.colorString())
	}

	// Read the input line by line
	sc := bufio.NewScanner(r)
	i := 0
	for sc.Scan() {

		line := bytes.NewBuffer(sc.Bytes())

		ss, err := statementsFromJSON(line, makePrefix(i))
		i++
		if err != nil {
			return exitFormStatements, fmt.Errorf("failed to form statements: %s", err)
		}

		// Go's maps do not have well-defined ordering, but we want a consistent
		// output for a given input, so we must sort the statements
		if opts&optNoSort == 0 {
			sort.Sort(ss)
		}

		if opts&optMonochrome > 0 {
			for _, s := range ss {
				fmt.Fprintln(w, s.String())
			}
		} else {
			for _, s := range ss {
				fmt.Fprintln(w, s.colorString())
			}
		}
	}
	if err := sc.Err(); err != nil {
		return exitFormStatements, fmt.Errorf("error reading multiline input: %s", err)
	}

	return exitOK, nil

}

// ungron is the reverse of gron. Given assignment statements as input,
// it returns JSON. The only option is optMonochrome
func ungron(r io.Reader, w io.Writer, opts int) (int, error) {
	scanner := bufio.NewScanner(r)

	// Make a list of statements from the input
	var ss statements
	for scanner.Scan() {
		s := statementFromString(scanner.Text())
		ss.add(s)
	}
	if err := scanner.Err(); err != nil {
		return exitReadInput, fmt.Errorf("failed to read input statements")
	}

	// turn the statements into a single merged interface{} type
	merged, err := ss.toInterface()
	if err != nil {
		return exitParseStatements, err
	}

	// If there's only one top level key and it's "json", make that the top level thing
	mergedMap, ok := merged.(map[string]interface{})
	if ok {
		if len(mergedMap) == 1 {
			if _, exists := mergedMap["json"]; exists {
				merged = mergedMap["json"]
			}
		}
	}

	// Marshal the output into JSON to display to the user
	out := &bytes.Buffer{}
	enc := json.NewEncoder(out)
	enc.SetIndent("", "  ")
	enc.SetEscapeHTML(false)
	err = enc.Encode(merged)
	if err != nil {
		return exitJSONEncode, errors.Wrap(err, "failed to convert statements to JSON")
	}
	j := out.Bytes()

	// If the output isn't monochrome, add color to the JSON
	if opts&optMonochrome == 0 {
		c, err := colorizeJSON(j)

		// If we failed to colorize the JSON for whatever reason,
		// we'll just fall back to monochrome output, otherwise
		// replace the monochrome JSON with glorious technicolor
		if err == nil {
			j = c
		}
	}

	fmt.Fprintf(w, "%s\n", j)

	return exitOK, nil
}

func colorizeJSON(src []byte) ([]byte, error) {
	out := &bytes.Buffer{}
	f := jsoncolor.NewFormatter()

	f.StringColor = strColor
	f.ObjectColor = braceColor
	f.ArrayColor = braceColor
	f.FieldColor = bareColor
	f.NumberColor = numColor
	f.TrueColor = boolColor
	f.FalseColor = boolColor
	f.NullColor = boolColor

	err := f.Format(out, src)
	if err != nil {
		return out.Bytes(), err
	}
	return out.Bytes(), nil
}

func fatal(code int, err error) {
	fmt.Fprintf(os.Stderr, "%s\n", err)
	os.Exit(code)
}
<|endoftext|>


<commit_before><commit_msg>cleans up function ordering<commit_after><|endoftext|>


<commit_before><commit_msg>cheesy checking of url/endpoint<commit_after><|endoftext|>


<commit_before><commit_msg>Minor changes<commit_after><|endoftext|>


<commit_before><commit_msg>hello world main<commit_after><|endoftext|>


<commit_before><commit_msg>Use US when country is empty<commit_after><|endoftext|>


<commit_before><commit_msg>short urls beginning with my own short url are autodetected, for future use to avoid rewrites.<commit_after><|endoftext|>


<commit_before><commit_msg>all source domains<commit_after><|endoftext|>


<commit_before><commit_msg>Minor bug fix<commit_after><|endoftext|>


<commit_before><commit_msg>all: move connection error printing into once.Do<commit_after><|endoftext|>


<commit_before>/**
 * keys
 * This module handles all the key generation logic for NaCl and AES.
 * It passes off reading from the rand reader to the entropy module to
 * limit code duplication. It also provides helper methods to turn byte
 * array data into the proper key types so they can be used in the crypto
 * methods where they are needed, and read and write key files to disk.
 */
package msg

import (
	"crypto/rand"
	"crypto/sha1"
	"crypto/sha256"
	"encoding/base64"
	"golang.org/x/crypto/nacl/box"
	"golang.org/x/crypto/pbkdf2"
	"io/ioutil"
	"log"
	"os"
)

// Consts used in this module
const PBKDF2_SALT_BYTES = 32
const PBKDF2_PASSWORD_BYTES = 32
const PBKDF2_KEY_BYTES = 32
const PBKDF2_CYCLES = 5000

// Include getters to make it clear other modules within cryptoUtils are using
// these constants
const NACL_FINGERPRINT_BYTES = 20

func getNACLFingerprintLength() int {
	return NACL_FINGERPRINT_BYTES
}

const NACL_KEY_BYTES = 32

/**
 * GenAES256Key
 * Generate a completley random 32 byte key using PBKDF2 with a completely random
 * "passphrase" and salt. sha256 is being used as the hashing function.
 */
func GenAES256Key() []byte {
	password := ReadFromRand(PBKDF2_PASSWORD_BYTES)
	salt := ReadFromRand(PBKDF2_SALT_BYTES)

	return pbkdf2.Key(password, salt, PBKDF2_CYCLES, PBKDF2_KEY_BYTES, sha256.New)
}

/**
 * GenerateNACLKeyPair
 * Helper method that wraps NaCl's built in key generation function.
 */
func GenerateNACLKeyPair() (pubKey, privKey *[32]byte) {
	pubKey, privKey, err := box.GenerateKey(rand.Reader)
	if err != nil {
		log.Fatal(err)
	}
	return
}

/**
 * WriteNACLKeyFile
 * Helper method that encodes NACL keys to base64 then writes them to disk.
 */
func WriteNACLKeyFile(keyFilePath string, key *[32]byte, keyComment string, perms os.FileMode) {
	encodedKey := base64.StdEncoding.EncodeToString(key[:])

	err := ioutil.WriteFile(keyFilePath, []byte(encodedKey + keyComment), perms)
	if err != nil {
		log.Fatal(err)
	}
}

/**
 * ReadNACLKeyFile
 * Helper method to read NACL key files that were written to disk with WriteNACLKeyFile.
 */
func ReadNACLKeyFile(keyFilePath string) string {
	keyFile, err := ioutil.ReadFile(keyFilePath)
	if err != nil {
		log.Fatal(err)
	}

	return string(keyFile)
}

/**
 * NACLKeyFromFile
 * Helper method that reads files containing an NaCl key string and returns an NaCl key.
 */
func NACLKeyFromFile(filePath string) *[32]byte {
	return StringToNACLKey(ReadNACLKeyFile(filePath))
}

/**
 * StringToNACLKey
 * Helper method to read encoded NACL key data strings and return an NACL key.
 * Because this is a helper method, we assume the string passed in is encoded
 * as base64.
 */
func StringToNACLKey(encodedKeyString string) *[32]byte {
	keyBytes, err := base64.StdEncoding.DecodeString(encodedKeyString[:44])
	if err != nil {
		log.Fatal(err)
	}

	naclKey := BytesToNACLKey(keyBytes)
	return naclKey
}

/**
 * GetNACLKeyFingerprint
 * Generates 20 byte fingerprints for NaCl Keys using sha1
 * This is only meant to be used with PUBLIC keys.
 */
func GetNACLKeyFingerprint(key *[32]byte) []byte {
	h := sha1.New()
	h.Write(key[:])
	return h.Sum(nil)
}

/**
 * BytesToNACLKey
 * Helper that takes in a byte array and casts it to *[32]byte so it can be used
 * as an NACL key. This is used by ReadNACLKeyFile.
 */
func BytesToNACLKey(bytes []byte) *[32]byte {
	if len(bytes) < 32 {
		log.Fatal("Too few bytes! This does not appear to be a valid NACL Key")
	}
	var naclKey [32]byte
	copy(naclKey[:], bytes[:])
	return &naclKey
}
<commit_msg>Go fmt ./...<commit_after>/**
 * keys
 * This module handles all the key generation logic for NaCl and AES.
 * It passes off reading from the rand reader to the entropy module to
 * limit code duplication. It also provides helper methods to turn byte
 * array data into the proper key types so they can be used in the crypto
 * methods where they are needed, and read and write key files to disk.
 */
package msg

import (
	"crypto/rand"
	"crypto/sha1"
	"crypto/sha256"
	"encoding/base64"
	"golang.org/x/crypto/nacl/box"
	"golang.org/x/crypto/pbkdf2"
	"io/ioutil"
	"log"
	"os"
)

// Consts used in this module
const PBKDF2_SALT_BYTES = 32
const PBKDF2_PASSWORD_BYTES = 32
const PBKDF2_KEY_BYTES = 32
const PBKDF2_CYCLES = 5000

// Include getters to make it clear other modules within cryptoUtils are using
// these constants
const NACL_FINGERPRINT_BYTES = 20

func getNACLFingerprintLength() int {
	return NACL_FINGERPRINT_BYTES
}

const NACL_KEY_BYTES = 32

/**
 * GenAES256Key
 * Generate a completley random 32 byte key using PBKDF2 with a completely random
 * "passphrase" and salt. sha256 is being used as the hashing function.
 */
func GenAES256Key() []byte {
	password := ReadFromRand(PBKDF2_PASSWORD_BYTES)
	salt := ReadFromRand(PBKDF2_SALT_BYTES)

	return pbkdf2.Key(password, salt, PBKDF2_CYCLES, PBKDF2_KEY_BYTES, sha256.New)
}

/**
 * GenerateNACLKeyPair
 * Helper method that wraps NaCl's built in key generation function.
 */
func GenerateNACLKeyPair() (pubKey, privKey *[32]byte) {
	pubKey, privKey, err := box.GenerateKey(rand.Reader)
	if err != nil {
		log.Fatal(err)
	}
	return
}

/**
 * WriteNACLKeyFile
 * Helper method that encodes NACL keys to base64 then writes them to disk.
 */
func WriteNACLKeyFile(keyFilePath string, key *[32]byte, keyComment string, perms os.FileMode) {
	encodedKey := base64.StdEncoding.EncodeToString(key[:])

	err := ioutil.WriteFile(keyFilePath, []byte(encodedKey+keyComment), perms)
	if err != nil {
		log.Fatal(err)
	}
}

/**
 * ReadNACLKeyFile
 * Helper method to read NACL key files that were written to disk with WriteNACLKeyFile.
 */
func ReadNACLKeyFile(keyFilePath string) string {
	keyFile, err := ioutil.ReadFile(keyFilePath)
	if err != nil {
		log.Fatal(err)
	}

	return string(keyFile)
}

/**
 * NACLKeyFromFile
 * Helper method that reads files containing an NaCl key string and returns an NaCl key.
 */
func NACLKeyFromFile(filePath string) *[32]byte {
	return StringToNACLKey(ReadNACLKeyFile(filePath))
}

/**
 * StringToNACLKey
 * Helper method to read encoded NACL key data strings and return an NACL key.
 * Because this is a helper method, we assume the string passed in is encoded
 * as base64.
 */
func StringToNACLKey(encodedKeyString string) *[32]byte {
	keyBytes, err := base64.StdEncoding.DecodeString(encodedKeyString[:44])
	if err != nil {
		log.Fatal(err)
	}

	naclKey := BytesToNACLKey(keyBytes)
	return naclKey
}

/**
 * GetNACLKeyFingerprint
 * Generates 20 byte fingerprints for NaCl Keys using sha1
 * This is only meant to be used with PUBLIC keys.
 */
func GetNACLKeyFingerprint(key *[32]byte) []byte {
	h := sha1.New()
	h.Write(key[:])
	return h.Sum(nil)
}

/**
 * BytesToNACLKey
 * Helper that takes in a byte array and casts it to *[32]byte so it can be used
 * as an NACL key. This is used by ReadNACLKeyFile.
 */
func BytesToNACLKey(bytes []byte) *[32]byte {
	if len(bytes) < 32 {
		log.Fatal("Too few bytes! This does not appear to be a valid NACL Key")
	}
	var naclKey [32]byte
	copy(naclKey[:], bytes[:])
	return &naclKey
}
<|endoftext|>


<commit_before>package bitcoincash

import (
	"github.com/OpenBazaar/wallet-interface"
	"github.com/btcsuite/btcd/chaincfg"
	hd "github.com/btcsuite/btcutil/hdkeychain"
	"github.com/btcsuite/goleveldb/leveldb/errors"
)

const LOOKAHEADWINDOW = 100

type KeyManager struct {
	datastore wallet.Keys
	params    *chaincfg.Params

	internalKey *hd.ExtendedKey
	externalKey *hd.ExtendedKey
}

func NewKeyManager(db wallet.Keys, params *chaincfg.Params, masterPrivKey *hd.ExtendedKey) (*KeyManager, error) {
	internal, external, err := Bip44Derivation(masterPrivKey)
	if err != nil {
		return nil, err
	}
	km := &KeyManager{
		datastore:   db,
		params:      params,
		internalKey: internal,
		externalKey: external,
	}
	if err := km.lookahead(); err != nil {
		return nil, err
	}
	return km, nil
}

// m / purpose' / coin_type' / account' / change / address_index
func Bip44Derivation(masterPrivKey *hd.ExtendedKey) (internal, external *hd.ExtendedKey, err error) {
	// Purpose = bip44
	fourtyFour, err := masterPrivKey.Child(hd.HardenedKeyStart + 44)
	if err != nil {
		return nil, nil, err
	}
	// Cointype = bitcoin
	bitcoin, err := fourtyFour.Child(hd.HardenedKeyStart + 0)
	if err != nil {
		return nil, nil, err
	}
	// Account = 0
	account, err := bitcoin.Child(hd.HardenedKeyStart + 0)
	if err != nil {
		return nil, nil, err
	}
	// Change(0) = external
	external, err = account.Child(0)
	if err != nil {
		return nil, nil, err
	}
	// Change(1) = internal
	internal, err = account.Child(1)
	if err != nil {
		return nil, nil, err
	}
	return internal, external, nil
}

func (km *KeyManager) GetCurrentKey(purpose wallet.KeyPurpose) (*hd.ExtendedKey, error) {
	i, err := km.datastore.GetUnused(purpose)
	if err != nil {
		return nil, err
	}
	if len(i) == 0 {
		return nil, errors.New("No unused keys in database")
	}
	return km.generateChildKey(purpose, uint32(i[0]))
}

func (km *KeyManager) GetFreshKey(purpose wallet.KeyPurpose) (*hd.ExtendedKey, error) {
	index, _, err := km.datastore.GetLastKeyIndex(purpose)
	var childKey *hd.ExtendedKey
	if err != nil {
		index = 0
	} else {
		index += 1
	}
	for {
		// There is a small possibility bip32 keys can be invalid. The procedure in such cases
		// is to discard the key and derive the next one. This loop will continue until a valid key
		// is derived.
		childKey, err = km.generateChildKey(purpose, uint32(index))
		if err == nil {
			break
		}
		index += 1
	}
	addr, err := childKey.Address(km.params)
	if err != nil {
		return nil, err
	}
	p := wallet.KeyPath{wallet.KeyPurpose(purpose), index}
	err = km.datastore.Put(addr.ScriptAddress(), p)
	if err != nil {
		return nil, err
	}
	return childKey, nil
}

func (km *KeyManager) GetKeys() []*hd.ExtendedKey {
	var keys []*hd.ExtendedKey
	keyPaths, err := km.datastore.GetAll()
	if err != nil {
		return keys
	}
	for _, path := range keyPaths {
		k, err := km.generateChildKey(path.Purpose, uint32(path.Index))
		if err != nil {
			continue
		}
		keys = append(keys, k)
	}
	return keys
}

func (km *KeyManager) GetKeyForScript(scriptAddress []byte) (*hd.ExtendedKey, error) {
	keyPath, err := km.datastore.GetPathForKey(scriptAddress)
	if err != nil {
		key, err := km.datastore.GetKey(scriptAddress)
		if err != nil {
			return nil, err
		}
		hdKey := hd.NewExtendedKey(
			km.params.HDPrivateKeyID[:],
			key.Serialize(),
			make([]byte, 32),
			[]byte{0x00, 0x00, 0x00, 0x00},
			0,
			0,
			true)
		return hdKey, nil
	}
	return km.generateChildKey(keyPath.Purpose, uint32(keyPath.Index))
}

// Mark the given key as used and extend the lookahead window
func (km *KeyManager) MarkKeyAsUsed(scriptAddress []byte) error {
	if err := km.datastore.MarkKeyAsUsed(scriptAddress); err != nil {
		return err
	}
	return km.lookahead()
}

func (km *KeyManager) generateChildKey(purpose wallet.KeyPurpose, index uint32) (*hd.ExtendedKey, error) {
	if purpose == wallet.EXTERNAL {
		return km.externalKey.Child(index)
	} else if purpose == wallet.INTERNAL {
		return km.internalKey.Child(index)
	}
	return nil, errors.New("Unknown key purpose")
}

func (km *KeyManager) lookahead() error {
	lookaheadWindows := km.datastore.GetLookaheadWindows()
	for purpose, size := range lookaheadWindows {
		if size < LOOKAHEADWINDOW {
			for i := 0; i < (LOOKAHEADWINDOW - size); i++ {
				_, err := km.GetFreshKey(purpose)
				if err != nil {
					return err
				}
			}
		}
	}
	return nil
}
<commit_msg>Switch bip44 derivation to use BitcoinCash coin type<commit_after>package bitcoincash

import (
	"github.com/OpenBazaar/wallet-interface"
	"github.com/btcsuite/btcd/chaincfg"
	hd "github.com/btcsuite/btcutil/hdkeychain"
	"github.com/btcsuite/goleveldb/leveldb/errors"
)

const LOOKAHEADWINDOW = 100

type KeyManager struct {
	datastore wallet.Keys
	params    *chaincfg.Params

	internalKey *hd.ExtendedKey
	externalKey *hd.ExtendedKey
}

func NewKeyManager(db wallet.Keys, params *chaincfg.Params, masterPrivKey *hd.ExtendedKey) (*KeyManager, error) {
	internal, external, err := Bip44Derivation(masterPrivKey)
	if err != nil {
		return nil, err
	}
	km := &KeyManager{
		datastore:   db,
		params:      params,
		internalKey: internal,
		externalKey: external,
	}
	if err := km.lookahead(); err != nil {
		return nil, err
	}
	return km, nil
}

// m / purpose' / coin_type' / account' / change / address_index
func Bip44Derivation(masterPrivKey *hd.ExtendedKey) (internal, external *hd.ExtendedKey, err error) {
	// Purpose = bip44
	fourtyFour, err := masterPrivKey.Child(hd.HardenedKeyStart + 44)
	if err != nil {
		return nil, nil, err
	}
	// Cointype = bitcoin
	bitcoin, err := fourtyFour.Child(hd.HardenedKeyStart + 145)
	if err != nil {
		return nil, nil, err
	}
	// Account = 0
	account, err := bitcoin.Child(hd.HardenedKeyStart + 0)
	if err != nil {
		return nil, nil, err
	}
	// Change(0) = external
	external, err = account.Child(0)
	if err != nil {
		return nil, nil, err
	}
	// Change(1) = internal
	internal, err = account.Child(1)
	if err != nil {
		return nil, nil, err
	}
	return internal, external, nil
}

func (km *KeyManager) GetCurrentKey(purpose wallet.KeyPurpose) (*hd.ExtendedKey, error) {
	i, err := km.datastore.GetUnused(purpose)
	if err != nil {
		return nil, err
	}
	if len(i) == 0 {
		return nil, errors.New("No unused keys in database")
	}
	return km.generateChildKey(purpose, uint32(i[0]))
}

func (km *KeyManager) GetFreshKey(purpose wallet.KeyPurpose) (*hd.ExtendedKey, error) {
	index, _, err := km.datastore.GetLastKeyIndex(purpose)
	var childKey *hd.ExtendedKey
	if err != nil {
		index = 0
	} else {
		index += 1
	}
	for {
		// There is a small possibility bip32 keys can be invalid. The procedure in such cases
		// is to discard the key and derive the next one. This loop will continue until a valid key
		// is derived.
		childKey, err = km.generateChildKey(purpose, uint32(index))
		if err == nil {
			break
		}
		index += 1
	}
	addr, err := childKey.Address(km.params)
	if err != nil {
		return nil, err
	}
	p := wallet.KeyPath{wallet.KeyPurpose(purpose), index}
	err = km.datastore.Put(addr.ScriptAddress(), p)
	if err != nil {
		return nil, err
	}
	return childKey, nil
}

func (km *KeyManager) GetKeys() []*hd.ExtendedKey {
	var keys []*hd.ExtendedKey
	keyPaths, err := km.datastore.GetAll()
	if err != nil {
		return keys
	}
	for _, path := range keyPaths {
		k, err := km.generateChildKey(path.Purpose, uint32(path.Index))
		if err != nil {
			continue
		}
		keys = append(keys, k)
	}
	return keys
}

func (km *KeyManager) GetKeyForScript(scriptAddress []byte) (*hd.ExtendedKey, error) {
	keyPath, err := km.datastore.GetPathForKey(scriptAddress)
	if err != nil {
		key, err := km.datastore.GetKey(scriptAddress)
		if err != nil {
			return nil, err
		}
		hdKey := hd.NewExtendedKey(
			km.params.HDPrivateKeyID[:],
			key.Serialize(),
			make([]byte, 32),
			[]byte{0x00, 0x00, 0x00, 0x00},
			0,
			0,
			true)
		return hdKey, nil
	}
	return km.generateChildKey(keyPath.Purpose, uint32(keyPath.Index))
}

// Mark the given key as used and extend the lookahead window
func (km *KeyManager) MarkKeyAsUsed(scriptAddress []byte) error {
	if err := km.datastore.MarkKeyAsUsed(scriptAddress); err != nil {
		return err
	}
	return km.lookahead()
}

func (km *KeyManager) generateChildKey(purpose wallet.KeyPurpose, index uint32) (*hd.ExtendedKey, error) {
	if purpose == wallet.EXTERNAL {
		return km.externalKey.Child(index)
	} else if purpose == wallet.INTERNAL {
		return km.internalKey.Child(index)
	}
	return nil, errors.New("Unknown key purpose")
}

func (km *KeyManager) lookahead() error {
	lookaheadWindows := km.datastore.GetLookaheadWindows()
	for purpose, size := range lookaheadWindows {
		if size < LOOKAHEADWINDOW {
			for i := 0; i < (LOOKAHEADWINDOW - size); i++ {
				_, err := km.GetFreshKey(purpose)
				if err != nil {
					return err
				}
			}
		}
	}
	return nil
}
<|endoftext|>