<commit_before>package srtp

import (
	"crypto/aes"
	"crypto/cipher"
	"crypto/hmac"
	"crypto/sha1"
	"encoding/binary"

	"github.com/pions/webrtc/pkg/rtp"
	"github.com/pkg/errors"
)

const (
	labelEncryption        = 0x00
	labelSalt              = 0x02
	labelAuthenticationTag = 0x01

	keyLen  = 16
	saltLen = 14

	maxROCDisorder    = 100
	maxSequenceNumber = 65535
)

// Context represents a SRTP cryptographic context
// which is a tuple of <SSRC, destination network address, destination transport port number>
type Context struct {
	ssrc uint32

	rolloverCounter      uint32
	rolloverHasProcessed bool
	lastSequenceNumber   uint16

	masterKey  []byte
	masterSalt []byte

	sessionKey     []byte
	sessionSalt    []byte
	sessionAuthTag []byte

	block cipher.Block
}

/*
  Note to reader
  lines prefixed with '-' are added by Sean-Der
  lines without that prefix are from RFC
*/

// CreateContext creates a new SRTP Context
func CreateContext(masterKey, masterSalt []byte, profile string, ssrc uint32) (c *Context, err error) {
	if masterKeyLen := len(masterKey); masterKeyLen != keyLen {
		return c, errors.Errorf("SRTP Master Key must be len %d, got %d", masterKey, keyLen)
	} else if masterSaltLen := len(masterSalt); masterSaltLen != saltLen {
		return c, errors.Errorf("SRTP Salt must be len %d, got %d", saltLen, masterSaltLen)
	}

	c = &Context{
		masterKey:  masterKey,
		masterSalt: masterSalt,
		ssrc:       ssrc,
	}

	if c.sessionKey, err = c.generateSessionKey(); err != nil {
		return nil, err
	}

	if c.sessionSalt, err = c.generateSessionSalt(); err != nil {
		return nil, err
	}

	if c.sessionAuthTag, err = c.generateSessionAuthTag(); err != nil {
		return nil, err
	}

	c.block, err = aes.NewCipher(c.sessionKey)
	if err != nil {
		return nil, err
	}

	return c, nil
}

func (c *Context) generateSessionKey() ([]byte, error) {
	// https://tools.ietf.org/html/rfc3711#appendix-B.3
	// The input block for AES-CM is generated by exclusive-oring the master salt with the
	// concatenation of the encryption key label 0x00 with (index DIV kdr),
	// - index is 'rollover count' and DIV is 'divided by'
	sessionKey := make([]byte, len(c.masterSalt))
	copy(sessionKey, c.masterSalt)

	labelAndIndexOverKdr := []byte{labelEncryption, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}
	for i, j := len(labelAndIndexOverKdr)-1, len(sessionKey)-1; i >= 0; i, j = i-1, j-1 {
		sessionKey[j] = sessionKey[j] ^ labelAndIndexOverKdr[i]
	}

	// then padding on the right with two null octets (which implements the multiply-by-2^16 operation, see Section 4.3.3).
	sessionKey = append(sessionKey, []byte{0x00, 0x00}...)

	//The resulting value is then AES-CM- encrypted using the master key to get the cipher key.
	block, err := aes.NewCipher(c.masterKey)
	if err != nil {
		return nil, err
	}

	block.Encrypt(sessionKey, sessionKey)
	return sessionKey, nil
}

func (c *Context) generateSessionSalt() ([]byte, error) {
	// https://tools.ietf.org/html/rfc3711#appendix-B.3
	// The input block for AES-CM is generated by exclusive-oring the master salt with
	// the concatenation of the encryption salt label
	sessionSalt := make([]byte, len(c.masterSalt))
	copy(sessionSalt, c.masterSalt)

	labelAndIndexOverKdr := []byte{labelSalt, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}
	for i, j := len(labelAndIndexOverKdr)-1, len(sessionSalt)-1; i >= 0; i, j = i-1, j-1 {
		sessionSalt[j] = sessionSalt[j] ^ labelAndIndexOverKdr[i]
	}

	// That value is padded and encrypted as above.
	sessionSalt = append(sessionSalt, []byte{0x00, 0x00}...)
	block, err := aes.NewCipher(c.masterKey)
	if err != nil {
		return nil, err
	}

	block.Encrypt(sessionSalt, sessionSalt)
	return sessionSalt[0:saltLen], nil
}
func (c *Context) generateSessionAuthTag() ([]byte, error) {
	// https://tools.ietf.org/html/rfc3711#appendix-B.3
	// We now show how the auth key is generated.  The input block for AES-
	// CM is generated as above, but using the authentication key label.
	sessionAuthTag := make([]byte, len(c.masterSalt))
	copy(sessionAuthTag, c.masterSalt)

	labelAndIndexOverKdr := []byte{labelAuthenticationTag, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}
	for i, j := len(labelAndIndexOverKdr)-1, len(sessionAuthTag)-1; i >= 0; i, j = i-1, j-1 {
		sessionAuthTag[j] = sessionAuthTag[j] ^ labelAndIndexOverKdr[i]
	}

	// That value is padded and encrypted as above.
	// - We need to do multiple runs at key size (20) is larger then source
	firstRun := append(sessionAuthTag, []byte{0x00, 0x00}...)
	secondRun := append(sessionAuthTag, []byte{0x00, 0x01}...)
	block, err := aes.NewCipher(c.masterKey)
	if err != nil {
		return nil, err
	}

	block.Encrypt(firstRun, firstRun)
	block.Encrypt(secondRun, secondRun)
	return append(firstRun, secondRun[:4]...), nil
}

// Generate IV https://tools.ietf.org/html/rfc3711#section-4.1.1
// where the 128-bit integer value IV SHALL be defined by the SSRC, the
// SRTP packet index i, and the SRTP session salting key k_s, as below.
// - ROC = a 32-bit unsigned rollover counter (ROC), which records how many
// -       times the 16-bit RTP sequence number has been reset to zero after
// -       passing through 65,535
// i = 2^16 * ROC + SEQ
// IV = (salt*2 ^ 16) | (ssrc*2 ^ 64) | (i*2 ^ 16)
func (c *Context) generateCounter(sequenceNumber uint16) []byte {
	counter := make([]byte, 16)

	binary.BigEndian.PutUint32(counter[4:], c.ssrc)
	binary.BigEndian.PutUint32(counter[8:], c.rolloverCounter)
	binary.BigEndian.PutUint32(counter[12:], uint32(sequenceNumber)<<16)

	for i := range c.sessionSalt {
		counter[i] = counter[i] ^ c.sessionSalt[i]
	}

	return counter
}

// https://tools.ietf.org/html/rfc3550#appendix-A.1
func (c *Context) updateRolloverCount(sequenceNumber uint16) {
	if !c.rolloverHasProcessed {
		c.rolloverHasProcessed = true
	} else if sequenceNumber == 0 { // We exactly hit the rollover count

		// Only update rolloverCounter if lastSequenceNumber is greater then maxROCDisorder
		// otherwise we already incremented for disorder
		if c.lastSequenceNumber > maxROCDisorder {
			c.rolloverCounter++
		}
	} else if c.lastSequenceNumber < maxROCDisorder && sequenceNumber > (maxSequenceNumber-maxROCDisorder) {
		// Our last sequence number incremented because we crossed 0, but then our current number was within maxROCDisorder of the max
		// So we fell behind, drop to account for jitter
		c.rolloverCounter--
	} else if sequenceNumber < maxROCDisorder && c.lastSequenceNumber > (maxSequenceNumber-maxROCDisorder) {
		// our current is within a maxROCDisorder of 0
		// and our last sequence number was a high sequence number, increment to account for jitter
		c.rolloverCounter++
	}
	c.lastSequenceNumber = sequenceNumber
}

// DecryptPacket decrypts a RTP packet with an encrypted payload
func (c *Context) DecryptPacket(packet *rtp.Packet) bool {
	c.updateRolloverCount(packet.SequenceNumber)

	stream := cipher.NewCTR(c.block, c.generateCounter(packet.SequenceNumber))
	stream.XORKeyStream(packet.Payload, packet.Payload)

	// TODO remove tags, need to assert value
	packet.Payload = packet.Payload[:len(packet.Payload)-10]

	// Replace payload with decrypted
	packet.Raw = packet.Raw[0:packet.PayloadOffset]
	packet.Raw = append(packet.Raw, packet.Payload...)

	return true
}

func (c *Context) addAuthTag(packet *rtp.Packet) error {
	// https://tools.ietf.org/html/rfc3711#section-4.2
	// In the case of SRTP, M SHALL consist of the Authenticated
	// Portion of the packet (as specified in Figure 1) concatenated with
	// the ROC, M = Authenticated Portion || ROC;
	//
	// The pre-defined authentication transform for SRTP is HMAC-SHA1
	// [RFC2104].  With HMAC-SHA1, the SRTP_PREFIX_LENGTH (Figure 3) SHALL
	// be 0.  For SRTP (respectively SRTCP), the HMAC SHALL be applied to
	// the session authentication key and M as specified above, i.e.,
	// HMAC(k_a, M).  The HMAC output SHALL then be truncated to the n_tag
	// left-most bits.
	// - Authenticated portion of the packet is everything BEFORE MKI
	// - k_a is the session message authentication key
	// - n_tag is the bit-length of the output authentication tag

	mac := hmac.New(sha1.New, c.sessionAuthTag)
	fullPkt, err := packet.Marshal()
	if err != nil {
		return err
	}

	fullPkt = append(fullPkt, make([]byte, 4)...)
	binary.BigEndian.PutUint32(fullPkt[len(fullPkt)-4:], c.rolloverCounter)

	if _, err := mac.Write(fullPkt); err != nil {
		return err
	}

	packet.Payload = append(packet.Payload, mac.Sum(nil)[0:10]...)
	return nil
}

// EncryptPacket Encrypts a SRTP packet in place
func (c *Context) EncryptPacket(packet *rtp.Packet) bool {
	c.updateRolloverCount(packet.SequenceNumber)

	stream := cipher.NewCTR(c.block, c.generateCounter(packet.SequenceNumber))
	stream.XORKeyStream(packet.Payload, packet.Payload)

	if err := c.addAuthTag(packet); err != nil {
		return false
	}

	return true
}
<commit_msg>Silence gosec SHA1 blacklist warning<commit_after>package srtp

import (
	"crypto/aes"
	"crypto/cipher"
	"crypto/hmac"
	"crypto/sha1" // #nosec
	"encoding/binary"

	"github.com/pions/webrtc/pkg/rtp"
	"github.com/pkg/errors"
)

const (
	labelEncryption        = 0x00
	labelSalt              = 0x02
	labelAuthenticationTag = 0x01

	keyLen  = 16
	saltLen = 14

	maxROCDisorder    = 100
	maxSequenceNumber = 65535
)

// Context represents a SRTP cryptographic context
// which is a tuple of <SSRC, destination network address, destination transport port number>
type Context struct {
	ssrc uint32

	rolloverCounter      uint32
	rolloverHasProcessed bool
	lastSequenceNumber   uint16

	masterKey  []byte
	masterSalt []byte

	sessionKey     []byte
	sessionSalt    []byte
	sessionAuthTag []byte

	block cipher.Block
}

/*
  Note to reader
  lines prefixed with '-' are added by Sean-Der
  lines without that prefix are from RFC
*/

// CreateContext creates a new SRTP Context
func CreateContext(masterKey, masterSalt []byte, profile string, ssrc uint32) (c *Context, err error) {
	if masterKeyLen := len(masterKey); masterKeyLen != keyLen {
		return c, errors.Errorf("SRTP Master Key must be len %d, got %d", masterKey, keyLen)
	} else if masterSaltLen := len(masterSalt); masterSaltLen != saltLen {
		return c, errors.Errorf("SRTP Salt must be len %d, got %d", saltLen, masterSaltLen)
	}

	c = &Context{
		masterKey:  masterKey,
		masterSalt: masterSalt,
		ssrc:       ssrc,
	}

	if c.sessionKey, err = c.generateSessionKey(); err != nil {
		return nil, err
	}

	if c.sessionSalt, err = c.generateSessionSalt(); err != nil {
		return nil, err
	}

	if c.sessionAuthTag, err = c.generateSessionAuthTag(); err != nil {
		return nil, err
	}

	c.block, err = aes.NewCipher(c.sessionKey)
	if err != nil {
		return nil, err
	}

	return c, nil
}

func (c *Context) generateSessionKey() ([]byte, error) {
	// https://tools.ietf.org/html/rfc3711#appendix-B.3
	// The input block for AES-CM is generated by exclusive-oring the master salt with the
	// concatenation of the encryption key label 0x00 with (index DIV kdr),
	// - index is 'rollover count' and DIV is 'divided by'
	sessionKey := make([]byte, len(c.masterSalt))
	copy(sessionKey, c.masterSalt)

	labelAndIndexOverKdr := []byte{labelEncryption, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}
	for i, j := len(labelAndIndexOverKdr)-1, len(sessionKey)-1; i >= 0; i, j = i-1, j-1 {
		sessionKey[j] = sessionKey[j] ^ labelAndIndexOverKdr[i]
	}

	// then padding on the right with two null octets (which implements the multiply-by-2^16 operation, see Section 4.3.3).
	sessionKey = append(sessionKey, []byte{0x00, 0x00}...)

	//The resulting value is then AES-CM- encrypted using the master key to get the cipher key.
	block, err := aes.NewCipher(c.masterKey)
	if err != nil {
		return nil, err
	}

	block.Encrypt(sessionKey, sessionKey)
	return sessionKey, nil
}

func (c *Context) generateSessionSalt() ([]byte, error) {
	// https://tools.ietf.org/html/rfc3711#appendix-B.3
	// The input block for AES-CM is generated by exclusive-oring the master salt with
	// the concatenation of the encryption salt label
	sessionSalt := make([]byte, len(c.masterSalt))
	copy(sessionSalt, c.masterSalt)

	labelAndIndexOverKdr := []byte{labelSalt, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}
	for i, j := len(labelAndIndexOverKdr)-1, len(sessionSalt)-1; i >= 0; i, j = i-1, j-1 {
		sessionSalt[j] = sessionSalt[j] ^ labelAndIndexOverKdr[i]
	}

	// That value is padded and encrypted as above.
	sessionSalt = append(sessionSalt, []byte{0x00, 0x00}...)
	block, err := aes.NewCipher(c.masterKey)
	if err != nil {
		return nil, err
	}

	block.Encrypt(sessionSalt, sessionSalt)
	return sessionSalt[0:saltLen], nil
}
func (c *Context) generateSessionAuthTag() ([]byte, error) {
	// https://tools.ietf.org/html/rfc3711#appendix-B.3
	// We now show how the auth key is generated.  The input block for AES-
	// CM is generated as above, but using the authentication key label.
	sessionAuthTag := make([]byte, len(c.masterSalt))
	copy(sessionAuthTag, c.masterSalt)

	labelAndIndexOverKdr := []byte{labelAuthenticationTag, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}
	for i, j := len(labelAndIndexOverKdr)-1, len(sessionAuthTag)-1; i >= 0; i, j = i-1, j-1 {
		sessionAuthTag[j] = sessionAuthTag[j] ^ labelAndIndexOverKdr[i]
	}

	// That value is padded and encrypted as above.
	// - We need to do multiple runs at key size (20) is larger then source
	firstRun := append(sessionAuthTag, []byte{0x00, 0x00}...)
	secondRun := append(sessionAuthTag, []byte{0x00, 0x01}...)
	block, err := aes.NewCipher(c.masterKey)
	if err != nil {
		return nil, err
	}

	block.Encrypt(firstRun, firstRun)
	block.Encrypt(secondRun, secondRun)
	return append(firstRun, secondRun[:4]...), nil
}

// Generate IV https://tools.ietf.org/html/rfc3711#section-4.1.1
// where the 128-bit integer value IV SHALL be defined by the SSRC, the
// SRTP packet index i, and the SRTP session salting key k_s, as below.
// - ROC = a 32-bit unsigned rollover counter (ROC), which records how many
// -       times the 16-bit RTP sequence number has been reset to zero after
// -       passing through 65,535
// i = 2^16 * ROC + SEQ
// IV = (salt*2 ^ 16) | (ssrc*2 ^ 64) | (i*2 ^ 16)
func (c *Context) generateCounter(sequenceNumber uint16) []byte {
	counter := make([]byte, 16)

	binary.BigEndian.PutUint32(counter[4:], c.ssrc)
	binary.BigEndian.PutUint32(counter[8:], c.rolloverCounter)
	binary.BigEndian.PutUint32(counter[12:], uint32(sequenceNumber)<<16)

	for i := range c.sessionSalt {
		counter[i] = counter[i] ^ c.sessionSalt[i]
	}

	return counter
}

// https://tools.ietf.org/html/rfc3550#appendix-A.1
func (c *Context) updateRolloverCount(sequenceNumber uint16) {
	if !c.rolloverHasProcessed {
		c.rolloverHasProcessed = true
	} else if sequenceNumber == 0 { // We exactly hit the rollover count

		// Only update rolloverCounter if lastSequenceNumber is greater then maxROCDisorder
		// otherwise we already incremented for disorder
		if c.lastSequenceNumber > maxROCDisorder {
			c.rolloverCounter++
		}
	} else if c.lastSequenceNumber < maxROCDisorder && sequenceNumber > (maxSequenceNumber-maxROCDisorder) {
		// Our last sequence number incremented because we crossed 0, but then our current number was within maxROCDisorder of the max
		// So we fell behind, drop to account for jitter
		c.rolloverCounter--
	} else if sequenceNumber < maxROCDisorder && c.lastSequenceNumber > (maxSequenceNumber-maxROCDisorder) {
		// our current is within a maxROCDisorder of 0
		// and our last sequence number was a high sequence number, increment to account for jitter
		c.rolloverCounter++
	}
	c.lastSequenceNumber = sequenceNumber
}

// DecryptPacket decrypts a RTP packet with an encrypted payload
func (c *Context) DecryptPacket(packet *rtp.Packet) bool {
	c.updateRolloverCount(packet.SequenceNumber)

	stream := cipher.NewCTR(c.block, c.generateCounter(packet.SequenceNumber))
	stream.XORKeyStream(packet.Payload, packet.Payload)

	// TODO remove tags, need to assert value
	packet.Payload = packet.Payload[:len(packet.Payload)-10]

	// Replace payload with decrypted
	packet.Raw = packet.Raw[0:packet.PayloadOffset]
	packet.Raw = append(packet.Raw, packet.Payload...)

	return true
}

func (c *Context) addAuthTag(packet *rtp.Packet) error {
	// https://tools.ietf.org/html/rfc3711#section-4.2
	// In the case of SRTP, M SHALL consist of the Authenticated
	// Portion of the packet (as specified in Figure 1) concatenated with
	// the ROC, M = Authenticated Portion || ROC;
	//
	// The pre-defined authentication transform for SRTP is HMAC-SHA1
	// [RFC2104].  With HMAC-SHA1, the SRTP_PREFIX_LENGTH (Figure 3) SHALL
	// be 0.  For SRTP (respectively SRTCP), the HMAC SHALL be applied to
	// the session authentication key and M as specified above, i.e.,
	// HMAC(k_a, M).  The HMAC output SHALL then be truncated to the n_tag
	// left-most bits.
	// - Authenticated portion of the packet is everything BEFORE MKI
	// - k_a is the session message authentication key
	// - n_tag is the bit-length of the output authentication tag

	mac := hmac.New(sha1.New, c.sessionAuthTag)
	fullPkt, err := packet.Marshal()
	if err != nil {
		return err
	}

	fullPkt = append(fullPkt, make([]byte, 4)...)
	binary.BigEndian.PutUint32(fullPkt[len(fullPkt)-4:], c.rolloverCounter)

	if _, err := mac.Write(fullPkt); err != nil {
		return err
	}

	packet.Payload = append(packet.Payload, mac.Sum(nil)[0:10]...)
	return nil
}

// EncryptPacket Encrypts a SRTP packet in place
func (c *Context) EncryptPacket(packet *rtp.Packet) bool {
	c.updateRolloverCount(packet.SequenceNumber)

	stream := cipher.NewCTR(c.block, c.generateCounter(packet.SequenceNumber))
	stream.XORKeyStream(packet.Payload, packet.Payload)

	if err := c.addAuthTag(packet); err != nil {
		return false
	}

	return true
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"time"
)

const defaultTimeout = time.Second * 30

type Step interface {
	Execute(Runtime, TaskReporter) error
	Description() string
}

type StopStep struct {
	services []string
}

func NewStopStep(config StepConfig) Step {
	return &StopStep{
		services: config.Stop,
	}
}

func (s *StopStep) Execute(runtime Runtime, reporter TaskReporter) error {
	return runtime.StopServices(s.services...)
}

func (s *StopStep) Description() string {
	return fmt.Sprintf("<Stop: %v>", s.services)
}

type StartStep struct {
	services []string
}

func NewStartStep(config StepConfig) Step {
	return &StartStep{
		services: config.Start,
	}
}

func (s *StartStep) Execute(runtime Runtime, reporter TaskReporter) error {
	return runtime.StartServices(s.services...)
}

func (s *StartStep) Description() string {
	return fmt.Sprintf("<Start: %v>", s.services)
}

type WaitStep struct {
	task TaskConfig
	env  TaskEnvironment
}

func NewWaitStep(task TaskConfig, env TaskEnvironment) Step {
	return &WaitStep{
		task: task,
		env:  env,
	}
}

func (s *WaitStep) Execute(runtime Runtime, reporter TaskReporter) error {
	timeout := time.After(defaultTimeout)
	for {
		if err := runtime.ExecuteTask(s.task, s.env, reporter); err == nil {
			return nil
		}
		select {
		case <-timeout:
			return fmt.Errorf("Task never completed successfully")
		default:
			continue
		}
	}
}

func (s *WaitStep) Description() string {
	return fmt.Sprintf("<Wait: %s>", s.task.Name)
}

type AssertStep struct {
	task TaskConfig
	env  TaskEnvironment
}

func NewAssertStep(task TaskConfig, env TaskEnvironment) Step {
	return &AssertStep{
		task: task,
		env:  env,
	}
}

func (s *AssertStep) Execute(runtime Runtime, reporter TaskReporter) error {
	return runtime.ExecuteTask(s.task, s.env, reporter)
}

func (s *AssertStep) Description() string {
	return fmt.Sprintf("<Assert: %s>", s.task.Name)
}

type FailStep struct {
	task TaskConfig
	env  TaskEnvironment
}

func NewFailStep(task TaskConfig, env TaskEnvironment) Step {
	return &FailStep{
		task: task,
		env:  env,
	}
}

func (s *FailStep) Execute(runtime Runtime, reporter TaskReporter) error {
	if err := runtime.ExecuteTask(s.task, s.env, reporter); err == nil {
		return fmt.Errorf("Expected task to fail!")
	}
	return nil
}

func (s *FailStep) Description() string {
	return fmt.Sprintf("<Fail: %s>", s.task.Name)
}
<commit_msg>Simplify loop<commit_after>package main

import (
	"fmt"
	"time"
)

const defaultTimeout = time.Second * 30

type Step interface {
	Execute(Runtime, TaskReporter) error
	Description() string
}

type StopStep struct {
	services []string
}

func NewStopStep(config StepConfig) Step {
	return &StopStep{
		services: config.Stop,
	}
}

func (s *StopStep) Execute(runtime Runtime, reporter TaskReporter) error {
	return runtime.StopServices(s.services...)
}

func (s *StopStep) Description() string {
	return fmt.Sprintf("<Stop: %v>", s.services)
}

type StartStep struct {
	services []string
}

func NewStartStep(config StepConfig) Step {
	return &StartStep{
		services: config.Start,
	}
}

func (s *StartStep) Execute(runtime Runtime, reporter TaskReporter) error {
	return runtime.StartServices(s.services...)
}

func (s *StartStep) Description() string {
	return fmt.Sprintf("<Start: %v>", s.services)
}

type WaitStep struct {
	task TaskConfig
	env  TaskEnvironment
}

func NewWaitStep(task TaskConfig, env TaskEnvironment) Step {
	return &WaitStep{
		task: task,
		env:  env,
	}
}

func (s *WaitStep) Execute(runtime Runtime, reporter TaskReporter) error {
	timeout := time.After(defaultTimeout)
	select {
	case <-timeout:
		return fmt.Errorf("Task never completed successfully")
	default:
		if err := runtime.ExecuteTask(s.task, s.env, reporter); err == nil {
			return nil
		}
	}
}

func (s *WaitStep) Description() string {
	return fmt.Sprintf("<Wait: %s>", s.task.Name)
}

type AssertStep struct {
	task TaskConfig
	env  TaskEnvironment
}

func NewAssertStep(task TaskConfig, env TaskEnvironment) Step {
	return &AssertStep{
		task: task,
		env:  env,
	}
}

func (s *AssertStep) Execute(runtime Runtime, reporter TaskReporter) error {
	return runtime.ExecuteTask(s.task, s.env, reporter)
}

func (s *AssertStep) Description() string {
	return fmt.Sprintf("<Assert: %s>", s.task.Name)
}

type FailStep struct {
	task TaskConfig
	env  TaskEnvironment
}

func NewFailStep(task TaskConfig, env TaskEnvironment) Step {
	return &FailStep{
		task: task,
		env:  env,
	}
}

func (s *FailStep) Execute(runtime Runtime, reporter TaskReporter) error {
	if err := runtime.ExecuteTask(s.task, s.env, reporter); err == nil {
		return fmt.Errorf("Expected task to fail!")
	}
	return nil
}

func (s *FailStep) Description() string {
	return fmt.Sprintf("<Fail: %s>", s.task.Name)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"

	"github.com/Felamande/filesync/log"
	"github.com/go-martini/martini"

	"os/signal"

	"github.com/Felamande/filesync/syncer"
)

func main() {
	s := syncer.New()
	m := martini.Classic()
	l := log.NewFileLogger("./.log/http.log")

	SigChan := make(chan os.Signal, 2)
	signal.Notify(SigChan, os.Interrupt, os.Kill)
	go handleSignal(SigChan, s)

	m.Map(s)
	m.Map(l)
	m.Post("/new", NewPair)
	m.Get("/new", HelloNewPair)
	go s.Run()
	http.ListenAndServe(":20000", m)

}

func NewPair(logger *log.FileLogger, s *syncer.Syncer, w http.ResponseWriter, r *http.Request) string {
	r.ParseForm()
	lName := r.FormValue("left")
	rName := r.FormValue("right")

	err := s.NewPair(syncer.SyncConfig{false, false, false}, lName, rName)

	if err != nil {
		return err.Error()
	}

	return "Success"

}

func HelloNewPair() string {
	return "<!DOCTYPE html>Hello"
}

func handleSignal(c chan os.Signal, s *syncer.Syncer) {
	for {
		select {
		case sig := <-c:
			fmt.Println(sig.String())
			c := syncer.SavedConfig{}
			for _, pair := range s.SyncPairs {
				fmt.Println(pair.Left.Uri())
				c.Pairs = append(c.Pairs, syncer.SyncPairConfig{
					Left:   pair.Left.Uri(),
					Right:  pair.Right.Uri(),
					Config: pair.Config,
				})
				b, _ := json.Marshal(c.Pairs)
				ioutil.WriteFile("config.json", b, 0777)
				os.Exit(0)
			}
		}
	}
}
<commit_msg>Implement system service.<commit_after>package main

import (
	"flag"
	"fmt"
	"net/http"
	"os"
	"strings"

	svc "github.com/kardianos/service"

	"github.com/Felamande/filesync/syncer"
)

var run *bool = flag.Bool("run", false, "Run in the shell. -svcctl will be disabled.")
var controls *string = flag.String("svcctl", "install,start", "value:[start,stop,restart,install,uninstall], can be multiple values separated by commas")
var help *bool = flag.Bool("help", false, "Get help")

func main() {
	flag.Parse()

	if len(os.Args) == 1 {
		flag.Usage()
		return
	}

	if *help {
		flag.Usage()
		return
	}

	p := &Program{}
	s, err := svc.New(p, &svc.Config{
		Name:        "Filesync",
		DisplayName: "FileSync Service",
		Description: "Filesync is a simple tool to sync files between multiple directory pairs.",
		Arguments:   []string{"-run"},
	})

	if err != nil {
		fmt.Println(err.Error())
		return
	}

	if *run {
		err := s.Run()
		fmt.Println("-run with error: ", err.Error())
		return
	}

	Actions := strings.Split(*controls, ",")
	for _, action := range Actions {
		err := svc.Control(s, action)
		fmt.Println(err)
	}

}

func HelloNewPair() string {
	return "<!DOCTYPE html>Hello"
}

func NewPair(s *syncer.Syncer, r *http.Request) string {
	err := r.ParseForm()
	if err != nil {
		return err.Error()
	}

	lName := r.FormValue("left")
	rName := r.FormValue("right")
	err = s.NewPair(syncer.SyncConfig{true, true, true}, lName, rName)
	if err != nil {
		return err.Error()
	}
	return lName + " ==> " + rName

}
<|endoftext|>


<commit_before>package main

import (
	"./mysql";
	"fmt";
	"os";
)

func main() {
	//dbh, err := mysql.Connect("unix", "", "/var/run/mysqld/mysqld.sock", "test", "", ""); // Without password, user test, without database.
	dbh, err := mysql.Connect("tcp", "", "127.0.0.1:3306", "test", "test", "test");	// With password, user test, with database.
	if err != nil {
		fmt.Printf("%s\n", err);
		os.Exit(1);
	}
	dbh.Use("test");	//Select database

	res := new(mysql.MySQLResponse);
	res, err = dbh.Query("INSERT INTO test (sha1) VALUES(SHA1('foo'))");

	fmt.Printf("%s\n", res);
	if err != nil {
		fmt.Printf("%s\n", err);
		os.Exit(1);
	}
	res, err = dbh.Query(fmt.Sprintf("DELETE FROM test WHERE id=%d", res.InsertId));
	fmt.Printf("%s\n", res);
	res, err = dbh.Query("SELECT * FROM test");
	fmt.Printf("%s\n", res);
	for row := res.FetchRow(); row != nil; row = res.FetchRow() {
		for i := 0; i < len(row.Data); i++ {
			fmt.Printf("%s\t", res.ResultSet.Fields[i])
		}
		fmt.Printf("\n");
		for i := 0; i < len(row.Data); i++ {
			fmt.Printf("%s\t", row.Data[i])
		}
		fmt.Printf("\n");
		if err != nil {
			fmt.Printf("%s\n", err);
			os.Exit(1);
		}
	}
	res, err = dbh.Query("SHOW PROCESSLIST");
	fmt.Printf("%s %s\n", res, err);
	for row := res.FetchRow(); row != nil; row = res.FetchRow() {
		for i := 0; i < len(row.Data); i++ {
			fmt.Printf("%s\t", res.ResultSet.Fields[i])
		}
		fmt.Printf("\n");
		for i := 0; i < len(row.Data); i++ {
			fmt.Printf("%s\t", row.Data[i])
		}
		fmt.Printf("\n");
		if err != nil {
			fmt.Printf("%s\n", err);
			os.Exit(1);
		}
	}
	dbh.Quit();
}
<commit_msg>Testing FetchRowMap()<commit_after>package main

import (
	"./mysql";
	"fmt";
	"os";
)

func main() {
	//dbh, err := mysql.Connect("unix", "", "/var/run/mysqld/mysqld.sock", "test", "", ""); // Without password, user test, without database.
	dbh, err := mysql.Connect("tcp", "", "127.0.0.1:3306", "test", "test", "test");	// With password, user test, with database.
	if err != nil {
		fmt.Printf("%s\n", err);
		os.Exit(1);
	}
	dbh.Use("test");	//Select database

	res := new(mysql.MySQLResponse);
	res, err = dbh.Query("INSERT INTO test (sha1) VALUES(SHA1('foo'))");

	fmt.Printf("%s\n", res);
	if err != nil {
		fmt.Printf("%s\n", err);
		os.Exit(1);
	}
	res, err = dbh.Query(fmt.Sprintf("DELETE FROM test WHERE id=%d", res.InsertId));
	fmt.Printf("%s\n", res);
	res, err = dbh.Query("SELECT * FROM test");
	fmt.Printf("-----%s\n", res);
	for rowmap := res.FetchRowMap(); rowmap != nil; rowmap = res.FetchRowMap() {
		fmt.Printf("%#v\n", rowmap);
	}
	res, err = dbh.Query("SHOW PROCESSLIST");
	fmt.Printf("%s %s\n", res, err);
	for row := res.FetchRow(); row != nil; row = res.FetchRow() {
		for i := 0; i < len(row.Data); i++ {
			fmt.Printf("%s\t", res.ResultSet.Fields[i])
		}
		fmt.Printf("\n");
		for i := 0; i < len(row.Data); i++ {
			fmt.Printf("%s\t", row.Data[i])
		}
		fmt.Printf("\n");
		if err != nil {
			fmt.Printf("%s\n", err);
			os.Exit(1);
		}
	}
	dbh.Quit();
}
<|endoftext|>


<commit_before>package util

import (
	"github.com/realglobe-Inc/go-lib-rg/erro"
	"net"
	"net/http"
	"strconv"
)

func FreePort() (port int, err error) {
	lis, err := net.Listen("tcp", ":0")
	if err != nil {
		return 0, erro.Wrap(err)
	}
	lis.Close()

	_, portStr, err := net.SplitHostPort(lis.Addr().String())
	if err != nil {
		return 0, erro.Wrap(err)
	}

	port, err = strconv.Atoi(portStr)
	if err != nil {
		return 0, erro.Wrap(err)
	}

	return port, nil
}

// テスト用の HTTP サーバー。
type TestHttpServer struct {
	lis    net.Listener
	respCh chan *testHttpServerResponse
}

type testHttpServerResponse struct {
	status int
	header http.Header
	body   []byte

	reqCh chan<- *http.Request
}

func NewTestHttpServer(port int) (*TestHttpServer, error) {
	lis, err := net.Listen("tcp", ":"+strconv.Itoa(port))
	if err != nil {
		return nil, erro.Wrap(err)
	}

	respCh := make(chan *testHttpServerResponse, 100)

	mux := http.NewServeMux()
	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		resp := <-respCh
		resp.reqCh <- r
		for key, values := range resp.header {
			for _, value := range values {
				w.Header().Add(key, value)
			}
		}
		w.WriteHeader(resp.status)
		if resp.body != nil {
			w.Write(resp.body)
		}
	})

	go func() {
		http.Serve(lis, mux)
	}()

	return &TestHttpServer{lis, respCh}, nil
}

// 次に返させるレスポンスを与える。
func (server *TestHttpServer) AddResponse(status int, header http.Header, body []byte) <-chan *http.Request {
	reqCh := make(chan *http.Request, 1)
	server.respCh <- &testHttpServerResponse{status, header, body, reqCh}
	return reqCh
}

// 待ち受けソケットを閉じる。
func (server *TestHttpServer) Close() error {
	if err := server.lis.Close(); err != nil {
		return erro.Wrap(err)
	}
	return nil
}
<commit_msg>ボディ付きでも動くようにした<commit_after>package util

import (
	"bytes"
	"github.com/realglobe-Inc/go-lib-rg/erro"
	"io/ioutil"
	"net"
	"net/http"
	"strconv"
)

func FreePort() (port int, err error) {
	lis, err := net.Listen("tcp", ":0")
	if err != nil {
		return 0, erro.Wrap(err)
	}
	lis.Close()

	_, portStr, err := net.SplitHostPort(lis.Addr().String())
	if err != nil {
		return 0, erro.Wrap(err)
	}

	port, err = strconv.Atoi(portStr)
	if err != nil {
		return 0, erro.Wrap(err)
	}

	return port, nil
}

// テスト用の HTTP サーバー。
type TestHttpServer struct {
	lis    net.Listener
	respCh chan *testHttpServerResponse
}

type testHttpServerResponse struct {
	status int
	header http.Header
	body   []byte

	reqCh chan<- *http.Request
}

func NewTestHttpServer(port int) (*TestHttpServer, error) {
	lis, err := net.Listen("tcp", ":"+strconv.Itoa(port))
	if err != nil {
		return nil, erro.Wrap(err)
	}

	respCh := make(chan *testHttpServerResponse, 100)

	mux := http.NewServeMux()
	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		// ここを抜けるときに勝手に Close されるので、Close されても問題無いように置き換える。
		req := *r
		if buff, err := ioutil.ReadAll(r.Body); err != nil {
			err := erro.Wrap(err)
			log.Err(erro.Unwrap(err))
			log.Debug(err)
		} else {
			req.Body = ioutil.NopCloser(bytes.NewReader(buff))
		}

		resp := <-respCh
		resp.reqCh <- &req
		for key, values := range resp.header {
			for _, value := range values {
				w.Header().Add(key, value)
			}
		}
		w.WriteHeader(resp.status)
		if resp.body != nil {
			w.Write(resp.body)
		}
	})

	go func() {
		http.Serve(lis, mux)
	}()

	return &TestHttpServer{lis, respCh}, nil
}

// 次に返させるレスポンスを与える。
func (server *TestHttpServer) AddResponse(status int, header http.Header, body []byte) <-chan *http.Request {
	reqCh := make(chan *http.Request, 1)
	server.respCh <- &testHttpServerResponse{status, header, body, reqCh}
	return reqCh
}

// 待ち受けソケットを閉じる。
func (server *TestHttpServer) Close() error {
	if err := server.lis.Close(); err != nil {
		return erro.Wrap(err)
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"strings"

	"github.com/fatih/color"
	"github.com/gammons/todolist/todolist"
	"github.com/skratchdot/open-golang/open"
)

const (
	VERSION = "0.3.1"
)

func main() {
	if len(os.Args) <= 1 {
		usage()
		os.Exit(0)
	}
	input := strings.Join(os.Args[1:], " ")
	routeInput(os.Args[1], input)
}

func usage() {
	blue := color.New(color.FgBlue)
	cyan := color.New(color.FgCyan)
	yellow := color.New(color.FgYellow)

	blueBold := blue.Add(color.Bold)

	fmt.Printf("todo v%s, a simple, command line based, GTD-style todo manager\n", VERSION)

	blueBold.Println("\nAdding todos")
	fmt.Println("  the 'a' command adds todos.")
	fmt.Println("  You can also optionally specify a due date.")
	fmt.Println("  Specify a due date by putting 'due <date>' at the end, where <date> is in (tod|today|tom|tomorrow|mon|tue|wed|thu|fri|sat|sun)")
	fmt.Println("\n  Examples for adding a todo:")
	yellow.Println("\ttodo a Meeting with @bob about +importantPrject due today")
	yellow.Println("\ttodo a +work +verify did @john fix the build\\?")

	blueBold.Println("\nListing todos")
	fmt.Println("  When listing todos, you can filter and group the output.\n")

	fmt.Println("  todo l due (tod|today|tom|tomorrow|overdue|this week|next week|mon|tue|wed|thu|fri|sat|sun|none)")
	fmt.Println("  todo l overdue")

	cyan.Println("  Filtering by date:\n")
	yellow.Println("\ttodo l due tod")
	fmt.Println("\tlists all todos due today\n")
	yellow.Println("\ttodo l due tom")
	fmt.Println("\tlists all todos due tomorrow\n")
	yellow.Println("\ttodo l due mon")
	fmt.Println("\tlists all todos due monday\n")
	yellow.Println("\ttodo l overdue")
	fmt.Println("\tlists all todos where the due date is in the past\n")
	yellow.Println("\ttodo agenda")
	fmt.Println("\tlists all todos where the due date is today or in the past\n")

	cyan.Println("  Grouping:")
	fmt.Println("  You can group todos by context or project.")
	yellow.Println("\ttodo l by c")
	fmt.Println("\tlists all todos grouped by context\n")
	yellow.Println("\ttodo l by p")
	fmt.Println("\tlists all todos grouped by project\n")

	cyan.Println("  Grouping and filtering:")
	fmt.Println("  Of course, you can combine grouping and filtering to get a nice formatted list.\n")
	yellow.Println("\ttodo l due today by c")
	fmt.Println("\tlists all todos due today grouped by context\n")
	yellow.Println("\ttodo l +project due this week by c")
	fmt.Println("\tlists all todos due today for +project, grouped by context\n")
	yellow.Println("\ttodo l @frank due tom by p")
	fmt.Println("\tlists all todos due tomorrow concerining @frank for +project, grouped by project\n")

	blueBold.Println("\nCompleting and uncompleting ")
	fmt.Println("Complete and Uncomplete a todo by its Id:\n")
	yellow.Println("\ttodo c 33")
	fmt.Println("\tCompletes a todo with id 33\n")
	yellow.Println("\ttodo uc 33")
	fmt.Println("\tUncompletes a todo with id 33\n")

	blueBold.Println("\nArchiving")
	fmt.Println("You can archive todos once they are done, or if you might come back to them.")
	fmt.Println("By default, todo will only show unarchived todos.\n")
	yellow.Println("\ttodo ar 33")
	fmt.Println("\tArchives a todo with id 33\n")
	yellow.Println("\ttodo ac")
	fmt.Println("\tArchives all completed todos\n")
	yellow.Println("\ttodo l archived")
	fmt.Println("\tlist all archived todos\n")

	blueBold.Println("\nEditing due dates")
	yellow.Println("\ttodo e 33 due mon")
	fmt.Println("\tEdits the todo with 33 and sets the due date to this coming Monday\n")
	yellow.Println("\ttodo e 33 due none")
	fmt.Println("\tEdits the todo with 33 and removes the due date\n")

	blueBold.Println("\nDeleting")
	yellow.Println("\ttodo d 33")
	fmt.Println("\tDeletes a todo with id 33\n")
	fmt.Println("Todolist was lovingly crafted by Grant Ammons (https://twitter.com/gammons).")
	fmt.Println("For full documentation, please visit http://todolist.site")
}

func routeInput(command string, input string) {
	app := todolist.NewApp()
	switch command {
	case "l", "list", "agenda":
		app.ListTodos(input)
	case "a", "add":
		app.AddTodo(input)
	case "d", "del":
		app.DeleteTodo(input)
	case "c", "complete":
		app.CompleteTodo(input)
	case "uc", "uncomplete":
		app.UncompleteTodo(input)
	case "ar", "archive":
		app.ArchiveTodo(input)
	case "uar", "unarchive":
		app.UnarchiveTodo(input)
	case "ac":
		app.ArchiveCompleted()
	case "e", "edit":
		app.EditTodoDue(input)
	case "init":
		app.InitializeRepo()
	case "web":
		// Only open default browser if .todos.json exists
		pwd, err := os.Getwd()
    if err != nil {
      fmt.Println(err)
      os.Exit(1)
    }

		if _, err := os.Stat(pwd + "/.todos.json"); err == nil {
			web := todolist.NewWebapp()
			fmt.Println("Now serving todolist web.\nHead to http://localhost:7890 to see your todo list!")
			open.Start("http://localhost:7890")
			web.Run()
		} else {
			fmt.Println("No todo file found!")
			fmt.Println("Initialize a new todo repo by running 'todo init'")
			os.Exit(1)
		}
	}
}
<commit_msg>Only open browser if .todos.json exists<commit_after>package main

import (
	"fmt"
	"os"
	"strings"

	"github.com/fatih/color"
	"github.com/gammons/todolist/todolist"
	"github.com/skratchdot/open-golang/open"
)

const (
	VERSION = "0.3.1"
)

func main() {
	if len(os.Args) <= 1 {
		usage()
		os.Exit(0)
	}
	input := strings.Join(os.Args[1:], " ")
	routeInput(os.Args[1], input)
}

func usage() {
	blue := color.New(color.FgBlue)
	cyan := color.New(color.FgCyan)
	yellow := color.New(color.FgYellow)

	blueBold := blue.Add(color.Bold)

	fmt.Printf("todo v%s, a simple, command line based, GTD-style todo manager\n", VERSION)

	blueBold.Println("\nAdding todos")
	fmt.Println("  the 'a' command adds todos.")
	fmt.Println("  You can also optionally specify a due date.")
	fmt.Println("  Specify a due date by putting 'due <date>' at the end, where <date> is in (tod|today|tom|tomorrow|mon|tue|wed|thu|fri|sat|sun)")
	fmt.Println("\n  Examples for adding a todo:")
	yellow.Println("\ttodo a Meeting with @bob about +importantPrject due today")
	yellow.Println("\ttodo a +work +verify did @john fix the build\\?")

	blueBold.Println("\nListing todos")
	fmt.Println("  When listing todos, you can filter and group the output.\n")

	fmt.Println("  todo l due (tod|today|tom|tomorrow|overdue|this week|next week|mon|tue|wed|thu|fri|sat|sun|none)")
	fmt.Println("  todo l overdue")

	cyan.Println("  Filtering by date:\n")
	yellow.Println("\ttodo l due tod")
	fmt.Println("\tlists all todos due today\n")
	yellow.Println("\ttodo l due tom")
	fmt.Println("\tlists all todos due tomorrow\n")
	yellow.Println("\ttodo l due mon")
	fmt.Println("\tlists all todos due monday\n")
	yellow.Println("\ttodo l overdue")
	fmt.Println("\tlists all todos where the due date is in the past\n")
	yellow.Println("\ttodo agenda")
	fmt.Println("\tlists all todos where the due date is today or in the past\n")

	cyan.Println("  Grouping:")
	fmt.Println("  You can group todos by context or project.")
	yellow.Println("\ttodo l by c")
	fmt.Println("\tlists all todos grouped by context\n")
	yellow.Println("\ttodo l by p")
	fmt.Println("\tlists all todos grouped by project\n")

	cyan.Println("  Grouping and filtering:")
	fmt.Println("  Of course, you can combine grouping and filtering to get a nice formatted list.\n")
	yellow.Println("\ttodo l due today by c")
	fmt.Println("\tlists all todos due today grouped by context\n")
	yellow.Println("\ttodo l +project due this week by c")
	fmt.Println("\tlists all todos due today for +project, grouped by context\n")
	yellow.Println("\ttodo l @frank due tom by p")
	fmt.Println("\tlists all todos due tomorrow concerining @frank for +project, grouped by project\n")

	blueBold.Println("\nCompleting and uncompleting ")
	fmt.Println("Complete and Uncomplete a todo by its Id:\n")
	yellow.Println("\ttodo c 33")
	fmt.Println("\tCompletes a todo with id 33\n")
	yellow.Println("\ttodo uc 33")
	fmt.Println("\tUncompletes a todo with id 33\n")

	blueBold.Println("\nArchiving")
	fmt.Println("You can archive todos once they are done, or if you might come back to them.")
	fmt.Println("By default, todo will only show unarchived todos.\n")
	yellow.Println("\ttodo ar 33")
	fmt.Println("\tArchives a todo with id 33\n")
	yellow.Println("\ttodo ac")
	fmt.Println("\tArchives all completed todos\n")
	yellow.Println("\ttodo l archived")
	fmt.Println("\tlist all archived todos\n")

	blueBold.Println("\nEditing due dates")
	yellow.Println("\ttodo e 33 due mon")
	fmt.Println("\tEdits the todo with 33 and sets the due date to this coming Monday\n")
	yellow.Println("\ttodo e 33 due none")
	fmt.Println("\tEdits the todo with 33 and removes the due date\n")

	blueBold.Println("\nDeleting")
	yellow.Println("\ttodo d 33")
	fmt.Println("\tDeletes a todo with id 33\n")
	fmt.Println("Todolist was lovingly crafted by Grant Ammons (https://twitter.com/gammons).")
	fmt.Println("For full documentation, please visit http://todolist.site")
}

func routeInput(command string, input string) {
	app := todolist.NewApp()
	switch command {
	case "l", "list", "agenda":
		app.ListTodos(input)
	case "a", "add":
		app.AddTodo(input)
	case "d", "del":
		app.DeleteTodo(input)
	case "c", "complete":
		app.CompleteTodo(input)
	case "uc", "uncomplete":
		app.UncompleteTodo(input)
	case "ar", "archive":
		app.ArchiveTodo(input)
	case "uar", "unarchive":
		app.UnarchiveTodo(input)
	case "ac":
		app.ArchiveCompleted()
	case "e", "edit":
		app.EditTodoDue(input)
	case "init":
		app.InitializeRepo()
	case "web":
		// Only open default browser if .todos.json exists
		pwd, err := os.Getwd()
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}

		if _, err := os.Stat(pwd + "/.todos.json"); err == nil {
			web := todolist.NewWebapp()
			fmt.Println("Now serving todolist web.\nHead to http://localhost:7890 to see your todo list!")
			open.Start("http://localhost:7890")
			web.Run()
		} else {
			fmt.Println("No todo file found!")
			fmt.Println("Initialize a new todo repo by running 'todo init'")
			os.Exit(1)
		}
	}
}
<|endoftext|>


<commit_before>package CloudForest

import ()

//Tree represents a single decision tree.
type Tree struct {
	//Tree int
	Root *Node
}

//AddNode adds a node a the specified path with the specivied pred value and/or
//Splitter. Paths are specified in the same format as in rf-aces sf files, as a
//string of 'L' and 'R'. Nodes must be added from the root up as the case where
//the path specifies a node whose parent does not allready exist in the tree is
//not handled well.
func (t *Tree) AddNode(path string, pred Num, splitter *Splitter) {
	n := new(Node)
	n.Pred = pred
	n.Splitter = splitter
	if t.Root == nil {
		t.Root = n
	} else {
		loc := t.Root
		for i := 0; i < len(path); i++ {
			switch path[i : i+1] {
			case "L":
				if loc.Left == nil {
					loc.Left = n
				}
				loc = loc.Left
			case "R":
				if loc.Right == nil {
					loc.Right = n
				}
				loc = loc.Right
			}

		}
	}

}

//Returns the arrays of all spliters of a tree.
func (t *Tree) GetSplits(fm *FeatureMatrix, fbycase *SparseCounter, relativeSplitCount *SparseCounter) []Splitter {
	splitters := make([]Splitter, 0)
	ncases := len(fm.Data[0].Data) // grab the number of samples for the first feature
	cases := make([]int, ncases)   //make an array that large
	for i, _ := range cases {
		cases[i] = i
	}

	t.Root.Recurse(func(n *Node, cases []int) {
		//if we're on a splitting node
		if fbycase != nil && n.Splitter != nil {
			//add this splitter to the list
			splitters = append(splitters, Splitter{n.Splitter.Feature, n.Splitter.Numerical, n.Splitter.Value, n.Splitter.Left, n.Splitter.Right})
			f_id := n.Splitter.Feature               //get the feature at this splitter
			f := fm.Data[fm.Map[n.Splitter.Feature]] //get the feature at this splitter
			for _, c := range cases {                //for each case

				if f.Missing[c] == false { //if there isa value for this case
					fbycase.Add(c, fm.Map[f_id], 1) //count the number of times each case is present for a split by a feature
					fvalue := f.Back[f.Data[c]]     //what is the feature value for this case?

					switch {
					case n.Splitter.Left[fvalue]: //if the value was split to the left
						relativeSplitCount.Add(c, fm.Map[f_id], -1) //subtract one
					case n.Splitter.Right[fvalue]:
						relativeSplitCount.Add(c, fm.Map[f_id], 1) //add one
					}
				}
			}
		}
	}, fm, cases)
	return splitters //return the array of all splitters from the tree

}

//GetLeaves is called by the leaf count utility to
//gather statistics about the nodes of a tree including the sets of cases at
//"leaf" nodes that aren't split further and the number of times each feature
//is used to split away each case.
func (t *Tree) GetLeaves(fm *FeatureMatrix, fbycase *SparseCounter) []Leaf {
	leaves := make([]Leaf, 0)
	ncases := len(fm.Data[0].Data)
	cases := make([]int, 0, ncases)
	for i := 0; i < ncases; i++ {
		cases = append(cases, i)
	}

	t.Root.Recurse(func(n *Node, cases []int) {
		if n.Left == nil && n.Right == nil { // I'm in a leaf node
			leaves = append(leaves, Leaf{cases, n.Pred})
		}
		if fbycase != nil && n.Splitter != nil { //I'm not in a leaf node?
			for _, c := range cases {
				fbycase.Add(c, fm.Map[n.Splitter.Feature], 1)
			}
		}

	}, fm, cases)
	return leaves

}

//Tree.Vote casts a vote for the predicted value of each case in fm *FeatureMatrix.
//into bb *BallotBox. Since BallotBox is not thread safe trees should not vote
//into the same BallotBox in parralel.
func (t *Tree) Vote(fm *FeatureMatrix, bb *BallotBox) {
	ncases := len(fm.Data[0].Data)
	cases := make([]int, 0, ncases)
	for i := 0; i < ncases; i++ {
		cases = append(cases, i)
	}

	t.Root.Recurse(func(n *Node, cases []int) {
		if n.Left == nil && n.Right == nil {
			// I'm in a leaf node
			for i := 0; i < len(cases); i++ {
				bb.Vote(cases[i], n.Pred)
			}
		}
	}, fm, cases)
}

type Leaf struct {
	Cases []int
	Pred  Num
}
<commit_msg>Tree only votes numerically<commit_after>package CloudForest

import ()

//Tree represents a single decision tree.
type Tree struct {
	//Tree int
	Root *Node
}

//AddNode adds a node a the specified path with the specivied pred value and/or
//Splitter. Paths are specified in the same format as in rf-aces sf files, as a
//string of 'L' and 'R'. Nodes must be added from the root up as the case where
//the path specifies a node whose parent does not allready exist in the tree is
//not handled well.
func (t *Tree) AddNode(path string, pred Num, splitter *Splitter) {
	n := new(Node)
	n.Pred = pred
	n.Splitter = splitter
	if t.Root == nil {
		t.Root = n
	} else {
		loc := t.Root
		for i := 0; i < len(path); i++ {
			switch path[i : i+1] {
			case "L":
				if loc.Left == nil {
					loc.Left = n
				}
				loc = loc.Left
			case "R":
				if loc.Right == nil {
					loc.Right = n
				}
				loc = loc.Right
			}

		}
	}

}

//Returns the arrays of all spliters of a tree.
func (t *Tree) GetSplits(fm *FeatureMatrix, fbycase *SparseCounter, relativeSplitCount *SparseCounter) []Splitter {
	splitters := make([]Splitter, 0)
	ncases := len(fm.Data[0].Data) // grab the number of samples for the first feature
	cases := make([]int, ncases)   //make an array that large
	for i, _ := range cases {
		cases[i] = i
	}

	t.Root.Recurse(func(n *Node, cases []int) {
		//if we're on a splitting node
		if fbycase != nil && n.Splitter != nil {
			//add this splitter to the list
			splitters = append(splitters, Splitter{n.Splitter.Feature, n.Splitter.Numerical, n.Splitter.Value, n.Splitter.Left, n.Splitter.Right})
			f_id := n.Splitter.Feature               //get the feature at this splitter
			f := fm.Data[fm.Map[n.Splitter.Feature]] //get the feature at this splitter
			for _, c := range cases {                //for each case

				if f.Missing[c] == false { //if there isa value for this case
					fbycase.Add(c, fm.Map[f_id], 1) //count the number of times each case is present for a split by a feature
					fvalue := f.Back[f.Data[c]]     //what is the feature value for this case?

					switch {
					case n.Splitter.Left[fvalue]: //if the value was split to the left
						relativeSplitCount.Add(c, fm.Map[f_id], -1) //subtract one
					case n.Splitter.Right[fvalue]:
						relativeSplitCount.Add(c, fm.Map[f_id], 1) //add one
					}
				}
			}
		}
	}, fm, cases)
	return splitters //return the array of all splitters from the tree

}

//GetLeaves is called by the leaf count utility to
//gather statistics about the nodes of a tree including the sets of cases at
//"leaf" nodes that aren't split further and the number of times each feature
//is used to split away each case.
func (t *Tree) GetLeaves(fm *FeatureMatrix, fbycase *SparseCounter) []Leaf {
	leaves := make([]Leaf, 0)
	ncases := len(fm.Data[0].Data)
	cases := make([]int, 0, ncases)
	for i := 0; i < ncases; i++ {
		cases = append(cases, i)
	}

	t.Root.Recurse(func(n *Node, cases []int) {
		if n.Left == nil && n.Right == nil { // I'm in a leaf node
			leaves = append(leaves, Leaf{cases, n.Pred})
		}
		if fbycase != nil && n.Splitter != nil { //I'm not in a leaf node?
			for _, c := range cases {
				fbycase.Add(c, fm.Map[n.Splitter.Feature], 1)
			}
		}

	}, fm, cases)
	return leaves

}

//Leaf is a struct for storing the index of the cases at a terminal "Leaf" node
//along with the Numeric predicted value.
type Leaf struct {
	Cases []int
	Pred  Num
}

//Tree.Vote casts a vote for the predicted value of each case in fm *FeatureMatrix.
//into bb *BallotBox. Since BallotBox is not thread safe trees should not vote
//into the same BallotBox in parralel.
func (t *Tree) Vote(fm *FeatureMatrix, bb *BallotBox) {
	ncases := len(fm.Data[0].Data)
	cases := make([]int, 0, ncases)
	for i := 0; i < ncases; i++ {
		cases = append(cases, i)
	}

	t.Root.Recurse(func(n *Node, cases []int) {
		if n.Left == nil && n.Right == nil {
			// I'm in a leaf node
			for i := 0; i < len(cases); i++ {
				bb.VoteNum(cases[i], n.Pred)
			}
		}
	}, fm, cases)
}
<|endoftext|>


<commit_before>// ***************************************************************************
//
//  Copyright 2017 David (Dizzy) Smith, dizzyd@dizzyd.com
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.
// ***************************************************************************

package main

import (
	"archive/zip"
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"time"
	"unicode/utf8"

	"golang.org/x/net/http2"

	"github.com/Jeffail/gabs"
	"github.com/viki-org/dnscache"
)

const connTimeout = time.Duration(5) * time.Second

var resolver = dnscache.New(time.Minute * 15)
var getterClient = NewHttpClient(true)
var redirectClient = NewHttpClient(false)

func NewHttpClient(followRedirects bool) http.Client {
	t := http.Transport{
		MaxIdleConnsPerHost:   10,
		ResponseHeaderTimeout: time.Duration(10 * time.Second),
		ExpectContinueTimeout: time.Duration(10 * time.Second),
		Dial: func(network string, address string) (net.Conn, error) {
			separator := strings.LastIndex(address, ":")
			ip, _ := resolver.FetchOne(address[:separator])
			ipStr := ip.String()
			if ip.To4() == nil {
				// IPv6 address; need to wrap it in brackets
				ipStr = fmt.Sprintf("[%s]", ipStr)
			}
			conn, err := net.DialTimeout("tcp", ipStr+address[separator:], connTimeout)
			if err != nil {
				return nil, err
			}
			return conn, nil
		},
	}
	err := http2.ConfigureTransport(&t)
	if err != nil {
		fmt.Printf("Err configuring http2: %+v\n", err)
	}

	if !followRedirects {
		return http.Client{Transport: &t, CheckRedirect: func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse }}
	}
	return http.Client{Transport: &t}

}

func HttpGet(url string) (*http.Response, error) {
	req, _ := http.NewRequest("GET", url, nil)
	req.Header.Add("User-Agent", "Mozilla/5.0 AppleWebKit/589.6 (KHTML, like Gecko) Brave Chrome/79.1.3945.88 Safari/577.36")
	return getterClient.Do(req)
}

func downloadHttpFile(url string, targetFile string) error {
	resp, err := HttpGet(url)
	if err != nil {
		return fmt.Errorf("failed to retrieve %s: %+v", url, err)
	}
	defer resp.Body.Close()

	// Make sure all directories exist for the given filename
	err = os.MkdirAll(path.Dir(targetFile), 0700)
	if err != nil {
		return fmt.Errorf("failed to create directories for %s: %+v", targetFile, err)
	}

	// Copy the stream into the filename
	return writeStream(targetFile, resp.Body)
}

func downloadHttpFileToDir(url string, targetDir string, skipIfExists bool) (string, error) {
	// Start the download
	resp, err := HttpGet(url)
	if err != nil {
		return "", fmt.Errorf("failed to download %s: %+v", url, err)
	}
	defer resp.Body.Close()

	// If we didn't get back a 200, bail
	if resp.StatusCode != 200 {
		return "", fmt.Errorf("failed to download %s status %d", url, resp.StatusCode)
	}

	// Extract the filename from the actual request (after following all redirects)
	filename := filepath.Base(resp.Request.URL.Path)

	// Check for Content-Disposition header
	attachmentID := resp.Header.Get("Content-Disposition")
	if strings.HasPrefix(attachmentID, "attachment; filename=") {
		filename = strings.TrimPrefix(attachmentID, "attachment; filename=")
	}

	// Cleanup the filename
	filename = strings.Replace(filename, " r", "-", -1)
	filename = strings.Replace(filename, " ", "-", -1)
	filename = strings.Replace(filename, "+", "-", -1)
	filename = strings.Replace(filename, "(", "-", -1)
	filename = strings.Replace(filename, ")", "", -1)
	filename = strings.Replace(filename, "[", "-", -1)
	filename = strings.Replace(filename, "]", "", -1)
	filename = strings.Replace(filename, "'", "", -1)
	filename = filepath.Join(targetDir, filename)

	if skipIfExists && fileExists(filename) {
		return filepath.Base(filename), nil
	}

	// Save the stream of the response to the file
	fmt.Printf("Downloading %s\n", filepath.Base(filename))

	err = writeStream(filename, resp.Body)
	if err != nil {
		return "", fmt.Errorf("failed to write %s: %+v", filename, err)
	}

	return filepath.Base(filename), nil
}

func findJSONFile(z *zip.ReadCloser, name string) (*gabs.Container, error) {
	for _, f := range z.File {
		if f.Name == name {
			freader, err := f.Open()
			if err != nil {
				return nil, err
			}

			json, err := gabs.ParseJSONBuffer(freader)
			if err != nil {
				return nil, fmt.Errorf("failed to parse JSON %s: %+v", name, err)
			}
			return json, nil
		}
	}

	return nil, fmt.Errorf("failed to find %s", name)
}

func writeStream(filename string, data io.Reader) error {
	// Construct a filename to hold the stream while writing; once the download is complete, we'll move it into place
	// and delete the temporary file. This ensures that partial/failed streams are properly detected.
	tempFilename := filename + ".part"

	// Create the temporary file
	f, err := os.Create(tempFilename)
	if err != nil {
		return fmt.Errorf("failed to create %s: %v", filename, err)
	}
	defer f.Close()

	// Stream the data into the temp file
	writer := bufio.NewWriter(f)
	_, err = io.Copy(writer, data)
	if err != nil {
		return fmt.Errorf("failed to write %s: %v", filename, err)
	}
	writer.Flush()
	f.Close()

	// Ok, write completed successfully, move the file
	err = os.Rename(tempFilename, filename)
	if err != nil {
		return fmt.Errorf("failed to rename %s: %+v", tempFilename, err)
	}

	return nil
}

func fileExists(filename string) bool {
	_, err := os.Stat(filename)
	return err == nil || os.IsExist(err)
}

func dirExists(dirname string) bool {
	stat, err := os.Stat(dirname)
	return err == nil && stat.IsDir()
}

func readStringFromUrl(url string) (string, error) {
	res, err := HttpGet(url)
	if err != nil {
		return "", fmt.Errorf("Failed to read string from %s: %+v", url, err)
	}
	defer res.Body.Close()

	if res.StatusCode != 200 {
		return "", fmt.Errorf("Failed to read string from %s: HTTP %d", url, res.StatusCode)
	}

	// Dump the body into a string
	buf := new(bytes.Buffer)
	buf.ReadFrom(res.Body)
	return strings.TrimSpace(buf.String()), nil
}

func writeJSON(json *gabs.Container, filename string) error {
	jsonStr := json.StringIndent("", " ")
	return ioutil.WriteFile(filename, []byte(jsonStr), 0644)
}

func readStringFile(filename string) (string, error) {
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return "", err
	}

	return string(data), nil
}

func writeStringFile(filename, data string) error {
	// Ensure all the necessary directories exist
	err := os.MkdirAll(path.Dir(filename), 0700)
	if err != nil {
		return err
	}

	return ioutil.WriteFile(filename, []byte(data), 0644)
}

func parseVersion(version string) (int, int, int, error) {
	parts := strings.SplitN(version, ".", 3)
	// Walk over all the parts and convert to ints
	intParts := make([]int, len(parts))
	for i := 0; i < len(parts); i++ {
		value, err := strconv.Atoi(parts[i])
		if err != nil {
			intParts[i] = -1
		} else {
			intParts[i] = value
		}
	}

	if len(intParts) > 2 {
		return intParts[0], intParts[1], intParts[2], nil
	} else if len(intParts) > 1 {
		return intParts[0], intParts[1], 0, nil
	} else {
		return -1, -1, -1, fmt.Errorf("invalid version %s", version)
	}
}

func stripBadUTF8(s string) string {
	// Noop if we've already got a valid string
	if utf8.ValidString(s) {
		return s
	}

	// Walk the string, checking each rune
	v := make([]rune, 0, len(s))
	for i, r := range s {
		if r == utf8.RuneError {
			_, size := utf8.DecodeRuneInString(s[i:])
			if size == 1 {
				continue
			}
		}
		v = append(v, r)
	}
	return string(v)
}

func getJSONFromURL(url string) (*gabs.Container, error) {
	res, e := HttpGet(url)
	if e != nil {
		return nil, fmt.Errorf("Failed to complete HTTP request: %s %+v", url, e)
	}
	defer res.Body.Close()

	if res.StatusCode != 200 {
		return nil, fmt.Errorf("Failed to retrieve %s: %d", url, res.StatusCode)
	}

	// Parse the data using gabs
	return gabs.ParseJSONBuffer(res.Body)
}

func intValue(c *gabs.Container, path string) (int, error) {
	data := c.Path(path).Data()
	switch v := data.(type) {
	case int:
		return v, nil
	case float64:
		return int(v), nil
	default:
		return 0, fmt.Errorf("Invalid type for %s: %+v", path, data)
	}
}

func hasAnyPrefix(url string, prefixes ...string) bool {
	for _, p := range prefixes {
		if strings.HasPrefix(url, p) {
			return true
		}
	}

	return false
}

func getJavaMainClass(jarfile string) (string, error) {
	zr, err := zip.OpenReader(jarfile)
	if err != nil {
		return "", err
	}
	defer zr.Close()

	for _, f := range zr.File {
		if f.Name == "META-INF/MANIFEST.MF" {
			manifest, err := f.Open()
			if err != nil {
				return "", err
			}

			scanner := bufio.NewScanner(manifest)
			for scanner.Scan() {
				line := scanner.Text()
				if strings.HasPrefix(line, "Main-Class:") {
					return strings.TrimSpace(strings.TrimPrefix(line, "Main-Class:")), nil
				}
			}
		}
	}
	return "", fmt.Errorf("main class not found")
}

func urlJoin(urlBase string, paths ...string) (string, error) {
	// Parse the base URL
	u, err := url.Parse(urlBase)
	if err != nil {
		return "", fmt.Errorf("invalid url %s: %+v", urlBase, err)
	}

	// Append all the provided paths to the base URL path
	u.Path = path.Join(append([]string{u.Path}, paths...)...)
	return u.String(), nil
}
<commit_msg>Minor fix to error message<commit_after>// ***************************************************************************
//
//  Copyright 2017 David (Dizzy) Smith, dizzyd@dizzyd.com
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.
// ***************************************************************************

package main

import (
	"archive/zip"
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"time"
	"unicode/utf8"

	"golang.org/x/net/http2"

	"github.com/Jeffail/gabs"
	"github.com/viki-org/dnscache"
)

const connTimeout = time.Duration(5) * time.Second

var resolver = dnscache.New(time.Minute * 15)
var getterClient = NewHttpClient(true)
var redirectClient = NewHttpClient(false)

func NewHttpClient(followRedirects bool) http.Client {
	t := http.Transport{
		MaxIdleConnsPerHost:   10,
		ResponseHeaderTimeout: time.Duration(10 * time.Second),
		ExpectContinueTimeout: time.Duration(10 * time.Second),
		Dial: func(network string, address string) (net.Conn, error) {
			separator := strings.LastIndex(address, ":")
			ip, _ := resolver.FetchOne(address[:separator])
			ipStr := ip.String()
			if ip.To4() == nil {
				// IPv6 address; need to wrap it in brackets
				ipStr = fmt.Sprintf("[%s]", ipStr)
			}
			conn, err := net.DialTimeout("tcp", ipStr+address[separator:], connTimeout)
			if err != nil {
				return nil, err
			}
			return conn, nil
		},
	}
	err := http2.ConfigureTransport(&t)
	if err != nil {
		fmt.Printf("Error configuring http2: %+v\n", err)
	}

	if !followRedirects {
		return http.Client{Transport: &t, CheckRedirect: func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse }}
	}
	return http.Client{Transport: &t}

}

func HttpGet(url string) (*http.Response, error) {
	req, _ := http.NewRequest("GET", url, nil)
	req.Header.Add("User-Agent", "Mozilla/5.0 AppleWebKit/589.6 (KHTML, like Gecko) Brave Chrome/79.1.3945.88 Safari/577.36")
	return getterClient.Do(req)
}

func downloadHttpFile(url string, targetFile string) error {
	resp, err := HttpGet(url)
	if err != nil {
		return fmt.Errorf("failed to retrieve %s: %+v", url, err)
	}
	defer resp.Body.Close()

	// Make sure all directories exist for the given filename
	err = os.MkdirAll(path.Dir(targetFile), 0700)
	if err != nil {
		return fmt.Errorf("failed to create directories for %s: %+v", targetFile, err)
	}

	// Copy the stream into the filename
	return writeStream(targetFile, resp.Body)
}

func downloadHttpFileToDir(url string, targetDir string, skipIfExists bool) (string, error) {
	// Start the download
	resp, err := HttpGet(url)
	if err != nil {
		return "", fmt.Errorf("failed to download %s: %+v", url, err)
	}
	defer resp.Body.Close()

	// If we didn't get back a 200, bail
	if resp.StatusCode != 200 {
		return "", fmt.Errorf("failed to download %s status %d", url, resp.StatusCode)
	}

	// Extract the filename from the actual request (after following all redirects)
	filename := filepath.Base(resp.Request.URL.Path)

	// Check for Content-Disposition header
	attachmentID := resp.Header.Get("Content-Disposition")
	if strings.HasPrefix(attachmentID, "attachment; filename=") {
		filename = strings.TrimPrefix(attachmentID, "attachment; filename=")
	}

	// Cleanup the filename
	filename = strings.Replace(filename, " r", "-", -1)
	filename = strings.Replace(filename, " ", "-", -1)
	filename = strings.Replace(filename, "+", "-", -1)
	filename = strings.Replace(filename, "(", "-", -1)
	filename = strings.Replace(filename, ")", "", -1)
	filename = strings.Replace(filename, "[", "-", -1)
	filename = strings.Replace(filename, "]", "", -1)
	filename = strings.Replace(filename, "'", "", -1)
	filename = filepath.Join(targetDir, filename)

	if skipIfExists && fileExists(filename) {
		return filepath.Base(filename), nil
	}

	// Save the stream of the response to the file
	fmt.Printf("Downloading %s\n", filepath.Base(filename))

	err = writeStream(filename, resp.Body)
	if err != nil {
		return "", fmt.Errorf("failed to write %s: %+v", filename, err)
	}

	return filepath.Base(filename), nil
}

func findJSONFile(z *zip.ReadCloser, name string) (*gabs.Container, error) {
	for _, f := range z.File {
		if f.Name == name {
			freader, err := f.Open()
			if err != nil {
				return nil, err
			}

			json, err := gabs.ParseJSONBuffer(freader)
			if err != nil {
				return nil, fmt.Errorf("failed to parse JSON %s: %+v", name, err)
			}
			return json, nil
		}
	}

	return nil, fmt.Errorf("failed to find %s", name)
}

func writeStream(filename string, data io.Reader) error {
	// Construct a filename to hold the stream while writing; once the download is complete, we'll move it into place
	// and delete the temporary file. This ensures that partial/failed streams are properly detected.
	tempFilename := filename + ".part"

	// Create the temporary file
	f, err := os.Create(tempFilename)
	if err != nil {
		return fmt.Errorf("failed to create %s: %v", filename, err)
	}
	defer f.Close()

	// Stream the data into the temp file
	writer := bufio.NewWriter(f)
	_, err = io.Copy(writer, data)
	if err != nil {
		return fmt.Errorf("failed to write %s: %v", filename, err)
	}
	writer.Flush()
	f.Close()

	// Ok, write completed successfully, move the file
	err = os.Rename(tempFilename, filename)
	if err != nil {
		return fmt.Errorf("failed to rename %s: %+v", tempFilename, err)
	}

	return nil
}

func fileExists(filename string) bool {
	_, err := os.Stat(filename)
	return err == nil || os.IsExist(err)
}

func dirExists(dirname string) bool {
	stat, err := os.Stat(dirname)
	return err == nil && stat.IsDir()
}

func readStringFromUrl(url string) (string, error) {
	res, err := HttpGet(url)
	if err != nil {
		return "", fmt.Errorf("Failed to read string from %s: %+v", url, err)
	}
	defer res.Body.Close()

	if res.StatusCode != 200 {
		return "", fmt.Errorf("Failed to read string from %s: HTTP %d", url, res.StatusCode)
	}

	// Dump the body into a string
	buf := new(bytes.Buffer)
	buf.ReadFrom(res.Body)
	return strings.TrimSpace(buf.String()), nil
}

func writeJSON(json *gabs.Container, filename string) error {
	jsonStr := json.StringIndent("", " ")
	return ioutil.WriteFile(filename, []byte(jsonStr), 0644)
}

func readStringFile(filename string) (string, error) {
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return "", err
	}

	return string(data), nil
}

func writeStringFile(filename, data string) error {
	// Ensure all the necessary directories exist
	err := os.MkdirAll(path.Dir(filename), 0700)
	if err != nil {
		return err
	}

	return ioutil.WriteFile(filename, []byte(data), 0644)
}

func parseVersion(version string) (int, int, int, error) {
	parts := strings.SplitN(version, ".", 3)
	// Walk over all the parts and convert to ints
	intParts := make([]int, len(parts))
	for i := 0; i < len(parts); i++ {
		value, err := strconv.Atoi(parts[i])
		if err != nil {
			intParts[i] = -1
		} else {
			intParts[i] = value
		}
	}

	if len(intParts) > 2 {
		return intParts[0], intParts[1], intParts[2], nil
	} else if len(intParts) > 1 {
		return intParts[0], intParts[1], 0, nil
	} else {
		return -1, -1, -1, fmt.Errorf("invalid version %s", version)
	}
}

func stripBadUTF8(s string) string {
	// Noop if we've already got a valid string
	if utf8.ValidString(s) {
		return s
	}

	// Walk the string, checking each rune
	v := make([]rune, 0, len(s))
	for i, r := range s {
		if r == utf8.RuneError {
			_, size := utf8.DecodeRuneInString(s[i:])
			if size == 1 {
				continue
			}
		}
		v = append(v, r)
	}
	return string(v)
}

func getJSONFromURL(url string) (*gabs.Container, error) {
	res, e := HttpGet(url)
	if e != nil {
		return nil, fmt.Errorf("Failed to complete HTTP request: %s %+v", url, e)
	}
	defer res.Body.Close()

	if res.StatusCode != 200 {
		return nil, fmt.Errorf("Failed to retrieve %s: %d", url, res.StatusCode)
	}

	// Parse the data using gabs
	return gabs.ParseJSONBuffer(res.Body)
}

func intValue(c *gabs.Container, path string) (int, error) {
	data := c.Path(path).Data()
	switch v := data.(type) {
	case int:
		return v, nil
	case float64:
		return int(v), nil
	default:
		return 0, fmt.Errorf("Invalid type for %s: %+v", path, data)
	}
}

func hasAnyPrefix(url string, prefixes ...string) bool {
	for _, p := range prefixes {
		if strings.HasPrefix(url, p) {
			return true
		}
	}

	return false
}

func getJavaMainClass(jarfile string) (string, error) {
	zr, err := zip.OpenReader(jarfile)
	if err != nil {
		return "", err
	}
	defer zr.Close()

	for _, f := range zr.File {
		if f.Name == "META-INF/MANIFEST.MF" {
			manifest, err := f.Open()
			if err != nil {
				return "", err
			}

			scanner := bufio.NewScanner(manifest)
			for scanner.Scan() {
				line := scanner.Text()
				if strings.HasPrefix(line, "Main-Class:") {
					return strings.TrimSpace(strings.TrimPrefix(line, "Main-Class:")), nil
				}
			}
		}
	}
	return "", fmt.Errorf("main class not found")
}

func urlJoin(urlBase string, paths ...string) (string, error) {
	// Parse the base URL
	u, err := url.Parse(urlBase)
	if err != nil {
		return "", fmt.Errorf("invalid url %s: %+v", urlBase, err)
	}

	// Append all the provided paths to the base URL path
	u.Path = path.Join(append([]string{u.Path}, paths...)...)
	return u.String(), nil
}
<|endoftext|>


<commit_before>// Copyright 2016 Factom Foundation
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package main

import (
	"crypto/sha256"
	"encoding/hex"
	"fmt"
)

// exidCollector accumulates the external ids from the command line -e and -E
// flags
var exidCollector [][]byte

// extids will be a flag receiver for adding chains and entries
// In ASCII
type extidsAscii []string

func (e *extidsAscii) String() string {
	return fmt.Sprint(*e)
}

func (e *extidsAscii) Set(s string) error {
	*e = append(*e, s)
	exidCollector = append(exidCollector[:], []byte(s))
	return nil
}

// extids will be a flag receiver for adding chains and entries
// In HEX
type extidsHex []string

func (e *extidsHex) String() string {
	return fmt.Sprint(*e)
}

func (e *extidsHex) Set(s string) error {
	*e = append(*e, s)
	b, err := hex.DecodeString(s)
	if err != nil {
		return err
	}
	exidCollector = append(exidCollector[:], b)
	return nil
}

// nameCollector accumulates the components of a chain name from the command
// line -n and -N flags
var nameCollector [][]byte

// namesAscii will be a flag receiver for ASCII chain names.
type namesAscii []string

func (n *namesAscii) String() string {
	return fmt.Sprint(*n)
}

func (n *namesAscii) Set(s string) error {
	*n = append(*n, s)
	nameCollector = append(nameCollector[:], []byte(s))
	return nil
}

// namesHex will be a flag receiver for HEX encoded chain names.
type namesHex []string

func (n *namesHex) String() string {
	return fmt.Sprint(*n)
}

func (n *namesHex) Set(s string) error {
	*n = append(*n, s)
	b, err := hex.DecodeString(s)
	if err != nil {
		return err
	}
	nameCollector = append(nameCollector[:], b)
	return nil
}

func nametoid(name [][]byte) string {
	hs := sha256.New()
	for _, v := range name {
		h := sha256.Sum256(v)
		hs.Write(h[:])
	}
	return hex.EncodeToString(hs.Sum(nil))
}
<commit_msg>properties<commit_after>// Copyright 2016 Factom Foundation
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package main

import (
	"crypto/sha256"
	"encoding/hex"
	"fmt"
)

// exidCollector accumulates the external ids from the command line -e and -E
// flags
var exidCollector [][]byte

// extids will be a flag receiver for adding chains and entries
// In ASCII
type extidsAscii []string

func (e *extidsAscii) String() string {
	return fmt.Sprint(*e)
}

func (e *extidsAscii) Set(s string) error {
	*e = append(*e, s)
	exidCollector = append(exidCollector[:], []byte(s))
	return nil
}

// extids will be a flag receiver for adding chains and entries
// In HEX
type extidsHex []string

func (e *extidsHex) String() string {
	return fmt.Sprint(*e)
}

func (e *extidsHex) Set(s string) error {
	*e = append(*e, s)
	b, err := hex.DecodeString(s)
	if err != nil {
		return err
	}
	exidCollector = append(exidCollector[:], b)
	return nil
}

// nameCollector accumulates the components of a chain name from the command
// line -n and -N flags
var nameCollector [][]byte

// namesAscii will be a flag receiver for ASCII chain names.
type namesAscii []string

func (n *namesAscii) String() string {
	return fmt.Sprint(*n)
}

func (n *namesAscii) Set(s string) error {
	*n = append(*n, s)
	nameCollector = append(nameCollector[:], []byte(s))
	return nil
}

// namesHex will be a flag receiver for HEX encoded chain names.
type namesHex []string

func (n *namesHex) String() string {
	return fmt.Sprint(*n)
}

func (n *namesHex) Set(s string) error {
	*n = append(*n, s)
	b, err := hex.DecodeString(s)
	if err != nil {
		return err
	}
	nameCollector = append(nameCollector[:], b)
	return nil
}

// nametoid computes a chainid from the chain name components
func nametoid(name [][]byte) string {
	hs := sha256.New()
	for _, v := range name {
		h := sha256.Sum256(v)
		hs.Write(h[:])
	}
	return hex.EncodeToString(hs.Sum(nil))
}
<|endoftext|>


<commit_before>// Copyright 2016 Factom Foundation
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package factom

import (
	"bytes"
	"crypto/sha256"
	"crypto/sha512"
	"encoding/binary"
	"fmt"
	"time"
)

const (
	ZeroHash = "0000000000000000000000000000000000000000000000000000000000000000"
)

var (
	factomdServer = "localhost:8088"
	walletServer  = "localhost:8089"
)

// SetFactomdServer sets the gloabal target for the factomd server
func SetFactomdServer(s string) {
	factomdServer = s
}

// SetWalletServer sets the global target for the fctwallet server
func SetWalletServer(s string) {
	walletServer = s
}

// FactomdServer returns the global server string for debugging
func FactomdServer() string {
	return factomdServer
}

// FactomdServer returns the global wallet server string for debugging
func WalletServer() string {
	return walletServer
}

// milliTime returns a 6 byte slice representing the unix time in milliseconds
func milliTime() (r []byte) {
	buf := new(bytes.Buffer)
	t := time.Now().UnixNano()
	m := t / 1e6
	binary.Write(buf, binary.BigEndian, m)
	return buf.Bytes()[2:]
}

// shad Double Sha256 Hash; sha256(sha256(data))
func shad(data []byte) []byte {
	h1 := sha256.Sum256(data)
	h2 := sha256.Sum256(h1[:])
	return h2[:]
}

// sha52 Sha512+Sha256 Hash; sha256(sha512(data)+data)
func sha52(data []byte) []byte {
	h1 := sha512.Sum512(data)
	h2 := sha256.Sum256(append(h1[:], data...))
	return h2[:]
}

func entryCost(e *Entry) (int8, error) {
	p, err := e.MarshalBinary()
	if err != nil {
		return 0, err
	}

	// caulculate the length exluding the header size 35 for Milestone 1
	l := len(p) - 35

	if l > 10240 {
		return 10, fmt.Errorf("Entry cannot be larger than 10KB")
	}

	// n is the capacity of the entry payment in KB
	n := int8(l / 1024)

	if r := l % 1024; r > 0 {
		n += 1
	}

	if n < 1 {
		n = 1
	}
	return n, nil
}
<commit_msg>code reorg<commit_after>// Copyright 2016 Factom Foundation
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package factom

import (
	"bytes"
	"crypto/sha256"
	"crypto/sha512"
	"encoding/binary"
	"fmt"
	"time"
)

const (
	// ZeroHash is the string of all 00s
	ZeroHash = "0000000000000000000000000000000000000000000000000000000000000000"
)

var (
	factomdServer = "localhost:8088"
	walletServer  = "localhost:8089"
)

// SetFactomdServer sets the gloabal target for the factomd server
func SetFactomdServer(s string) {
	factomdServer = s
}

// SetWalletServer sets the global target for the fctwallet server
func SetWalletServer(s string) {
	walletServer = s
}

// FactomdServer returns the global server string for debugging
func FactomdServer() string {
	return factomdServer
}

// WalletServer returns the global wallet server string for debugging
func WalletServer() string {
	return walletServer
}

// entryCost calculates the cost of adding the Entry to factom
func entryCost(e *Entry) (int8, error) {
	p, err := e.MarshalBinary()
	if err != nil {
		return 0, err
	}

	// caulculate the length exluding the header size 35 for Milestone 1
	l := len(p) - 35

	if l > 10240 {
		return 10, fmt.Errorf("Entry cannot be larger than 10KB")
	}

	// n is the capacity of the entry payment in KB
	n := int8(l / 1024)

	if r := l % 1024; r > 0 {
		n++
	}

	if n < 1 {
		n = 1
	}
	return n, nil
}

// milliTime returns a 6 byte slice representing the unix time in milliseconds
func milliTime() (r []byte) {
	buf := new(bytes.Buffer)
	t := time.Now().UnixNano()
	m := t / 1e6
	binary.Write(buf, binary.BigEndian, m)
	return buf.Bytes()[2:]
}

// shad Double Sha256 Hash; sha256(sha256(data))
func shad(data []byte) []byte {
	h1 := sha256.Sum256(data)
	h2 := sha256.Sum256(h1[:])
	return h2[:]
}

// sha52 Sha512+Sha256 Hash; sha256(sha512(data)+data)
func sha52(data []byte) []byte {
	h1 := sha512.Sum512(data)
	h2 := sha256.Sum256(append(h1[:], data...))
	return h2[:]
}
<|endoftext|>


<commit_before>package sofa

import (
	"net/url"
	"strings"
)

// FutonURL attempts to correctly convert any CouchDB path into a URL to be used to
// access the same documents through the Futon web GUI.
func (con *Connection) FutonURL(path string) url.URL {
	patharr := strings.Split(strings.Trim(path, "/"), "/")

	furl := con.URL("/")
	furl.Path = urlConcat(furl.Path, "_utils/")

	if len(patharr) == 0 || patharr[0] == "" {
		return furl
	}

	isDatabaseURL := false
	if len(patharr) == 1 {
		isDatabaseURL = true
	} else {
		switch patharr[1] {
		case "_design", "_all_docs":
			isDatabaseURL = true
		}
	}

	furl.RawQuery = strings.TrimLeft(path, "/")

	if isDatabaseURL {
		furl.Path = urlConcat(furl.Path, "database.html")
		return furl
	}

	furl.Path = urlConcat(furl.Path, "document.html")
	return furl
}
<commit_msg>Add a utility function for converting booleans<commit_after>package sofa

import (
	"net/url"
	"strings"
)

// FutonURL attempts to correctly convert any CouchDB path into a URL to be used to
// access the same documents through the Futon web GUI.
func (con *Connection) FutonURL(path string) url.URL {
	patharr := strings.Split(strings.Trim(path, "/"), "/")

	furl := con.URL("/")
	furl.Path = urlConcat(furl.Path, "_utils/")

	if len(patharr) == 0 || patharr[0] == "" {
		return furl
	}

	isDatabaseURL := false
	if len(patharr) == 1 {
		isDatabaseURL = true
	} else {
		switch patharr[1] {
		case "_design", "_all_docs":
			isDatabaseURL = true
		}
	}

	furl.RawQuery = strings.TrimLeft(path, "/")

	if isDatabaseURL {
		furl.Path = urlConcat(furl.Path, "database.html")
		return furl
	}

	furl.Path = urlConcat(furl.Path, "document.html")
	return furl
}

func Boolean(b bool) BooleanParameter {
	if b {
		return True
	}

	return False
}
<|endoftext|>


<commit_before>package mirror

import (
	"reflect"
)

// VisitType
type VisitType func(t *reflect.StructField, typeIndex, depth int) error

type typeWalker struct {
	indices map[reflect.Type]int
	types   []reflect.Type
}

func (walker *typeWalker) index(t reflect.Type) (int, bool) {
	idx, known := walker.indices[t]
	if known {
		return idx, true
	}
	idx = len(walker.types)
	walker.indices[t] = idx
	walker.types = append(walker.types, t)
	return idx, false
}

func (walker *typeWalker) walk(t *reflect.StructField, visit VisitType) error {
	if t == nil || visit == nil {
		return nil
	}
	type stackNode struct {
		field      *reflect.StructField
		currentIdx int
		parentIdx  int
		depth      int
		known      bool
	}
	addNodes := func(stack []stackNode, nodes ...stackNode) []stackNode {
		return append(nodes, stack...)
	}
	idx, known := walker.index(t.Type)
	stack := []stackNode{{t, idx, idx, 0, known}}
	var node stackNode
	for len(stack) > 0 {
		node, stack = stack[0], stack[1:]
		err := visit(node.field, node.currentIdx, node.depth)
		if err != nil {
			return err
		}
		if node.known {
			// to prevent endless loops, don't follow known types
			continue
		}
		// follow container types
		parentIdx := node.currentIdx
		depth := node.depth + 1
		switch t := node.field.Type; t.Kind() {
		case reflect.Struct:
			for i, iLimit := 0, t.NumField(); i < iLimit; i++ {
				// add each field from struct to stack
				field := t.Field(i)
				typeIdx, known := walker.index(field.Type)
				stack = addNodes(stack, stackNode{&field, typeIdx, parentIdx, depth, known})
			}
		case reflect.Ptr, reflect.Array, reflect.Slice, reflect.Map, reflect.Chan:
			// add element to stack
			field := &reflect.StructField{Type: t.Elem()}
			typeIdx, known := walker.index(field.Type)
			stack = addNodes(stack, stackNode{field, typeIdx, parentIdx, depth, known})
		}
	}
	return nil
}

// Walk will call visit on t and each type in t.
// It will follow struct fields, pointers, arrays, slices, maps and chans
// and call visit on each element.
// For struct fields, Walk will only follow into fields at the first occurence of their type.
// Function handling for reflect.Interface and reflect.Method has to be done by visit.
func Walk(t reflect.Type, visit VisitType) error {
	walker := &typeWalker{indices: make(map[reflect.Type]int)}
	return walker.walk(&reflect.StructField{Type: t}, visit)
}
<commit_msg>at it...<commit_after>package mirror

import (
	"reflect"
)

// VisitType
type VisitType func(t *reflect.StructField, typeIndex, depth int) error

type typeWalker struct {
	indices map[reflect.Type]int
	types   []reflect.Type
}

func (walker *typeWalker) index(t reflect.Type) (int, bool) {
	idx, known := walker.indices[t]
	if known {
		return idx, true
	}
	idx = len(walker.types)
	walker.indices[t] = idx
	walker.types = append(walker.types, t)
	return idx, false
}

func (walker *typeWalker) walk(t *reflect.StructField, visit VisitType) error {
	if t == nil || visit == nil {
		return nil
	}
	type stackNode struct {
		field      *reflect.StructField
		currentIdx int
		parentIdx  int
		depth      int
		known      bool
	}
	idx, known := walker.index(t.Type)
	stack := []stackNode{{t, idx, idx, 0, known}}
	var node stackNode
	for len(stack) > 0 {
		node, stack = stack[0], stack[1:]
		err := visit(node.field, node.currentIdx, node.depth)
		if err != nil {
			return err
		}
		if node.known {
			// to prevent endless loops, don't follow known types
			continue
		}
		// follow container types
		parentIdx := node.currentIdx
		depth := node.depth + 1
		switch t := node.field.Type; t.Kind() {
		case reflect.Struct:
			fields := t.NumField()
			newNodes := make([]stackNode, 0, len(stack)+fields)
			for i := 0; i < fields; i++ {
				// add each field from struct to stack
				field := t.Field(i)
				typeIdx, known := walker.index(field.Type)
				newNodes = append(newNodes, stackNode{&field, typeIdx, parentIdx, depth, known})
			}
			newNodes = append(newNodes, stack...)
		case reflect.Ptr, reflect.Array, reflect.Slice, reflect.Map, reflect.Chan:
			// add element to stack
			field := &reflect.StructField{Type: t.Elem()}
			typeIdx, known := walker.index(field.Type)
			stack = append([]stackNodes{stackNode{&field, typeIdx, parentIdx, depth, known}}, stack...)
		}
	}
	return nil
}

// Walk will call visit on t and each type in t.
// It will follow struct fields, pointers, arrays, slices, maps and chans
// and call visit on each element.
// For struct fields, Walk will only follow into fields at the first occurence of their type.
// Function handling for reflect.Interface and reflect.Method has to be done by visit.
func Walk(t reflect.Type, visit VisitType) error {
	walker := &typeWalker{indices: make(map[reflect.Type]int)}
	return walker.walk(&reflect.StructField{Type: t}, visit)
}
<|endoftext|>


<commit_before>package someutils

import (
	"errors"
	"flag"
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
)

func init() {
	Register(Util{
		"wget",
		Wget})
}

type WgetOptions struct {
}

func Wget(call []string) error {

	//options := WgetOptions{}
	flagSet := flag.NewFlagSet("wget", flag.ContinueOnError)
/*
	options.IsPerl = flagSet.Bool("P", false, "Perl-style regex")
	options.IsExtended = flagSet.Bool("E", true, "Extended regex (default)")
	options.IsIgnoreCase = flagSet.Bool("i", false, "ignore case")
	options.IsPrintFilename = flagSet.Bool("H", true, "print the file name for each match")
	options.IsPrintLineNumber = flagSet.Bool("n", false, "print the line number for each match")
	options.IsInvertMatch = flagSet.Bool("v", false, "invert match")
*/
	helpFlag := flagSet.Bool("help", false, "Show this help")

	err := flagSet.Parse(splitSingleHyphenOpts(call[1:]))
	if err != nil {
		return err
	}
	if *helpFlag {
		println("`grep` [options] PATTERN [files...]")
		flagSet.PrintDefaults()
		return nil
	}
	args := flagSet.Args()
	if len(args) < 1 {
		flagSet.PrintDefaults()
		return errors.New("Not enough args")
	}
	if len(args) > 0 {
		links := args
		return wget(links)
	} else {
		if IsPipingStdin() {
			//check STDIN
			return wget([]string{})
		} else {
			//NOT piping.
			return errors.New("Not enough args")
		}
	}
}

func wget(links []string) error {
	for _, link := range links {
		err := wgetOne(link)
		if err != nil {
			return err
		}
	}
	return nil
}
func wgetOne(link string) error {
	resp, err := http.Get(link)
	defer resp.Body.Close()
	filename := filepath.Base(resp.Request.URL.Path)
	if filename == "" || filename == "/" || filename == "." {
		filename = "index.htm"
	}
	fmt.Printf("filename: %v\n", filename)
	out, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer out.Close()

	n, err := io.Copy(out, resp.Body)
	fmt.Printf("%v b written to %v\n",n, filename)
	if err != nil {
		return err
	}
	err = out.Close()
	return err
	//return errors.New("Not implemented")
}
<commit_msg>improving filename resolution<commit_after>package someutils

import (
	"errors"
	"flag"
	"fmt"
	"io"
	"mime"
	"net/http"
	"os"
	"path/filepath"
	"strings"
)

func init() {
	Register(Util{
		"wget",
		Wget})
}

type WgetOptions struct {
}

func Wget(call []string) error {

	//options := WgetOptions{}
	flagSet := flag.NewFlagSet("wget", flag.ContinueOnError)
/*
	options.IsPerl = flagSet.Bool("P", false, "Perl-style regex")
	options.IsExtended = flagSet.Bool("E", true, "Extended regex (default)")
	options.IsIgnoreCase = flagSet.Bool("i", false, "ignore case")
	options.IsPrintFilename = flagSet.Bool("H", true, "print the file name for each match")
	options.IsPrintLineNumber = flagSet.Bool("n", false, "print the line number for each match")
	options.IsInvertMatch = flagSet.Bool("v", false, "invert match")
*/
	helpFlag := flagSet.Bool("help", false, "Show this help")

	err := flagSet.Parse(splitSingleHyphenOpts(call[1:]))
	if err != nil {
		return err
	}
	if *helpFlag {
		println("`grep` [options] PATTERN [files...]")
		flagSet.PrintDefaults()
		return nil
	}
	args := flagSet.Args()
	if len(args) < 1 {
		flagSet.PrintDefaults()
		return errors.New("Not enough args")
	}
	if len(args) > 0 {
		links := args
		return wget(links)
	} else {
		if IsPipingStdin() {
			//check STDIN
			return wget([]string{})
		} else {
			//NOT piping.
			return errors.New("Not enough args")
		}
	}
}

func wget(links []string) error {
	for _, link := range links {
		err := wgetOne(link)
		if err != nil {
			return err
		}
	}
	return nil
}

func wgetOne(link string) error {
	if !strings.Contains(link, ":") {
		link = "http://" + link
	}
	resp, err := http.Get(link)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	filename := filepath.Base(resp.Request.URL.Path)
	//invalid filenames ...
	if filename == "" || filename == "/" ||filename == "\\" || filename == "." {
		filename = "index"
	}
	if !strings.Contains(filename, ".") {
		ct := resp.Header.Get("Content-Type")
		//println(ct)
		ext := "htm"
		mediatype, _, err := mime.ParseMediaType(ct)
		if err != nil {
			fmt.Printf("mime error: %v\n", err)
		} else {
			fmt.Printf("mime type: %v (from Content-Type %v)\n", mediatype, ct)
			slash := strings.Index(mediatype, "/")
			if slash != -1 {
				_, sub := mediatype[:slash], mediatype[slash+1:]
				if sub != "" {
					ext = sub
				}
			}
		}
		filename = filename + "." + ext
	}
	fmt.Printf("filename: %v\n", filename)
	out, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer out.Close()

	n, err := io.Copy(out, resp.Body)
	fmt.Printf("%v bytes written to %v\n",n, filename)
	if err != nil {
		return err
	}
	err = out.Close()
	return err
	//return errors.New("Not implemented")
}
<|endoftext|>