<commit_before>package main

import (
	"crypto/tls"
	log "github.com/Sirupsen/logrus"
	"github.com/aymerick/douceur/inliner"
	"github.com/drone/drone-go/template"
	"github.com/jaytaylor/html2text"
	"gopkg.in/gomail.v2"
)

type (
	Repo struct {
		FullName string
		Owner    string
		Name     string
		SCM      string
		Link     string
		Avatar   string
		Branch   string
		Private  bool
		Trusted  bool
	}

	Remote struct {
		URL string
	}

	Author struct {
		Name   string
		Email  string
		Avatar string
	}

	Commit struct {
		Sha     string
		Ref     string
		Branch  string
		Link    string
		Message string
		Author  Author
	}

	Build struct {
		Number   int
		Event    string
		Status   string
		Link     string
		Created  int64
		Started  int64
		Finished int64
	}

	PrevBuild struct {
		Status string
		Number int
	}

	PrevCommit struct {
		Sha string
	}

	Prev struct {
		Build  PrevBuild
		Commit PrevCommit
	}

	Job struct {
		Status   string
		ExitCode int
		Started  int64
		Finished int64
	}

	Yaml struct {
		Signed   bool
		Verified bool
	}

	Config struct {
		From           string
		Host           string
		Port           int
		Username       string
		Password       string
		SkipVerify     bool
		Recipients     []string
		RecipientsOnly bool
		Subject        string
		Body           string
		Attachment     string
	}

	Plugin struct {
		Repo        Repo
		Remote      Remote
		Commit      Commit
		Build       Build
		Prev        Prev
		Job         Job
		Yaml        Yaml
		Tag         string
		PullRequest int
		DeployTo    string
		Config      Config
	}
)

// Exec will send emails over SMTP
func (p Plugin) Exec() error {
	var dialer *gomail.Dialer

	if !p.Config.RecipientsOnly {
		exists := false
		for _, recipient := range p.Config.Recipients {
			if recipient == p.Commit.Author.Email {
				exists = true
			}
		}

		if !exists {
			p.Config.Recipients = append(p.Config.Recipients, p.Commit.Author.Email)
		}
	}

	if p.Config.Username == "" && p.Config.Password == "" {
		dialer = &gomail.Dialer{Host: p.Config.Host, Port: p.Config.Port}
	} else {
		dialer = gomail.NewDialer(p.Config.Host, p.Config.Port, p.Config.Username, p.Config.Password)
	}
	if p.Config.SkipVerify {
		dialer.TLSConfig = &tls.Config{InsecureSkipVerify: true}
	}

	closer, err := dialer.Dial()
	if err != nil {
		log.Errorf("Error while dialing SMTP server: %v", err)
		return err
	}

	type Context struct {
		Repo        Repo
		Remote      Remote
		Commit      Commit
		Build       Build
		Prev        Prev
		Job         Job
		Yaml        Yaml
		Tag         string
		PullRequest int
		DeployTo    string
	}
	ctx := Context{
		Repo:        p.Repo,
		Remote:      p.Remote,
		Commit:      p.Commit,
		Build:       p.Build,
		Prev:        p.Prev,
		Job:         p.Job,
		Yaml:        p.Yaml,
		Tag:         p.Tag,
		PullRequest: p.PullRequest,
		DeployTo:    p.DeployTo,
	}

	// Render body in HTML and plain text
	renderedBody, err := template.RenderTrim(p.Config.Body, ctx)
	if err != nil {
		log.Errorf("Could not render body template: %v", err)
		return err
	}
	html, err := inliner.Inline(renderedBody)
	if err != nil {
		log.Errorf("Could not inline rendered body: %v", err)
		return err
	}
	plainBody, err := html2text.FromString(html)
	if err != nil {
		log.Errorf("Could not convert html to text: %v", err)
		return err
	}

	// Render subject
	subject, err := template.RenderTrim(p.Config.Subject, ctx)
	if err != nil {
		log.Errorf("Could not render subject template: %v", err)
		return err
	}

	// Send emails
	message := gomail.NewMessage()
	for _, recipient := range p.Config.Recipients {
		message.SetHeader("From", p.Config.From)
		message.SetAddressHeader("To", recipient, "")
		message.SetHeader("Subject", subject)
		message.AddAlternative("text/plain", plainBody)
		message.AddAlternative("text/html", html)

		if p.Config.Attachment != "" {
			message.Attach(p.Config.Attachment)
		}

		if err := gomail.Send(closer, message); err != nil {
			log.Errorf("Could not send email to %q: %v", recipient, err)
			return err
		}
		message.Reset()
	}

	return nil
}
<commit_msg>fix: invalid address<commit_after>package main

import (
	"crypto/tls"
	log "github.com/Sirupsen/logrus"
	"github.com/aymerick/douceur/inliner"
	"github.com/drone/drone-go/template"
	"github.com/jaytaylor/html2text"
	"gopkg.in/gomail.v2"
)

type (
	Repo struct {
		FullName string
		Owner    string
		Name     string
		SCM      string
		Link     string
		Avatar   string
		Branch   string
		Private  bool
		Trusted  bool
	}

	Remote struct {
		URL string
	}

	Author struct {
		Name   string
		Email  string
		Avatar string
	}

	Commit struct {
		Sha     string
		Ref     string
		Branch  string
		Link    string
		Message string
		Author  Author
	}

	Build struct {
		Number   int
		Event    string
		Status   string
		Link     string
		Created  int64
		Started  int64
		Finished int64
	}

	PrevBuild struct {
		Status string
		Number int
	}

	PrevCommit struct {
		Sha string
	}

	Prev struct {
		Build  PrevBuild
		Commit PrevCommit
	}

	Job struct {
		Status   string
		ExitCode int
		Started  int64
		Finished int64
	}

	Yaml struct {
		Signed   bool
		Verified bool
	}

	Config struct {
		From           string
		Host           string
		Port           int
		Username       string
		Password       string
		SkipVerify     bool
		Recipients     []string
		RecipientsOnly bool
		Subject        string
		Body           string
		Attachment     string
	}

	Plugin struct {
		Repo        Repo
		Remote      Remote
		Commit      Commit
		Build       Build
		Prev        Prev
		Job         Job
		Yaml        Yaml
		Tag         string
		PullRequest int
		DeployTo    string
		Config      Config
	}
)

// Exec will send emails over SMTP
func (p Plugin) Exec() error {
	var dialer *gomail.Dialer

	if !p.Config.RecipientsOnly {
		exists := false
		for _, recipient := range p.Config.Recipients {
			if recipient == p.Commit.Author.Email {
				exists = true
			}
		}

		if !exists {
			p.Config.Recipients = append(p.Config.Recipients, p.Commit.Author.Email)
		}
	}

	if p.Config.Username == "" && p.Config.Password == "" {
		dialer = &gomail.Dialer{Host: p.Config.Host, Port: p.Config.Port}
	} else {
		dialer = gomail.NewDialer(p.Config.Host, p.Config.Port, p.Config.Username, p.Config.Password)
	}
	if p.Config.SkipVerify {
		dialer.TLSConfig = &tls.Config{InsecureSkipVerify: true}
	}

	closer, err := dialer.Dial()
	if err != nil {
		log.Errorf("Error while dialing SMTP server: %v", err)
		return err
	}

	type Context struct {
		Repo        Repo
		Remote      Remote
		Commit      Commit
		Build       Build
		Prev        Prev
		Job         Job
		Yaml        Yaml
		Tag         string
		PullRequest int
		DeployTo    string
	}
	ctx := Context{
		Repo:        p.Repo,
		Remote:      p.Remote,
		Commit:      p.Commit,
		Build:       p.Build,
		Prev:        p.Prev,
		Job:         p.Job,
		Yaml:        p.Yaml,
		Tag:         p.Tag,
		PullRequest: p.PullRequest,
		DeployTo:    p.DeployTo,
	}

	// Render body in HTML and plain text
	renderedBody, err := template.RenderTrim(p.Config.Body, ctx)
	if err != nil {
		log.Errorf("Could not render body template: %v", err)
		return err
	}
	html, err := inliner.Inline(renderedBody)
	if err != nil {
		log.Errorf("Could not inline rendered body: %v", err)
		return err
	}
	plainBody, err := html2text.FromString(html)
	if err != nil {
		log.Errorf("Could not convert html to text: %v", err)
		return err
	}

	// Render subject
	subject, err := template.RenderTrim(p.Config.Subject, ctx)
	if err != nil {
		log.Errorf("Could not render subject template: %v", err)
		return err
	}

	// Send emails
	message := gomail.NewMessage()
	for _, recipient := range p.Config.Recipients {
		if len(recipient) == 0 {
			continue
		}
		message.SetHeader("From", p.Config.From)
		message.SetAddressHeader("To", recipient, "")
		message.SetHeader("Subject", subject)
		message.AddAlternative("text/plain", plainBody)
		message.AddAlternative("text/html", html)

		if p.Config.Attachment != "" {
			message.Attach(p.Config.Attachment)
		}

		if err := gomail.Send(closer, message); err != nil {
			log.Errorf("Could not send email to %q: %v", recipient, err)
			return err
		}
		message.Reset()
	}

	return nil
}
<|endoftext|>


<commit_before>/*
 *
 * Copyright 2014, Google Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met:
 *
 *     * Redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above
 * copyright notice, this list of conditions and the following disclaimer
 * in the documentation and/or other materials provided with the
 * distribution.
 *     * Neither the name of Google Inc. nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 */

package grpc

import (
	"time"

	"golang.org/x/net/context"
	"google.golang.org/grpc/transport"
)

// Picker picks a Conn for RPC requests.
// This is EXPERIMENTAL and Please do not implement your own Picker for now.
type Picker interface {
	// Init does initial processing for the Picker, e.g., initiate some connections.
	Init(cc *ClientConn) error
	// Pick blocks until either a transport.ClientTransport is ready for the upcoming RPC
	// or some error happens.
	Pick(ctx context.Context) (transport.ClientTransport, error)
	// State returns the connectivity state of the underlying connections.
	State() ConnectivityState
	// WaitForStateChange blocks until the state changes to something other than
	// the sourceState or timeout fires on cc. It returns false if timeout fires,
	// and true otherwise.
	WaitForStateChange(timeout time.Duration, sourceState ConnectivityState) bool
	// Close closes all the Conn's owned by this Picker.
	Close() error
}

// unicastPicker is the default Picker which is used when there is no custom Picker
// specified by users. It always picks the same Conn.
type unicastPicker struct {
	conn *Conn
}

func (p *unicastPicker) Init(cc *ClientConn) error {
	c, err := NewConn(cc)
	if err != nil {
		return err
	}
	p.conn = c
	return nil
}

func (p *unicastPicker) Pick(ctx context.Context) (transport.ClientTransport, error) {
	return p.conn.Wait(ctx)
}

func (p *unicastPicker) State() ConnectivityState {
	return p.conn.State()
}

func (p *unicastPicker) WaitForStateChange(timeout time.Duration, sourceState ConnectivityState) bool {
	return p.conn.WaitForStateChange(timeout, sourceState)
}

func (p *unicastPicker) Close() error {
	if p.conn != nil {
		return p.conn.Close()
	}
	return nil
}
<commit_msg>picker: fix capitalization in comment<commit_after>/*
 *
 * Copyright 2014, Google Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met:
 *
 *     * Redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above
 * copyright notice, this list of conditions and the following disclaimer
 * in the documentation and/or other materials provided with the
 * distribution.
 *     * Neither the name of Google Inc. nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 */

package grpc

import (
	"time"

	"golang.org/x/net/context"
	"google.golang.org/grpc/transport"
)

// Picker picks a Conn for RPC requests.
// This is EXPERIMENTAL and please do not implement your own Picker for now.
type Picker interface {
	// Init does initial processing for the Picker, e.g., initiate some connections.
	Init(cc *ClientConn) error
	// Pick blocks until either a transport.ClientTransport is ready for the upcoming RPC
	// or some error happens.
	Pick(ctx context.Context) (transport.ClientTransport, error)
	// State returns the connectivity state of the underlying connections.
	State() ConnectivityState
	// WaitForStateChange blocks until the state changes to something other than
	// the sourceState or timeout fires on cc. It returns false if timeout fires,
	// and true otherwise.
	WaitForStateChange(timeout time.Duration, sourceState ConnectivityState) bool
	// Close closes all the Conn's owned by this Picker.
	Close() error
}

// unicastPicker is the default Picker which is used when there is no custom Picker
// specified by users. It always picks the same Conn.
type unicastPicker struct {
	conn *Conn
}

func (p *unicastPicker) Init(cc *ClientConn) error {
	c, err := NewConn(cc)
	if err != nil {
		return err
	}
	p.conn = c
	return nil
}

func (p *unicastPicker) Pick(ctx context.Context) (transport.ClientTransport, error) {
	return p.conn.Wait(ctx)
}

func (p *unicastPicker) State() ConnectivityState {
	return p.conn.State()
}

func (p *unicastPicker) WaitForStateChange(timeout time.Duration, sourceState ConnectivityState) bool {
	return p.conn.WaitForStateChange(timeout, sourceState)
}

func (p *unicastPicker) Close() error {
	if p.conn != nil {
		return p.conn.Close()
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/codegangsta/cli"
)

var CLIFlags = []cli.Flag{
	cli.StringFlag{
		Name:  "p, provider",
		Usage: "Path to provider for fetching secrets",
	},
	cli.StringFlag{
		Name:  "f",
		Value: "secrets.yml",
		Usage: "Path to secrets.yml",
	},
	cli.StringSliceFlag{
		Name:  "D",
		Value: &cli.StringSlice{},
		Usage: "var=value causes substitution of value to $var",
	},
	cli.StringFlag{
		Name:  "yaml",
		Usage: "secrets.yml as a literal string",
	},
	cli.StringSliceFlag{
		Name:  "ignore, i",
		Value: &cli.StringSlice{},
		Usage: "Ignore the specified key if is isn't accessible or doesn’t exist",
	},
}
<commit_msg>remove duplicate flags.go<commit_after><|endoftext|>


<commit_before>package main // import "github.com/johnpeterharvey/pinger"

import (
  "errors"
  "log"
  "net/http"
  "os"
  "strconv"
  "strings"
  "time"
)

func main() {
  log.Print("Starting up...")

  interval, settings, err := GetSettings()
  if err != nil {
    log.Print(err)
    os.Exit(1)
  }

  client := &http.Client{}

  for {
    DoCall(client, settings)
    log.Printf("Sleeping for %d seconds", interval)
    time.Sleep(time.Duration(interval) * time.Second)
  }
  log.Printf("Exiting\n")
}

func GetSettings() (int, map[string]string, error) {
  target := os.Getenv("TARGET_URL")
  method := os.Getenv("METHOD")
  interval, err := strconv.Atoi(os.Getenv("INTERVAL"))

  if target == "" || method == "" || err != nil || interval < 0 {
    return -1, map[string]string{}, errors.New("Environment variables were not set, returning error")
  }

  return interval, map[string]string{
    "target": target,
    "method": method},
    nil
}

func DoCall(client *http.Client, settings map[string]string) {
  log.Printf("Trying HTTP %s to %s", settings["method"], settings["target"])

  req, _ := http.NewRequest(settings["method"], settings["target"], strings.NewReader("{}"))
  req.Header.Add("Accept", "application/json")
  req.Header.Add("Content-Type", "application/json")
  resp, err := client.Do(req)

  if err != nil {
    log.Printf("Error! Received error while contacting target! %s", err)
  } else {
    log.Printf("Received response %d\n", resp.StatusCode)
    if resp.StatusCode < 200 || resp.StatusCode >= 300 {
      log.Printf("Error! Received unexpected status code from target! %d - %s", resp.StatusCode, err)
    }
    defer resp.Body.Close()
  }
}
<commit_msg>Run through formatter, add commenting, remove empty %s from error message, and remove unreachable log line<commit_after>package main // import "github.com/johnpeterharvey/pinger"

import (
	"errors"
	"log"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"
)

func main() {
	log.Print("Starting up...")

	interval, settings, err := GetSettings()
	if err != nil {
		log.Print(err)
		os.Exit(1)
	}

	client := &http.Client{}

	for {
		DoCall(client, settings)
		log.Printf("Sleeping for %d seconds", interval)
		time.Sleep(time.Duration(interval) * time.Second)
	}
}

// GetSettings Read required settings from environment variables
func GetSettings() (int, map[string]string, error) {
	target := os.Getenv("TARGET_URL")
	method := os.Getenv("METHOD")
	interval, err := strconv.Atoi(os.Getenv("INTERVAL"))

	if target == "" || method == "" || err != nil || interval < 0 {
		return -1, map[string]string{}, errors.New("Environment variables were not set, returning error")
	}

	return interval, map[string]string{
			"target": target,
			"method": method},
		nil
}

// DoCall Do an HTTP call out to the server, with the specified properties
func DoCall(client *http.Client, settings map[string]string) {
	log.Printf("Trying HTTP %s to %s", settings["method"], settings["target"])

	req, _ := http.NewRequest(settings["method"], settings["target"], strings.NewReader("{}"))
	req.Header.Add("Accept", "application/json")
	req.Header.Add("Content-Type", "application/json")
	resp, err := client.Do(req)

	if err != nil {
		log.Printf("Error! Received error while contacting target! %s", err)
	} else {
		log.Printf("Received response %d\n", resp.StatusCode)
		if resp.StatusCode < 200 || resp.StatusCode >= 300 {
			log.Printf("Error! Received unexpected status code from target! %d", resp.StatusCode)
		}
		defer resp.Body.Close()
	}
}
<|endoftext|>


<commit_before>//go:generate stringer -type=mType

package lorawan

import (
	"crypto/aes"
	"encoding/binary"
	"encoding/hex"
	"errors"
	"fmt"
	"strings"

	"github.com/jacobsa/crypto/cmac"
)

// MType represents the message type.
type mType byte

// Major defines the major version of data message.
type major byte

// Supported message types (MType)
const (
	JoinRequest         mType = 0
	JoinAccept          mType = (1 << 5)
	UnconfirmedDataUp   mType = (1 << 6)
	UnconfirmedDataDown mType = (1 << 6) ^ (1 << 5)
	ConfirmedDataUp     mType = (1 << 7)
	ConfirmedDataDown   mType = (1 << 7) ^ (1 << 5)
	Proprietary         mType = (1 << 7) ^ (1 << 6) ^ (1 << 5)
)

// Supported major versions
const (
	LoRaWANR1 major = 0
)

// AES128Key represents a 128 bit AES key.
type AES128Key [16]byte

// String implements fmt.Stringer.
func (k AES128Key) String() string {
	return hex.EncodeToString(k[:])
}

// MarshalJSON implements json.Marshaler.
func (k AES128Key) MarshalJSON() ([]byte, error) {
	return []byte(`"` + k.String() + `"`), nil
}

// UnmarshalJSON implements json.Unmarshaler.
func (k *AES128Key) UnmarshalJSON(data []byte) error {
	hexStr := strings.Trim(string(data), `"`)
	b, err := hex.DecodeString(hexStr)
	if err != nil {
		return err
	}
	if len(b) != len(k) {
		return fmt.Errorf("lorawan: exactly %d bytes are expected", len(k))
	}
	copy(k[:], b)
	return nil
}

// MHDR represents the MAC header.
type MHDR struct {
	MType mType
	Major major
}

// MarshalBinary marshals the object in binary form.
func (h MHDR) MarshalBinary() ([]byte, error) {
	return []byte{byte(h.Major) ^ byte(h.MType)}, nil
}

// UnmarshalBinary decodes the object from binary form.
func (h *MHDR) UnmarshalBinary(data []byte) error {
	if len(data) != 1 {
		return errors.New("lorawan: 1 byte of data is expected")
	}
	h.Major = major(data[0] & ((1 << 1) ^ (1 << 0)))
	h.MType = mType(data[0] & ((1 << 7) ^ (1 << 6) ^ (1 << 5)))
	return nil
}

// PHYPayload represents the physical payload. Use NewPhyPayload for creating
// a new PHYPayload.
type PHYPayload struct {
	MHDR       MHDR
	MACPayload Payload
	MIC        [4]byte
	uplink     bool
}

// NewPHYPayload returns a new PHYPayload instance set to either uplink or downlink.
// This is needed since there is a difference in how uplink and downlink
// payloads are (un)marshalled and encrypted / decrypted.
func NewPHYPayload(uplink bool) PHYPayload {
	return PHYPayload{uplink: uplink}
}

// calculateMIC calculates and returns the MIC.
func (p PHYPayload) calculateMIC(key AES128Key) ([]byte, error) {
	if p.MACPayload == nil {
		return []byte{}, errors.New("lorawan: MACPayload should not be empty")
	}

	macPayload, ok := p.MACPayload.(*MACPayload)
	if !ok {
		return []byte{}, errors.New("lorawan: MACPayload should be of type *MACPayload")
	}

	var b []byte
	var err error
	var micBytes []byte

	b, err = p.MHDR.MarshalBinary()
	if err != nil {
		return nil, err
	}
	micBytes = append(micBytes, b...)

	b, err = macPayload.MarshalBinary()
	if err != nil {
		return nil, err
	}
	micBytes = append(micBytes, b...)

	b0 := make([]byte, 16)
	b0[0] = 0x49
	if !p.uplink {
		b0[5] = 1
	}
	b, err = macPayload.FHDR.DevAddr.MarshalBinary()
	if err != nil {
		return nil, err
	}
	copy(b0[6:10], b)
	binary.LittleEndian.PutUint32(b0[10:14], macPayload.FHDR.FCnt)
	b0[15] = byte(len(micBytes))

	hash, err := cmac.New(key[:])
	if err != nil {
		return nil, err
	}

	if _, err = hash.Write(b0); err != nil {
		return nil, err
	}
	if _, err = hash.Write(micBytes); err != nil {
		return nil, err
	}

	hb := hash.Sum([]byte{})
	if len(hb) < 4 {
		return nil, errors.New("lorawan: the hash returned less than 4 bytes")
	}
	return hb[0:4], nil
}

// calculateJoinRequestMIC calculates and returns the join-request MIC.
func (p PHYPayload) calculateJoinRequestMIC(key AES128Key) ([]byte, error) {
	if p.MACPayload == nil {
		return []byte{}, errors.New("lorawan: MACPayload should not be empty")
	}
	jrPayload, ok := p.MACPayload.(*JoinRequestPayload)
	if !ok {
		return []byte{}, errors.New("lorawan: MACPayload should be of type *JoinRequestPayload")
	}

	micBytes := make([]byte, 0, 19)

	b, err := p.MHDR.MarshalBinary()
	if err != nil {
		return []byte{}, err
	}
	micBytes = append(micBytes, b...)

	b, err = jrPayload.MarshalBinary()
	if err != nil {
		return nil, err
	}
	micBytes = append(micBytes, b...)

	hash, err := cmac.New(key[:])
	if err != nil {
		return []byte{}, err
	}
	if _, err = hash.Write(micBytes); err != nil {
		return nil, err
	}
	hb := hash.Sum([]byte{})
	if len(hb) < 4 {
		return []byte{}, errors.New("lorawan: the hash returned less than 4 bytes")
	}
	return hb[0:4], nil
}

// calculateJoinAcceptMIC calculates and returns the join-accept MIC.
func (p PHYPayload) calculateJoinAcceptMIC(key AES128Key) ([]byte, error) {
	if p.MACPayload == nil {
		return []byte{}, errors.New("lorawan: MACPayload should not be empty")
	}
	jaPayload, ok := p.MACPayload.(*JoinAcceptPayload)
	if !ok {
		return []byte{}, errors.New("lorawan: MACPayload should be of type *JoinAcceptPayload")
	}

	micBytes := make([]byte, 0, 13)

	b, err := p.MHDR.MarshalBinary()
	if err != nil {
		return []byte{}, err
	}
	micBytes = append(micBytes, b...)

	b, err = jaPayload.MarshalBinary()
	if err != nil {
		return nil, err
	}
	micBytes = append(micBytes, b...)

	hash, err := cmac.New(key[:])
	if err != nil {
		return []byte{}, err
	}
	if _, err = hash.Write(micBytes); err != nil {
		return nil, err
	}
	hb := hash.Sum([]byte{})
	if len(hb) < 4 {
		return []byte{}, errors.New("lorawan: the hash returned less than 4 bytes")
	}
	return hb[0:4], nil
}

// SetMIC calculates and sets the MIC field.
func (p *PHYPayload) SetMIC(key AES128Key) error {
	var mic []byte
	var err error

	switch p.MACPayload.(type) {
	case *JoinRequestPayload:
		mic, err = p.calculateJoinRequestMIC(key)
	case *JoinAcceptPayload:
		mic, err = p.calculateJoinAcceptMIC(key)
	default:
		mic, err = p.calculateMIC(key)
	}

	if err != nil {
		return err
	}
	if len(mic) != 4 {
		return errors.New("lorawan: a MIC of 4 bytes is expected")
	}
	for i, v := range mic {
		p.MIC[i] = v
	}
	return nil
}

// ValidateMIC returns if the MIC is valid.
func (p PHYPayload) ValidateMIC(key AES128Key) (bool, error) {
	var mic []byte
	var err error

	switch p.MACPayload.(type) {
	case *JoinRequestPayload:
		mic, err = p.calculateJoinRequestMIC(key)
	case *JoinAcceptPayload:
		mic, err = p.calculateJoinAcceptMIC(key)
	default:
		mic, err = p.calculateMIC(key)
	}

	if err != nil {
		return false, err
	}
	if len(mic) != 4 {
		return false, errors.New("lorawan: a MIC of 4 bytes is expected")
	}
	for i, v := range mic {
		if p.MIC[i] != v {
			return false, nil
		}
	}
	return true, nil
}

// EncryptMACPayload encrypts the MACPayload with the given key. Note that this
// should only be done when the MACPayload is a JoinAcceptPayload.
// Note that the encryption should be performed after SetMIC since the MIC
// is part of the encrypted content.
func (p *PHYPayload) EncryptMACPayload(key AES128Key) error {
	if _, ok := p.MACPayload.(*JoinAcceptPayload); !ok {
		return errors.New("lorawan: EncryptMACPayload can only be for *JoinAcceptPayload")
	}

	pt, err := p.MACPayload.MarshalBinary()
	if err != nil {
		return err
	}

	// in the 1.0 spec instead of DLSettings there is RFU field. the assumption
	// is made that this should have been DLSettings.

	pt = append(pt, p.MIC[0:4]...)
	if len(pt)%16 != 0 {
		return errors.New("lorawan: plaintext should be a multiple of 16")
	}

	block, err := aes.NewCipher(key[:])
	if err != nil {
		return err
	}
	if block.BlockSize() != 16 {
		return errors.New("lorawan: block-size of 16 bytes is expected")
	}
	ct := make([]byte, len(pt))
	for i := 0; i < len(pt)/16; i++ {
		// decrypt is used for encryption, so that the node only has to implement
		// the aes encryption (see lorawan specs)
		block.Decrypt(ct[i*16:(i*16)+16], pt[i*16:(i*16)+16])
	}
	p.MACPayload = &DataPayload{Bytes: ct}
	return nil
}

// DecryptMACPayload decrypts the MACPayload with the given key.
func (p *PHYPayload) DecryptMACPayload(key AES128Key) error {
	dp, ok := p.MACPayload.(*DataPayload)
	if !ok {
		return errors.New("lorawan: MACPayload should be of type *DataPayload")
	}
	if len(dp.Bytes)%16 != 0 {
		return errors.New("lorawan: the DataPayload should be a multiple of 16")
	}

	block, err := aes.NewCipher(key[:])
	if err != nil {
		return err
	}
	if block.BlockSize() != 16 {
		return errors.New("lorawan: block-size of 16 bytes is expected")
	}
	pt := make([]byte, len(dp.Bytes))
	for i := 0; i < len(pt)/16; i++ {
		block.Encrypt(pt[i*16:i*16+16], dp.Bytes[i*16:i*16+16])
	}
	p.MACPayload = &JoinAcceptPayload{}
	return p.MACPayload.UnmarshalBinary(pt[0 : len(pt)-4]) // - MIC
}

// MarshalBinary marshals the object in binary form.
func (p PHYPayload) MarshalBinary() ([]byte, error) {
	if p.MACPayload == nil {
		return []byte{}, errors.New("lorawan: MACPayload should not be nil")
	}

	if mpl, ok := p.MACPayload.(*MACPayload); ok {
		mpl.uplink = p.uplink
	}

	var out []byte
	var b []byte
	var err error

	if b, err = p.MHDR.MarshalBinary(); err != nil {
		return []byte{}, err
	}
	out = append(out, b...)

	if b, err = p.MACPayload.MarshalBinary(); err != nil {
		return []byte{}, err
	}
	out = append(out, b...)
	out = append(out, p.MIC[0:len(p.MIC)]...)
	return out, nil
}

// UnmarshalBinary decodes the object from binary form.
func (p *PHYPayload) UnmarshalBinary(data []byte) error {
	if len(data) < 5 {
		return errors.New("lorawan: at least 5 bytes needed to decode PHYPayload")
	}

	if err := p.MHDR.UnmarshalBinary(data[0:1]); err != nil {
		return err
	}

	switch p.MHDR.MType {
	case JoinRequest:
		p.MACPayload = &JoinRequestPayload{}
	case JoinAccept:
		p.MACPayload = &DataPayload{}
	default:
		p.MACPayload = &MACPayload{uplink: p.uplink}
	}

	if err := p.MACPayload.UnmarshalBinary(data[1 : len(data)-4]); err != nil {
		return err
	}
	for i := 0; i < 4; i++ {
		p.MIC[i] = data[len(data)-4+i]
	}
	return nil
}

// GobEncode implements the gob.GobEncoder interface.
func (p PHYPayload) GobEncode() ([]byte, error) {
	out := make([]byte, 1)
	if p.uplink {
		out[0] = 1
	}

	b, err := p.MarshalBinary()
	if err != nil {
		return nil, err
	}
	out = append(out, b...)
	return out, nil
}

// GobDecode implements the gob.GobEncoder interface.
func (p *PHYPayload) GobDecode(data []byte) error {
	if len(data) < 1 {
		return errors.New("lorawan: at least 1 byte needed for GobDecode")
	}
	if data[0] == 1 {
		p.uplink = true
	}
	return p.UnmarshalBinary(data[1:])
}
<commit_msg>Add note about 32 bit FCnt when validating the MIC.<commit_after>//go:generate stringer -type=mType

package lorawan

import (
	"crypto/aes"
	"encoding/binary"
	"encoding/hex"
	"errors"
	"fmt"
	"strings"

	"github.com/jacobsa/crypto/cmac"
)

// MType represents the message type.
type mType byte

// Major defines the major version of data message.
type major byte

// Supported message types (MType)
const (
	JoinRequest         mType = 0
	JoinAccept          mType = (1 << 5)
	UnconfirmedDataUp   mType = (1 << 6)
	UnconfirmedDataDown mType = (1 << 6) ^ (1 << 5)
	ConfirmedDataUp     mType = (1 << 7)
	ConfirmedDataDown   mType = (1 << 7) ^ (1 << 5)
	Proprietary         mType = (1 << 7) ^ (1 << 6) ^ (1 << 5)
)

// Supported major versions
const (
	LoRaWANR1 major = 0
)

// AES128Key represents a 128 bit AES key.
type AES128Key [16]byte

// String implements fmt.Stringer.
func (k AES128Key) String() string {
	return hex.EncodeToString(k[:])
}

// MarshalJSON implements json.Marshaler.
func (k AES128Key) MarshalJSON() ([]byte, error) {
	return []byte(`"` + k.String() + `"`), nil
}

// UnmarshalJSON implements json.Unmarshaler.
func (k *AES128Key) UnmarshalJSON(data []byte) error {
	hexStr := strings.Trim(string(data), `"`)
	b, err := hex.DecodeString(hexStr)
	if err != nil {
		return err
	}
	if len(b) != len(k) {
		return fmt.Errorf("lorawan: exactly %d bytes are expected", len(k))
	}
	copy(k[:], b)
	return nil
}

// MHDR represents the MAC header.
type MHDR struct {
	MType mType
	Major major
}

// MarshalBinary marshals the object in binary form.
func (h MHDR) MarshalBinary() ([]byte, error) {
	return []byte{byte(h.Major) ^ byte(h.MType)}, nil
}

// UnmarshalBinary decodes the object from binary form.
func (h *MHDR) UnmarshalBinary(data []byte) error {
	if len(data) != 1 {
		return errors.New("lorawan: 1 byte of data is expected")
	}
	h.Major = major(data[0] & ((1 << 1) ^ (1 << 0)))
	h.MType = mType(data[0] & ((1 << 7) ^ (1 << 6) ^ (1 << 5)))
	return nil
}

// PHYPayload represents the physical payload. Use NewPhyPayload for creating
// a new PHYPayload.
type PHYPayload struct {
	MHDR       MHDR
	MACPayload Payload
	MIC        [4]byte
	uplink     bool
}

// NewPHYPayload returns a new PHYPayload instance set to either uplink or downlink.
// This is needed since there is a difference in how uplink and downlink
// payloads are (un)marshalled and encrypted / decrypted.
func NewPHYPayload(uplink bool) PHYPayload {
	return PHYPayload{uplink: uplink}
}

// calculateMIC calculates and returns the MIC.
func (p PHYPayload) calculateMIC(key AES128Key) ([]byte, error) {
	if p.MACPayload == nil {
		return []byte{}, errors.New("lorawan: MACPayload should not be empty")
	}

	macPayload, ok := p.MACPayload.(*MACPayload)
	if !ok {
		return []byte{}, errors.New("lorawan: MACPayload should be of type *MACPayload")
	}

	var b []byte
	var err error
	var micBytes []byte

	b, err = p.MHDR.MarshalBinary()
	if err != nil {
		return nil, err
	}
	micBytes = append(micBytes, b...)

	b, err = macPayload.MarshalBinary()
	if err != nil {
		return nil, err
	}
	micBytes = append(micBytes, b...)

	b0 := make([]byte, 16)
	b0[0] = 0x49
	if !p.uplink {
		b0[5] = 1
	}
	b, err = macPayload.FHDR.DevAddr.MarshalBinary()
	if err != nil {
		return nil, err
	}
	copy(b0[6:10], b)
	binary.LittleEndian.PutUint32(b0[10:14], macPayload.FHDR.FCnt)
	b0[15] = byte(len(micBytes))

	hash, err := cmac.New(key[:])
	if err != nil {
		return nil, err
	}

	if _, err = hash.Write(b0); err != nil {
		return nil, err
	}
	if _, err = hash.Write(micBytes); err != nil {
		return nil, err
	}

	hb := hash.Sum([]byte{})
	if len(hb) < 4 {
		return nil, errors.New("lorawan: the hash returned less than 4 bytes")
	}
	return hb[0:4], nil
}

// calculateJoinRequestMIC calculates and returns the join-request MIC.
func (p PHYPayload) calculateJoinRequestMIC(key AES128Key) ([]byte, error) {
	if p.MACPayload == nil {
		return []byte{}, errors.New("lorawan: MACPayload should not be empty")
	}
	jrPayload, ok := p.MACPayload.(*JoinRequestPayload)
	if !ok {
		return []byte{}, errors.New("lorawan: MACPayload should be of type *JoinRequestPayload")
	}

	micBytes := make([]byte, 0, 19)

	b, err := p.MHDR.MarshalBinary()
	if err != nil {
		return []byte{}, err
	}
	micBytes = append(micBytes, b...)

	b, err = jrPayload.MarshalBinary()
	if err != nil {
		return nil, err
	}
	micBytes = append(micBytes, b...)

	hash, err := cmac.New(key[:])
	if err != nil {
		return []byte{}, err
	}
	if _, err = hash.Write(micBytes); err != nil {
		return nil, err
	}
	hb := hash.Sum([]byte{})
	if len(hb) < 4 {
		return []byte{}, errors.New("lorawan: the hash returned less than 4 bytes")
	}
	return hb[0:4], nil
}

// calculateJoinAcceptMIC calculates and returns the join-accept MIC.
func (p PHYPayload) calculateJoinAcceptMIC(key AES128Key) ([]byte, error) {
	if p.MACPayload == nil {
		return []byte{}, errors.New("lorawan: MACPayload should not be empty")
	}
	jaPayload, ok := p.MACPayload.(*JoinAcceptPayload)
	if !ok {
		return []byte{}, errors.New("lorawan: MACPayload should be of type *JoinAcceptPayload")
	}

	micBytes := make([]byte, 0, 13)

	b, err := p.MHDR.MarshalBinary()
	if err != nil {
		return []byte{}, err
	}
	micBytes = append(micBytes, b...)

	b, err = jaPayload.MarshalBinary()
	if err != nil {
		return nil, err
	}
	micBytes = append(micBytes, b...)

	hash, err := cmac.New(key[:])
	if err != nil {
		return []byte{}, err
	}
	if _, err = hash.Write(micBytes); err != nil {
		return nil, err
	}
	hb := hash.Sum([]byte{})
	if len(hb) < 4 {
		return []byte{}, errors.New("lorawan: the hash returned less than 4 bytes")
	}
	return hb[0:4], nil
}

// SetMIC calculates and sets the MIC field.
func (p *PHYPayload) SetMIC(key AES128Key) error {
	var mic []byte
	var err error

	switch p.MACPayload.(type) {
	case *JoinRequestPayload:
		mic, err = p.calculateJoinRequestMIC(key)
	case *JoinAcceptPayload:
		mic, err = p.calculateJoinAcceptMIC(key)
	default:
		mic, err = p.calculateMIC(key)
	}

	if err != nil {
		return err
	}
	if len(mic) != 4 {
		return errors.New("lorawan: a MIC of 4 bytes is expected")
	}
	for i, v := range mic {
		p.MIC[i] = v
	}
	return nil
}

// ValidateMIC returns if the MIC is valid.
// When using 32 bit frame counters, only the least-signification 16 bits are
// sent / received. In order to validate the MIC, the receiver needs to set
// the FCnt to the full 32 bit value (based on the observation of the traffic).
// See section '4.3.1.5 Frame counter (FCnt)' of the LoRaWAN 1.0 specification
// for more details.
func (p PHYPayload) ValidateMIC(key AES128Key) (bool, error) {
	var mic []byte
	var err error

	switch p.MACPayload.(type) {
	case *JoinRequestPayload:
		mic, err = p.calculateJoinRequestMIC(key)
	case *JoinAcceptPayload:
		mic, err = p.calculateJoinAcceptMIC(key)
	default:
		mic, err = p.calculateMIC(key)
	}

	if err != nil {
		return false, err
	}
	if len(mic) != 4 {
		return false, errors.New("lorawan: a MIC of 4 bytes is expected")
	}
	for i, v := range mic {
		if p.MIC[i] != v {
			return false, nil
		}
	}
	return true, nil
}

// EncryptMACPayload encrypts the MACPayload with the given key. Note that this
// should only be done when the MACPayload is a JoinAcceptPayload.
// Note that the encryption should be performed after SetMIC since the MIC
// is part of the encrypted content.
func (p *PHYPayload) EncryptMACPayload(key AES128Key) error {
	if _, ok := p.MACPayload.(*JoinAcceptPayload); !ok {
		return errors.New("lorawan: EncryptMACPayload can only be for *JoinAcceptPayload")
	}

	pt, err := p.MACPayload.MarshalBinary()
	if err != nil {
		return err
	}

	// in the 1.0 spec instead of DLSettings there is RFU field. the assumption
	// is made that this should have been DLSettings.

	pt = append(pt, p.MIC[0:4]...)
	if len(pt)%16 != 0 {
		return errors.New("lorawan: plaintext should be a multiple of 16")
	}

	block, err := aes.NewCipher(key[:])
	if err != nil {
		return err
	}
	if block.BlockSize() != 16 {
		return errors.New("lorawan: block-size of 16 bytes is expected")
	}
	ct := make([]byte, len(pt))
	for i := 0; i < len(pt)/16; i++ {
		// decrypt is used for encryption, so that the node only has to implement
		// the aes encryption (see lorawan specs)
		block.Decrypt(ct[i*16:(i*16)+16], pt[i*16:(i*16)+16])
	}
	p.MACPayload = &DataPayload{Bytes: ct}
	return nil
}

// DecryptMACPayload decrypts the MACPayload with the given key.
func (p *PHYPayload) DecryptMACPayload(key AES128Key) error {
	dp, ok := p.MACPayload.(*DataPayload)
	if !ok {
		return errors.New("lorawan: MACPayload should be of type *DataPayload")
	}
	if len(dp.Bytes)%16 != 0 {
		return errors.New("lorawan: the DataPayload should be a multiple of 16")
	}

	block, err := aes.NewCipher(key[:])
	if err != nil {
		return err
	}
	if block.BlockSize() != 16 {
		return errors.New("lorawan: block-size of 16 bytes is expected")
	}
	pt := make([]byte, len(dp.Bytes))
	for i := 0; i < len(pt)/16; i++ {
		block.Encrypt(pt[i*16:i*16+16], dp.Bytes[i*16:i*16+16])
	}
	p.MACPayload = &JoinAcceptPayload{}
	return p.MACPayload.UnmarshalBinary(pt[0 : len(pt)-4]) // - MIC
}

// MarshalBinary marshals the object in binary form.
func (p PHYPayload) MarshalBinary() ([]byte, error) {
	if p.MACPayload == nil {
		return []byte{}, errors.New("lorawan: MACPayload should not be nil")
	}

	if mpl, ok := p.MACPayload.(*MACPayload); ok {
		mpl.uplink = p.uplink
	}

	var out []byte
	var b []byte
	var err error

	if b, err = p.MHDR.MarshalBinary(); err != nil {
		return []byte{}, err
	}
	out = append(out, b...)

	if b, err = p.MACPayload.MarshalBinary(); err != nil {
		return []byte{}, err
	}
	out = append(out, b...)
	out = append(out, p.MIC[0:len(p.MIC)]...)
	return out, nil
}

// UnmarshalBinary decodes the object from binary form.
func (p *PHYPayload) UnmarshalBinary(data []byte) error {
	if len(data) < 5 {
		return errors.New("lorawan: at least 5 bytes needed to decode PHYPayload")
	}

	if err := p.MHDR.UnmarshalBinary(data[0:1]); err != nil {
		return err
	}

	switch p.MHDR.MType {
	case JoinRequest:
		p.MACPayload = &JoinRequestPayload{}
	case JoinAccept:
		p.MACPayload = &DataPayload{}
	default:
		p.MACPayload = &MACPayload{uplink: p.uplink}
	}

	if err := p.MACPayload.UnmarshalBinary(data[1 : len(data)-4]); err != nil {
		return err
	}
	for i := 0; i < 4; i++ {
		p.MIC[i] = data[len(data)-4+i]
	}
	return nil
}

// GobEncode implements the gob.GobEncoder interface.
func (p PHYPayload) GobEncode() ([]byte, error) {
	out := make([]byte, 1)
	if p.uplink {
		out[0] = 1
	}

	b, err := p.MarshalBinary()
	if err != nil {
		return nil, err
	}
	out = append(out, b...)
	return out, nil
}

// GobDecode implements the gob.GobEncoder interface.
func (p *PHYPayload) GobDecode(data []byte) error {
	if len(data) < 1 {
		return errors.New("lorawan: at least 1 byte needed for GobDecode")
	}
	if data[0] == 1 {
		p.uplink = true
	}
	return p.UnmarshalBinary(data[1:])
}
<|endoftext|>


<commit_before>package registry

import (
	"crypto/tls"
	"crypto/x509"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/autogen/dockerversion"
	"github.com/docker/docker/pkg/parsers/kernel"
	"github.com/docker/docker/pkg/timeoutconn"
	"github.com/docker/docker/pkg/transport"
	"github.com/docker/docker/pkg/useragent"
)

var (
	ErrAlreadyExists = errors.New("Image already exists")
	ErrDoesNotExist  = errors.New("Image does not exist")
	errLoginRequired = errors.New("Authentication is required.")
)

type TimeoutType uint32

const (
	NoTimeout TimeoutType = iota
	ReceiveTimeout
	ConnectTimeout
)

// dockerUserAgent is the User-Agent the Docker client uses to identify itself.
// It is populated on init(), comprising version information of different components.
var dockerUserAgent string

func init() {
	httpVersion := make([]useragent.VersionInfo, 0, 6)
	httpVersion = append(httpVersion, useragent.VersionInfo{"docker", dockerversion.VERSION})
	httpVersion = append(httpVersion, useragent.VersionInfo{"go", runtime.Version()})
	httpVersion = append(httpVersion, useragent.VersionInfo{"git-commit", dockerversion.GITCOMMIT})
	if kernelVersion, err := kernel.GetKernelVersion(); err == nil {
		httpVersion = append(httpVersion, useragent.VersionInfo{"kernel", kernelVersion.String()})
	}
	httpVersion = append(httpVersion, useragent.VersionInfo{"os", runtime.GOOS})
	httpVersion = append(httpVersion, useragent.VersionInfo{"arch", runtime.GOARCH})

	dockerUserAgent = useragent.AppendVersions("", httpVersion...)
}

type httpsRequestModifier struct {
	mu        sync.Mutex
	tlsConfig *tls.Config
}

// DRAGONS(tiborvass): If someone wonders why do we set tlsconfig in a roundtrip,
// it's because it's so as to match the current behavior in master: we generate the
// certpool on every-goddam-request. It's not great, but it allows people to just put
// the certs in /etc/docker/certs.d/.../ and let docker "pick it up" immediately. Would
// prefer an fsnotify implementation, but that was out of scope of my refactoring.
func (m *httpsRequestModifier) ModifyRequest(req *http.Request) error {
	var (
		roots   *x509.CertPool
		certs   []tls.Certificate
		hostDir string
	)

	if req.URL.Scheme == "https" {
		hasFile := func(files []os.FileInfo, name string) bool {
			for _, f := range files {
				if f.Name() == name {
					return true
				}
			}
			return false
		}

		if runtime.GOOS == "windows" {
			hostDir = path.Join(os.TempDir(), "/docker/certs.d", req.URL.Host)
		} else {
			hostDir = path.Join("/etc/docker/certs.d", req.URL.Host)
		}
		logrus.Debugf("hostDir: %s", hostDir)
		fs, err := ioutil.ReadDir(hostDir)
		if err != nil && !os.IsNotExist(err) {
			return nil
		}

		for _, f := range fs {
			if strings.HasSuffix(f.Name(), ".crt") {
				if roots == nil {
					roots = x509.NewCertPool()
				}
				logrus.Debugf("crt: %s", hostDir+"/"+f.Name())
				data, err := ioutil.ReadFile(filepath.Join(hostDir, f.Name()))
				if err != nil {
					return err
				}
				roots.AppendCertsFromPEM(data)
			}
			if strings.HasSuffix(f.Name(), ".cert") {
				certName := f.Name()
				keyName := certName[:len(certName)-5] + ".key"
				logrus.Debugf("cert: %s", hostDir+"/"+f.Name())
				if !hasFile(fs, keyName) {
					return fmt.Errorf("Missing key %s for certificate %s", keyName, certName)
				}
				cert, err := tls.LoadX509KeyPair(filepath.Join(hostDir, certName), path.Join(hostDir, keyName))
				if err != nil {
					return err
				}
				certs = append(certs, cert)
			}
			if strings.HasSuffix(f.Name(), ".key") {
				keyName := f.Name()
				certName := keyName[:len(keyName)-4] + ".cert"
				logrus.Debugf("key: %s", hostDir+"/"+f.Name())
				if !hasFile(fs, certName) {
					return fmt.Errorf("Missing certificate %s for key %s", certName, keyName)
				}
			}
		}
		m.mu.Lock()
		m.tlsConfig.RootCAs = roots
		m.tlsConfig.Certificates = certs
		m.mu.Unlock()
	}
	return nil
}

func NewTransport(timeout TimeoutType, secure bool) http.RoundTripper {
	tlsConfig := &tls.Config{
		// Avoid fallback to SSL protocols < TLS1.0
		MinVersion:         tls.VersionTLS10,
		InsecureSkipVerify: !secure,
	}

	tr := &http.Transport{
		DisableKeepAlives: true,
		Proxy:             http.ProxyFromEnvironment,
		TLSClientConfig:   tlsConfig,
	}

	switch timeout {
	case ConnectTimeout:
		tr.Dial = func(proto string, addr string) (net.Conn, error) {
			// Set the connect timeout to 30 seconds to allow for slower connection
			// times...
			d := net.Dialer{Timeout: 30 * time.Second, DualStack: true}

			conn, err := d.Dial(proto, addr)
			if err != nil {
				return nil, err
			}
			// Set the recv timeout to 10 seconds
			conn.SetDeadline(time.Now().Add(10 * time.Second))
			return conn, nil
		}
	case ReceiveTimeout:
		tr.Dial = func(proto string, addr string) (net.Conn, error) {
			d := net.Dialer{DualStack: true}

			conn, err := d.Dial(proto, addr)
			if err != nil {
				return nil, err
			}
			conn = timeoutconn.New(conn, 1*time.Minute)
			return conn, nil
		}
	}

	if secure {
		// note: httpsTransport also handles http transport
		// but for HTTPS, it sets up the certs
		return transport.NewTransport(tr, &httpsRequestModifier{tlsConfig: tlsConfig})
	}

	return tr
}

// DockerHeaders returns request modifiers that ensure requests have
// the User-Agent header set to dockerUserAgent and that metaHeaders
// are added.
func DockerHeaders(metaHeaders http.Header) []transport.RequestModifier {
	modifiers := []transport.RequestModifier{
		transport.NewHeaderRequestModifier(http.Header{"User-Agent": []string{dockerUserAgent}}),
	}
	if metaHeaders != nil {
		modifiers = append(modifiers, transport.NewHeaderRequestModifier(metaHeaders))
	}
	return modifiers
}

func HTTPClient(transport http.RoundTripper) *http.Client {
	if transport == nil {
		transport = NewTransport(ConnectTimeout, true)
	}

	return &http.Client{
		Transport:     transport,
		CheckRedirect: AddRequiredHeadersToRedirectedRequests,
	}
}

func trustedLocation(req *http.Request) bool {
	var (
		trusteds = []string{"docker.com", "docker.io"}
		hostname = strings.SplitN(req.Host, ":", 2)[0]
	)
	if req.URL.Scheme != "https" {
		return false
	}

	for _, trusted := range trusteds {
		if hostname == trusted || strings.HasSuffix(hostname, "."+trusted) {
			return true
		}
	}
	return false
}

func AddRequiredHeadersToRedirectedRequests(req *http.Request, via []*http.Request) error {
	if via != nil && via[0] != nil {
		if trustedLocation(req) && trustedLocation(via[0]) {
			req.Header = via[0].Header
			return nil
		}
		for k, v := range via[0].Header {
			if k != "Authorization" {
				for _, vv := range v {
					req.Header.Add(k, vv)
				}
			}
		}
	}
	return nil
}
<commit_msg>Remove RC4 from the list of registry cipher suites<commit_after>package registry

import (
	"crypto/tls"
	"crypto/x509"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/autogen/dockerversion"
	"github.com/docker/docker/pkg/parsers/kernel"
	"github.com/docker/docker/pkg/timeoutconn"
	"github.com/docker/docker/pkg/tlsconfig"
	"github.com/docker/docker/pkg/transport"
	"github.com/docker/docker/pkg/useragent"
)

var (
	ErrAlreadyExists = errors.New("Image already exists")
	ErrDoesNotExist  = errors.New("Image does not exist")
	errLoginRequired = errors.New("Authentication is required.")
)

type TimeoutType uint32

const (
	NoTimeout TimeoutType = iota
	ReceiveTimeout
	ConnectTimeout
)

// dockerUserAgent is the User-Agent the Docker client uses to identify itself.
// It is populated on init(), comprising version information of different components.
var dockerUserAgent string

func init() {
	httpVersion := make([]useragent.VersionInfo, 0, 6)
	httpVersion = append(httpVersion, useragent.VersionInfo{"docker", dockerversion.VERSION})
	httpVersion = append(httpVersion, useragent.VersionInfo{"go", runtime.Version()})
	httpVersion = append(httpVersion, useragent.VersionInfo{"git-commit", dockerversion.GITCOMMIT})
	if kernelVersion, err := kernel.GetKernelVersion(); err == nil {
		httpVersion = append(httpVersion, useragent.VersionInfo{"kernel", kernelVersion.String()})
	}
	httpVersion = append(httpVersion, useragent.VersionInfo{"os", runtime.GOOS})
	httpVersion = append(httpVersion, useragent.VersionInfo{"arch", runtime.GOARCH})

	dockerUserAgent = useragent.AppendVersions("", httpVersion...)
}

type httpsRequestModifier struct {
	mu        sync.Mutex
	tlsConfig *tls.Config
}

// DRAGONS(tiborvass): If someone wonders why do we set tlsconfig in a roundtrip,
// it's because it's so as to match the current behavior in master: we generate the
// certpool on every-goddam-request. It's not great, but it allows people to just put
// the certs in /etc/docker/certs.d/.../ and let docker "pick it up" immediately. Would
// prefer an fsnotify implementation, but that was out of scope of my refactoring.
func (m *httpsRequestModifier) ModifyRequest(req *http.Request) error {
	var (
		roots   *x509.CertPool
		certs   []tls.Certificate
		hostDir string
	)

	if req.URL.Scheme == "https" {
		hasFile := func(files []os.FileInfo, name string) bool {
			for _, f := range files {
				if f.Name() == name {
					return true
				}
			}
			return false
		}

		if runtime.GOOS == "windows" {
			hostDir = path.Join(os.TempDir(), "/docker/certs.d", req.URL.Host)
		} else {
			hostDir = path.Join("/etc/docker/certs.d", req.URL.Host)
		}
		logrus.Debugf("hostDir: %s", hostDir)
		fs, err := ioutil.ReadDir(hostDir)
		if err != nil && !os.IsNotExist(err) {
			return nil
		}

		for _, f := range fs {
			if strings.HasSuffix(f.Name(), ".crt") {
				if roots == nil {
					roots = x509.NewCertPool()
				}
				logrus.Debugf("crt: %s", hostDir+"/"+f.Name())
				data, err := ioutil.ReadFile(filepath.Join(hostDir, f.Name()))
				if err != nil {
					return err
				}
				roots.AppendCertsFromPEM(data)
			}
			if strings.HasSuffix(f.Name(), ".cert") {
				certName := f.Name()
				keyName := certName[:len(certName)-5] + ".key"
				logrus.Debugf("cert: %s", hostDir+"/"+f.Name())
				if !hasFile(fs, keyName) {
					return fmt.Errorf("Missing key %s for certificate %s", keyName, certName)
				}
				cert, err := tls.LoadX509KeyPair(filepath.Join(hostDir, certName), path.Join(hostDir, keyName))
				if err != nil {
					return err
				}
				certs = append(certs, cert)
			}
			if strings.HasSuffix(f.Name(), ".key") {
				keyName := f.Name()
				certName := keyName[:len(keyName)-4] + ".cert"
				logrus.Debugf("key: %s", hostDir+"/"+f.Name())
				if !hasFile(fs, certName) {
					return fmt.Errorf("Missing certificate %s for key %s", certName, keyName)
				}
			}
		}
		m.mu.Lock()
		m.tlsConfig.RootCAs = roots
		m.tlsConfig.Certificates = certs
		m.mu.Unlock()
	}
	return nil
}

func NewTransport(timeout TimeoutType, secure bool) http.RoundTripper {
	tlsConfig := &tls.Config{
		// Avoid fallback to SSL protocols < TLS1.0
		MinVersion:         tls.VersionTLS10,
		InsecureSkipVerify: !secure,
		CipherSuites:       tlsconfig.DefaultServerAcceptedCiphers,
	}

	tr := &http.Transport{
		DisableKeepAlives: true,
		Proxy:             http.ProxyFromEnvironment,
		TLSClientConfig:   tlsConfig,
	}

	switch timeout {
	case ConnectTimeout:
		tr.Dial = func(proto string, addr string) (net.Conn, error) {
			// Set the connect timeout to 30 seconds to allow for slower connection
			// times...
			d := net.Dialer{Timeout: 30 * time.Second, DualStack: true}

			conn, err := d.Dial(proto, addr)
			if err != nil {
				return nil, err
			}
			// Set the recv timeout to 10 seconds
			conn.SetDeadline(time.Now().Add(10 * time.Second))
			return conn, nil
		}
	case ReceiveTimeout:
		tr.Dial = func(proto string, addr string) (net.Conn, error) {
			d := net.Dialer{DualStack: true}

			conn, err := d.Dial(proto, addr)
			if err != nil {
				return nil, err
			}
			conn = timeoutconn.New(conn, 1*time.Minute)
			return conn, nil
		}
	}

	if secure {
		// note: httpsTransport also handles http transport
		// but for HTTPS, it sets up the certs
		return transport.NewTransport(tr, &httpsRequestModifier{tlsConfig: tlsConfig})
	}

	return tr
}

// DockerHeaders returns request modifiers that ensure requests have
// the User-Agent header set to dockerUserAgent and that metaHeaders
// are added.
func DockerHeaders(metaHeaders http.Header) []transport.RequestModifier {
	modifiers := []transport.RequestModifier{
		transport.NewHeaderRequestModifier(http.Header{"User-Agent": []string{dockerUserAgent}}),
	}
	if metaHeaders != nil {
		modifiers = append(modifiers, transport.NewHeaderRequestModifier(metaHeaders))
	}
	return modifiers
}

func HTTPClient(transport http.RoundTripper) *http.Client {
	if transport == nil {
		transport = NewTransport(ConnectTimeout, true)
	}

	return &http.Client{
		Transport:     transport,
		CheckRedirect: AddRequiredHeadersToRedirectedRequests,
	}
}

func trustedLocation(req *http.Request) bool {
	var (
		trusteds = []string{"docker.com", "docker.io"}
		hostname = strings.SplitN(req.Host, ":", 2)[0]
	)
	if req.URL.Scheme != "https" {
		return false
	}

	for _, trusted := range trusteds {
		if hostname == trusted || strings.HasSuffix(hostname, "."+trusted) {
			return true
		}
	}
	return false
}

func AddRequiredHeadersToRedirectedRequests(req *http.Request, via []*http.Request) error {
	if via != nil && via[0] != nil {
		if trustedLocation(req) && trustedLocation(via[0]) {
			req.Header = via[0].Header
			return nil
		}
		for k, v := range via[0].Header {
			if k != "Authorization" {
				for _, vv := range v {
					req.Header.Add(k, vv)
				}
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package regression

/// SimpleLinear is a simple linear regression model.
type SimpleLinear struct {
	α float64
	β float64
}

/// NewSimpleLinear fits a simple linear regression model.
func NewSimpleLinear(x, y []float64) *SimpleLinear {
	α := 0.0
	β := 0.0
	return &SimpleLinear{α, β}
}

/// Compute evaluate the model at a point.
func (self *SimpleLinear) Compute(x float64) float64 {
	return self.α*x + self.β
}
<commit_msg>regression: fix the comments<commit_after>package regression

// SimpleLinear is a simple linear regression model.
//
// https://en.wikipedia.org/wiki/Simple_linear_regression
type SimpleLinear struct {
	α float64
	β float64
}

// NewSimpleLinear fits a simple linear regression model.
func NewSimpleLinear(x, y []float64) *SimpleLinear {
	α := 0.0
	β := 0.0
	return &SimpleLinear{α, β}
}

// Compute evaluate the model at a point.
func (self *SimpleLinear) Compute(x float64) float64 {
	return self.α*x + self.β
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
)

const (
	episodeListFilename = "episodes.json"
)

type EpisodeMetadata struct {
	Media    string
	Image    string
	Season   int
	Episode  int
	Metadata *OmdbTVEpisode // This can be 'null' if OMDB didn't return info for this ep.
}

type BySeasonThenEpisode []EpisodeMetadata

func (a BySeasonThenEpisode) Len() int      { return len(a) }
func (a BySeasonThenEpisode) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a BySeasonThenEpisode) Less(i, j int) bool {
	return a[i].Season*1000+a[i].Episode < a[j].Season*1000+a[j].Episode
}

/// Regenerates the list of episodes for the given show path.
func generateEpisodeList(showPath string, paths Paths) {

	log.Println("Generating episode list:", showPath)

	var episodes []EpisodeMetadata

	files, _ := ioutil.ReadDir(showPath) // Assume this works.
	for _, fileInfo := range files {
		if fileInfo.IsDir() {
			// Parse the 'SxEy'
			_, season, episode, _ := showSeasonEpisodeFromFile(fileInfo.Name())

			// Load the metadata.
			var metadata *OmdbTVEpisode = nil
			metadataPath := filepath.Join(filepath.Join(showPath, fileInfo.Name()), metadataFilename)
			metadataData, metadataErr := ioutil.ReadFile(metadataPath)
			if metadataErr == nil {
				var m OmdbTVEpisode
				if err := json.Unmarshal(metadataData, &m); err == nil {
					metadata = &m
				}
			}

			epPath := filepath.Join(showPath, fileInfo.Name())
			mediaPath, _ := filepath.Rel(paths.Root, filepath.Join(epPath, hlsFilename))
			imagePath, _ := filepath.Rel(paths.Root, filepath.Join(epPath, imageFilename))

			ep := EpisodeMetadata{
				Media:    mediaPath,
				Image:    imagePath,
				Season:   season,
				Episode:  episode,
				Metadata: metadata,
			}
			episodes = append(episodes, ep)
		}
	}

	// Sort
	sort.Sort(BySeasonThenEpisode(episodes))

	// Save.
	data, _ := json.Marshal(episodes)
	outPath := filepath.Join(showPath, episodeListFilename)
	ioutil.WriteFile(outPath, data, os.ModePerm)

	log.Println("Successfully generated episode list")
}

TODO make index.html with links to each ep

type ShowMetadata struct {
	Image    string
	Metadata interface{}
	Episodes []interface{}
}

type ByName []ShowMetadata

func (a ByName) Len() int           { return len(a) }
func (a ByName) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByName) Less(i, j int) bool { return a[i].Image < a[j].Image }

/// Regenerates the list of all tv shows.
func generateShowList(paths Paths) {

	log.Println("Generating show list")

	var shows []ShowMetadata

	files, _ := ioutil.ReadDir(paths.TV) // Assume this works.
	for _, fileInfo := range files {
		if fileInfo.IsDir() {

			showPath := filepath.Join(paths.TV, fileInfo.Name())

			// Load the metadata.
			metadataData, metadataErr := ioutil.ReadFile(filepath.Join(showPath, metadataFilename))
			if metadataErr != nil {
				continue
			}
			var metadata interface{}
			if err := json.Unmarshal(metadataData, &metadata); err != nil {
				continue
			}

			// Load the episodes.
			episodesData, episodesErr := ioutil.ReadFile(filepath.Join(showPath, episodeListFilename))
			if episodesErr != nil {
				continue
			}
			var episodes []interface{}
			if err := json.Unmarshal(episodesData, &episodes); err != nil {
				continue
			}

			imagePath, _ := filepath.Rel(paths.Root, filepath.Join(showPath, imageFilename))
			s := ShowMetadata{
				Image:    imagePath,
				Metadata: metadata,
				Episodes: episodes,
			}
			shows = append(shows, s)
		}
	}

	sort.Sort(ByName(shows))

	// Save.
	data, _ := json.MarshalIndent(shows, "", "    ")
	outPath := filepath.Join(paths.TV, metadataFilename)
	ioutil.WriteFile(outPath, data, os.ModePerm)
}
<commit_msg>HTML for show<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
)

const (
	episodeListFilename = "episodes.json"
)

type EpisodeMetadata struct {
	Media    string
	Image    string
	Season   int
	Episode  int
	Metadata *OmdbTVEpisode // This can be 'null' if OMDB didn't return info for this ep.
}

type BySeasonThenEpisode []EpisodeMetadata

func (a BySeasonThenEpisode) Len() int      { return len(a) }
func (a BySeasonThenEpisode) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a BySeasonThenEpisode) Less(i, j int) bool {
	return a[i].Season*1000+a[i].Episode < a[j].Season*1000+a[j].Episode
}

/// Regenerates the list of episodes for the given show path.
func generateEpisodeList(showPath string, paths Paths) {

	log.Println("Generating episode list:", showPath)

	var episodes []EpisodeMetadata

	files, _ := ioutil.ReadDir(showPath) // Assume this works.
	for _, fileInfo := range files {
		if fileInfo.IsDir() {
			// Parse the 'SxEy'
			_, season, episode, _ := showSeasonEpisodeFromFile(fileInfo.Name())

			// Load the metadata.
			var metadata *OmdbTVEpisode = nil
			metadataPath := filepath.Join(filepath.Join(showPath, fileInfo.Name()), metadataFilename)
			metadataData, metadataErr := ioutil.ReadFile(metadataPath)
			if metadataErr == nil {
				var m OmdbTVEpisode
				if err := json.Unmarshal(metadataData, &m); err == nil {
					metadata = &m
				}
			}

			epPath := filepath.Join(showPath, fileInfo.Name())
			mediaPath, _ := filepath.Rel(paths.Root, filepath.Join(epPath, hlsFilename))
			imagePath, _ := filepath.Rel(paths.Root, filepath.Join(epPath, imageFilename))

			ep := EpisodeMetadata{
				Media:    mediaPath,
				Image:    imagePath,
				Season:   season,
				Episode:  episode,
				Metadata: metadata,
			}
			episodes = append(episodes, ep)
		}
	}

	// Sort
	sort.Sort(BySeasonThenEpisode(episodes))

	// Save.
	data, _ := json.Marshal(episodes)
	outPath := filepath.Join(showPath, episodeListFilename)
	ioutil.WriteFile(outPath, data, os.ModePerm)

	generateEpisodeListHTML(showPath, episodes, paths)

	log.Println("Successfully generated episode list")
}

func generateEpisodeListHTML(showPath string, episodes []EpisodeMetadata, paths Paths) {
	html := htmlStart
	isLeft := true
	trOpen := false

	for _, episode := range episodes {
		if isLeft {
			html += "
"
			trOpen = true
		}

		linkPath, _ := filepath.Rel(showPath, filepath.Join(paths.Root, episode.Media))
		imagePath, _ := filepath.Rel(showPath, filepath.Join(paths.Root, episode.Image))
		name := fmt.Sprintf("S%02d E%02d", episode.Season, episode.Episode)
		if episode.Metadata != nil {
			name = name + " " + episode.Metadata.Title
		}

		h := htmlTd
		h = strings.Replace(h, "LINK", linkPath, -1)
		h = strings.Replace(h, "IMAGE", imagePath, -1)
		h = strings.Replace(h, "NAME", name, -1)
		html += h

		if !isLeft {
			html += "
"
			trOpen = false
		}

		isLeft = !isLeft
	}

	if trOpen {
		html += "
"
	}
	html += htmlEnd

	// Save.
	outPath := filepath.Join(showPath, indexHtml)
	ioutil.WriteFile(outPath, []byte(html), os.ModePerm)
}

type ShowMetadata struct {
	Image    string
	Metadata interface{}
	Episodes []interface{}
}

type ByName []ShowMetadata

func (a ByName) Len() int           { return len(a) }
func (a ByName) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByName) Less(i, j int) bool { return a[i].Image < a[j].Image }

/// Regenerates the list of all tv shows.
func generateShowList(paths Paths) {

	log.Println("Generating show list")

	var shows []ShowMetadata

	files, _ := ioutil.ReadDir(paths.TV) // Assume this works.
	for _, fileInfo := range files {
		if fileInfo.IsDir() {

			showPath := filepath.Join(paths.TV, fileInfo.Name())

			// Load the metadata.
			metadataData, metadataErr := ioutil.ReadFile(filepath.Join(showPath, metadataFilename))
			if metadataErr != nil {
				continue
			}
			var metadata interface{}
			if err := json.Unmarshal(metadataData, &metadata); err != nil {
				continue
			}

			// Load the episodes.
			episodesData, episodesErr := ioutil.ReadFile(filepath.Join(showPath, episodeListFilename))
			if episodesErr != nil {
				continue
			}
			var episodes []interface{}
			if err := json.Unmarshal(episodesData, &episodes); err != nil {
				continue
			}

			imagePath, _ := filepath.Rel(paths.Root, filepath.Join(showPath, imageFilename))
			s := ShowMetadata{
				Image:    imagePath,
				Metadata: metadata,
				Episodes: episodes,
			}
			shows = append(shows, s)
		}
	}

	sort.Sort(ByName(shows))

	// Save.
	data, _ := json.MarshalIndent(shows, "", "    ")
	outPath := filepath.Join(paths.TV, metadataFilename)
	ioutil.WriteFile(outPath, data, os.ModePerm)
}
<|endoftext|>


<commit_before>// Copyright 2014 SteelSeries ApS.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package implements a basic LISP interpretor for embedding in a go program for scripting.
// This file implements the frame data type.

package golisp

import (
	"gopkg.in/fatih/set.v0"
	"strings"
)

type FrameMap map[string]*Data

func (self *FrameMap) hasSlotLocally(key string) bool {
	_, ok := (*self)[key]
	return ok
}

func (self *FrameMap) localSlots() []string {
	slots := make([]string, 0, len(*self))
	for k, _ := range *self {
		slots = append(slots, k)
	}
	return slots
}

func isParentKey(key string) bool {
	return strings.HasSuffix(key, "*:")
}

func (self *FrameMap) hasParentSlots() bool {
	for k, _ := range *self {
		if isParentKey(k) {
			return true
		}
	}

	return false
}

func (self *FrameMap) parentSlots() []string {
	slots := make([]string, 0, len(*self))
	for k, _ := range *self {
		if isParentKey(k) {
			slots = append(slots, k)
		}
	}
	return slots
}

func (self *FrameMap) Parents() []*FrameMap {
	parents := make([]*FrameMap, 0, 0)
	for k, v := range *self {
		if isParentKey(k) {
			parents = append(parents, FrameValue(v))
		}
	}
	return parents
}

func toSet(items []string) *set.Set {
	s := set.New()
	for _, i := range items {
		s.Add(i)
	}
	return s
}

func (self *FrameMap) inheritedValueSlots() []string {
	parentFrames := self.Parents()
	inheritedSlots := make([][]string, len(parentFrames))
	for _, p := range parentFrames {
		inheritedSlots = append(inheritedSlots, p.inheritedValueSlots())
	}

	slots := set.New()
	for _, s := range self.localSlots() {
		if !isParentKey(s) {
			slots.Add(s)
		}
	}

	for _, is := range inheritedSlots {
		slots.Merge(toSet(is))
	}

	return set.StringSlice(slots)
}

func (self *FrameMap) HasSlot(key string) bool {
	if self.hasSlotLocally(key) {
		return true
	}

	if !self.hasParentSlots() {
		return false
	}

	for _, p := range self.Parents() {
		if p.HasSlot(key) {
			return true
		}
	}

	return false
}

func (self *FrameMap) Get(key string) *Data {
	v, ok := (*self)[key]
	if ok {
		return v
	}

	for _, p := range self.Parents() {
		v := p.Get(key)
		if v != nil {
			return v
		}
	}

	return nil
}

func (self *FrameMap) Remove(key string) bool {
	if !self.hasSlotLocally(key) {
		return false
	}
	delete(*self, key)
	return true
}

func (self *FrameMap) Set(key string, value *Data) *Data {
	if !self.HasSlot(key) || self.hasSlotLocally(key) {
		(*self)[key] = value
		return value
	}

	for _, p := range self.Parents() {
		v := p.Set(key, value)
		if v != nil {
			return v
		}
	}
	return nil
}

func (self *FrameMap) Clone() *FrameMap {
	f := make(FrameMap)
	for k, v := range *self {
		f[k] = v
	}
	return &f
}
<commit_msg>Remove unused code, add protection against circular parent graphs and diamond inheritance.<commit_after>// Copyright 2014 SteelSeries ApS.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package implements a basic LISP interpretor for embedding in a go program for scripting.
// This file implements the frame data type.

package golisp

import (
	"gopkg.in/fatih/set.v0"
	"strings"
)

type FrameMap map[string]*Data

func (self *FrameMap) hasSlotLocally(key string) bool {
	_, ok := (*self)[key]
	return ok
}

func (self *FrameMap) localSlots() []string {
	slots := make([]string, 0, len(*self))
	for k, _ := range *self {
		slots = append(slots, k)
	}
	return slots
}

func isParentKey(key string) bool {
	return strings.HasSuffix(key, "*:")
}

func (self *FrameMap) hasParentSlots() bool {
	for k, _ := range *self {
		if isParentKey(k) {
			return true
		}
	}

	return false
}

func (self *FrameMap) parentSlots() []string {
	slots := make([]string, 0, len(*self))
	for k, _ := range *self {
		if isParentKey(k) {
			slots = append(slots, k)
		}
	}
	return slots
}

func (self *FrameMap) Parents() []*FrameMap {
	parents := make([]*FrameMap, 0, 0)
	for k, v := range *self {
		if isParentKey(k) {
			parents = append(parents, FrameValue(v))
		}
	}
	return parents
}

//------------------------------------------------------------

func (self *FrameMap) hasSlotHelper(key string, v *set.Set) bool {
	if v.Has(self) {
		return false
	}

	v.Add(self)

	if self.hasSlotLocally(key) {
		return true
	}

	if !self.hasParentSlots() {
		return false
	}

	for _, p := range self.Parents() {
		if p.hasSlotHelper(key, v) {
			return true
		}
	}

	return false
}

func (self *FrameMap) HasSlot(key string) bool {
	visited := set.New()
	return self.hasSlotHelper(key, visited)
}

//------------------------------------------------------------

func (self *FrameMap) getHelper(key string, v *set.Set) *Data {
	if v.Has(self) {
		return nil
	}

	v.Add(self)

	val, ok := (*self)[key]
	if ok {
		return val
	}

	for _, p := range self.Parents() {
		val := p.getHelper(key, v)
		if val != nil {
			return val
		}
	}

	return nil
}

func (self *FrameMap) Get(key string) *Data {
	visited := set.New()
	return self.getHelper(key, visited)
}

//------------------------------------------------------------

func (self *FrameMap) Remove(key string) bool {
	if !self.hasSlotLocally(key) {
		return false
	}
	delete(*self, key)
	return true
}

//------------------------------------------------------------

func (self *FrameMap) setHelper(key string, value *Data, v *set.Set) *Data {
	if v.Has(self) {
		return nil
	}

	v.Add(self)

	if !self.HasSlot(key) || self.hasSlotLocally(key) {
		(*self)[key] = value
		return value
	}

	for _, p := range self.Parents() {
		v := p.Set(key, value)
		if v != nil {
			return v
		}
	}
	return nil
}

func (self *FrameMap) Set(key string, value *Data) *Data {
	visited := set.New()
	return self.setHelper(key, value, visited)
}

//------------------------------------------------------------

func (self *FrameMap) Clone() *FrameMap {
	f := make(FrameMap)
	for k, v := range *self {
		f[k] = v
	}
	return &f
}
<|endoftext|>


<commit_before>package minion

import (
	"bytes"
	"encoding/json"
	"log"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/dnaeon/gru/classifier"
	"github.com/dnaeon/gru/task"
	"github.com/dnaeon/gru/utils"

	"code.google.com/p/go-uuid/uuid"
	etcdclient "github.com/coreos/etcd/client"
	"golang.org/x/net/context"
)

// Minions keyspace in etcd
const EtcdMinionSpace = "/gru/minion"

// Etcd Minion
type etcdMinion struct {
	// Name of this minion
	name string

	// Minion root node in etcd
	rootDir string

	// Minion queue node in etcd
	queueDir string

	// Log directory to keep previously executed tasks
	logDir string

	// Root node for classifiers in etcd
	classifierDir string

	// Minion unique identifier
	id uuid.UUID

	// KeysAPI client to etcd
	kapi etcdclient.KeysAPI
}

// Creates a new etcd minion
func NewEtcdMinion(name string, cfg etcdclient.Config) Minion {
	c, err := etcdclient.New(cfg)
	if err != nil {
		log.Fatal(err)
	}

	kapi := etcdclient.NewKeysAPI(c)
	id := utils.GenerateUUID(name)
	rootDir := filepath.Join(EtcdMinionSpace, id.String())
	queueDir := filepath.Join(rootDir, "queue")
	classifierDir := filepath.Join(rootDir, "classifier")
	logDir := filepath.Join(rootDir, "log")

	m := &etcdMinion{
		name:          name,
		rootDir:       rootDir,
		queueDir:      queueDir,
		classifierDir: classifierDir,
		logDir:        logDir,
		id:            id,
		kapi:          kapi,
	}

	return m
}

// Set the human-readable name of the minion in etcd
func (m *etcdMinion) setName() error {
	nameKey := filepath.Join(m.rootDir, "name")
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
	}

	_, err := m.kapi.Set(context.Background(), nameKey, m.Name(), opts)

	return err
}

// Set the time the minion was last seen in seconds since the Epoch
func (m *etcdMinion) setLastseen(s int64) error {
	lastseenKey := filepath.Join(m.rootDir, "lastseen")
	lastseenValue := strconv.FormatInt(s, 10)
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
	}

	_, err := m.kapi.Set(context.Background(), lastseenKey, lastseenValue, opts)

	return err
}

// Checks for any pending tasks in queue
func (m *etcdMinion) checkQueue(c chan<- *task.Task) error {
	opts := &etcdclient.GetOptions{
		Recursive: true,
		Sort:      true,
	}

	// Get backlog tasks if any
	resp, err := m.kapi.Get(context.Background(), m.queueDir, opts)
	if err != nil {
		return nil
	}

	backlog := resp.Node.Nodes
	if len(backlog) == 0 {
		// No backlog tasks found
		return nil
	}

	log.Printf("Found %d tasks in queue", len(backlog))
	for _, node := range backlog {
		task, err := EtcdUnmarshalTask(node)
		m.kapi.Delete(context.Background(), node.Key, nil)

		if err != nil {
			continue
		}

		c <- task
	}

	return nil
}

// Runs periodic jobs such as refreshing classifiers and
// updating the lastseen time
func (m *etcdMinion) periodicRunner(ticker *time.Ticker) error {
	for {
		// Update minion classifiers
		m.Classify()

		// Update lastseen time
		now := time.Now().Unix()
		err := m.setLastseen(now)
		if err != nil {
			log.Printf("Failed to update lastseen time: %s\n", err)
		}

		<-ticker.C
	}

	return nil
}

// Processes new tasks
func (m *etcdMinion) processTask(t *task.Task) error {
	defer m.SaveTaskResult(t)

	var buf bytes.Buffer
	cmd := exec.Command(t.Command, t.Args...)
	cmd.Stdout = &buf
	cmd.Stderr = &buf

	log.Printf("Processing task %s\n", t.TaskID)

	cmdError := cmd.Run()
	t.TimeProcessed = time.Now().Unix()
	t.Result = buf.String()

	if cmdError != nil {
		log.Printf("Failed to process task %s\n", t.TaskID)
		t.Error = cmdError.Error()
	} else {
		log.Printf("Finished processing task %s\n", t.TaskID)
	}

	return cmdError
}

// Saves the task result
func (m *etcdMinion) SaveTaskResult(t *task.Task) error {
	// Task key in etcd
	taskKey := filepath.Join(m.logDir, t.TaskID.String())

	// Serialize task to JSON
	data, err := json.Marshal(t)
	if err != nil {
		log.Printf("Failed to serialize task %s: %s\n", t.TaskID, err)
		return err
	}

	// Save the task result in etcd
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
	}

	_, err = m.kapi.Set(context.Background(), taskKey, string(data), opts)
	if err != nil {
		log.Printf("Failed to save task %s: %s\n", t.TaskID, err)
	}

	return err
}

// Unmarshals task from etcd
func EtcdUnmarshalTask(node *etcdclient.Node) (*task.Task, error) {
	task := new(task.Task)
	err := json.Unmarshal([]byte(node.Value), &task)

	if err != nil {
		log.Printf("Invalid task %s: %s\n", node.Key, err)
	}

	return task, err
}

// Returns the minion unique identifier
func (m *etcdMinion) ID() uuid.UUID {
	return m.id
}

// Returns the assigned name of the minion
func (m *etcdMinion) Name() string {
	return m.name
}

// Classifies the minion
func (m *etcdMinion) Classify() error {
	// Classifiers in etcd expire after an hour
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
		TTL:       time.Hour,
	}

	// Update classifiers
	for key, _ := range classifier.Registry {
		klassifier, err := classifier.Get(key)

		if err != nil {
			continue
		}

		// Serialize classifier to JSON and save it in etcd
		data, err := json.Marshal(klassifier)
		if err != nil {
			log.Printf("Failed to serialize classifier: %s\n", key)
			continue
		}

		// Classifier key in etcd
		klassifierKey := filepath.Join(m.classifierDir, key)
		_, err = m.kapi.Set(context.Background(), klassifierKey, string(data), opts)

		if err != nil {
			log.Printf("Failed to set classifier %s: %s\n", key, err)
		}
	}

	return nil
}

// Monitors etcd for new tasks
func (m *etcdMinion) TaskListener(c chan<- *task.Task) error {
	watcherOpts := &etcdclient.WatcherOptions{
		Recursive: true,
	}
	watcher := m.kapi.Watcher(m.queueDir, watcherOpts)

	for {
		resp, err := watcher.Next(context.Background())
		if err != nil {
			log.Printf("Failed to read task: %s\n", err)
			continue
		}

		// Ignore "delete" events when removing a task from the queue
		action := strings.ToLower(resp.Action)
		if strings.EqualFold(action, "delete") {
			continue
		}

		// Remove task from the queue
		task, err := EtcdUnmarshalTask(resp.Node)
		m.kapi.Delete(context.Background(), resp.Node.Key, nil)

		if err != nil {
			continue
		}

		log.Printf("Received task %s\n", task.TaskID)

		c <- task
	}

	return nil
}

// Processes new tasks
func (m *etcdMinion) TaskRunner(c <-chan *task.Task) error {
	for {
		task := <-c

		task.TimeReceived = time.Now().Unix()

		if task.IsConcurrent {
			go m.processTask(task)
		} else {
			m.processTask(task)
		}
	}

	return nil
}

// Main entry point of the minion
func (m *etcdMinion) Serve() error {
	// Channel on which we send the quit signal
	quit := make(chan os.Signal, 1)
	signal.Notify(quit, os.Interrupt)

	m.setName()
	log.Printf("Minion %s is ready to serve", m.ID())

	// Run periodic tasks every fifteen minutes
	ticker := time.NewTicker(time.Minute * 15)
	go m.periodicRunner(ticker)

	// Check for pending tasks in queue first
	tasks := make(chan *task.Task)
	go m.TaskRunner(tasks)
	m.checkQueue(tasks)

	go m.TaskListener(tasks)

	// Block until a stop signal is received
	s := <-quit
	log.Printf("Received %s signal, shutting down", s)
	close(quit)
	close(tasks)

	return nil
}
<commit_msg>Log an error when a task fails to be unmarshaled<commit_after>package minion

import (
	"bytes"
	"encoding/json"
	"log"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/dnaeon/gru/classifier"
	"github.com/dnaeon/gru/task"
	"github.com/dnaeon/gru/utils"

	"code.google.com/p/go-uuid/uuid"
	etcdclient "github.com/coreos/etcd/client"
	"golang.org/x/net/context"
)

// Minions keyspace in etcd
const EtcdMinionSpace = "/gru/minion"

// Etcd Minion
type etcdMinion struct {
	// Name of this minion
	name string

	// Minion root node in etcd
	rootDir string

	// Minion queue node in etcd
	queueDir string

	// Log directory to keep previously executed tasks
	logDir string

	// Root node for classifiers in etcd
	classifierDir string

	// Minion unique identifier
	id uuid.UUID

	// KeysAPI client to etcd
	kapi etcdclient.KeysAPI
}

// Creates a new etcd minion
func NewEtcdMinion(name string, cfg etcdclient.Config) Minion {
	c, err := etcdclient.New(cfg)
	if err != nil {
		log.Fatal(err)
	}

	kapi := etcdclient.NewKeysAPI(c)
	id := utils.GenerateUUID(name)
	rootDir := filepath.Join(EtcdMinionSpace, id.String())
	queueDir := filepath.Join(rootDir, "queue")
	classifierDir := filepath.Join(rootDir, "classifier")
	logDir := filepath.Join(rootDir, "log")

	m := &etcdMinion{
		name:          name,
		rootDir:       rootDir,
		queueDir:      queueDir,
		classifierDir: classifierDir,
		logDir:        logDir,
		id:            id,
		kapi:          kapi,
	}

	return m
}

// Set the human-readable name of the minion in etcd
func (m *etcdMinion) setName() error {
	nameKey := filepath.Join(m.rootDir, "name")
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
	}

	_, err := m.kapi.Set(context.Background(), nameKey, m.Name(), opts)

	return err
}

// Set the time the minion was last seen in seconds since the Epoch
func (m *etcdMinion) setLastseen(s int64) error {
	lastseenKey := filepath.Join(m.rootDir, "lastseen")
	lastseenValue := strconv.FormatInt(s, 10)
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
	}

	_, err := m.kapi.Set(context.Background(), lastseenKey, lastseenValue, opts)

	return err
}

// Checks for any pending tasks in queue
func (m *etcdMinion) checkQueue(c chan<- *task.Task) error {
	opts := &etcdclient.GetOptions{
		Recursive: true,
		Sort:      true,
	}

	// Get backlog tasks if any
	resp, err := m.kapi.Get(context.Background(), m.queueDir, opts)
	if err != nil {
		return nil
	}

	backlog := resp.Node.Nodes
	if len(backlog) == 0 {
		// No backlog tasks found
		return nil
	}

	log.Printf("Found %d tasks in queue", len(backlog))
	for _, node := range backlog {
		task, err := EtcdUnmarshalTask(node)
		m.kapi.Delete(context.Background(), node.Key, nil)

		if err != nil {
			continue
		}

		c <- task
	}

	return nil
}

// Runs periodic jobs such as refreshing classifiers and
// updating the lastseen time
func (m *etcdMinion) periodicRunner(ticker *time.Ticker) error {
	for {
		// Update minion classifiers
		m.Classify()

		// Update lastseen time
		now := time.Now().Unix()
		err := m.setLastseen(now)
		if err != nil {
			log.Printf("Failed to update lastseen time: %s\n", err)
		}

		<-ticker.C
	}

	return nil
}

// Processes new tasks
func (m *etcdMinion) processTask(t *task.Task) error {
	defer m.SaveTaskResult(t)

	var buf bytes.Buffer
	cmd := exec.Command(t.Command, t.Args...)
	cmd.Stdout = &buf
	cmd.Stderr = &buf

	log.Printf("Processing task %s\n", t.TaskID)

	cmdError := cmd.Run()
	t.TimeProcessed = time.Now().Unix()
	t.Result = buf.String()

	if cmdError != nil {
		log.Printf("Failed to process task %s\n", t.TaskID)
		t.Error = cmdError.Error()
	} else {
		log.Printf("Finished processing task %s\n", t.TaskID)
	}

	return cmdError
}

// Saves the task result
func (m *etcdMinion) SaveTaskResult(t *task.Task) error {
	// Task key in etcd
	taskKey := filepath.Join(m.logDir, t.TaskID.String())

	// Serialize task to JSON
	data, err := json.Marshal(t)
	if err != nil {
		log.Printf("Failed to serialize task %s: %s\n", t.TaskID, err)
		return err
	}

	// Save the task result in etcd
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
	}

	_, err = m.kapi.Set(context.Background(), taskKey, string(data), opts)
	if err != nil {
		log.Printf("Failed to save task %s: %s\n", t.TaskID, err)
	}

	return err
}

// Unmarshals task from etcd
func EtcdUnmarshalTask(node *etcdclient.Node) (*task.Task, error) {
	task := new(task.Task)
	err := json.Unmarshal([]byte(node.Value), &task)

	return task, err
}

// Returns the minion unique identifier
func (m *etcdMinion) ID() uuid.UUID {
	return m.id
}

// Returns the assigned name of the minion
func (m *etcdMinion) Name() string {
	return m.name
}

// Classifies the minion
func (m *etcdMinion) Classify() error {
	// Classifiers in etcd expire after an hour
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
		TTL:       time.Hour,
	}

	// Update classifiers
	for key, _ := range classifier.Registry {
		klassifier, err := classifier.Get(key)

		if err != nil {
			continue
		}

		// Serialize classifier to JSON and save it in etcd
		data, err := json.Marshal(klassifier)
		if err != nil {
			log.Printf("Failed to serialize classifier: %s\n", key)
			continue
		}

		// Classifier key in etcd
		klassifierKey := filepath.Join(m.classifierDir, key)
		_, err = m.kapi.Set(context.Background(), klassifierKey, string(data), opts)

		if err != nil {
			log.Printf("Failed to set classifier %s: %s\n", key, err)
		}
	}

	return nil
}

// Monitors etcd for new tasks
func (m *etcdMinion) TaskListener(c chan<- *task.Task) error {
	watcherOpts := &etcdclient.WatcherOptions{
		Recursive: true,
	}
	watcher := m.kapi.Watcher(m.queueDir, watcherOpts)

	for {
		resp, err := watcher.Next(context.Background())
		if err != nil {
			log.Printf("Failed to read task: %s\n", err)
			continue
		}

		// Ignore "delete" events when removing a task from the queue
		action := strings.ToLower(resp.Action)
		if strings.EqualFold(action, "delete") {
			continue
		}

		// Remove task from the queue
		task, err := EtcdUnmarshalTask(resp.Node)
		m.kapi.Delete(context.Background(), resp.Node.Key, nil)

		if err != nil {
			log.Printf("Invalid task %s: %s\n", resp.Node.Key, err)
			continue
		}

		log.Printf("Received task %s\n", task.TaskID)

		c <- task
	}

	return nil
}

// Processes new tasks
func (m *etcdMinion) TaskRunner(c <-chan *task.Task) error {
	for {
		task := <-c

		task.TimeReceived = time.Now().Unix()

		if task.IsConcurrent {
			go m.processTask(task)
		} else {
			m.processTask(task)
		}
	}

	return nil
}

// Main entry point of the minion
func (m *etcdMinion) Serve() error {
	// Channel on which we send the quit signal
	quit := make(chan os.Signal, 1)
	signal.Notify(quit, os.Interrupt)

	m.setName()
	log.Printf("Minion %s is ready to serve", m.ID())

	// Run periodic tasks every fifteen minutes
	ticker := time.NewTicker(time.Minute * 15)
	go m.periodicRunner(ticker)

	// Check for pending tasks in queue first
	tasks := make(chan *task.Task)
	go m.TaskRunner(tasks)
	m.checkQueue(tasks)

	go m.TaskListener(tasks)

	// Block until a stop signal is received
	s := <-quit
	log.Printf("Received %s signal, shutting down", s)
	close(quit)
	close(tasks)

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Jean Niklas L'orange.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package edn

import (
	"bytes"
	"unicode/utf8"
)

func newline(dst *bytes.Buffer, prefix, indent string, depth int) {
	dst.WriteByte('\n')
	dst.WriteString(prefix)
	for i := 0; i < depth; i++ {
		dst.WriteString(indent)
	}
}

// Indent appends to dst an indented form of the EDN-encoded src. Each EDN
// collection begins on a new, indented line beginning with prefix followed by
// one or more copies of indent according to the indentation nesting. The data
// appended to dst does not begin with the prefix nor any indentation, and has
// no trailing newline, to make it easier to embed inside other formatted EDN
// data.
//
// Indent filters away whitespace, including comments and discards.
func Indent(dst *bytes.Buffer, src []byte, prefix, indent string) error {
	origLen := dst.Len()
	var lex lexer
	lex.reset()
	tokStack := newTokenStack()
	curType := tokenError
	curSize := 0
	d := NewDecoder(bytes.NewBuffer(src))
	depth := 0
	for {
		bs, tt, err := d.nextToken()
		if err != nil {
			dst.Truncate(origLen)
			return err
		}
		err = tokStack.push(tt)
		if err != nil {
			dst.Truncate(origLen)
			return err
		}
		prevType := curType
		prevSize := curSize
		if len(tokStack.toks) > 0 {
			curType = tokStack.peek()
			curSize = tokStack.peekCount()
		}
		switch tt {
		case tokenMapStart, tokenVectorStart, tokenListStart, tokenSetStart:
			if prevType == tokenMapStart {
				dst.WriteByte(' ')
			} else if depth > 0 {
				newline(dst, prefix, indent, depth)
			}
			dst.Write(bs)
			depth++
		case tokenVectorEnd, tokenListEnd, tokenMapEnd: // tokenSetEnd == tokenMapEnd
			depth--
			if prevSize > 0 { // suppress indent for empty collections
				newline(dst, prefix, indent, depth)
			}
			// all of these are of length 1 in bytes, so utilise this for perf
			dst.WriteByte(bs[0])
		case tokenTag:
			// need to know what the previous type was.
			switch prevType {
			case tokenMapStart:
				if prevSize%2 == 0 { // If previous size modulo 2 is equal to 0, we're a key
					if prevSize > 0 {
						dst.WriteByte(',')
					}
					newline(dst, prefix, indent, depth)
				} else { // We're a value, add a space after the key
					dst.WriteByte(' ')
				}
				dst.Write(bs)
				dst.WriteByte(' ')
			case tokenSetStart, tokenVectorStart, tokenListStart:
				newline(dst, prefix, indent, depth)
				dst.Write(bs)
				dst.WriteByte(' ')
			default: // tokenError or nested tag
				dst.Write(bs)
				dst.WriteByte(' ')
			}
		default:
			switch prevType {
			case tokenMapStart:
				if prevSize%2 == 0 { // If previous size modulo 2 is equal to 0, we're a key
					if prevSize > 0 {
						dst.WriteByte(',')
					}
					newline(dst, prefix, indent, depth)
				} else { // We're a value, add a space after the key
					dst.WriteByte(' ')
				}
				dst.Write(bs)
			case tokenSetStart, tokenVectorStart, tokenListStart:
				newline(dst, prefix, indent, depth)
				dst.Write(bs)
			default: // toplevel or nested tag. This should collapse the whole tag tower
				dst.Write(bs)
			}
		}
		if tokStack.done() {
			break
		}
	}
	return nil
}

// PPrintOpts is a configuration map for PPrint. The values in this struct has
// no effect as of now.
type PPrintOpts struct {
	RightMargin int
	MiserWidth  int
}

func pprintIndent(dst *bytes.Buffer, shift int) {
	dst.WriteByte('\n')
	for i := 0; i < shift; i++ {
		dst.WriteByte(' ') // TODO: This may be slower than caching the size as a byte slice
	}
}

// PPrint appends to dst an indented form of the EDN-encoded src. This
// implementation attempts to write idiomatic/readable EDN values, in a fashion
// close to (but not quite equal to) clojure.pprint/pprint.
//
// PPrint filters away whitespace, including comments and discards.
func PPrint(dst *bytes.Buffer, src []byte, opt *PPrintOpts) error {
	origLen := dst.Len()
	var lex lexer
	lex.reset()
	tokStack := newTokenStack()
	shift := []int{0}
	col := 0
	prevColl := false
	prevCollStart := 0
	curType := tokenError
	curSize := 0
	d := NewDecoder(bytes.NewBuffer(src))
	for {
		bs, tt, err := d.nextToken()
		if err != nil {
			dst.Truncate(origLen)
			return err
		}
		err = tokStack.push(tt)
		if err != nil {
			dst.Truncate(origLen)
			return err
		}
		prevType := curType
		prevSize := curSize
		if len(tokStack.toks) > 0 {
			curType = tokStack.peek()
			curSize = tokStack.peekCount()
		}
		// Indentation
		switch tt {
		case tokenVectorEnd, tokenListEnd, tokenMapEnd:
		default:
			switch prevType {
			case tokenMapStart:
				if prevSize%2 == 0 && prevSize > 0 {
					dst.WriteByte(',')
					pprintIndent(dst, shift[len(shift)-1])
					col = shift[len(shift)-1]
				} else if prevSize%2 == 1 { // We're a value, add a space after the key
					dst.WriteByte(' ')
					col++
				}
			case tokenSetStart, tokenVectorStart, tokenListStart:
				if prevColl {
					// begin on new line where prevColl started
					// This will look so strange for heterogenous maps.
					pprintIndent(dst, prevCollStart)
					col = prevCollStart
				} else if prevSize > 0 {
					dst.WriteByte(' ')
					col++
				}
			}
		}
		switch tt {
		case tokenMapStart, tokenVectorStart, tokenListStart, tokenSetStart:
			dst.Write(bs)
			col += len(bs)             // either 2 or 1
			shift = append(shift, col) // we only use maps for now, but we'll utilise this more thoroughly later on
		case tokenVectorEnd, tokenListEnd, tokenMapEnd: // tokenSetEnd == tokenMapEnd
			dst.WriteByte(bs[0]) // all of these are of length 1 in bytes, so this is ok
			prevCollStart = shift[len(shift)-1] - 1
			shift = shift[:len(shift)-1]
		case tokenTag:
			bslen := utf8.RuneCount(bs)
			dst.Write(bs)
			dst.WriteByte(' ')
			col += bslen + 1
		default:
			bslen := utf8.RuneCount(bs)
			dst.Write(bs)
			col += bslen
		}
		prevColl = (tt == tokenMapEnd || tt == tokenVectorEnd || tt == tokenListEnd)
		if tokStack.done() {
			break
		}
	}
	return nil
}
<commit_msg>pprint: Add PPrintStream<commit_after>// Copyright 2015 Jean Niklas L'orange.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package edn

import (
	"bytes"
	"io"
	"unicode/utf8"
)

var (
	// we can't call it spaceBytes not to conflict with decode.go's spaceBytes.
	spaceOutputBytes = []byte(" ")
	commaOutputBytes = []byte(",")
)

func newline(dst io.Writer, prefix, indent string, depth int) {
	dst.Write([]byte{'\n'})
	dst.Write([]byte(prefix))
	for i := 0; i < depth; i++ {
		dst.Write([]byte(indent))
	}
}

// Indent appends to dst an indented form of the EDN-encoded src. Each EDN
// collection begins on a new, indented line beginning with prefix followed by
// one or more copies of indent according to the indentation nesting. The data
// appended to dst does not begin with the prefix nor any indentation, and has
// no trailing newline, to make it easier to embed inside other formatted EDN
// data.
//
// Indent filters away whitespace, including comments and discards.
func Indent(dst *bytes.Buffer, src []byte, prefix, indent string) error {
	var lex lexer
	lex.reset()
	tokStack := newTokenStack()
	curType := tokenError
	curSize := 0
	d := NewDecoder(bytes.NewBuffer(src))
	depth := 0
	for {
		bs, tt, err := d.nextToken()
		if err != nil {
			return err
		}
		err = tokStack.push(tt)
		if err != nil {
			return err
		}
		prevType := curType
		prevSize := curSize
		if len(tokStack.toks) > 0 {
			curType = tokStack.peek()
			curSize = tokStack.peekCount()
		}
		switch tt {
		case tokenMapStart, tokenVectorStart, tokenListStart, tokenSetStart:
			if prevType == tokenMapStart {
				dst.WriteByte(' ')
			} else if depth > 0 {
				newline(dst, prefix, indent, depth)
			}
			dst.Write(bs)
			depth++
		case tokenVectorEnd, tokenListEnd, tokenMapEnd: // tokenSetEnd == tokenMapEnd
			depth--
			if prevSize > 0 { // suppress indent for empty collections
				newline(dst, prefix, indent, depth)
			}
			// all of these are of length 1 in bytes, so utilise this for perf
			dst.WriteByte(bs[0])
		case tokenTag:
			// need to know what the previous type was.
			switch prevType {
			case tokenMapStart:
				if prevSize%2 == 0 { // If previous size modulo 2 is equal to 0, we're a key
					if prevSize > 0 {
						dst.WriteByte(',')
					}
					newline(dst, prefix, indent, depth)
				} else { // We're a value, add a space after the key
					dst.WriteByte(' ')
				}
				dst.Write(bs)
				dst.WriteByte(' ')
			case tokenSetStart, tokenVectorStart, tokenListStart:
				newline(dst, prefix, indent, depth)
				dst.Write(bs)
				dst.WriteByte(' ')
			default: // tokenError or nested tag
				dst.Write(bs)
				dst.WriteByte(' ')
			}
		default:
			switch prevType {
			case tokenMapStart:
				if prevSize%2 == 0 { // If previous size modulo 2 is equal to 0, we're a key
					if prevSize > 0 {
						dst.WriteByte(',')
					}
					newline(dst, prefix, indent, depth)
				} else { // We're a value, add a space after the key
					dst.WriteByte(' ')
				}
				dst.Write(bs)
			case tokenSetStart, tokenVectorStart, tokenListStart:
				newline(dst, prefix, indent, depth)
				dst.Write(bs)
			default: // toplevel or nested tag. This should collapse the whole tag tower
				dst.Write(bs)
			}
		}
		if tokStack.done() {
			break
		}
	}
	return nil
}

// PPrintOpts is a configuration map for PPrint. The values in this struct has
// no effect as of now.
type PPrintOpts struct {
	RightMargin int
	MiserWidth  int
}

func pprintIndent(dst io.Writer, shift int) {
	dst.Write([]byte{'\n'})

	spaces := make([]byte, shift)

	// TODO: This may be slower than caching the size as a byte slice
	for i, _ := range spaces {
		spaces[i] = ' '
	}

	dst.Write(spaces)
}

// PPrint writes to dst an indented form of the EDN-encoded src. This
// implementation attempts to write idiomatic/readable EDN values, in a fashion
// close to (but not quite equal to) clojure.pprint/pprint.
//
// PPrint filters away whitespace, including comments and discards.
func PPrint(dst *bytes.Buffer, src []byte, opt *PPrintOpts) error {
	origLen := dst.Len()
	err := PPrintStream(dst, bytes.NewBuffer(src), opt)
	if err != nil {
		dst.Truncate(origLen)
	}
	return err
}

// PPrintStream is an implementation of PPrint for generic readers and writers
func PPrintStream(dst io.Writer, src io.Reader, opt *PPrintOpts) error {
	var lex lexer
	lex.reset()
	tokStack := newTokenStack()
	shift := []int{0}
	col := 0
	prevColl := false
	prevCollStart := 0
	curType := tokenError
	curSize := 0
	d := NewDecoder(src)

	for {
		bs, tt, err := d.nextToken()
		if err != nil {
			return err
		}
		err = tokStack.push(tt)
		if err != nil {
			return err
		}
		prevType := curType
		prevSize := curSize
		if len(tokStack.toks) > 0 {
			curType = tokStack.peek()
			curSize = tokStack.peekCount()
		}
		// Indentation
		switch tt {
		case tokenVectorEnd, tokenListEnd, tokenMapEnd:
		default:
			switch prevType {
			case tokenMapStart:
				if prevSize%2 == 0 && prevSize > 0 {
					dst.Write(commaOutputBytes)
					pprintIndent(dst, shift[len(shift)-1])
					col = shift[len(shift)-1]
				} else if prevSize%2 == 1 { // We're a value, add a space after the key
					dst.Write(spaceOutputBytes)
					col++
				}
			case tokenSetStart, tokenVectorStart, tokenListStart:
				if prevColl {
					// begin on new line where prevColl started
					// This will look so strange for heterogenous maps.
					pprintIndent(dst, prevCollStart)
					col = prevCollStart
				} else if prevSize > 0 {
					dst.Write(spaceOutputBytes)
					col++
				}
			}
		}
		switch tt {
		case tokenMapStart, tokenVectorStart, tokenListStart, tokenSetStart:
			dst.Write(bs)
			col += len(bs)             // either 2 or 1
			shift = append(shift, col) // we only use maps for now, but we'll utilise this more thoroughly later on
		case tokenVectorEnd, tokenListEnd, tokenMapEnd: // tokenSetEnd == tokenMapEnd
			dst.Write(bs[:1]) // all of these are of length 1 in bytes, so this is ok
			prevCollStart = shift[len(shift)-1] - 1
			shift = shift[:len(shift)-1]
		case tokenTag:
			bslen := utf8.RuneCount(bs)
			dst.Write(bs)
			dst.Write(spaceOutputBytes)
			col += bslen + 1
		default:
			bslen := utf8.RuneCount(bs)
			dst.Write(bs)
			col += bslen
		}
		prevColl = (tt == tokenMapEnd || tt == tokenVectorEnd || tt == tokenListEnd)
		if tokStack.done() {
			break
		}
	}
	return nil
}
<|endoftext|>