<commit_before>package main

import (
	"crypto/hmac"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"

	zmq "github.com/alecthomas/gozmq"
	uuid "github.com/nu7hatch/gouuid"
)

// MsgHeader encodes header info for ZMQ messages
type MsgHeader struct {
	MsgID    string `json:"msg_id"`
	Username string `json:"username"`
	Session  string `json:"session"`
	MsgType  string `json:"msg_type"`
}

// ComposedMsg represents an entire message in a high-level structure.
type ComposedMsg struct {
	Header       MsgHeader
	ParentHeader MsgHeader
	Metadata     map[string]interface{}
	Content      interface{}
}

// InvalidSignatureError is returned when the signature on a received message does not
// validate.
type InvalidSignatureError struct{}

func (e *InvalidSignatureError) Error() string {
	return "A message had an invalid signature"
}

// WireMsgToComposedMsg translates a multipart ZMQ messages received from a socket into
// a ComposedMsg struct and a slice of return identities. This includes verifying the
// message signature.
func WireMsgToComposedMsg(msgparts [][]byte, signkey []byte) (msg ComposedMsg,
	identities [][]byte, err error) {
	i := 0
	for string(msgparts[i]) != "<IDS|MSG>" {
		i++
	}
	identities = msgparts[:i]
	// msgparts[i] is the delimiter

	// Validate signature
	if len(signkey) != 0 {
		mac := hmac.New(sha256.New, signkey)
		for _, msgpart := range msgparts[i+2 : i+6] {
			mac.Write(msgpart)
		}
		signature := make([]byte, hex.DecodedLen(len(msgparts[i+1])))
		hex.Decode(signature, msgparts[i+1])
		if !hmac.Equal(mac.Sum(nil), signature) {
			return msg, nil, &InvalidSignatureError{}
		}
	}
	json.Unmarshal(msgparts[i+2], &msg.Header)
	json.Unmarshal(msgparts[i+3], &msg.ParentHeader)
	json.Unmarshal(msgparts[i+4], &msg.Metadata)
	json.Unmarshal(msgparts[i+5], &msg.Content)
	return
}

// ToWireMsg translates a ComposedMsg into a multipart ZMQ message ready to send, and
// signs it. This does not add the return identities or the delimiter.
func (msg ComposedMsg) ToWireMsg(signkey []byte) (msgparts [][]byte) {
	msgparts = make([][]byte, 5)
	header, _ := json.Marshal(msg.Header)
	msgparts[1] = header
	parentHeader, _ := json.Marshal(msg.ParentHeader)
	msgparts[2] = parentHeader
	if msg.Metadata == nil {
		msg.Metadata = make(map[string]interface{})
	}
	metadata, _ := json.Marshal(msg.Metadata)
	msgparts[3] = metadata
	content, _ := json.Marshal(msg.Content)
	msgparts[4] = content

	// Sign the message
	if len(signkey) != 0 {
		mac := hmac.New(sha256.New, signkey)
		for _, msgpart := range msgparts[1:] {
			mac.Write(msgpart)
		}
		msgparts[0] = make([]byte, hex.EncodedLen(mac.Size()))
		hex.Encode(msgparts[0], mac.Sum(nil))
	}
	return
}

// MsgReceipt represents a received message, its return identities, and the sockets for
// communication.
type MsgReceipt struct {
	Msg        ComposedMsg
	Identities [][]byte
	Sockets    SocketGroup
}

// SendResponse sends a message back to return identites of the received message.
func (receipt *MsgReceipt) SendResponse(socket *zmq.Socket, msg ComposedMsg) {
	socket.SendMultipart(receipt.Identities, zmq.SNDMORE)
	socket.Send([]byte("<IDS|MSG>"), zmq.SNDMORE)
	socket.SendMultipart(msg.ToWireMsg(receipt.Sockets.Key), 0)
	logger.Println("<--", msg.Header.MsgType)
	logger.Printf("%+v\n", msg.Content)
}

// NewMsg creates a new ComposedMsg to respond to a parent message. This includes setting
// up its headers.
func NewMsg(msgType string, parent ComposedMsg) (msg ComposedMsg) {
	msg.ParentHeader = parent.Header
	msg.Header.Session = parent.Header.Session
	msg.Header.Username = parent.Header.Username
	msg.Header.MsgType = msgType
	u, _ := uuid.NewV4()
	msg.Header.MsgID = u.String()
	return
}
<commit_msg>more error handling<commit_after>package main

import (
	"crypto/hmac"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"log"

	zmq "github.com/alecthomas/gozmq"
	uuid "github.com/nu7hatch/gouuid"
	"github.com/pkg/errors"
)

// MsgHeader encodes header info for ZMQ messages
type MsgHeader struct {
	MsgID    string `json:"msg_id"`
	Username string `json:"username"`
	Session  string `json:"session"`
	MsgType  string `json:"msg_type"`
}

// ComposedMsg represents an entire message in a high-level structure.
type ComposedMsg struct {
	Header       MsgHeader
	ParentHeader MsgHeader
	Metadata     map[string]interface{}
	Content      interface{}
}

// InvalidSignatureError is returned when the signature on a received message does not
// validate.
type InvalidSignatureError struct{}

func (e *InvalidSignatureError) Error() string {
	return "A message had an invalid signature"
}

// WireMsgToComposedMsg translates a multipart ZMQ messages received from a socket into
// a ComposedMsg struct and a slice of return identities. This includes verifying the
// message signature.
func WireMsgToComposedMsg(msgparts [][]byte, signkey []byte) (msg ComposedMsg,
	identities [][]byte, err error) {
	i := 0
	for string(msgparts[i]) != "<IDS|MSG>" {
		i++
	}
	identities = msgparts[:i]

	// Validate signature
	if len(signkey) != 0 {
		mac := hmac.New(sha256.New, signkey)
		for _, msgpart := range msgparts[i+2 : i+6] {
			mac.Write(msgpart)
		}
		signature := make([]byte, hex.DecodedLen(len(msgparts[i+1])))
		hex.Decode(signature, msgparts[i+1])
		if !hmac.Equal(mac.Sum(nil), signature) {
			return msg, nil, &InvalidSignatureError{}
		}
	}
	json.Unmarshal(msgparts[i+2], &msg.Header)
	json.Unmarshal(msgparts[i+3], &msg.ParentHeader)
	json.Unmarshal(msgparts[i+4], &msg.Metadata)
	json.Unmarshal(msgparts[i+5], &msg.Content)
	return
}

// ToWireMsg translates a ComposedMsg into a multipart ZMQ message ready to send, and
// signs it. This does not add the return identities or the delimiter.
func (msg ComposedMsg) ToWireMsg(signkey []byte) ([][]byte, error) {

	msgparts := make([][]byte, 5)
	header, err := json.Marshal(msg.Header)
	if err != nil {
		return msgparts, errors.Wrap(err, "Could not marshal message header")
	}
	msgparts[1] = header

	parentHeader, err := json.Marshal(msg.ParentHeader)
	if err != nil {
		return msgparts, errors.Wrap(err, "Could not marshal parent header")
	}
	msgparts[2] = parentHeader

	if msg.Metadata == nil {
		msg.Metadata = make(map[string]interface{})
	}
	metadata, err := json.Marshal(msg.Metadata)
	if err != nil {
		return msgparts, errors.Wrap(err, "Could not marshal metadata")
	}
	msgparts[3] = metadata

	content, err := json.Marshal(msg.Content)
	if err != nil {
		return msgparts, errors.Wrap(err, "Could not marshal content")
	}
	msgparts[4] = content

	// Sign the message
	if len(signkey) != 0 {
		mac := hmac.New(sha256.New, signkey)
		for _, msgpart := range msgparts[1:] {
			mac.Write(msgpart)
		}
		msgparts[0] = make([]byte, hex.EncodedLen(mac.Size()))
		hex.Encode(msgparts[0], mac.Sum(nil))
	}
	return msgparts, nil
}

// MsgReceipt represents a received message, its return identities, and the sockets for
// communication.
type MsgReceipt struct {
	Msg        ComposedMsg
	Identities [][]byte
	Sockets    SocketGroup
}

// SendResponse sends a message back to return identites of the received message.
func (receipt *MsgReceipt) SendResponse(socket *zmq.Socket, msg ComposedMsg) {

	socket.SendMultipart(receipt.Identities, zmq.SNDMORE)
	socket.Send([]byte("<IDS|MSG>"), zmq.SNDMORE)

	msgParts, err := msg.ToWireMsg(receipt.Sockets.Key)
	if err != nil {
		log.Fatalln(err)
	}
	socket.SendMultipart(msgParts, 0)
	logger.Println("<--", msg.Header.MsgType)
	logger.Printf("%+v\n", msg.Content)
}

// NewMsg creates a new ComposedMsg to respond to a parent message. This includes setting
// up its headers.
func NewMsg(msgType string, parent ComposedMsg) (msg ComposedMsg) {
	msg.ParentHeader = parent.Header
	msg.Header.Session = parent.Header.Session
	msg.Header.Username = parent.Header.Username
	msg.Header.MsgType = msgType
	u, err := uuid.NewV4()
	if err != nil {
		log.Fatalln(errors.Wrap(err, "Could not generate UUID"))
	}
	msg.Header.MsgID = u.String()
	return
}
<|endoftext|>


<commit_before>package main

import (
	"archive/zip"
	"encoding/binary"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"runtime"
	"strings"

	"github.com/blang/semver"
	"github.com/dustin/go-humanize"
	"github.com/itchio/butler/comm"
	"github.com/itchio/go-itchio"
	"github.com/itchio/wharf/pwr"
	"github.com/itchio/wharf/tlc"
	"github.com/itchio/wharf/wire"
	"github.com/kardianos/osext"
)

var updateBaseURL = fmt.Sprintf("https://dl.itch.ovh/butler/%s-%s", runtime.GOOS, runtime.GOARCH)

func which() {
	p, err := osext.Executable()
	must(err)

	comm.Logf("You're running butler %s, from the following path:", versionString)
	comm.Logf("%s", p)
}

func file(path string) {
	stats, err := os.Lstat(path)
	if os.IsNotExist(err) {
		comm.Dief("%s: no such file or directory")
	}
	must(err)

	if stats.IsDir() {
		comm.Logf("%s: directory", path)
		return
	}

	if stats.Size() == 0 {
		comm.Logf("%s: empty file. peaceful.", path)
		return
	}

	prettySize := humanize.Bytes(uint64(stats.Size()))

	reader, err := os.Open(path)
	must(err)

	var magic int32
	must(binary.Read(reader, wire.Endianness, &magic))

	switch magic {
	case pwr.PatchMagic:
		{
			ph := &pwr.PatchHeader{}
			rctx := wire.NewReadContext(reader)
			must(rctx.ReadMessage(ph))

			rctx, err = pwr.DecompressWire(rctx, ph.GetCompression())
			must(err)
			container := &tlc.Container{}
			must(rctx.ReadMessage(container)) // target container
			container.Reset()
			must(rctx.ReadMessage(container)) // source container

			comm.Logf("%s: %s wharf patch file (%s) with %s", path, prettySize, ph.GetCompression().ToString(), container.Stats())
		}

	case pwr.SignatureMagic:
		{
			sh := &pwr.SignatureHeader{}
			rctx := wire.NewReadContext(reader)
			must(rctx.ReadMessage(sh))

			rctx, err = pwr.DecompressWire(rctx, sh.GetCompression())
			must(err)
			container := &tlc.Container{}
			must(rctx.ReadMessage(container))

			comm.Logf("%s: %s wharf signature file (%s) with %s", path, prettySize, sh.GetCompression().ToString(), container.Stats())
		}

	default:
		_, err := reader.Seek(0, os.SEEK_SET)
		must(err)

		wasZip := func() bool {
			zr, err := zip.NewReader(reader, stats.Size())
			if err != nil {
				if err != zip.ErrFormat {
					must(err)
				}
				return false
			}

			container, err := tlc.WalkZip(zr, func(fi os.FileInfo) bool { return true })
			must(err)

			comm.Logf("%s: %s zip file with %s", path, prettySize, container.Stats())
			return true
		}()

		if !wasZip {
			comm.Logf("%s: not sure - try the file(1) command if your system has it!", path)
		}
	}
}

func ls(path string) {
	stats, err := os.Lstat(path)
	if os.IsNotExist(err) {
		comm.Dief("%s: no such file or directory")
	}
	must(err)

	if stats.IsDir() {
		comm.Logf("%s: directory", path)
		return
	}

	if stats.Size() == 0 {
		comm.Logf("%s: empty file. peaceful.", path)
		return
	}

	log := func(line string) {
		comm.Logf(line)
	}

	reader, err := os.Open(path)
	must(err)

	var magic int32
	must(binary.Read(reader, wire.Endianness, &magic))

	switch magic {
	case pwr.PatchMagic:
		{
			h := &pwr.PatchHeader{}
			rctx := wire.NewReadContext(reader)
			must(rctx.ReadMessage(h))

			rctx, err = pwr.DecompressWire(rctx, h.GetCompression())
			must(err)
			container := &tlc.Container{}
			must(rctx.ReadMessage(container))
			log("pre-patch container:")
			container.Print(log)

			container.Reset()
			must(rctx.ReadMessage(container))
			log("================================")
			log("post-patch container:")
			container.Print(log)
		}

	case pwr.SignatureMagic:
		{
			h := &pwr.SignatureHeader{}
			rctx := wire.NewReadContext(reader)
			must(rctx.ReadMessage(h))

			rctx, err = pwr.DecompressWire(rctx, h.GetCompression())
			must(err)
			container := &tlc.Container{}
			must(rctx.ReadMessage(container))
			container.Print(log)
		}

	default:
		_, err := reader.Seek(0, os.SEEK_SET)
		must(err)

		wasZip := func() bool {
			zr, err := zip.NewReader(reader, stats.Size())
			if err != nil {
				if err != zip.ErrFormat {
					must(err)
				}
				return false
			}

			container, err := tlc.WalkZip(zr, func(fi os.FileInfo) bool { return true })
			must(err)
			container.Print(log)
			return true
		}()

		if !wasZip {
			comm.Logf("%s: not sure - try the file(1) command if your system has it!", path)
		}
	}
}

func versionCheck() {
	currentVer, latestVer, err := queryLatestVersion()
	if err != nil {
		comm.Logf("Version check failed: %s", err.Error())
	}

	if currentVer == nil || latestVer == nil {
		return
	}

	if latestVer.GT(*currentVer) {
		comm.Notice("New version available",
			[]string{
				fmt.Sprintf("Current version: %s", version),
				fmt.Sprintf("Latest version:  %s", latestVer),
				"",
				"Run `butler upgrade` to get it.",
			})
	}
}

func queryLatestVersion() (*semver.Version, *semver.Version, error) {
	if *appArgs.quiet {
		return nil, nil, nil
	}

	if version == "head" {
		comm.Opf("Bleeding-edge, skipping version check")
		return nil, nil, nil
	}

	currentVer, err := semver.Make(version)
	if err != nil {
		return nil, nil, err
	}

	c := itchio.ClientWithKey("x")

	latestURL := fmt.Sprintf("%s/LATEST", updateBaseURL)
	req, err := http.NewRequest("GET", latestURL, nil)
	if err != nil {
		return nil, nil, err
	}

	res, err := c.Do(req)
	if err != nil {
		return nil, nil, err
	}

	if res.StatusCode != 200 {
		err := fmt.Errorf("HTTP %d: %s", res.StatusCode, latestURL)
		return nil, nil, err
	}

	buf, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, nil, err
	}

	latestVersion := strings.TrimLeft(strings.Trim(string(buf), " \r\n"), "v")

	latestVer, err := semver.Make(latestVersion)
	if err != nil {
		return nil, nil, err
	}

	return &currentVer, &latestVer, nil
}

func upgrade(head bool) {
	must(doUpgrade(head))
}

func doUpgrade(head bool) error {
	if head {
		if !comm.YesNo("Do you want to upgrade to the bleeding-edge version? Things may break!") {
			comm.Logf("Okay, not upgrading. Bye!")
			return nil
		}

		return applyUpgrade("head", "head")
	}

	if version == "head" {
		comm.Statf("Bleeding-edge, not upgrading to stable unless told to.")
		comm.Logf("(Use `--head` if you want to upgrade to the latest bleeding-edge version)")
		return nil
	}

	comm.Opf("Looking for upgrades...")

	currentVer, latestVer, err := queryLatestVersion()
	if err != nil {
		return fmt.Errorf("Version check failed: %s", err.Error())
	}

	if latestVer == nil || currentVer.GTE(*latestVer) {
		comm.Statf("Your butler is up-to-date. Have a nice day!")
		return nil
	}

	comm.Statf("Current version: %s", currentVer.String())
	comm.Statf("Latest version : %s", latestVer.String())

	if !comm.YesNo("Do you want to upgrade now?") {
		comm.Logf("Okay, not upgrading. Bye!")
		return nil
	}

	must(applyUpgrade(currentVer.String(), latestVer.String()))
	return nil
}

func applyUpgrade(before string, after string) error {
	execPath, err := osext.Executable()
	if err != nil {
		return err
	}

	oldPath := execPath + ".old"
	newPath := execPath + ".new"

	ext := ""
	if runtime.GOOS == "windows" {
		ext = ".exe"
	}

	fragment := fmt.Sprintf("v%s", after)
	if after == "head" {
		fragment = "head"
	}
	execURL := fmt.Sprintf("%s/%s/butler%s", updateBaseURL, fragment, ext)
	comm.Opf("%s", execURL)

	err = os.RemoveAll(newPath)
	if err != nil {
		return err
	}

	dl(execURL, newPath)
	err = os.Chmod(newPath, os.FileMode(0755))
	if err != nil {
		return err
	}

	comm.Opf("Backing up current version to %s just in case...", oldPath)
	err = os.Rename(execPath, oldPath)
	if err != nil {
		return err
	}

	err = os.Rename(newPath, execPath)
	if err != nil {
		return err
	}

	err = os.Remove(oldPath)
	if err != nil {
		return err
	}

	comm.Statf("Upgraded butler from %s to %s. Have a nice day!", before, after)
	return nil
}
<commit_msg>Don't even try to remove self exec on windows<commit_after>package main

import (
	"archive/zip"
	"encoding/binary"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"runtime"
	"strings"

	"github.com/blang/semver"
	"github.com/dustin/go-humanize"
	"github.com/itchio/butler/comm"
	"github.com/itchio/go-itchio"
	"github.com/itchio/wharf/pwr"
	"github.com/itchio/wharf/tlc"
	"github.com/itchio/wharf/wire"
	"github.com/kardianos/osext"
)

var updateBaseURL = fmt.Sprintf("https://dl.itch.ovh/butler/%s-%s", runtime.GOOS, runtime.GOARCH)

func which() {
	p, err := osext.Executable()
	must(err)

	comm.Logf("You're running butler %s, from the following path:", versionString)
	comm.Logf("%s", p)
}

func file(path string) {
	stats, err := os.Lstat(path)
	if os.IsNotExist(err) {
		comm.Dief("%s: no such file or directory")
	}
	must(err)

	if stats.IsDir() {
		comm.Logf("%s: directory", path)
		return
	}

	if stats.Size() == 0 {
		comm.Logf("%s: empty file. peaceful.", path)
		return
	}

	prettySize := humanize.Bytes(uint64(stats.Size()))

	reader, err := os.Open(path)
	must(err)

	var magic int32
	must(binary.Read(reader, wire.Endianness, &magic))

	switch magic {
	case pwr.PatchMagic:
		{
			ph := &pwr.PatchHeader{}
			rctx := wire.NewReadContext(reader)
			must(rctx.ReadMessage(ph))

			rctx, err = pwr.DecompressWire(rctx, ph.GetCompression())
			must(err)
			container := &tlc.Container{}
			must(rctx.ReadMessage(container)) // target container
			container.Reset()
			must(rctx.ReadMessage(container)) // source container

			comm.Logf("%s: %s wharf patch file (%s) with %s", path, prettySize, ph.GetCompression().ToString(), container.Stats())
		}

	case pwr.SignatureMagic:
		{
			sh := &pwr.SignatureHeader{}
			rctx := wire.NewReadContext(reader)
			must(rctx.ReadMessage(sh))

			rctx, err = pwr.DecompressWire(rctx, sh.GetCompression())
			must(err)
			container := &tlc.Container{}
			must(rctx.ReadMessage(container))

			comm.Logf("%s: %s wharf signature file (%s) with %s", path, prettySize, sh.GetCompression().ToString(), container.Stats())
		}

	default:
		_, err := reader.Seek(0, os.SEEK_SET)
		must(err)

		wasZip := func() bool {
			zr, err := zip.NewReader(reader, stats.Size())
			if err != nil {
				if err != zip.ErrFormat {
					must(err)
				}
				return false
			}

			container, err := tlc.WalkZip(zr, func(fi os.FileInfo) bool { return true })
			must(err)

			comm.Logf("%s: %s zip file with %s", path, prettySize, container.Stats())
			return true
		}()

		if !wasZip {
			comm.Logf("%s: not sure - try the file(1) command if your system has it!", path)
		}
	}
}

func ls(path string) {
	stats, err := os.Lstat(path)
	if os.IsNotExist(err) {
		comm.Dief("%s: no such file or directory")
	}
	must(err)

	if stats.IsDir() {
		comm.Logf("%s: directory", path)
		return
	}

	if stats.Size() == 0 {
		comm.Logf("%s: empty file. peaceful.", path)
		return
	}

	log := func(line string) {
		comm.Logf(line)
	}

	reader, err := os.Open(path)
	must(err)

	var magic int32
	must(binary.Read(reader, wire.Endianness, &magic))

	switch magic {
	case pwr.PatchMagic:
		{
			h := &pwr.PatchHeader{}
			rctx := wire.NewReadContext(reader)
			must(rctx.ReadMessage(h))

			rctx, err = pwr.DecompressWire(rctx, h.GetCompression())
			must(err)
			container := &tlc.Container{}
			must(rctx.ReadMessage(container))
			log("pre-patch container:")
			container.Print(log)

			container.Reset()
			must(rctx.ReadMessage(container))
			log("================================")
			log("post-patch container:")
			container.Print(log)
		}

	case pwr.SignatureMagic:
		{
			h := &pwr.SignatureHeader{}
			rctx := wire.NewReadContext(reader)
			must(rctx.ReadMessage(h))

			rctx, err = pwr.DecompressWire(rctx, h.GetCompression())
			must(err)
			container := &tlc.Container{}
			must(rctx.ReadMessage(container))
			container.Print(log)
		}

	default:
		_, err := reader.Seek(0, os.SEEK_SET)
		must(err)

		wasZip := func() bool {
			zr, err := zip.NewReader(reader, stats.Size())
			if err != nil {
				if err != zip.ErrFormat {
					must(err)
				}
				return false
			}

			container, err := tlc.WalkZip(zr, func(fi os.FileInfo) bool { return true })
			must(err)
			container.Print(log)
			return true
		}()

		if !wasZip {
			comm.Logf("%s: not sure - try the file(1) command if your system has it!", path)
		}
	}
}

func versionCheck() {
	currentVer, latestVer, err := queryLatestVersion()
	if err != nil {
		comm.Logf("Version check failed: %s", err.Error())
	}

	if currentVer == nil || latestVer == nil {
		return
	}

	if latestVer.GT(*currentVer) {
		comm.Notice("New version available",
			[]string{
				fmt.Sprintf("Current version: %s", version),
				fmt.Sprintf("Latest version:  %s", latestVer),
				"",
				"Run `butler upgrade` to get it.",
			})
	}
}

func queryLatestVersion() (*semver.Version, *semver.Version, error) {
	if *appArgs.quiet {
		return nil, nil, nil
	}

	if version == "head" {
		comm.Opf("Bleeding-edge, skipping version check")
		return nil, nil, nil
	}

	currentVer, err := semver.Make(version)
	if err != nil {
		return nil, nil, err
	}

	c := itchio.ClientWithKey("x")

	latestURL := fmt.Sprintf("%s/LATEST", updateBaseURL)
	req, err := http.NewRequest("GET", latestURL, nil)
	if err != nil {
		return nil, nil, err
	}

	res, err := c.Do(req)
	if err != nil {
		return nil, nil, err
	}

	if res.StatusCode != 200 {
		err := fmt.Errorf("HTTP %d: %s", res.StatusCode, latestURL)
		return nil, nil, err
	}

	buf, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, nil, err
	}

	latestVersion := strings.TrimLeft(strings.Trim(string(buf), " \r\n"), "v")

	latestVer, err := semver.Make(latestVersion)
	if err != nil {
		return nil, nil, err
	}

	return &currentVer, &latestVer, nil
}

func upgrade(head bool) {
	must(doUpgrade(head))
}

func doUpgrade(head bool) error {
	if head {
		if !comm.YesNo("Do you want to upgrade to the bleeding-edge version? Things may break!") {
			comm.Logf("Okay, not upgrading. Bye!")
			return nil
		}

		return applyUpgrade("head", "head")
	}

	if version == "head" {
		comm.Statf("Bleeding-edge, not upgrading to stable unless told to.")
		comm.Logf("(Use `--head` if you want to upgrade to the latest bleeding-edge version)")
		return nil
	}

	comm.Opf("Looking for upgrades...")

	currentVer, latestVer, err := queryLatestVersion()
	if err != nil {
		return fmt.Errorf("Version check failed: %s", err.Error())
	}

	if latestVer == nil || currentVer.GTE(*latestVer) {
		comm.Statf("Your butler is up-to-date. Have a nice day!")
		return nil
	}

	comm.Statf("Current version: %s", currentVer.String())
	comm.Statf("Latest version : %s", latestVer.String())

	if !comm.YesNo("Do you want to upgrade now?") {
		comm.Logf("Okay, not upgrading. Bye!")
		return nil
	}

	must(applyUpgrade(currentVer.String(), latestVer.String()))
	return nil
}

func applyUpgrade(before string, after string) error {
	execPath, err := osext.Executable()
	if err != nil {
		return err
	}

	oldPath := execPath + ".old"
	newPath := execPath + ".new"

	ext := ""
	if runtime.GOOS == "windows" {
		ext = ".exe"
	}

	fragment := fmt.Sprintf("v%s", after)
	if after == "head" {
		fragment = "head"
	}
	execURL := fmt.Sprintf("%s/%s/butler%s", updateBaseURL, fragment, ext)
	comm.Opf("%s", execURL)

	err = os.RemoveAll(newPath)
	if err != nil {
		return err
	}

	dl(execURL, newPath)
	err = os.Chmod(newPath, os.FileMode(0755))
	if err != nil {
		return err
	}

	comm.Opf("Backing up current version to %s just in case...", oldPath)
	err = os.Rename(execPath, oldPath)
	if err != nil {
		return err
	}

	err = os.Rename(newPath, execPath)
	if err != nil {
		return err
	}

	err = os.Remove(oldPath)
	if err != nil {
		if os.IsPermission(err) && runtime.GOOS == "windows" {
			// poor windows doesn't like us removing executables from under it
			// I vote we move on and let butler.exe.old hang around.
		} else {
			return err
		}
	}

	comm.Statf("Upgraded butler from %s to %s. Have a nice day!", before, after)
	return nil
}
<|endoftext|>


<commit_before>// this file contains a pre-processor to pull some stuff out of the markdown file before parsing it

package gnosis

import (
	"bufio"
	"io"
	"os"
)

type pageMetadata struct {
	Metadata []string
	Topics   []string
	Page     []byte
}

var pdata pageMetadata

func (pdata *PageMetadata) lineIsTitle(line []byte) bool {
	finalLength := len(line)
	i := 0

	// if the row doesn't start with tabs, spaces, or ='s
	if (data[i] != ' ' && data[i] != '=') && data[i] != '\t' {
		return false
	}

	// skip any spaces or tabs at the start
	for data[i] == ' ' || data[i] == '\t' {
		i++
	}

	// if the next item's not a =, bail out
	if data[i] != '=' {
		return false
	}

	// run through all of the ='s
	for data[i] == '=' {
		i++
	}

	if data[i] != ' ' && data[i] != '\t' && data[i] != '\n' {
		return false
	}

	//ditch all spaces after any ='s
	for data[i] == ' ' || data[i] == '\t' {
		i++
	}

	if finalLength == i+1 {
		return true
	} else {
		return false
	}
}

func (pdata *PageMetadata) ProcessMetadata(line []byte) error {

}

func (pdata *PageMetadata) LoadPage(pageName string) error {
	f, err := os.Open(pageName)
	reader := bufio.NewReader(f)
	upperLine, fullLine, err := reader.ReadLine()

	// inspect the first line you read
	if err != nil {
		return err
	} else if !fullLine {
		return err.New("first line I read wasn't a full line")
	} else if lineIsTitle(upperLine) {
		return err.New("first line looks an awful lot like the underside of the title o.O")
	}

	lowerLine, fullLine, err := reader.ReadLine()

	// inspect the lower line
	if err != nil {
		return err
	} else if !fullLine {
		return err.New("second line I read wasn't a full line")
	} else if lineIsTitle(lowerLine) {
		// if the second line is a title, read the rest of the page in
		// you don't have any metadata to work with here, move on
		upperLine.Append('\n')
		upperLine.Append(lowerLine)
		upperLine.Append('\n')

		_, err = reader.Read(lowerLine)
		if err != nil {
			return err
		}
	}

	// if you're at this point, the first line is metadata
	// you gotta process it and work with the next line
	// so let's just read through the file until we hit the title
	for !lineIsTitle(lowerLine) {
		// process the line
		ProcessMetadata(upperLine)
		// shift the lower line up
		upperLine = lowerLine
		// read in a new lower line
		lowerLine, fullLine, err := reader.ReadLine()
		if err != nil {
			return err
		} else if !fullLine {
			return err.New("I filled my buffer with a line")
		}
	}

	// by this point, I should have read everything in - let's read the rest and just return it
	upperLine.Append('\n')
	upperLine.Append(lowerLine)
	upperLine.Append('\n')

	_, err = reader.Read(lowerLine)
	return err
}
<commit_msg>finished writing parser for the page - pulling out keywords and tags<commit_after>// this file contains a pre-processor to pull some stuff out of the markdown file before parsing it

package gnosis

import (
	"bufio"
	"io"
	"os"
)

type pageMetadata struct {
	Keywords []string
	Topics   []string
	Page     []byte
}

var pdata pageMetadata

func (pdata *PageMetadata) lineIsTitle(line []byte) bool {
	finalLength := len(line)
	i := 0

	// if the row doesn't start with tabs, spaces, or ='s
	if (data[i] != ' ' && data[i] != '=') && data[i] != '\t' {
		return false
	}

	// skip any spaces or tabs at the start
	for data[i] == ' ' || data[i] == '\t' {
		i++
	}

	// if the next item's not a =, bail out
	if data[i] != '=' {
		return false
	}

	// run through all of the ='s
	for data[i] == '=' {
		i++
	}

	if data[i] != ' ' && data[i] != '\t' && data[i] != '\n' {
		return false
	}

	//ditch all spaces after any ='s
	for data[i] == ' ' || data[i] == '\t' {
		i++
	}

	if finalLength == i+1 {
		return true
	} else {
		return false
	}
}

func (pdata *PageMetadata) checkMatch(input []byte, looking []byte, tracker []string) {
	if input[:len(looking)] == looking {
		// trim off the target from the []byte
		value := input[len(looking):]

		// trim spaces at the start and at the end
		for value[0] == ' ' || value[0] == '\t' {
			value = value[1:]
		}
		for value[len(value)-1] == ' ' || value[len(value)-1] == '\t' {
			value = value[len(value)-1:]
		}

		// replace any spaces in the middle with -'s, and suppress double spaces
		for i := 0; i < len(value); i++ {
			if value[i] == ' ' || value[i] == '\t' {
				if value[i-1] != '-' {
					value[i] = '-'
				} else {
					value = value[:i-1] + value[i:]
				}
			}
		}

		// now just add the value to the array that you're tracking
		tracker.Append(string(value))
	}
}

func (pdata *PageMetadata) ProcessMetadata(line []byte) error {
	checkMatch(line, "tag=", pdata.Topics)
	checkMatch(line, "tag =", pdata.Topics)
	checkMatch(line, "topic=", pdata.Topics)
	checkMatch(line, "topic =", pdata.Topics)
	checkMatch(line, "category=", pdata.Topics)
	checkMatch(line, "category =", pdata.Topics)

	checkMatch(line, "keyword=", pdata.Keywords)
	checkMatch(line, "keyword =", pdata.Keywords)
	checkMatch(line, "keywords=", pdata.Keywords)
	checkMatch(line, "keywords =", pdata.Keywords)
	checkMatch(line, "meta=", pdata.Keywords)
	checkMatch(line, "meta =", pdata.Keywords)
}

func (pdata *PageMetadata) LoadPage(pageName string) error {
	f, err := os.Open(pageName)
	reader := bufio.NewReader(f)
	upperLine, fullLine, err := reader.ReadLine()

	// inspect the first line you read
	if err != nil {
		return err
	} else if !fullLine {
		return err.New("first line I read wasn't a full line")
	} else if lineIsTitle(upperLine) {
		return err.New("first line looks an awful lot like the underside of the title o.O")
	}

	lowerLine, fullLine, err := reader.ReadLine()

	// inspect the lower line
	if err != nil {
		return err
	} else if !fullLine {
		return err.New("second line I read wasn't a full line")
	} else if lineIsTitle(lowerLine) {
		// if the second line is a title, read the rest of the page in
		// you don't have any metadata to work with here, move on
		upperLine.Append('\n')
		upperLine.Append(lowerLine)
		upperLine.Append('\n')

		_, err = reader.Read(lowerLine)
		if err != nil {
			return err
		}
	}

	// if you're at this point, the first line is metadata
	// you gotta process it and work with the next line
	// so let's just read through the file until we hit the title
	for !lineIsTitle(lowerLine) {
		// process the line
		ProcessMetadata(upperLine)
		// shift the lower line up
		upperLine = lowerLine
		// read in a new lower line
		lowerLine, fullLine, err := reader.ReadLine()
		if err != nil {
			return err
		} else if !fullLine {
			return err.New("I filled my buffer with a line")
		}
	}

	// by this point, I should have read everything in - let's read the rest and just return it
	upperLine.Append('\n')
	upperLine.Append(lowerLine)
	upperLine.Append('\n')

	_, err = reader.Read(lowerLine)
	return err
}
<|endoftext|>


<commit_before>package hoverfly
<commit_msg>metadata interface<commit_after>package hoverfly

import (
	"bytes"
	"fmt"

	"github.com/boltdb/bolt"
)

// Metadata - interface to store and retrieve any metadata that is related to Hoverfly
type Metadata interface {
	Set(key, value []byte) error
	Get(key []byte) ([]byte, error)
	Delete(key []byte) error
	GetAll() ([]MetaObject, error)
	CloseDB()
}

<|endoftext|>


<commit_before>package registry

import (
	"github.com/dotcloud/docker/auth"
	"github.com/dotcloud/docker/utils"
	"strings"
	"testing"
)

var (
	IMAGE_ID = "42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d"
	TOKEN    = []string{"fake-token"}
	REPO     = "foo42/bar"
)

func spawnTestRegistry(t *testing.T) *Registry {
	authConfig := &auth.AuthConfig{}
	r, err := NewRegistry(authConfig, utils.NewHTTPRequestFactory(), makeURL("/v1/"))
	if err != nil {
		t.Fatal(err)
	}
	return r
}

func TestPingRegistryEndpoint(t *testing.T) {
	err := pingRegistryEndpoint(makeURL("/v1/"))
	if err != nil {
		t.Fatal(err)
	}
}

func TestGetRemoteHistory(t *testing.T) {
	r := spawnTestRegistry(t)
	hist, err := r.GetRemoteHistory(IMAGE_ID, makeURL("/v1/"), TOKEN)
	if err != nil {
		t.Fatal(err)
	}
	assertEqual(t, len(hist), 2, "Expected 2 images in history")
	assertEqual(t, hist[0], IMAGE_ID, "Expected "+IMAGE_ID+"as first ancestry")
	assertEqual(t, hist[1], "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20",
		"Unexpected second ancestry")
}

func TestLookupRemoteImage(t *testing.T) {
	r := spawnTestRegistry(t)
	found := r.LookupRemoteImage(IMAGE_ID, makeURL("/v1/"), TOKEN)
	assertEqual(t, found, true, "Expected remote lookup to succeed")
	found = r.LookupRemoteImage("abcdef", makeURL("/v1/"), TOKEN)
	assertEqual(t, found, false, "Expected remote lookup to fail")
}

func TestGetRemoteImageJSON(t *testing.T) {
	r := spawnTestRegistry(t)
	json, size, err := r.GetRemoteImageJSON(IMAGE_ID, makeURL("/v1/"), TOKEN)
	if err != nil {
		t.Fatal(err)
	}
	assertEqual(t, size, 154, "Expected size 154")
	if len(json) <= 0 {
		t.Fatal("Expected non-empty json")
	}

	_, _, err = r.GetRemoteImageJSON("abcdef", makeURL("/v1/"), TOKEN)
	if err == nil {
		t.Fatal("Expected image not found error")
	}
}

func TestGetRemoteImageLayer(t *testing.T) {
	r := spawnTestRegistry(t)
	data, err := r.GetRemoteImageLayer(IMAGE_ID, makeURL("/v1/"), TOKEN)
	if err != nil {
		t.Fatal(err)
	}
	if data == nil {
		t.Fatal("Expected non-nil data result")
	}

	_, err = r.GetRemoteImageLayer("abcdef", makeURL("/v1/"), TOKEN)
	if err == nil {
		t.Fatal("Expected image not found error")
	}
}

func TestGetRemoteTags(t *testing.T) {
	r := spawnTestRegistry(t)
	tags, err := r.GetRemoteTags([]string{makeURL("/v1/")}, REPO, TOKEN)
	if err != nil {
		t.Fatal(err)
	}
	assertEqual(t, len(tags), 1, "Expected one tag")
	assertEqual(t, tags["latest"], IMAGE_ID, "Expected tag latest to map to "+IMAGE_ID)

	_, err = r.GetRemoteTags([]string{makeURL("/v1/")}, "foo42/baz", TOKEN)
	if err == nil {
		t.Fatal("Expected error when fetching tags for bogus repo")
	}
}

func TestGetRepositoryData(t *testing.T) {
	r := spawnTestRegistry(t)
	data, err := r.GetRepositoryData("foo42/bar")
	if err != nil {
		t.Fatal(err)
	}
	assertEqual(t, len(data.ImgList), 2, "Expected 2 images in ImgList")
	assertEqual(t, len(data.Endpoints), 1, "Expected one endpoint in Endpoints")
}

func TestPushImageJSONRegistry(t *testing.T) {
	r := spawnTestRegistry(t)
	imgData := &ImgData{
		ID:       "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20",
		Checksum: "sha256:1ac330d56e05eef6d438586545ceff7550d3bdcb6b19961f12c5ba714ee1bb37",
	}

	err := r.PushImageJSONRegistry(imgData, []byte{0x42, 0xdf, 0x0}, makeURL("/v1/"), TOKEN)
	if err != nil {
		t.Fatal(err)
	}
}

func TestPushImageLayerRegistry(t *testing.T) {
	r := spawnTestRegistry(t)
	layer := strings.NewReader("")
	_, err := r.PushImageLayerRegistry(IMAGE_ID, layer, makeURL("/v1/"), TOKEN, []byte{})
	if err != nil {
		t.Fatal(err)
	}
}

func TestResolveRepositoryName(t *testing.T) {
	_, _, err := ResolveRepositoryName("https://github.com/dotcloud/docker")
	assertEqual(t, err, ErrInvalidRepositoryName, "Expected error invalid repo name")
	ep, repo, err := ResolveRepositoryName("fooo/bar")
	if err != nil {
		t.Fatal(err)
	}
	assertEqual(t, ep, auth.IndexServerAddress(), "Expected endpoint to be index server address")
	assertEqual(t, repo, "fooo/bar", "Expected resolved repo to be foo/bar")

	u := makeURL("")[7:]
	ep, repo, err = ResolveRepositoryName(u + "/private/moonbase")
	if err != nil {
		t.Fatal(err)
	}
	assertEqual(t, ep, "http://"+u+"/v1/", "Expected endpoint to be "+u)
	assertEqual(t, repo, "private/moonbase", "Expected endpoint to be private/moonbase")
}

func TestPushRegistryTag(t *testing.T) {
	r := spawnTestRegistry(t)
	err := r.PushRegistryTag("foo42/bar", IMAGE_ID, "stable", makeURL("/v1/"), TOKEN)
	if err != nil {
		t.Fatal(err)
	}
}

func TestPushImageJSONIndex(t *testing.T) {
	r := spawnTestRegistry(t)
	imgData := []*ImgData{
		{
			ID:       "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20",
			Checksum: "sha256:1ac330d56e05eef6d438586545ceff7550d3bdcb6b19961f12c5ba714ee1bb37",
		},
		{
			ID:       "42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d",
			Checksum: "sha256:bea7bf2e4bacd479344b737328db47b18880d09096e6674165533aa994f5e9f2",
		},
	}
	repoData, err := r.PushImageJSONIndex("foo42/bar", imgData, false, nil)
	if err != nil {
		t.Fatal(err)
	}
	if repoData == nil {
		t.Fatal("Expected RepositoryData object")
	}
	repoData, err = r.PushImageJSONIndex("foo42/bar", imgData, true, []string{r.indexEndpoint})
	if err != nil {
		t.Fatal(err)
	}
	if repoData == nil {
		t.Fatal("Expected RepositoryData object")
	}
}

func TestSearchRepositories(t *testing.T) {
	r := spawnTestRegistry(t)
	results, err := r.SearchRepositories("supercalifragilisticepsialidocious")
	if err != nil {
		t.Fatal(err)
	}
	if results == nil {
		t.Fatal("Expected non-nil SearchResults object")
	}
	assertEqual(t, results.NumResults, 0, "Expected 0 search results")
}

func TestValidRepositoryName(t *testing.T) {
	if err := validateRepositoryName("docker/docker"); err != nil {
		t.Fatal(err)
	}
	if err := validateRepositoryName("docker/Docker"); err == nil {
		t.Log("Repository name should be invalid")
		t.Fail()
	}
}
<commit_msg>Fixed registry unit tests<commit_after>package registry

import (
	"github.com/dotcloud/docker/auth"
	"github.com/dotcloud/docker/utils"
	"strings"
	"testing"
)

var (
	IMAGE_ID = "42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d"
	TOKEN    = []string{"fake-token"}
	REPO     = "foo42/bar"
)

func spawnTestRegistry(t *testing.T) *Registry {
	authConfig := &auth.AuthConfig{}
	r, err := NewRegistry(authConfig, utils.NewHTTPRequestFactory(), makeURL("/v1/"))
	if err != nil {
		t.Fatal(err)
	}
	return r
}

func TestPingRegistryEndpoint(t *testing.T) {
	standalone, err := pingRegistryEndpoint(makeURL("/v1/"))
	if err != nil {
		t.Fatal(err)
	}
	assertEqual(t, standalone, true, "Expected standalone to be true (default)")
}

func TestGetRemoteHistory(t *testing.T) {
	r := spawnTestRegistry(t)
	hist, err := r.GetRemoteHistory(IMAGE_ID, makeURL("/v1/"), TOKEN)
	if err != nil {
		t.Fatal(err)
	}
	assertEqual(t, len(hist), 2, "Expected 2 images in history")
	assertEqual(t, hist[0], IMAGE_ID, "Expected "+IMAGE_ID+"as first ancestry")
	assertEqual(t, hist[1], "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20",
		"Unexpected second ancestry")
}

func TestLookupRemoteImage(t *testing.T) {
	r := spawnTestRegistry(t)
	found := r.LookupRemoteImage(IMAGE_ID, makeURL("/v1/"), TOKEN)
	assertEqual(t, found, true, "Expected remote lookup to succeed")
	found = r.LookupRemoteImage("abcdef", makeURL("/v1/"), TOKEN)
	assertEqual(t, found, false, "Expected remote lookup to fail")
}

func TestGetRemoteImageJSON(t *testing.T) {
	r := spawnTestRegistry(t)
	json, size, err := r.GetRemoteImageJSON(IMAGE_ID, makeURL("/v1/"), TOKEN)
	if err != nil {
		t.Fatal(err)
	}
	assertEqual(t, size, 154, "Expected size 154")
	if len(json) <= 0 {
		t.Fatal("Expected non-empty json")
	}

	_, _, err = r.GetRemoteImageJSON("abcdef", makeURL("/v1/"), TOKEN)
	if err == nil {
		t.Fatal("Expected image not found error")
	}
}

func TestGetRemoteImageLayer(t *testing.T) {
	r := spawnTestRegistry(t)
	data, err := r.GetRemoteImageLayer(IMAGE_ID, makeURL("/v1/"), TOKEN)
	if err != nil {
		t.Fatal(err)
	}
	if data == nil {
		t.Fatal("Expected non-nil data result")
	}

	_, err = r.GetRemoteImageLayer("abcdef", makeURL("/v1/"), TOKEN)
	if err == nil {
		t.Fatal("Expected image not found error")
	}
}

func TestGetRemoteTags(t *testing.T) {
	r := spawnTestRegistry(t)
	tags, err := r.GetRemoteTags([]string{makeURL("/v1/")}, REPO, TOKEN)
	if err != nil {
		t.Fatal(err)
	}
	assertEqual(t, len(tags), 1, "Expected one tag")
	assertEqual(t, tags["latest"], IMAGE_ID, "Expected tag latest to map to "+IMAGE_ID)

	_, err = r.GetRemoteTags([]string{makeURL("/v1/")}, "foo42/baz", TOKEN)
	if err == nil {
		t.Fatal("Expected error when fetching tags for bogus repo")
	}
}

func TestGetRepositoryData(t *testing.T) {
	r := spawnTestRegistry(t)
	data, err := r.GetRepositoryData("foo42/bar")
	if err != nil {
		t.Fatal(err)
	}
	assertEqual(t, len(data.ImgList), 2, "Expected 2 images in ImgList")
	assertEqual(t, len(data.Endpoints), 1, "Expected one endpoint in Endpoints")
}

func TestPushImageJSONRegistry(t *testing.T) {
	r := spawnTestRegistry(t)
	imgData := &ImgData{
		ID:       "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20",
		Checksum: "sha256:1ac330d56e05eef6d438586545ceff7550d3bdcb6b19961f12c5ba714ee1bb37",
	}

	err := r.PushImageJSONRegistry(imgData, []byte{0x42, 0xdf, 0x0}, makeURL("/v1/"), TOKEN)
	if err != nil {
		t.Fatal(err)
	}
}

func TestPushImageLayerRegistry(t *testing.T) {
	r := spawnTestRegistry(t)
	layer := strings.NewReader("")
	_, err := r.PushImageLayerRegistry(IMAGE_ID, layer, makeURL("/v1/"), TOKEN, []byte{})
	if err != nil {
		t.Fatal(err)
	}
}

func TestResolveRepositoryName(t *testing.T) {
	_, _, err := ResolveRepositoryName("https://github.com/dotcloud/docker")
	assertEqual(t, err, ErrInvalidRepositoryName, "Expected error invalid repo name")
	ep, repo, err := ResolveRepositoryName("fooo/bar")
	if err != nil {
		t.Fatal(err)
	}
	assertEqual(t, ep, auth.IndexServerAddress(), "Expected endpoint to be index server address")
	assertEqual(t, repo, "fooo/bar", "Expected resolved repo to be foo/bar")

	u := makeURL("")[7:]
	ep, repo, err = ResolveRepositoryName(u + "/private/moonbase")
	if err != nil {
		t.Fatal(err)
	}
	assertEqual(t, ep, "http://"+u+"/v1/", "Expected endpoint to be "+u)
	assertEqual(t, repo, "private/moonbase", "Expected endpoint to be private/moonbase")
}

func TestPushRegistryTag(t *testing.T) {
	r := spawnTestRegistry(t)
	err := r.PushRegistryTag("foo42/bar", IMAGE_ID, "stable", makeURL("/v1/"), TOKEN)
	if err != nil {
		t.Fatal(err)
	}
}

func TestPushImageJSONIndex(t *testing.T) {
	r := spawnTestRegistry(t)
	imgData := []*ImgData{
		{
			ID:       "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20",
			Checksum: "sha256:1ac330d56e05eef6d438586545ceff7550d3bdcb6b19961f12c5ba714ee1bb37",
		},
		{
			ID:       "42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d",
			Checksum: "sha256:bea7bf2e4bacd479344b737328db47b18880d09096e6674165533aa994f5e9f2",
		},
	}
	repoData, err := r.PushImageJSONIndex("foo42/bar", imgData, false, nil)
	if err != nil {
		t.Fatal(err)
	}
	if repoData == nil {
		t.Fatal("Expected RepositoryData object")
	}
	repoData, err = r.PushImageJSONIndex("foo42/bar", imgData, true, []string{r.indexEndpoint})
	if err != nil {
		t.Fatal(err)
	}
	if repoData == nil {
		t.Fatal("Expected RepositoryData object")
	}
}

func TestSearchRepositories(t *testing.T) {
	r := spawnTestRegistry(t)
	results, err := r.SearchRepositories("supercalifragilisticepsialidocious")
	if err != nil {
		t.Fatal(err)
	}
	if results == nil {
		t.Fatal("Expected non-nil SearchResults object")
	}
	assertEqual(t, results.NumResults, 0, "Expected 0 search results")
}

func TestValidRepositoryName(t *testing.T) {
	if err := validateRepositoryName("docker/docker"); err != nil {
		t.Fatal(err)
	}
	if err := validateRepositoryName("docker/Docker"); err == nil {
		t.Log("Repository name should be invalid")
		t.Fail()
	}
}
<|endoftext|>


<commit_before>/*
   Copyright (c) 2016, Percona LLC and/or its affiliates. All rights reserved.

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published by
   the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>
*/

package main

import (
	"encoding/json"
	"flag"
	"fmt"
	golog "log"
	"os"
	"os/signal"
	"os/user"
	"runtime"
	"syscall"
	"time"

	"github.com/percona/pmm/proto"
	pc "github.com/percona/pmm/proto/config"
	"github.com/percona/qan-agent/agent"
	"github.com/percona/qan-agent/agent/release"
	"github.com/percona/qan-agent/client"
	"github.com/percona/qan-agent/data"
	"github.com/percona/qan-agent/instance"
	"github.com/percona/qan-agent/log"
	"github.com/percona/qan-agent/mrms"
	"github.com/percona/qan-agent/mysql"
	"github.com/percona/qan-agent/pct"
	pctCmd "github.com/percona/qan-agent/pct/cmd"
	"github.com/percona/qan-agent/qan"
	qanAnalyzerFactory "github.com/percona/qan-agent/qan/analyzer/factory"
	"github.com/percona/qan-agent/query"
	"github.com/percona/qan-agent/ticker"
)

var (
	flagBasedir string
	flagPidFile string
	flagListen  string
	flagPing    bool
	flagVersion bool
)

func init() {
	if os.Getenv("GOMAXPROCS") == "" {
		runtime.GOMAXPROCS(1)
	}

	golog.SetFlags(golog.Ldate | golog.Ltime | golog.Lmicroseconds | golog.Lshortfile)
	golog.SetOutput(os.Stdout)

	// PMM-1842: TODO: pid-file is now dummy flag to keep backward compatibility, it's not used anymore.
	// Remove it after we decide v1.0.5 is old enough to not be supported.
	// https://github.com/percona/pmm-client/commit/2dd9f028e6f8930f9f0185d13f2053d3683b8e64#diff-10f4cb4dc1c681a08cdba7b20c30b3f1
	flag.StringVar(&flagPidFile, "pid-file", "", "PID file")

	flag.StringVar(&flagBasedir, "basedir", pct.DEFAULT_BASEDIR, "Agent basedir")
	flag.StringVar(&flagListen, "listen", agent.DEFAULT_LISTEN, "Agent interface address")
	flag.BoolVar(&flagPing, "ping", false, "Ping API")
	flag.BoolVar(&flagVersion, "version", false, "Print version")

	flag.Parse()

	// We don't accept any positional arguments
	if len(flag.Args()) != 0 {
		flag.Usage()
		os.Exit(1)
	}
}

func main() {
	// //////////////////////////////////////////////////////////////////////
	// Handle options like -version which don't start the agent.
	// //////////////////////////////////////////////////////////////////////

	agentVersion := fmt.Sprintf("percona-qan-agent %s", release.VERSION)
	if flagVersion {
		fmt.Println(agentVersion)
		return
	}

	if err := pct.Basedir.Init(flagBasedir); err != nil {
		fmt.Printf("Error initializing basedir %s: %s", flagBasedir, err)
		os.Exit(1)
	}

	// Read agent.conf to get API hostname and agent UUID.
	agentConfigFile := pct.Basedir.ConfigFile("agent")
	if !pct.FileExists(agentConfigFile) {
		fmt.Printf("Agent config file %s does not exist", agentConfigFile)
		os.Exit(1)
	}

	bytes, err := agent.LoadConfig()
	if err != nil {
		fmt.Printf("Error reading agent config file %s: %s", agentConfigFile, err)
		os.Exit(1)
	}
	agentConfig := &pc.Agent{}
	if err := json.Unmarshal(bytes, agentConfig); err != nil {
		fmt.Printf("Error decoding agent config file %s: %s", agentConfigFile, err)
		os.Exit(1)
	}

	apiURL := agentConfig.ApiHostname + agentConfig.ApiPath
	fmt.Printf("# Version: %s\n", agentVersion)
	fmt.Printf("# Basedir: %s\n", pct.Basedir.Path())
	//fmt.Printf("# Listen:  %s\n", flagListen)
	fmt.Printf("# PID:     %d\n", os.Getpid())
	fmt.Printf("# API:     %s\n", apiURL)
	fmt.Printf("# UUID:    %s\n", agentConfig.UUID)

	// -ping and exit.
	if flagPing {
		t0 := time.Now()
		code, err := pct.Ping(apiURL)
		d := time.Now().Sub(t0)
		if err != nil || code != 200 {
			fmt.Printf("Ping FAIL (%d %d %s)", d, code, err)
			os.Exit(1)
		}
		fmt.Printf("Ping OK (%s)", d)
		return
	}

	// //////////////////////////////////////////////////////////////////////
	// Run the agent
	// //////////////////////////////////////////////////////////////////////

	err = run(agentConfig) // run the agent

	if err != nil {
		golog.Println(err)
		os.Exit(1)
	}
}

func run(agentConfig *pc.Agent) error {
	golog.Println("Starting agent...")
	var stopErr error
	defer func() {
		if stopErr == nil {
			golog.Println("Agent has stopped")
		}
	}()

	// //////////////////////////////////////////////////////////////////////
	// Internal services, factories, and other dependencies
	// //////////////////////////////////////////////////////////////////////
	pctCmd.Factory = &pctCmd.RealCmdFactory{}
	nowFunc := func() int64 { return time.Now().UTC().UnixNano() }
	clock := ticker.NewClock(&ticker.RealTickerFactory{}, nowFunc)

	// //////////////////////////////////////////////////////////////////////
	// Start API interface
	// //////////////////////////////////////////////////////////////////////

	// The API interface provides low-level functionality to websocket clients.
	// To be useful, it must connect once to get resource links. Do this async
	// so in case we're offline the agent still starts and collects data. We
	// can spool and send data later when API is online.
	api := pct.NewAPI(agentConfig.ServerUser, agentConfig.ServerPassword, agentConfig.ServerSSL, agentConfig.ServerInsecureSSL)
	go func() {
		haveWarned := false
		for {
			if err := api.Connect(agentConfig.ApiHostname, agentConfig.ApiPath, agentConfig.UUID); err != nil {
				if !haveWarned {
					golog.Printf("Cannot connect to API: %s. Verify that the"+
						" agent UUID and API hostname printed above are"+
						" correct and that no network issues prevent this"+
						" host from accessing the API. Connection"+
						" attempts to API will continue until successful, but"+
						" additional errors will not be logged, and agent"+
						" will not send data until connected to API.", err)
					haveWarned = true // don't flood log in case we're offline for a long time
				}
				time.Sleep(3 * time.Second)
				continue
			}
			golog.Println("API is ready")
			return
		}
	}()

	// //////////////////////////////////////////////////////////////////////
	// Agent services
	// //////////////////////////////////////////////////////////////////////

	// Log relay
	logChan := make(chan proto.LogEntry, log.BUFFER_SIZE*2)
	logClient, err := client.NewWebsocketClient(pct.NewLogger(logChan, "log-ws"), api, "log", nil)
	if err != nil {
		return err
	}
	logManager := log.NewManager(
		logClient,
		logChan,
	)
	if err := logManager.Start(); err != nil {
		return fmt.Errorf("error starting log manager: %s", err)
	}

	// MRMS (MySQL Restart Monitoring Service)
	mrmsMonitor := mrms.NewRealMonitor(
		pct.NewLogger(logChan, "mrms-monitor"),
		&mysql.RealConnectionFactory{},
	)
	mrmsManager := mrms.NewManager(
		pct.NewLogger(logChan, "mrms-manager"),
		mrmsMonitor,
	)
	if err := mrmsManager.Start(); err != nil {
		return fmt.Errorf("error starting mrms manager: %s", err)
	}

	// Instance manager
	itManager := instance.NewManager(
		pct.NewLogger(logChan, "instance-manager"),
		pct.Basedir.Dir("instance"),
		api,
		mrmsMonitor,
	)
	if err := itManager.Start(); err != nil {
		return fmt.Errorf("error starting instance manager: %s", err)
	}

	// Data spooler and sender
	hostname, _ := os.Hostname()
	dataClient, err := client.NewWebsocketClient(pct.NewLogger(logChan, "data-ws"), api, "data", nil)
	if err != nil {
		return err
	}
	dataManager := data.NewManager(
		pct.NewLogger(logChan, "data"),
		pct.Basedir.Dir("data"),
		pct.Basedir.Dir("trash"),
		hostname,
		dataClient,
	)
	if err := dataManager.Start(); err != nil {
		return fmt.Errorf("error starting data manager: %s", err)
	}

	// Query (real-time EXPLAIN, SHOW CREATE TABLE, etc.)
	queryManager := query.NewManager(
		pct.NewLogger(logChan, "query"),
		itManager.Repo(),
	)
	if err := queryManager.Start(); err != nil {
		return fmt.Errorf("error starting query manager: %s", err)
	}

	// Query Analytics
	qanManager := qan.NewManager(
		pct.NewLogger(logChan, "qan"),
		itManager.Repo(),
		qanAnalyzerFactory.New(
			logChan,
			dataManager.Spooler(),
			clock,
			mrmsMonitor,
			itManager.Repo(),
		),
	)
	if err := qanManager.Start(); err != nil {
		return fmt.Errorf("Error starting qan manager: %s", err)
	}

	// //////////////////////////////////////////////////////////////////////
	// Create and start the agent
	// //////////////////////////////////////////////////////////////////////

	cmdClient, err := client.NewWebsocketClient(pct.NewLogger(logChan, "agent-ws"), api, "cmd", nil)
	if err != nil {
		return err
	}
	agentLogger := pct.NewLogger(logChan, "agent")
	agentRouter := agent.NewAgent(
		agentConfig,
		agentLogger,
		cmdClient,
		flagListen,
		map[string]pct.ServiceManager{ // agent services
			"log":      logManager,
			"data":     dataManager,
			"qan":      qanManager,
			"instance": itManager,
			"mrms":     mrmsManager,
			"query":    queryManager,
		},
	)

	// Run the agent, wait for it to stop, signal, or crash.
	stopChan := make(chan error, 2)
	go func() {
		defer func() {
			if err := recover(); err != nil {
				errMsg := fmt.Sprintf("Agent crashed: %s", err)
				golog.Println(errMsg)
				agentLogger.Error(errMsg)
				stopChan <- fmt.Errorf("%s", errMsg)
			}
		}()
		stopChan <- agentRouter.Run() // ----- RUN THE AGENT -----
	}()

	golog.Println("Agent is ready")

	// //////////////////////////////////////////////////////////////////////
	// Signal handlers
	// //////////////////////////////////////////////////////////////////////

	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)

	reconnectSigChan := make(chan os.Signal, 1)
	signal.Notify(reconnectSigChan, syscall.SIGHUP) // kill -HUP PID

	intSigChan := make(chan os.Signal, 1)
	signal.Notify(intSigChan, syscall.SIGINT) // CTRL-C

	// //////////////////////////////////////////////////////////////////////
	// Wait for agent stop, signals, etc.
	// //////////////////////////////////////////////////////////////////////

SIGNAL_LOOP:
	for {
		select {
		case stopErr = <-stopChan:
			// stopErr is logged in defer
			break SIGNAL_LOOP // stop running
		case sig := <-sigChan:
			msg := fmt.Sprintf("Caught %s signal, shutting down", sig)
			agentLogger.Info(msg)
			golog.Println(msg)
			break SIGNAL_LOOP // stop running
		case <-intSigChan:
			msg := "Caught CTRL-C (SIGINT), shutting down"
			agentLogger.Warn(msg)
			golog.Println(msg)
			break SIGNAL_LOOP // stop running
		case <-reconnectSigChan:
			u, _ := user.Current()
			cmd := &proto.Cmd{
				Ts:        time.Now().UTC(),
				User:      u.Username + " (SIGHUP)",
				AgentUUID: agentConfig.UUID,
				Service:   "agent",
				Cmd:       "Reconnect",
			}
			agentRouter.Handle(cmd)
		}
	}

	// //////////////////////////////////////////////////////////////////////
	// Clean up, undo any changes we made to MySQL.
	/////////////////////////////////////////////////////////////////////////

	// Disable slow log or perf schema. It's not terrible if we don't, but
	// it's a lot better if we do.
	golog.Println("Stopping QAN...")
	if err := qanManager.Stop(); err != nil {
		msg := fmt.Sprintf("Cannot stop QAN: %s", err)
		agentLogger.Warn(msg)
		golog.Printf(msg)
	}

	golog.Println("Waiting 2 seconds to flush agent log to API...")
	time.Sleep(2 * time.Second) // wait for last replies and log entries

	return stopErr
}
<commit_msg>PMM-2615 Add newlines to logs.<commit_after>/*
   Copyright (c) 2016, Percona LLC and/or its affiliates. All rights reserved.

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published by
   the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>
*/

package main

import (
	"encoding/json"
	"flag"
	"fmt"
	golog "log"
	"os"
	"os/signal"
	"os/user"
	"runtime"
	"syscall"
	"time"

	"github.com/percona/pmm/proto"
	pc "github.com/percona/pmm/proto/config"
	"github.com/percona/qan-agent/agent"
	"github.com/percona/qan-agent/agent/release"
	"github.com/percona/qan-agent/client"
	"github.com/percona/qan-agent/data"
	"github.com/percona/qan-agent/instance"
	"github.com/percona/qan-agent/log"
	"github.com/percona/qan-agent/mrms"
	"github.com/percona/qan-agent/mysql"
	"github.com/percona/qan-agent/pct"
	pctCmd "github.com/percona/qan-agent/pct/cmd"
	"github.com/percona/qan-agent/qan"
	qanAnalyzerFactory "github.com/percona/qan-agent/qan/analyzer/factory"
	"github.com/percona/qan-agent/query"
	"github.com/percona/qan-agent/ticker"
)

var (
	flagBasedir string
	flagPidFile string
	flagListen  string
	flagPing    bool
	flagVersion bool
)

func init() {
	if os.Getenv("GOMAXPROCS") == "" {
		runtime.GOMAXPROCS(1)
	}

	golog.SetFlags(golog.Ldate | golog.Ltime | golog.Lmicroseconds | golog.Lshortfile)
	golog.SetOutput(os.Stdout)

	// PMM-1842: TODO: pid-file is now dummy flag to keep backward compatibility, it's not used anymore.
	// Remove it after we decide v1.0.5 is old enough to not be supported.
	// https://github.com/percona/pmm-client/commit/2dd9f028e6f8930f9f0185d13f2053d3683b8e64#diff-10f4cb4dc1c681a08cdba7b20c30b3f1
	flag.StringVar(&flagPidFile, "pid-file", "", "PID file")

	flag.StringVar(&flagBasedir, "basedir", pct.DEFAULT_BASEDIR, "Agent basedir")
	flag.StringVar(&flagListen, "listen", agent.DEFAULT_LISTEN, "Agent interface address")
	flag.BoolVar(&flagPing, "ping", false, "Ping API")
	flag.BoolVar(&flagVersion, "version", false, "Print version")

	flag.Parse()

	// We don't accept any positional arguments
	if len(flag.Args()) != 0 {
		flag.Usage()
		os.Exit(1)
	}
}

func main() {
	// //////////////////////////////////////////////////////////////////////
	// Handle options like -version which don't start the agent.
	// //////////////////////////////////////////////////////////////////////

	agentVersion := fmt.Sprintf("percona-qan-agent %s", release.VERSION)
	if flagVersion {
		fmt.Println(agentVersion)
		return
	}

	if err := pct.Basedir.Init(flagBasedir); err != nil {
		fmt.Printf("Error initializing basedir %s: %s\n", flagBasedir, err)
		os.Exit(1)
	}

	// Read agent.conf to get API hostname and agent UUID.
	agentConfigFile := pct.Basedir.ConfigFile("agent")
	if !pct.FileExists(agentConfigFile) {
		fmt.Printf("Agent config file %s does not exist\n", agentConfigFile)
		os.Exit(1)
	}

	bytes, err := agent.LoadConfig()
	if err != nil {
		fmt.Printf("Error reading agent config file %s: %s\n", agentConfigFile, err)
		os.Exit(1)
	}
	agentConfig := &pc.Agent{}
	if err := json.Unmarshal(bytes, agentConfig); err != nil {
		fmt.Printf("Error decoding agent config file %s: %s\n", agentConfigFile, err)
		os.Exit(1)
	}

	apiURL := agentConfig.ApiHostname + agentConfig.ApiPath
	fmt.Printf("# Version: %s\n", agentVersion)
	fmt.Printf("# Basedir: %s\n", pct.Basedir.Path())
	//fmt.Printf("# Listen:  %s\n", flagListen)
	fmt.Printf("# PID:     %d\n", os.Getpid())
	fmt.Printf("# API:     %s\n", apiURL)
	fmt.Printf("# UUID:    %s\n", agentConfig.UUID)

	// -ping and exit.
	if flagPing {
		t0 := time.Now()
		code, err := pct.Ping(apiURL)
		d := time.Now().Sub(t0)
		if err != nil || code != 200 {
			fmt.Printf("Ping FAIL (%d %d %s)\n", d, code, err)
			os.Exit(1)
		}
		fmt.Printf("Ping OK (%s)\n", d)
		return
	}

	// //////////////////////////////////////////////////////////////////////
	// Run the agent
	// //////////////////////////////////////////////////////////////////////

	err = run(agentConfig) // run the agent

	if err != nil {
		golog.Println(err)
		os.Exit(1)
	}
}

func run(agentConfig *pc.Agent) error {
	golog.Println("Starting agent...")
	var stopErr error
	defer func() {
		if stopErr == nil {
			golog.Println("Agent has stopped")
		}
	}()

	// //////////////////////////////////////////////////////////////////////
	// Internal services, factories, and other dependencies
	// //////////////////////////////////////////////////////////////////////
	pctCmd.Factory = &pctCmd.RealCmdFactory{}
	nowFunc := func() int64 { return time.Now().UTC().UnixNano() }
	clock := ticker.NewClock(&ticker.RealTickerFactory{}, nowFunc)

	// //////////////////////////////////////////////////////////////////////
	// Start API interface
	// //////////////////////////////////////////////////////////////////////

	// The API interface provides low-level functionality to websocket clients.
	// To be useful, it must connect once to get resource links. Do this async
	// so in case we're offline the agent still starts and collects data. We
	// can spool and send data later when API is online.
	api := pct.NewAPI(agentConfig.ServerUser, agentConfig.ServerPassword, agentConfig.ServerSSL, agentConfig.ServerInsecureSSL)
	go func() {
		haveWarned := false
		for {
			if err := api.Connect(agentConfig.ApiHostname, agentConfig.ApiPath, agentConfig.UUID); err != nil {
				if !haveWarned {
					golog.Printf("Cannot connect to API: %s. Verify that the"+
						" agent UUID and API hostname printed above are"+
						" correct and that no network issues prevent this"+
						" host from accessing the API. Connection"+
						" attempts to API will continue until successful, but"+
						" additional errors will not be logged, and agent"+
						" will not send data until connected to API.", err)
					haveWarned = true // don't flood log in case we're offline for a long time
				}
				time.Sleep(3 * time.Second)
				continue
			}
			golog.Println("API is ready")
			return
		}
	}()

	// //////////////////////////////////////////////////////////////////////
	// Agent services
	// //////////////////////////////////////////////////////////////////////

	// Log relay
	logChan := make(chan proto.LogEntry, log.BUFFER_SIZE*2)
	logClient, err := client.NewWebsocketClient(pct.NewLogger(logChan, "log-ws"), api, "log", nil)
	if err != nil {
		return err
	}
	logManager := log.NewManager(
		logClient,
		logChan,
	)
	if err := logManager.Start(); err != nil {
		return fmt.Errorf("error starting log manager: %s", err)
	}

	// MRMS (MySQL Restart Monitoring Service)
	mrmsMonitor := mrms.NewRealMonitor(
		pct.NewLogger(logChan, "mrms-monitor"),
		&mysql.RealConnectionFactory{},
	)
	mrmsManager := mrms.NewManager(
		pct.NewLogger(logChan, "mrms-manager"),
		mrmsMonitor,
	)
	if err := mrmsManager.Start(); err != nil {
		return fmt.Errorf("error starting mrms manager: %s", err)
	}

	// Instance manager
	itManager := instance.NewManager(
		pct.NewLogger(logChan, "instance-manager"),
		pct.Basedir.Dir("instance"),
		api,
		mrmsMonitor,
	)
	if err := itManager.Start(); err != nil {
		return fmt.Errorf("error starting instance manager: %s", err)
	}

	// Data spooler and sender
	hostname, _ := os.Hostname()
	dataClient, err := client.NewWebsocketClient(pct.NewLogger(logChan, "data-ws"), api, "data", nil)
	if err != nil {
		return err
	}
	dataManager := data.NewManager(
		pct.NewLogger(logChan, "data"),
		pct.Basedir.Dir("data"),
		pct.Basedir.Dir("trash"),
		hostname,
		dataClient,
	)
	if err := dataManager.Start(); err != nil {
		return fmt.Errorf("error starting data manager: %s", err)
	}

	// Query (real-time EXPLAIN, SHOW CREATE TABLE, etc.)
	queryManager := query.NewManager(
		pct.NewLogger(logChan, "query"),
		itManager.Repo(),
	)
	if err := queryManager.Start(); err != nil {
		return fmt.Errorf("error starting query manager: %s", err)
	}

	// Query Analytics
	qanManager := qan.NewManager(
		pct.NewLogger(logChan, "qan"),
		itManager.Repo(),
		qanAnalyzerFactory.New(
			logChan,
			dataManager.Spooler(),
			clock,
			mrmsMonitor,
			itManager.Repo(),
		),
	)
	if err := qanManager.Start(); err != nil {
		return fmt.Errorf("Error starting qan manager: %s", err)
	}

	// //////////////////////////////////////////////////////////////////////
	// Create and start the agent
	// //////////////////////////////////////////////////////////////////////

	cmdClient, err := client.NewWebsocketClient(pct.NewLogger(logChan, "agent-ws"), api, "cmd", nil)
	if err != nil {
		return err
	}
	agentLogger := pct.NewLogger(logChan, "agent")
	agentRouter := agent.NewAgent(
		agentConfig,
		agentLogger,
		cmdClient,
		flagListen,
		map[string]pct.ServiceManager{ // agent services
			"log":      logManager,
			"data":     dataManager,
			"qan":      qanManager,
			"instance": itManager,
			"mrms":     mrmsManager,
			"query":    queryManager,
		},
	)

	// Run the agent, wait for it to stop, signal, or crash.
	stopChan := make(chan error, 2)
	go func() {
		defer func() {
			if err := recover(); err != nil {
				errMsg := fmt.Sprintf("Agent crashed: %s", err)
				golog.Println(errMsg)
				agentLogger.Error(errMsg)
				stopChan <- fmt.Errorf("%s", errMsg)
			}
		}()
		stopChan <- agentRouter.Run() // ----- RUN THE AGENT -----
	}()

	golog.Println("Agent is ready")

	// //////////////////////////////////////////////////////////////////////
	// Signal handlers
	// //////////////////////////////////////////////////////////////////////

	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)

	reconnectSigChan := make(chan os.Signal, 1)
	signal.Notify(reconnectSigChan, syscall.SIGHUP) // kill -HUP PID

	intSigChan := make(chan os.Signal, 1)
	signal.Notify(intSigChan, syscall.SIGINT) // CTRL-C

	// //////////////////////////////////////////////////////////////////////
	// Wait for agent stop, signals, etc.
	// //////////////////////////////////////////////////////////////////////

SIGNAL_LOOP:
	for {
		select {
		case stopErr = <-stopChan:
			// stopErr is logged in defer
			break SIGNAL_LOOP // stop running
		case sig := <-sigChan:
			msg := fmt.Sprintf("Caught %s signal, shutting down", sig)
			agentLogger.Info(msg)
			golog.Println(msg)
			break SIGNAL_LOOP // stop running
		case <-intSigChan:
			msg := "Caught CTRL-C (SIGINT), shutting down"
			agentLogger.Warn(msg)
			golog.Println(msg)
			break SIGNAL_LOOP // stop running
		case <-reconnectSigChan:
			u, _ := user.Current()
			cmd := &proto.Cmd{
				Ts:        time.Now().UTC(),
				User:      u.Username + " (SIGHUP)",
				AgentUUID: agentConfig.UUID,
				Service:   "agent",
				Cmd:       "Reconnect",
			}
			agentRouter.Handle(cmd)
		}
	}

	// //////////////////////////////////////////////////////////////////////
	// Clean up, undo any changes we made to MySQL.
	/////////////////////////////////////////////////////////////////////////

	// Disable slow log or perf schema. It's not terrible if we don't, but
	// it's a lot better if we do.
	golog.Println("Stopping QAN...")
	if err := qanManager.Stop(); err != nil {
		msg := fmt.Sprintf("Cannot stop QAN: %s", err)
		agentLogger.Warn(msg)
		golog.Printf(msg)
	}

	golog.Println("Waiting 2 seconds to flush agent log to API...")
	time.Sleep(2 * time.Second) // wait for last replies and log entries

	return stopErr
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"time"
)

func main() {

	// the main communication channel
	c := make(chan string, 10)

	// delay mechanism for service restarts
	backoff := 1

	// testing (will close chan once after 5 seconds)
	go stopAfter5(c)

	// main loop
	for {

		// checking how long the service was running
		// if long enough, the backoff is reset to 1
		t := time.Now()
		err := runService(c)
		if err != nil {
			log.Println(err)
		}

		// new communication channel for the restarted
		// service.
		c = make(chan string, 10)

		// exponential backoff with maximum 32 sec delays.
		if time.Since(t) < time.Duration(2)*time.Second {
			log.Printf("Restarting service in %vs\n", backoff)
			time.Sleep(time.Duration(backoff) * time.Second)
			if backoff < 32 {
				backoff *= 2
			}
		}
	}
}

func runService(c chan string) (err error) {

	// channel for fatal errors
	// quit if receive from this.
	e := make(chan error)

	go doSomething1(c, e)
	go doSomething2(c, e)

	for {
		select {
		case err = <-e:
			if _, ok := <-c; ok {

				// closing the channel makes sure that
				// all goroutines writing to it will panic
				// panic must be recovered in each goroutine !!!
				close(c)
			}
			return

		// Important! always check at read if the channel is closed
		// reading from closed channel floods an empty string.
		case m, ok := <-c:

			// if the channel is closed, quit.
			if ok {
				fmt.Println(m)
			} else {
				return
			}
		}
	}
}

func stopAfter5(c chan string) {
	select {
	case <-time.After(5 * time.Second):
		close(c)
	}
}

func doSomething1(c chan string, e chan error) {

	defer handlePanic(e)

	for {
		select {
		case <-time.After(1 * time.Second):
			c <- "hello from 1"
		}
	}
}

// sending hello every 500 ms
func doSomething2(c chan string, e chan error) {

	time.Sleep(500 * time.Millisecond)
	defer handlePanic(e)

	for {
		select {
		case <-time.After(1 * time.Second):
			c <- "hello from 2"
		}
	}
}

func handlePanic(e chan error) {
	if r := recover(); r != nil {
		log.Println(r)
	}
	select {

	// sending the error on channel e makes the "parent" service stop.
	case e <- fmt.Errorf("Process exited"):
	default:
	}
}
<commit_msg>wording changed<commit_after>package main

import (
	"fmt"
	"log"
	"time"
)

func main() {

	// the main communication channel
	c := make(chan string, 10)

	// delay mechanism for service restarts
	backoff := 1

	// testing (will close chan once after 5 seconds)
	go stopAfter5(c)

	// main loop
	for {

		// checking how long the service was running
		// if long enough, the backoff is reset to 1
		t := time.Now()
		err := runService(c)
		if err != nil {
			log.Println(err)
		}

		// new communication channel for the restarted
		// service.
		c = make(chan string, 10)

		// exponential backoff with maximum 32 sec delays.
		if time.Since(t) < time.Duration(2)*time.Second {
			log.Printf("Restarting service in %vs\n", backoff)
			time.Sleep(time.Duration(backoff) * time.Second)
			if backoff < 32 {
				backoff *= 2
			}
		}
	}
}

func runService(c chan string) (err error) {

	// channel for fatal errors
	// return if receive from this.
	e := make(chan error)

	go doSomething1(c, e)
	go doSomething2(c, e)

	for {
		select {
		case err = <-e:
			if _, ok := <-c; ok {

				// closing the channel makes sure that
				// all goroutines writing to it will panic
				// panic must be recovered in each goroutine !!!
				close(c)
			}
			return

		// Important! always check at read if the channel is closed
		// reading from closed channel floods an empty string.
		case m, ok := <-c:

			// if the channel is closed, quit.
			if ok {
				fmt.Println(m)
			} else {
				return
			}
		}
	}
}

func stopAfter5(c chan string) {
	select {
	case <-time.After(5 * time.Second):
		close(c)
	}
}

func doSomething1(c chan string, e chan error) {

	defer handlePanic(e)

	for {
		select {
		case <-time.After(1 * time.Second):
			c <- "hello from 1"
		}
	}
}

// sending hello every 500 ms
func doSomething2(c chan string, e chan error) {

	time.Sleep(500 * time.Millisecond)
	defer handlePanic(e)

	for {
		select {
		case <-time.After(1 * time.Second):
			c <- "hello from 2"
		}
	}
}

func handlePanic(e chan error) {
	if r := recover(); r != nil {
		log.Println(r)
	}
	select {

	// sending the error on channel e makes the "parent" service stop.
	case e <- fmt.Errorf("Process exited"):
	default:
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package disks

import (
	"fmt"
	"os/exec"

	"github.com/coreos/ignition/config"
	"github.com/coreos/ignition/src/exec/stages"
	"github.com/coreos/ignition/src/exec/util"
	"github.com/coreos/ignition/src/log"
	"github.com/coreos/ignition/src/sgdisk"
	"github.com/coreos/ignition/src/systemd"
)

const (
	name = "disks"
)

func init() {
	stages.Register(creator{})
}

type creator struct{}

func (creator) Create(logger *log.Logger, root string) stages.Stage {
	return &stage{
		DestDir: util.DestDir(root),
		logger:  logger,
	}
}

func (creator) Name() string {
	return name
}

type stage struct {
	logger *log.Logger
	util.DestDir
}

func (stage) Name() string {
	return name
}

func (s stage) Run(config config.Config) bool {

	if err := s.createPartitions(config); err != nil {
		s.logger.Crit("create partitions failed: %v", err)
		return false
	}

	if err := s.createRaids(config); err != nil {
		s.logger.Crit("failed to create raids: %v", err)
		return false
	}

	if err := s.createFilesystems(config); err != nil {
		s.logger.Crit("failed to create filesystems: %v", err)
		return false
	}

	return true
}

func (s stage) waitOnDevices(devs []string, ctxt string) error {
	err := s.logger.LogOp(func() error { return systemd.WaitOnDevices(devs, ctxt) }, "waiting for devices %v", devs)
	if err != nil {
		return fmt.Errorf("failed to wait on %s devs: %v", ctxt, err)
	}
	return nil
}

func (s stage) createPartitions(config config.Config) error {
	if len(config.Storage.Disks) == 0 {
		return nil
	}
	s.logger.PushPrefix("createPartitions")
	defer s.logger.PopPrefix()

	devs := []string{}
	for _, disk := range config.Storage.Disks {
		devs = append(devs, string(disk.Device))
	}

	if err := s.waitOnDevices(devs, "disks"); err != nil {
		return err
	}

	for _, dev := range config.Storage.Disks {
		err := s.logger.LogOp(func() error {
			op := sgdisk.Begin(string(dev.Device))
			if dev.WipeTable {
				s.logger.Info("wiping partition table on %q", dev.Device)
				op.WipeTable(true)
			}

			for _, part := range dev.Partitions {
				err := op.CreatePartition(sgdisk.Partition{
					Number: part.Number,
					Length: part.Size.Value(), // TODO(vc): normalize units... sectors? do something sane.
					Offset: part.Start.Value(),
					Label:  string(part.Label),
				})
				if err != nil {
					return fmt.Errorf("create failure: %v", err)
				}
			}

			if err := op.Commit(); err != nil {
				return fmt.Errorf("commit failure: %v", err)
			}
			return nil
		}, "partitioning %q", dev.Device)
		if err != nil {
			return err
		}
	}

	return nil
}

func (s stage) createRaids(config config.Config) error {
	if len(config.Storage.Arrays) == 0 {
		return nil
	}
	s.logger.PushPrefix("createRaids")
	defer s.logger.PopPrefix()

	devs := []string{}
	for _, array := range config.Storage.Arrays {
		for _, dev := range array.Devices {
			devs = append(devs, string(dev))
		}

		for _, dev := range array.Spares {
			devs = append(devs, string(dev))
		}
	}

	if err := s.waitOnDevices(devs, "raids"); err != nil {
		return err
	}

	for _, md := range config.Storage.Arrays {
		// FIXME(vc): this is utterly flummoxed by a preexisting md.Name, the magic of device-resident md metadata really interferes with us.
		// It's as if what ignition really needs is to turn off automagic md probing/running before getting started.
		args := []string{
			"--create", md.Name,
			"--force",
			"--run",
			"--level", md.Level,
			"--raid-devices", fmt.Sprintf("%d", len(md.Devices)),
			// FIXME(vc): md.Spares, this doesn't map well with mdadm's cli, change config.
		}
		for _, dev := range md.Devices {
			args = append(args, string(dev))
		}

		cmd := exec.Command("/sbin/mdadm", args...)
		err := s.logger.LogOp(cmd.Run, "creating %q", md)
		if err != nil {
			return fmt.Errorf("mdadm failed: %v", err)
		}
	}

	return nil
}

func (s stage) createFilesystems(config config.Config) error {
	if len(config.Storage.Filesystems) == 0 {
		return nil
	}
	s.logger.PushPrefix("createFilesystems")
	defer s.logger.PopPrefix()

	devs := []string{}
	for _, fs := range config.Storage.Filesystems {
		devs = append(devs, string(fs.Device))
	}

	if err := s.waitOnDevices(devs, "filesystems"); err != nil {
		return err
	}

	for _, fs := range config.Storage.Filesystems {
		mkfs := ""
		args := []string(fs.Options)
		switch fs.Format {
		case "btrfs":
			mkfs = "/sbin/mkfs.btrfs"
			args = append(args, "--force")
		case "ext4":
			mkfs = "/sbin/mkfs.ext4"
			args = append(args, "-F")
		default:
			return fmt.Errorf("unsupported filesystem format: %q", fs.Format)
		}

		args = append(args, string(fs.Device))
		cmd := exec.Command(mkfs, args...)

		err := s.logger.LogOp(cmd.Run, "creating %q filesystem on %q", fs.Format, string(fs.Device))
		if err != nil {
			return fmt.Errorf("failed to run %q: %v %v", mkfs, err, args)
		}

		// TODO(vc): apply fs.Files, which requires mounting somewhere...
	}

	return nil
}
<commit_msg>storage: write the files in config.Filesystem<commit_after>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package disks

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"syscall"

	"github.com/coreos/ignition/config"
	"github.com/coreos/ignition/src/exec/stages"
	"github.com/coreos/ignition/src/exec/util"
	"github.com/coreos/ignition/src/log"
	"github.com/coreos/ignition/src/sgdisk"
	"github.com/coreos/ignition/src/systemd"
)

const (
	name = "disks"
)

func init() {
	stages.Register(creator{})
}

type creator struct{}

func (creator) Create(logger *log.Logger, root string) stages.Stage {
	return &stage{
		DestDir: util.DestDir(root),
		logger:  logger,
	}
}

func (creator) Name() string {
	return name
}

type stage struct {
	logger *log.Logger
	util.DestDir
}

func (stage) Name() string {
	return name
}

func (s stage) Run(config config.Config) bool {

	if err := s.createPartitions(config); err != nil {
		s.logger.Crit("create partitions failed: %v", err)
		return false
	}

	if err := s.createRaids(config); err != nil {
		s.logger.Crit("failed to create raids: %v", err)
		return false
	}

	if err := s.createFilesystems(config); err != nil {
		s.logger.Crit("failed to create filesystems: %v", err)
		return false
	}

	return true
}

func (s stage) waitOnDevices(devs []string, ctxt string) error {
	err := s.logger.LogOp(func() error { return systemd.WaitOnDevices(devs, ctxt) }, "waiting for devices %v", devs)
	if err != nil {
		return fmt.Errorf("failed to wait on %s devs: %v", ctxt, err)
	}
	return nil
}

func (s stage) createPartitions(config config.Config) error {
	if len(config.Storage.Disks) == 0 {
		return nil
	}
	s.logger.PushPrefix("createPartitions")
	defer s.logger.PopPrefix()

	devs := []string{}
	for _, disk := range config.Storage.Disks {
		devs = append(devs, string(disk.Device))
	}

	if err := s.waitOnDevices(devs, "disks"); err != nil {
		return err
	}

	for _, dev := range config.Storage.Disks {
		err := s.logger.LogOp(func() error {
			op := sgdisk.Begin(string(dev.Device))
			if dev.WipeTable {
				s.logger.Info("wiping partition table on %q", dev.Device)
				op.WipeTable(true)
			}

			for _, part := range dev.Partitions {
				err := op.CreatePartition(sgdisk.Partition{
					Number: part.Number,
					Length: part.Size.Value(), // TODO(vc): normalize units... sectors? do something sane.
					Offset: part.Start.Value(),
					Label:  string(part.Label),
				})
				if err != nil {
					return fmt.Errorf("create failure: %v", err)
				}
			}

			if err := op.Commit(); err != nil {
				return fmt.Errorf("commit failure: %v", err)
			}
			return nil
		}, "partitioning %q", dev.Device)
		if err != nil {
			return err
		}
	}

	return nil
}

func (s stage) createRaids(config config.Config) error {
	if len(config.Storage.Arrays) == 0 {
		return nil
	}
	s.logger.PushPrefix("createRaids")
	defer s.logger.PopPrefix()

	devs := []string{}
	for _, array := range config.Storage.Arrays {
		for _, dev := range array.Devices {
			devs = append(devs, string(dev))
		}

		for _, dev := range array.Spares {
			devs = append(devs, string(dev))
		}
	}

	if err := s.waitOnDevices(devs, "raids"); err != nil {
		return err
	}

	for _, md := range config.Storage.Arrays {
		// FIXME(vc): this is utterly flummoxed by a preexisting md.Name, the magic of device-resident md metadata really interferes with us.
		// It's as if what ignition really needs is to turn off automagic md probing/running before getting started.
		args := []string{
			"--create", md.Name,
			"--force",
			"--run",
			"--level", md.Level,
			"--raid-devices", fmt.Sprintf("%d", len(md.Devices)),
			// FIXME(vc): md.Spares, this doesn't map well with mdadm's cli, change config.
		}
		for _, dev := range md.Devices {
			args = append(args, string(dev))
		}

		cmd := exec.Command("/sbin/mdadm", args...)
		err := s.logger.LogOp(cmd.Run, "creating %q", md)
		if err != nil {
			return fmt.Errorf("mdadm failed: %v", err)
		}
	}

	return nil
}

func (s stage) createFilesystems(config config.Config) error {
	if len(config.Storage.Filesystems) == 0 {
		return nil
	}
	s.logger.PushPrefix("createFilesystems")
	defer s.logger.PopPrefix()

	devs := []string{}
	for _, fs := range config.Storage.Filesystems {
		devs = append(devs, string(fs.Device))
	}

	if err := s.waitOnDevices(devs, "filesystems"); err != nil {
		return err
	}

	for _, fs := range config.Storage.Filesystems {
		mkfs := ""
		args := []string(fs.Options)
		switch fs.Format {
		case "btrfs":
			mkfs = "/sbin/mkfs.btrfs"
			args = append(args, "--force")
		case "ext4":
			mkfs = "/sbin/mkfs.ext4"
			args = append(args, "-F")
		default:
			return fmt.Errorf("unsupported filesystem format: %q", fs.Format)
		}

		args = append(args, string(fs.Device))
		cmd := exec.Command(mkfs, args...)

		err := s.logger.LogOp(cmd.Run, "creating %q filesystem on %q", fs.Format, string(fs.Device))
		if err != nil {
			return fmt.Errorf("failed to run %q: %v %v", mkfs, err, args)
		}

		if err := s.createFiles(fs); err != nil {
			return fmt.Errorf("failed to create files %q: %v", fs.Device, err)
		}
	}

	return nil
}

func (s stage) createFiles(fs config.Filesystem) error {
	if len(fs.Files) == 0 {
		return nil
	}
	s.logger.PushPrefix("createFiles")
	defer s.logger.PopPrefix()

	mnt, err := ioutil.TempDir("", "ignition-files")
	if err != nil {
		return fmt.Errorf("failed to create temp directory: %v", err)
	}
	defer os.Remove(mnt)

	dev := string(fs.Device)
	format := string(fs.Format)

	err = s.logger.LogOp(func() error { return syscall.Mount(dev, mnt, format, 0, "") }, "mounting %q at %q", dev, mnt)
	if err != nil {
		return fmt.Errorf("failed to mount device %q at %q: %v", dev, mnt, err)
	}
	defer s.logger.LogOp(func() error { return syscall.Unmount(mnt, 0) }, "unmounting %q at %q", dev, mnt)

	dest := util.DestDir(mnt)
	for _, f := range fs.Files {
		err := s.logger.LogOp(func() error { return dest.WriteFile(&f) }, "writing file %q", string(f.Path))
		if err != nil {
			return fmt.Errorf("failed to create file %q: %v", f.Path, err)
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"path/filepath"

	"github.com/mkideal/cli"
)

type coverT struct {
	cli.Helper
}

var cover = &cli.Command{
	Name: "cover",
	Desc: "Update the artwork for all files with cover.jpg",
	Argv: func() interface{} { return new(coverT) },
	Fn: func(ctx *cli.Context) error {
		if fileExists("cover.jpg") {
			files, _ := filepath.Glob("*.mp3")
			for i := 0; i < len(files); i++ {
				fmt.Println("setting cover on file: " + files[i])
				err := setPicture("cover.jpg", files[i])
				if err != nil {
					log.Fatal("could not save cover", err)
				}
			}
		} else {
			log.Fatal("could not find file: cover.jpg")
		}

		return nil
	},
}
<commit_msg>allow the cover image file to be specified<commit_after>package main

import (
	"fmt"
	"log"
	"path/filepath"

	"github.com/mkideal/cli"
)

type coverT struct {
	cli.Helper
	File string `cli:"file" usage:"A specific image file to use as the cover"`
}

const defaultCoverImage string = "cover.jpg"

var cover = &cli.Command{
	Name: "cover",
	Desc: "Update the artwork for all files with cover.jpg",
	Argv: func() interface{} { return new(coverT) },
	Fn: func(ctx *cli.Context) error {
		argv := ctx.Argv().(*coverT)

		var coverImage string

		if checkNotEmpty(argv.File) {
			coverImage = argv.File
		} else {
			coverImage = defaultCoverImage
		}

		if fileExists(coverImage) {
			files, _ := filepath.Glob("*.mp3")
			for i := 0; i < len(files); i++ {
				fmt.Println("setting cover on file: " + files[i])
				err := setPicture(coverImage, files[i])
				if err != nil {
					log.Fatal("could not save cover", err)
				}
			}
		} else {
			log.Fatal("could not find file: " + coverImage)
		}

		return nil
	},
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package html

import (
	"bytes"
	"io"
	"os"
	"strconv"
)

// A TokenType is the type of a Token.
type TokenType int

const (
	// ErrorToken means that an error occurred during tokenization.
	ErrorToken TokenType = iota
	// TextToken means a text node.
	TextToken
	// A StartTagToken looks like .
	StartTagToken
	// An EndTagToken looks like .
	EndTagToken
	// A SelfClosingTagToken tag looks like
.
	SelfClosingTagToken
	// A CommentToken looks like .
	CommentToken
)

// String returns a string representation of the TokenType.
func (t TokenType) String() string {
	switch t {
	case ErrorToken:
		return "Error"
	case TextToken:
		return "Text"
	case StartTagToken:
		return "StartTag"
	case EndTagToken:
		return "EndTag"
	case SelfClosingTagToken:
		return "SelfClosingTag"
	case CommentToken:
		return "Comment"
	}
	return "Invalid(" + strconv.Itoa(int(t)) + ")"
}

// An Attribute is an attribute key-value pair. Key is alphabetic (and hence
// does not contain escapable characters like '&', '<' or '>'), and Val is
// unescaped (it looks like "a<b" rather than "a<b").
type Attribute struct {
	Key, Val string
}

// A Token consists of a TokenType and some Data (tag name for start and end
// tags, content for text and comments). A tag Token may also contain a slice
// of Attributes. Data is unescaped for all Tokens (it looks like "a<b" rather
// than "a<b").
type Token struct {
	Type TokenType
	Data string
	Attr []Attribute
}

// tagString returns a string representation of a tag Token's Data and Attr.
func (t Token) tagString() string {
	if len(t.Attr) == 0 {
		return t.Data
	}
	buf := bytes.NewBuffer(nil)
	buf.WriteString(t.Data)
	for _, a := range t.Attr {
		buf.WriteByte(' ')
		buf.WriteString(a.Key)
		buf.WriteString(`="`)
		escape(buf, a.Val)
		buf.WriteByte('"')
	}
	return buf.String()
}

// String returns a string representation of the Token.
func (t Token) String() string {
	switch t.Type {
	case ErrorToken:
		return ""
	case TextToken:
		return EscapeString(t.Data)
	case StartTagToken:
		return "<" + t.tagString() + ">"
	case EndTagToken:
		return "</" + t.tagString() + ">"
	case SelfClosingTagToken:
		return "<" + t.tagString() + "/>"
	case CommentToken:
		return ""
	}
	return "Invalid(" + strconv.Itoa(int(t.Type)) + ")"
}

// A Tokenizer returns a stream of HTML Tokens.
type Tokenizer struct {
	// If ReturnComments is set, Next returns comment tokens;
	// otherwise it skips over comments (default).
	ReturnComments bool

	// r is the source of the HTML text.
	r io.Reader
	// tt is the TokenType of the most recently read token. If tt == Error
	// then err is the error associated with trying to read that token.
	tt  TokenType
	err os.Error
	// buf[p0:p1] holds the raw data of the most recent token.
	// buf[p1:] is buffered input that will yield future tokens.
	p0, p1 int
	buf    []byte
}

// Error returns the error associated with the most recent ErrorToken token.
// This is typically os.EOF, meaning the end of tokenization.
func (z *Tokenizer) Error() os.Error {
	if z.tt != ErrorToken {
		return nil
	}
	return z.err
}

// Raw returns the unmodified text of the current token. Calling Next, Token,
// Text, TagName or TagAttr may change the contents of the returned slice.
func (z *Tokenizer) Raw() []byte {
	return z.buf[z.p0:z.p1]
}

// readByte returns the next byte from the input stream, doing a buffered read
// from z.r into z.buf if necessary. z.buf[z.p0:z.p1] remains a contiguous byte
// slice that holds all the bytes read so far for the current token.
func (z *Tokenizer) readByte() (byte, os.Error) {
	if z.p1 >= len(z.buf) {
		// Our buffer is exhausted and we have to read from z.r.
		// We copy z.buf[z.p0:z.p1] to the beginning of z.buf. If the length
		// z.p1 - z.p0 is more than half the capacity of z.buf, then we
		// allocate a new buffer before the copy.
		c := cap(z.buf)
		d := z.p1 - z.p0
		var buf1 []byte
		if 2*d > c {
			buf1 = make([]byte, d, 2*c)
		} else {
			buf1 = z.buf[0:d]
		}
		copy(buf1, z.buf[z.p0:z.p1])
		z.p0, z.p1, z.buf = 0, d, buf1[0:d]
		// Now that we have copied the live bytes to the start of the buffer,
		// we read from z.r into the remainder.
		n, err := z.r.Read(buf1[d:cap(buf1)])
		if err != nil {
			return 0, err
		}
		z.buf = buf1[0 : d+n]
	}
	x := z.buf[z.p1]
	z.p1++
	return x, nil
}

// readTo keeps reading bytes until x is found.
func (z *Tokenizer) readTo(x uint8) os.Error {
	for {
		c, err := z.readByte()
		if err != nil {
			return err
		}
		switch c {
		case x:
			return nil
		case '\\':
			_, err = z.readByte()
			if err != nil {
				return err
			}
		}
	}
	panic("unreachable")
}

// nextMarkupDeclaration returns the next TokenType starting with "<!".
func (z *Tokenizer) nextMarkupDeclaration() (TokenType, os.Error) {
	// TODO: check for <!DOCTYPE ... >, don't just assume that it's a comment.
	for i := 0; i < 2; i++ {
		c, err := z.readByte()
		if err != nil {
			return TextToken, err
		}
		if c != '-' {
			return z.nextText(), nil
		}
	}
	// " from the right.
		// ".
	CommentToken
)

// String returns a string representation of the TokenType.
func (t TokenType) String() string {
	switch t {
	case ErrorToken:
		return "Error"
	case TextToken:
		return "Text"
	case StartTagToken:
		return "StartTag"
	case EndTagToken:
		return "EndTag"
	case SelfClosingTagToken:
		return "SelfClosingTag"
	case CommentToken:
		return "Comment"
	}
	return "Invalid(" + strconv.Itoa(int(t)) + ")"
}

// An Attribute is an attribute key-value pair. Key is alphabetic (and hence
// does not contain escapable characters like '&', '<' or '>'), and Val is
// unescaped (it looks like "a<b" rather than "a<b").
type Attribute struct {
	Key, Val string
}

// A Token consists of a TokenType and some Data (tag name for start and end
// tags, content for text and comments). A tag Token may also contain a slice
// of Attributes. Data is unescaped for all Tokens (it looks like "a<b" rather
// than "a<b").
type Token struct {
	Type TokenType
	Data string
	Attr []Attribute
}

// tagString returns a string representation of a tag Token's Data and Attr.
func (t Token) tagString() string {
	if len(t.Attr) == 0 {
		return t.Data
	}
	buf := bytes.NewBuffer(nil)
	buf.WriteString(t.Data)
	for _, a := range t.Attr {
		buf.WriteByte(' ')
		buf.WriteString(a.Key)
		buf.WriteString(`="`)
		escape(buf, a.Val)
		buf.WriteByte('"')
	}
	return buf.String()
}

// String returns a string representation of the Token.
func (t Token) String() string {
	switch t.Type {
	case ErrorToken:
		return ""
	case TextToken:
		return EscapeString(t.Data)
	case StartTagToken:
		return "<" + t.tagString() + ">"
	case EndTagToken:
		return "</" + t.tagString() + ">"
	case SelfClosingTagToken:
		return "<" + t.tagString() + "/>"
	case CommentToken:
		return ""
	}
	return "Invalid(" + strconv.Itoa(int(t.Type)) + ")"
}

// A Tokenizer returns a stream of HTML Tokens.
type Tokenizer struct {
	// If ReturnComments is set, Next returns comment tokens;
	// otherwise it skips over comments (default).
	ReturnComments bool

	// r is the source of the HTML text.
	r io.Reader
	// tt is the TokenType of the most recently read token. If tt == Error
	// then err is the error associated with trying to read that token.
	tt  TokenType
	err os.Error
	// buf[p0:p1] holds the raw data of the most recent token.
	// buf[p1:] is buffered input that will yield future tokens.
	p0, p1 int
	buf    []byte
}

// Error returns the error associated with the most recent ErrorToken token.
// This is typically os.EOF, meaning the end of tokenization.
func (z *Tokenizer) Error() os.Error {
	if z.tt != ErrorToken {
		return nil
	}
	return z.err
}

// Raw returns the unmodified text of the current token. Calling Next, Token,
// Text, TagName or TagAttr may change the contents of the returned slice.
func (z *Tokenizer) Raw() []byte {
	return z.buf[z.p0:z.p1]
}

// readByte returns the next byte from the input stream, doing a buffered read
// from z.r into z.buf if necessary. z.buf[z.p0:z.p1] remains a contiguous byte
// slice that holds all the bytes read so far for the current token.
func (z *Tokenizer) readByte() (byte, os.Error) {
	if z.p1 >= len(z.buf) {
		// Our buffer is exhausted and we have to read from z.r.
		// We copy z.buf[z.p0:z.p1] to the beginning of z.buf. If the length
		// z.p1 - z.p0 is more than half the capacity of z.buf, then we
		// allocate a new buffer before the copy.
		c := cap(z.buf)
		d := z.p1 - z.p0
		var buf1 []byte
		if 2*d > c {
			buf1 = make([]byte, d, 2*c)
		} else {
			buf1 = z.buf[0:d]
		}
		copy(buf1, z.buf[z.p0:z.p1])
		z.p0, z.p1, z.buf = 0, d, buf1[0:d]
		// Now that we have copied the live bytes to the start of the buffer,
		// we read from z.r into the remainder.
		n, err := z.r.Read(buf1[d:cap(buf1)])
		if err != nil {
			return 0, err
		}
		z.buf = buf1[0 : d+n]
	}
	x := z.buf[z.p1]
	z.p1++
	return x, nil
}

// readTo keeps reading bytes until x is found.
func (z *Tokenizer) readTo(x uint8) os.Error {
	for {
		c, err := z.readByte()
		if err != nil {
			return err
		}
		switch c {
		case x:
			return nil
		case '\\':
			_, err = z.readByte()
			if err != nil {
				return err
			}
		}
	}
	panic("unreachable")
}

// nextMarkupDeclaration returns the next TokenType starting with "<!".
func (z *Tokenizer) nextMarkupDeclaration() (TokenType, os.Error) {
	// TODO: check for <!DOCTYPE ... >, don't just assume that it's a comment.
	for i := 0; i < 2; i++ {
		c, err := z.readByte()
		if err != nil {
			return TextToken, err
		}
		if c != '-' {
			return z.nextText(), nil
		}
	}
	// " from the right.
		// "


<commit_before>// Copyright 2015 Luke Shumaker

package httpentity

import (
	"net/http"
	"net/url"
	"strings"
)

func (r *Router) Route(req Request, u *url.URL) (res Response) {
	u, mimetype := normalizeURL(u)
	if mimetype != "" {
		// the file extension overrides the Accept: header
		req.Headers.Set("Accept", mimetype)
	}

	defer r.finish(req, u, &res)

	for _, middleware := range r.Middlewares {
		middleware.Before(&req)
		defer middleware.After(req, &res)
	}
	res = r.route(req, u)
	return
}

func (h *Router) serveHTTP(w http.ResponseWriter, r *http.Request) (res Response) {
	// adapt the request from `net/http` format to `httpentity` format
	req := Request{
		Scheme:  "http",
		Method:  r.Method,
		Headers: r.Header,
		Query:   r.URL.Query(),
		Entity:  nil,
		Things:  map[string]interface{}{},
	}
	if r.TLS != nil {
		req.Scheme = "https"
	}
	if h.TrustForwarded {
		if scheme := req.Headers.Get("X-Forwarded-Proto"); scheme != "" {
			fmt.Printf("X-Forwarded-TrustForwarded: %v\n", h.TrustForwarded)
			req.Scheme = scheme
		}
		if str := req.Headers.Get("Forwarded"); str != "" {
			parts := strings.Split(str, ";")
			for i := range parts {
				ary := strings.SplitN(parts[i], "=", 2)
				if len(ary) == 2 {
					if strings.EqualFold("proto", ary[0]) {
						req.Scheme = ary[1]
					}
				}
			}
		}
	}
	u, mimetype := normalizeURL(r.URL)
	if mimetype != "" {
		// the file extension overrides the Accept: header
		req.Headers.Set("Accept", mimetype)
	}

	defer h.finish(req, u, &res)

	// parse the submitted entity
	switch req.Method {
	case "POST", "PUT", "PATCH":
		resperr := req.readEntity(h, r.Body, r.Header.Get("Content-Type"))
		if resperr != nil {
			return *resperr
		}
	}

	// run the request
	for _, middleware := range h.Middlewares {
		middleware.Before(&req)
		defer middleware.After(req, &res)
	}
	res = h.route(req, u)

	return
}

func (h *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	res := h.serveHTTP(w, req)
	// Adapt the response from `httpentity` format to `net/http` format
	for k, v := range res.Headers {
		w.Header().Set(k, strings.Join(v, ", "))
	}
	w.WriteHeader(int(res.Status))
	res.writeEntity(w)
}
<commit_msg>oops, I left a debug statement in a couple of commits ago<commit_after>// Copyright 2015 Luke Shumaker

package httpentity

import (
	"net/http"
	"net/url"
	"strings"
)

func (r *Router) Route(req Request, u *url.URL) (res Response) {
	u, mimetype := normalizeURL(u)
	if mimetype != "" {
		// the file extension overrides the Accept: header
		req.Headers.Set("Accept", mimetype)
	}

	defer r.finish(req, u, &res)

	for _, middleware := range r.Middlewares {
		middleware.Before(&req)
		defer middleware.After(req, &res)
	}
	res = r.route(req, u)
	return
}

func (h *Router) serveHTTP(w http.ResponseWriter, r *http.Request) (res Response) {
	// adapt the request from `net/http` format to `httpentity` format
	req := Request{
		Scheme:  "http",
		Method:  r.Method,
		Headers: r.Header,
		Query:   r.URL.Query(),
		Entity:  nil,
		Things:  map[string]interface{}{},
	}
	if r.TLS != nil {
		req.Scheme = "https"
	}
	if h.TrustForwarded {
		if scheme := req.Headers.Get("X-Forwarded-Proto"); scheme != "" {
			req.Scheme = scheme
		}
		if str := req.Headers.Get("Forwarded"); str != "" {
			parts := strings.Split(str, ";")
			for i := range parts {
				ary := strings.SplitN(parts[i], "=", 2)
				if len(ary) == 2 {
					if strings.EqualFold("proto", ary[0]) {
						req.Scheme = ary[1]
					}
				}
			}
		}
	}
	u, mimetype := normalizeURL(r.URL)
	if mimetype != "" {
		// the file extension overrides the Accept: header
		req.Headers.Set("Accept", mimetype)
	}

	defer h.finish(req, u, &res)

	// parse the submitted entity
	switch req.Method {
	case "POST", "PUT", "PATCH":
		resperr := req.readEntity(h, r.Body, r.Header.Get("Content-Type"))
		if resperr != nil {
			return *resperr
		}
	}

	// run the request
	for _, middleware := range h.Middlewares {
		middleware.Before(&req)
		defer middleware.After(req, &res)
	}
	res = h.route(req, u)

	return
}

func (h *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	res := h.serveHTTP(w, req)
	// Adapt the response from `httpentity` format to `net/http` format
	for k, v := range res.Headers {
		w.Header().Set(k, strings.Join(v, ", "))
	}
	w.WriteHeader(int(res.Status))
	res.writeEntity(w)
}
<|endoftext|>