<commit_before>package main

import (
	"os"
)

type CodeGenerator struct {
	writer          Writer
	lineChannel     chan string
	source          Source
	root            *Node
	namePrefix      string
	nameSuffix      string
	nameSpaceTagMap map[string]string
	globalNodeMap   map[string]*Node
}

func (v *CodeGenerator) init(root *Node, globalNodeMap map[string]*Node, namePrefix string, nameSpaceTagMap map[string]string, nameSuffix string, codegenDir string, codegenGoFile string, source Source, lineChannel chan string) (Writer, error) {
	v.root = root
	v.namePrefix = namePrefix
	v.nameSuffix = nameSuffix
	v.lineChannel = lineChannel
	v.source = source
	v.nameSpaceTagMap = nameSpaceTagMap
	v.globalNodeMap = globalNodeMap

	var err error
	err = os.RemoveAll(codegenDir)
	if err != nil {
		return nil, err
	}

	if _, err := os.Stat(codegenDir); os.IsNotExist(err) {
		err = os.Mkdir(codegenDir, 0777)
		if err != nil {
			return nil, err
		}
	}

	v.writer = new(fileWriter)
	v.writer.open(codegenDir+"/"+codegenGoFile, lineChannel)

	return v.writer, err

}

func (v *CodeGenerator) generateCodePre(hasStartElements bool, url bool, writeJson bool, writeXml bool) {
	v.lineChannel <- "package main"
	v.lineChannel <- " "

	v.lineChannel <- "import ("
	if !hasStartElements {
		v.lineChannel <- "	\"os\""
		v.lineChannel <- "	\"log\""
		v.lineChannel <- ")"
		return
	}
	if url {
		v.lineChannel <- "	\"net/http\""
	}
	v.lineChannel <- "	\"encoding/xml\""
	if writeJson {
		v.lineChannel <- "\"encoding/json\""
	}

	if writeJson || writeXml {
		v.lineChannel <- "	\"fmt\""
	}
	v.lineChannel <- "	\"os\""
	v.lineChannel <- "      \"bufio\""
	v.lineChannel <- "      \"log\""
	v.lineChannel <- "      \"compress/bzip2\""
	v.lineChannel <- "      \"compress/gzip\""
	v.lineChannel <- "      \"io\""
	v.lineChannel <- "      \"strings\""
	v.lineChannel <- ")"

	v.lineChannel <- "\n\n"

	makeGenericReaderCode(v.lineChannel)
}

func (v *CodeGenerator) makeName(name string) string {
	//return v.namePrefix + capitalizeFirstLetter(name) + v.nameSuffix
	return v.namePrefix + cleanName(name) + v.nameSuffix
}

func (v *CodeGenerator) generateConvertToCode(firstNode *Node, writeJson bool, writeXml bool, prettyPrint bool) {
	v.lineChannel <- "func main(){"
	v.lineChannel <- " "
	v.makeDecoder(firstNode, writeJson, writeXml, prettyPrint)
	v.lineChannel <- "}"
}

func (v *CodeGenerator) generateVerifyCode(hasStartElements bool, globalTagAttributes map[string]([]*FQN), url bool) {
	v.lineChannel <- "func main(){"
	v.lineChannel <- " "
	if !hasStartElements {
		v.lineChannel <- "log.Print(\"++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\")"
		v.lineChannel <- "log.Print(\"++++++ No StartElements encountered: possible error in XML or bug+++++++\")"
		v.lineChannel <- "log.Print(\"++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\")"
		v.lineChannel <- "os.Exit(42)"
		v.lineChannel <- "}"
		return
	}

	for _, node := range v.globalNodeMap {
		v.makeDecoder(node, false, false, false)
	}
	v.lineChannel <- "}"

}

func (v *CodeGenerator) printTokenExtractor(name string, space string, spaceTag string, writeJson bool, writeXml bool, prettyPrint bool) {
	v.lineChannel <- "			if se.Name.Local == \"" + name + "\" && se.Name.Space == \"" + space + "\" {"
	if !writeJson && !writeXml {
		v.lineChannel <- "				count += 1"
	}
	structName := makeTypeGeneric(name, spaceTag, v.namePrefix, v.nameSuffix)
	v.lineChannel <- "				var item " + structName
	v.lineChannel <- "				decoder.DecodeElement(&item, &se)"
	if writeJson {
		if prettyPrint {
			v.lineChannel <- "				b, err := json.MarshalIndent(item, \"\", \" \")"
		} else {
			v.lineChannel <- "				b, err := json.Marshal(item)"
		}
	} else {
		if writeXml {
			v.lineChannel <- "				tmp := struct {"
			v.lineChannel <- "				" + structName
			v.lineChannel <- "				XMLName struct{} `xml:\"" + name + "\"`"
			v.lineChannel <- "				}{" + structName + ": item}"

			v.lineChannel <- "				if err := enc.Encode(tmp); err != nil {"
			v.lineChannel <- "				    fmt.Printf(\"error: %v\\n\", err)"
			v.lineChannel <- "				}"
		}
	}

	if writeJson || writeXml {
		v.lineChannel <- "				if err != nil {"
		v.lineChannel <- "				    log.Fatal(err)"
		v.lineChannel <- "				}"
	}
	if writeJson {
		v.lineChannel <- "				fmt.Println(string(b))"
	}
	v.lineChannel <- ""
	v.lineChannel <- "			}"
}

func (v *CodeGenerator) makeReader(url bool) {
	if !url {
		v.makeFileReader()
	} else {
		v.makeUrlReader()
	}

}

func (v *CodeGenerator) makeUrlReader() {
	v.lineChannel <- "        res, err := http.Get(\"" + v.source.getName() + "\")"
	v.lineChannel <- "        if err != nil {"
	v.lineChannel <- "        		log.Fatal(err)"
	v.lineChannel <- "        		return"
	v.lineChannel <- "        	}"
	v.lineChannel <- "        if res.StatusCode != 200 {"
	v.lineChannel <- "        		log.Fatal(\"ERROR: bad http status code != 200: \", res.StatusCode)"
	v.lineChannel <- "        		return"
	v.lineChannel <- "        	}"
	v.lineChannel <- "        reader := res.Body"
}

func (v *CodeGenerator) makeFileReader() {

	v.lineChannel <- "reader, xmlFile, err := genericReader(\"../" + v.source.getName() + "\")"

	v.lineChannel <- "if err != nil {"
	v.lineChannel <- "log.Fatal(err)"
	v.lineChannel <- "return"
	v.lineChannel <- "}"
	v.lineChannel <- "defer xmlFile.Close()"
}

func (v *CodeGenerator) makeCountVar(node *Node) string {
	return "count_" + node.name + "_" + v.nameSpaceTagMap[node.space]
}

func (v *CodeGenerator) makeTagSpaceMap(varName string) string {
	s := "\n\t" + varName + ":= map[string]string{"
	for _, v := range v.globalNodeMap {
		s += "\n\t\t\"" + v.name + "\":\"" + v.space + "\","
	}
	s += "\n\t}"
	return s
}

func (v *CodeGenerator) makeTagSpaceTagMap(nameSpaceTagMap map[string]string, varName string) string {
	s := "\n\t" + varName + ":= map[string]string{"
	for k, v := range nameSpaceTagMap {
		s += "\n\t\t\"" + k + "\":\"" + v + "\","
	}
	s += "\n\t}"
	return s
}

func (v *CodeGenerator) makeDecoder(node *Node, writeJson bool, writeXml bool, prettyPrint bool) {
	v.lineChannel <- "     {"
	if !writeJson && !writeXml {
		v.lineChannel <- "       count:= 0"
	}

	v.makeReader(url)
	if writeXml {
		v.lineChannel <- "enc := xml.NewEncoder(os.Stdout)"
		if prettyPrint {
			v.lineChannel <- "enc.Indent(\"  \", \"    \")"
		}
	}

	v.lineChannel <- "decoder := xml.NewDecoder(reader)"

	v.lineChannel <- "	for{"
	v.lineChannel <- "		token, _ := decoder.Token()"
	v.lineChannel <- "		if token == nil {"
	v.lineChannel <- "		    break"
	v.lineChannel <- "		}"
	v.lineChannel <- "		switch se := token.(type) {"
	v.lineChannel <- "		case xml.StartElement:"

	v.printTokenExtractor(node.name, node.space, node.spaceTag, writeJson, writeXml, prettyPrint)
	v.lineChannel <- "	   }"
	v.lineChannel <- "        }"
	spaceTag := ""
	if node.spaceTag != "" {
		spaceTag = node.spaceTag + ":"
	}

	if !writeJson && !writeXml {
		v.lineChannel <- "     log.Print(\"Number of " + spaceTag + node.name + "= \", count)"
	}

	v.lineChannel <- "    }"
}

func makeGenericReaderCode(lineChannel chan string) {
	lineChannel <- "func genericReader(filename string) (io.Reader, *os.File, error) {"
	lineChannel <- "    file, err := os.Open(filename)"
	lineChannel <- "    if err != nil {"
	lineChannel <- "        return nil, nil, err"
	lineChannel <- "    }"
	lineChannel <- "    if strings.HasSuffix(filename, \"bz2\") {"
	lineChannel <- "        return bufio.NewReader(bzip2.NewReader(bufio.NewReader(file))), file, err"
	lineChannel <- "    }"
	lineChannel <- ""
	lineChannel <- "    if strings.HasSuffix(filename, \"gz\") {"
	lineChannel <- "        reader, err := gzip.NewReader(bufio.NewReader(file))"
	lineChannel <- "        if err != nil {"
	lineChannel <- "            return nil, nil, err"
	lineChannel <- "        }"
	lineChannel <- "        return bufio.NewReader(reader), file, err"
	lineChannel <- "    }"
	lineChannel <- "    return bufio.NewReader(file), file, err"
	lineChannel <- "}"
}
<commit_msg>Cleanup: removed codegen.go<commit_after><|endoftext|>


<commit_before>// Copyright 2017 Aleksandr Demakin. All rights reserved.

package coincap

import (
	"encoding/json"
	"net/http"

	gosio "github.com/graarh/golang-socketio"
	"github.com/graarh/golang-socketio/transport"
	"github.com/pkg/errors"
)

// HistoryInterval* consts are used in History() request.
const (
	HistoryInterval1Day    = "1day"
	HistoryInterval7Days   = "7day"
	HistoryInterval30Days  = "30day"
	HistoryInterval90Days  = "90day"
	HistoryInterval180Days = "180day"
	HistoryInterval365Days = "365day"
)

const (
	cAPIURL = "http://coincap.io/"
	cWsURL  = "socket.coincap.io"
)

type market struct {
	Time                 int64
	Shapeshift           bool
	Published            bool
	IsBuy                bool
	Short                string
	Long                 string
	Position24           json.Number
	Position             json.Number
	Perc                 json.Number
	Volume               json.Number
	USDVolume            json.Number
	Cap24hrChange        json.Number
	Mktcap               json.Number
	Supply               json.Number
	Cap24hrChangePercent json.Number
	VwapData             *json.Number
	VwapDataBTC          *json.Number
}

// Front is a reply for /front path.
type Front struct {
	market
	Price json.Number
}

// Trade is a websocket message from 'trades' channel.
type Trade struct {
	Coin string
	Msg  Front
}

// Global is a websocket message from 'global' channel,
// and a reply for /global path.
type Global struct {
	BTCPrice      json.Number
	BTCCap        json.Number
	AltCap        json.Number
	Dom           json.Number
	BitnodesCount json.Number
}

// Mapping is a reply for /map path.
type Mapping struct {
	Name    string
	Symbol  string
	Aliases []string
}

// Page is a reply for /page path.
type Page struct {
	Global
	priceAndCap
	market
	USDPrice      json.Number
	HomeURL       string
	ExplorerURL   string
	Twitter       string
	DiscussionURL string
	Mineable      bool
	Premined      bool
	PreminedSig   bool
}

type priceAndCap struct {
	Price      [][2]json.Number
	Market_Cap [][2]json.Number
}

// History is a reply for /history path.
type History struct {
	priceAndCap
	Volume [][2]json.Number
}

// Client send API requests and parses responses.
// It also can be used for subscription on websocket.
type Client struct {
	cl *http.Client
}

// New returns new Client.
func New() *Client {
	return &Client{cl: &http.Client{}}
}

// Coins requests /coins path.
func (c *Client) Coins() ([]string, error) {
	var result []string
	if err := c.get("coins", &result); err != nil {
		return nil, err
	}
	return result, nil
}

// CoinsXCP requests coins/xcp path
func (c *Client) CoinsXCP() ([]string, error) {
	var result []string
	if err := c.get("coins/xcp", &result); err != nil {
		return nil, err
	}
	return result, nil
}

// CoinsXCPAll requests coins/xcp/all path.
func (c *Client) CoinsXCPAll() ([]string, error) {
	var result []string
	if err := c.get("coins/xcp/all", &result); err != nil {
		return nil, err
	}
	return result, nil
}

// Map requests /map path.
func (c *Client) Map() ([]Mapping, error) {
	var result []Mapping
	if err := c.get("map", &result); err != nil {
		return nil, err
	}
	return result, nil
}

// Global requests /global path.
func (c *Client) Global() (Global, error) {
	var result Global
	err := c.get("global", &result)
	return result, err
}

// Front requests /front path.
func (c *Client) Front() ([]Front, error) {
	var result []Front
	if err := c.get("front", &result); err != nil {
		return nil, err
	}
	return result, nil
}

// FrontXCP requests front/xcp path.
func (c *Client) FrontXCP() ([]Front, error) {
	var result []Front
	if err := c.get("front/xcp", &result); err != nil {
		return nil, err
	}
	return result, nil
}

// Page requests /page path for given symbol.
func (c *Client) Page(symb string) (*Page, error) {
	var result Page
	if err := c.get("page/"+symb, &result); err != nil {
		return nil, err
	}
	return &result, nil
}

// History requests /history path for given symbol.
//	interval can be either empty (returns all history on a coin),
//	or one of the HistoryInterval* consts.
func (c *Client) History(symb, interval string) (*History, error) {
	var result History
	if len(interval) > 0 {
		interval += "/"
	}
	if err := c.get("history/"+interval+symb, &result); err != nil {
		return nil, err
	}
	return &result, nil
}

func (c *Client) get(url string, value interface{}) error {
	resp, err := c.cl.Get(cAPIURL + url)
	if err != nil {
		return errors.Wrap(err, "http request error")
	}
	defer resp.Body.Close()
	if err := json.NewDecoder(resp.Body).Decode(value); err != nil {
		return errors.Wrap(err, "failed to decode request")
	}
	return nil
}

// SubscribeGlobal subscribes for websocket messages on 'global' channel.
// All incoming messages are sent to 'dataChan'.
// If there are errors during subscription, it returns an error immediately.
// Otherwise, it blocks, waiting for an error, or stop signal.
// If an error occures, it will be returned as the result.
// Send to/close of 'stopChan' will close connection, and nil will be returned.
func (c *Client) SubscribeGlobal(dataChan chan<- *Global, stopChan <-chan struct{}) error {
	type wrapper struct {
		Message Global
	}
	return c.subscribe("global", func(ch *gosio.Channel, gm *wrapper) {
		dataChan <- &gm.Message
	}, stopChan)
}

// SubscribeTrades subscribes for websocket messages on 'trades' channel.
// All incoming messages are sent to 'dataChan'.
// If there are errors during subscription, it returns an error immediately.
// Otherwise, it blocks, waiting for an error, or stop signal.
// If an error occures, it will be returned as the result.
// Send to/close of 'stopChan' will close connection, and nil will be returned.
func (c *Client) SubscribeTrades(dataChan chan<- *Trade, stopChan <-chan struct{}) error {
	type wrapper struct {
		Message Trade
	}
	return c.subscribe("trades", func(ch *gosio.Channel, tm *wrapper) {
		dataChan <- &tm.Message
	}, stopChan)
}

func (c *Client) subscribe(method string, handler interface{}, stopChan <-chan struct{}) error {
	client, err := gosio.Dial(gosio.GetUrl(cWsURL, 80, false), transport.GetDefaultWebsocketTransport())
	if err != nil {
		return errors.Wrap(err, "coincap: ws dial error")
	}
	defer client.Close()
	errCh := make(chan error, 2)
	err = client.On(gosio.OnDisconnection, func(ch *gosio.Channel) {
		errCh <- errors.Errorf("websocket disconnected on channel %s", ch.Id())
	})
	if err != nil {
		return errors.Wrap(err, "failed to setup disconnect handler")
	}
	err = client.On(gosio.OnError, func(ch *gosio.Channel) {
		errCh <- errors.Errorf("websocket error on channel %s", ch.Id())
	})
	if err != nil {
		return errors.Wrap(err, "failed to setup error handler")
	}
	if err = client.On(method, handler); err != nil {
		return errors.Wrap(err, "failed to setup message handler")
	}
	select {
	case err := <-errCh:
		return err
	case <-stopChan:
		return nil
	}
}
<commit_msg>comments: add package comment.<commit_after>// Copyright 2017 Aleksandr Demakin. All rights reserved.

// Package coincap is a simple coincap.io API wrapper library.
// Note, that in many structs json.Number is used instead of string or float64.
// This is because for some unknown reason coincap may send
// same fields as numbers or as strings.
package coincap

import (
	"encoding/json"
	"net/http"

	gosio "github.com/graarh/golang-socketio"
	"github.com/graarh/golang-socketio/transport"
	"github.com/pkg/errors"
)

// HistoryInterval* consts are used in History() request.
const (
	HistoryInterval1Day    = "1day"
	HistoryInterval7Days   = "7day"
	HistoryInterval30Days  = "30day"
	HistoryInterval90Days  = "90day"
	HistoryInterval180Days = "180day"
	HistoryInterval365Days = "365day"
)

const (
	cAPIURL = "http://coincap.io/"
	cWsURL  = "socket.coincap.io"
)

type market struct {
	Time                 int64
	Shapeshift           bool
	Published            bool
	IsBuy                bool
	Short                string
	Long                 string
	Position24           json.Number
	Position             json.Number
	Perc                 json.Number
	Volume               json.Number
	USDVolume            json.Number
	Cap24hrChange        json.Number
	Mktcap               json.Number
	Supply               json.Number
	Cap24hrChangePercent json.Number
	VwapData             *json.Number
	VwapDataBTC          *json.Number
}

// Front is a reply for /front path.
type Front struct {
	market
	Price json.Number
}

// Trade is a websocket message from 'trades' channel.
type Trade struct {
	Coin string
	Msg  Front
}

// Global is a websocket message from 'global' channel,
// and a reply for /global path.
type Global struct {
	BTCPrice      json.Number
	BTCCap        json.Number
	AltCap        json.Number
	Dom           json.Number
	BitnodesCount json.Number
}

// Mapping is a reply for /map path.
type Mapping struct {
	Name    string
	Symbol  string
	Aliases []string
}

// Page is a reply for /page path.
type Page struct {
	Global
	priceAndCap
	market
	USDPrice      json.Number
	HomeURL       string
	ExplorerURL   string
	Twitter       string
	DiscussionURL string
	Mineable      bool
	Premined      bool
	PreminedSig   bool
}

type priceAndCap struct {
	Price      [][2]json.Number
	Market_Cap [][2]json.Number
}

// History is a reply for /history path.
type History struct {
	priceAndCap
	Volume [][2]json.Number
}

// Client send API requests and parses responses.
// It also can be used for subscription on websocket.
type Client struct {
	cl *http.Client
}

// New returns new Client.
func New() *Client {
	return &Client{cl: &http.Client{}}
}

// Coins requests /coins path.
func (c *Client) Coins() ([]string, error) {
	var result []string
	if err := c.get("coins", &result); err != nil {
		return nil, err
	}
	return result, nil
}

// CoinsXCP requests coins/xcp path
func (c *Client) CoinsXCP() ([]string, error) {
	var result []string
	if err := c.get("coins/xcp", &result); err != nil {
		return nil, err
	}
	return result, nil
}

// CoinsXCPAll requests coins/xcp/all path.
func (c *Client) CoinsXCPAll() ([]string, error) {
	var result []string
	if err := c.get("coins/xcp/all", &result); err != nil {
		return nil, err
	}
	return result, nil
}

// Map requests /map path.
func (c *Client) Map() ([]Mapping, error) {
	var result []Mapping
	if err := c.get("map", &result); err != nil {
		return nil, err
	}
	return result, nil
}

// Global requests /global path.
func (c *Client) Global() (Global, error) {
	var result Global
	err := c.get("global", &result)
	return result, err
}

// Front requests /front path.
func (c *Client) Front() ([]Front, error) {
	var result []Front
	if err := c.get("front", &result); err != nil {
		return nil, err
	}
	return result, nil
}

// FrontXCP requests front/xcp path.
func (c *Client) FrontXCP() ([]Front, error) {
	var result []Front
	if err := c.get("front/xcp", &result); err != nil {
		return nil, err
	}
	return result, nil
}

// Page requests /page path for given symbol.
func (c *Client) Page(symb string) (*Page, error) {
	var result Page
	if err := c.get("page/"+symb, &result); err != nil {
		return nil, err
	}
	return &result, nil
}

// History requests /history path for given symbol.
//	interval can be either empty (returns all history on a coin),
//	or one of the HistoryInterval* consts.
func (c *Client) History(symb, interval string) (*History, error) {
	var result History
	if len(interval) > 0 {
		interval += "/"
	}
	if err := c.get("history/"+interval+symb, &result); err != nil {
		return nil, err
	}
	return &result, nil
}

func (c *Client) get(url string, value interface{}) error {
	resp, err := c.cl.Get(cAPIURL + url)
	if err != nil {
		return errors.Wrap(err, "http request error")
	}
	defer resp.Body.Close()
	if err := json.NewDecoder(resp.Body).Decode(value); err != nil {
		return errors.Wrap(err, "failed to decode request")
	}
	return nil
}

// SubscribeGlobal subscribes for websocket messages on 'global' channel.
// All incoming messages are sent to 'dataChan'.
// If there are errors during subscription, it returns an error immediately.
// Otherwise, it blocks, waiting for an error, or stop signal.
// If an error occures, it will be returned as the result.
// Send to/close of 'stopChan' will close connection, and nil will be returned.
func (c *Client) SubscribeGlobal(dataChan chan<- *Global, stopChan <-chan struct{}) error {
	type wrapper struct {
		Message Global
	}
	return c.subscribe("global", func(ch *gosio.Channel, gm *wrapper) {
		dataChan <- &gm.Message
	}, stopChan)
}

// SubscribeTrades subscribes for websocket messages on 'trades' channel.
// All incoming messages are sent to 'dataChan'.
// If there are errors during subscription, it returns an error immediately.
// Otherwise, it blocks, waiting for an error, or stop signal.
// If an error occures, it will be returned as the result.
// Send to/close of 'stopChan' will close connection, and nil will be returned.
func (c *Client) SubscribeTrades(dataChan chan<- *Trade, stopChan <-chan struct{}) error {
	type wrapper struct {
		Message Trade
	}
	return c.subscribe("trades", func(ch *gosio.Channel, tm *wrapper) {
		dataChan <- &tm.Message
	}, stopChan)
}

func (c *Client) subscribe(method string, handler interface{}, stopChan <-chan struct{}) error {
	client, err := gosio.Dial(gosio.GetUrl(cWsURL, 80, false), transport.GetDefaultWebsocketTransport())
	if err != nil {
		return errors.Wrap(err, "coincap: ws dial error")
	}
	defer client.Close()
	errCh := make(chan error, 2)
	err = client.On(gosio.OnDisconnection, func(ch *gosio.Channel) {
		errCh <- errors.Errorf("websocket disconnected on channel %s", ch.Id())
	})
	if err != nil {
		return errors.Wrap(err, "failed to setup disconnect handler")
	}
	err = client.On(gosio.OnError, func(ch *gosio.Channel) {
		errCh <- errors.Errorf("websocket error on channel %s", ch.Id())
	})
	if err != nil {
		return errors.Wrap(err, "failed to setup error handler")
	}
	if err = client.On(method, handler); err != nil {
		return errors.Wrap(err, "failed to setup message handler")
	}
	select {
	case err := <-errCh:
		return err
	case <-stopChan:
		return nil
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"sort"
	"strings"
)

func init() {
	Topics = append(Topics, TopicSet{
		{
			Name:        "commands",
			Description: "list all commands",
			Hidden:      true,
			Commands: CommandSet{
				{
					Topic:            "commands",
					Description:      "list all commands",
					Flags:            []Flag{{Name: "json"}},
					DisableAnalytics: true,
					Run: func(ctx *Context) {
						commands := AllCommands().Sort()
						if ctx.Flags["json"] == true {
							commands.loadUsages()
							commands.loadFullHelp()
							doc := map[string]interface{}{"topics": Topics, "commands": commands}
							s, _ := json.Marshal(doc)
							Println(string(s))
							return
						}
						for _, command := range commands {
							if command.Hidden {
								continue
							}
							if command.Command == "" {
								Printf("%s\n", command.Topic)
							} else {
								Printf("%s:%s\n", command.Topic, command.Command)
							}
						}
					},
				},
			},
		},
	}...)
}

// Command represents a CLI command.
// For example, in the command `heroku apps:create` the command would be `create`.
// They must have a Topic name that links to a real topic's name.
type Command struct {
	Topic            string             `json:"topic"`
	Command          string             `json:"command,omitempty"`
	Plugin           string             `json:"plugin"`
	Usage            string             `json:"usage"`
	Description      string             `json:"description"`
	Default          bool               `json:"default"`
	Help             string             `json:"help"`
	FullHelp         string             `json:"fullHelp"`
	Hidden           bool               `json:"hidden"`
	NeedsApp         bool               `json:"needsApp"`
	WantsApp         bool               `json:"wantsApp"`
	NeedsOrg         bool               `json:"needsOrg"`
	WantsOrg         bool               `json:"wantsOrg"`
	NeedsAuth        bool               `json:"needsAuth"`
	VariableArgs     bool               `json:"variableArgs"`
	DisableAnalytics bool               `json:"disableAnalytics"`
	Args             []Arg              `json:"args"`
	Flags            []Flag             `json:"flags"`
	Run              func(ctx *Context) `json:"-"`
}

func (c Command) String() string {
	if c.Command == "" {
		return c.Topic
	}
	return c.Topic + ":" + c.Command
}

func CommandUsage(c *Command) string {
	if c.Usage != "" {
		return c.Usage
	}
	return c.String() + argsString(c.Args)
}

func (c *Command) buildFlagHelp() string {
	flags := c.Flags
	if c.NeedsApp || c.WantsApp {
		flags = append(flags, *AppFlag, *RemoteFlag)
	}
	if c.NeedsOrg || c.WantsOrg {
		flags = append(flags, *OrgFlag)
	}
	lines := make([]string, 0, len(flags))
	for _, flag := range flags {
		if flag.Hidden {
			continue
		}
		if flag.Description == "" {
			lines = append(lines, flag.String())
		} else {
			lines = append(lines, fmt.Sprintf("%-20s # %s", flag.String(), flag.Description))
		}
	}
	return strings.Join(lines, "\n")
}

func (c *Command) buildFullHelp() string {
	sections := make([]string, 0, 3)
	if c.Description != "" {
		sections = append(sections, c.Description)
	}
	flagHelp := c.buildFlagHelp()
	if flagHelp != "" {
		sections = append(sections, flagHelp)
	}
	if c.Help != "" {
		sections = append(sections, c.Help)
	}
	return strings.TrimSuffix(strings.Join(sections, "\n\n"), "\n")
}

func (c *Command) unexpectedFlagErr(flag string) {
	flagHelp := c.buildFlagHelp()
	cmd := "heroku " + c.String()
	if flagHelp == "" {
		ExitWithMessage(
			`Error: Unexpected flag %s
Usage: %s
This command does not take any flags.

See more information with %s`,
			red(flag),
			cyan("heroku "+CommandUsage(c)),
			cyan(cmd+" --help"),
		)
	}
	ExitWithMessage(
		`Error: Unexpected flag %s
Usage: %s

This flag is invalid for this command. Here are the accepted flags:
%s

See more information with %s`,
		red(flag),
		cyan("heroku "+CommandUsage(c)),
		flagHelp,
		cyan(cmd+" --help"),
	)
}

func (c *Command) appNeededErr() {
	ExitWithMessage(
		`Error: No app specified
Usage: %s
We don't know which app to run this on.
Run this command from inside an app folder or specify which app to use with %s

https://devcenter.heroku.com/articles/using-the-cli#app-commands`,
		cyan("heroku "+CommandUsage(c)+" --app APP"),
		cyan("--app APP"),
	)
}

func (c *Command) unexpectedArgumentsErr(args []string) {
	cmd := "heroku " + c.String()
	ExitWithMessage(
		`Error: Unexpected %s %s
Usage: %s
You gave this command too many arguments. Try the command again without these extra arguments.

See more information with %s`,
		plural("argument", len(args)),
		red(strings.Join(args, " ")),
		cyan("heroku "+CommandUsage(c)),
		cyan(cmd+" --help"),
	)
}

// CommandSet is a slice of Command structs with some helper methods.
type CommandSet []*Command

// Find finds a command and topic matching the cmd string
func (commands CommandSet) Find(cmd string) *Command {
	var topic, command string
	tc := strings.SplitN(cmd, ":", 2)
	topic = tc[0]
	if len(tc) > 1 {
		command = tc[1]
	}
	for _, c := range commands {
		if c.Topic == topic && (c.Command == command || c.Default && command == "") {
			return c
		}
	}
	return nil
}

func (commands CommandSet) loadUsages() {
	for _, c := range commands {
		c.Usage = CommandUsage(c)
	}
}

func (commands CommandSet) loadFullHelp() {
	for _, c := range commands {
		if c.FullHelp == "" {
			c.FullHelp = c.buildFullHelp()
		}
	}
}

func (commands CommandSet) Len() int {
	return len(commands)
}

func (commands CommandSet) Less(i, j int) bool {
	if commands[i].Topic == commands[j].Topic {
		return commands[i].Command < commands[j].Command
	}
	return commands[i].Topic < commands[j].Topic
}

func (commands CommandSet) Swap(i, j int) {
	commands[i], commands[j] = commands[j], commands[i]
}

// NonHidden returns the commands that are not hidden
func (commands CommandSet) NonHidden() []*Command {
	to := make([]*Command, 0, len(commands))
	for _, command := range commands {
		if !command.Hidden {
			to = append(to, command)
		}
	}
	return to
}

// Sort sorts
func (commands CommandSet) Sort() CommandSet {
	sort.Sort(commands)
	return commands
}

// Arg defines an argument for a command.
// These will be parsed in Go and passed to the Run method in the Context struct.
type Arg struct {
	Name     string `json:"name"`
	Optional bool   `json:"optional"`
	Hidden   bool   `json:"hidden"`
}

func (a *Arg) String() string {
	if a.Optional {
		return "[" + strings.ToUpper(a.Name) + "]"
	}
	return strings.ToUpper(a.Name)
}

func argsString(args []Arg) string {
	var buffer bytes.Buffer
	for _, arg := range args {
		if arg.Hidden {
			continue
		}
		if arg.Optional {
			buffer.WriteString(" [" + strings.ToUpper(arg.Name) + "]")
		} else {
			buffer.WriteString(" " + strings.ToUpper(arg.Name))
		}
	}
	return buffer.String()
}

// AllCommands gets all go/core/user commands
func AllCommands() CommandSet {
	commands := Topics.Commands()
	commands = append(commands, corePlugins.Commands()...)
	commands = append(commands, userPlugins.Commands()...)
	return commands
}
<commit_msg>govet<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"sort"
	"strings"
)

func init() {
	Topics = append(Topics, TopicSet{
		{
			Name:        "commands",
			Description: "list all commands",
			Hidden:      true,
			Commands: CommandSet{
				{
					Topic:            "commands",
					Description:      "list all commands",
					Flags:            []Flag{{Name: "json"}},
					DisableAnalytics: true,
					Run: func(ctx *Context) {
						commands := AllCommands().Sort()
						if ctx.Flags["json"] == true {
							commands.loadUsages()
							commands.loadFullHelp()
							doc := map[string]interface{}{"topics": Topics, "commands": commands}
							s, _ := json.Marshal(doc)
							Println(string(s))
							return
						}
						for _, command := range commands {
							if command.Hidden {
								continue
							}
							if command.Command == "" {
								Printf("%s\n", command.Topic)
							} else {
								Printf("%s:%s\n", command.Topic, command.Command)
							}
						}
					},
				},
			},
		},
	}...)
}

// Command represents a CLI command.
// For example, in the command `heroku apps:create` the command would be `create`.
// They must have a Topic name that links to a real topic's name.
type Command struct {
	Topic            string             `json:"topic"`
	Command          string             `json:"command,omitempty"`
	Plugin           string             `json:"plugin"`
	Usage            string             `json:"usage"`
	Description      string             `json:"description"`
	Default          bool               `json:"default"`
	Help             string             `json:"help"`
	FullHelp         string             `json:"fullHelp"`
	Hidden           bool               `json:"hidden"`
	NeedsApp         bool               `json:"needsApp"`
	WantsApp         bool               `json:"wantsApp"`
	NeedsOrg         bool               `json:"needsOrg"`
	WantsOrg         bool               `json:"wantsOrg"`
	NeedsAuth        bool               `json:"needsAuth"`
	VariableArgs     bool               `json:"variableArgs"`
	DisableAnalytics bool               `json:"disableAnalytics"`
	Args             []Arg              `json:"args"`
	Flags            []Flag             `json:"flags"`
	Run              func(ctx *Context) `json:"-"`
}

func (c Command) String() string {
	if c.Command == "" {
		return c.Topic
	}
	return c.Topic + ":" + c.Command
}

// CommandUsage generates the usage for a command
func CommandUsage(c *Command) string {
	if c.Usage != "" {
		return c.Usage
	}
	return c.String() + argsString(c.Args)
}

func (c *Command) buildFlagHelp() string {
	flags := c.Flags
	if c.NeedsApp || c.WantsApp {
		flags = append(flags, *AppFlag, *RemoteFlag)
	}
	if c.NeedsOrg || c.WantsOrg {
		flags = append(flags, *OrgFlag)
	}
	lines := make([]string, 0, len(flags))
	for _, flag := range flags {
		if flag.Hidden {
			continue
		}
		if flag.Description == "" {
			lines = append(lines, flag.String())
		} else {
			lines = append(lines, fmt.Sprintf("%-20s # %s", flag.String(), flag.Description))
		}
	}
	return strings.Join(lines, "\n")
}

func (c *Command) buildFullHelp() string {
	sections := make([]string, 0, 3)
	if c.Description != "" {
		sections = append(sections, c.Description)
	}
	flagHelp := c.buildFlagHelp()
	if flagHelp != "" {
		sections = append(sections, flagHelp)
	}
	if c.Help != "" {
		sections = append(sections, c.Help)
	}
	return strings.TrimSuffix(strings.Join(sections, "\n\n"), "\n")
}

func (c *Command) unexpectedFlagErr(flag string) {
	flagHelp := c.buildFlagHelp()
	cmd := "heroku " + c.String()
	if flagHelp == "" {
		ExitWithMessage(
			`Error: Unexpected flag %s
Usage: %s
This command does not take any flags.

See more information with %s`,
			red(flag),
			cyan("heroku "+CommandUsage(c)),
			cyan(cmd+" --help"),
		)
	}
	ExitWithMessage(
		`Error: Unexpected flag %s
Usage: %s

This flag is invalid for this command. Here are the accepted flags:
%s

See more information with %s`,
		red(flag),
		cyan("heroku "+CommandUsage(c)),
		flagHelp,
		cyan(cmd+" --help"),
	)
}

func (c *Command) appNeededErr() {
	ExitWithMessage(
		`Error: No app specified
Usage: %s
We don't know which app to run this on.
Run this command from inside an app folder or specify which app to use with %s

https://devcenter.heroku.com/articles/using-the-cli#app-commands`,
		cyan("heroku "+CommandUsage(c)+" --app APP"),
		cyan("--app APP"),
	)
}

func (c *Command) unexpectedArgumentsErr(args []string) {
	cmd := "heroku " + c.String()
	ExitWithMessage(
		`Error: Unexpected %s %s
Usage: %s
You gave this command too many arguments. Try the command again without these extra arguments.

See more information with %s`,
		plural("argument", len(args)),
		red(strings.Join(args, " ")),
		cyan("heroku "+CommandUsage(c)),
		cyan(cmd+" --help"),
	)
}

// CommandSet is a slice of Command structs with some helper methods.
type CommandSet []*Command

// Find finds a command and topic matching the cmd string
func (commands CommandSet) Find(cmd string) *Command {
	var topic, command string
	tc := strings.SplitN(cmd, ":", 2)
	topic = tc[0]
	if len(tc) > 1 {
		command = tc[1]
	}
	for _, c := range commands {
		if c.Topic == topic && (c.Command == command || c.Default && command == "") {
			return c
		}
	}
	return nil
}

func (commands CommandSet) loadUsages() {
	for _, c := range commands {
		c.Usage = CommandUsage(c)
	}
}

func (commands CommandSet) loadFullHelp() {
	for _, c := range commands {
		if c.FullHelp == "" {
			c.FullHelp = c.buildFullHelp()
		}
	}
}

func (commands CommandSet) Len() int {
	return len(commands)
}

func (commands CommandSet) Less(i, j int) bool {
	if commands[i].Topic == commands[j].Topic {
		return commands[i].Command < commands[j].Command
	}
	return commands[i].Topic < commands[j].Topic
}

func (commands CommandSet) Swap(i, j int) {
	commands[i], commands[j] = commands[j], commands[i]
}

// NonHidden returns the commands that are not hidden
func (commands CommandSet) NonHidden() []*Command {
	to := make([]*Command, 0, len(commands))
	for _, command := range commands {
		if !command.Hidden {
			to = append(to, command)
		}
	}
	return to
}

// Sort sorts
func (commands CommandSet) Sort() CommandSet {
	sort.Sort(commands)
	return commands
}

// Arg defines an argument for a command.
// These will be parsed in Go and passed to the Run method in the Context struct.
type Arg struct {
	Name     string `json:"name"`
	Optional bool   `json:"optional"`
	Hidden   bool   `json:"hidden"`
}

func (a *Arg) String() string {
	if a.Optional {
		return "[" + strings.ToUpper(a.Name) + "]"
	}
	return strings.ToUpper(a.Name)
}

func argsString(args []Arg) string {
	var buffer bytes.Buffer
	for _, arg := range args {
		if arg.Hidden {
			continue
		}
		if arg.Optional {
			buffer.WriteString(" [" + strings.ToUpper(arg.Name) + "]")
		} else {
			buffer.WriteString(" " + strings.ToUpper(arg.Name))
		}
	}
	return buffer.String()
}

// AllCommands gets all go/core/user commands
func AllCommands() CommandSet {
	commands := Topics.Commands()
	commands = append(commands, corePlugins.Commands()...)
	commands = append(commands, userPlugins.Commands()...)
	return commands
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"sort"
	"strconv"
	"strings"
)

func init() {
	CLITopics = append(CLITopics, Topics{
		{
			Name:        "commands",
			Description: "list all commands",
			Hidden:      true,
			Commands: Commands{
				{
					Topic:            "commands",
					Description:      "list all commands",
					Flags:            []Flag{{Name: "json"}},
					DisableAnalytics: true,
					Run: func(ctx *Context) {
						commands := AllCommands().Sort()
						if ctx.Flags["json"] == true {
							commands.loadUsages()
							commands.loadFullHelp()
							doc := map[string]interface{}{"topics": CLITopics, "commands": commands}
							s, _ := json.Marshal(doc)
							Println(string(s))
							return
						}
						for _, command := range commands {
							if command.Hidden {
								continue
							}
							if command.Command == "" {
								Printf("%s\n", command.Topic)
							} else {
								Printf("%s:%s\n", command.Topic, command.Command)
							}
						}
					},
				},
			},
		},
	}...)
}

// Command represents a CLI command.
// For example, in the command `heroku apps:create` the command would be `create`.
// They must have a Topic name that links to a real topic's name.
type Command struct {
	Topic            string             `json:"topic"`
	Command          string             `json:"command,omitempty"`
	Plugin           string             `json:"plugin"`
	Usage            string             `json:"usage"`
	Description      string             `json:"description"`
	Default          bool               `json:"default"`
	Help             string             `json:"help"`
	FullHelp         string             `json:"fullHelp"`
	Hidden           bool               `json:"hidden"`
	NeedsApp         bool               `json:"needsApp"`
	WantsApp         bool               `json:"wantsApp"`
	NeedsOrg         bool               `json:"needsOrg"`
	WantsOrg         bool               `json:"wantsOrg"`
	NeedsAuth        bool               `json:"needsAuth"`
	VariableArgs     bool               `json:"variableArgs"`
	DisableAnalytics bool               `json:"disableAnalytics"`
	Args             []Arg              `json:"args"`
	Flags            Flags              `json:"flags"`
	Run              func(ctx *Context) `json:"-"`
}

func (c Command) String() string {
	if c.Command == "" {
		return c.Topic
	}
	return c.Topic + ":" + c.Command
}

// CommandUsage generates the usage for a command
func CommandUsage(c *Command) string {
	if c.Usage != "" {
		return c.Usage
	}
	return c.String() + argsString(c.Args)
}

func (c *Command) buildFlagHelp() string {
	flags := c.Flags
	if c.NeedsApp || c.WantsApp {
		flags = append(flags, *AppFlag, *RemoteFlag)
	}
	if c.NeedsOrg || c.WantsOrg {
		flags = append(flags, *OrgFlag)
	}
	flags.Sort()
	lines := make([]string, 0, len(flags))
	longestFlag := 20
	for _, flag := range flags {
		if l := len(flag.String()); l > longestFlag {
			longestFlag = l
		}
	}
	for _, flag := range flags {
		if flag.Hidden {
			continue
		}
		if flag.Description == "" {
			lines = append(lines, flag.String())
		} else {
			lines = append(lines, fmt.Sprintf("%-"+strconv.Itoa(longestFlag)+"s # %s", flag.String(), flag.Description))
		}
	}
	return strings.Join(lines, "\n")
}

func (c *Command) buildFullHelp() string {
	sections := make([]string, 0, 3)
	if c.Description != "" {
		sections = append(sections, c.Description)
	}
	flagHelp := c.buildFlagHelp()
	if flagHelp != "" {
		sections = append(sections, flagHelp)
	}
	if c.Help != "" {
		sections = append(sections, c.Help)
	}
	return strings.TrimSuffix(strings.Join(sections, "\n\n"), "\n")
}

func (c *Command) unexpectedFlagErr(flag string) {
	flagHelp := c.buildFlagHelp()
	cmd := "heroku " + c.String()
	if flagHelp == "" {
		ExitWithMessage(
			`Error: Unexpected flag %s
Usage: %s
This command does not take any flags.

See more information with %s`,
			red(flag),
			cyan("heroku "+CommandUsage(c)),
			cyan(cmd+" --help"),
		)
	}
	ExitWithMessage(
		`Error: Unexpected flag %s
Usage: %s

This flag is invalid for this command. Here are the accepted flags:
%s

See more information with %s`,
		red(flag),
		cyan("heroku "+CommandUsage(c)),
		flagHelp,
		cyan(cmd+" --help"),
	)
}

func (c *Command) appNeededErr() {
	ExitWithMessage(
		`Error: No app specified
Usage: %s
We don't know which app to run this on.
Run this command from inside an app folder or specify which app to use with %s

https://devcenter.heroku.com/articles/using-the-cli#app-commands`,
		cyan("heroku "+CommandUsage(c)+" --app APP"),
		cyan("--app APP"),
	)
}

func (c *Command) unexpectedArgumentsErr(args []string) {
	cmd := "heroku " + c.String()
	ExitWithMessage(
		`Error: Unexpected %s %s
Usage: %s
You gave this command too many arguments. Try the command again without these extra arguments.

See more information with %s`,
		plural("argument", len(args)),
		red(strings.Join(args, " ")),
		cyan("heroku "+CommandUsage(c)),
		cyan(cmd+" --help"),
	)
}

// Commands is a slice of Command structs with some helper methods.
type Commands []*Command

// Find finds a command and topic matching the cmd string
func (commands Commands) Find(cmd string) *Command {
	var topic, command string
	tc := strings.SplitN(cmd, ":", 2)
	topic = tc[0]
	if len(tc) > 1 {
		command = tc[1]
	}
	for _, c := range commands {
		if c.Topic == topic && (c.Command == command || c.Default && command == "") {
			return c
		}
	}
	return nil
}

func (commands Commands) loadUsages() {
	for _, c := range commands {
		c.Usage = CommandUsage(c)
	}
}

func (commands Commands) loadFullHelp() {
	for _, c := range commands {
		if c.FullHelp == "" {
			c.FullHelp = c.buildFullHelp()
		}
	}
}

func (commands Commands) Len() int {
	return len(commands)
}

func (commands Commands) Less(i, j int) bool {
	if commands[i].Topic == commands[j].Topic {
		return commands[i].Command < commands[j].Command
	}
	return commands[i].Topic < commands[j].Topic
}

func (commands Commands) Swap(i, j int) {
	commands[i], commands[j] = commands[j], commands[i]
}

// NonHidden returns the commands that are not hidden
func (commands Commands) NonHidden() []*Command {
	to := make([]*Command, 0, len(commands))
	for _, command := range commands {
		if !command.Hidden {
			to = append(to, command)
		}
	}
	return to
}

// Sort sorts
func (commands Commands) Sort() Commands {
	sort.Sort(commands)
	return commands
}

// Arg defines an argument for a command.
// These will be parsed in Go and passed to the Run method in the Context struct.
type Arg struct {
	Name     string `json:"name"`
	Optional bool   `json:"optional"`
	Hidden   bool   `json:"hidden"`
}

func (a *Arg) String() string {
	if a.Optional {
		return "[" + strings.ToUpper(a.Name) + "]"
	}
	return strings.ToUpper(a.Name)
}

func argsString(args []Arg) string {
	var buffer bytes.Buffer
	for _, arg := range args {
		if arg.Hidden {
			continue
		}
		if arg.Optional {
			buffer.WriteString(" [" + strings.ToUpper(arg.Name) + "]")
		} else {
			buffer.WriteString(" " + strings.ToUpper(arg.Name))
		}
	}
	return buffer.String()
}

// AllCommands gets all go/core/user commands
func AllCommands() Commands {
	commands := CLITopics.Commands()
	commands = append(commands, UserPlugins.Commands()...)
	commands = append(commands, CorePlugins.Commands()...)
	return commands
}
<commit_msg>show all topics<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"sort"
	"strconv"
	"strings"
)

func init() {
	CLITopics = append(CLITopics, Topics{
		{
			Name:        "commands",
			Description: "list all commands",
			Hidden:      true,
			Commands: Commands{
				{
					Topic:            "commands",
					Description:      "list all commands",
					Flags:            []Flag{{Name: "json"}},
					DisableAnalytics: true,
					Run: func(ctx *Context) {
						commands := AllCommands().Sort()
						if ctx.Flags["json"] == true {
							commands.loadUsages()
							commands.loadFullHelp()
							doc := map[string]interface{}{"topics": AllTopics(), "commands": commands}
							s, _ := json.Marshal(doc)
							Println(string(s))
							return
						}
						for _, command := range commands {
							if command.Hidden {
								continue
							}
							if command.Command == "" {
								Printf("%s\n", command.Topic)
							} else {
								Printf("%s:%s\n", command.Topic, command.Command)
							}
						}
					},
				},
			},
		},
	}...)
}

// Command represents a CLI command.
// For example, in the command `heroku apps:create` the command would be `create`.
// They must have a Topic name that links to a real topic's name.
type Command struct {
	Topic            string             `json:"topic"`
	Command          string             `json:"command,omitempty"`
	Plugin           string             `json:"plugin"`
	Usage            string             `json:"usage"`
	Description      string             `json:"description"`
	Default          bool               `json:"default"`
	Help             string             `json:"help"`
	FullHelp         string             `json:"fullHelp"`
	Hidden           bool               `json:"hidden"`
	NeedsApp         bool               `json:"needsApp"`
	WantsApp         bool               `json:"wantsApp"`
	NeedsOrg         bool               `json:"needsOrg"`
	WantsOrg         bool               `json:"wantsOrg"`
	NeedsAuth        bool               `json:"needsAuth"`
	VariableArgs     bool               `json:"variableArgs"`
	DisableAnalytics bool               `json:"disableAnalytics"`
	Args             []Arg              `json:"args"`
	Flags            Flags              `json:"flags"`
	Run              func(ctx *Context) `json:"-"`
}

func (c Command) String() string {
	if c.Command == "" {
		return c.Topic
	}
	return c.Topic + ":" + c.Command
}

// CommandUsage generates the usage for a command
func CommandUsage(c *Command) string {
	if c.Usage != "" {
		return c.Usage
	}
	return c.String() + argsString(c.Args)
}

func (c *Command) buildFlagHelp() string {
	flags := c.Flags
	if c.NeedsApp || c.WantsApp {
		flags = append(flags, *AppFlag, *RemoteFlag)
	}
	if c.NeedsOrg || c.WantsOrg {
		flags = append(flags, *OrgFlag)
	}
	flags.Sort()
	lines := make([]string, 0, len(flags))
	longestFlag := 20
	for _, flag := range flags {
		if l := len(flag.String()); l > longestFlag {
			longestFlag = l
		}
	}
	for _, flag := range flags {
		if flag.Hidden {
			continue
		}
		if flag.Description == "" {
			lines = append(lines, flag.String())
		} else {
			lines = append(lines, fmt.Sprintf("%-"+strconv.Itoa(longestFlag)+"s # %s", flag.String(), flag.Description))
		}
	}
	return strings.Join(lines, "\n")
}

func (c *Command) buildFullHelp() string {
	sections := make([]string, 0, 3)
	if c.Description != "" {
		sections = append(sections, c.Description)
	}
	flagHelp := c.buildFlagHelp()
	if flagHelp != "" {
		sections = append(sections, flagHelp)
	}
	if c.Help != "" {
		sections = append(sections, c.Help)
	}
	return strings.TrimSuffix(strings.Join(sections, "\n\n"), "\n")
}

func (c *Command) unexpectedFlagErr(flag string) {
	flagHelp := c.buildFlagHelp()
	cmd := "heroku " + c.String()
	if flagHelp == "" {
		ExitWithMessage(
			`Error: Unexpected flag %s
Usage: %s
This command does not take any flags.

See more information with %s`,
			red(flag),
			cyan("heroku "+CommandUsage(c)),
			cyan(cmd+" --help"),
		)
	}
	ExitWithMessage(
		`Error: Unexpected flag %s
Usage: %s

This flag is invalid for this command. Here are the accepted flags:
%s

See more information with %s`,
		red(flag),
		cyan("heroku "+CommandUsage(c)),
		flagHelp,
		cyan(cmd+" --help"),
	)
}

func (c *Command) appNeededErr() {
	ExitWithMessage(
		`Error: No app specified
Usage: %s
We don't know which app to run this on.
Run this command from inside an app folder or specify which app to use with %s

https://devcenter.heroku.com/articles/using-the-cli#app-commands`,
		cyan("heroku "+CommandUsage(c)+" --app APP"),
		cyan("--app APP"),
	)
}

func (c *Command) unexpectedArgumentsErr(args []string) {
	cmd := "heroku " + c.String()
	ExitWithMessage(
		`Error: Unexpected %s %s
Usage: %s
You gave this command too many arguments. Try the command again without these extra arguments.

See more information with %s`,
		plural("argument", len(args)),
		red(strings.Join(args, " ")),
		cyan("heroku "+CommandUsage(c)),
		cyan(cmd+" --help"),
	)
}

// Commands is a slice of Command structs with some helper methods.
type Commands []*Command

// Find finds a command and topic matching the cmd string
func (commands Commands) Find(cmd string) *Command {
	var topic, command string
	tc := strings.SplitN(cmd, ":", 2)
	topic = tc[0]
	if len(tc) > 1 {
		command = tc[1]
	}
	for _, c := range commands {
		if c.Topic == topic && (c.Command == command || c.Default && command == "") {
			return c
		}
	}
	return nil
}

func (commands Commands) loadUsages() {
	for _, c := range commands {
		c.Usage = CommandUsage(c)
	}
}

func (commands Commands) loadFullHelp() {
	for _, c := range commands {
		if c.FullHelp == "" {
			c.FullHelp = c.buildFullHelp()
		}
	}
}

func (commands Commands) Len() int {
	return len(commands)
}

func (commands Commands) Less(i, j int) bool {
	if commands[i].Topic == commands[j].Topic {
		return commands[i].Command < commands[j].Command
	}
	return commands[i].Topic < commands[j].Topic
}

func (commands Commands) Swap(i, j int) {
	commands[i], commands[j] = commands[j], commands[i]
}

// NonHidden returns the commands that are not hidden
func (commands Commands) NonHidden() []*Command {
	to := make([]*Command, 0, len(commands))
	for _, command := range commands {
		if !command.Hidden {
			to = append(to, command)
		}
	}
	return to
}

// Sort sorts
func (commands Commands) Sort() Commands {
	sort.Sort(commands)
	return commands
}

// Arg defines an argument for a command.
// These will be parsed in Go and passed to the Run method in the Context struct.
type Arg struct {
	Name     string `json:"name"`
	Optional bool   `json:"optional"`
	Hidden   bool   `json:"hidden"`
}

func (a *Arg) String() string {
	if a.Optional {
		return "[" + strings.ToUpper(a.Name) + "]"
	}
	return strings.ToUpper(a.Name)
}

func argsString(args []Arg) string {
	var buffer bytes.Buffer
	for _, arg := range args {
		if arg.Hidden {
			continue
		}
		if arg.Optional {
			buffer.WriteString(" [" + strings.ToUpper(arg.Name) + "]")
		} else {
			buffer.WriteString(" " + strings.ToUpper(arg.Name))
		}
	}
	return buffer.String()
}

// AllCommands gets all go/core/user commands
func AllCommands() Commands {
	commands := CLITopics.Commands()
	commands = append(commands, UserPlugins.Commands()...)
	commands = append(commands, CorePlugins.Commands()...)
	return commands
}
<|endoftext|>


<commit_before>package renter

import (
	"errors"
	"fmt"
	// "io"
	"net"
	// "os"
	"time"

	"github.com/NebulousLabs/Sia/consensus"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/hash"
	"github.com/NebulousLabs/Sia/sia/components"
)

// TODO: ALSO WARNING: There's a bunch of code duplication here as a result of
// trying to get the release out. If you edit a part of this, make sure both
// halves (small file and big file) get the update, or save us all the trouble
// and dedup the code.

// ClientFundFileContract takes a template FileContract and returns a
// partial transaction containing an input for the contract, but no signatures.
//
// TODO: We need to get the id of the contract before we can start doing
// re-uploading.
func (r *Renter) proposeContract(filename string, duration consensus.BlockHeight) (fp FilePiece, err error) {
	err = errors.New("proposeContract is not implemented - needs to be merged with other code")
	return

	/*
		// Open the file, create a merkle hash.
		file, err := os.Open(filename)
		if err != nil {
			return
		}
		defer file.Close()
		info, err := file.Stat()
		if err != nil {
			return
		}
		merkle, err := hash.ReaderMerkleRoot(file, hash.CalculateSegments(uint64(info.Size())))
		if err != nil {
			return
		}
		// reset read position
		if _, err = file.Seek(0, 0); err != nil {
			return
		}

		// Find a host. If the search or the negotiation is unsuccessful,
		// hostdb.FlagHost() will be called and another host will be requested. If
		// there is an internal error (no hosts, or an unsuccessful flagging for
		// example), the loop will break.
		var host components.HostEntry
		var fileContract consensus.FileContract
		for {
			host, err = r.hostDB.RandomHost()
			if err != nil {
				return
			}

			// Fill out the contract according to the whims of the host.
			// The contract fund: (burn * duration + price * full duration) * filesize
			delay := consensus.BlockHeight(20)
			contractFund := (host.Price*consensus.Currency(duration+delay) + host.Burn*consensus.Currency(duration)) * consensus.Currency(info.Size())
			fileContract = consensus.FileContract{
				ContractFund:       contractFund,
				FileMerkleRoot:     merkle,
				FileSize:           uint64(info.Size()),
				Start:              r.state.Height() + delay,
				End:                r.state.Height() + duration + delay,
				ChallengeWindow:    host.Window,
				Tolerance:          host.Tolerance,
				ValidProofPayout:   host.Price * consensus.Currency(info.Size()) * consensus.Currency(host.Window),
				ValidProofAddress:  host.CoinAddress,
				MissedProofPayout:  host.Burn * consensus.Currency(info.Size()) * consensus.Currency(host.Window),
				MissedProofAddress: consensus.CoinAddress{}, // The empty address is the burn address.
			}

			// Fund the client portion of the transaction.
			minerFee := consensus.Currency(10) // TODO: ask wallet.
			renterPortion := host.Price * consensus.Currency(duration+delay) * consensus.Currency(fileContract.FileSize)
			var id string
			id, err = r.wallet.RegisterTransaction(consensus.Transaction{})
			if err != nil {
				return
			}
			err = r.wallet.FundTransaction(id, renterPortion+minerFee)
			if err != nil {
				return
			}
			err = r.wallet.AddMinerFee(id, minerFee)
			if err != nil {
				return
			}
			err = r.wallet.AddFileContract(id, fileContract)
			if err != nil {
				return
			}
			var transaction consensus.Transaction
			transaction, err = r.wallet.SignTransaction(id, false)
			if err != nil {
				return
			}

			// Negotiate the contract to the host.
			err = host.IPAddress.Call("NegotiateContract", func(conn net.Conn) error {
				// send contract
				if _, err := encoding.WriteObject(conn, transaction); err != nil {
					return err
				}
				// read response
				var response string
				if err := encoding.ReadObject(conn, &response, 128); err != nil {
					return err
				}
				if response != components.AcceptContractResponse {
					return errors.New(response)
				}
				// host accepted, so transmit file data
				// (no prefix needed, since FileSize is included in the metadata)
				_, err = io.CopyN(conn, file, info.Size())
				return err
			})
			if err == nil {
				break
			}

			fmt.Println("Problem from NegotiateContract:", err)
			err = r.hostDB.FlagHost(host.ID)
			if err != nil {
				return
			}
		}

		// Record the file into the renter database.
		fp = FilePiece{
			Host:     host,
			Contract: fileContract,
		}

		return
	*/
}

// TODO: Do the uploading in parallel.
func (r *Renter) RentFile(rfp components.RentFileParameters) (err error) {
	r.mu.Lock()
	defer r.mu.Unlock()

	_, exists := r.files[rfp.Nickname]
	if exists {
		return errors.New("file of that nickname already exists")
	}

	// Make an entry for this file.
	var pieces []FilePiece
	for i := 0; i < rfp.TotalPieces; i++ {
		var piece FilePiece
		piece, err = r.proposeContract(rfp.Filepath, consensus.BlockHeight(2000+1000*i))
		if err != nil {
			return
		}
		pieces = append(pieces, piece)
	}

	r.files[rfp.Nickname] = FileEntry{Pieces: pieces}
	return
}

// TODO: Do the uploading in parallel.
//
// On mutexes: cannot do network stuff with a lock on, so we need to get the
// lock, get the contracts, and then drop the lock.
func (r *Renter) RentSmallFile(rsfp components.RentSmallFileParameters) (err error) {
	r.mu.RLock()
	_, exists := r.files[rsfp.Nickname]
	if exists {
		return errors.New("file of that nickname already exists")
	}
	r.mu.RUnlock()

	// Make an entry for this file.
	var pieces []FilePiece
	for i := 0; i < rsfp.TotalPieces; i++ {
		var piece FilePiece
		piece, err = r.proposeSmallContract(rsfp.FullFile, consensus.BlockHeight(800))
		if err != nil {
			return
		}
		pieces = append(pieces, piece)
		r.mu.Lock()
		r.files[rsfp.Nickname] = FileEntry{Pieces: pieces}
		r.mu.Unlock()
	}

	return
}

func (r *Renter) proposeSmallContract(fullFile []byte, duration consensus.BlockHeight) (fp FilePiece, err error) {
	merkle, err := hash.BytesMerkleRoot(fullFile)
	if err != nil {
		return
	}

	// Find a host. If the search or the negotiation is unsuccessful,
	// hostdb.FlagHost() will be called and another host will be requested. If
	// there is an internal error (no hosts, or an unsuccessful flagging for
	// example), the loop will break.
	var host components.HostEntry
	var fileContract consensus.FileContract
	var contractID consensus.ContractID
	for {
		host, err = r.hostDB.RandomHost()
		if err != nil {
			return
		}

		// Fill out the contract according to the whims of the host.
		// The contract fund: (burn * duration + price * full duration) * filesize
		delay := consensus.BlockHeight(20)
		contractFund := (host.Price*consensus.Currency(duration+delay) + host.Burn*consensus.Currency(duration)) * consensus.Currency(int64(len(fullFile)))
		fileContract = consensus.FileContract{
			ContractFund:       contractFund,
			FileMerkleRoot:     merkle,
			FileSize:           uint64(len(fullFile)),
			Start:              r.state.Height() + delay,
			End:                r.state.Height() + duration + delay,
			ChallengeWindow:    host.Window,
			Tolerance:          host.Tolerance,
			ValidProofPayout:   host.Price * consensus.Currency(len(fullFile)) * consensus.Currency(host.Window),
			ValidProofAddress:  host.CoinAddress,
			MissedProofPayout:  host.Burn * consensus.Currency(len(fullFile)) * consensus.Currency(host.Window),
			MissedProofAddress: consensus.CoinAddress{}, // The empty address is the burn address.
		}

		// Fund the client portion of the transaction.
		minerFee := consensus.Currency(10) // TODO: ask wallet.
		renterPortion := host.Price * consensus.Currency(duration+delay) * consensus.Currency(fileContract.FileSize)
		var id string
		id, err = r.wallet.RegisterTransaction(consensus.Transaction{})
		if err != nil {
			return
		}

		// Try to fund the transaction, and wait if there isn't enough money.
		err = r.wallet.FundTransaction(id, renterPortion+minerFee)
		if err != nil && err != components.LowBalanceErr {
			return
		}
		for err == components.LowBalanceErr {
			// TODO: This is a dirty hack - the system will try to get the file
			// through until it has enough money to actually get the file
			// through. Significant problem :(

			// There should be no locks at this point.
			time.Sleep(time.Second * 30)
			err = r.wallet.FundTransaction(id, renterPortion+minerFee)
		}

		err = r.wallet.AddMinerFee(id, minerFee)
		if err != nil {
			return
		}
		err = r.wallet.AddFileContract(id, fileContract)
		if err != nil {
			return
		}
		var transaction consensus.Transaction
		transaction, err = r.wallet.SignTransaction(id, false)
		if err != nil {
			return
		}
		contractID = transaction.FileContractID(0)

		// Negotiate the contract to the host.
		err = host.IPAddress.Call("NegotiateContract", func(conn net.Conn) error {
			// send contract
			if _, err := encoding.WriteObject(conn, transaction); err != nil {
				return err
			}
			// read response
			var response string
			if err := encoding.ReadObject(conn, &response, 128); err != nil {
				return err
			}
			if response != components.AcceptContractResponse {
				return errors.New(response)
			}
			// host accepted, so transmit file data
			// (no prefix needed, since FileSize is included in the metadata)
			_, err = conn.Write(fullFile)
			return err
		})
		if err == nil {
			break
		}

		fmt.Println("Problem from NegotiateContract:", err)
		err = r.hostDB.FlagHost(host.ID)
		if err != nil {
			return
		}
	}

	// Record the file into the renter database.
	fp = FilePiece{
		Host:       host,
		Contract:   fileContract,
		ContractID: contractID,
	}

	return
}
<commit_msg>add checking for empty string when uploading<commit_after>package renter

import (
	"errors"
	"fmt"
	// "io"
	"net"
	// "os"
	"time"

	"github.com/NebulousLabs/Sia/consensus"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/hash"
	"github.com/NebulousLabs/Sia/sia/components"
)

// TODO: ALSO WARNING: There's a bunch of code duplication here as a result of
// trying to get the release out. If you edit a part of this, make sure both
// halves (small file and big file) get the update, or save us all the trouble
// and dedup the code.

// ClientFundFileContract takes a template FileContract and returns a
// partial transaction containing an input for the contract, but no signatures.
//
// TODO: We need to get the id of the contract before we can start doing
// re-uploading.
func (r *Renter) proposeContract(filename string, duration consensus.BlockHeight) (fp FilePiece, err error) {
	err = errors.New("proposeContract is not implemented - needs to be merged with other code")
	return

	/*
		// Open the file, create a merkle hash.
		file, err := os.Open(filename)
		if err != nil {
			return
		}
		defer file.Close()
		info, err := file.Stat()
		if err != nil {
			return
		}
		merkle, err := hash.ReaderMerkleRoot(file, hash.CalculateSegments(uint64(info.Size())))
		if err != nil {
			return
		}
		// reset read position
		if _, err = file.Seek(0, 0); err != nil {
			return
		}

		// Find a host. If the search or the negotiation is unsuccessful,
		// hostdb.FlagHost() will be called and another host will be requested. If
		// there is an internal error (no hosts, or an unsuccessful flagging for
		// example), the loop will break.
		var host components.HostEntry
		var fileContract consensus.FileContract
		for {
			host, err = r.hostDB.RandomHost()
			if err != nil {
				return
			}

			// Fill out the contract according to the whims of the host.
			// The contract fund: (burn * duration + price * full duration) * filesize
			delay := consensus.BlockHeight(20)
			contractFund := (host.Price*consensus.Currency(duration+delay) + host.Burn*consensus.Currency(duration)) * consensus.Currency(info.Size())
			fileContract = consensus.FileContract{
				ContractFund:       contractFund,
				FileMerkleRoot:     merkle,
				FileSize:           uint64(info.Size()),
				Start:              r.state.Height() + delay,
				End:                r.state.Height() + duration + delay,
				ChallengeWindow:    host.Window,
				Tolerance:          host.Tolerance,
				ValidProofPayout:   host.Price * consensus.Currency(info.Size()) * consensus.Currency(host.Window),
				ValidProofAddress:  host.CoinAddress,
				MissedProofPayout:  host.Burn * consensus.Currency(info.Size()) * consensus.Currency(host.Window),
				MissedProofAddress: consensus.CoinAddress{}, // The empty address is the burn address.
			}

			// Fund the client portion of the transaction.
			minerFee := consensus.Currency(10) // TODO: ask wallet.
			renterPortion := host.Price * consensus.Currency(duration+delay) * consensus.Currency(fileContract.FileSize)
			var id string
			id, err = r.wallet.RegisterTransaction(consensus.Transaction{})
			if err != nil {
				return
			}
			err = r.wallet.FundTransaction(id, renterPortion+minerFee)
			if err != nil {
				return
			}
			err = r.wallet.AddMinerFee(id, minerFee)
			if err != nil {
				return
			}
			err = r.wallet.AddFileContract(id, fileContract)
			if err != nil {
				return
			}
			var transaction consensus.Transaction
			transaction, err = r.wallet.SignTransaction(id, false)
			if err != nil {
				return
			}

			// Negotiate the contract to the host.
			err = host.IPAddress.Call("NegotiateContract", func(conn net.Conn) error {
				// send contract
				if _, err := encoding.WriteObject(conn, transaction); err != nil {
					return err
				}
				// read response
				var response string
				if err := encoding.ReadObject(conn, &response, 128); err != nil {
					return err
				}
				if response != components.AcceptContractResponse {
					return errors.New(response)
				}
				// host accepted, so transmit file data
				// (no prefix needed, since FileSize is included in the metadata)
				_, err = io.CopyN(conn, file, info.Size())
				return err
			})
			if err == nil {
				break
			}

			fmt.Println("Problem from NegotiateContract:", err)
			err = r.hostDB.FlagHost(host.ID)
			if err != nil {
				return
			}
		}

		// Record the file into the renter database.
		fp = FilePiece{
			Host:     host,
			Contract: fileContract,
		}

		return
	*/
}

// TODO: Do the uploading in parallel.
func (r *Renter) RentFile(rfp components.RentFileParameters) (err error) {
	r.mu.Lock()
	defer r.mu.Unlock()

	_, exists := r.files[rfp.Nickname]
	if exists {
		return errors.New("file of that nickname already exists")
	}

	// Make an entry for this file.
	var pieces []FilePiece
	for i := 0; i < rfp.TotalPieces; i++ {
		var piece FilePiece
		piece, err = r.proposeContract(rfp.Filepath, consensus.BlockHeight(2000+1000*i))
		if err != nil {
			return
		}
		pieces = append(pieces, piece)
	}

	r.files[rfp.Nickname] = FileEntry{Pieces: pieces}
	return
}

// TODO: Do the uploading in parallel.
//
// On mutexes: cannot do network stuff with a lock on, so we need to get the
// lock, get the contracts, and then drop the lock.
func (r *Renter) RentSmallFile(rsfp components.RentSmallFileParameters) (err error) {
	r.mu.RLock()
	_, exists := r.files[rsfp.Nickname]
	if exists {
		return errors.New("file of that nickname already exists")
	} else if rsfp.Nickname == "" {
		return errors.New("cannot use empty string for nickname")
	}
	r.mu.RUnlock()

	// Make an entry for this file.
	var pieces []FilePiece
	for i := 0; i < rsfp.TotalPieces; i++ {
		var piece FilePiece
		piece, err = r.proposeSmallContract(rsfp.FullFile, consensus.BlockHeight(800))
		if err != nil {
			return
		}
		pieces = append(pieces, piece)
		r.mu.Lock()
		r.files[rsfp.Nickname] = FileEntry{Pieces: pieces}
		r.mu.Unlock()
	}

	return
}

func (r *Renter) proposeSmallContract(fullFile []byte, duration consensus.BlockHeight) (fp FilePiece, err error) {
	merkle, err := hash.BytesMerkleRoot(fullFile)
	if err != nil {
		return
	}

	// Find a host. If the search or the negotiation is unsuccessful,
	// hostdb.FlagHost() will be called and another host will be requested. If
	// there is an internal error (no hosts, or an unsuccessful flagging for
	// example), the loop will break.
	var host components.HostEntry
	var fileContract consensus.FileContract
	var contractID consensus.ContractID
	for {
		host, err = r.hostDB.RandomHost()
		if err != nil {
			return
		}

		// Fill out the contract according to the whims of the host.
		// The contract fund: (burn * duration + price * full duration) * filesize
		delay := consensus.BlockHeight(20)
		contractFund := (host.Price*consensus.Currency(duration+delay) + host.Burn*consensus.Currency(duration)) * consensus.Currency(int64(len(fullFile)))
		fileContract = consensus.FileContract{
			ContractFund:       contractFund,
			FileMerkleRoot:     merkle,
			FileSize:           uint64(len(fullFile)),
			Start:              r.state.Height() + delay,
			End:                r.state.Height() + duration + delay,
			ChallengeWindow:    host.Window,
			Tolerance:          host.Tolerance,
			ValidProofPayout:   host.Price * consensus.Currency(len(fullFile)) * consensus.Currency(host.Window),
			ValidProofAddress:  host.CoinAddress,
			MissedProofPayout:  host.Burn * consensus.Currency(len(fullFile)) * consensus.Currency(host.Window),
			MissedProofAddress: consensus.CoinAddress{}, // The empty address is the burn address.
		}

		// Fund the client portion of the transaction.
		minerFee := consensus.Currency(10) // TODO: ask wallet.
		renterPortion := host.Price * consensus.Currency(duration+delay) * consensus.Currency(fileContract.FileSize)
		var id string
		id, err = r.wallet.RegisterTransaction(consensus.Transaction{})
		if err != nil {
			return
		}

		// Try to fund the transaction, and wait if there isn't enough money.
		err = r.wallet.FundTransaction(id, renterPortion+minerFee)
		if err != nil && err != components.LowBalanceErr {
			return
		}
		for err == components.LowBalanceErr {
			// TODO: This is a dirty hack - the system will try to get the file
			// through until it has enough money to actually get the file
			// through. Significant problem :(

			// There should be no locks at this point.
			time.Sleep(time.Second * 30)
			err = r.wallet.FundTransaction(id, renterPortion+minerFee)
		}

		err = r.wallet.AddMinerFee(id, minerFee)
		if err != nil {
			return
		}
		err = r.wallet.AddFileContract(id, fileContract)
		if err != nil {
			return
		}
		var transaction consensus.Transaction
		transaction, err = r.wallet.SignTransaction(id, false)
		if err != nil {
			return
		}
		contractID = transaction.FileContractID(0)

		// Negotiate the contract to the host.
		err = host.IPAddress.Call("NegotiateContract", func(conn net.Conn) error {
			// send contract
			if _, err := encoding.WriteObject(conn, transaction); err != nil {
				return err
			}
			// read response
			var response string
			if err := encoding.ReadObject(conn, &response, 128); err != nil {
				return err
			}
			if response != components.AcceptContractResponse {
				return errors.New(response)
			}
			// host accepted, so transmit file data
			// (no prefix needed, since FileSize is included in the metadata)
			_, err = conn.Write(fullFile)
			return err
		})
		if err == nil {
			break
		}

		fmt.Println("Problem from NegotiateContract:", err)
		err = r.hostDB.FlagHost(host.ID)
		if err != nil {
			return
		}
	}

	// Record the file into the renter database.
	fp = FilePiece{
		Host:       host,
		Contract:   fileContract,
		ContractID: contractID,
	}

	return
}
<|endoftext|>


<commit_before>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package testbase

import (
	"go/build"
	"path/filepath"
	"sort"
	"strings"

	gc "launchpad.net/gocheck"
)

// FindJujuCoreImports returns a sorted list of juju-core packages that are
// imported by the packageName parameter.  The resulting list removes the
// common prefix "launchpad.net/juju-core/" leaving just the short names.
func FindJujuCoreImports(c *gc.C, packageName string) []string {
	var imports []string

	for _, root := range build.Default.SrcDirs() {
		fullpath := filepath.Join(root, packageName)
		pkg, err := build.ImportDir(fullpath, 0)
		if err == nil {
			imports = pkg.Imports
			break
		}
	}
	if imports == nil {
		c.Fatalf(packageName + " not found")
	}

	var result []string
	const prefix = "launchpad.net/juju-core/"
	for _, name := range imports {
		if strings.HasPrefix(name, prefix) {
			result = append(result, name[len(prefix):])
		}
	}
	sort.Strings(result)
	return result
}
<commit_msg>FindJujuCoreImports now recursively checks for all imported packages<commit_after>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package testbase

import (
	"go/build"
	"path/filepath"
	"sort"
	"strings"

	gc "launchpad.net/gocheck"
)

const jujuPkgPrefix = "launchpad.net/juju-core/"

// FindJujuCoreImports returns a sorted list of juju-core packages that are
// imported by the packageName parameter.  The resulting list removes the
// common prefix "launchpad.net/juju-core/" leaving just the short names.
func FindJujuCoreImports(c *gc.C, packageName string) []string {

	var result []string
	allpkgs := make(map[string]bool)

	findJujuCoreImports(c, packageName, allpkgs)

	for name := range allpkgs {
		result = append(result, name[len(jujuPkgPrefix):])
	}

	sort.Strings(result)
	return result
}

// findJujuCoreImports recursively adds all imported packages of given
// package (packageName) to allpkgs map.
func findJujuCoreImports(c *gc.C, packageName string, allpkgs map[string]bool) {

	var imports []string

	for _, root := range build.Default.SrcDirs() {
		fullpath := filepath.Join(root, packageName)
		pkg, err := build.ImportDir(fullpath, 0)
		if err == nil {
			imports = pkg.Imports
			break
		}
	}
	if imports == nil {
		c.Fatalf(packageName + " not found")
	}

	for _, name := range imports {
		if strings.HasPrefix(name, jujuPkgPrefix) {
			allpkgs[name] = true
			findJujuCoreImports(c, name, allpkgs)
		}
	}

}
<|endoftext|>


<commit_before>package main

import (
	"io"
	"rtos"

	"stm32/hal/dma"
	"stm32/hal/gpio"
	"stm32/hal/irq"
	"stm32/hal/system"
	"stm32/hal/system/timer/systick"
	"stm32/hal/usart"
)

var tts *usart.Driver

func init() {
	system.SetupPLL(8, 1, 48/8)
	systick.Setup(2e6)

	gpio.A.EnableClock(true)
	tx := gpio.A.Pin(9)

	tx.Setup(&gpio.Config{Mode: gpio.Alt})
	tx.SetAltFunc(gpio.USART1_AF1)
	d := dma.DMA1
	d.EnableClock(true)
	tts = usart.NewDriver(usart.USART1, d.Channel(2, 0), nil, nil)
	tts.Periph().EnableClock(true)
	tts.Periph().SetBaudRate(115200)
	tts.Periph().Enable()
	tts.EnableTx()

	rtos.IRQ(irq.USART1).Enable()
	rtos.IRQ(irq.DMA1_Channel2_3).Enable()
}

type MorseWriter struct {
	W io.Writer
}

func (w *MorseWriter) Write(s []byte) (n int, err error) {
	var buf [8]byte
	for _, c := range s {
		if c < ' ' {
			continue
		}
		if 'a' <= c && c <= 'z' {
			c -= 'a' - 'A' // Convert to upper case.
		}
		if c > 'Z' {
			continue
		}
		var symbol morseSymbol
		if c == ' ' {
			symbol.length = 2
			buf[0] = ' '
			buf[1] = ' '
		} else {
			symbol = morseSymbols[c-'!']
			for i := uint(0); i < uint(symbol.length); i++ {
				if (symbol.code>>i)&1 != 0 {
					buf[i] = '-'
				} else {
					buf[i] = '.'
				}
			}
		}
		buf[symbol.length] = ' '
		m, err := w.W.Write(buf[:symbol.length+1])
		n += m
		if err != nil {
			return n, err
		}
	}
	return n, nil
}

func main() {
	w := &MorseWriter{tts}
	io.WriteString(w, "Hello, World!\r\n")
}

func ttsISR() {
	tts.ISR()
}

func ttsDMAISR() {
	tts.TxDMAISR()
}

//c:__attribute__((section(".ISRs")))
var ISRs = [...]func(){
	irq.USART1:          ttsISR,
	irq.DMA1_Channel2_3: ttsDMAISR,
}

type morseSymbol struct {
	code, length byte
}

//emgo:const
var morseSymbols = [...]morseSymbol{
	{1<<0 | 1<<1 | 1<<2, 4}, // ! ---.
	{1<<1 | 1<<4, 6},        // " .-..-.
	{},                      // #
	{1<<3 | 1<<6, 7},        // $ ...-..-
	{},                      // %
	{},                      // &
	{1<<1 | 1<<2 | 1<<3 | 1<<4, 6}, // ' .----.
	{1<<0 | 1<<2 | 1<<3, 5},        // ( -.--.
	{1<<0 | 1<<2 | 1<<3 | 1<<5, 6}, // ) -.--.-
	{},                                    // *
	{1<<1 | 1<<3, 5},                      // + .-.-.
	{1<<0 | 1<<1 | 1<<4 | 1<<5, 6},        // , --..--
	{1<<0 | 1<<5, 6},                      // - -....-
	{1<<1 | 1<<3 | 1<<5, 6},               // . .-.-.-
	{1<<0 | 1<<3, 5},                      // / -..-.
	{1<<0 | 1<<1 | 1<<2 | 1<<3 | 1<<4, 5}, // 0 -----
	{1<<1 | 1<<2 | 1<<3 | 1<<4, 5},        // 1 .----
	{1<<2 | 1<<3 | 1<<4, 5},               // 2 ..---
	{1<<3 | 1<<4, 5},                      // 3 ...--
	{1 << 4, 5},                           // 4 ....-
	{0, 5},                                // 5 .....
	{1 << 0, 5},                           // 6 -....
	{1<<0 | 1<<1, 5},                      // 7 --...
	{1<<0 | 1<<1 | 1<<2, 5},               // 8 ---..
	{1<<0 | 1<<1 | 1<<2 | 1<<3, 5},        // 9 ----.
	{1<<0 | 1<<1 | 1<<2, 6},               // : ---...
	{1<<0 | 1<<2 | 1<<4, 6},               // ; -.-.-.
	{},                      // <
	{1<<0 | 1<<4, 5},        // = -...-
	{},                      // >
	{1<<2 | 1<<3, 6},        // ? ..--..
	{1<<1 | 1<<2 | 1<<4, 6}, // @ .--.-.
	{1 << 1, 2},             // A .-
	{1 << 0, 4},             // B -...
	{1<<0 | 1<<2, 4},        // C -.-.
	{1 << 0, 3},             // D -..
	{0, 1},                  // E .
	{1 << 2, 4},             // F ..-.
	{1<<0 | 1<<1, 3},        // G --.
	{0, 4},                  // H ....
	{0, 2},                  // I ..
	{1<<1 | 1<<2 | 1<<3, 4}, // J .---
	{1<<0 | 1<<2, 3},        // K -.-
	{1 << 1, 4},             // L .-..
	{1<<0 | 1<<1, 2},        // M --
	{1 << 0, 2},             // N -.
	{1<<0 | 1<<1 | 1<<2, 3}, // O ---
	{1<<1 | 1<<2, 4},        // P .--.
	{1<<0 | 1<<1 | 1<<3, 4}, // Q --.-
	{1 << 1, 3},             // R .-.
	{0, 3},                  // S ...
	{1 << 0, 1},             // T -
	{1 << 2, 3},             // U ..-
	{1 << 3, 4},             // V ...-
	{1<<1 | 1<<2, 3},        // W .--
	{1<<0 | 1<<3, 4},        // X -..-
	{1<<0 | 1<<2 | 1<<3, 4}, // Y -.--
	{1<<0 | 1<<1, 4},        // Z --..
}
<commit_msg>examples/f030-demo-board/morseuart: Small change.<commit_after>package main

import (
	"io"
	"rtos"

	"stm32/hal/dma"
	"stm32/hal/gpio"
	"stm32/hal/irq"
	"stm32/hal/system"
	"stm32/hal/system/timer/systick"
	"stm32/hal/usart"
)

var tts *usart.Driver

func init() {
	system.SetupPLL(8, 1, 48/8)
	systick.Setup(2e6)

	gpio.A.EnableClock(true)
	tx := gpio.A.Pin(9)

	tx.Setup(&gpio.Config{Mode: gpio.Alt})
	tx.SetAltFunc(gpio.USART1_AF1)
	d := dma.DMA1
	d.EnableClock(true)
	tts = usart.NewDriver(usart.USART1, d.Channel(2, 0), nil, nil)
	tts.Periph().EnableClock(true)
	tts.Periph().SetBaudRate(115200)
	tts.Periph().Enable()
	tts.EnableTx()

	rtos.IRQ(irq.USART1).Enable()
	rtos.IRQ(irq.DMA1_Channel2_3).Enable()
}

type MorseWriter struct {
	W io.Writer
}

func (w *MorseWriter) Write(s []byte) (n int, err error) {
	var buf [8]byte
	for _, c := range s {
		if c < ' ' {
			continue
		}
		if 'a' <= c && c <= 'z' {
			c -= 'a' - 'A' // Convert to upper case.
		}
		if c > 'Z' {
			continue
		}
		var symbol morseSymbol
		if c == ' ' || c == '\n' {
			symbol.length = 2
			buf[0] = ' '
			buf[1] = ' '
		} else {
			symbol = morseSymbols[c-'!']
			for i := uint(0); i < uint(symbol.length); i++ {
				if (symbol.code>>i)&1 != 0 {
					buf[i] = '-'
				} else {
					buf[i] = '.'
				}
			}
		}
		buf[symbol.length] = ' '
		m, err := w.W.Write(buf[:symbol.length+1])
		n += m
		if err != nil {
			return n, err
		}
	}
	return n, nil
}

func main() {
	s := "Hello, World!\r\n"
	w := &MorseWriter{tts}

	io.WriteString(tts, s)
	io.WriteString(w, s)
}

func ttsISR() {
	tts.ISR()
}

func ttsDMAISR() {
	tts.TxDMAISR()
}

//c:__attribute__((section(".ISRs")))
var ISRs = [...]func(){
	irq.USART1:          ttsISR,
	irq.DMA1_Channel2_3: ttsDMAISR,
}

type morseSymbol struct {
	code, length byte
}

//emgo:const
var morseSymbols = [...]morseSymbol{
	{1<<0 | 1<<1 | 1<<2, 4}, // ! ---.
	{1<<1 | 1<<4, 6},        // " .-..-.
	{},                      // #
	{1<<3 | 1<<6, 7},        // $ ...-..-
	{},                      // %
	{},                      // &
	{1<<1 | 1<<2 | 1<<3 | 1<<4, 6}, // ' .----.
	{1<<0 | 1<<2 | 1<<3, 5},        // ( -.--.
	{1<<0 | 1<<2 | 1<<3 | 1<<5, 6}, // ) -.--.-
	{},                                    // *
	{1<<1 | 1<<3, 5},                      // + .-.-.
	{1<<0 | 1<<1 | 1<<4 | 1<<5, 6},        // , --..--
	{1<<0 | 1<<5, 6},                      // - -....-
	{1<<1 | 1<<3 | 1<<5, 6},               // . .-.-.-
	{1<<0 | 1<<3, 5},                      // / -..-.
	{1<<0 | 1<<1 | 1<<2 | 1<<3 | 1<<4, 5}, // 0 -----
	{1<<1 | 1<<2 | 1<<3 | 1<<4, 5},        // 1 .----
	{1<<2 | 1<<3 | 1<<4, 5},               // 2 ..---
	{1<<3 | 1<<4, 5},                      // 3 ...--
	{1 << 4, 5},                           // 4 ....-
	{0, 5},                                // 5 .....
	{1 << 0, 5},                           // 6 -....
	{1<<0 | 1<<1, 5},                      // 7 --...
	{1<<0 | 1<<1 | 1<<2, 5},               // 8 ---..
	{1<<0 | 1<<1 | 1<<2 | 1<<3, 5},        // 9 ----.
	{1<<0 | 1<<1 | 1<<2, 6},               // : ---...
	{1<<0 | 1<<2 | 1<<4, 6},               // ; -.-.-.
	{},                      // <
	{1<<0 | 1<<4, 5},        // = -...-
	{},                      // >
	{1<<2 | 1<<3, 6},        // ? ..--..
	{1<<1 | 1<<2 | 1<<4, 6}, // @ .--.-.
	{1 << 1, 2},             // A .-
	{1 << 0, 4},             // B -...
	{1<<0 | 1<<2, 4},        // C -.-.
	{1 << 0, 3},             // D -..
	{0, 1},                  // E .
	{1 << 2, 4},             // F ..-.
	{1<<0 | 1<<1, 3},        // G --.
	{0, 4},                  // H ....
	{0, 2},                  // I ..
	{1<<1 | 1<<2 | 1<<3, 4}, // J .---
	{1<<0 | 1<<2, 3},        // K -.-
	{1 << 1, 4},             // L .-..
	{1<<0 | 1<<1, 2},        // M --
	{1 << 0, 2},             // N -.
	{1<<0 | 1<<1 | 1<<2, 3}, // O ---
	{1<<1 | 1<<2, 4},        // P .--.
	{1<<0 | 1<<1 | 1<<3, 4}, // Q --.-
	{1 << 1, 3},             // R .-.
	{0, 3},                  // S ...
	{1 << 0, 1},             // T -
	{1 << 2, 3},             // U ..-
	{1 << 3, 4},             // V ...-
	{1<<1 | 1<<2, 3},        // W .--
	{1<<0 | 1<<3, 4},        // X -..-
	{1<<0 | 1<<2 | 1<<3, 4}, // Y -.--
	{1<<0 | 1<<1, 4},        // Z --..
}
<|endoftext|>


<commit_before>package tradingdays

import (
	"fmt"
	"testing"
)

func TestReadCSV(*testing.T) {
	//readCSV()
	fmt.Println(exceptDays)
}
<commit_msg>Add `BenchmarkIsOpen`.<commit_after>package tradingdays

import (
	"fmt"
	"testing"
	"time"

	"github.com/toomore/gogrs/utils"
)

func TestReadCSV(*testing.T) {
	//readCSV()
	fmt.Println(exceptDays)
	fmt.Println(time.Date(2015, 9, 29, 0, 0, 0, 0, time.Local))
	fmt.Println(IsOpen(2015, 4, 17, time.Local))
	fmt.Println(IsOpen(2015, 4, 18, time.Local))
	fmt.Println(IsOpen(2015, 4, 20, time.Local))
	fmt.Println(IsOpen(2015, 4, 25, time.Local))
}

func BenchmarkIsOpen(t *testing.B) {
	for i := 0; i < t.N; i++ {
		IsOpen(2015, 4, 19, utils.TaipeiTimeZone)
	}
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
   License, v. 2.0. If a copy of the MPL was not distributed with this
   file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/

package main

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strconv"
	"sync"
	"time"

	"github.com/gorhill/cronexpr"
	"github.com/mozilla/tls-observatory/certificate"
	"github.com/mozilla/tls-observatory/database"

	"gopkg.in/yaml.v2"
)

type Configuration struct {
	Runs []Run
	Smtp struct {
		Host string
		Port int
		From string
		Auth struct {
			User, Pass string
		}
	}
	Slack struct {
		Username, IconEmoji, Webhook string
	}
}

type Run struct {
	Targets       []string
	Assertions    []Assertion
	Cron          string
	Notifications NotificationsConf
}

type Assertion struct {
	Certificate struct {
		Validity struct {
			NotBefore string
			NotAfter  string
		}
	}
	Analysis struct {
		Analyzer string
		Result   string `json:"result"`
	}
}

type NotificationsConf struct {
	Irc struct {
		Channels []string
	}
	Email struct {
		Recipients []string
	}
	Slack struct {
		Channels []string
	}
}

var (
	cfgFile     string
	observatory string
	debug       bool
	conf        Configuration
)

func main() {
	flag.StringVar(&observatory, "observatory", "https://tls-observatory.services.mozilla.com", "URL of the observatory")
	flag.StringVar(&cfgFile, "c", "/etc/tls-observatory/runner.yaml", "YAML configuration file")
	flag.BoolVar(&debug, "debug", false, "Set debug logging")
	flag.Parse()
	conf = getConf(cfgFile)
	exit := make(chan bool)
	for i, run := range conf.Runs {
		go run.start(i)
	}
	<-exit
}

func (r Run) start(id int) {
	for {
		cexpr, err := cronexpr.Parse(r.Cron)
		if err != nil {
			log.Printf("Failed to parse cron expression %q: %v", r.Cron, err)
			time.Sleep(time.Minute)
			continue
		}
		// sleep until the next run is scheduled to happen
		nrun := cexpr.Next(time.Now())
		waitduration := nrun.Sub(time.Now())
		log.Printf("[info] run %d will start at %v (in %v)", id, nrun, waitduration)
		time.Sleep(waitduration)

		notifchan := make(chan Notification)
		done := make(chan bool)
		go processNotifications(notifchan, done)
		var wg sync.WaitGroup
		for _, target := range r.Targets {
			log.Printf("[info] run %d starting scan of target %q", id, target)
			id, err := r.scan(target)
			debugprint("got scan id %d", id)
			if err != nil {
				log.Printf("[error] failed to launch against %q: %v", target, err)
				continue
			}
			wg.Add(1)
			go r.evaluate(id, notifchan, &wg)
		}
		wg.Wait()
		close(notifchan)
		<-done
	}
}

type scan struct {
	ID int64 `json:"scan_id"`
}

func (r Run) scan(target string) (id int64, err error) {
	defer func() {
		if e := recover(); e != nil {
			err = fmt.Errorf("scan(target=%q) -> %v", target, e)
		}
	}()
	resp, err := http.Post(observatory+"/api/v1/scan?target="+target, "application/json", nil)
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		panic(err)
	}
	if resp.StatusCode != http.StatusOK {
		panic(fmt.Sprintf("Scan failed. HTTP %d: %s", resp.StatusCode, body))
	}
	var s scan
	err = json.Unmarshal(body, &s)
	if err != nil {
		panic(err)
	}
	if s.ID < 1 {
		panic("failed to launch scan on target " + target)
	}
	id = s.ID
	return
}

func (r Run) evaluate(id int64, notifchan chan Notification, wg *sync.WaitGroup) {
	defer func() {
		if e := recover(); e != nil {
			log.Printf("[error] evaluate(id=%q) -> %v", id, e)
		}
		wg.Done()
	}()
	var (
		results database.Scan
		cert    certificate.Certificate
		err     error
	)
	for {
		resp, err := http.Get(fmt.Sprintf("%s/api/v1/results?id=%d", observatory, id))
		if err != nil {
			panic(err)
		}
		defer resp.Body.Close()
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			panic(err)
		}
		err = json.Unmarshal(body, &results)
		if err != nil {
			panic(err)
		}
		if results.Complperc >= 100 {
			debugprint("scan id %d completed", id)
			break
		}
		time.Sleep(5 * time.Second)
	}
	debugprint("getting certificate id %d", results.Cert_id)
	if !results.Has_tls && results.Cert_id < 1 {
		log.Printf("[info] target %q is not TLS enabled", results.Target)
		return
	}
	cert, err = getCert(results.Cert_id)
	if err != nil {
		panic(err)
	}
	for _, a := range r.Assertions {
		r.AssertNotBefore(a, results.Target, cert.Validity.NotBefore, notifchan)
		r.AssertNotAfter(a, results.Target, cert.Validity.NotAfter, notifchan)
		r.AssertAnalysis(a, results, cert, notifchan)
	}
	return
}

func getCert(id int64) (cert certificate.Certificate, err error) {
	defer func() {
		if e := recover(); e != nil {
			err = fmt.Errorf("getCert(id=%q) -> %v", e)
		}
	}()
	resp, err := http.Get(fmt.Sprintf("%s/api/v1/certificate?id=%d", observatory, id))
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		panic(err)
	}
	err = json.Unmarshal(body, &cert)
	if err != nil {
		panic(err)
	}
	return
}

func debugprint(format string, a ...interface{}) {
	if debug {
		log.Printf("[debug] "+format, a...)
	}
}

// getConf first read the configuration from a local YAML file then overrides it
// with the content of the TLSOBS_RUNNER_CONF var (which much contain a full yaml file
// encoded in base64), and then overrides the SMTP settings with various SMTP env var
func getConf(cfg string) (c Configuration) {
	// load the local configuration file
	fd, err := ioutil.ReadFile(cfg)
	if err != nil {
		log.Fatal(err)
	}
	err = yaml.Unmarshal(fd, &c)
	if err != nil {
		log.Fatalf("error: %v", err)
	}
	// iterate over notifications in targets and unbase64 the values
	for i, run := range c.Runs {
		for j, rcpt := range run.Notifications.Email.Recipients {
			if len(rcpt) < 5 || rcpt[0:4] != "b64:" {
				continue
			}
			data, err := base64.StdEncoding.DecodeString(rcpt[4:])
			if err != nil {
				log.Fatalf("error while decoding b64 recipient: %v", err)
			}
			c.Runs[i].Notifications.Email.Recipients[j] = fmt.Sprintf("%s", bytes.TrimRight(data, "\n"))
		}
	}
	if os.Getenv("TLSOBS_RUNNER_SMTP_HOST") != "" {
		c.Smtp.Host = os.Getenv("TLSOBS_RUNNER_SMTP_HOST")
	}
	if os.Getenv("TLSOBS_RUNNER_SMTP_PORT") != "" {
		var err error
		c.Smtp.Port, err = strconv.Atoi(os.Getenv("TLSOBS_RUNNER_SMTP_PORT"))
		if err != nil {
			log.Printf("[error] failed to read smtp port from env variable: %v", err)
		}
	}
	if os.Getenv("TLSOBS_RUNNER_SMTP_FROM") != "" {
		c.Smtp.From = os.Getenv("TLSOBS_RUNNER_SMTP_FROM")
	}
	if os.Getenv("TLSOBS_RUNNER_SMTP_AUTH_USER") != "" {
		c.Smtp.Auth.User = os.Getenv("TLSOBS_RUNNER_SMTP_AUTH_USER")
	}
	if os.Getenv("TLSOBS_RUNNER_SMTP_AUTH_PASS") != "" {
		c.Smtp.Auth.Pass = os.Getenv("TLSOBS_RUNNER_SMTP_AUTH_PASS")
	}
	if os.Getenv("TLSOBS_RUNNER_SLACK_USERNAME") != "" {
		c.Slack.Username = os.Getenv("TLSOBS_RUNNER_USERNAME")
	}
	if os.Getenv("TLSOBS_RUNNER_SLACK_ICONEMOJI") != "" {
		c.Slack.IconEmoji = os.Getenv("TLSOBS_RUNNER_SLACK_ICONEMOJI")
	}
	if os.Getenv("TLSOBS_RUNNER_SLACK_WEBHOOK") != "" {
		c.Slack.Webhook = os.Getenv("TLSOBS_RUNNER_SLACK_WEBHOOK")
	}
	return c
}
<commit_msg>fix go vet issues<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
   License, v. 2.0. If a copy of the MPL was not distributed with this
   file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/

package main

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strconv"
	"sync"
	"time"

	"github.com/gorhill/cronexpr"
	"github.com/mozilla/tls-observatory/certificate"
	"github.com/mozilla/tls-observatory/database"

	"gopkg.in/yaml.v2"
)

type Configuration struct {
	Runs []Run
	Smtp struct {
		Host string
		Port int
		From string
		Auth struct {
			User, Pass string
		}
	}
	Slack struct {
		Username, IconEmoji, Webhook string
	}
}

type Run struct {
	Targets       []string
	Assertions    []Assertion
	Cron          string
	Notifications NotificationsConf
}

type Assertion struct {
	Certificate struct {
		Validity struct {
			NotBefore string
			NotAfter  string
		}
	}
	Analysis struct {
		Analyzer string
		Result   string `json:"result"`
	}
}

type NotificationsConf struct {
	Irc struct {
		Channels []string
	}
	Email struct {
		Recipients []string
	}
	Slack struct {
		Channels []string
	}
}

var (
	cfgFile     string
	observatory string
	debug       bool
	conf        Configuration
)

func main() {
	flag.StringVar(&observatory, "observatory", "https://tls-observatory.services.mozilla.com", "URL of the observatory")
	flag.StringVar(&cfgFile, "c", "/etc/tls-observatory/runner.yaml", "YAML configuration file")
	flag.BoolVar(&debug, "debug", false, "Set debug logging")
	flag.Parse()
	conf = getConf(cfgFile)
	exit := make(chan bool)
	for i, run := range conf.Runs {
		go run.start(i)
	}
	<-exit
}

func (r Run) start(id int) {
	for {
		cexpr, err := cronexpr.Parse(r.Cron)
		if err != nil {
			log.Printf("Failed to parse cron expression %q: %v", r.Cron, err)
			time.Sleep(time.Minute)
			continue
		}
		// sleep until the next run is scheduled to happen
		nrun := cexpr.Next(time.Now())
		waitduration := nrun.Sub(time.Now())
		log.Printf("[info] run %d will start at %v (in %v)", id, nrun, waitduration)
		time.Sleep(waitduration)

		notifchan := make(chan Notification)
		done := make(chan bool)
		go processNotifications(notifchan, done)
		var wg sync.WaitGroup
		for _, target := range r.Targets {
			log.Printf("[info] run %d starting scan of target %q", id, target)
			id, err := r.scan(target)
			debugprint("got scan id %d", id)
			if err != nil {
				log.Printf("[error] failed to launch against %q: %v", target, err)
				continue
			}
			wg.Add(1)
			go r.evaluate(id, notifchan, &wg)
		}
		wg.Wait()
		close(notifchan)
		<-done
	}
}

type scan struct {
	ID int64 `json:"scan_id"`
}

func (r Run) scan(target string) (id int64, err error) {
	defer func() {
		if e := recover(); e != nil {
			err = fmt.Errorf("scan(target=%q) -> %v", target, e)
		}
	}()
	resp, err := http.Post(observatory+"/api/v1/scan?target="+target, "application/json", nil)
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		panic(err)
	}
	if resp.StatusCode != http.StatusOK {
		panic(fmt.Sprintf("Scan failed. HTTP %d: %s", resp.StatusCode, body))
	}
	var s scan
	err = json.Unmarshal(body, &s)
	if err != nil {
		panic(err)
	}
	if s.ID < 1 {
		panic("failed to launch scan on target " + target)
	}
	id = s.ID
	return
}

func (r Run) evaluate(id int64, notifchan chan Notification, wg *sync.WaitGroup) {
	defer func() {
		if e := recover(); e != nil {
			log.Printf("[error] evaluate(id=%q) -> %v", id, e)
		}
		wg.Done()
	}()
	var (
		results database.Scan
		cert    certificate.Certificate
		err     error
	)
	for {
		resp, err := http.Get(fmt.Sprintf("%s/api/v1/results?id=%d", observatory, id))
		if err != nil {
			panic(err)
		}
		defer resp.Body.Close()
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			panic(err)
		}
		err = json.Unmarshal(body, &results)
		if err != nil {
			panic(err)
		}
		if results.Complperc >= 100 {
			debugprint("scan id %d completed", id)
			break
		}
		time.Sleep(5 * time.Second)
	}
	debugprint("getting certificate id %d", results.Cert_id)
	if !results.Has_tls && results.Cert_id < 1 {
		log.Printf("[info] target %q is not TLS enabled", results.Target)
		return
	}
	cert, err = getCert(results.Cert_id)
	if err != nil {
		panic(err)
	}
	for _, a := range r.Assertions {
		r.AssertNotBefore(a, results.Target, cert.Validity.NotBefore, notifchan)
		r.AssertNotAfter(a, results.Target, cert.Validity.NotAfter, notifchan)
		r.AssertAnalysis(a, results, cert, notifchan)
	}
	return
}

func getCert(id int64) (cert certificate.Certificate, err error) {
	defer func() {
		if e := recover(); e != nil {
			err = fmt.Errorf("getCert(id=%q) -> %v", cert.ID, e)
		}
	}()
	resp, err := http.Get(fmt.Sprintf("%s/api/v1/certificate?id=%d", observatory, id))
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		panic(err)
	}
	err = json.Unmarshal(body, &cert)
	if err != nil {
		panic(err)
	}
	return
}

func debugprint(format string, a ...interface{}) {
	if debug {
		log.Printf("[debug] "+format, a...)
	}
}

// getConf first read the configuration from a local YAML file then overrides it
// with the content of the TLSOBS_RUNNER_CONF var (which much contain a full yaml file
// encoded in base64), and then overrides the SMTP settings with various SMTP env var
func getConf(cfg string) (c Configuration) {
	// load the local configuration file
	fd, err := ioutil.ReadFile(cfg)
	if err != nil {
		log.Fatal(err)
	}
	err = yaml.Unmarshal(fd, &c)
	if err != nil {
		log.Fatalf("error: %v", err)
	}
	// iterate over notifications in targets and unbase64 the values
	for i, run := range c.Runs {
		for j, rcpt := range run.Notifications.Email.Recipients {
			if len(rcpt) < 5 || rcpt[0:4] != "b64:" {
				continue
			}
			data, err := base64.StdEncoding.DecodeString(rcpt[4:])
			if err != nil {
				log.Fatalf("error while decoding b64 recipient: %v", err)
			}
			c.Runs[i].Notifications.Email.Recipients[j] = fmt.Sprintf("%s", bytes.TrimRight(data, "\n"))
		}
	}
	if os.Getenv("TLSOBS_RUNNER_SMTP_HOST") != "" {
		c.Smtp.Host = os.Getenv("TLSOBS_RUNNER_SMTP_HOST")
	}
	if os.Getenv("TLSOBS_RUNNER_SMTP_PORT") != "" {
		var err error
		c.Smtp.Port, err = strconv.Atoi(os.Getenv("TLSOBS_RUNNER_SMTP_PORT"))
		if err != nil {
			log.Printf("[error] failed to read smtp port from env variable: %v", err)
		}
	}
	if os.Getenv("TLSOBS_RUNNER_SMTP_FROM") != "" {
		c.Smtp.From = os.Getenv("TLSOBS_RUNNER_SMTP_FROM")
	}
	if os.Getenv("TLSOBS_RUNNER_SMTP_AUTH_USER") != "" {
		c.Smtp.Auth.User = os.Getenv("TLSOBS_RUNNER_SMTP_AUTH_USER")
	}
	if os.Getenv("TLSOBS_RUNNER_SMTP_AUTH_PASS") != "" {
		c.Smtp.Auth.Pass = os.Getenv("TLSOBS_RUNNER_SMTP_AUTH_PASS")
	}
	if os.Getenv("TLSOBS_RUNNER_SLACK_USERNAME") != "" {
		c.Slack.Username = os.Getenv("TLSOBS_RUNNER_USERNAME")
	}
	if os.Getenv("TLSOBS_RUNNER_SLACK_ICONEMOJI") != "" {
		c.Slack.IconEmoji = os.Getenv("TLSOBS_RUNNER_SLACK_ICONEMOJI")
	}
	if os.Getenv("TLSOBS_RUNNER_SLACK_WEBHOOK") != "" {
		c.Slack.Webhook = os.Getenv("TLSOBS_RUNNER_SLACK_WEBHOOK")
	}
	return c
}
<|endoftext|>