<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"regexp"
	"runtime/debug"
	"sort"
	"strings"
	"time"
)

type Fortune struct {
	wr         http.ResponseWriter
	rq         *http.Request
	deck       *Deck
	scoreCards *Deck
}

func init() {
	debug := flag.Bool("d", false, "debug")
	flag.Parse()

	if !*debug {
		log.SetOutput(ioutil.Discard)
	}
}

func main() {
	var fortune Fortune
	fortune.scoreCards = &Deck{}
	fortune.scoreCards.init()
	fortune.restoreScores()

	fmt.Println("Listening on http://localhost:8080")

	err := http.ListenAndServe(":8080", &fortune)
	if err != nil {
		fmt.Println(err)
	}
}

func (f *Fortune) ServeHTTP(wr http.ResponseWriter, rq *http.Request) {
	defer func() {
		obj := recover()
		if obj != nil {
			msg := fmt.Sprintf("
Error: %v\nStack: %v
", obj, string(debug.Stack()))
			io.WriteString(wr, msg)
			fmt.Println(msg)
		}
	}()

	f.wr = wr
	f.rq = rq

	path := rq.URL.Path
	contentType := "text/html"
	root := "."

	switch {
	case strings.HasPrefix(path, "/playing-cards/"):
		contentType = "image/png"
		wr.Header().Set("cache-control", "max-age=86400")

	case strings.HasPrefix(path, "/js/"):
		contentType = "application/javascript"

	case strings.HasPrefix(path, "/css/"):
		contentType = "text/css"

	case strings.HasPrefix(path, "/html/"):
		contentType = "text/html"

	case path == "/":
		contentType = "text/html"
		path = "/html/main.html"
		fmt.Printf("%s: Visitor from %s\n", time.Now(), rq.RemoteAddr)

	case path == "/init":
		f.init()
		path = ""

	case path == "/deal":
		f.deal()
		path = ""

	case path == "/fortune":
		f.fortune()
		path = ""

	case path == "/scores":
		f.scores()
		path = ""
	}

	if len(path) > 0 {
		wr.Header().Set("Content-Type", contentType)
		data, err := ioutil.ReadFile(root + path)
		if err == nil {
			wr.Write(data)
		} else {
			fmt.Fprint(wr, err)
		}
	}
}

func (f *Fortune) init() {
	f.deck = &Deck{}
	f.deck.init()
	f.deck.shuffle()
	f.deck.Cards = f.deck.Cards[:21]

	type Response struct {
		Cards []*Card
		Error string
	}

	response := &Response{
		Cards: f.deck.Cards,
	}
	data, err := json.Marshal(response)
	if err != nil {
		response.Error = err.Error()
	}
	f.wr.Header().Set("Content-Type", "application/json")
	f.wr.Write(data)
}

func (f *Fortune) deal() {
	type RequestCard struct {
		Image string
	}
	type Request struct {
		Cards []RequestCard
		Row   int
		Count int
	}
	type Response struct {
		Row1  []*Card
		Row2  []*Card
		Row3  []*Card
		Card  string
		Error string
	}

	response := &Response{}
	reqData, err := ioutil.ReadAll(f.rq.Body)
	if err != nil {
		response.Error = err.Error()
	} else {
		request := &Request{}
		err = json.Unmarshal(reqData, request)
		if err != nil {
			response.Error = err.Error()
		}
		f.deck = &Deck{}
		for _, card := range request.Cards {
			f.deck.Cards = append(f.deck.Cards, &Card{Image: card.Image})
		}
		if len(request.Cards) == 21 {
			if request.Row == 0 {
				response.Row1 = f.deck.Cards[:7]
				response.Row2 = f.deck.Cards[7:14]
				response.Row3 = f.deck.Cards[14:]
			} else {
				f.deck.placeMiddle(request.Row)
				f.deck.deal()
				response.Row1 = f.deck.Row1
				response.Row2 = f.deck.Row2
				response.Row3 = f.deck.Row3
			}
		} else {
			response.Error += "\nDeck should have 21 cards."
		}
		log.Printf("request: %v\n", request)
		if request.Count == 3 {
			response.Card = f.deck.Row2[3].Image
			f.scoreCard(response.Card)
			log.Printf("memorized card: %s\n", response.Card)
		}
	}

	data, err := json.Marshal(response)
	if err != nil {
		response.Error = err.Error()
	}
	f.wr.Header().Set("Content-Type", "application/json")
	f.wr.Write(data)
}

func (f *Fortune) fortune() {
	words := map[string]string{
		"2C.png":  "passion",
		"2D.png":  "wealth",
		"2H.png":  "love",
		"2S.png":  "law",
		"3C.png":  "interest",
		"3D.png":  "rich",
		"3H.png":  "like",
		"3S.png":  "rule",
		"4C.png":  "positive",
		"4D.png":  "gold",
		"4H.png":  "nice",
		"4S.png":  "command",
		"5C.png":  "real",
		"5D.png":  "money",
		"5H.png":  "related",
		"5S.png":  "advise",
		"6C.png":  "growing",
		"6D.png":  "fortune",
		"6H.png":  "good",
		"6S.png":  "statement",
		"7C.png":  "study",
		"7D.png":  "well",
		"7H.png":  "sweet",
		"7S.png":  "court",
		"8C.png":  "understand",
		"8D.png":  "cash",
		"8H.png":  "protect",
		"8S.png":  "action",
		"9C.png":  "hobby",
		"9D.png":  "stock",
		"9H.png":  "live",
		"9S.png":  "act",
		"10C.png": "knowledge",
		"10D.png": "value",
		"10H.png": "friend",
		"10S.png": "order",
		"JC.png":  "student",
		"JD.png":  "banker",
		"JH.png":  "husband",
		"JS.png":  "judge",
		"QC.png":  "nurse",
		"QD.png":  "actress",
		"QH.png":  "wife",
		"QS.png":  "queen",
		"KC.png":  "researcher",
		"KD.png":  "ceo",
		"KH.png":  "lover",
		"KS.png":  "congressman",
		"AC.png":  "president",
		"AD.png":  "thesaurus",
		"AH.png":  "family",
		"AS.png":  "country",
	}
	type Request struct {
		Card string
	}
	type Response struct {
		Tweet string
		Error string
	}

	response := &Response{}

	request := &Request{}
	reqData, err := ioutil.ReadAll(f.rq.Body)
	err = json.Unmarshal(reqData, request)
	if err != nil {
		response.Error = err.Error()
	}

	key, _ := words[request.Card]
	url := "https://twitter.com/search?f=realtime&q=" + key
	resp, err := http.Get(url)
	if err != nil {
		response.Error = "Error: " + err.Error()
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		response.Error = "Error: " + err.Error()
	}
	resp.Body.Close()

	search := regexp.MustCompile(`<p class="TweetTextSize .*>.*</p>`)
	tweets := search.FindAllString(string(body), -1)

	tweet := "Unable to fetch tweets."
	if len(tweets) > 0 {
		index := rand.Intn(len(tweets))
		tweet = tweets[index]
	}
	response.Tweet = tweet
	fmt.Printf("Visitor=%s word=%s fortune=%s\n", f.rq.RemoteAddr, key, tweet)

	data, err := json.Marshal(response)
	if err != nil {
		response.Error = err.Error()
	}
	f.wr.Header().Set("Content-Type", "application/json")
	f.wr.Write(data)
}

func (f *Fortune) scoreCard(memorizedCard string) {
	for _, card := range f.scoreCards.Cards {
		if card.Image == memorizedCard {
			card.Score++
			break
		}
	}
	f.saveScores()
}

func (f *Fortune) scores() {
	type Response struct {
		ScoreCards []*Card
		Error      string
	}

	response := &Response{}
	sort.Sort(f.scoreCards)
	response.ScoreCards = f.scoreCards.Cards

	data, err := json.Marshal(response)
	if err != nil {
		response.Error = err.Error()
	}
	f.wr.Header().Set("Content-Type", "application/json")
	f.wr.Write(data)
}

func (f *Fortune) saveScores() {
	data, err := json.Marshal(f.scoreCards)
	if err != nil {
		fmt.Println(err)
		return
	}
	err = ioutil.WriteFile("scores.json", data, 0644)
	if err != nil {
		fmt.Println(err)
		return
	}
}

func (f *Fortune) restoreScores() {
	data, err := ioutil.ReadFile("scores.json")
	if err != nil {
		fmt.Println(err)
		return
	}
	err = json.Unmarshal(data, f.scoreCards)
	if err != nil {
		fmt.Println(err)
		return
	}
}
<commit_msg>fixing a bug<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"regexp"
	"runtime/debug"
	"sort"
	"strings"
	"time"
)

type Fortune struct {
	deck       *Deck
	scoreCards *Deck
}

func init() {
	debug := flag.Bool("d", false, "debug")
	flag.Parse()

	if !*debug {
		log.SetOutput(ioutil.Discard)
	}
}

func main() {
	var fortune Fortune
	fortune.scoreCards = &Deck{}
	fortune.scoreCards.init()
	fortune.restoreScores()

	fmt.Println("Listening on http://localhost:8080")

	err := http.ListenAndServe(":8080", &fortune)
	if err != nil {
		fmt.Println(err)
	}
}

func (f *Fortune) ServeHTTP(wr http.ResponseWriter, rq *http.Request) {
	defer func() {
		obj := recover()
		if obj != nil {
			msg := fmt.Sprintf("
Error: %v\nStack: %v
", obj, string(debug.Stack()))
			io.WriteString(wr, msg)
			fmt.Println(msg)
		}
	}()

	path := rq.URL.Path
	contentType := "text/html"
	root := "."

	switch {
	case strings.HasPrefix(path, "/playing-cards/"):
		contentType = "image/png"
		wr.Header().Set("cache-control", "max-age=86400")

	case strings.HasPrefix(path, "/js/"):
		contentType = "application/javascript"

	case strings.HasPrefix(path, "/css/"):
		contentType = "text/css"

	case strings.HasPrefix(path, "/html/"):
		contentType = "text/html"
	}

	switch path {
	case "/":
		contentType = "text/html"
		path = "/html/main.html"
		fmt.Printf("%s: Visitor from %s\n", time.Now(), rq.RemoteAddr)

	case "/init":
		f.init(wr, rq)
		path = ""

	case "/deal":
		f.deal(wr, rq)
		path = ""

	case "/fortune":
		f.fortune(wr, rq)
		path = ""

	case "/scores":
		f.scores(wr, rq)
		path = ""
	}

	if len(path) > 0 {
		wr.Header().Set("Content-Type", contentType)
		data, err := ioutil.ReadFile(root + path)
		if err == nil {
			wr.Write(data)
		} else {
			fmt.Fprint(wr, err)
		}
	}
}

func (f *Fortune) init(wr http.ResponseWriter, rq *http.Request) {
	f.deck = &Deck{}
	f.deck.init()
	f.deck.shuffle()
	f.deck.Cards = f.deck.Cards[:21]

	type Response struct {
		Cards []*Card
		Error string
	}

	response := &Response{
		Cards: f.deck.Cards,
	}
	data, err := json.Marshal(response)
	if err != nil {
		response.Error = err.Error()
	}
	wr.Header().Set("Content-Type", "application/json")
	wr.Write(data)
}

func (f *Fortune) deal(wr http.ResponseWriter, rq *http.Request) {
	type RequestCard struct {
		Image string
	}
	type Request struct {
		Cards []RequestCard
		Row   int
		Count int
	}
	type Response struct {
		Row1  []*Card
		Row2  []*Card
		Row3  []*Card
		Card  string
		Error string
	}

	response := &Response{}
	reqData, err := ioutil.ReadAll(rq.Body)
	if err != nil {
		response.Error = err.Error()
	} else {
		request := &Request{}
		err = json.Unmarshal(reqData, request)
		if err != nil {
			response.Error = err.Error()
		}
		f.deck = &Deck{}
		for _, card := range request.Cards {
			f.deck.Cards = append(f.deck.Cards, &Card{Image: card.Image})
		}
		if len(request.Cards) == 21 {
			if request.Row == 0 {
				response.Row1 = f.deck.Cards[:7]
				response.Row2 = f.deck.Cards[7:14]
				response.Row3 = f.deck.Cards[14:]
			} else {
				f.deck.placeMiddle(request.Row)
				f.deck.deal()
				response.Row1 = f.deck.Row1
				response.Row2 = f.deck.Row2
				response.Row3 = f.deck.Row3
			}
		} else {
			response.Error += "\nDeck should have 21 cards."
		}
		log.Printf("request: %v\n", request)
		if request.Count == 3 {
			response.Card = f.deck.Row2[3].Image
			f.scoreCard(response.Card)
			log.Printf("memorized card: %s\n", response.Card)
		}
	}

	data, err := json.Marshal(response)
	if err != nil {
		response.Error = err.Error()
	}
	wr.Header().Set("Content-Type", "application/json")
	wr.Write(data)
}

func (f *Fortune) fortune(wr http.ResponseWriter, rq *http.Request) {
	words := map[string]string{
		"2C.png":  "passion",
		"2D.png":  "wealth",
		"2H.png":  "love",
		"2S.png":  "law",
		"3C.png":  "interest",
		"3D.png":  "rich",
		"3H.png":  "like",
		"3S.png":  "rule",
		"4C.png":  "positive",
		"4D.png":  "gold",
		"4H.png":  "nice",
		"4S.png":  "command",
		"5C.png":  "real",
		"5D.png":  "money",
		"5H.png":  "related",
		"5S.png":  "advise",
		"6C.png":  "growing",
		"6D.png":  "fortune",
		"6H.png":  "good",
		"6S.png":  "statement",
		"7C.png":  "study",
		"7D.png":  "well",
		"7H.png":  "sweet",
		"7S.png":  "court",
		"8C.png":  "understand",
		"8D.png":  "cash",
		"8H.png":  "protect",
		"8S.png":  "action",
		"9C.png":  "hobby",
		"9D.png":  "stock",
		"9H.png":  "live",
		"9S.png":  "act",
		"10C.png": "knowledge",
		"10D.png": "value",
		"10H.png": "friend",
		"10S.png": "order",
		"JC.png":  "student",
		"JD.png":  "banker",
		"JH.png":  "husband",
		"JS.png":  "judge",
		"QC.png":  "nurse",
		"QD.png":  "actress",
		"QH.png":  "wife",
		"QS.png":  "queen",
		"KC.png":  "researcher",
		"KD.png":  "ceo",
		"KH.png":  "lover",
		"KS.png":  "congressman",
		"AC.png":  "president",
		"AD.png":  "thesaurus",
		"AH.png":  "family",
		"AS.png":  "country",
	}
	type Request struct {
		Card string
	}
	type Response struct {
		Tweet string
		Error string
	}

	response := &Response{}

	request := &Request{}
	reqData, err := ioutil.ReadAll(rq.Body)
	err = json.Unmarshal(reqData, request)
	if err != nil {
		response.Error = err.Error()
	}

	key, _ := words[request.Card]
	url := "https://twitter.com/search?f=realtime&q=" + key
	resp, err := http.Get(url)
	if err != nil {
		response.Error = "Error: " + err.Error()
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		response.Error = "Error: " + err.Error()
	}
	resp.Body.Close()

	search := regexp.MustCompile(`<p class="TweetTextSize .*>.*</p>`)
	tweets := search.FindAllString(string(body), -1)

	tweet := "Unable to fetch tweets."
	if len(tweets) > 0 {
		index := rand.Intn(len(tweets))
		tweet = tweets[index]
	}
	response.Tweet = tweet
	fmt.Printf("Visitor=%s word=%s fortune=%s\n", rq.RemoteAddr, key, tweet)

	data, err := json.Marshal(response)
	if err != nil {
		response.Error = err.Error()
	}
	wr.Header().Set("Content-Type", "application/json")
	wr.Write(data)
}

func (f *Fortune) scoreCard(memorizedCard string) {
	for _, card := range f.scoreCards.Cards {
		if card.Image == memorizedCard {
			card.Score++
			break
		}
	}
	f.saveScores()
}

func (f *Fortune) scores(wr http.ResponseWriter, rq *http.Request) {
	type Response struct {
		ScoreCards []*Card
		Error      string
	}

	response := &Response{}
	sort.Sort(f.scoreCards)
	response.ScoreCards = f.scoreCards.Cards

	data, err := json.Marshal(response)
	if err != nil {
		response.Error = err.Error()
	}
	wr.Header().Set("Content-Type", "application/json")
	wr.Write(data)
}

func (f *Fortune) saveScores() {
	data, err := json.Marshal(f.scoreCards)
	if err != nil {
		fmt.Println(err)
		return
	}
	err = ioutil.WriteFile("scores.json", data, 0644)
	if err != nil {
		fmt.Println(err)
		return
	}
}

func (f *Fortune) restoreScores() {
	data, err := ioutil.ReadFile("scores.json")
	if err != nil {
		fmt.Println(err)
		return
	}
	err = json.Unmarshal(data, f.scoreCards)
	if err != nil {
		fmt.Println(err)
		return
	}
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/hashicorp/consul/api"
	"gopkg.in/urfave/cli.v1"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"log"
	"os"
	"strings"
)

func readFile(file string) map[interface{}]interface{} {
	contents := make(map[interface{}]interface{})

	data, err := ioutil.ReadFile(file)
	if err != nil {
		log.Fatalf("Unable to read %s: %s", file, err)
	}

	err = yaml.Unmarshal(data, &contents)
	if err != nil {
		log.Fatalf("Unable to parse %s: %s", file, err)
	}

	return contents
}

func process(data interface{}, output *map[string]interface{}) {
	var keys []string
	if output == nil {
		tmp := make(map[string]interface{})
		output = &tmp
	}

	stringMarshallers := map[string]stringMarshaller{
		"!env$":  envMarshal,
		"!yaml$": yamlMarshal,
		"!json$": jsonMarshal,
	}

	gt := &Traverser{}

	gt.Node = func(keys []string, data interface{}) {
		if data == nil {
			data = (interface{})("")
		}
		(*output)[strings.Join(keys, "/")] = data
	}

	gt.Map = func(keys []string, key string, data interface{}) {
		for pattern, fn := range stringMarshallers {
			if value, applied := applyIfKeyMatch(key, pattern, fn, data); applied {
				(*output)[strings.Join(append(keys, key), "/")] = value
				return
			}
		}

		gt.Traverse(data, append(keys, key))
	}

	gt.Traverse(data, keys)
}

func main() {
	app := cli.NewApp()
	app.Name = "consulsync"
	app.Usage = "Sync YAML / JSON files to consul KV"
	app.Version = "0.0.1"

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "consul-address",
			Usage: "JSON / YAML file",
		},
		cli.StringFlag{
			Name:  "datacenter",
			Usage: "Datacenter",
		},
		cli.StringFlag{
			Name:  "kv-prefix",
			Usage: "KV prefix to sync",
		},
		cli.BoolFlag{
			Name:  "dryrun",
			Usage: "Do not perform changes",
		},
		cli.BoolFlag{
			Name:  "quiet",
			Usage: "Do not print output",
		},
	}

	app.Action = func(c *cli.Context) error {
		out := make(map[string]interface{})
		for _, file := range c.Args() {
			data := readFile(file)
			process(interface{}(data), &out)
		}

		config := &api.Config{
			Address:    c.GlobalString("consul-address"),
			Datacenter: c.GlobalString("datacenter"),
		}

		consul, err := api.NewClient(config)
		if err != nil {
			panic(err)
		}

		consulData := fetchConsulData(consul, c.GlobalString("kv-prefix"))
		diff := diffConsulData(c.GlobalString("kv-prefix"), consulData, out)

		if !c.Bool("quiet") {
			diff.Print()
		}

		if !c.Bool("dryrun") {
			applyDiff(consul, diff)
		}

		return nil
	}

	app.Run(os.Args)
}
<commit_msg>Trims tag from tagged keys<commit_after>package main

import (
	"fmt"
	"github.com/hashicorp/consul/api"
	"gopkg.in/urfave/cli.v1"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"log"
	"os"
	"strings"
)

func readFile(file string) map[interface{}]interface{} {
	contents := make(map[interface{}]interface{})

	data, err := ioutil.ReadFile(file)
	if err != nil {
		log.Fatalf("Unable to read %s: %s", file, err)
	}

	err = yaml.Unmarshal(data, &contents)
	if err != nil {
		log.Fatalf("Unable to parse %s: %s", file, err)
	}

	return contents
}

func process(data interface{}, output *map[string]interface{}) {
	var keys []string
	if output == nil {
		tmp := make(map[string]interface{})
		output = &tmp
	}

	stringMarshallers := map[string]stringMarshaller{
		"!env":  envMarshal,
		"!yaml": yamlMarshal,
		"!json": jsonMarshal,
	}

	gt := &Traverser{}

	gt.Node = func(keys []string, data interface{}) {
		if data == nil {
			data = (interface{})("")
		}
		(*output)[strings.Join(keys, "/")] = data
	}

	gt.Map = func(keys []string, key string, data interface{}) {
		for tag, fn := range stringMarshallers {
			pattern := fmt.Sprintf("%s$", tag)
			if value, applied := applyIfKeyMatch(key, pattern, fn, data); applied {
				(*output)[strings.Join(append(keys, strings.TrimSuffix(key, tag)), "/")] = value
				return
			}
		}

		gt.Traverse(data, append(keys, key))
	}

	gt.Traverse(data, keys)
}

func main() {
	app := cli.NewApp()
	app.Name = "consulsync"
	app.Usage = "Sync YAML / JSON files to consul KV"
	app.Version = "0.0.1"

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "consul-address",
			Usage: "JSON / YAML file",
		},
		cli.StringFlag{
			Name:  "datacenter",
			Usage: "Datacenter",
		},
		cli.StringFlag{
			Name:  "kv-prefix",
			Usage: "KV prefix to sync",
		},
		cli.BoolFlag{
			Name:  "dryrun",
			Usage: "Do not perform changes",
		},
		cli.BoolFlag{
			Name:  "quiet",
			Usage: "Do not print output",
		},
	}

	app.Action = func(c *cli.Context) error {
		out := make(map[string]interface{})
		for _, file := range c.Args() {
			data := readFile(file)
			process(interface{}(data), &out)
		}

		config := &api.Config{
			Address:    c.GlobalString("consul-address"),
			Datacenter: c.GlobalString("datacenter"),
		}

		consul, err := api.NewClient(config)
		if err != nil {
			panic(err)
		}

		consulData := fetchConsulData(consul, c.GlobalString("kv-prefix"))
		diff := diffConsulData(c.GlobalString("kv-prefix"), consulData, out)

		if !c.Bool("quiet") {
			diff.Print()
		}

		if !c.Bool("dryrun") {
			applyDiff(consul, diff)
		}

		return nil
	}

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>// Copyright (c) 2015, Nathan Youngman. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"crypto/md5"
	"flag"
	"fmt"
	"io"
	"log"
	"mime"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/mitchellh/goamz/aws" // http://gopkg.in/amz.v2
	"github.com/mitchellh/goamz/s3"
)

type file struct {
	path         string // relative path
	absPath      string // absolute path
	size         int64
	lastModified time.Time
}

var wg sync.WaitGroup

func main() {
	var accessKey, secretKey, sourcePath, regionName, bucketName string
	var numberOfWorkers int
	var help bool

	// Usage example:
	// s3up -source=public/ -bucket=origin.edmontongo.org -key=$AWS_ACCESS_KEY_ID -secret=$AWS_SECRET_ACCESS_KEY

	flag.StringVar(&accessKey, "key", "", "Access Key ID for AWS")
	flag.StringVar(&secretKey, "secret", "", "Secret Access Key for AWS")
	flag.StringVar(&regionName, "region", "us-east-1", "Name of region for AWS")
	flag.StringVar(&bucketName, "bucket", "", "Destination bucket name on AWS")
	flag.StringVar(&sourcePath, "source", ".", "path of files to upload")
	flag.IntVar(&numberOfWorkers, "workers", 10, "number of workers to upload files")
	flag.BoolVar(&help, "h", false, "help")

	flag.Parse()

	fmt.Println("s3up 0.1.0, (c) 2015 Nathan Youngman.")

	if help {
		flag.Usage()
		return
	}

	var auth aws.Auth

	if accessKey != "" && secretKey != "" {
		auth = aws.Auth{AccessKey: accessKey, SecretKey: secretKey}
	} else if accessKey != "" || secretKey != "" {
		// provided one but not both
		fmt.Println("AWS key and secret are required.")
		flag.Usage()
		os.Exit(1)
	} else {
		// TODO: Getenv AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
		// load credentials from file
		var err error
		auth, err = aws.SharedAuth()
		if err != nil {
			fmt.Println("Credentials not found in ~/.aws/credentials")
			flag.Usage()
			os.Exit(1)
		}
	}

	if bucketName == "" {
		fmt.Println("AWS bucket is required.")
		flag.Usage()
		os.Exit(1)
	}

	// Get bucket with bucketName

	region := aws.Regions[regionName]
	s := s3.New(auth, region)
	b := s.Bucket(bucketName)

	filesToUpload := make(chan file)
	errs := make(chan error, 1)
	for i := 0; i < numberOfWorkers; i++ {
		wg.Add(1)
		go worker(filesToUpload, b, errs)
	}

	plan(sourcePath, b, filesToUpload)

	wg.Wait()

	// if any errors occurred during upload, exit with an error status code
	select {
	case <-errs:
		fmt.Println("Errors occurred while upload files.")
		os.Exit(1)
	default:
	}
}

// plan figures out which files need to be uploaded.
func plan(sourcePath string, destBucket *s3.Bucket, uploadFiles chan<- file) {
	// List all files in the remote bucket
	contents, err := destBucket.GetBucketContents()
	if err != nil {
		log.Fatal(err)
	}
	remoteFiles := *contents

	// All local files at sourcePath
	localFiles := make(chan file)
	go walk(sourcePath, localFiles)

	for f := range localFiles {
		// default: upload because local file not found on remote.
		up := true
		reason := "not found"

		if key, ok := remoteFiles[f.path]; ok {
			up, reason = shouldOverwrite(f, key)
			// remove from map, whatever is leftover should be deleted:
			delete(remoteFiles, f.path)
		}

		if up {
			fmt.Printf("%s %s, uploading.\n", f.path, reason)
			uploadFiles <- f
		} else {
			fmt.Printf("%s %s, skipping.\n", f.path, reason)
		}
	}
	close(uploadFiles)

	// any remote files not found locally should be removed:
	var filesToDelete = make([]string, 0, len(remoteFiles))
	for key := range remoteFiles {
		fmt.Printf("%s not found in source, deleting.\n", key)
		filesToDelete = append(filesToDelete, key)
	}
	cleanup(filesToDelete, destBucket)
}

// walk a local directory
func walk(basePath string, files chan<- file) {
	filepath.Walk(basePath, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() {
			// skip hidden directories like .git
			if strings.HasPrefix(info.Name(), ".") {
				return filepath.SkipDir
			}
		} else {
			if info.Name() == ".DS_Store" {
				return nil
			}
			abs, err := filepath.Abs(path)
			if err != nil {
				return err
			}
			rel, err := filepath.Rel(basePath, path)
			if err != nil {
				return err
			}
			files <- file{path: rel, absPath: abs, size: info.Size(), lastModified: info.ModTime()}
		}
		return nil
	})
	close(files)
}

// shouldOverwrite uses size or md5 to determine what needs to be uploaded
func shouldOverwrite(source file, dest s3.Key) (up bool, reason string) {
	if source.size != dest.Size {
		return true, "file size mismatch"
	}

	sourceMod := source.lastModified.UTC().Format(time.RFC3339)
	if sourceMod == dest.LastModified {
		// no need to upload if times match, but different times may just be drift
		return false, "last modified match"
	}

	etag, err := calculateETag(source.absPath)
	if err != nil {
		log.Fatalf("Error calculating ETag for %s: %v", source.absPath, err)
	}
	if dest.ETag == etag {
		return false, "etags match"
	}
	return true, "etags mismatch"
}

func calculateETag(path string) (string, error) {
	f, err := os.Open(path)
	if err != nil {
		return "", err
	}
	defer f.Close()

	h := md5.New()
	_, err = io.Copy(h, f)
	if err != nil {
		return "", err
	}
	etag := fmt.Sprintf("\"%x\"", h.Sum(nil))
	return etag, nil
}

func cleanup(paths []string, destBucket *s3.Bucket) error {
	// only can delete 1000 at a time, TODO: split if needed
	return destBucket.MultiDel(paths)
}

// worker uploads files
func worker(filesToUpload <-chan file, destBucket *s3.Bucket, errs chan<- error) {
	for f := range filesToUpload {
		err := upload(f, destBucket)
		if err != nil {
			fmt.Printf("Error uploading %s: %s\n", f.path, err)
			// if there are no errors on the channel, put this one there
			select {
			case errs <- err:
			default:
			}
		}
	}

	wg.Done()
}

func upload(source file, destBucket *s3.Bucket) error {
	f, err := os.Open(source.absPath)
	if err != nil {
		return err
	}
	defer f.Close()

	contentType := mime.TypeByExtension(filepath.Ext(source.path))
	if contentType == "" {
		contentType = "application/octet-stream"
	}

	return destBucket.PutReader(source.path, f, source.size, contentType, "public-read")
}
<commit_msg>grammar<commit_after>// Copyright (c) 2015, Nathan Youngman. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"crypto/md5"
	"flag"
	"fmt"
	"io"
	"log"
	"mime"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/mitchellh/goamz/aws" // http://gopkg.in/amz.v2
	"github.com/mitchellh/goamz/s3"
)

type file struct {
	path         string // relative path
	absPath      string // absolute path
	size         int64
	lastModified time.Time
}

var wg sync.WaitGroup

func main() {
	var accessKey, secretKey, sourcePath, regionName, bucketName string
	var numberOfWorkers int
	var help bool

	// Usage example:
	// s3up -source=public/ -bucket=origin.edmontongo.org -key=$AWS_ACCESS_KEY_ID -secret=$AWS_SECRET_ACCESS_KEY

	flag.StringVar(&accessKey, "key", "", "Access Key ID for AWS")
	flag.StringVar(&secretKey, "secret", "", "Secret Access Key for AWS")
	flag.StringVar(&regionName, "region", "us-east-1", "Name of region for AWS")
	flag.StringVar(&bucketName, "bucket", "", "Destination bucket name on AWS")
	flag.StringVar(&sourcePath, "source", ".", "path of files to upload")
	flag.IntVar(&numberOfWorkers, "workers", 10, "number of workers to upload files")
	flag.BoolVar(&help, "h", false, "help")

	flag.Parse()

	fmt.Println("s3up 0.1.0, (c) 2015 Nathan Youngman.")

	if help {
		flag.Usage()
		return
	}

	var auth aws.Auth

	if accessKey != "" && secretKey != "" {
		auth = aws.Auth{AccessKey: accessKey, SecretKey: secretKey}
	} else if accessKey != "" || secretKey != "" {
		// provided one but not both
		fmt.Println("AWS key and secret are required.")
		flag.Usage()
		os.Exit(1)
	} else {
		// TODO: Getenv AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
		// load credentials from file
		var err error
		auth, err = aws.SharedAuth()
		if err != nil {
			fmt.Println("Credentials not found in ~/.aws/credentials")
			flag.Usage()
			os.Exit(1)
		}
	}

	if bucketName == "" {
		fmt.Println("AWS bucket is required.")
		flag.Usage()
		os.Exit(1)
	}

	// Get bucket with bucketName

	region := aws.Regions[regionName]
	s := s3.New(auth, region)
	b := s.Bucket(bucketName)

	filesToUpload := make(chan file)
	errs := make(chan error, 1)
	for i := 0; i < numberOfWorkers; i++ {
		wg.Add(1)
		go worker(filesToUpload, b, errs)
	}

	plan(sourcePath, b, filesToUpload)

	wg.Wait()

	// if any errors occurred during upload, exit with an error status code
	select {
	case <-errs:
		fmt.Println("Errors occurred while uploading files.")
		os.Exit(1)
	default:
	}
}

// plan figures out which files need to be uploaded.
func plan(sourcePath string, destBucket *s3.Bucket, uploadFiles chan<- file) {
	// List all files in the remote bucket
	contents, err := destBucket.GetBucketContents()
	if err != nil {
		log.Fatal(err)
	}
	remoteFiles := *contents

	// All local files at sourcePath
	localFiles := make(chan file)
	go walk(sourcePath, localFiles)

	for f := range localFiles {
		// default: upload because local file not found on remote.
		up := true
		reason := "not found"

		if key, ok := remoteFiles[f.path]; ok {
			up, reason = shouldOverwrite(f, key)
			// remove from map, whatever is leftover should be deleted:
			delete(remoteFiles, f.path)
		}

		if up {
			fmt.Printf("%s %s, uploading.\n", f.path, reason)
			uploadFiles <- f
		} else {
			fmt.Printf("%s %s, skipping.\n", f.path, reason)
		}
	}
	close(uploadFiles)

	// any remote files not found locally should be removed:
	var filesToDelete = make([]string, 0, len(remoteFiles))
	for key := range remoteFiles {
		fmt.Printf("%s not found in source, deleting.\n", key)
		filesToDelete = append(filesToDelete, key)
	}
	cleanup(filesToDelete, destBucket)
}

// walk a local directory
func walk(basePath string, files chan<- file) {
	filepath.Walk(basePath, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() {
			// skip hidden directories like .git
			if strings.HasPrefix(info.Name(), ".") {
				return filepath.SkipDir
			}
		} else {
			if info.Name() == ".DS_Store" {
				return nil
			}
			abs, err := filepath.Abs(path)
			if err != nil {
				return err
			}
			rel, err := filepath.Rel(basePath, path)
			if err != nil {
				return err
			}
			files <- file{path: rel, absPath: abs, size: info.Size(), lastModified: info.ModTime()}
		}
		return nil
	})
	close(files)
}

// shouldOverwrite uses size or md5 to determine what needs to be uploaded
func shouldOverwrite(source file, dest s3.Key) (up bool, reason string) {
	if source.size != dest.Size {
		return true, "file size mismatch"
	}

	sourceMod := source.lastModified.UTC().Format(time.RFC3339)
	if sourceMod == dest.LastModified {
		// no need to upload if times match, but different times may just be drift
		return false, "last modified match"
	}

	etag, err := calculateETag(source.absPath)
	if err != nil {
		log.Fatalf("Error calculating ETag for %s: %v", source.absPath, err)
	}
	if dest.ETag == etag {
		return false, "etags match"
	}
	return true, "etags mismatch"
}

func calculateETag(path string) (string, error) {
	f, err := os.Open(path)
	if err != nil {
		return "", err
	}
	defer f.Close()

	h := md5.New()
	_, err = io.Copy(h, f)
	if err != nil {
		return "", err
	}
	etag := fmt.Sprintf("\"%x\"", h.Sum(nil))
	return etag, nil
}

func cleanup(paths []string, destBucket *s3.Bucket) error {
	// only can delete 1000 at a time, TODO: split if needed
	return destBucket.MultiDel(paths)
}

// worker uploads files
func worker(filesToUpload <-chan file, destBucket *s3.Bucket, errs chan<- error) {
	for f := range filesToUpload {
		err := upload(f, destBucket)
		if err != nil {
			fmt.Printf("Error uploading %s: %s\n", f.path, err)
			// if there are no errors on the channel, put this one there
			select {
			case errs <- err:
			default:
			}
		}
	}

	wg.Done()
}

func upload(source file, destBucket *s3.Bucket) error {
	f, err := os.Open(source.absPath)
	if err != nil {
		return err
	}
	defer f.Close()

	contentType := mime.TypeByExtension(filepath.Ext(source.path))
	if contentType == "" {
		contentType = "application/octet-stream"
	}

	return destBucket.PutReader(source.path, f, source.size, contentType, "public-read")
}
<|endoftext|>


<commit_before>package frame

// Interface Sequence is used to manipulate with frames, which can be used more
// than one time (e.g. APIC, COMM, USLT, SYLT and etc.)
type Sequencer interface {
	AddFrame(Framer)
	Frames() []Framer
}
<commit_msg>Make clearer comment in Sequencer interface<commit_after>package frame

// Interface Sequence is used to manipulate with frames, which can be in tag
// more than one (e.g. APIC, COMM, USLT, SYLT and etc.)
type Sequencer interface {
	AddFrame(Framer)
	Frames() []Framer
}
<|endoftext|>


<commit_before>package main

import "flag"
import "fmt"
import "html/template"
import "io/ioutil"
import "log"
import "net/http"
import "runtime"
import "strconv"
import "sync"
import "time"

import "github.com/hatstand/hodoor/dash"
import "github.com/hatstand/hodoor/doorbell"
import "github.com/hatstand/hodoor/model"
import "github.com/hatstand/hodoor/webpush"
import "github.com/stianeikeland/go-rpio"
import wp "github.com/SherClockHolmes/webpush-go"

var port = flag.Int("port", 8080, "Port to start HTTP server on")
var deviceIndex = flag.Int("device", 2, "Audio device to listen with")
var threshold = flag.Int("threshold", 3000, "Arbitrary threshold for doorbell activation")
var webpushKey = flag.String("key", "", "Private key for sending webpush requests")
var GPIOPin = flag.Int("pin", 18, "GPIO pin to toggle to open door")
var delaySeconds = flag.Int("delay", 5, "Time in seconds to hold door open")

type gpioHandler struct {
  lock sync.Mutex
  pin rpio.Pin
  db *model.Database
}

func GpioHandler(pin rpio.Pin) *gpioHandler {
  db, err := model.OpenDatabase("db")
  if err != nil {
    log.Fatal("Failed to open database: ", err)
  }
  return &gpioHandler{pin:pin, db:db}
}

func (f *gpioHandler) HandleButtonPress() {
  log.Printf("Dash button pressed!")
  f.openDoor()
}

func (f *gpioHandler) openDoor() {
  timer := time.NewTimer(*delaySeconds * time.Second)
  go func() {
    f.lock.Lock()
    defer f.lock.Unlock()
    log.Printf("Toggling door on pin %d for %d seconds", f.pin, *delaySeconds)
    f.pin.Output()
    f.pin.High()
    defer f.pin.Low()
    <-timer.C
  }()
}

func (f *gpioHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
  switch path := r.URL.Path; path {
    case "/":
      f.handleRoot(w, r)
    case "/hodoor":
      f.handleOpenDoor(w, r)
    case "/subscribe":
      f.handleSubscribe(w, r)
    case "/ping":
      f.handlePing(w, r)
    default:
      f.handleRoot(w, r)
  }
}

func (f *gpioHandler) handleRoot(w http.ResponseWriter, r *http.Request) {
  t, err := template.ParseFiles("templates/index.html")
  if err != nil {
    log.Fatal(err)
  }

  t.Execute(w, nil)
}

func (f *gpioHandler) handleOpenDoor(w http.ResponseWriter, r *http.Request) {
  t, err := template.ParseFiles("templates/hodoor.html")
  if err != nil {
    log.Fatal(err)
  }

  type TemplateOutput struct {
    Pin rpio.Pin
    Delay int
  }
  output := &TemplateOutput{f.pin, DelaySeconds}

  t.Execute(w, output)
  f.openDoor()
}

func (f *gpioHandler) handleSubscribe(w http.ResponseWriter, r *http.Request) {
  body, _ := ioutil.ReadAll(r.Body)
  sub, err := webpush.SubscriptionFromJSON(body)
  if err != nil {
    log.Printf("Failed to parse subscription: %v", err)
    http.Error(w, "Failed to parse subscription", 400)
    return
  }
  defer r.Body.Close()
  log.Printf("Subscribing user: %v", sub)
  f.db.Subscribe(sub)
}

func (f *gpioHandler) handlePing(w http.ResponseWriter, r *http.Request) {
  fmt.Fprintf(w, "Pinging subscribers")
  f.notifySubscribers("Ping!")
}

func (f *gpioHandler) notifySubscribers(message string) error {
  subs, err := f.db.GetSubscriptions()
  if err != nil {
    log.Printf("Failed to fetch subscribers: ", err)
    return err
  }
  for _, sub := range(subs) {
    go func(sub *wp.Subscription) {
      log.Printf("Sending webpush to endpoint: %v", sub.Endpoint)
      err := webpush.Send([]byte(message), sub, *webpushKey, 60)
      if err != nil {
        log.Printf("Failed to send webpush: %v", err)
      } else {
        log.Printf("Sent webpush successfully")
      }
    }(sub)
  }
  runtime.Gosched()
  return nil
}

func (f *gpioHandler) HandleDoorBell() {
  log.Println("Doorbell handled")
  f.notifySubscribers("DING DONG")
}

func main() {
  flag.Parse()
  runtime.GOMAXPROCS(6)

  err := rpio.Open()
  defer rpio.Close()

  if err != nil {
    log.Fatal(err)
  }

  handler := GpioHandler(rpio.Pin(*GPIOPin))

  go func() {
    err := dash.Listen(handler)
    if err != nil {
      log.Fatal(err)
    }
  }()

  go func() {
    err := doorbell.Listen(*deviceIndex, *threshold, handler)
    if err != nil {
      log.Fatal(err)
    }
  }()

  http.Handle("/hodoor", handler)
  http.Handle("/", handler)
  http.Handle("/subscribe", handler)
  http.Handle("/ping", handler)
  http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("static"))))
  log.Printf("Starting HTTP Server on port: %d", *port)
  go http.ListenAndServe(":" + strconv.Itoa(*port), nil)
  select{}
}
<commit_msg>Rudimentary Google Assistant integration<commit_after>package main

import "encoding/json"
import "flag"
import "html/template"
import "io/ioutil"
import "log"
import "net/http"
import "runtime"
import "strconv"
import "sync"
import "time"

import "github.com/hatstand/hodoor/dash"
import "github.com/hatstand/hodoor/doorbell"
import "github.com/hatstand/hodoor/model"
import "github.com/hatstand/hodoor/webpush"
import "github.com/stianeikeland/go-rpio"
import wp "github.com/SherClockHolmes/webpush-go"

var port = flag.Int("port", 8080, "Port to start HTTP server on")
var deviceIndex = flag.Int("device", 2, "Audio device to listen with")
var threshold = flag.Int("threshold", 3000, "Arbitrary threshold for doorbell activation")
var webpushKey = flag.String("key", "", "Private key for sending webpush requests")
var GPIOPin = flag.Int("pin", 18, "GPIO pin to toggle to open door")
var delaySeconds = flag.Duration("delay", 5 * time.Second, "Time in seconds to hold door open")

type AssistantResponse struct {
  Speech string `json:"speech"`
  DisplayText string `json:"displayText"`
}

type gpioHandler struct {
  lock sync.Mutex
  pin rpio.Pin
  db *model.Database
}

func GpioHandler(pin rpio.Pin) *gpioHandler {
  db, err := model.OpenDatabase("db")
  if err != nil {
    log.Fatal("Failed to open database: ", err)
  }
  return &gpioHandler{pin:pin, db:db}
}

func (f *gpioHandler) HandleButtonPress() {
  log.Printf("Dash button pressed!")
  f.openDoor()
}

func (f *gpioHandler) openDoor() {
  timer := time.NewTimer(*delaySeconds)
  go func() {
    f.lock.Lock()
    defer f.lock.Unlock()
    log.Printf("Toggling door on pin %d for %d seconds", f.pin, *delaySeconds)
    f.pin.Output()
    f.pin.High()
    defer f.pin.Low()
    <-timer.C
  }()
}

func (f *gpioHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
  switch path := r.URL.Path; path {
    case "/":
      f.handleRoot(w, r)
    case "/hodoor":
      f.handleOpenDoor(w, r)
    case "/subscribe":
      f.handleSubscribe(w, r)
    case "/ping":
      f.handlePing(w, r)
    default:
      f.handleRoot(w, r)
  }
}

func (f *gpioHandler) handleRoot(w http.ResponseWriter, r *http.Request) {
  t, err := template.ParseFiles("templates/index.html")
  if err != nil {
    log.Fatal(err)
  }

  t.Execute(w, nil)
}

func (f *gpioHandler) handleOpenDoor(w http.ResponseWriter, r *http.Request) {
  t, err := template.ParseFiles("templates/hodoor.html")
  if err != nil {
    log.Fatal(err)
  }

  type TemplateOutput struct {
    Pin rpio.Pin
    Delay int
  }
  output := &TemplateOutput{f.pin, int(delaySeconds.Seconds())}

  t.Execute(w, output)
  f.openDoor()
}

func (f *gpioHandler) handleSubscribe(w http.ResponseWriter, r *http.Request) {
  body, _ := ioutil.ReadAll(r.Body)
  sub, err := webpush.SubscriptionFromJSON(body)
  if err != nil {
    log.Printf("Failed to parse subscription: %v", err)
    http.Error(w, "Failed to parse subscription", 400)
    return
  }
  defer r.Body.Close()
  log.Printf("Subscribing user: %v", sub)
  f.db.Subscribe(sub)
}

func (f *gpioHandler) handlePing(w http.ResponseWriter, r *http.Request) {
  w.Header().Set("Content-Type", "application/json")

  resp := AssistantResponse{"Opening door", "Opening door"}
  j, err := json.Marshal(resp)
  if err != nil {
    http.Error(w, "Failed to serialise JSON", 500)
    return
  }

  err = f.notifySubscribers("Ping!")
  if err != nil {
    http.Error(w, "Failed to notify subscribers", 500)
    return
  }
  w.Write(j)
}

func (f *gpioHandler) notifySubscribers(message string) error {
  subs, err := f.db.GetSubscriptions()
  if err != nil {
    log.Printf("Failed to fetch subscribers: ", err)
    return err
  }
  for _, sub := range(subs) {
    go func(sub *wp.Subscription) {
      log.Printf("Sending webpush to endpoint: %v", sub.Endpoint)
      err := webpush.Send([]byte(message), sub, *webpushKey, 60)
      if err != nil {
        log.Printf("Failed to send webpush: %v", err)
      } else {
        log.Printf("Sent webpush successfully")
      }
    }(sub)
  }
  runtime.Gosched()
  return nil
}

func (f *gpioHandler) HandleDoorBell() {
  log.Println("Doorbell handled")
  f.notifySubscribers("DING DONG")
}

func main() {
  flag.Parse()
  runtime.GOMAXPROCS(6)

  err := rpio.Open()
  defer rpio.Close()

  if err != nil {
    log.Fatal(err)
  }

  handler := GpioHandler(rpio.Pin(*GPIOPin))

  go func() {
    err := dash.Listen(handler)
    if err != nil {
      log.Fatal(err)
    }
  }()

  go func() {
    err := doorbell.Listen(*deviceIndex, *threshold, handler)
    if err != nil {
      log.Fatal(err)
    }
  }()

  http.Handle("/hodoor", handler)
  http.Handle("/", handler)
  http.Handle("/subscribe", handler)
  http.Handle("/ping", handler)
  http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("static"))))
  log.Printf("Starting HTTP Server on port: %d", *port)
  go http.ListenAndServe(":" + strconv.Itoa(*port), nil)
  select{}
}
<|endoftext|>


<commit_before>// Copyright 2013 The Chihaya Authors. All rights reserved.
// Use of this source code is governed by the BSD 2-Clause license,
// which can be found in the LICENSE file.

package main

import (
	"flag"
	"log"
	"os"
	"os/signal"
	"runtime"
	"runtime/pprof"

	"github.com/pushrax/chihaya/config"
	"github.com/pushrax/chihaya/server"
)

var (
	profile    bool
	configPath string
)

func init() {
	flag.BoolVar(&profile, "profile", false, "Generate profiling data for pprof into chihaya.cpu")
	flag.StringVar(&configPath, "config", "", "The location of a valid configuration file.")
}

func main() {
	flag.Parse()
	runtime.GOMAXPROCS(runtime.NumCPU())

	if profile {
		log.Println("Running with profiling enabled")
		f, err := os.Create("chihaya.cpu")
		if err != nil {
			log.Fatalf("Failed to create profile file: %s\n", err)
		}
		defer f.Close()
		pprof.StartCPUProfile(f)
	}

	if configPath == "" {
		log.Fatalf("Must specify a configuration file")
	}
	conf, err := config.New(configPath)
	if err != nil {
		log.Fatalf("Failed to parse configuration file: %s\n", err)
	}
	s := server.New(conf)

	go func() {
		c := make(chan os.Signal, 1)
		signal.Notify(c, os.Interrupt)
		<-c

		if profile {
			pprof.StopCPUProfile()
		}

		log.Println("Caught interrupt, shutting down..")
		err := s.Stop()
		if err != nil {
			panic("Failed to shutdown cleanly")
		}
		log.Println("Shutdown successfully")
		<-c
		os.Exit(0)
	}()

	err = s.ListenAndServe()
	if err != nil {
		log.Fatalf("Failed to start server: %s\n", err)
	}
}
<commit_msg>accidentally removed this err check<commit_after>// Copyright 2013 The Chihaya Authors. All rights reserved.
// Use of this source code is governed by the BSD 2-Clause license,
// which can be found in the LICENSE file.

package main

import (
	"flag"
	"log"
	"os"
	"os/signal"
	"runtime"
	"runtime/pprof"

	"github.com/pushrax/chihaya/config"
	"github.com/pushrax/chihaya/server"
)

var (
	profile    bool
	configPath string
)

func init() {
	flag.BoolVar(&profile, "profile", false, "Generate profiling data for pprof into chihaya.cpu")
	flag.StringVar(&configPath, "config", "", "The location of a valid configuration file.")
}

func main() {
	flag.Parse()
	runtime.GOMAXPROCS(runtime.NumCPU())

	if profile {
		log.Println("Running with profiling enabled")
		f, err := os.Create("chihaya.cpu")
		if err != nil {
			log.Fatalf("Failed to create profile file: %s\n", err)
		}
		defer f.Close()
		pprof.StartCPUProfile(f)
	}

	if configPath == "" {
		log.Fatalf("Must specify a configuration file")
	}
	conf, err := config.New(configPath)
	if err != nil {
		log.Fatalf("Failed to parse configuration file: %s\n", err)
	}
	s, err := server.New(conf)
	if err != nil {
		log.Fatalf("Failed to create server: %s\n", err)
	}

	go func() {
		c := make(chan os.Signal, 1)
		signal.Notify(c, os.Interrupt)
		<-c

		if profile {
			pprof.StopCPUProfile()
		}

		log.Println("Caught interrupt, shutting down..")
		err := s.Stop()
		if err != nil {
			panic("Failed to shutdown cleanly")
		}
		log.Println("Shutdown successfully")
		<-c
		os.Exit(0)
	}()

	err = s.ListenAndServe()
	if err != nil {
		log.Fatalf("Failed to start server: %s\n", err)
	}
}
<|endoftext|>


<commit_before>package core

import (
	"path/filepath"

	"github.com/mitchellh/go-homedir"
)

var directory string = "~/.config/runcom"
var file string = "storage.json"

const Command = "runcom"
const Name = "Runcom"

func Directory() string {
	dir, err := homedir.Expand(directory)
	if err != nil {
		return ""
	}
	return dir
}

func PluginDirectory(plugin string) string {
	dir := Directory()
	return filepath.Join(dir, plugin)
}
<commit_msg>Cleaning.<commit_after>package core

import (
	"path/filepath"

	"github.com/mitchellh/go-homedir"
)

var directory = "~/.config/runcom"
var file = "storage.json"

const Command = "runcom"
const Name = "Runcom"

func Directory() string {
	dir, err := homedir.Expand(directory)
	if err != nil {
		return ""
	}
	return dir
}

func PluginDirectory(plugin string) string {
	dir := Directory()
	return filepath.Join(dir, plugin)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"io"
	"math/rand"
	"os"
	"time"

	"github.com/lucapette/fakedata/pkg/fakedata"
	flag "github.com/spf13/pflag"
)

var version = "main"

func generatorsHelp(generators fakedata.Generators) string {
	max := 0
	for _, gen := range generators {
		if len(gen.Name) > max {
			max = len(gen.Name)
		}
	}

	buffer := &bytes.Buffer{}
	pattern := fmt.Sprintf("%%-%ds%%s\n", max+2) //+2 makes the output more readable
	for _, gen := range generators {
		fmt.Fprintf(buffer, pattern, gen.Name, gen.Desc)
	}

	return buffer.String()
}

func isPipe() bool {
	stat, err := os.Stdin.Stat()
	if err != nil {
		fmt.Printf("error checking shell pipe: %v", err)
	}
	// Check if template data is piped to fakedata
	return (stat.Mode() & os.ModeCharDevice) == 0
}

func findTemplate(path string) string {
	if path != "" {
		tp, err := os.ReadFile(path)
		if err != nil {
			fmt.Printf("unable to read input: %s", err)
			os.Exit(1)
		}

		return string(tp)
	}

	if isPipe() {
		tp, err := io.ReadAll(os.Stdin)
		if err != nil {
			fmt.Printf("unable to read input: %s", err)
			os.Exit(1)
		}

		return string(tp)
	}

	return ""
}

func main() {
	var (
		generatorsFlag  = flag.BoolP("generators", "G", false, "lists available generators")
		generatorFlag   = flag.StringP("generator", "g", "", "show help for a specific generator")
		constraintsFlag = flag.BoolP("generators-with-constraints", "c", false, "lists available generators with constraints")
		limitFlag       = flag.IntP("limit", "l", 10, "limits rows up to n")
		streamFlag      = flag.BoolP("stream", "S", false, "streams rows till the end of time")
		formatFlag      = flag.StringP("format", "f", "column", "generates rows in f format. Available formats: column|sql")
		tableFlag       = flag.StringP("table", "t", "TABLE", "table name of the sql format")
		separatorFlag   = flag.StringP("separator", "s", " ", "specifies separator for the column format")
		templateFlag    = flag.StringP("template", "T", "", "Use template as input")
		completionFlag  = flag.StringP("completion", "C", "", "print shell completion function, pass shell name as argument (\"bash\", \"zsh\" or \"fish\")")
		versionFlag     = flag.BoolP("version", "v", false, "shows version information")
		helpFlag        = flag.BoolP("help", "h", false, "shows help")
	)

	flag.Usage = func() {
		fmt.Print("Usage: fakedata [option ...] generator...\n\n")
		flag.PrintDefaults()
	}
	flag.Parse()

	if *helpFlag {
		flag.Usage()
		os.Exit(0)
	}

	if *completionFlag != "" {
		completion, err := fakedata.GetCompletionFunc(*completionFlag)
		if err != nil {
			fmt.Println(err)
		}
		fmt.Printf("%s\n", completion)
		os.Exit(0)
	}

	if *versionFlag {
		fmt.Println(version)
		os.Exit(0)
	}

	generators := fakedata.NewGenerators()

	if *generatorsFlag {
		fmt.Print(generatorsHelp(generators))
		os.Exit(0)
	}

	if *generatorFlag != "" {
		if generator := generators.FindByName(*generatorFlag); generator != nil {
			fmt.Printf("Description: %s\n\nExample:\n\n", generator.Desc)
			for i := 0; i < 5; i++ {
				fn := generator.Func
				if generator.IsCustom() {
					custom, err := generator.CustomFunc("")
					if err != nil {
						fmt.Printf("could not generate example: %v", err)
						os.Exit(1)
					}

					fn = custom
				}
				fmt.Println(fn())
			}
		}
		os.Exit(0)
	}

	if *constraintsFlag {
		fmt.Print(generatorsHelp(generators.WithConstraints()))
		os.Exit(0)
	}

	rand.Seed(time.Now().UnixNano())

	if tmpl := findTemplate(*templateFlag); tmpl != "" {
		if err := fakedata.ExecuteTemplate(tmpl, *limitFlag, *streamFlag); err != nil {
			fmt.Println(err)
			os.Exit(1)
		}

		os.Exit(0)
	}

	if len(flag.Args()) == 0 {
		flag.Usage()
		os.Exit(0)
	}

	columns, err := fakedata.NewColumns(flag.Args())
	if err != nil {
		fmt.Printf("%v\n\n", err)
		flag.Usage()
		os.Exit(1)
	}

	var formatter fakedata.Formatter
	if *formatFlag == "column" {
		formatter = fakedata.NewColumnFormatter(*separatorFlag)
	} else if *formatFlag == "sql" {
		formatter = fakedata.NewSQLFormatter(*tableFlag)
	} else {
		fmt.Printf("unknown format: %s\n\n", *formatFlag)
		flag.Usage()
		os.Exit(1)
	}

	if *streamFlag {
		for {
			fmt.Println(columns.GenerateRow(formatter))
		}
	}
	for i := 0; i < *limitFlag; i++ {
		fmt.Println(columns.GenerateRow(formatter))
	}
}
<commit_msg>Better<commit_after>package main

import (
	"bytes"
	"fmt"
	"io"
	"math/rand"
	"os"
	"time"

	"github.com/lucapette/fakedata/pkg/fakedata"
	flag "github.com/spf13/pflag"
)

var version = "main"

func generatorsHelp(generators fakedata.Generators) string {
	max := 0
	for _, gen := range generators {
		if len(gen.Name) > max {
			max = len(gen.Name)
		}
	}

	buffer := &bytes.Buffer{}
	pattern := fmt.Sprintf("%%-%ds%%s\n", max+2) //+2 makes the output more readable
	for _, gen := range generators {
		fmt.Fprintf(buffer, pattern, gen.Name, gen.Desc)
	}

	return buffer.String()
}

func isPipe() bool {
	stat, err := os.Stdin.Stat()
	if err != nil {
		fmt.Printf("error checking shell pipe: %v", err)
	}
	// Check if template data is piped to fakedata
	return (stat.Mode() & os.ModeCharDevice) == 0
}

func findTemplate(path string) string {
	if path != "" {
		tp, err := os.ReadFile(path)
		if err != nil {
			fmt.Printf("unable to read input: %s", err)
			os.Exit(1)
		}

		return string(tp)
	}

	if isPipe() {
		tp, err := io.ReadAll(os.Stdin)
		if err != nil {
			fmt.Printf("unable to read input: %s", err)
			os.Exit(1)
		}

		return string(tp)
	}

	return ""
}

func main() {
	var (
		generatorsFlag  = flag.BoolP("generators", "G", false, "lists available generators")
		generatorFlag   = flag.StringP("generator", "g", "", "show help for a specific generator")
		constraintsFlag = flag.BoolP("generators-with-constraints", "c", false, "lists available generators with constraints")
		limitFlag       = flag.IntP("limit", "l", 10, "limits rows up to n")
		streamFlag      = flag.BoolP("stream", "S", false, "streams rows till the end of time")
		formatFlag      = flag.StringP("format", "f", "column", "generates rows in f format. Available formats: column|sql")
		tableFlag       = flag.StringP("table", "t", "TABLE", "table name of the sql format")
		separatorFlag   = flag.StringP("separator", "s", " ", "specifies separator for the column format")
		templateFlag    = flag.StringP("template", "T", "", "Use template as input")
		completionFlag  = flag.StringP("completion", "C", "", "print shell completion function, pass shell name as argument (\"bash\", \"zsh\" or \"fish\")")
		versionFlag     = flag.BoolP("version", "v", false, "shows version information")
		helpFlag        = flag.BoolP("help", "h", false, "shows help")
	)

	flag.Usage = func() {
		fmt.Print("Usage: fakedata [option ...] generator...\n\n")
		flag.PrintDefaults()
	}
	flag.Parse()

	if *helpFlag {
		flag.Usage()
		os.Exit(0)
	}

	if *completionFlag != "" {
		completion, err := fakedata.GetCompletionFunc(*completionFlag)
		if err != nil {
			fmt.Println(err)
		}
		fmt.Printf("%s\n", completion)
		os.Exit(0)
	}

	if *versionFlag {
		fmt.Println(version)
		os.Exit(0)
	}

	generators := fakedata.NewGenerators()

	if *generatorsFlag {
		fmt.Print(generatorsHelp(generators))
		os.Exit(0)
	}

	if *generatorFlag != "" {
		if generator := generators.FindByName(*generatorFlag); generator != nil {
			fmt.Printf("Description: %s\n\nExample:\n\n", generator.Desc)
			for i := 0; i < 5; i++ {
				fn := generator.Func
				if generator.IsCustom() {
					custom, err := generator.CustomFunc("")
					if err != nil {
						fmt.Printf("could not generate example: %v", err)
						os.Exit(1)
					}

					fn = custom
				}
				fmt.Println(fn())
			}
		}
		os.Exit(0)
	}

	if *constraintsFlag {
		fmt.Print(generatorsHelp(generators.WithConstraints()))
		os.Exit(0)
	}

	rand.Seed(time.Now().UnixNano())

	if tmpl := findTemplate(*templateFlag); tmpl != "" {
		if err := fakedata.ExecuteTemplate(tmpl, *limitFlag, *streamFlag); err != nil {
			fmt.Println(err)
			os.Exit(1)
		}

		return
	}

	if len(flag.Args()) == 0 {
		flag.Usage()
		os.Exit(0)
	}

	columns, err := fakedata.NewColumns(flag.Args())
	if err != nil {
		fmt.Printf("%v\n\n", err)
		flag.Usage()
		os.Exit(1)
	}

	var formatter fakedata.Formatter
	if *formatFlag == "column" {
		formatter = fakedata.NewColumnFormatter(*separatorFlag)
	} else if *formatFlag == "sql" {
		formatter = fakedata.NewSQLFormatter(*tableFlag)
	} else {
		fmt.Printf("unknown format: %s\n\n", *formatFlag)
		flag.Usage()
		os.Exit(1)
	}

	if *streamFlag {
		for {
			fmt.Println(columns.GenerateRow(formatter))
		}
	}
	for i := 0; i < *limitFlag; i++ {
		fmt.Println(columns.GenerateRow(formatter))
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/mattn/go-colorable"
	"github.com/tonnerre/golang-pretty"

	"gopkg.in/alecthomas/kingpin.v2"
	"gopkg.in/inconshreveable/log15.v2"

	"gopkg.in/rightscale/rsc.v4/log"
	"gopkg.in/rightscale/rsc.v4/rsapi"
)

var (
	app         = kingpin.New("right_st", "A command-line application for managing RightScripts")
	version     = app.Flag("version", "Print version").Short('v').Bool()
	debug       = app.Flag("debug", "Debug mode").Short('d').Bool()
	configFile  = app.Flag("config", "Set the config file path.").Short('c').Default(defaultConfigFile()).String()
	environment = app.Flag("environment", "Set the RightScale login environment.").Short('e').String()

	rightScript = app.Command("rightscript", "RightScript stuff")

	rightScriptList       = rightScript.Command("list", "List RightScripts")
	rightScriptListFilter = rightScriptList.Flag("filter", "Filter by name").Required().String()

	rightScriptUpload     = rightScript.Command("upload", "Upload a RightScript")
	rightScriptUploadFile = rightScriptUpload.Flag("file", "File or directory to upload").Short('f').String()

	rightScriptDownload     = rightScript.Command("download", "Download a RightScript to a file or files")
	rightScriptDownloadName = rightScriptDownload.Flag("name", "Script Name").Short('s').String()
	rightScriptDownloadId   = rightScriptDownload.Flag("id", "Script ID").Short('i').Int()

	rightScriptMetadata     = rightScript.Command("metadata", "Add RightScript YAML metadata comments to a file or files")
	rightScriptMetadataFile = rightScriptMetadata.Flag("file", "File or directory to set metadata for").Short('f').String()

	rightScriptValidate      = rightScript.Command("validate", "Validate RightScript YAML metadata comments in a file or files")
	rightScriptValidatePaths = rightScriptValidate.Arg("path", "Path to script file or directory containing script files").Required().ExistingFilesOrDirs()
)

func main() {
	command := kingpin.MustParse(app.Parse(os.Args[1:]))

	err := readConfig(*configFile, *environment)
	client := config.environment.Client15()
	if err != nil {
		fmt.Fprintf(os.Stderr, "%s: Error reading config file: %s\n", filepath.Base(os.Args[0]), err)
		os.Exit(1)
	}

	// Handle logginng
	handler := log15.StreamHandler(colorable.NewColorableStdout(), log15.TerminalFormat())
	log15.Root().SetHandler(handler)
	log.Logger.SetHandler(handler)
	app.Writer(os.Stdout)

	switch command {
	case rightScriptList.FullCommand():
		rightscriptLocator := client.RightScriptLocator("/api/right_scripts")
		var apiParams = rsapi.APIParams{"filter": []string{"name==" + *rightScriptListFilter}}
		fmt.Printf("LIST %s", *rightScriptListFilter)
		rightscripts, err := rightscriptLocator.Index(
			apiParams,
		)
		if err != nil {
			fatalError("%#v", err)
		}
		for _, rs := range rightscripts {
			fmt.Printf("%s\n", rs.Name)
		}
	case rightScriptUpload.FullCommand():
		fmt.Println(*rightScriptUpload)
	case rightScriptDownload.FullCommand():
		fmt.Println(*rightScriptDownload)
	case rightScriptMetadata.FullCommand():
		fmt.Println(*rightScriptMetadata)
	case rightScriptValidate.FullCommand():
		for _, path := range *rightScriptValidatePaths {
			info, err := os.Stat(path)
			if err != nil {
				fmt.Fprintf(os.Stderr, "%s: %s\n", filepath.Base(os.Args[0]), err)
				os.Exit(1)
			}
			if info.IsDir() {
				// TODO: recurse?
			} else {
				validateRightScript(path)
			}
		}
	}

	fmt.Println("Start - options:", *configFile, config, *rightScript)

	fmt.Println("Done -- authenticated")
}

func fatalError(format string, v ...interface{}) {
	msg := fmt.Sprintf(format, v)
	fmt.Println(msg)
	os.Exit(1)
}

func validateRightScript(path string) {
	script, err := os.Open(path)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%s: %s\n", filepath.Base(os.Args[0]), err)
		os.Exit(1)
	}
	defer script.Close()
	pretty.Println(ParseRightScriptMetadata(script))
}
<commit_msg>MD5 sum support<commit_after>package main

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"io"
	"os"
	"path/filepath"

	"github.com/mattn/go-colorable"
	"github.com/tonnerre/golang-pretty"

	"gopkg.in/alecthomas/kingpin.v2"
	"gopkg.in/inconshreveable/log15.v2"

	"gopkg.in/rightscale/rsc.v4/log"
	"gopkg.in/rightscale/rsc.v4/rsapi"
)

var (
	app         = kingpin.New("right_st", "A command-line application for managing RightScripts")
	version     = app.Flag("version", "Print version").Short('v').Bool()
	debug       = app.Flag("debug", "Debug mode").Short('d').Bool()
	configFile  = app.Flag("config", "Set the config file path.").Short('c').Default(defaultConfigFile()).String()
	environment = app.Flag("environment", "Set the RightScale login environment.").Short('e').String()

	rightScript = app.Command("rightscript", "RightScript stuff")

	rightScriptList       = rightScript.Command("list", "List RightScripts")
	rightScriptListFilter = rightScriptList.Flag("filter", "Filter by name").Required().String()

	rightScriptUpload     = rightScript.Command("upload", "Upload a RightScript")
	rightScriptUploadFile = rightScriptUpload.Flag("file", "File or directory to upload").Short('f').String()

	rightScriptDownload     = rightScript.Command("download", "Download a RightScript to a file or files")
	rightScriptDownloadName = rightScriptDownload.Flag("name", "Script Name").Short('s').String()
	rightScriptDownloadId   = rightScriptDownload.Flag("id", "Script ID").Short('i').Int()

	rightScriptMetadata     = rightScript.Command("metadata", "Add RightScript YAML metadata comments to a file or files")
	rightScriptMetadataFile = rightScriptMetadata.Flag("file", "File or directory to set metadata for").Short('f').String()

	rightScriptValidate      = rightScript.Command("validate", "Validate RightScript YAML metadata comments in a file or files")
	rightScriptValidatePaths = rightScriptValidate.Arg("path", "Path to script file or directory containing script files").Required().ExistingFilesOrDirs()
)

func main() {
	command := kingpin.MustParse(app.Parse(os.Args[1:]))

	err := readConfig(*configFile, *environment)
	client := config.environment.Client15()
	if err != nil {
		fmt.Fprintf(os.Stderr, "%s: Error reading config file: %s\n", filepath.Base(os.Args[0]), err)
		os.Exit(1)
	}

	// Handle logginng
	handler := log15.StreamHandler(colorable.NewColorableStdout(), log15.TerminalFormat())
	log15.Root().SetHandler(handler)
	log.Logger.SetHandler(handler)
	app.Writer(os.Stdout)

	switch command {
	case rightScriptList.FullCommand():
		rightscriptLocator := client.RightScriptLocator("/api/right_scripts")
		var apiParams = rsapi.APIParams{"filter": []string{"name==" + *rightScriptListFilter}}
		fmt.Printf("LIST %s", *rightScriptListFilter)
		rightscripts, err := rightscriptLocator.Index(
			apiParams,
		)
		if err != nil {
			fatalError("%#v", err)
		}
		for _, rs := range rightscripts {
			fmt.Printf("%s\n", rs.Name)
		}
	case rightScriptUpload.FullCommand():
		fmt.Println(*rightScriptUpload)
	case rightScriptDownload.FullCommand():
		fmt.Println(*rightScriptDownload)
	case rightScriptMetadata.FullCommand():
		fmt.Println(*rightScriptMetadata)
	case rightScriptValidate.FullCommand():
		for _, path := range *rightScriptValidatePaths {
			info, err := os.Stat(path)
			if err != nil {
				fmt.Fprintf(os.Stderr, "%s: %s\n", filepath.Base(os.Args[0]), err)
				os.Exit(1)
			}
			if info.IsDir() {
				// TODO: recurse?
			} else {
				err = validateRightScript(path)
				if err != nil {
					fmt.Fprintf(os.Stderr, "%s: %s\n", filepath.Base(os.Args[0]), err)
					os.Exit(1)
				}
			}
		}
	}

	fmt.Println("Start - options:", *configFile, config, *rightScript)

	fmt.Println("Done -- authenticated")
}

func fatalError(format string, v ...interface{}) {
	msg := fmt.Sprintf(format, v)
	fmt.Println(msg)
	os.Exit(1)
}

func validateRightScript(path string) error {
	script, err := os.Open(path)
	if err != nil {
		return err
	}
	defer script.Close()

	metadata, err := ParseRightScriptMetadata(script)
	if err != nil {
		return err
	}
	pretty.Println(metadata)

	for _, attachment := range metadata.Attachments {
		md5, err := md5Attachment(path, attachment)
		if err != nil {
			return err
		}
		fmt.Println(attachment, md5)
	}

	return nil
}

func md5Attachment(script, attachment string) (string, error) {
	path := filepath.Join(filepath.Dir(script), attachment)
	file, err := os.Open(path)
	if err != nil {
		return "", err
	}
	defer file.Close()
	hash := md5.New()

	_, err = io.Copy(hash, file)
	if err != nil {
		return "", err
	}

	return hex.EncodeToString(hash.Sum(nil)), nil
}
<|endoftext|>


<commit_before>// fbgdl is a Facebook Graph downloader. It cycles through as many users
// as it is told (or MaxUint64) and stores them in a database.
package main

import (
	"database/sql"
	"encoding/json"
	"flag"
	"fmt"
	_ "github.com/mattn/go-sqlite3"
	"io/ioutil"
	"log"
	"math"
	"net/http"
	"strconv"
	"time"
)

const dbFile = "fbgraph.db"
const graphBase = "https://graph.facebook.com"

// userUrl takes a user ID and returns the Facebook graph URL for that user.
func userUrl(uid uint64) string {
	return fmt.Sprintf("%s/%d", graphBase, uid)
}

// Type GraphUser represents an entry from the Graph. It is not suitable
// for storing, but contains the data to be converted to a User type
// that can be stored in the database.
type GraphUser struct {
	Id       string `json:"id"`
	Name     string `json:"name"`
	First    string `json:"first_name"`
	Last     string `json:"last_name"`
	Link     string `json:"link"`
	Username string `json:"username"`
	Gender   string `json:"gender"`
	Locale   string `json:"locale"`
	Error    struct {
		Message string `json:"message"`
		Type    string `json:"type"`
		Code    int    `json:"code"`
	} `json:"error"`
}

// Failed returns true if the UID was an invalid Graph user.
func (gu *GraphUser) Failed() bool {
	if gu.Error.Message != "" {
		return true
	}
	return false
}

// ToUser converts a GraphUser to a User.
func (gu *GraphUser) ToUser() (u *User, err error) {
	if gu.Failed() {
		err = fmt.Errorf(gu.Error.Message)
		return
	}
	u = new(User)

	n, err := strconv.ParseUint(gu.Id, 10, 64)
	if err != nil {
		return
	}

	nString := fmt.Sprintf("%d", n)
	if nString != gu.Id {
		err = fmt.Errorf("invalid id conversion")
		return
	}

	u.Id = n
	u.Name = gu.Name
	u.First = gu.First
	u.Last = gu.Last
	u.Link = gu.Link
	u.Username = gu.Username
	u.Gender = gu.Gender
	u.Locale = gu.Locale
	return
}

// Type User is a representation of a graph user suitable for storing
// in the database.
type User struct {
	Id       uint64
	Name     string
	First    string
	Last     string
	Link     string
	Username string
	Gender   string
	Locale   string
}

// Method Store is used to save a user to the database.
func (u *User) Store() (err error) {
	db, err := sql.Open("sqlite3", dbFile)
	if err != nil {
		return
	}
	defer db.Close()

	_, err = db.Exec(`insert into users values (?, ?, ?, ?, ?, ?, ?, ?)`,
		u.Id, u.Name, u.First, u.Last, u.Link, u.Username, u.Gender,
		u.Locale)
	return
}

// checkDatabase looks for the database file, and makes sure it has the
// appropriate table.
func checkDatabase() {
	db, err := sql.Open("sqlite3", dbFile)
	if err != nil {
		return
	}
	defer db.Close()

	var missingTable = fmt.Errorf("no such table: users")

	_, err = db.Exec("select count(*) from users")
	if err != nil && err.Error() == missingTable.Error() {
		fmt.Println("creating table")
		err = createDB()
	}
	if err != nil {
		panic("[!] fbgdl: opening profile database: " +
			err.Error())
	}
}

// createDB is responsible for creating the database.
func createDB() (err error) {
	db, err := sql.Open("sqlite3", dbFile)
	if err != nil {
		return
	}
	defer db.Close()

	_, err = db.Exec(`create table users
                          (id integer primary key unique not null,
                           name text,
                           first text,
                           last text,
                           link text,
                           username text,
                           gender text,
                           locale text)`)
	return
}

func getLastUser() (count uint64, err error) {
	db, err := sql.Open("sqlite3", dbFile)
	if err != nil {
		return
	}
	defer db.Close()

	row := db.QueryRow("select count(*) from users")
	err = row.Scan(&count)
	if err != nil {
		return
	}
	if count == 0 {
		return
	}

	row = db.QueryRow("select max(id) from users")
	err = row.Scan(&count)
        if err == nil {
                count++
        }
	return
}

// fetchUser grabs a user from the Graph, storing the user in the database
// if it is a valid user. Otherwise, an error is returned.
func fetchUser(uid uint64) (u *User, err error) {
	resp, err := http.Get(userUrl(uid))
	if err != nil {
		return
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)

	gu := new(GraphUser)
	err = json.Unmarshal(body, &gu)
	if err != nil {
		return
	}

	u, err = gu.ToUser()
	if err != nil {
		return
	}

	err = u.Store()
	return
}

// Download the graph!
func main() {
	checkDatabase()

	start, err := getLastUser()
	if err != nil {
		log.Fatal(err.Error())
	}

	fMaxUid := flag.Uint64("u", math.MaxUint64, "max uid to grab")
	flag.Parse()

	if *fMaxUid < start {
		log.Fatal("max uid is less than starting uid")
	} else {
		log.Printf("grabbing uids from %d to %d\n", start, *fMaxUid)
	}

	var ErrLimit = fmt.Errorf("(#4) Application request limit reached")
	var total uint64
	for uid := start; uid < *fMaxUid; uid++ {
		u, err := fetchUser(uid)
		if err != nil {
			logMsg := fmt.Sprintf("failed uid %d: %s", uid,
				err.Error())
			log.Println(logMsg)
			if err.Error() == ErrLimit.Error() {
				uid--
				<-time.After(1 * time.Hour)
				continue
			}
		} else {
			total++
			logMsg := fmt.Sprintf("stored uid %d (%s)", uid,
				u.Username)
			log.Println(logMsg)
			if total > 0 && total%1000 == 0 {
				log.Printf("%d users stored\n", total)
			}
		}
	}
}
<commit_msg>go fmt<commit_after>// fbgdl is a Facebook Graph downloader. It cycles through as many users
// as it is told (or MaxUint64) and stores them in a database.
package main

import (
	"database/sql"
	"encoding/json"
	"flag"
	"fmt"
	_ "github.com/mattn/go-sqlite3"
	"io/ioutil"
	"log"
	"math"
	"net/http"
	"strconv"
	"time"
)

const dbFile = "fbgraph.db"
const graphBase = "https://graph.facebook.com"

// userUrl takes a user ID and returns the Facebook graph URL for that user.
func userUrl(uid uint64) string {
	return fmt.Sprintf("%s/%d", graphBase, uid)
}

// Type GraphUser represents an entry from the Graph. It is not suitable
// for storing, but contains the data to be converted to a User type
// that can be stored in the database.
type GraphUser struct {
	Id       string `json:"id"`
	Name     string `json:"name"`
	First    string `json:"first_name"`
	Last     string `json:"last_name"`
	Link     string `json:"link"`
	Username string `json:"username"`
	Gender   string `json:"gender"`
	Locale   string `json:"locale"`
	Error    struct {
		Message string `json:"message"`
		Type    string `json:"type"`
		Code    int    `json:"code"`
	} `json:"error"`
}

// Failed returns true if the UID was an invalid Graph user.
func (gu *GraphUser) Failed() bool {
	if gu.Error.Message != "" {
		return true
	}
	return false
}

// ToUser converts a GraphUser to a User.
func (gu *GraphUser) ToUser() (u *User, err error) {
	if gu.Failed() {
		err = fmt.Errorf(gu.Error.Message)
		return
	}
	u = new(User)

	n, err := strconv.ParseUint(gu.Id, 10, 64)
	if err != nil {
		return
	}

	nString := fmt.Sprintf("%d", n)
	if nString != gu.Id {
		err = fmt.Errorf("invalid id conversion")
		return
	}

	u.Id = n
	u.Name = gu.Name
	u.First = gu.First
	u.Last = gu.Last
	u.Link = gu.Link
	u.Username = gu.Username
	u.Gender = gu.Gender
	u.Locale = gu.Locale
	return
}

// Type User is a representation of a graph user suitable for storing
// in the database.
type User struct {
	Id       uint64
	Name     string
	First    string
	Last     string
	Link     string
	Username string
	Gender   string
	Locale   string
}

// Method Store is used to save a user to the database.
func (u *User) Store() (err error) {
	db, err := sql.Open("sqlite3", dbFile)
	if err != nil {
		return
	}
	defer db.Close()

	_, err = db.Exec(`insert into users values (?, ?, ?, ?, ?, ?, ?, ?)`,
		u.Id, u.Name, u.First, u.Last, u.Link, u.Username, u.Gender,
		u.Locale)
	return
}

// checkDatabase looks for the database file, and makes sure it has the
// appropriate table.
func checkDatabase() {
	db, err := sql.Open("sqlite3", dbFile)
	if err != nil {
		return
	}
	defer db.Close()

	var missingTable = fmt.Errorf("no such table: users")

	_, err = db.Exec("select count(*) from users")
	if err != nil && err.Error() == missingTable.Error() {
		fmt.Println("creating table")
		err = createDB()
	}
	if err != nil {
		panic("[!] fbgdl: opening profile database: " +
			err.Error())
	}
}

// createDB is responsible for creating the database.
func createDB() (err error) {
	db, err := sql.Open("sqlite3", dbFile)
	if err != nil {
		return
	}
	defer db.Close()

	_, err = db.Exec(`create table users
                          (id integer primary key unique not null,
                           name text,
                           first text,
                           last text,
                           link text,
                           username text,
                           gender text,
                           locale text)`)
	return
}

func getLastUser() (count uint64, err error) {
	db, err := sql.Open("sqlite3", dbFile)
	if err != nil {
		return
	}
	defer db.Close()

	row := db.QueryRow("select count(*) from users")
	err = row.Scan(&count)
	if err != nil {
		return
	}
	if count == 0 {
		return
	}

	row = db.QueryRow("select max(id) from users")
	err = row.Scan(&count)
	if err == nil {
		count++
	}
	return
}

// fetchUser grabs a user from the Graph, storing the user in the database
// if it is a valid user. Otherwise, an error is returned.
func fetchUser(uid uint64) (u *User, err error) {
	resp, err := http.Get(userUrl(uid))
	if err != nil {
		return
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)

	gu := new(GraphUser)
	err = json.Unmarshal(body, &gu)
	if err != nil {
		return
	}

	u, err = gu.ToUser()
	if err != nil {
		return
	}

	err = u.Store()
	return
}

// Download the graph!
func main() {
	checkDatabase()

	start, err := getLastUser()
	if err != nil {
		log.Fatal(err.Error())
	}

	fMaxUid := flag.Uint64("u", math.MaxUint64, "max uid to grab")
	flag.Parse()

	if *fMaxUid < start {
		log.Fatal("max uid is less than starting uid")
	} else {
		log.Printf("grabbing uids from %d to %d\n", start, *fMaxUid)
	}

	var ErrLimit = fmt.Errorf("(#4) Application request limit reached")
	var total uint64
	for uid := start; uid < *fMaxUid; uid++ {
		u, err := fetchUser(uid)
		if err != nil {
			logMsg := fmt.Sprintf("failed uid %d: %s", uid,
				err.Error())
			log.Println(logMsg)
			if err.Error() == ErrLimit.Error() {
				uid--
				<-time.After(1 * time.Hour)
				continue
			}
		} else {
			total++
			logMsg := fmt.Sprintf("stored uid %d (%s)", uid,
				u.Username)
			log.Println(logMsg)
			if total > 0 && total%1000 == 0 {
				log.Printf("%d users stored\n", total)
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/md5"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
)

func main() {
	if len(os.Args) == 1 {
		fmt.Println("Syntax: dmlivewiki_checksum <path> [-y]")
		return
	}

	filepath := os.Args[1]

	// Ignore error, it returns false
	// even if it doesn't exist
	isDirectory, _ := isDirectory(filepath)
	if !isDirectory {
		fmt.Println("Error: target is not a directory")
		return
	}

	if !shouldContinue(filepath) {
		return
	}

	files, _ := ioutil.ReadDir(filepath)
	for _, file := range files {
		if file.IsDir() {
			ffp := createFile(filepath, file.Name(), "ffp")
			processDirectory(path.Join(filepath, file.Name()), 1, ffp, "ffp")
			ffp.Close()

			md5 := createFile(filepath, file.Name(), "md5")
			processDirectory(path.Join(filepath, file.Name()), 1, md5, "md5")
			md5.Close()
		}
	}
}

func shouldContinue(filepath string) bool {
	// Ask to continue or just process?
	// Hacky!
	if len(os.Args) > 2 {
		if os.Args[2] == "-y" {
			return true
		}
	}

	fmt.Println("The following filepath will be processed: ", filepath)
	fmt.Print("Continue? (y/n): ")
	text := ""
	fmt.Scanln(&text)
	if text != "y" {
		return false
	}
	return true
}

func processDirectory(filepath string, depth int, out *os.File, mode string) {
	files, _ := ioutil.ReadDir(filepath)
	if len(files) == 0 {
		if mode == "ffp" {
			fmt.Println("Empty folder found:", filepath)
		}
		return
	}

	var parser func(string, string, int) string
	if mode == "ffp" {
		parser = ffpParse
	} else if mode == "md5" {
		parser = md5Parse
	}

	for _, file := range files {
		name := file.Name()

		if file.IsDir() {
			processDirectory(path.Join(filepath, name), depth+1, out, mode)
		} else if (path.Ext(name) != ".md5") && !file.IsDir() {
			if result := parser(filepath, name, depth); result != "" {
				out.WriteString(result)
			}
		}
	}
}

func md5Parse(filepath string, name string, depth int) string {
	data, err := ioutil.ReadFile(path.Join(filepath, name))

	if err != nil {
		panic(err)
	}

	return fmt.Sprintf("%x *%s%s\n", md5.Sum(data), getLastPathComponents(filepath, depth), name)
}

func ffpParse(filepath string, name string, depth int) string {
	if path.Ext(name) != ".flac" {
		return ""
	}

	data, err := exec.Command(
		"metaflac",
		"--show-md5sum",
		path.Join(filepath, name),
	).Output()

	if err != nil {
		panic(err)
	}

	return fmt.Sprintf("%s%s:%s", getLastPathComponents(filepath, depth), name, data)
}
<commit_msg>Move to codegansta/cli<commit_after>package main

import (
	"crypto/md5"
	"fmt"
	"github.com/codegangsta/cli"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
)

func main() {
	app := cli.NewApp()
	app.Name = "dmlivewiki_checksum"
	app.Usage = "" // todo

	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "yes, y",
			Usage: "skip the confirmation input",
		},
	}

	app.Action = mainAction
	app.Version = "1.0.2"

	app.Run(os.Args)
}

func mainAction(c *cli.Context) {
	if len(c.Args()) != 1 {
		fmt.Println("Syntax: dmlivewiki_checksum [options] <path>")
		return
	}

	filepath := c.Args()[0]
	print(filepath)

	// Ignore error, it returns false
	// even if it doesn't exist
	isDirectory, _ := isDirectory(filepath)
	if !isDirectory {
		fmt.Println("Error: target is not a directory")
		return
	}

	if !shouldContinue(c, filepath) {
		return
	}

	files, _ := ioutil.ReadDir(filepath)
	for _, file := range files {
		if file.IsDir() {
			ffp := createFile(filepath, file.Name(), "ffp")
			processDirectory(path.Join(filepath, file.Name()), 1, ffp, "ffp")
			ffp.Close()

			md5 := createFile(filepath, file.Name(), "md5")
			processDirectory(path.Join(filepath, file.Name()), 1, md5, "md5")
			md5.Close()
		}
	}
}

func shouldContinue(c *cli.Context, filepath string) bool {
	// Ask to continue or just process?
	// Hacky!
	if c.Bool("yes") {
		return true
	}

	fmt.Println("The following filepath will be processed: ", filepath)
	fmt.Print("Continue? (y/n): ")
	text := ""
	fmt.Scanln(&text)
	if text != "y" {
		return false
	}
	return true
}

func processDirectory(filepath string, depth int, out *os.File, mode string) {
	files, _ := ioutil.ReadDir(filepath)
	if len(files) == 0 {
		if mode == "ffp" {
			fmt.Println("Empty folder found:", filepath)
		}
		return
	}

	var parser func(string, string, int) string
	if mode == "ffp" {
		parser = ffpParse
	} else if mode == "md5" {
		parser = md5Parse
	}

	for _, file := range files {
		name := file.Name()

		if file.IsDir() {
			processDirectory(path.Join(filepath, name), depth+1, out, mode)
		} else if (path.Ext(name) != ".md5") && !file.IsDir() {
			if result := parser(filepath, name, depth); result != "" {
				out.WriteString(result)
			}
		}
	}
}

func md5Parse(filepath string, name string, depth int) string {
	data, err := ioutil.ReadFile(path.Join(filepath, name))

	if err != nil {
		panic(err)
	}

	return fmt.Sprintf("%x *%s%s\n", md5.Sum(data), getLastPathComponents(filepath, depth), name)
}

func ffpParse(filepath string, name string, depth int) string {
	if path.Ext(name) != ".flac" {
		return ""
	}

	data, err := exec.Command(
		"metaflac",
		"--show-md5sum",
		path.Join(filepath, name),
	).Output()

	if err != nil {
		panic(err)
	}

	return fmt.Sprintf("%s%s:%s", getLastPathComponents(filepath, depth), name, data)
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/mitchellh/packer/packer/plugin"
	"github.com/mitchellh/packer/packer"
	"github.com/mitchellh/packer/common"
	"github.com/mitchellh/packer/builder/docker"
	"github.com/mitchellh/packer/post-processor/docker-import"
	"encoding/json"
	"log"
	"fmt"
	"os"
	"os/exec"
	"bytes"
	"bufio"
	"text/template"
	"strings"
	"regexp"
	"errors"
)
const BuilderId = "packer.post-processor.docker-dockerfile"

func main() {
	server, err := plugin.Server()
	if err != nil {
		log.Printf("[ERR] %s", err)
		os.Exit(1)
	}
	server.RegisterPostProcessor(new(PostProcessor))
	server.Serve()
}

type PostProcessor struct {
	c Config
	t *template.Template
	docker_build_fn func(*bytes.Buffer) (string, error) // to facilitate easy testing
	tpl *packer.ConfigTemplate
}

type Config struct {
	common.PackerConfig     `mapstructure:",squash"`

	Expose []string         `mapstructure:"expose"`
	User string             `mapstructure:"user"`
	Env map[string]string   `mapstructure:"env"`
	Volume []string         `mapstructure:"volume"`
	WorkDir string          `mapstructure:"workdir"`
	Entrypoint interface{}  `mapstructure:"entrypoint"`
	Cmd interface{}         `mapstructure:"cmd"`
	ImageId string
}

func (p *PostProcessor) Configure(raw_config ...interface{}) error {
	var err error
	_, err = common.DecodeConfig(&p.c, raw_config...)
	if err != nil {
		return err
	}
	p.docker_build_fn = docker_build // configure the build function
	if err = p.prepare_config_template(); err != nil { return err }

	return nil
}

func (p *PostProcessor) prepare_config_template() error {
	tpl, err := packer.NewConfigTemplate()
	if err != nil { return err }

	tpl.UserVars = p.c.PackerUserVars
	p.tpl = tpl

	return nil
}

func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) {
	if artifact.BuilderId() != dockerimport.BuilderId {
		err := fmt.Errorf(
			"Unknown artifact type: %s\nCan only tag from Docker builder artifacts.",
			artifact.BuilderId())
		return nil, false, err
	}

	dockerfile, template_err := p.render_template(artifact.Id())
	if template_err != nil { // could not render template
		return nil, false, template_err
	}
	log.Printf("[DEBUG] Dockerfile: %s\n", dockerfile.String())

	if image_id, err := p.docker_build_fn(dockerfile); err != nil { // docker build command failed
		return nil, false, err
	} else {
		ui.Message("Built image: " + image_id)
		new_artifact := &docker.ImportArtifact{
			BuilderIdValue: dockerimport.BuilderId,
			Driver: &docker.DockerDriver{Ui: ui, Tpl: nil},
			IdValue: image_id,
		}
		log.Printf("[DEBUG] artifact: %#v\n", new_artifact)
		return new_artifact, true, nil
	}
}

// Render a variable template using packer.ConfigTemplate primed with user variables
// You must call p.prepare_config_template() before using this function
func (p *PostProcessor) render(var_tmpl string) string {
	rendered, err := p.tpl.Process(var_tmpl, nil)
	if err != nil {
		panic(err)
	}
	return rendered
}

// Process a variable of unknown type. This function will call render() to render any packer user variables
// This function will panic if it can't handle the variable.
func (p *PostProcessor) process_var(variable interface{}) string {
	errs := new(packer.MultiError)

	render_string_or_slice := func(field interface{}) interface{} {
		switch t := field.(type) {
		case []string:
			ary := make([]string, 0, len(t))
			for _, item := range t {
				ary = append(ary, p.render(item))
			}
			return ary
		case []interface{}:
			ary := make([]string, 0, len(t))
			for _, item := range t {
				ary = append(ary, p.render(item.(string)))
			}
			return ary
		case string: return p.render(t)
		case nil: return nil
		default:
			errs = packer.MultiErrorAppend(errs, fmt.Errorf("Error processing %s: not a string or a string array", field))
			return nil
		}
	}

	switch t := variable.(type) {
	case []string: return json_dump_slice(render_string_or_slice(t))
	case []interface{}: return json_dump_slice(render_string_or_slice(t))
	case string: return p.render(variable.(string))
	case nil: return ""
	default: panic(errors.New("not sure how to handle type"))
	}
	if len(errs.Errors) > 0 {
		panic(errs)
	}
	return ""
}

func json_dump_slice(data interface{}) string {
	if res, err := json.Marshal(data); err != nil {
		panic(err)
	} else {
		return string(res)
	}
}

func (p *PostProcessor)render_template(id string) (buf *bytes.Buffer, _err error) {
	template_str := `FROM {{ .ImageId }}
{{ if .Volume }}VOLUME {{ stringify .Volume }}
{{ end }}{{ if .Expose }}EXPOSE {{ join .Expose " " }}
{{ end }}{{ if .WorkDir }}WORKDIR {{ .WorkDir }}
{{ end }}{{ if .User }}USER {{ .User }}
{{ end }}{{ if .Env }}{{ range $k, $v := .Env }}ENV {{ $k }} {{ render $v }}
{{ end }}{{ end }}{{ if .Entrypoint }}ENTRYPOINT {{ stringify .Entrypoint }}
{{ end }}{{ if .Cmd }}CMD {{ stringify .Cmd }}{{ end }}`
	template_buffer := new(bytes.Buffer)
	template_writer := bufio.NewWriter(template_buffer)

	p.c.ImageId = id

	defer func() {
		if err := recover(); err != nil {
			switch t_err := err.(type) {
			case error: _err = t_err // caught panic, return error to caller
			case string: _err = errors.New(t_err)
			default:
			}
		}
	}()

	t, err := template.New("dockerfile").Funcs(template.FuncMap{
		"stringify": p.process_var,
		"join": strings.Join,
		"render": func(s string) string {
			return p.render(s)
		},
	}).Parse(template_str)
	if err != nil {
		return nil, err
	}

	if err := t.Execute(template_writer, p.c); err != nil {
		return nil, err
	}
	template_writer.Flush()

	return template_buffer, nil
}

func docker_build(stdin *bytes.Buffer) (string, error) {
	var stderr bytes.Buffer
	var stdout bytes.Buffer
	cmd := exec.Command("docker", "build", "--force-rm=true", "--no-cache=true", "-q", "-")
	cmd.Stdin = stdin
	cmd.Stderr = &stderr
	cmd.Stdout = &stdout

	if err := cmd.Start(); err != nil {
		return "", err
	}
	if err := cmd.Wait(); err != nil {
		log.Printf("[ERR] error while running docker build. error: %s, command output: %s", err, stderr.String())
		return "", err
	}
	log.Println("Docker build command output:\n" + stdout.String())
	lines := strings.Split(stdout.String(), "\n")
	last_line := lines[len(lines) - 2] // we seem to have a trailing empty line
	image_id_regexp := regexp.MustCompile("Successfully built ([a-f0-9]+)")
	if matches := image_id_regexp.FindStringSubmatch(last_line); len(matches) > 0 {
		image_id := matches[len(matches) - 1]
		return image_id, nil
	} else {
		return "", errors.New("Could not parse `docker build` output")
	}
}
<commit_msg>Added more logging<commit_after>package main

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/mitchellh/packer/builder/docker"
	"github.com/mitchellh/packer/common"
	"github.com/mitchellh/packer/packer"
	"github.com/mitchellh/packer/packer/plugin"
	"github.com/mitchellh/packer/post-processor/docker-import"
	"log"
	"os"
	"os/exec"
	"regexp"
	"strings"
	"text/template"
)

const BuilderId = "packer.post-processor.docker-dockerfile"

func main() {
	server, err := plugin.Server()
	if err != nil {
		log.Printf("[ERR] %s", err)
		os.Exit(1)
	}
	server.RegisterPostProcessor(new(PostProcessor))
	server.Serve()
}

type PostProcessor struct {
	c               Config
	t               *template.Template
	docker_build_fn func(*bytes.Buffer) (string, error) // to facilitate easy testing
	tpl             *packer.ConfigTemplate
}

type Config struct {
	common.PackerConfig `mapstructure:",squash"`

	Expose     []string          `mapstructure:"expose"`
	User       string            `mapstructure:"user"`
	Env        map[string]string `mapstructure:"env"`
	Volume     []string          `mapstructure:"volume"`
	WorkDir    string            `mapstructure:"workdir"`
	Entrypoint interface{}       `mapstructure:"entrypoint"`
	Cmd        interface{}       `mapstructure:"cmd"`
	ImageId    string
}

func (p *PostProcessor) Configure(raw_config ...interface{}) error {
	var err error
	_, err = common.DecodeConfig(&p.c, raw_config...)
	if err != nil {
		return err
	}
	p.docker_build_fn = docker_build // configure the build function
	if err = p.prepare_config_template(); err != nil {
		return err
	}

	return nil
}

func (p *PostProcessor) prepare_config_template() error {
	tpl, err := packer.NewConfigTemplate()
	if err != nil {
		return err
	}

	tpl.UserVars = p.c.PackerUserVars
	p.tpl = tpl

	return nil
}

func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) {
	if artifact.BuilderId() != dockerimport.BuilderId {
		err := fmt.Errorf(
			"Unknown artifact type: %s\nCan only tag from Docker builder artifacts.",
			artifact.BuilderId())
		return nil, false, err
	}

	dockerfile, template_err := p.render_template(artifact.Id())
	if template_err != nil { // could not render template
		return nil, false, template_err
	}

	log.Printf("[DEBUG] Dockerfile: %s\n", dockerfile.String())

	if image_id, err := p.docker_build_fn(dockerfile); err != nil { // docker build command failed
		ui.Error("docker build command failed: " + err.Error())
		return nil, false, err
	} else {
		ui.Message("Built image: " + image_id)
		new_artifact := &docker.ImportArtifact{
			BuilderIdValue: dockerimport.BuilderId,
			Driver:         &docker.DockerDriver{Ui: ui, Tpl: nil},
			IdValue:        image_id,
		}
		log.Printf("[DEBUG] artifact: %#v\n", new_artifact)
		return new_artifact, true, nil
	}
}

// Render a variable template using packer.ConfigTemplate primed with user variables
// You must call p.prepare_config_template() before using this function
func (p *PostProcessor) render(var_tmpl string) string {
	rendered, err := p.tpl.Process(var_tmpl, nil)
	if err != nil {
		panic(err)
	}
	return rendered
}

// Process a variable of unknown type. This function will call render() to render any packer user variables
// This function will panic if it can't handle the variable.
func (p *PostProcessor) process_var(variable interface{}) string {
	errs := new(packer.MultiError)

	render_string_or_slice := func(field interface{}) interface{} {
		switch t := field.(type) {
		case []string:
			ary := make([]string, 0, len(t))
			for _, item := range t {
				ary = append(ary, p.render(item))
			}
			return ary
		case []interface{}:
			ary := make([]string, 0, len(t))
			for _, item := range t {
				ary = append(ary, p.render(item.(string)))
			}
			return ary
		case string:
			return p.render(t)
		case nil:
			return nil
		default:
			errs = packer.MultiErrorAppend(errs, fmt.Errorf("Error processing %s: not a string or a string array", field))
			return nil
		}
	}

	switch t := variable.(type) {
	case []string:
		return json_dump_slice(render_string_or_slice(t))
	case []interface{}:
		return json_dump_slice(render_string_or_slice(t))
	case string:
		return p.render(variable.(string))
	case nil:
		return ""
	default:
		panic(errors.New("not sure how to handle type"))
	}
	if len(errs.Errors) > 0 {
		panic(errs)
	}
	return ""
}

func json_dump_slice(data interface{}) string {
	if res, err := json.Marshal(data); err != nil {
		panic(err)
	} else {
		return string(res)
	}
}

func (p *PostProcessor) render_template(id string) (buf *bytes.Buffer, _err error) {
	template_str := `FROM {{ .ImageId }}
{{ if .Volume }}VOLUME {{ stringify .Volume }}
{{ end }}{{ if .Expose }}EXPOSE {{ join .Expose " " }}
{{ end }}{{ if .WorkDir }}WORKDIR {{ .WorkDir }}
{{ end }}{{ if .User }}USER {{ .User }}
{{ end }}{{ if .Env }}{{ range $k, $v := .Env }}ENV {{ $k }} {{ render $v }}
{{ end }}{{ end }}{{ if .Entrypoint }}ENTRYPOINT {{ stringify .Entrypoint }}
{{ end }}{{ if .Cmd }}CMD {{ stringify .Cmd }}{{ end }}`
	template_buffer := new(bytes.Buffer)
	template_writer := bufio.NewWriter(template_buffer)

	p.c.ImageId = id

	defer func() {
		if err := recover(); err != nil {
			switch t_err := err.(type) {
			case error:
				_err = t_err // caught panic, return error to caller
			case string:
				_err = errors.New(t_err)
			default:
			}
		}
	}()

	t, err := template.New("dockerfile").Funcs(template.FuncMap{
		"stringify": p.process_var,
		"join":      strings.Join,
		"render": func(s string) string {
			return p.render(s)
		},
	}).Parse(template_str)
	if err != nil {
		return nil, err
	}

	if err := t.Execute(template_writer, p.c); err != nil {
		return nil, err
	}
	template_writer.Flush()

	return template_buffer, nil
}

func docker_build(stdin *bytes.Buffer) (string, error) {
	var stderr bytes.Buffer
	var stdout bytes.Buffer
	cmd := exec.Command("docker", "build", "--force-rm=true", "--no-cache=true", "-q", "-")
	cmd.Stdin = stdin
	cmd.Stderr = &stderr
	cmd.Stdout = &stdout

	if err := cmd.Start(); err != nil {
		log.Println("[ERR] docker build failed to start")
		return "", err
	}
	if err := cmd.Wait(); err != nil {
		log.Printf("[ERR] error while running docker build. error: %s, command output: %s", err, stderr.String())
		return "", err
	}
	log.Println("Docker build command output:\n" + stdout.String())
	lines := strings.Split(stdout.String(), "\n")
	last_line := lines[len(lines)-2] // we seem to have a trailing empty line
	image_id_regexp := regexp.MustCompile("Successfully built ([a-f0-9]+)")
	if matches := image_id_regexp.FindStringSubmatch(last_line); len(matches) > 0 {
		image_id := matches[len(matches)-1]
		return image_id, nil
	} else {
		return "", errors.New("Could not parse `docker build` output")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"io/ioutil"
	"log"
	"net/http"
	"text/template"
)

var templates = template.Must(template.ParseFiles("bin/pages/error.html", "bin/pages/index.html"))

func main() {
	log.Println("Starting server.")
	http.HandleFunc("/", rootHandler)
	http.HandleFunc("/script", scriptHandler)
	http.HandleFunc("/error", func(w http.ResponseWriter, r *http.Request) {
		renderError(w, "test error", errors.New("test error, please ignore"))
	})
	err := http.ListenAndServe(":8000", nil)
	if err != nil {
		log.Println("Failed to start server:", err)
		return
	}
	log.Println("Stopping server.")
}

/*
rootHandler handles the index page.
*/
func rootHandler(w http.ResponseWriter, r *http.Request) {
	// catch any non-index accessess
	if r.RequestURI != "/" {
		w.WriteHeader(http.StatusNotFound)
		renderError(w, "page doesn't exist", errors.New("no handler defined"))
		return
	}
	err := templates.ExecuteTemplate(w, "index.html", nil)
	if err != nil {
		renderError(w, "server error", err)
		return
	}
}

/*
scriptHandler allows the client to fetch the script for the html files.
*/
func scriptHandler(w http.ResponseWriter, r *http.Request) {
	data, err := ioutil.ReadFile("bin/pages/script.js")
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		renderError(w, "failed to fetch resources", err)
		return
	}
	_, err = w.Write(data)
	if err != nil {
		log.Println("Failed to write script data:", err)
	}
}

/*
renderError renders the user render page and logs the error for the server.
*/
func renderError(w http.ResponseWriter, reason string, err error) {
	log.Printf("User error: <%s> Explanation given: <%s>.", err, reason)
	err = templates.ExecuteTemplate(w, "error.html", reason)
	if err != nil {
		log.Println("renderError failed on template execute:", err)
	}
}
<commit_msg>better renderError<commit_after>package main

import (
	"errors"
	"io/ioutil"
	"log"
	"net/http"
	"text/template"
)

var templates = template.Must(template.ParseFiles("bin/pages/error.html", "bin/pages/index.html"))

func main() {
	log.Println("Starting server.")
	http.HandleFunc("/", rootHandler)
	http.HandleFunc("/script", scriptHandler)
	http.HandleFunc("/error", func(w http.ResponseWriter, r *http.Request) {
		renderError(http.StatusOK, w, "test error", errors.New("test error, please ignore"))
	})
	err := http.ListenAndServe(":8000", nil)
	if err != nil {
		log.Println("Failed to start server:", err)
		return
	}
	log.Println("Stopping server.")
}

/*
rootHandler handles the index page.
*/
func rootHandler(w http.ResponseWriter, r *http.Request) {
	// catch any non-index accessess
	if r.RequestURI != "/" {
		renderError(http.StatusNotFound, w, "page doesn't exist", errors.New("no handler defined"))
		return
	}
	err := templates.ExecuteTemplate(w, "index.html", nil)
	if err != nil {
		renderError(http.StatusInternalServerError, w, "server error", err)
		return
	}
}

/*
scriptHandler allows the client to fetch the script for the html files.
*/
func scriptHandler(w http.ResponseWriter, r *http.Request) {
	data, err := ioutil.ReadFile("bin/pages/script.js")
	if err != nil {
		renderError(http.StatusInternalServerError, w, "failed to fetch resources", err)
		return
	}
	_, err = w.Write(data)
	if err != nil {
		renderError(http.StatusInternalServerError, w, "failed to write resources", err)
	}
}

/*
renderError renders the user render page and logs the error for the server.
*/
func renderError(status int, w http.ResponseWriter, reason string, err error) {
	log.Printf("User error: <%s> Explanation given: <%s>.", err, reason)
	w.WriteHeader(status)
	err = templates.ExecuteTemplate(w, "error.html", reason)
	if err != nil {
		log.Println("renderError failed on template execute:", err)
	}
}
<|endoftext|>