<commit_before>// Copyright 2014 James McGuire. All rights reserved.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"crypto/tls"
	"database/sql"
	"encoding/json"
	//"flag"
	"fmt"
	"html"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"regexp"
	"strings"
	"syscall"
	"time"
	"unicode/utf8"

	"github.com/PuerkitoBio/goquery"
	irc "github.com/fluffle/goirc/client"
	//"github.com/fluffle/goirc/logging/glog"
	_ "github.com/go-sql-driver/mysql"
	"github.com/mvdan/xurls"
	"golang.org/x/net/html/charset"
)

var (
	config         Config
	httpRegex      = regexp.MustCompile(`https?://.*`)
	findWhiteSpace = regexp.MustCompile(`\s+`)
	db             *sql.DB
	badWords       = make(map[string]*regexp.Regexp)
)

const FREENODE = "irc.freenode.net"

type Config struct {
	Channels             []string
	DBConn               string
	Nick                 string
	Ident                string
	FullName             string
	FlickrAPIKey         string
	WolframAPIKey        string
	OpenWeatherMapAPIKey string
	IRCPass              string
	RebuildWords         bool
	Commands             []struct {
		Channel  string
		Commands []struct {
			Name string
			Text string
		}
	}
	BadWords []struct {
		Word  string
		Query string
	}
}

func getCommand(line *irc.Line) string {
	splitmessage := strings.Split(line.Text(), " ")
	cmd := strings.TrimSpace(splitmessage[0])
	return cmd
}

// Try and grab the title for any URL's posted in the channel
func sendUrl(channel, unparsedURL string, conn *irc.Conn, nick string) {
	if !httpRegex.MatchString(unparsedURL) {
		unparsedURL = `http://` + unparsedURL
	}
	postedUrl, err := url.Parse(unparsedURL)
	if err != nil {
		log.Println(err)
		return
	}
	log.Println("Fetching title for " + postedUrl.String() + " In channel " + channel)

	resp, err := http.Get(postedUrl.String())
	if err != nil {
		log.Println(err)
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode >= 400 {
		log.Println("http server return error.")
		return
	}
	respbody := []byte{}
	if resp.Header.Get("Content-Type") == "" {
		buf := make([]byte, 512)
		bufsize, err := resp.Body.Read(buf)
		if err != nil {
			log.Println("adding content type failed")
		}
		resp.Header.Set("Content-Type", http.DetectContentType(buf[:bufsize]))
		respbody = append(respbody, buf[:bufsize]...)
	}

	if !strings.Contains(resp.Header.Get("Content-Type"), "text/html") {
		log.Println("content-type is not text/html")
		return
	}

	utf8Body, err := charset.NewReader(resp.Body, resp.Header.Get("Content-Type"))
	if err != nil {
		log.Println("Error converting page to utf8:", err)
		return
	}
	restofbody, err := ioutil.ReadAll(io.LimitReader(utf8Body, 50000))
	if err != nil {
		log.Println("Error reading posted link:", err)
		return
	}
	respbody = append(respbody, restofbody...)
	query, err := goquery.NewDocumentFromReader(bytes.NewReader(respbody))
	if err != nil {
		log.Println("Error parsing HTML tree:", err)
		return
	}
	title := query.Find("title").Text()
	title = strings.TrimSpace(title)
	if len(title) == 0 || !utf8.ValidString(title) {
		return
	}
	// Example:
	// Title: sadbox . org (at sadbox.org)
	hostNick := fmt.Sprintf(" (%s)", postedUrl.Host)
	formattedTitle := html.UnescapeString(title)
	formattedTitle = findWhiteSpace.ReplaceAllString(formattedTitle, " ")
	if len(formattedTitle) > conn.Config().SplitLen-len(hostNick)-1 {
		formattedTitle = formattedTitle[:conn.Config().SplitLen-len(hostNick)-1]
	}
	formattedTitle = formattedTitle + hostNick
	log.Println(formattedTitle)
	conn.Privmsg(channel, formattedTitle)
}

func logMessage(conn *irc.Conn, line *irc.Line) {
	_, err := db.Exec("insert into messages (Nick, Ident, Host, Src, Cmd, Channel,"+
		" Message, Time) values (?, ?, ?, ?, ?, ?, ?, ?)", line.Nick, line.Ident,
		line.Host, line.Src, line.Cmd, line.Target(), line.Text(), line.Time)
	if err != nil {
		log.Println(err)
	}
	err = updateWords(line.Nick, line.Text())
	if err != nil {
		log.Println(err)
	}
}

func checkForUrl(conn *irc.Conn, line *irc.Line) {
	if strings.HasPrefix(line.Text(), "#") {
		return
	}
	urllist := make(map[string]struct{})
	for _, item := range xurls.Relaxed.FindAllString(line.Text(), -1) {
		urllist[item] = struct{}{}
	}
	numlinks := 0
	for item, _ := range urllist {
		numlinks++
		if numlinks > 3 {
			break
		}
		go sendUrl(line.Target(), item, conn, line.Nick)
	}
}

func cst(conn *irc.Conn, line *irc.Line) {
	if line.Nick != "sadbox" || getCommand(line) != "!cst" {
		return
	}
	go conn.Privmsg(line.Target(), "\u00039,13#CSTMASTERRACE")
}

// Commands that are read in from the config file
func configCommands(conn *irc.Conn, line *irc.Line) {
	splitmessage := strings.Split(line.Text(), " ")
AllConfigs:
	for _, commandConfig := range config.Commands {
		if commandConfig.Channel == line.Target() || commandConfig.Channel == "default" {
			for _, command := range commandConfig.Commands {
				if getCommand(line) == command.Name {
					var response string
					if len(splitmessage) >= 2 {
						response = fmt.Sprintf("%s: %s", splitmessage[1], command.Text)
					} else {
						response = command.Text
					}
					go conn.Privmsg(line.Target(), response)
					break AllConfigs
				}
			}
		}
	}
}

func init() {
	log.Println("Starting sadbot")

	rand.Seed(time.Now().UTC().UnixNano())

	configfile, err := os.Open("config.json")
	if err != nil {
		log.Fatal(err)
	}
	err = json.NewDecoder(configfile).Decode(&config)
	if err != nil {
		log.Fatal(err)
	}

	for _, word := range config.BadWords {
		badWords[word.Word] = regexp.MustCompile(word.Query)
	}

	log.Println("Loaded config file!")
	log.Printf("Joining: %s", config.Channels)
	log.Printf("Nick: %s", config.Nick)
	log.Printf("Ident: %s", config.Ident)
	log.Printf("FullName: %s", config.FullName)

	numcommands := 0
	for _, commandConfig := range config.Commands {
		for _, command := range commandConfig.Commands {
			numcommands++
			log.Printf("%d %s/%s: %s", numcommands, commandConfig.Channel, command.Name, command.Text)
		}
	}
	log.Printf("Found %d commands", numcommands)
}

func main() {
	//flag.Parse()
	//glog.Init()
	var err error
	db, err = sql.Open("mysql", config.DBConn)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()
	db.SetMaxIdleConns(100)
	db.SetMaxOpenConns(200)

	go makeMarkov()

	buildchan := make(chan os.Signal, 1)
	signal.Notify(buildchan, syscall.SIGUSR1)
	go func() {
		for _ = range buildchan {
			genTables()
		}
	}()

	ircConfig := irc.NewConfig(config.Nick, config.Ident, config.FullName)
	ircConfig.SSL = true
	ircConfig.SSLConfig = &tls.Config{ServerName: FREENODE}
	ircConfig.Server = FREENODE
	ircConfig.Pass = config.Nick + ":" + config.IRCPass

	c := irc.Client(ircConfig)

	c.HandleFunc(irc.CONNECTED,
		func(conn *irc.Conn, line *irc.Line) {
			for _, channel := range config.Channels {
				log.Printf("Joining %s", channel)
				conn.Join(channel)
			}
			log.Println("Connected!")
		})
	quit := make(chan bool)

	c.HandleFunc(irc.DISCONNECTED,
		func(conn *irc.Conn, line *irc.Line) { print("disconnected!"); quit <- true })

	// Handle all the things
	c.HandleFunc(irc.PRIVMSG, logMessage)
	c.HandleFunc(irc.ACTION, logMessage)

	c.HandleFunc(irc.PRIVMSG, checkForUrl)
	c.HandleFunc(irc.ACTION, checkForUrl)

	c.HandleFunc(irc.PRIVMSG, haata)
	c.HandleFunc(irc.PRIVMSG, wolfram)
	c.HandleFunc(irc.PRIVMSG, meeba)
	c.HandleFunc(irc.PRIVMSG, markov)
	c.HandleFunc(irc.PRIVMSG, dance)
	c.HandleFunc(irc.PRIVMSG, cst)
	c.HandleFunc(irc.PRIVMSG, roll)
	c.HandleFunc(irc.PRIVMSG, btc)
	c.HandleFunc(irc.PRIVMSG, lastSeen)
	c.HandleFunc(irc.PRIVMSG, showWeather)
	c.HandleFunc(irc.PRIVMSG, showQuote)
	c.HandleFunc(irc.PRIVMSG, configCommands)

	if err := c.Connect(); err != nil {
		log.Fatalln("Connection error: %s\n", err)
	}

	<-quit
	log.Fatal("SHITTING THE FUCK DOWN")
}
<commit_msg>Only use the first title found in the head section<commit_after>// Copyright 2014 James McGuire. All rights reserved.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"crypto/tls"
	"database/sql"
	"encoding/json"
	//"flag"
	"fmt"
	"html"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"regexp"
	"strings"
	"syscall"
	"time"
	"unicode/utf8"

	"github.com/PuerkitoBio/goquery"
	irc "github.com/fluffle/goirc/client"
	//"github.com/fluffle/goirc/logging/glog"
	_ "github.com/go-sql-driver/mysql"
	"github.com/mvdan/xurls"
	"golang.org/x/net/html/charset"
)

var (
	config         Config
	httpRegex      = regexp.MustCompile(`https?://.*`)
	findWhiteSpace = regexp.MustCompile(`\s+`)
	db             *sql.DB
	badWords       = make(map[string]*regexp.Regexp)
)

const FREENODE = "irc.freenode.net"

type Config struct {
	Channels             []string
	DBConn               string
	Nick                 string
	Ident                string
	FullName             string
	FlickrAPIKey         string
	WolframAPIKey        string
	OpenWeatherMapAPIKey string
	IRCPass              string
	RebuildWords         bool
	Commands             []struct {
		Channel  string
		Commands []struct {
			Name string
			Text string
		}
	}
	BadWords []struct {
		Word  string
		Query string
	}
}

func getCommand(line *irc.Line) string {
	splitmessage := strings.Split(line.Text(), " ")
	cmd := strings.TrimSpace(splitmessage[0])
	return cmd
}

// Try and grab the title for any URL's posted in the channel
func sendUrl(channel, unparsedURL string, conn *irc.Conn, nick string) {
	if !httpRegex.MatchString(unparsedURL) {
		unparsedURL = `http://` + unparsedURL
	}
	postedUrl, err := url.Parse(unparsedURL)
	if err != nil {
		log.Println(err)
		return
	}
	log.Println("Fetching title for " + postedUrl.String() + " In channel " + channel)

	resp, err := http.Get(postedUrl.String())
	if err != nil {
		log.Println(err)
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode >= 400 {
		log.Println("http server return error.")
		return
	}
	respbody := []byte{}
	if resp.Header.Get("Content-Type") == "" {
		buf := make([]byte, 512)
		bufsize, err := resp.Body.Read(buf)
		if err != nil {
			log.Println("adding content type failed")
		}
		resp.Header.Set("Content-Type", http.DetectContentType(buf[:bufsize]))
		respbody = append(respbody, buf[:bufsize]...)
	}

	if !strings.Contains(resp.Header.Get("Content-Type"), "text/html") {
		log.Println("content-type is not text/html")
		return
	}

	utf8Body, err := charset.NewReader(resp.Body, resp.Header.Get("Content-Type"))
	if err != nil {
		log.Println("Error converting page to utf8:", err)
		return
	}
	restofbody, err := ioutil.ReadAll(io.LimitReader(utf8Body, 50000))
	if err != nil {
		log.Println("Error reading posted link:", err)
		return
	}
	respbody = append(respbody, restofbody...)
	query, err := goquery.NewDocumentFromReader(bytes.NewReader(respbody))
	if err != nil {
		log.Println("Error parsing HTML tree:", err)
		return
	}
	title := query.Find("head").Find("title").First().Text()
	title = strings.TrimSpace(title)
	if len(title) == 0 || !utf8.ValidString(title) {
		return
	}
	// Example:
	// Title: sadbox . org (at sadbox.org)
	hostNick := fmt.Sprintf(" (%s)", postedUrl.Host)
	formattedTitle := html.UnescapeString(title)
	formattedTitle = findWhiteSpace.ReplaceAllString(formattedTitle, " ")
	if len(formattedTitle) > conn.Config().SplitLen-len(hostNick)-1 {
		formattedTitle = formattedTitle[:conn.Config().SplitLen-len(hostNick)-1]
	}
	formattedTitle = formattedTitle + hostNick
	log.Println(formattedTitle)
	conn.Privmsg(channel, formattedTitle)
}

func logMessage(conn *irc.Conn, line *irc.Line) {
	_, err := db.Exec("insert into messages (Nick, Ident, Host, Src, Cmd, Channel,"+
		" Message, Time) values (?, ?, ?, ?, ?, ?, ?, ?)", line.Nick, line.Ident,
		line.Host, line.Src, line.Cmd, line.Target(), line.Text(), line.Time)
	if err != nil {
		log.Println(err)
	}
	err = updateWords(line.Nick, line.Text())
	if err != nil {
		log.Println(err)
	}
}

func checkForUrl(conn *irc.Conn, line *irc.Line) {
	if strings.HasPrefix(line.Text(), "#") {
		return
	}
	urllist := make(map[string]struct{})
	for _, item := range xurls.Relaxed.FindAllString(line.Text(), -1) {
		urllist[item] = struct{}{}
	}
	numlinks := 0
	for item, _ := range urllist {
		numlinks++
		if numlinks > 3 {
			break
		}
		go sendUrl(line.Target(), item, conn, line.Nick)
	}
}

func cst(conn *irc.Conn, line *irc.Line) {
	if line.Nick != "sadbox" || getCommand(line) != "!cst" {
		return
	}
	go conn.Privmsg(line.Target(), "\u00039,13#CSTMASTERRACE")
}

// Commands that are read in from the config file
func configCommands(conn *irc.Conn, line *irc.Line) {
	splitmessage := strings.Split(line.Text(), " ")
AllConfigs:
	for _, commandConfig := range config.Commands {
		if commandConfig.Channel == line.Target() || commandConfig.Channel == "default" {
			for _, command := range commandConfig.Commands {
				if getCommand(line) == command.Name {
					var response string
					if len(splitmessage) >= 2 {
						response = fmt.Sprintf("%s: %s", splitmessage[1], command.Text)
					} else {
						response = command.Text
					}
					go conn.Privmsg(line.Target(), response)
					break AllConfigs
				}
			}
		}
	}
}

func init() {
	log.Println("Starting sadbot")

	rand.Seed(time.Now().UTC().UnixNano())

	configfile, err := os.Open("config.json")
	if err != nil {
		log.Fatal(err)
	}
	err = json.NewDecoder(configfile).Decode(&config)
	if err != nil {
		log.Fatal(err)
	}

	for _, word := range config.BadWords {
		badWords[word.Word] = regexp.MustCompile(word.Query)
	}

	log.Println("Loaded config file!")
	log.Printf("Joining: %s", config.Channels)
	log.Printf("Nick: %s", config.Nick)
	log.Printf("Ident: %s", config.Ident)
	log.Printf("FullName: %s", config.FullName)

	numcommands := 0
	for _, commandConfig := range config.Commands {
		for _, command := range commandConfig.Commands {
			numcommands++
			log.Printf("%d %s/%s: %s", numcommands, commandConfig.Channel, command.Name, command.Text)
		}
	}
	log.Printf("Found %d commands", numcommands)
}

func main() {
	//flag.Parse()
	//glog.Init()
	var err error
	db, err = sql.Open("mysql", config.DBConn)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()
	db.SetMaxIdleConns(100)
	db.SetMaxOpenConns(200)

	go makeMarkov()

	buildchan := make(chan os.Signal, 1)
	signal.Notify(buildchan, syscall.SIGUSR1)
	go func() {
		for _ = range buildchan {
			genTables()
		}
	}()

	ircConfig := irc.NewConfig(config.Nick, config.Ident, config.FullName)
	ircConfig.SSL = true
	ircConfig.SSLConfig = &tls.Config{ServerName: FREENODE}
	ircConfig.Server = FREENODE
	ircConfig.Pass = config.Nick + ":" + config.IRCPass

	c := irc.Client(ircConfig)

	c.HandleFunc(irc.CONNECTED,
		func(conn *irc.Conn, line *irc.Line) {
			for _, channel := range config.Channels {
				log.Printf("Joining %s", channel)
				conn.Join(channel)
			}
			log.Println("Connected!")
		})
	quit := make(chan bool)

	c.HandleFunc(irc.DISCONNECTED,
		func(conn *irc.Conn, line *irc.Line) { print("disconnected!"); quit <- true })

	// Handle all the things
	c.HandleFunc(irc.PRIVMSG, logMessage)
	c.HandleFunc(irc.ACTION, logMessage)

	c.HandleFunc(irc.PRIVMSG, checkForUrl)
	c.HandleFunc(irc.ACTION, checkForUrl)

	c.HandleFunc(irc.PRIVMSG, haata)
	c.HandleFunc(irc.PRIVMSG, wolfram)
	c.HandleFunc(irc.PRIVMSG, meeba)
	c.HandleFunc(irc.PRIVMSG, markov)
	c.HandleFunc(irc.PRIVMSG, dance)
	c.HandleFunc(irc.PRIVMSG, cst)
	c.HandleFunc(irc.PRIVMSG, roll)
	c.HandleFunc(irc.PRIVMSG, btc)
	c.HandleFunc(irc.PRIVMSG, lastSeen)
	c.HandleFunc(irc.PRIVMSG, showWeather)
	c.HandleFunc(irc.PRIVMSG, showQuote)
	c.HandleFunc(irc.PRIVMSG, configCommands)

	if err := c.Connect(); err != nil {
		log.Fatalln("Connection error: %s\n", err)
	}

	<-quit
	log.Fatal("SHITTING THE FUCK DOWN")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"sort"
	"strconv"
	"strings"
)

var commonFoods []string
var goodContextWords []string
var badContextWords []string
var pairing map[[2]string]int
var uniqueFoods []string

// func gaussianFilter(data []int) (filtered []int) {

func init() {
	loadData()

	pairing = make(map[[2]string]int, 221777)

	f, err := ioutil.ReadFile("resources/pairing.csv")
	if err != nil {
		panic(err)
	}
	for _, l := range strings.Split(string(f), "\n") {
		arr := strings.Split(string(l), ",")
		if len(arr) == 3 {
			k := arr[:2]
			sort.Sort(sort.StringSlice(k))
			key := [2]string{k[0], k[1]}

			val, err := strconv.Atoi(arr[2])
			if err != nil {
				panic(err)
			}
			pairing[key] = val

		}
	}
	fmt.Println(pairing[[2]string{"mint oil", "pinto bean"}])
	fmt.Println(len(pairing), len(uniqueFoods))

}

func main() {

	// fmt.Println(commonFoods)
	// fmt.Println(badContextWords)
	// fmt.Println(goodContextWords)

	testContent := `Hi
Ingredients
2 cups all-purpose flour
1/4 cup sugar
1 tablespoon baking powder
1/2 teaspoon salt
1 1/2 cups milk
1 tablespoon plus 1 teaspoon vanilla extract
2 large eggs, separated, plus 2 additional egg whites
1 stick (8 tablespoons) salted butter, melted, plus softened butter, for serving
Warm syrup, for serving
Directions
Watch how to make this recipe.
Special equipment: Waffle iron
Preheat the waffle iron to the regular setting.
Sift together the flour, sugar, baking powder and salt in a bowl. In a separate bowl, whisk together the milk, vanilla and 2 egg yolks. Pour over the dry ingredients and very gently stir         until halfway combined. Pour in the melted butter and continue mixing very gently until combined.
In a separate bowl using a
whisk
(or a
mixer
), beat the 4 egg whites until stiff. Slowly fold them into the batter, stopping short of mixing them all the way         through.
Scoop the
batter
into your
waffle iron
in batches and cook according to its directions (lean toward the
waffles
being a little deep golden and
crisp
!). Serve immediately with softened butter and warm syrup.
Recipe courtesy of Ree Drummond`
	testContent = parseURL("http://www.foodnetwork.com/recipes/food-network-kitchens/slow-cooker-pork-tacos-recipe.html")

	text := getIndredientText(testContent)

	text = strings.ToLower(text)
	text = strings.Replace(text, ",", "", -1)
	text = strings.Replace(text, ".", "", -1)
	text = strings.Replace(text, "!", "", -1)
	text = strings.Replace(text, "?", "", -1)
	text = strings.Replace(text, "olive oil", "olivoil", -1)
	text = strings.Replace(text, "cider vinegar", "vinegar", -1)
	text = strings.Replace(text, "nutrition", "", -1)
	fmt.Println(text)

	fmt.Println("\nFrom food pairing:")
	var ingredients []string
	for _, food := range commonFoods {
		if strings.Contains(text, " "+food) && len(food) > 1 {
			ingredients = append(ingredients, food)
		}
	}

	score := float64(0)
	foods := float64(0)
	for i, food1 := range ingredients {
		for j, food2 := range ingredients {
			if j > i {
				score += float64(pairing[[2]string{food1, food2}])
				score += float64(pairing[[2]string{food2, food1}])
				if pairing[[2]string{food1, food2}] > 0 {
					fmt.Println(food1, food2, pairing[[2]string{food1, food2}])
				}
				if pairing[[2]string{food2, food1}] > 0 {
					fmt.Println(food2, food1, pairing[[2]string{food2, food1}])
				}
				if score > 0 {
					foods += float64(1)
				}
			}
		}
	}
	fmt.Println(score, foods, score/foods)

}
<commit_msg>Updated with new url<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"sort"
	"strconv"
	"strings"
)

var commonFoods []string
var goodContextWords []string
var badContextWords []string
var pairing map[[2]string]int
var uniqueFoods []string

// func gaussianFilter(data []int) (filtered []int) {

func init() {
	loadData()

	pairing = make(map[[2]string]int, 221777)

	f, err := ioutil.ReadFile("resources/pairing.csv")
	if err != nil {
		panic(err)
	}
	for _, l := range strings.Split(string(f), "\n") {
		arr := strings.Split(string(l), ",")
		if len(arr) == 3 {
			k := arr[:2]
			sort.Sort(sort.StringSlice(k))
			key := [2]string{k[0], k[1]}

			val, err := strconv.Atoi(arr[2])
			if err != nil {
				panic(err)
			}
			pairing[key] = val

		}
	}
	fmt.Println(pairing[[2]string{"mint oil", "pinto bean"}])
	fmt.Println(len(pairing), len(uniqueFoods))

}

func main() {

	// fmt.Println(commonFoods)
	// fmt.Println(badContextWords)
	// fmt.Println(goodContextWords)

	testContent := `Hi
Ingredients
2 cups all-purpose flour
1/4 cup sugar
1 tablespoon baking powder
1/2 teaspoon salt
1 1/2 cups milk
1 tablespoon plus 1 teaspoon vanilla extract
2 large eggs, separated, plus 2 additional egg whites
1 stick (8 tablespoons) salted butter, melted, plus softened butter, for serving
Warm syrup, for serving
Directions
Watch how to make this recipe.
Special equipment: Waffle iron
Preheat the waffle iron to the regular setting.
Sift together the flour, sugar, baking powder and salt in a bowl. In a separate bowl, whisk together the milk, vanilla and 2 egg yolks. Pour over the dry ingredients and very gently stir         until halfway combined. Pour in the melted butter and continue mixing very gently until combined.
In a separate bowl using a
whisk
(or a
mixer
), beat the 4 egg whites until stiff. Slowly fold them into the batter, stopping short of mixing them all the way         through.
Scoop the
batter
into your
waffle iron
in batches and cook according to its directions (lean toward the
waffles
being a little deep golden and
crisp
!). Serve immediately with softened butter and warm syrup.
Recipe courtesy of Ree Drummond`
	testContent = parseURL("http://allrecipes.com/recipe/18897/company-couscous/")
	text := getIndredientText(testContent)

	text = strings.ToLower(text)
	text = strings.Replace(text, ",", "", -1)
	text = strings.Replace(text, ".", "", -1)
	text = strings.Replace(text, "!", "", -1)
	text = strings.Replace(text, "?", "", -1)
	text = strings.Replace(text, "olive oil", "olivoil", -1)
	text = strings.Replace(text, "cider vinegar", "vinegar", -1)
	text = strings.Replace(text, "nutrition", "", -1)
	fmt.Println(text)

	fmt.Println("\nFrom food pairing:")
	var ingredients []string
	for _, food := range commonFoods {
		if strings.Contains(text, " "+food) && len(food) > 1 {
			ingredients = append(ingredients, food)
		}
	}

	score := float64(0)
	foods := float64(0)
	for i, food1 := range ingredients {
		for j, food2 := range ingredients {
			if j > i {
				score += float64(pairing[[2]string{food1, food2}])
				score += float64(pairing[[2]string{food2, food1}])
				if pairing[[2]string{food1, food2}] > 0 {
					fmt.Println(food1, food2, pairing[[2]string{food1, food2}])
				}
				if pairing[[2]string{food2, food1}] > 0 {
					fmt.Println(food2, food1, pairing[[2]string{food2, food1}])
				}
				if score > 0 {
					foods += float64(1)
				}
			}
		}
	}
	fmt.Println(score, foods, score/foods)

}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/fatih/color"
	"github.com/howeyc/fsnotify"
)

func handle(ev *fsnotify.FileEvent) {
	commandline := strings.Fields(flag.Args()[0])
	cmd := exec.Command(commandline[0], commandline[1:]...)

	pipe, err := cmd.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}

	fmt.Print("\033[H\033[2J") // Clear the screen
	log.Println(fmt.Sprintf("Running %s...", commandline[0]))
	if err = cmd.Start(); err != nil {
		log.Fatal(err)
	}

	buff := make([]byte, 1024)

	for {
		n, err := pipe.Read(buff)
		// Either if the pipe was empty or an EOF or other error was returned.
		if n == 0 && err == nil || err != nil {
			break
		}

		s := string(buff[:n])
		fmt.Print(s)
	}

	// Print red error message or green success message
	if err = cmd.Wait(); err != nil {
		red := color.New(color.FgRed, color.Bold).SprintfFunc()
		log.Println(red(err.Error()))
	} else {
		green := color.New(color.FgGreen, color.Bold).SprintfFunc()
		log.Println(green("Execution successful."))
	}
}

func main() {
	dir := flag.String("d", ".", "directory to watch")
	flag.Parse()

	if flag.NArg() != 1 {
		log.Fatal("One command line argument sting required.")
	}

	absdir, err := filepath.Abs(*dir)
	if err != nil {
		log.Fatal(err)
	}

	watcher, err := fsnotify.NewWatcher()
	defer watcher.Close()

	if err != nil {
		log.Fatal(err)
	}

	done := make(chan bool)

	// Process events
	go func() {
		for {
			select {
			case ev := <-watcher.Event:
				handle(ev)

			case err := <-watcher.Error:
				log.Println("error:", err)

			}
		}
	}()

	log.Printf("Watching %s...", absdir)
	err = watcher.Watch(*dir)
	if err != nil {
		log.Fatal(err)
	}

	<-done
}
<commit_msg>Remove absolute paths<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os/exec"
	"strings"

	"github.com/fatih/color"
	"github.com/howeyc/fsnotify"
)

func handle(ev *fsnotify.FileEvent) {
	commandline := strings.Fields(flag.Args()[0])
	cmd := exec.Command(commandline[0], commandline[1:]...)

	pipe, err := cmd.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}

	fmt.Print("\033[H\033[2J") // Clear the screen
	log.Println(fmt.Sprintf("Running %s...", commandline[0]))
	if err = cmd.Start(); err != nil {
		log.Fatal(err)
	}

	buff := make([]byte, 1024)

	for {
		n, err := pipe.Read(buff)
		// Either if the pipe was empty or an EOF or other error was returned.
		if n == 0 && err == nil || err != nil {
			break
		}

		fmt.Print(string(buff[:n]))
	}

	// Print red error message or green success message
	if err = cmd.Wait(); err != nil {
		red := color.New(color.FgRed, color.Bold).SprintfFunc()
		log.Println(red(err.Error()))
	} else {
		green := color.New(color.FgGreen, color.Bold).SprintfFunc()
		log.Println(green("Execution successful."))
	}
}

func main() {
	dir := flag.String("d", ".", "directory to watch")
	flag.Parse()

	if flag.NArg() != 1 {
		log.Fatal("One command line argument sting required.")
	}

	watcher, err := fsnotify.NewWatcher()
	defer watcher.Close()

	if err != nil {
		log.Fatal(err)
	}

	done := make(chan bool)

	// Process events
	go func() {
		for {
			select {
			case ev := <-watcher.Event:
				handle(ev)

			case err := <-watcher.Error:
				log.Println("error:", err)

			}
		}
	}()

	log.Printf("Watching %s...", *dir)
	err = watcher.Watch(*dir)
	if err != nil {
		log.Fatal(err)
	}

	<-done
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	color "github.com/fatih/color"
	//pretty "github.com/kr/pretty"
	goyaml "gopkg.in/yaml.v2"
	"io/ioutil"
	"os"
	"reflect"
)

func main() {
	file1 := os.Args[1]
	file2 := os.Args[2]

	buf, err := ioutil.ReadFile(file1)
	if err != nil {
		panic(err)
	}
	m1 := make(map[interface{}]interface{})
	err = goyaml.Unmarshal([]byte(buf), &m1)
	if err != nil {
		panic(err)
	}

	buf2, err := ioutil.ReadFile(file2)
	if err != nil {
		panic(err)
	}
	m2 := make(map[interface{}]interface{})
	err = goyaml.Unmarshal([]byte(buf2), &m2)
	if err != nil {
		panic(err)
	}

	m3 := make(map[interface{}]interface{})
	addLeftParam(m3, m1)
	addRightParam(m3, m2)

	ommitSameParam(m3)
	//pretty.Printf("%# v\n\n", m3)

	printDiffYaml(m3, "")
}

type Comparer struct {
	leftValue  interface{}
	rightValue interface{}
}

type Empty struct {
}

func addLeftParam(mergedMap, leftMap map[interface{}]interface{}) {
	for key, value := range leftMap {
		if value == nil || reflect.TypeOf(value).Kind() != reflect.Map {
			mergedMap[key] = value
		} else {
			if _, ok := mergedMap[key]; !ok {
				mergedMap[key] = make(map[interface{}]interface{})
			}
			addLeftParam(mergedMap[key].(map[interface{}]interface{}), leftMap[key].(map[interface{}]interface{}))
		}
	}
}

func addRightParam(mergedMap, rightMap map[interface{}]interface{}) {
	for key, value := range rightMap {
		if value == nil || reflect.TypeOf(value).Kind() != reflect.Map {
			if _, ok := mergedMap[key]; !ok {
				mergedMap[key] = Comparer{Empty{}, value}
			} else if mergedMap[key] == nil {
				mergedMap[key] = Comparer{nil, value}
			} else {
				mergedMap[key] = Comparer{mergedMap[key], value}
			}
		} else {
			if _, ok := mergedMap[key]; !ok {
				mergedMap[key] = make(map[interface{}]interface{})
			}
			if mergedMap[key] == nil || reflect.TypeOf(mergedMap[key]).Kind() != reflect.Map {
				mergedMap[key] = Comparer{mergedMap[key], value}
			} else {
				addRightParam(mergedMap[key].(map[interface{}]interface{}), rightMap[key].(map[interface{}]interface{}))
			}
		}
	}

	setComparer(mergedMap)
}

func setComparer(mergedMap map[interface{}]interface{}) {
	for key, value := range mergedMap {
		if value == nil {
			mergedMap[key] = Comparer{nil, Empty{}}
		} else if reflect.TypeOf(value).Kind() == reflect.Map {
			setComparer(mergedMap[key].(map[interface{}]interface{}))
		} else {
			vt := reflect.TypeOf(value).Kind()
			if vt != reflect.Map && vt != reflect.Struct {
				mergedMap[key] = Comparer{value, Empty{}}
			}
		}
	}
}

func ommitSameParam(m map[interface{}]interface{}) {
	for key, value := range m {
		if reflect.TypeOf(value).Kind() != reflect.Map {
			tmp := value.(Comparer)
			if reflect.DeepEqual(tmp.leftValue, tmp.rightValue) {
				delete(m, key)
			}
		} else {
			ommitSameParam(m[key].(map[interface{}]interface{}))
			if len(m[key].(map[interface{}]interface{})) == 0 {
				delete(m, key)
			}
		}
	}
}

func printDiffYaml(m map[interface{}]interface{}, indent string) {
	for key, value := range m {
		if reflect.TypeOf(value).Kind() == reflect.Map {
			println(indent + key.(string) + ":")
			printDiffYaml(m[key].(map[interface{}]interface{}), indent+"    ")
		} else {
			tmp := value.(Comparer)
			println(indent + key.(string) + ":")
			color.Set(color.FgRed)
			printValue(tmp.leftValue, indent+"  - ")
			color.Set(color.FgGreen)
			printValue(tmp.rightValue, indent+"  + ")
			color.Unset()
		}

	}
}

func printValue(value interface{}, indent string) {
	if value != nil {
		if reflect.TypeOf(value).Kind() < reflect.Array {
			fmt.Printf("%s%#v\n", indent, reflect.ValueOf(value).Interface())
		}

		switch reflect.TypeOf(value).Kind() {
		case reflect.Slice:
			fmt.Printf("%s[", indent)
			for i, v := range value.([]interface{}) {
				if i != 0 {
					fmt.Print(", ")
				}
				fmt.Printf("%#v", reflect.ValueOf(v).Interface())
			}
			println("]")
		case reflect.String:
			fmt.Printf("%s%#v\n", indent, reflect.ValueOf(value).Interface())
		case reflect.Map:
			for key, v := range value.(map[interface{}]interface{}) {
				println(indent + key.(string) + ":")
				printValue(v, "    "+indent)
			}
		}

	} else {
		println(indent + "null")
	}
}
<commit_msg>Print by alphabetical order<commit_after>package main

import (
	"fmt"
	color "github.com/fatih/color"
	//pretty "github.com/kr/pretty"
	goyaml "gopkg.in/yaml.v2"
	"io/ioutil"
	"os"
	"reflect"
	"sort"
)

func main() {
	file1 := os.Args[1]
	file2 := os.Args[2]

	buf, err := ioutil.ReadFile(file1)
	if err != nil {
		panic(err)
	}
	m1 := make(map[interface{}]interface{})
	err = goyaml.Unmarshal([]byte(buf), &m1)
	if err != nil {
		panic(err)
	}

	buf2, err := ioutil.ReadFile(file2)
	if err != nil {
		panic(err)
	}
	m2 := make(map[interface{}]interface{})
	err = goyaml.Unmarshal([]byte(buf2), &m2)
	if err != nil {
		panic(err)
	}

	m3 := make(map[interface{}]interface{})
	addLeftParam(m3, m1)
	addRightParam(m3, m2)

	ommitSameParam(m3)
	//pretty.Printf("%# v\n\n", m3)

	printDiffYaml(m3, "")
}

type Comparer struct {
	leftValue  interface{}
	rightValue interface{}
}

type Empty struct {
}

func addLeftParam(mergedMap, leftMap map[interface{}]interface{}) {
	for key, value := range leftMap {
		if value == nil || reflect.TypeOf(value).Kind() != reflect.Map {
			mergedMap[key] = value
		} else {
			if _, ok := mergedMap[key]; !ok {
				mergedMap[key] = make(map[interface{}]interface{})
			}
			addLeftParam(mergedMap[key].(map[interface{}]interface{}), leftMap[key].(map[interface{}]interface{}))
		}
	}
}

func addRightParam(mergedMap, rightMap map[interface{}]interface{}) {
	for key, value := range rightMap {
		if value == nil || reflect.TypeOf(value).Kind() != reflect.Map {
			if _, ok := mergedMap[key]; !ok {
				mergedMap[key] = Comparer{Empty{}, value}
			} else if mergedMap[key] == nil {
				mergedMap[key] = Comparer{nil, value}
			} else {
				mergedMap[key] = Comparer{mergedMap[key], value}
			}
		} else {
			if _, ok := mergedMap[key]; !ok {
				mergedMap[key] = make(map[interface{}]interface{})
			}
			if mergedMap[key] == nil || reflect.TypeOf(mergedMap[key]).Kind() != reflect.Map {
				mergedMap[key] = Comparer{mergedMap[key], value}
			} else {
				addRightParam(mergedMap[key].(map[interface{}]interface{}), rightMap[key].(map[interface{}]interface{}))
			}
		}
	}

	setComparer(mergedMap)
}

func setComparer(mergedMap map[interface{}]interface{}) {
	for key, value := range mergedMap {
		if value == nil {
			mergedMap[key] = Comparer{nil, Empty{}}
		} else if reflect.TypeOf(value).Kind() == reflect.Map {
			setComparer(mergedMap[key].(map[interface{}]interface{}))
		} else {
			vt := reflect.TypeOf(value).Kind()
			if vt != reflect.Map && vt != reflect.Struct {
				mergedMap[key] = Comparer{value, Empty{}}
			}
		}
	}
}

func ommitSameParam(m map[interface{}]interface{}) {
	for key, value := range m {
		if reflect.TypeOf(value).Kind() != reflect.Map {
			tmp := value.(Comparer)
			if reflect.DeepEqual(tmp.leftValue, tmp.rightValue) {
				delete(m, key)
			}
		} else {
			ommitSameParam(m[key].(map[interface{}]interface{}))
			if len(m[key].(map[interface{}]interface{})) == 0 {
				delete(m, key)
			}
		}
	}
}

func printDiffYaml(m map[interface{}]interface{}, indent string) {
	sortedMap := make([]string, len(m))
	i := 0
	for k, _ := range m {
		sortedMap[i] = k.(string)
		i++
	}
	sort.Strings(sortedMap)

	for _, v := range sortedMap {
		key := v
		value := m[key]
		if reflect.TypeOf(value).Kind() == reflect.Map {
			println(indent + key + ":")
			printDiffYaml(m[key].(map[interface{}]interface{}), indent+"    ")
		} else {
			tmp := value.(Comparer)
			println(indent + key + ":")
			color.Set(color.FgRed)
			printValue(tmp.leftValue, indent+"  - ")
			color.Set(color.FgGreen)
			printValue(tmp.rightValue, indent+"  + ")
			color.Unset()
		}

	}
}

func printValue(value interface{}, indent string) {
	if value != nil {
		if reflect.TypeOf(value).Kind() < reflect.Array {
			fmt.Printf("%s%#v\n", indent, reflect.ValueOf(value).Interface())
		}

		switch reflect.TypeOf(value).Kind() {
		case reflect.Slice:
			fmt.Printf("%s[", indent)
			for i, v := range value.([]interface{}) {
				if i != 0 {
					fmt.Print(", ")
				}
				fmt.Printf("%#v", reflect.ValueOf(v).Interface())
			}
			println("]")
		case reflect.String:
			fmt.Printf("%s%#v\n", indent, reflect.ValueOf(value).Interface())
		case reflect.Map:
			for key, v := range value.(map[interface{}]interface{}) {
				println(indent + key.(string) + ":")
				printValue(v, "    "+indent)
			}
		}

	} else {
		println(indent + "null")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/google/go-github/github"
	oauth "golang.org/x/oauth2"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"os"
)

type Token string

func (t Token) Token() (*oauth.Token, error) {
	return &oauth.Token{AccessToken: string(t)}, nil
}

type Label struct {
	Name  string
	Color string
}

type Milestone struct {
	Name string
}

type Config struct {
	Repos      []string
	Labels     []Label
	Milestones []Milestone
}

func setupClient() *github.Client {
	token, _ := Token(os.Getenv("GITHUB_TOKEN")).Token()
	var conf oauth.Config
	client := conf.Client(nil, token)

	return github.NewClient(client)
}

func main() {
	var conf Config
	filecontents, err := ioutil.ReadFile("config.yml.example")
	if err != nil {
		panic(err)
	}
	yaml.Unmarshal(filecontents, &conf)
	fmt.Printf("%#v\n", &conf)

	// retrieve information about the repos we're gonna manage
}
<commit_msg>use github.Label as the main data structure for Config<commit_after>package main

import (
	"fmt"
	"github.com/google/go-github/github"
	oauth "golang.org/x/oauth2"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"os"
)

type Token string

func (t Token) Token() (*oauth.Token, error) {
	return &oauth.Token{AccessToken: string(t)}, nil
}

type Milestone struct {
	Name string
}

type Config struct {
	Repos      []string
	Labels     []github.Label
	Milestones []Milestone
}

func setupClient() *github.Client {
	token, _ := Token(os.Getenv("GITHUB_TOKEN")).Token()
	var conf oauth.Config
	client := conf.Client(nil, token)

	return github.NewClient(client)
}

func main() {
	var conf Config
	filecontents, err := ioutil.ReadFile("config.yml.example")
	if err != nil {
		panic(err)
	}
	yaml.Unmarshal(filecontents, &conf)
	fmt.Printf("%#v\n", &conf)

	// retrieve information about the repos we're gonna manage
}
<|endoftext|>


<commit_before>package userd_k8s

import (
	"context"
	"errors"
	"fmt"
	"sync"

	"google.golang.org/grpc"
	empty "google.golang.org/protobuf/types/known/emptypb"
	corev1 "k8s.io/api/core/v1"
	k8err "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/client-go/discovery"

	"github.com/datawire/ambassador/pkg/kates"
	"github.com/datawire/dlib/dlog"
	"github.com/telepresenceio/telepresence/rpc/v2/daemon"
	"github.com/telepresenceio/telepresence/v2/pkg/client"
	"github.com/telepresenceio/telepresence/v2/pkg/client/actions"
)

type nameMeta struct {
	Name string `json:"name"`
}

type objName struct {
	nameMeta `json:"metadata"`
}

type Callbacks struct {
	SetDNSSearchPath func(ctx context.Context, in *daemon.Paths, opts ...grpc.CallOption) (*empty.Empty, error)
}

// k8sCluster is a Kubernetes cluster reference
type Cluster struct {
	*Config
	mappedNamespaces []string

	// Main
	client    *kates.Client
	callbacks Callbacks

	lastNamespaces []string

	// Currently intercepted namespaces by remote intercepts
	interceptedNamespaces map[string]struct{}

	// Currently intercepted namespaces by local intercepts
	localInterceptedNamespaces map[string]struct{}

	accLock         sync.Mutex
	accWait         chan struct{}
	LocalIntercepts map[string]string

	// Current Namespace snapshot, get set by acc.Update().
	curSnapshot struct {
		Namespaces []*objName
	}
}

func (kc *Cluster) ActualNamespace(namespace string) string {
	if namespace == "" {
		namespace = kc.Namespace
	}
	if !kc.namespaceExists(namespace) {
		namespace = ""
	}
	return namespace
}

// check uses a non-caching DiscoveryClientConfig to retrieve the server version
func (kc *Cluster) check(c context.Context) error {
	// The discover client is using context.TODO() so the timeout specified in our
	// context has no effect.
	errCh := make(chan error)
	go func() {
		dc, err := discovery.NewDiscoveryClientForConfig(kc.config)
		if err != nil {
			errCh <- err
			return
		}
		info, err := dc.ServerVersion()
		if err != nil {
			errCh <- err
			return
		}
		dlog.Infof(c, "Server version %s", info.GitVersion)
		close(errCh)
	}()

	select {
	case <-c.Done():
	case err := <-errCh:
		if err == nil {
			return nil
		}
		if c.Err() == nil {
			return fmt.Errorf("initial cluster check failed: %w", client.RunError(err))
		}
	}
	return c.Err()
}

// Deployments returns all deployments found in the given Namespace
func (kc *Cluster) Deployments(c context.Context, namespace string) ([]kates.Object, error) {
	var deployments []*kates.Deployment
	if err := kc.client.List(c, kates.Query{Kind: "Deployment", Namespace: namespace}, &deployments); err != nil {
		return nil, err
	}
	objs := make([]kates.Object, len(deployments))
	for i, dep := range deployments {
		objs[i] = dep
	}
	return objs, nil
}

// ReplicaSets returns all replica sets found in the given Namespace
func (kc *Cluster) ReplicaSets(c context.Context, namespace string) ([]kates.Object, error) {
	var replicaSets []*kates.ReplicaSet
	if err := kc.client.List(c, kates.Query{Kind: "ReplicaSet", Namespace: namespace}, &replicaSets); err != nil {
		return nil, err
	}
	objs := make([]kates.Object, len(replicaSets))
	for i, rs := range replicaSets {
		objs[i] = rs
	}
	return objs, nil
}

// StatefulSets returns all stateful sets found in the given Namespace
func (kc *Cluster) StatefulSets(c context.Context, namespace string) ([]kates.Object, error) {
	var statefulSets []*kates.StatefulSet
	if err := kc.client.List(c, kates.Query{Kind: "StatefulSet", Namespace: namespace}, &statefulSets); err != nil {
		return nil, err
	}
	objs := make([]kates.Object, len(statefulSets))
	for i, ss := range statefulSets {
		objs[i] = ss
	}
	return objs, nil
}

// Pods returns all pods found in the given Namespace
func (kc *Cluster) Pods(c context.Context, namespace string) ([]*kates.Pod, error) {
	var pods []*kates.Pod
	if err := kc.client.List(c, kates.Query{Kind: "Pod", Namespace: namespace}, &pods); err != nil {
		return nil, err
	}
	return pods, nil
}

// FindDeployment returns a deployment with the given name in the given namespace or nil
// if no such deployment could be found.
func (kc *Cluster) FindDeployment(c context.Context, namespace, name string) (*kates.Deployment, error) {
	dep := &kates.Deployment{
		TypeMeta:   kates.TypeMeta{Kind: "Deployment"},
		ObjectMeta: kates.ObjectMeta{Name: name, Namespace: namespace},
	}
	if err := kc.client.Get(c, dep, dep); err != nil {
		return nil, err
	}
	return dep, nil
}

// FindAgain returns a fresh version of the given object.
func (kc *Cluster) FindAgain(c context.Context, obj kates.Object) (kates.Object, error) {
	if err := kc.client.Get(c, obj, obj); err != nil {
		return nil, err
	}
	return obj, nil
}

// FindPodFromDeployName returns a pod with the given name-hex-hex
func (kc *Cluster) FindPodFromSelector(c context.Context, namespace string, selector map[string]string) (*kates.Pod, error) {
	pods, err := kc.Pods(c, namespace)
	if err != nil {
		return nil, err
	}

	for _, pod := range pods {
		podLabels := pod.GetLabels()
		// check if selector is in labels
		for key, val := range selector {
			if podLabels[key] == val {
				return pod, nil
			}
		}
	}

	return nil, errors.New("pod not found")
}

// FindPod returns a pod with the given name in the given namespace or nil
// if no such replica set could be found.
func (kc *Cluster) FindPod(c context.Context, namespace, name string) (*kates.Pod, error) {
	pod := &kates.Pod{
		TypeMeta:   kates.TypeMeta{Kind: "Pod"},
		ObjectMeta: kates.ObjectMeta{Name: name, Namespace: namespace},
	}
	if err := kc.client.Get(c, pod, pod); err != nil {
		return nil, err
	}
	return pod, nil
}

// FindWorkload returns a workload for the given name and namespace. We
// search in a specific order based on how we prefer workload objects:
// 1. Deployments
// 2. ReplicaSets
// 3. StatefulSets
// And return the kind as soon as we find one that matches
func (kc *Cluster) FindWorkload(c context.Context, namespace, name string) (kates.Object, error) {
	type workLoad struct {
		kind string
		obj  kates.Object
	}
	for _, wl := range []workLoad{{"Deployment", &kates.Deployment{}}, {"ReplicaSet", &kates.ReplicaSet{}}, {"StatefulSet", &kates.StatefulSet{}}} {
		wl.obj.(schema.ObjectKind).SetGroupVersionKind(schema.GroupVersionKind{Kind: wl.kind})
		wl.obj.SetName(name)
		wl.obj.SetNamespace(namespace)
		if err := kc.client.Get(c, wl.obj, wl.obj); err != nil {
			if kates.IsNotFound(err) {
				continue
			}
			return nil, err
		}
		return wl.obj, nil
	}
	return nil, k8err.NewNotFound(corev1.Resource("workload"), name+"."+namespace)
}

// FindSvc finds a service with the given name in the given Namespace and returns
// either a copy of that service or nil if no such service could be found.
func (kc *Cluster) FindSvc(c context.Context, namespace, name string) (*kates.Service, error) {
	rs := &kates.Service{
		TypeMeta:   kates.TypeMeta{Kind: "Service"},
		ObjectMeta: kates.ObjectMeta{Name: name, Namespace: namespace},
	}
	if err := kc.client.Get(c, rs, rs); err != nil {
		return nil, err
	}
	return rs, nil
}

// findAllSvc finds services with the given service type in all namespaces of the cluster returns
// a slice containing a copy of those services.
func (kc *Cluster) findAllSvcByType(c context.Context, svcType corev1.ServiceType) ([]*kates.Service, error) {
	// NOTE: This is expensive in terms of bandwidth on a large cluster. We currently only use this
	// to retrieve ingress info and that task could be moved to the traffic-manager instead.
	var svcs []*kates.Service
	if err := kc.client.List(c, kates.Query{Kind: "Service"}, &svcs); err != nil {
		return nil, err
	}
	var typedSvcs []*kates.Service
	for _, svc := range svcs {
		if svc.Spec.Type == svcType {
			typedSvcs = append(typedSvcs, svc)
			break
		}
	}
	return typedSvcs, nil
}

func (kc *Cluster) namespaceExists(namespace string) (exists bool) {
	kc.accLock.Lock()
	for _, n := range kc.lastNamespaces {
		if n == namespace {
			exists = true
			break
		}
	}
	kc.accLock.Unlock()
	return exists
}

func NewCluster(c context.Context, kubeFlags *Config, mappedNamespaces []string, callbacks Callbacks) (*Cluster, error) {
	// TODO: Add constructor to kates that takes an additional restConfig argument to prevent that kates recreates it.
	kc, err := kates.NewClientFromConfigFlags(kubeFlags.ConfigFlags)
	if err != nil {
		return nil, client.CheckTimeout(c, fmt.Errorf("k8s client create failed: %w", err))
	}

	ret := &Cluster{
		Config:           kubeFlags,
		mappedNamespaces: mappedNamespaces,
		client:           kc,
		callbacks:        callbacks,
		LocalIntercepts:  map[string]string{},
		accWait:          make(chan struct{}),
	}

	if err := ret.check(c); err != nil {
		return nil, err
	}

	dlog.Infof(c, "Context: %s", ret.Context)
	dlog.Infof(c, "Server: %s", ret.Server)

	return ret, nil
}

func (kc *Cluster) WaitUntilReady(ctx context.Context) error {
	select {
	case <-kc.accWait:
		return nil
	case <-ctx.Done():
		return ctx.Err()
	}
}

func (kc *Cluster) GetClusterId(ctx context.Context) string {
	clusterID, _ := actions.GetClusterID(ctx, kc.client)
	return clusterID
}

func (kc *Cluster) Client() *kates.Client {
	return kc.client
}

func (kc *Cluster) GetManagerNamespace() string {
	return kc.kubeconfigExtension.Manager.Namespace
}
<commit_msg>check all labels in selector<commit_after>package userd_k8s

import (
	"context"
	"errors"
	"fmt"
	"sync"

	"google.golang.org/grpc"
	empty "google.golang.org/protobuf/types/known/emptypb"
	corev1 "k8s.io/api/core/v1"
	k8err "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/client-go/discovery"

	"github.com/datawire/ambassador/pkg/kates"
	"github.com/datawire/dlib/dlog"
	"github.com/telepresenceio/telepresence/rpc/v2/daemon"
	"github.com/telepresenceio/telepresence/v2/pkg/client"
	"github.com/telepresenceio/telepresence/v2/pkg/client/actions"
)

type nameMeta struct {
	Name string `json:"name"`
}

type objName struct {
	nameMeta `json:"metadata"`
}

type Callbacks struct {
	SetDNSSearchPath func(ctx context.Context, in *daemon.Paths, opts ...grpc.CallOption) (*empty.Empty, error)
}

// k8sCluster is a Kubernetes cluster reference
type Cluster struct {
	*Config
	mappedNamespaces []string

	// Main
	client    *kates.Client
	callbacks Callbacks

	lastNamespaces []string

	// Currently intercepted namespaces by remote intercepts
	interceptedNamespaces map[string]struct{}

	// Currently intercepted namespaces by local intercepts
	localInterceptedNamespaces map[string]struct{}

	accLock         sync.Mutex
	accWait         chan struct{}
	LocalIntercepts map[string]string

	// Current Namespace snapshot, get set by acc.Update().
	curSnapshot struct {
		Namespaces []*objName
	}
}

func (kc *Cluster) ActualNamespace(namespace string) string {
	if namespace == "" {
		namespace = kc.Namespace
	}
	if !kc.namespaceExists(namespace) {
		namespace = ""
	}
	return namespace
}

// check uses a non-caching DiscoveryClientConfig to retrieve the server version
func (kc *Cluster) check(c context.Context) error {
	// The discover client is using context.TODO() so the timeout specified in our
	// context has no effect.
	errCh := make(chan error)
	go func() {
		dc, err := discovery.NewDiscoveryClientForConfig(kc.config)
		if err != nil {
			errCh <- err
			return
		}
		info, err := dc.ServerVersion()
		if err != nil {
			errCh <- err
			return
		}
		dlog.Infof(c, "Server version %s", info.GitVersion)
		close(errCh)
	}()

	select {
	case <-c.Done():
	case err := <-errCh:
		if err == nil {
			return nil
		}
		if c.Err() == nil {
			return fmt.Errorf("initial cluster check failed: %w", client.RunError(err))
		}
	}
	return c.Err()
}

// Deployments returns all deployments found in the given Namespace
func (kc *Cluster) Deployments(c context.Context, namespace string) ([]kates.Object, error) {
	var deployments []*kates.Deployment
	if err := kc.client.List(c, kates.Query{Kind: "Deployment", Namespace: namespace}, &deployments); err != nil {
		return nil, err
	}
	objs := make([]kates.Object, len(deployments))
	for i, dep := range deployments {
		objs[i] = dep
	}
	return objs, nil
}

// ReplicaSets returns all replica sets found in the given Namespace
func (kc *Cluster) ReplicaSets(c context.Context, namespace string) ([]kates.Object, error) {
	var replicaSets []*kates.ReplicaSet
	if err := kc.client.List(c, kates.Query{Kind: "ReplicaSet", Namespace: namespace}, &replicaSets); err != nil {
		return nil, err
	}
	objs := make([]kates.Object, len(replicaSets))
	for i, rs := range replicaSets {
		objs[i] = rs
	}
	return objs, nil
}

// StatefulSets returns all stateful sets found in the given Namespace
func (kc *Cluster) StatefulSets(c context.Context, namespace string) ([]kates.Object, error) {
	var statefulSets []*kates.StatefulSet
	if err := kc.client.List(c, kates.Query{Kind: "StatefulSet", Namespace: namespace}, &statefulSets); err != nil {
		return nil, err
	}
	objs := make([]kates.Object, len(statefulSets))
	for i, ss := range statefulSets {
		objs[i] = ss
	}
	return objs, nil
}

// Pods returns all pods found in the given Namespace
func (kc *Cluster) Pods(c context.Context, namespace string) ([]*kates.Pod, error) {
	var pods []*kates.Pod
	if err := kc.client.List(c, kates.Query{Kind: "Pod", Namespace: namespace}, &pods); err != nil {
		return nil, err
	}
	return pods, nil
}

// FindDeployment returns a deployment with the given name in the given namespace or nil
// if no such deployment could be found.
func (kc *Cluster) FindDeployment(c context.Context, namespace, name string) (*kates.Deployment, error) {
	dep := &kates.Deployment{
		TypeMeta:   kates.TypeMeta{Kind: "Deployment"},
		ObjectMeta: kates.ObjectMeta{Name: name, Namespace: namespace},
	}
	if err := kc.client.Get(c, dep, dep); err != nil {
		return nil, err
	}
	return dep, nil
}

// FindAgain returns a fresh version of the given object.
func (kc *Cluster) FindAgain(c context.Context, obj kates.Object) (kates.Object, error) {
	if err := kc.client.Get(c, obj, obj); err != nil {
		return nil, err
	}
	return obj, nil
}

// FindPodFromDeployName returns a pod with the given name-hex-hex
func (kc *Cluster) FindPodFromSelector(c context.Context, namespace string, selector map[string]string) (*kates.Pod, error) {
	pods, err := kc.Pods(c, namespace)
	if err != nil {
		return nil, err
	}

	for _, pod := range pods {
		podLabels := pod.GetLabels()
		match := true
		// check if selector is in labels
		for key, val := range selector {
			if podLabels[key] != val {
				match = false
			}
		}
		if match {
			return pod, nil
		}
	}

	return nil, errors.New("pod not found")
}

// FindPod returns a pod with the given name in the given namespace or nil
// if no such replica set could be found.
func (kc *Cluster) FindPod(c context.Context, namespace, name string) (*kates.Pod, error) {
	pod := &kates.Pod{
		TypeMeta:   kates.TypeMeta{Kind: "Pod"},
		ObjectMeta: kates.ObjectMeta{Name: name, Namespace: namespace},
	}
	if err := kc.client.Get(c, pod, pod); err != nil {
		return nil, err
	}
	return pod, nil
}

// FindWorkload returns a workload for the given name and namespace. We
// search in a specific order based on how we prefer workload objects:
// 1. Deployments
// 2. ReplicaSets
// 3. StatefulSets
// And return the kind as soon as we find one that matches
func (kc *Cluster) FindWorkload(c context.Context, namespace, name string) (kates.Object, error) {
	type workLoad struct {
		kind string
		obj  kates.Object
	}
	for _, wl := range []workLoad{{"Deployment", &kates.Deployment{}}, {"ReplicaSet", &kates.ReplicaSet{}}, {"StatefulSet", &kates.StatefulSet{}}} {
		wl.obj.(schema.ObjectKind).SetGroupVersionKind(schema.GroupVersionKind{Kind: wl.kind})
		wl.obj.SetName(name)
		wl.obj.SetNamespace(namespace)
		if err := kc.client.Get(c, wl.obj, wl.obj); err != nil {
			if kates.IsNotFound(err) {
				continue
			}
			return nil, err
		}
		return wl.obj, nil
	}
	return nil, k8err.NewNotFound(corev1.Resource("workload"), name+"."+namespace)
}

// FindSvc finds a service with the given name in the given Namespace and returns
// either a copy of that service or nil if no such service could be found.
func (kc *Cluster) FindSvc(c context.Context, namespace, name string) (*kates.Service, error) {
	rs := &kates.Service{
		TypeMeta:   kates.TypeMeta{Kind: "Service"},
		ObjectMeta: kates.ObjectMeta{Name: name, Namespace: namespace},
	}
	if err := kc.client.Get(c, rs, rs); err != nil {
		return nil, err
	}
	return rs, nil
}

// findAllSvc finds services with the given service type in all namespaces of the cluster returns
// a slice containing a copy of those services.
func (kc *Cluster) findAllSvcByType(c context.Context, svcType corev1.ServiceType) ([]*kates.Service, error) {
	// NOTE: This is expensive in terms of bandwidth on a large cluster. We currently only use this
	// to retrieve ingress info and that task could be moved to the traffic-manager instead.
	var svcs []*kates.Service
	if err := kc.client.List(c, kates.Query{Kind: "Service"}, &svcs); err != nil {
		return nil, err
	}
	var typedSvcs []*kates.Service
	for _, svc := range svcs {
		if svc.Spec.Type == svcType {
			typedSvcs = append(typedSvcs, svc)
			break
		}
	}
	return typedSvcs, nil
}

func (kc *Cluster) namespaceExists(namespace string) (exists bool) {
	kc.accLock.Lock()
	for _, n := range kc.lastNamespaces {
		if n == namespace {
			exists = true
			break
		}
	}
	kc.accLock.Unlock()
	return exists
}

func NewCluster(c context.Context, kubeFlags *Config, mappedNamespaces []string, callbacks Callbacks) (*Cluster, error) {
	// TODO: Add constructor to kates that takes an additional restConfig argument to prevent that kates recreates it.
	kc, err := kates.NewClientFromConfigFlags(kubeFlags.ConfigFlags)
	if err != nil {
		return nil, client.CheckTimeout(c, fmt.Errorf("k8s client create failed: %w", err))
	}

	ret := &Cluster{
		Config:           kubeFlags,
		mappedNamespaces: mappedNamespaces,
		client:           kc,
		callbacks:        callbacks,
		LocalIntercepts:  map[string]string{},
		accWait:          make(chan struct{}),
	}

	if err := ret.check(c); err != nil {
		return nil, err
	}

	dlog.Infof(c, "Context: %s", ret.Context)
	dlog.Infof(c, "Server: %s", ret.Server)

	return ret, nil
}

func (kc *Cluster) WaitUntilReady(ctx context.Context) error {
	select {
	case <-kc.accWait:
		return nil
	case <-ctx.Done():
		return ctx.Err()
	}
}

func (kc *Cluster) GetClusterId(ctx context.Context) string {
	clusterID, _ := actions.GetClusterID(ctx, kc.client)
	return clusterID
}

func (kc *Cluster) Client() *kates.Client {
	return kc.client
}

func (kc *Cluster) GetManagerNamespace() string {
	return kc.kubeconfigExtension.Manager.Namespace
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"net/http"
	"os"
	"os/signal"
	"syscall"

	"golang.org/x/net/context"

	"github.com/go-kit/kit/log"
	"github.com/microservices-demo/user/api"
	"github.com/microservices-demo/user/db"
	"github.com/microservices-demo/user/db/mongodb"
)

var dev bool
var verbose bool
var port string
var acc string

func init() {
	flag.StringVar(&port, "port", "8084", "Port on which to run")
	flag.BoolVar(&verbose, "verbose", false, "Verbose logging")
	db.Register("mongodb", &mongodb.Mongo{})
}

func main() {

	flag.Parse()
	// Mechanical stuff.
	errc := make(chan error)
	ctx := context.Background()

	// Log domain.
	var logger log.Logger
	{
		logger = log.NewLogfmtLogger(os.Stderr)
		logger = log.NewContext(logger).With("ts", log.DefaultTimestampUTC)
		logger = log.NewContext(logger).With("caller", log.DefaultCaller)
	}
	err := db.Init()
	if err != nil {
		logger.Log(err)
		os.Exit(1)
	}

	// Service domain.
	var service api.Service
	{
		service = api.NewFixedService()
		service = api.LoggingMiddleware(logger)(service)
	}

	// Endpoint domain.
	endpoints := api.MakeEndpoints(service)

	// Create and launch the HTTP server.
	go func() {
		logger.Log("transport", "HTTP", "port", port)
		handler := api.MakeHTTPHandler(ctx, endpoints, logger)
		errc <- http.ListenAndServe(fmt.Sprintf(":%v", port), handler)
	}()

	// Capture interrupts.
	go func() {
		c := make(chan os.Signal)
		signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
		errc <- fmt.Errorf("%s", <-c)
	}()

	logger.Log("exit", <-errc)
}
<commit_msg>removed verbose flag<commit_after>package main

import (
	"flag"
	"fmt"
	"net/http"
	"os"
	"os/signal"
	"syscall"

	"golang.org/x/net/context"

	"github.com/go-kit/kit/log"
	"github.com/microservices-demo/user/api"
	"github.com/microservices-demo/user/db"
	"github.com/microservices-demo/user/db/mongodb"
)

var dev bool
var port string
var acc string

func init() {
	flag.StringVar(&port, "port", "8084", "Port on which to run")
	db.Register("mongodb", &mongodb.Mongo{})
}

func main() {

	flag.Parse()
	// Mechanical stuff.
	errc := make(chan error)
	ctx := context.Background()

	// Log domain.
	var logger log.Logger
	{
		logger = log.NewLogfmtLogger(os.Stderr)
		logger = log.NewContext(logger).With("ts", log.DefaultTimestampUTC)
		logger = log.NewContext(logger).With("caller", log.DefaultCaller)
	}
	err := db.Init()
	if err != nil {
		logger.Log(err)
		os.Exit(1)
	}

	// Service domain.
	var service api.Service
	{
		service = api.NewFixedService()
		service = api.LoggingMiddleware(logger)(service)
	}

	// Endpoint domain.
	endpoints := api.MakeEndpoints(service)

	// Create and launch the HTTP server.
	go func() {
		logger.Log("transport", "HTTP", "port", port)
		handler := api.MakeHTTPHandler(ctx, endpoints, logger)
		errc <- http.ListenAndServe(fmt.Sprintf(":%v", port), handler)
	}()

	// Capture interrupts.
	go func() {
		c := make(chan os.Signal)
		signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
		errc <- fmt.Errorf("%s", <-c)
	}()

	logger.Log("exit", <-errc)
}
<|endoftext|>


<commit_before>package main

import (
	"intro2alg/datastructs"
	"intro2alg/geom"
	"intro2alg/sorts"
	"math/rand"
)

func main() {
	sorts.Tests()
	geom.Tests()
	datastructs.Tests()
}

func randomString(l int) string {
	bytes := make([]byte, l)
	for i := 0; i < l; i++ {
		bytes[i] = byte(randInt(65, 90))
	}
	return string(bytes)
}

func randInt(min int, max int) int {
	return min + rand.Intn(max-min)
}
<commit_msg>add base folder<commit_after>// run sample data for test
package main

import (
	"zjf/intro2alg/datastructs"
	"zjf/intro2alg/geom"
	"zjf/intro2alg/sorts"
	"math/rand"
)

func main() {
	sorts.Tests()
	geom.Tests()
	datastructs.Tests()
}

func randomString(l int) string {
	bytes := make([]byte, l)
	for i := 0; i < l; i++ {
		bytes[i] = byte(randInt(65, 90))
	}
	return string(bytes)
}

func randInt(min int, max int) int {
	return min + rand.Intn(max-min)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"github.com/asaskevich/govalidator"
	"github.com/codegangsta/negroni"
	"github.com/gorilla/mux"
	"github.com/unrolled/render"
	"gopkg.in/redis.v3"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"runtime"
)

type Logger struct {
	Debug *log.Logger
	Info  *log.Logger
	Warn  *log.Logger
	Error *log.Logger
}

func newLogger() *Logger {

	flag := log.Ldate | log.Ltime | log.Lshortfile

	return &Logger{
		log.New(os.Stdout, "DEBUG: ", flag),
		log.New(os.Stdout, "INFO: ", flag),
		log.New(os.Stdout, "WARN: ", flag),
		log.New(os.Stderr, "ERROR: ", flag),
	}
}

func (l *Logger) WriteErr(w http.ResponseWriter, err error) {
	_, fn, line, _ := runtime.Caller(1)
	l.Error.Printf("%s:%d:%v", fn, line, err)
	http.Error(w, "Sorry, an error has occurred", http.StatusInternalServerError)
}

type Movie struct {
	Title    string
	Actors   string
	Poster   string
	Year     string
	Plot     string
	Director string
	Rating   string `json:"imdbRating"`
	ImdbID   string `json:"imdbID"`
}

func (m *Movie) String() string {
	return m.Title
}

func (m *Movie) MarshalBinary() ([]byte, error) {
	return json.Marshal(m)
}

func (m *Movie) UnmarshalBinary(data []byte) error {
	return json.Unmarshal(data, m)
}

func (m *Movie) Save(db *redis.Client) error {
	return db.Set(m.ImdbID, m, 0).Err()
}

type MovieForm struct {
	Title string `valid:"required"`
}

func (f *MovieForm) Decode(r *http.Request) error {
	return decode(r, f)
}

type Error interface {
	error
	Status() int
}

type HTTPError struct {
	Code int
	Err  error
}

func (e HTTPError) Error() string {
	return e.Err.Error()
}

func (e HTTPError) Status() int {
	return e.Code
}

var errHTTPNotFound = HTTPError{http.StatusNotFound, errors.New("Not found")}

type Env struct {
	*render.Render
	DB  *redis.Client
	Log *Logger
}

type HandlerFunc func(e *Env, w http.ResponseWriter, r *http.Request) error

type Handler struct {
	*Env
	H HandlerFunc
}

func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	err := h.H(h.Env, w, r)
	if err != nil {
		switch e := err.(type) {
		case Error:
			h.Env.Log.Error.Printf("HTTP %d: %s", e.Status(), e)
			http.Error(w, e.Error(), e.Status())
		default:
			_, fn, line, _ := runtime.Caller(1)
			h.Env.Log.Error.Printf("%s:%d:%v", fn, line, err)
			http.Error(w, "Sorry, an error occurred", http.StatusInternalServerError)
		}
	}
}

// decodes JSON body of request and runs through validator
func decode(r *http.Request, data interface{}) error {
	defer r.Body.Close()
	if err := json.NewDecoder(r.Body).Decode(data); err != nil {
		return err
	}
	if _, err := govalidator.ValidateStruct(data); err != nil {
		return err
	}
	return nil
}

func getMovieFromOMDB(title string) (*Movie, error) {

	u, _ := url.Parse("http://omdbapi.com")

	q := u.Query()
	q.Set("t", title)

	u.RawQuery = q.Encode()

	resp, err := http.Get(u.String())
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	movie := &Movie{}
	if err := json.Unmarshal(body, movie); err != nil {
		return nil, err
	}

	return movie, nil
}

func getRandomMovie(db *redis.Client) (*Movie, error) {
	imdbID, err := db.RandomKey().Result()
	if err == redis.Nil {
		return nil, nil
	}
	if err != nil {
		return nil, err
	}
	return getMovie(db, imdbID)
}

func getMovie(db *redis.Client, imdbID string) (*Movie, error) {
	movie := &Movie{}
	if err := db.Get(imdbID).Scan(movie); err != nil {
		if err == redis.Nil {
			return nil, nil
		} else {
			return nil, err
		}
	}
	return movie, nil
}

func getMovies(db *redis.Client) ([]*Movie, error) {
	result := db.Keys("*")
	if err := result.Err(); err != nil {
		return nil, err
	}
	var movies []*Movie
	for _, imdbID := range result.Val() {
		movie := &Movie{}
		if err := db.Get(imdbID).Scan(movie); err == nil {
			movies = append(movies, movie)
		}
	}
	return movies, nil
}

var (
	env  = flag.String("env", "prod", "environment ('prod' or 'dev')")
	port = flag.String("port", "4000", "server port")
)

const (
	staticURL    = "/static/"
	staticDir    = "./dist/"
	devServerURL = "http://localhost:8080"
	redisAddr    = "localhost:6379"
)

func main() {

	flag.Parse()

	db := redis.NewClient(&redis.Options{
		Addr:     redisAddr,
		Password: "",
		DB:       0,
	})

	_, err := db.Ping().Result()
	if err != nil {
		panic(err)
	}

	router := mux.NewRouter()
	render := render.New()
	logger := newLogger()

	// static content
	router.PathPrefix(
		staticURL).Handler(http.StripPrefix(
		staticURL, http.FileServer(http.Dir(staticDir))))

	// index page
	router.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		var staticHost string
		if *env == "dev" {
			staticHost = devServerURL
		}

		ctx := map[string]string{
			"staticHost": staticHost,
			"env":        *env,
		}

		render.HTML(w, http.StatusOK, "index", ctx)
	})

	// API calls
	api := router.PathPrefix("/api/").Subrouter()

	e := &Env{render, db, logger}

	api.Handle("/", Handler{e, func(e *Env, w http.ResponseWriter, r *http.Request) error {
		movie, err := getRandomMovie(e.DB)
		if err != nil {
			return err
		}
		if movie == nil {
			return errHTTPNotFound
		}
		e.JSON(w, http.StatusOK, movie)
		return nil
	}}).Methods("GET")

	api.Handle("/movie/{id}", Handler{e, func(e *Env, w http.ResponseWriter, r *http.Request) error {
		movie, err := getMovie(e.DB, mux.Vars(r)["id"])
		if err != nil {
			return err
		}
		if movie == nil {
			return errHTTPNotFound
		}
		e.JSON(w, http.StatusOK, movie)
		return nil
	}}).Methods("GET")

	api.Handle("/movie/{id}", Handler{e, func(e *Env, w http.ResponseWriter, r *http.Request) error {
		if err := db.Del(mux.Vars(r)["id"]).Err(); err != nil {
			return err
		}
		e.Text(w, http.StatusOK, "Movie deleted")
		return nil
	}}).Methods("DELETE")

	api.Handle("/all/", Handler{e, func(e *Env, w http.ResponseWriter, r *http.Request) error {
		movies, err := getMovies(e.DB)
		if err != nil {
			return err
		}
		e.JSON(w, http.StatusOK, movies)
		return nil
	}}).Methods("GET")

	api.Handle("/", Handler{e, func(e *Env, w http.ResponseWriter, r *http.Request) error {

		f := &MovieForm{}
		if err := f.Decode(r); err != nil {
			return HTTPError{http.StatusBadRequest, err}
		}

		movie, err := getMovieFromOMDB(f.Title)
		if err != nil {
			return err
		}

		if movie.ImdbID == "" {
			return errHTTPNotFound
		}

		if err := movie.Save(e.DB); err != nil {
			return err
		}
		logger.Info.Printf("New movie %s added", movie)
		e.JSON(w, http.StatusOK, movie)
		return nil

	}}).Methods("POST")

	n := negroni.Classic()
	n.UseHandler(router)
	n.Run(":" + *port)

}
<commit_msg>Go tidy up<commit_after>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"github.com/asaskevich/govalidator"
	"github.com/codegangsta/negroni"
	"github.com/gorilla/mux"
	"github.com/unrolled/render"
	"gopkg.in/redis.v3"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"runtime"
)

type Logger struct {
	Debug *log.Logger
	Info  *log.Logger
	Warn  *log.Logger
	Error *log.Logger
}

func newLogger() *Logger {

	flag := log.Ldate | log.Ltime | log.Lshortfile

	return &Logger{
		log.New(os.Stdout, "DEBUG: ", flag),
		log.New(os.Stdout, "INFO: ", flag),
		log.New(os.Stdout, "WARN: ", flag),
		log.New(os.Stderr, "ERROR: ", flag),
	}
}

func getRandomMovie(e *Env, w http.ResponseWriter, r *http.Request) error {
	movie, err := e.DB.GetRandomMovie()
	if err != nil {
		return err
	}
	if movie == nil {
		return errHTTPNotFound
	}
	e.JSON(w, http.StatusOK, movie)
	return nil
}

func getMovie(e *Env, w http.ResponseWriter, r *http.Request) error {
	movie, err := e.DB.GetMovie(mux.Vars(r)["id"])
	if err != nil {
		return err
	}
	if movie == nil {
		return errHTTPNotFound
	}
	e.JSON(w, http.StatusOK, movie)
	return nil
}

func deleteMovie(e *Env, w http.ResponseWriter, r *http.Request) error {
	if err := e.DB.Del(mux.Vars(r)["id"]).Err(); err != nil {
		return err
	}
	e.Text(w, http.StatusOK, "Movie deleted")
	return nil
}

func getMovies(e *Env, w http.ResponseWriter, r *http.Request) error {
	movies, err := e.DB.GetMovies()
	if err != nil {
		return err
	}
	e.JSON(w, http.StatusOK, movies)
	return nil
}

func addMovie(e *Env, w http.ResponseWriter, r *http.Request) error {

	f := &MovieForm{}
	if err := f.Decode(r); err != nil {
		return HTTPError{http.StatusBadRequest, err}
	}

	movie, err := getMovieFromOMDB(f.Title)
	if err != nil {
		return err
	}

	if movie.ImdbID == "" {
		return errHTTPNotFound
	}

	if err := movie.Save(e.DB); err != nil {
		return err
	}
	e.Log.Info.Printf("New movie %s added", movie)
	e.JSON(w, http.StatusOK, movie)
	return nil

}

type Movie struct {
	Title    string
	Actors   string
	Poster   string
	Year     string
	Plot     string
	Director string
	Rating   string `json:"imdbRating"`
	ImdbID   string `json:"imdbID"`
}

func (m *Movie) String() string {
	return m.Title
}

func (m *Movie) MarshalBinary() ([]byte, error) {
	return json.Marshal(m)
}

func (m *Movie) UnmarshalBinary(data []byte) error {
	return json.Unmarshal(data, m)
}

func (m *Movie) Save(db *DB) error {
	return db.Set(m.ImdbID, m, 0).Err()
}

type MovieForm struct {
	Title string `valid:"required"`
}

func (f *MovieForm) Decode(r *http.Request) error {
	return decode(r, f)
}

type Error interface {
	error
	Status() int
}

type HTTPError struct {
	Code int
	Err  error
}

func (e HTTPError) Error() string {
	return e.Err.Error()
}

func (e HTTPError) Status() int {
	return e.Code
}

var errHTTPNotFound = HTTPError{http.StatusNotFound, errors.New("Not found")}

type Env struct {
	*render.Render
	DB  *DB
	Log *Logger
}

func (e *Env) Handler(fn HandlerFunc) Handler {
	return Handler{e, fn}
}

type HandlerFunc func(e *Env, w http.ResponseWriter, r *http.Request) error

type Handler struct {
	*Env
	H HandlerFunc
}

func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	err := h.H(h.Env, w, r)
	if err != nil {
		switch e := err.(type) {
		case Error:
			h.Env.Log.Error.Printf("HTTP %d: %s", e.Status(), e)
			http.Error(w, e.Error(), e.Status())
		default:
			_, fn, line, _ := runtime.Caller(1)
			h.Env.Log.Error.Printf("%s:%d:%v", fn, line, err)
			http.Error(w, "Sorry, an error occurred", http.StatusInternalServerError)
		}
	}
}

// decodes JSON body of request and runs through validator
func decode(r *http.Request, data interface{}) error {
	defer r.Body.Close()
	if err := json.NewDecoder(r.Body).Decode(data); err != nil {
		return err
	}
	if _, err := govalidator.ValidateStruct(data); err != nil {
		return err
	}
	return nil
}

func getMovieFromOMDB(title string) (*Movie, error) {

	u, _ := url.Parse("http://omdbapi.com")

	q := u.Query()
	q.Set("t", title)

	u.RawQuery = q.Encode()

	resp, err := http.Get(u.String())
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	movie := &Movie{}
	if err := json.Unmarshal(body, movie); err != nil {
		return nil, err
	}

	return movie, nil
}

type DB struct {
	*redis.Client
}

func (db *DB) GetRandomMovie() (*Movie, error) {
	imdbID, err := db.RandomKey().Result()
	if err == redis.Nil {
		return nil, nil
	}
	if err != nil {
		return nil, err
	}
	return db.GetMovie(imdbID)
}

func (db *DB) GetMovie(imdbID string) (*Movie, error) {
	movie := &Movie{}
	if err := db.Get(imdbID).Scan(movie); err != nil {
		if err == redis.Nil {
			return nil, nil
		} else {
			return nil, err
		}
	}
	return movie, nil
}

func (db *DB) GetMovies() ([]*Movie, error) {
	result := db.Keys("*")
	if err := result.Err(); err != nil {
		return nil, err
	}
	var movies []*Movie
	for _, imdbID := range result.Val() {
		movie := &Movie{}
		if err := db.Get(imdbID).Scan(movie); err == nil {
			movies = append(movies, movie)
		}
	}
	return movies, nil
}

var (
	env  = flag.String("env", "prod", "environment ('prod' or 'dev')")
	port = flag.String("port", "4000", "server port")
)

const (
	staticURL    = "/static/"
	staticDir    = "./dist/"
	devServerURL = "http://localhost:8080"
	redisAddr    = "localhost:6379"
)

func main() {

	flag.Parse()

	db := &DB{redis.NewClient(&redis.Options{
		Addr:     redisAddr,
		Password: "",
		DB:       0,
	})}

	_, err := db.Ping().Result()
	if err != nil {
		panic(err)
	}

	router := mux.NewRouter()
	render := render.New()
	logger := newLogger()

	// static content
	router.PathPrefix(
		staticURL).Handler(http.StripPrefix(
		staticURL, http.FileServer(http.Dir(staticDir))))

	// index page
	router.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		var staticHost string
		if *env == "dev" {
			staticHost = devServerURL
		}

		ctx := map[string]string{
			"staticHost": staticHost,
			"env":        *env,
		}

		render.HTML(w, http.StatusOK, "index", ctx)
	})

	// API calls
	api := router.PathPrefix("/api/").Subrouter()

	e := &Env{render, db, logger}

	api.Handle("/", e.Handler(getRandomMovie)).Methods("GET")
	api.Handle("/", e.Handler(addMovie)).Methods("POST")
	api.Handle("/movie/{id}", e.Handler(getMovie)).Methods("GET")
	api.Handle("/movie/{id}", e.Handler(deleteMovie)).Methods("DELETE")
	api.Handle("/all/", e.Handler(getMovies)).Methods("GET")

	n := negroni.Classic()
	n.UseHandler(router)
	n.Run(":" + *port)

}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"log"
	"net/http"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/inominate/apicache"

	"github.com/boltdb/bolt"
	"github.com/codegangsta/martini"
	_ "github.com/go-sql-driver/mysql"
	"github.com/inominate/session"
	"github.com/robfig/config"
)

var sesManager *session.SessionManager
var conf *config.Config
var exemptRoles []int64
var exemptChars []string

func setupMartini() *martini.ClassicMartini {
	r := martini.NewRouter()
	m := martini.New()
	m.Use(martini.Logger())
	m.Use(martini.Recovery())
	m.Use(SessionService())
	m.MapTo(r, (*martini.Routes)(nil))
	m.Action(r.Handle)
	return &martini.ClassicMartini{m, r}
}

func loadConfig() error {
	newConf, err := config.ReadDefault("purger.conf")
	if err != nil {
		return err
	}
	conf = newConf

	confStr, err := conf.String("purger", "exemptCharacters")
	if err == nil {
		exemptChars = strings.Split(strings.ToLower(confStr), ",")
		for k := range exemptChars {
			exemptChars[k] = strings.TrimSpace(exemptChars[k])
		}
	} else {
		log.Printf("No exempted characters found: %s", err)
		exemptChars = []string{}
	}

	confStr, err = conf.String("purger", "exemptRoles")
	if err == nil {
		exemptRolesStrings := strings.Split(confStr, ",")
		exemptRoles = make([]int64, len(exemptRolesStrings))
		for k, v := range exemptRolesStrings {
			v = strings.TrimSpace(v)
			if v == "" {
				continue
			}

			role, err := strconv.ParseInt(v, 10, 64)
			if err != nil || role == 0 || role < 0 {
				log.Printf("Error parsing role '%s': %s", v, err)
				continue
			}

			exemptRoles[k] = role
		}
	} else {
		log.Printf("No exempted roles found: %s", err)
		exemptRoles = []int64{}
	}

	return nil
}

var bdb *bolt.DB

func main() {
	var err error

	err = loadConfig()
	if err != nil {
		log.Fatalf("Failed to load config: %s", err)
	}

	c := conf
	keyid, err := c.Int("purger", "keyid")
	if err != nil {
		log.Fatalf("Failed to load config: %s", err)
	}
	vcode, err := c.String("purger", "vcode")
	if err != nil {
		log.Fatalf("Failed to load config: %s", err)
	}
	days, err := c.Int("purger", "maxIdleDays")
	if err != nil {
		log.Fatalf("Failed to load config: %s", err)
	}
	listen, err := c.String("purger", "listen")
	if err != nil {
		log.Fatalf("Failed to load config: %s", err)
	}
	if listen == "ENV" {
		listen = ":" + os.Getenv("PORT")
	}

	baseURL, _ := c.String("purger", "APIBaseURL")
	if baseURL == "" {
		baseURL = "https://api.eveonline.com/"
	}
	apiClient := apicache.NewClient(apicache.NilCache)
	apiClient.BaseURL = baseURL

	var memberDB *sql.DB
	var query string
	var cmt CorpMemberTracker

	memberDSN, _ := c.String("registered_characters", "DSN")
	memberURL, _ := c.String("registered_characters", "URL")
	if memberDSN != "" {
		memberDB, err = sql.Open("mysql", memberDSN)
		if err != nil {
			log.Fatalf("could not open database: %s", err)
		}

		query, _ = c.String("registered_characters", "all_query")
		if query == "" {
			log.Fatalf("registered_characters dsn specified but no all_query found.")
		}

		query, _ = c.String("registered_characters", "single_query")
		if query == "" {
			log.Fatalf("registered_characters dsn specified but no single_query found.")
		}
		cmt = NewSQLCorpMemberTracker(memberDB)
	} else if memberURL != "" {
		cmt = NewHTTPCorpMemberTracker(memberURL)
	}

	var store session.SessionStorage

	dbfile, _ := c.String("purger", "boltDB")
	if dbfile == "" {
		log.Fatalf("Need a boltDB file specified for the bolt database.")
	}

	bdb, err = bolt.Open("purger.db", 0644, &bolt.Options{Timeout: 100 * time.Millisecond})
	if err != nil {
		log.Fatalf("Failed to open db: %s", err)
	}
	store, _ = session.NewBoltStore(bdb, time.Hour*24)
	sesManager, _ = session.NewSessionManager(store, "slopemaker_session")

	loadState()

	m := setupMartini()

	m.Get("/", forceLogin, handleRoot)
	m.Get("/purger", forceLogin, handleRoot)

	m.Get("/strip", forceLogin, handleStrip)
	m.Post("/strip", forceLogin, handleStrip)

	m.Get("/boot", forceLogin, handleBoot)
	m.Post("/boot", forceLogin, handleBoot)

	m.Get("/stats", forceLogin, handleStats)

	m.Get("/login", displayLogin)
	m.Post("/login", handleLogin)

	m.Use(martini.Static("static", martini.StaticOptions{Prefix: "static/"}))

	go membersUpdater(apiClient, int64(keyid), vcode, time.Duration(days)*time.Hour*24, cmt)
	go http.ListenAndServe(listen, m)

	sch := make(chan os.Signal, 1)
	signal.Notify(sch, syscall.SIGHUP)
	for _ = range sch {
		saveState()
		err = loadConfig()
		if err != nil {
			log.Printf("Failed to reload config: %s", err)
			continue
		}
		log.Printf("Reloaded user configuration.")
	}
}
<commit_msg>fix bolt db config<commit_after>package main

import (
	"database/sql"
	"log"
	"net/http"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/inominate/apicache"

	"github.com/boltdb/bolt"
	"github.com/codegangsta/martini"
	_ "github.com/go-sql-driver/mysql"
	"github.com/inominate/session"
	"github.com/robfig/config"
)

var sesManager *session.SessionManager
var conf *config.Config
var exemptRoles []int64
var exemptChars []string

func setupMartini() *martini.ClassicMartini {
	r := martini.NewRouter()
	m := martini.New()
	m.Use(martini.Logger())
	m.Use(martini.Recovery())
	m.Use(SessionService())
	m.MapTo(r, (*martini.Routes)(nil))
	m.Action(r.Handle)
	return &martini.ClassicMartini{m, r}
}

func loadConfig() error {
	newConf, err := config.ReadDefault("purger.conf")
	if err != nil {
		return err
	}
	conf = newConf

	confStr, err := conf.String("purger", "exemptCharacters")
	if err == nil {
		exemptChars = strings.Split(strings.ToLower(confStr), ",")
		for k := range exemptChars {
			exemptChars[k] = strings.TrimSpace(exemptChars[k])
		}
	} else {
		log.Printf("No exempted characters found: %s", err)
		exemptChars = []string{}
	}

	confStr, err = conf.String("purger", "exemptRoles")
	if err == nil {
		exemptRolesStrings := strings.Split(confStr, ",")
		exemptRoles = make([]int64, len(exemptRolesStrings))
		for k, v := range exemptRolesStrings {
			v = strings.TrimSpace(v)
			if v == "" {
				continue
			}

			role, err := strconv.ParseInt(v, 10, 64)
			if err != nil || role == 0 || role < 0 {
				log.Printf("Error parsing role '%s': %s", v, err)
				continue
			}

			exemptRoles[k] = role
		}
	} else {
		log.Printf("No exempted roles found: %s", err)
		exemptRoles = []int64{}
	}

	return nil
}

var bdb *bolt.DB

func main() {
	var err error

	err = loadConfig()
	if err != nil {
		log.Fatalf("Failed to load config: %s", err)
	}

	c := conf
	keyid, err := c.Int("purger", "keyid")
	if err != nil {
		log.Fatalf("Failed to load config: %s", err)
	}
	vcode, err := c.String("purger", "vcode")
	if err != nil {
		log.Fatalf("Failed to load config: %s", err)
	}
	days, err := c.Int("purger", "maxIdleDays")
	if err != nil {
		log.Fatalf("Failed to load config: %s", err)
	}
	listen, err := c.String("purger", "listen")
	if err != nil {
		log.Fatalf("Failed to load config: %s", err)
	}
	if listen == "ENV" {
		listen = ":" + os.Getenv("PORT")
	}

	baseURL, _ := c.String("purger", "APIBaseURL")
	if baseURL == "" {
		baseURL = "https://api.eveonline.com/"
	}
	apiClient := apicache.NewClient(apicache.NilCache)
	apiClient.BaseURL = baseURL

	var memberDB *sql.DB
	var query string
	var cmt CorpMemberTracker

	memberDSN, _ := c.String("registered_characters", "DSN")
	memberURL, _ := c.String("registered_characters", "URL")
	if memberDSN != "" {
		memberDB, err = sql.Open("mysql", memberDSN)
		if err != nil {
			log.Fatalf("could not open database: %s", err)
		}

		query, _ = c.String("registered_characters", "all_query")
		if query == "" {
			log.Fatalf("registered_characters dsn specified but no all_query found.")
		}

		query, _ = c.String("registered_characters", "single_query")
		if query == "" {
			log.Fatalf("registered_characters dsn specified but no single_query found.")
		}
		cmt = NewSQLCorpMemberTracker(memberDB)
	} else if memberURL != "" {
		cmt = NewHTTPCorpMemberTracker(memberURL)
	}

	var store session.SessionStorage

	dbfile, _ := c.String("purger", "boltDB")
	if dbfile == "" {
		log.Fatalf("Need a boltDB file specified for the bolt database.")
	}

	bdb, err = bolt.Open(dbfile, 0644, &bolt.Options{Timeout: 100 * time.Millisecond})
	if err != nil {
		log.Fatalf("Failed to open db: %s", err)
	}
	store, _ = session.NewBoltStore(bdb, time.Hour*24)
	sesManager, _ = session.NewSessionManager(store, "slopemaker_session")

	loadState()

	m := setupMartini()

	m.Get("/", forceLogin, handleRoot)
	m.Get("/purger", forceLogin, handleRoot)

	m.Get("/strip", forceLogin, handleStrip)
	m.Post("/strip", forceLogin, handleStrip)

	m.Get("/boot", forceLogin, handleBoot)
	m.Post("/boot", forceLogin, handleBoot)

	m.Get("/stats", forceLogin, handleStats)

	m.Get("/login", displayLogin)
	m.Post("/login", handleLogin)

	m.Use(martini.Static("static", martini.StaticOptions{Prefix: "static/"}))

	go membersUpdater(apiClient, int64(keyid), vcode, time.Duration(days)*time.Hour*24, cmt)
	go http.ListenAndServe(listen, m)

	sch := make(chan os.Signal, 1)
	signal.Notify(sch, syscall.SIGHUP)
	for _ = range sch {
		saveState()
		err = loadConfig()
		if err != nil {
			log.Printf("Failed to reload config: %s", err)
			continue
		}
		log.Printf("Reloaded user configuration.")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path"
	"strings"
	"sync"
)

// Printed for -help, -h or with wrong number of arguments
const usage = `Usage: %s githubname backupdir

  githubname  github user or organization name to get the repositories from
  backupdir   directory path to save the repositories to

`

type repo struct {
	Name   string
	GitURL string `json:"git_url"`
}

const defaultMaxWorkers = 10
const defaultGithubAPI = "https://api.github.com"

// Get command line arguments and start updating repositories
func main() {
	// Flags
	verbose := flag.Bool("verbose", false, "print progress information")

	// Parse args
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, usage, os.Args[0])
		flag.PrintDefaults()
	}
	flag.Parse()
	args := flag.Args()
	if len(args) != 2 {
		flag.Usage()
		os.Exit(1)
	}

	// Logger for verbose mode
	var verboseLogger *log.Logger
	if *verbose {
		verboseLogger = log.New(os.Stderr, "", log.LstdFlags|log.LUTC)
	} else {
		verboseLogger = log.New(ioutil.Discard, "", 0)
	}

	backup(backupOpts{
		name:       args[0],
		dir:        args[1],
		httpClient: http.DefaultClient,
		// Log errors with line numbers
		err:     log.New(os.Stderr, "", log.LstdFlags|log.Lshortfile|log.LUTC),
		verbose: verboseLogger,
	})
}

type backupOpts struct {
	name       string
	dir        string
	httpClient *http.Client
	err        *log.Logger
	verbose    *log.Logger
}

// Update repos for the given options
func backup(opts backupOpts) {
	category, err := getCategory(opts.name, opts.httpClient)
	if err != nil {
		opts.err.Fatal(err)
	}

	url, err := setMaxPageSize(strings.Join([]string{defaultGithubAPI, category, opts.name, "repos"}, "/"))
	if err != nil {
		opts.err.Fatal(err)
	}

	repos, err := getRepos(url, opts.httpClient)
	if err != nil {
		opts.err.Fatal(err)
	}

	opts.verbose.Println("Backup for", category[:len(category)-1], opts.name, "with", len(repos), "repositories")

	jobs := make(chan repo)

	workers := defaultMaxWorkers
	if len(repos) < defaultMaxWorkers {
		workers = len(repos)
	}

	var wg sync.WaitGroup
	wg.Add(workers)

	for w := 0; w < workers; w++ {
		go func() {
			defer wg.Done()
			for r := range jobs {
				err := updateRepo(opts.dir, r, opts.verbose)
				if err != nil {
					opts.err.Println(err)
				}
			}
		}()
	}

	for _, r := range repos {
		jobs <- r
	}
	close(jobs)
	wg.Wait()
}

// Returns "users" or "orgs" depending on type of account
func getCategory(name string, client *http.Client) (string, error) {
	res, err := client.Get(strings.Join([]string{defaultGithubAPI, "users", name}, "/"))
	if err != nil {
		return "", fmt.Errorf("cannot get user info: %v", err)
	}
	defer func() {
		_ = res.Body.Close()
	}()
	if res.StatusCode >= 300 {
		return "", fmt.Errorf("bad response from %s: %v", res.Request.URL, res.Status)
	}

	var account struct {
		Type string
	}
	err = json.NewDecoder(res.Body).Decode(&account)
	if err != nil {
		return "", fmt.Errorf("cannot decode JSON response: %v", err)
	}

	if account.Type == "User" {
		return "users", nil
	}
	if account.Type == "Organization" {
		return "orgs", nil
	}
	return "", fmt.Errorf("unknown type of account %s for name %s", account.Type, name)
}

// Get repositories from Github.
// Follow "next" links recursivly.
func getRepos(u string, client *http.Client) ([]repo, error) {
	res, err := client.Get(u)
	if err != nil {
		return nil, fmt.Errorf("cannot get repos: %v", err)
	}
	defer func() {
		_ = res.Body.Close()
	}()
	if res.StatusCode >= 300 {
		return nil, fmt.Errorf("bad response from %s: %v", res.Request.URL, res.Status)
	}

	var repos []repo
	err = json.NewDecoder(res.Body).Decode(&repos)
	if err != nil {
		return nil, fmt.Errorf("cannot decode JSON response: %v", err)
	}

	linkHeader := res.Header["Link"]
	if len(linkHeader) > 0 {
		firstLink := strings.Split(linkHeader[0], ",")[0]
		if strings.Contains(firstLink, "rel=\"next\"") {
			urlInBrackets := strings.Split(firstLink, ";")[0]
			nextRepos, err := getRepos(urlInBrackets[1:len(urlInBrackets)-1], client)
			return append(repos, nextRepos...), err
		}
	}

	return repos, nil
}

//  Adds per_page=100 to a URL
func setMaxPageSize(rawURL string) (string, error) {
	u, err := url.Parse(rawURL)
	if err != nil {
		return "", fmt.Errorf("cannot parse url: %v", err)
	}
	q := u.Query()
	q.Set("per_page", "100")
	u.RawQuery = q.Encode()
	return u.String(), nil
}

// Clone new repo or pull in existing repo
func updateRepo(backupDir string, r repo, info *log.Logger) error {
	repoDir := path.Join(backupDir, r.Name)

	var cmd *exec.Cmd
	repoExists, err := exists(repoDir)
	if err != nil {
		return fmt.Errorf("cannot check if repo exists: %v", err)
	}
	if repoExists {
		info.Println("Updating	", r.Name)
		cmd = exec.Command("git", "pull")
		cmd.Dir = repoDir
	} else {
		info.Println("Cloning	", r.Name)
		cmd = exec.Command("git", "clone", r.GitURL, repoDir)
	}

	err = cmd.Run()
	if err != nil {
		// Give enough information to reproduce command
		return fmt.Errorf("error running command `%v` (`%v`) in dir `%v` with env `%v`: %v", cmd.Args, cmd.Path, cmd.Dir, cmd.Env, err)
	}
	return nil
}

// Check if a file or directory exists
func exists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err != nil {
		if os.IsNotExist(err) {
			return false, nil
		}
		return false, fmt.Errorf("cannot get stats for path `%s`: %v", path, err)
	}
	return true, nil
}
<commit_msg>getRepos without recursion<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path"
	"strings"
	"sync"
)

// Printed for -help, -h or with wrong number of arguments
const usage = `Usage: %s githubname backupdir

  githubname  github user or organization name to get the repositories from
  backupdir   directory path to save the repositories to

`

type repo struct {
	Name   string
	GitURL string `json:"git_url"`
}

const defaultMaxWorkers = 10
const defaultGithubAPI = "https://api.github.com"

// Get command line arguments and start updating repositories
func main() {
	// Flags
	verbose := flag.Bool("verbose", false, "print progress information")

	// Parse args
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, usage, os.Args[0])
		flag.PrintDefaults()
	}
	flag.Parse()
	args := flag.Args()
	if len(args) != 2 {
		flag.Usage()
		os.Exit(1)
	}

	// Logger for verbose mode
	var verboseLogger *log.Logger
	if *verbose {
		verboseLogger = log.New(os.Stderr, "", log.LstdFlags|log.LUTC)
	} else {
		verboseLogger = log.New(ioutil.Discard, "", 0)
	}

	backup(backupOpts{
		name:       args[0],
		dir:        args[1],
		httpClient: http.DefaultClient,
		// Log errors with line numbers
		err:     log.New(os.Stderr, "", log.LstdFlags|log.Lshortfile|log.LUTC),
		verbose: verboseLogger,
	})
}

type backupOpts struct {
	name       string
	dir        string
	httpClient *http.Client
	err        *log.Logger
	verbose    *log.Logger
}

// Update repos for the given options
func backup(opts backupOpts) {
	category, err := getCategory(opts.name, opts.httpClient)
	if err != nil {
		opts.err.Fatal(err)
	}

	url, err := setMaxPageSize(strings.Join([]string{defaultGithubAPI, category, opts.name, "repos"}, "/"))
	if err != nil {
		opts.err.Fatal(err)
	}

	repos, err := getRepos(url, opts.httpClient)
	if err != nil {
		opts.err.Fatal(err)
	}

	opts.verbose.Println("Backup for", category[:len(category)-1], opts.name, "with", len(repos), "repositories")

	jobs := make(chan repo)

	workers := defaultMaxWorkers
	if len(repos) < defaultMaxWorkers {
		workers = len(repos)
	}

	var wg sync.WaitGroup
	wg.Add(workers)

	for w := 0; w < workers; w++ {
		go func() {
			defer wg.Done()
			for r := range jobs {
				err := updateRepo(opts.dir, r, opts.verbose)
				if err != nil {
					opts.err.Println(err)
				}
			}
		}()
	}

	for _, r := range repos {
		jobs <- r
	}
	close(jobs)
	wg.Wait()
}

// Returns "users" or "orgs" depending on type of account
func getCategory(name string, client *http.Client) (string, error) {
	res, err := client.Get(strings.Join([]string{defaultGithubAPI, "users", name}, "/"))
	if err != nil {
		return "", fmt.Errorf("cannot get user info: %v", err)
	}
	defer func() {
		_ = res.Body.Close()
	}()
	if res.StatusCode >= 300 {
		return "", fmt.Errorf("bad response from %s: %v", res.Request.URL, res.Status)
	}

	var account struct {
		Type string
	}
	err = json.NewDecoder(res.Body).Decode(&account)
	if err != nil {
		return "", fmt.Errorf("cannot decode JSON response: %v", err)
	}

	if account.Type == "User" {
		return "users", nil
	}
	if account.Type == "Organization" {
		return "orgs", nil
	}
	return "", fmt.Errorf("unknown type of account %s for name %s", account.Type, name)
}

// Get repositories from Github.
// Follow all "next" links.
func getRepos(url string, client *http.Client) ([]repo, error) {
	var allRepos []repo

	// Go through all pages
	for {
		res, err := client.Get(url)
		if err != nil {
			return nil, fmt.Errorf("cannot get repos: %v", err)
		}
		defer func() {
			_ = res.Body.Close()
		}()
		if res.StatusCode >= 300 {
			return nil, fmt.Errorf("bad response from %s: %v", res.Request.URL, res.Status)
		}

		var repos []repo
		err = json.NewDecoder(res.Body).Decode(&repos)
		if err != nil {
			return nil, fmt.Errorf("cannot decode JSON response: %v", err)
		}

		allRepos = append(allRepos, repos...)

		linkHeader := res.Header["Link"]
		if len(linkHeader) == 0 {
			break
		}
		firstLink := strings.Split(linkHeader[0], ",")[0]
		if !strings.Contains(firstLink, "rel=\"next\"") {
			break
		}
		urlInBrackets := strings.Split(firstLink, ";")[0]
		// Set url for next iteration
		url = urlInBrackets[1 : len(urlInBrackets)-1]
	}

	return allRepos, nil
}

//  Adds per_page=100 to a URL
func setMaxPageSize(rawURL string) (string, error) {
	u, err := url.Parse(rawURL)
	if err != nil {
		return "", fmt.Errorf("cannot parse url: %v", err)
	}
	q := u.Query()
	q.Set("per_page", "100")
	u.RawQuery = q.Encode()
	return u.String(), nil
}

// Clone new repo or pull in existing repo
func updateRepo(backupDir string, r repo, info *log.Logger) error {
	repoDir := path.Join(backupDir, r.Name)

	var cmd *exec.Cmd
	repoExists, err := exists(repoDir)
	if err != nil {
		return fmt.Errorf("cannot check if repo exists: %v", err)
	}
	if repoExists {
		info.Println("Updating	", r.Name)
		cmd = exec.Command("git", "pull")
		cmd.Dir = repoDir
	} else {
		info.Println("Cloning	", r.Name)
		cmd = exec.Command("git", "clone", r.GitURL, repoDir)
	}

	err = cmd.Run()
	if err != nil {
		// Give enough information to reproduce command
		return fmt.Errorf("error running command `%v` (`%v`) in dir `%v` with env `%v`: %v", cmd.Args, cmd.Path, cmd.Dir, cmd.Env, err)
	}
	return nil
}

// Check if a file or directory exists
func exists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err != nil {
		if os.IsNotExist(err) {
			return false, nil
		}
		return false, fmt.Errorf("cannot get stats for path `%s`: %v", path, err)
	}
	return true, nil
}
<|endoftext|>


<commit_before>// get-headers prints the headers from GET requesting a URL
package main

import (
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	netURL "net/url"
	"os"
	"sync"
	"text/tabwriter"
	"time"

	"github.com/carlmjohnson/get-headers/prettyprint"
)

// Flag variables (set in flags.go on init)
var (
	etag       string
	gzip       bool
	ignoreBody bool
)

// Sentinal error to let us know if we're ignoring a redirect
var errRedirect = errors.New("redirected")

// Don't follow redirects
func checkRedirect(req *http.Request, via []*http.Request) error {
	return errRedirect
}

func die(err error) {
	if err != nil {
		fmt.Fprintf(os.Stderr, "get-time: %v\n", err)
		os.Exit(1)
	}
}

func main() {
	client := http.Client{
		CheckRedirect: checkRedirect,
		Transport: &http.Transport{
			DisableCompression: true,
		},
	}
	for _, url := range flag.Args() {

		req, err := http.NewRequest("GET", url, nil)
		die(err)

		req.Header = map[string][]string{}

		if gzip {
			req.Header["Accept-Encoding"] = []string{"gzip, deflate"}
		}

		if etag != "" {
			req.Header["If-None-Match"] = []string{etag}
		}

		start := time.Now()
		resp, err := client.Do(req)
		duration := time.Since(start)

		var (
			n  int64
			wg sync.WaitGroup
		)
		// Ignore the error if it's just our errRedirect
		switch urlErr, ok := err.(*netURL.Error); {
		case err == nil:
			if ignoreBody {
				break
			}

			wg.Add(1)
			go func() {
				// Copying to /dev/null just to make sure this is real
				n, err = io.Copy(ioutil.Discard, resp.Body)
				duration = time.Since(start)
				die(err)
				wg.Done()
			}()
		case ok && urlErr.Err == errRedirect:
		default:
			die(err)
		}

		fmt.Println("GET", url)
		fmt.Println(resp.Proto, resp.Status)
		fmt.Println()
		fmt.Println(prettyprint.ResponseHeader(resp.Header))
		wg.Wait()
		die(resp.Body.Close())

		tw := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
		fmt.Fprintf(tw, "Time\t%s\n", prettyprint.Duration(duration))
		if n != 0 {
			fmt.Fprintf(tw, "Content length\t%s\n", prettyprint.Size(n))
			bps := prettyprint.Size(float64(n) / duration.Seconds())
			fmt.Fprintf(tw, "Speed\t%s/s\n", bps)
		}
		tw.Flush()
	}
}
<commit_msg>Main: Use Header methods<commit_after>// get-headers prints the headers from GET requesting a URL
package main

import (
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	netURL "net/url"
	"os"
	"sync"
	"text/tabwriter"
	"time"

	"github.com/carlmjohnson/get-headers/prettyprint"
)

// Flag variables (set in flags.go on init)
var (
	etag       string
	gzip       bool
	ignoreBody bool
)

// Sentinal error to let us know if we're ignoring a redirect
var errRedirect = errors.New("redirected")

// Don't follow redirects
func checkRedirect(req *http.Request, via []*http.Request) error {
	return errRedirect
}

func die(err error) {
	if err != nil {
		fmt.Fprintf(os.Stderr, "get-time: %v\n", err)
		os.Exit(1)
	}
}

func main() {
	client := http.Client{
		CheckRedirect: checkRedirect,
		Transport: &http.Transport{
			DisableCompression: true,
		},
	}
	for _, url := range flag.Args() {

		req, err := http.NewRequest("GET", url, nil)
		die(err)

		if gzip {
			req.Header.Add("Accept-Encoding", "gzip, deflate")
		}

		if etag != "" {
			req.Header.Add("If-None-Match", etag)
		}

		start := time.Now()
		resp, err := client.Do(req)
		duration := time.Since(start)

		var (
			n  int64
			wg sync.WaitGroup
		)
		// Ignore the error if it's just our errRedirect
		switch urlErr, ok := err.(*netURL.Error); {
		case err == nil:
			if ignoreBody {
				break
			}

			wg.Add(1)
			go func() {
				// Copying to /dev/null just to make sure this is real
				n, err = io.Copy(ioutil.Discard, resp.Body)
				duration = time.Since(start)
				die(err)
				wg.Done()
			}()
		case ok && urlErr.Err == errRedirect:
		default:
			die(err)
		}

		fmt.Println("GET", url)
		fmt.Println(resp.Proto, resp.Status)
		fmt.Println()
		fmt.Println(prettyprint.ResponseHeader(resp.Header))
		wg.Wait()
		die(resp.Body.Close())

		tw := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
		fmt.Fprintf(tw, "Time\t%s\n", prettyprint.Duration(duration))
		if n != 0 {
			fmt.Fprintf(tw, "Content length\t%s\n", prettyprint.Size(n))
			bps := prettyprint.Size(float64(n) / duration.Seconds())
			fmt.Fprintf(tw, "Speed\t%s/s\n", bps)
		}
		tw.Flush()
	}
}
<|endoftext|>