<commit_before>package main

import (
	"fmt"
	"math/rand"
	"os"
	"strconv"
	"time"
)

//VARIABILI
var Matrix [][]*Element
var Altezza int
var Larghezza int
var SaluteIniziale int = 50
var CostoMovIniziale int = 5
var CostoSexIniziale int = 100
var EvoluzioneIniziale int = 0
var PremuraIniziale int = 100
var Clock uint
var NumClock uint

func main() { //FUNZIONE MAIN
	Clock = 1
	NumClock = 0
	rand.Seed(time.Now().UTC().UnixNano()) //inizializzazione rand
	height, err := strconv.Atoi(os.Args[1])
	if err != nil {
		panic("height not valid")
	}
	width, err2 := strconv.Atoi(os.Args[2])
	if err2 != nil {
		panic("width not valid")
	}
	Altezza = height
	Larghezza = width
	Matrix = make([][]*Element, height)
	for i := range Matrix { // inizializzazione matrice
		Matrix[i] = make([]*Element, width)
		for j := range Matrix[i] {
			chose := rand.Intn(3) //scelta rando cibo bug o vuoto (null)
			switch chose {
			case 0:
				Matrix[i][j] = new(Element) // insetto
				Matrix[i][j].IsFood = false
				Matrix[i][j].Age = 0
				Matrix[i][j].Health = SaluteIniziale
				Matrix[i][j].CostoMov = CostoMovIniziale
				Matrix[i][j].CostoSex = CostoSexIniziale
				Matrix[i][j].Evoluzione = EvoluzioneIniziale
				Matrix[i][j].Premura = PremuraIniziale
				Matrix[i][j].Razza = rand.Intn(2)
			case 1:
				Matrix[i][j] = nil //vuota
			case 2:
				Matrix[i][j] = new(Element) // cibo
				Matrix[i][j].IsFood = true
				Matrix[i][j].Health = 5
			}
		}
	}

	go ServiHTML() // fai partire il server html

	fmt.Println("Situazione iniziale: ")
	stampaMatrice()

	aggiorna()
}

func aggiorna() { //FUNZIONE AGGIORNA:	chiama la funzione muovi
	for {
		time.Sleep(time.Second * time.Duration(Clock))
		NumClock++
		for i := 0; i < Altezza; i++ {
			for j := 0; j < Larghezza; j++ {
				muovi(i, j)
			}
		}
		fmt.Printf("\nSituazione dopo %d movimenti:\n", NumClock)
		stampaMatrice()
	}
}

func muovi(h int, w int) { //FUNZIONE MUOVI:	aggiorna la posizione di tutti gli oggetti in tabella	// h verticale, w orizzontale
	elemento := Matrix[h][w]                //assegnamente del contenuto della cella in 'elemento'
	if elemento == nil || elemento.IsFood { //controllo se 'elemento' è cibo o un altro essere
		return
	}

	if elemento.Health<=0 {
		Matrix[h][w] = nil
	}
	direzCasOriz := rand.Intn(3) //numero da 0 a 2
	direzCasOriz--
	direzCasVert := rand.Intn(3)
	direzCasVert--
	nuovaPosizioneH := h + direzCasVert //aggiornamento posiozione verticale
	nuovaPosizioneW := w + direzCasOriz //aggiornamento posizione orizzontale

	if nuovaPosizioneH >= Altezza || nuovaPosizioneH < 0 ||
		nuovaPosizioneW >= Larghezza || nuovaPosizioneW < 0 { //se esce dai bordi
		return
	}

	tmpNewElem := Matrix[nuovaPosizioneH][nuovaPosizioneW]

	if tmpNewElem != nil {
		if tmpNewElem.Razza != elemento.Razza { //se non è dalla stessa razza
			if tmpNewElem.IsFood || (tmpNewElem.Health+tmpNewElem.Evoluzione) < (elemento.Health+elemento.Evoluzione) { // se e' cibo o un insetto piu debole
				elemento.Health += tmpNewElem.Health                //prelevamento energia essere fagocitato
				tmpNewElem = elemento //inglobamento essere perito
				elemento = nil
				tmpNewElem.Health -= tmpNewElem.CostoMov
			} else {	//perdita nel combattimento per la sopravvivenza
				tmpNewElem.Health += elemento.Health //il nemico prende l'energia
				elemento = nil
			}
		} else { //se sono amici
			if nuovaPosizioneH == h && nuovaPosizioneW == w { //se cerca di mangiare il suo amico
				muovi(h, w)
			}
		}
	} else { //si muove sulla nuova casella
		tmpNewElem = elemento
		elemento = nil
		tmpNewElem.Health -= tmpNewElem.CostoMov

		/*if rand.Intn(10) == 0 { //se ha fortuna (o sfortuna) si evolve
			if rand.Intn(3) == 0 {
				tmpNewElem.Evoluzione--
			} else {
				tmpNewElem.Evoluzione++
			}
		}*/

		if (tmpNewElem.Health-tmpNewElem.Premura)>tmpNewElem.CostoSex {		//se ha energia a sufficienza per riprodursi
			//tmpNewElem = Costruttore(elemento.Razza, elemento.Evoluzione, elemento.CostoMov, elemento.CostoSex, elemento.Premura, SaluteIniziale)
		}

	}
}

/*func stampaMatrice2() {
	for i := 0; i < Altezza; i++ {
		fmt.Printf("Riga %d:\n", i)
		for j := 0; j < Larghezza; j++ {
			var stringa string
			elem := Matrix[i][j]
			if elem == nil {
				stringa = "Vuota"
			} else {
				stringa = elem.String()
			}
			fmt.Printf("  Colonna %d: %s\n", j, stringa)
		}
	}
}*/

func stampaMatrice() {
	for i := 0; i < Altezza; i++ {
		for j := 0; j < Larghezza; j++ {
			if Matrix[i][j] == nil {
				fmt.Printf("    --  ")
			} else {
				if Matrix[i][j].IsFood {
					fmt.Printf("    CC  ")
				} else {
					fmt.Printf("%d   %d  ",Matrix[i][j].Razza, Matrix[i][j].Health)
				}
			}
		}
		fmt.Printf("\n")
	}
}
<commit_msg>Funge in parte<commit_after>package main

import (
	"fmt"
	"math/rand"
	"os"
	"strconv"
	"time"
)

//VARIABILI
var Matrix [][]*Element
var Altezza int
var Larghezza int
var SaluteIniziale int = 50
var CostoMovIniziale int = 5
var CostoSexIniziale int = 100
var EvoluzioneIniziale int = 0
var PremuraIniziale int = 100
var Clock uint
var NumClock uint

func main() { //FUNZIONE MAIN
	Clock = 1
	NumClock = 0
	rand.Seed(time.Now().UTC().UnixNano()) //inizializzazione rand
	height, err := strconv.Atoi(os.Args[1])
	if err != nil {
		panic("height not valid")
	}
	width, err2 := strconv.Atoi(os.Args[2])
	if err2 != nil {
		panic("width not valid")
	}
	Altezza = height
	Larghezza = width
	Matrix = make([][]*Element, height)
	for i := range Matrix { // inizializzazione matrice
		Matrix[i] = make([]*Element, width)
		for j := range Matrix[i] {
			chose := rand.Intn(3) //scelta rando cibo bug o vuoto (null)
			switch chose {
			case 0:
				Matrix[i][j] = new(Element) // insetto
				Matrix[i][j].IsFood = false
				Matrix[i][j].Age = 0
				Matrix[i][j].Health = SaluteIniziale
				Matrix[i][j].CostoMov = CostoMovIniziale
				Matrix[i][j].CostoSex = CostoSexIniziale
				Matrix[i][j].Evoluzione = EvoluzioneIniziale
				Matrix[i][j].Premura = PremuraIniziale
				Matrix[i][j].Razza = rand.Intn(2)
			case 1:
				Matrix[i][j] = nil //vuota
			case 2:
				Matrix[i][j] = new(Element) // cibo
				Matrix[i][j].IsFood = true
				Matrix[i][j].Health = 5
			}
		}
	}

	go ServiHTML() // fai partire il server html

	fmt.Println("Situazione iniziale: ")
	stampaMatrice()

	aggiorna()
}

func aggiorna() { //FUNZIONE AGGIORNA:	chiama la funzione muovi
	for {
		time.Sleep(time.Second * time.Duration(Clock))
		NumClock++
		for i := 0; i < Altezza; i++ {
			for j := 0; j < Larghezza; j++ {
				muovi(i, j)
			}
		}
		fmt.Printf("\nSituazione dopo %d movimenti:\n", NumClock)
		stampaMatrice()
	}
}

func muovi(h int, w int) { //FUNZIONE MUOVI:	aggiorna la posizione di tutti gli oggetti in tabella	// h verticale, w orizzontale
	elemento := Matrix[h][w]                //assegnamente del contenuto della cella in 'elemento'
	if elemento == nil || elemento.IsFood { //controllo se 'elemento' è cibo o un altro essere
		return
	}

	if elemento.Health<=0 {
		Matrix[h][w] = nil
		return
	}
	direzCasOriz := rand.Intn(3) //numero da 0 a 2
	direzCasOriz--
	direzCasVert := rand.Intn(3)
	direzCasVert--
	nuovaPosizioneH := h + direzCasVert //aggiornamento posiozione verticale
	nuovaPosizioneW := w + direzCasOriz //aggiornamento posizione orizzontale

	if nuovaPosizioneH >= Altezza || nuovaPosizioneH < 0 ||
		nuovaPosizioneW >= Larghezza || nuovaPosizioneW < 0 { //se esce dai bordi
		return
	}

	if Matrix[nuovaPosizioneH][nuovaPosizioneW] != nil {
		if Matrix[nuovaPosizioneH][nuovaPosizioneW].Razza != Matrix[h][w].Razza { //se non è dalla stessa razza
			if Matrix[nuovaPosizioneH][nuovaPosizioneW].IsFood || (Matrix[nuovaPosizioneH][nuovaPosizioneW].Health+Matrix[nuovaPosizioneH][nuovaPosizioneW].Evoluzione) < (Matrix[h][w].Health+elemento.Evoluzione) { // se e' cibo o un insetto piu debole
				Matrix[h][w].Health += Matrix[nuovaPosizioneH][nuovaPosizioneW].Health                //prelevamento energia essere fagocitato
				Matrix[nuovaPosizioneH][nuovaPosizioneW] = Matrix[h][w] //inglobamento essere perito
				Matrix[h][w] = nil
				Matrix[nuovaPosizioneH][nuovaPosizioneW].Health -= Matrix[nuovaPosizioneH][nuovaPosizioneW].CostoMov
			} else {	//perdita nel combattimento per la sopravvivenza
				Matrix[nuovaPosizioneH][nuovaPosizioneW].Health += Matrix[h][w].Health //il nemico prende l'energia
				Matrix[h][w] = nil
			}
		} else { //se sono amici
			if nuovaPosizioneH == h && nuovaPosizioneW == w { //se cerca di mangiare il suo amico
				muovi(h, w)
			}
		}
	} else { //si muove sulla nuova casella
		Matrix[nuovaPosizioneH][nuovaPosizioneW] = Matrix[h][w]
		Matrix[nuovaPosizioneH][nuovaPosizioneW].Health -= Matrix[nuovaPosizioneH][nuovaPosizioneW].CostoMov
		Matrix[h][w] = nil

		if rand.Intn(10) == 0 { //se ha fortuna (o sfortuna) si evolve
			if rand.Intn(3) == 0 {
				Matrix[nuovaPosizioneH][nuovaPosizioneW].Evoluzione--
			} else {
				Matrix[nuovaPosizioneH][nuovaPosizioneW].Evoluzione++
			}
		}

		if (Matrix[nuovaPosizioneH][nuovaPosizioneW].Health-Matrix[nuovaPosizioneH][nuovaPosizioneW].Premura)>Matrix[nuovaPosizioneH][nuovaPosizioneW].CostoSex {		//se ha energia a sufficienza per riprodursi
			//Matrix[nuovaPosizioneH][nuovaPosizioneW] = Costruttore(Matrix[h][w].Razza, Matrix[h][w].Evoluzione, Matrix[h][w].CostoMov, Matrix[h][w].CostoSex, Matrix[h][w].Premura, SaluteIniziale)
		}

	}
}

func stampaMatrice() {
	for i := 0; i < Altezza; i++ {
		for j := 0; j < Larghezza; j++ {
			if Matrix[i][j] == nil {
				fmt.Printf("    --  ")
			} else {
				if Matrix[i][j].IsFood {
					fmt.Printf("    CC  ")
				} else {
					fmt.Printf("%d   %d  ",Matrix[i][j].Razza, Matrix[i][j].Health)
				}
			}
		}
		fmt.Printf("\n")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path"
	"strings"
)

var usage = `Usage: %s githubname backupdir

githubname  github user or organization name to get the repositories from
backupdir   directory path to save the repositories to`

type Repo struct {
	Name   string
	GitUrl string `json:"git_url"`
}

var maxWorkers = 10
var githubApi = "https://api.github.com"

func main() {
	name, backupDir := parseArgs()

	category := getCategory(name)
	repos := getRepos(setMaxPageSize(strings.Join([]string{githubApi, category, name, "repos"}, "/")))

	fmt.Println("Backup for", category[:len(category)-1], name, "with", len(repos), "repositories")

	jobs := make(chan Repo)
	workers := maxWorkers
	if len(repos) < maxWorkers {
		workers = 1
	}
	for w := 0; w < workers; w++ {
		go func() {
			for repo := range jobs {
				updateRepo(backupDir, repo)
			}
		}()
	}
	for _, repo := range repos {
		jobs <- repo
	}
}

// Get the two positional arguments githubname and backupdir
func parseArgs() (string, string) {
	flag.Parse()
	args := flag.Args()
	if len(args) != 2 {
		fmt.Fprintf(os.Stderr, usage, os.Args[0])
		os.Exit(1)
	}
	return args[0], args[1]
}

// Returns "users" or "orgs" depending on type of account
func getCategory(name string) string {
	r, err := http.Get(strings.Join([]string{githubApi, "users", name}, "/"))
	if err != nil {
		panic(err)
	}
	defer r.Body.Close()
	if r.StatusCode >= 300 {
		panic(fmt.Sprint("Request to ", r.Request.URL, " with bad status code ", r.StatusCode))
	}

	var account struct {
		Type string
	}
	json.NewDecoder(r.Body).Decode(&account)

	if account.Type == "User" {
		return "users"
	}
	if account.Type == "Organization" {
		return "orgs"
	}
	panic(fmt.Sprint("Unknown type of account ", account.Type, " for name ", name))
}

// Get repositories from Github.
// Follow "next" links recursivly.
func getRepos(u string) []Repo {
	r, err := http.Get(u)
	if err != nil {
		panic(err)
	}
	defer r.Body.Close()

	if r.StatusCode >= 300 {
		panic(fmt.Sprint("Request to ", u, " with bad status code ", r.StatusCode))
	}

	var repos []Repo
	json.NewDecoder(r.Body).Decode(&repos)

	linkHeader := r.Header["Link"]
	if len(linkHeader) > 0 {
		firstLink := strings.Split(linkHeader[0], ",")[0]
		if strings.Contains(firstLink, "rel=\"next\"") {
			urlInBrackets := strings.Split(firstLink, ";")[0]
			return append(repos, getRepos(urlInBrackets[1:len(urlInBrackets)-1])...)
		}
	}

	return repos
}

func setMaxPageSize(rawUrl string) string {
	u, err := url.Parse(rawUrl)
	if err != nil {
		panic(err)
	}
	q := u.Query()
	q.Set("per_page", "100")
	u.RawQuery = q.Encode()
	return u.String()
}

// Clone new repo or pull in existing repo
func updateRepo(backupDir string, repo Repo) {
	repoDir := path.Join(backupDir, repo.Name)

	var cmd *exec.Cmd
	if exists(repoDir) {
		defer fmt.Println("Updated repository:", repo.Name)

		cmd = exec.Command("git", "pull")
		cmd.Dir = repoDir
	} else {
		defer fmt.Println("Cloned  repository:", repo.Name)

		cmd = exec.Command("git", "clone", repo.GitUrl, repoDir)
	}

	err := cmd.Run()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
	}
}

// Check if a file or directory exists
func exists(path string) bool {
	_, err := os.Stat(path)
	if err != nil {
		if os.IsNotExist(err) {
			return false
		} else {
			panic(err)
		}
	}
	return true
}

func min(x, y int) int {
	if x < y {
		return x
	}
	return y
}
<commit_msg>explain funcs<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path"
	"strings"
)

// Printed on for -h or with wrong arguments
var usage = `Usage: %s githubname backupdir

githubname  github user or organization name to get the repositories from
backupdir   directory path to save the repositories to`

type Repo struct {
	Name   string
	GitUrl string `json:"git_url"`
}

var maxWorkers = 10
var githubApi = "https://api.github.com"

// Get command line arguments and start updating repositories
func main() {
	name, backupDir := parseArgs()

	category := getCategory(name)
	repos := getRepos(setMaxPageSize(strings.Join([]string{githubApi, category, name, "repos"}, "/")))

	fmt.Println("Backup for", category[:len(category)-1], name, "with", len(repos), "repositories")

	jobs := make(chan Repo)
	workers := maxWorkers
	if len(repos) < maxWorkers {
		workers = 1
	}
	for w := 0; w < workers; w++ {
		go func() {
			for repo := range jobs {
				updateRepo(backupDir, repo)
			}
		}()
	}
	for _, repo := range repos {
		jobs <- repo
	}
}

// Get the two positional arguments githubname and backupdir
func parseArgs() (string, string) {
	flag.Parse()
	args := flag.Args()
	if len(args) != 2 {
		fmt.Fprintf(os.Stderr, usage, os.Args[0])
		os.Exit(1)
	}
	return args[0], args[1]
}

// Returns "users" or "orgs" depending on type of account
func getCategory(name string) string {
	r, err := http.Get(strings.Join([]string{githubApi, "users", name}, "/"))
	if err != nil {
		panic(err)
	}
	defer r.Body.Close()
	if r.StatusCode >= 300 {
		panic(fmt.Sprint("Request to ", r.Request.URL, " with bad status code ", r.StatusCode))
	}

	var account struct {
		Type string
	}
	json.NewDecoder(r.Body).Decode(&account)

	if account.Type == "User" {
		return "users"
	}
	if account.Type == "Organization" {
		return "orgs"
	}
	panic(fmt.Sprint("Unknown type of account ", account.Type, " for name ", name))
}

// Get repositories from Github.
// Follow "next" links recursivly.
func getRepos(u string) []Repo {
	r, err := http.Get(u)
	if err != nil {
		panic(err)
	}
	defer r.Body.Close()

	if r.StatusCode >= 300 {
		panic(fmt.Sprint("Request to ", u, " with bad status code ", r.StatusCode))
	}

	var repos []Repo
	json.NewDecoder(r.Body).Decode(&repos)

	linkHeader := r.Header["Link"]
	if len(linkHeader) > 0 {
		firstLink := strings.Split(linkHeader[0], ",")[0]
		if strings.Contains(firstLink, "rel=\"next\"") {
			urlInBrackets := strings.Split(firstLink, ";")[0]
			return append(repos, getRepos(urlInBrackets[1:len(urlInBrackets)-1])...)
		}
	}

	return repos
}

//  Adds per_page=100 to a URL
func setMaxPageSize(rawUrl string) string {
	u, err := url.Parse(rawUrl)
	if err != nil {
		panic(err)
	}
	q := u.Query()
	q.Set("per_page", "100")
	u.RawQuery = q.Encode()
	return u.String()
}

// Clone new repo or pull in existing repo
func updateRepo(backupDir string, repo Repo) {
	repoDir := path.Join(backupDir, repo.Name)

	var cmd *exec.Cmd
	if exists(repoDir) {
		defer fmt.Println("Updated repository:", repo.Name)

		cmd = exec.Command("git", "pull")
		cmd.Dir = repoDir
	} else {
		defer fmt.Println("Cloned  repository:", repo.Name)

		cmd = exec.Command("git", "clone", repo.GitUrl, repoDir)
	}

	err := cmd.Run()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
	}
}

// Check if a file or directory exists
func exists(path string) bool {
	_, err := os.Stat(path)
	if err != nil {
		if os.IsNotExist(err) {
			return false
		} else {
			panic(err)
		}
	}
	return true
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/base64"
	"fmt"
	"math/rand"
	"net/http"
	"sort"
	"strings"
	"text/template"
	"time"

	"github.com/codegangsta/martini"
	"github.com/russross/blackfriday"

	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
)

var PostTemplate = template.Must(template.ParseFiles("public2/pagetempl.html"))
var HomeTemplate = template.Must(template.ParseFiles("public2/hometempl.html"))

type Post struct {
	Author  string
	Content string `datastore:",noindex"`
	Date    time.Time
	Slug    string
	Title   string
	Type    string // Possible types [Comedy,Hardware,Mystery,Networking,Problem,Quirk]
	R1      string // Override of the Rec 1
	R2      string // Override of the Rec 2
}

type PostFormatted struct {
	Author  string
	Content string `datastore:",noindex"`
	Date    string
	Slug    string
	Title   string
}

func main() {
	m := martini.Classic()
	m.Get("/post/:name", ReadPost)
	m.Get("/raw/:name", ReadRawPost)
	m.Post("/admin/new", PublishPost)
	m.Get("/admin/", Admin)
	m.Get("/", ListPosts)
	m.Get("/all", ListPosts)
	m.Get("/rss.xml", GetRSS)
	m.Get("/sitemap.xml", GetSitemap)
	m.Get("/admin/run_gc", Run_GC)
	m.Get("/admin/backup.tar", Producebackup)
	m.Get("/admin/remove/:name", RemovePost)
	m.Post("/admin/uploadfile", UploadFile)

	m.Get("/lessons/:year/:month/:day/:title", MigrateOldURLS)
	m.Get("/lessons/:year/:month/:day/:title/", MigrateOldURLS)
	m.Get("/errors/:year/:month/:day/:title", MigrateOldURLS)
	m.Get("/errors/:year/:month/:day/:title/", MigrateOldURLS)
	m.Get("/posts/errors/:year/:month/:day/:title", MigrateOldURLS)
	m.Get("/posts/errors/:year/:month/:day/:title/", MigrateOldURLS)
	m.Get("/asset/:tag", ReadFile)

	m.Use(func(res http.ResponseWriter, req *http.Request) {
		res.Header().Add("Cache-Control", "public")
		res.Header().Add("X-Served-By", GimmeDC(res, req))
		res.Header().Add("X-Served-For", req.Header.Get("CF-RAY"))
	})

	m.Use(martini.Static("public2"))

	http.Handle("/", m)

	PostTitleCache = make(map[string]Post)
	appengine.Main()
}

func MigrateOldURLS(rw http.ResponseWriter, req *http.Request, params martini.Params) {
	http.Redirect(rw, req, fmt.Sprintf("https://blog.benjojo.co.uk/post/%s-%s-%s-%s.md", params["year"], params["month"], params["day"], params["title"]), http.StatusMovedPermanently)
}

func ReadPost(rw http.ResponseWriter, req *http.Request, params martini.Params) {
	// c := appengine.NewContext(r)
	// key := datastore.NewIncompleteKey(c, "Greeting", PostKey(c))
	// _, err := datastore.Put(c, key, &g)
	if len(PostTitleCache) == 0 {
		forceUpdatePostTitleCache(rw, req)
	}

	c := appengine.NewContext(req)
	k := datastore.NewKey(c, "Post", params["name"], 0, nil)
	post := Post{}
	err := datastore.Get(c, k, &post)

	if err != nil {
		if fmt.Sprint(err) == "datastore: no such entity" {
			http.Error(rw, "This blog post cannot be found, Please check your URL", http.StatusNotFound)
			return
		}

		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}

	if post.Type != "" {
		findReccomendations(&post)
	}

	postd, _ := base64.StdEncoding.DecodeString(post.Content)
	// post.Content = strings.Replace(string(postd), "\n", "\r\n\r\n", -1)
	post.Content = string(postd)
	output := blackfriday.MarkdownCommon([]byte(post.Content))
	lines := strings.Split(string(postd), "\n")
	layoutData := struct {
		Title   string
		Content string
		Date    string
		// Rec links at the bottom
		HasReccomendations bool
		FirstRecLink       string
		FirstTitle         string
		FirstYear          int
		SecondRecLink      string
		SecondTitle        string
		SecondYear         int
		RandomLink         string
		RandomTitle        string
		RandomYear         int
	}{
		Title:   lines[0],
		Content: string(output),
		Date:    post.Date.Format("Jan 2 2006"),
	}
	if post.R1 != "" && post.R2 != "" {
		layoutData.HasReccomendations = true
		layoutData.FirstRecLink = "/post/" + post.R1
		layoutData.FirstTitle = PostTitleCache[post.R1].Title
		layoutData.FirstYear = PostTitleCache[post.R1].Date.Year()
		layoutData.SecondRecLink = "/post/" + post.R2
		layoutData.SecondTitle = PostTitleCache[post.R2].Title
		layoutData.SecondYear = PostTitleCache[post.R2].Date.Year()

		for _, v := range PostTitleCache {
			if !strings.HasPrefix(v.Slug, "DRAFT") {
				layoutData.RandomLink = "/post/" + v.Slug
				layoutData.RandomTitle = PostTitleCache[v.Slug].Title
				layoutData.RandomYear = PostTitleCache[v.Slug].Date.Year()
				break
			}
		}

	}

	err = PostTemplate.Execute(rw, layoutData)
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
	}
}

func findReccomendations(Incoming *Post) {
	rand.Seed(Incoming.Date.Unix())
	// Make an array of candidates
	Candidates := make([]string, 0)
	for _, v := range PostTitleCache {
		if v.Type == Incoming.Type {
			if v.Date.Unix() < Incoming.Date.Unix() {
				// If the post is older
				if strings.HasPrefix("DRAFT-", v.Title) {
					Candidates = append(Candidates, v.Slug)
				}
			}
		}
	}
	if len(Candidates) < 2 {
		return
	}

	sort.Strings(Candidates)

	if Incoming.R1 == "" {
		Incoming.R1 = Candidates[rand.Intn(len(Candidates)-1)]
	}

	if Incoming.R2 == "" {
		Incoming.R2 = Candidates[rand.Intn(len(Candidates)-1)]
	}
}

func ReadRawPost(rw http.ResponseWriter, req *http.Request, params martini.Params) {
	c := appengine.NewContext(req)
	k := datastore.NewKey(c, "Post", params["name"], 0, nil)
	post := Post{}
	err := datastore.Get(c, k, &post)
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}

	postd, _ := base64.StdEncoding.DecodeString(post.Content)
	rw.Write(postd)
}

var PostTitleCache map[string]Post

func updatePostCache(Posts []Post) {
	for _, v := range Posts {
		PostTitleCache[v.Slug] = v
	}
}

func forceUpdatePostTitleCache(rw http.ResponseWriter, req *http.Request) {
	c := appengine.NewContext(req)
	q := datastore.NewQuery("Post").Order("-Date").Limit(100)
	posts := make([]Post, 0, 100)

	if _, err := q.GetAll(c, &posts); err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}
	updatePostCache(posts)
}

func ListPosts(rw http.ResponseWriter, req *http.Request) {
	c := appengine.NewContext(req)
	q := datastore.NewQuery("Post").Order("-Date").Limit(100)
	posts := make([]Post, 0, 100)

	if _, err := q.GetAll(c, &posts); err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}
	updatePostCache(posts)

	FormattedPosts := make([]PostFormatted, 0)

	for _, v := range posts {
		if !strings.HasPrefix(v.Slug, "DRAFT-") {
			newpost := PostFormatted{
				Author:  v.Author,
				Content: v.Content,
				Date:    v.Date.Format("2006-01-02 15:04:05"),
				Slug:    v.Slug,
				Title:   v.Title,
			}
			FormattedPosts = append(FormattedPosts, newpost)
		}
	}

	layoutData := struct {
		Posts []PostFormatted
	}{
		Posts: FormattedPosts,
	}

	err := HomeTemplate.Execute(rw, layoutData)
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
	}

}
<commit_msg>Fix findReccomendations only reccomending drafts<commit_after>package main

import (
	"encoding/base64"
	"fmt"
	"math/rand"
	"net/http"
	"sort"
	"strings"
	"text/template"
	"time"

	"github.com/codegangsta/martini"
	"github.com/russross/blackfriday"

	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
)

var PostTemplate = template.Must(template.ParseFiles("public2/pagetempl.html"))
var HomeTemplate = template.Must(template.ParseFiles("public2/hometempl.html"))

type Post struct {
	Author  string
	Content string `datastore:",noindex"`
	Date    time.Time
	Slug    string
	Title   string
	Type    string // Possible types [Comedy,Hardware,Mystery,Networking,Problem,Quirk]
	R1      string // Override of the Rec 1
	R2      string // Override of the Rec 2
}

type PostFormatted struct {
	Author  string
	Content string `datastore:",noindex"`
	Date    string
	Slug    string
	Title   string
}

func main() {
	m := martini.Classic()
	m.Get("/post/:name", ReadPost)
	m.Get("/raw/:name", ReadRawPost)
	m.Post("/admin/new", PublishPost)
	m.Get("/admin/", Admin)
	m.Get("/", ListPosts)
	m.Get("/all", ListPosts)
	m.Get("/rss.xml", GetRSS)
	m.Get("/sitemap.xml", GetSitemap)
	m.Get("/admin/run_gc", Run_GC)
	m.Get("/admin/backup.tar", Producebackup)
	m.Get("/admin/remove/:name", RemovePost)
	m.Post("/admin/uploadfile", UploadFile)

	m.Get("/lessons/:year/:month/:day/:title", MigrateOldURLS)
	m.Get("/lessons/:year/:month/:day/:title/", MigrateOldURLS)
	m.Get("/errors/:year/:month/:day/:title", MigrateOldURLS)
	m.Get("/errors/:year/:month/:day/:title/", MigrateOldURLS)
	m.Get("/posts/errors/:year/:month/:day/:title", MigrateOldURLS)
	m.Get("/posts/errors/:year/:month/:day/:title/", MigrateOldURLS)
	m.Get("/asset/:tag", ReadFile)

	m.Use(func(res http.ResponseWriter, req *http.Request) {
		res.Header().Add("Cache-Control", "public")
		res.Header().Add("X-Served-By", GimmeDC(res, req))
		res.Header().Add("X-Served-For", req.Header.Get("CF-RAY"))
	})

	m.Use(martini.Static("public2"))

	http.Handle("/", m)

	PostTitleCache = make(map[string]Post)
	appengine.Main()
}

func MigrateOldURLS(rw http.ResponseWriter, req *http.Request, params martini.Params) {
	http.Redirect(rw, req, fmt.Sprintf("https://blog.benjojo.co.uk/post/%s-%s-%s-%s.md", params["year"], params["month"], params["day"], params["title"]), http.StatusMovedPermanently)
}

func ReadPost(rw http.ResponseWriter, req *http.Request, params martini.Params) {
	// c := appengine.NewContext(r)
	// key := datastore.NewIncompleteKey(c, "Greeting", PostKey(c))
	// _, err := datastore.Put(c, key, &g)
	if len(PostTitleCache) == 0 {
		forceUpdatePostTitleCache(rw, req)
	}

	c := appengine.NewContext(req)
	k := datastore.NewKey(c, "Post", params["name"], 0, nil)
	post := Post{}
	err := datastore.Get(c, k, &post)

	if err != nil {
		if fmt.Sprint(err) == "datastore: no such entity" {
			http.Error(rw, "This blog post cannot be found, Please check your URL", http.StatusNotFound)
			return
		}

		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}

	if post.Type != "" {
		findReccomendations(&post)
	}

	postd, _ := base64.StdEncoding.DecodeString(post.Content)
	// post.Content = strings.Replace(string(postd), "\n", "\r\n\r\n", -1)
	post.Content = string(postd)
	output := blackfriday.MarkdownCommon([]byte(post.Content))
	lines := strings.Split(string(postd), "\n")
	layoutData := struct {
		Title   string
		Content string
		Date    string
		// Rec links at the bottom
		HasReccomendations bool
		FirstRecLink       string
		FirstTitle         string
		FirstYear          int
		SecondRecLink      string
		SecondTitle        string
		SecondYear         int
		RandomLink         string
		RandomTitle        string
		RandomYear         int
	}{
		Title:   lines[0],
		Content: string(output),
		Date:    post.Date.Format("Jan 2 2006"),
	}
	if post.R1 != "" && post.R2 != "" {
		layoutData.HasReccomendations = true
		layoutData.FirstRecLink = "/post/" + post.R1
		layoutData.FirstTitle = PostTitleCache[post.R1].Title
		layoutData.FirstYear = PostTitleCache[post.R1].Date.Year()
		layoutData.SecondRecLink = "/post/" + post.R2
		layoutData.SecondTitle = PostTitleCache[post.R2].Title
		layoutData.SecondYear = PostTitleCache[post.R2].Date.Year()

		for _, v := range PostTitleCache {
			if !strings.HasPrefix(v.Slug, "DRAFT") {
				layoutData.RandomLink = "/post/" + v.Slug
				layoutData.RandomTitle = PostTitleCache[v.Slug].Title
				layoutData.RandomYear = PostTitleCache[v.Slug].Date.Year()
				break
			}
		}

	}

	err = PostTemplate.Execute(rw, layoutData)
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
	}
}

func findReccomendations(Incoming *Post) {
	rand.Seed(Incoming.Date.Unix())
	// Make an array of candidates
	Candidates := make([]string, 0)
	for _, v := range PostTitleCache {
		if v.Type == Incoming.Type {
			if v.Date.Unix() < Incoming.Date.Unix() {
				// If the post is older
				if !strings.HasPrefix("DRAFT-", v.Title) {
					Candidates = append(Candidates, v.Slug)
				}
			}
		}
	}
	if len(Candidates) < 2 {
		return
	}

	sort.Strings(Candidates)

	if Incoming.R1 == "" {
		Incoming.R1 = Candidates[rand.Intn(len(Candidates)-1)]
	}

	if Incoming.R2 == "" {
		Incoming.R2 = Candidates[rand.Intn(len(Candidates)-1)]
	}
}

func ReadRawPost(rw http.ResponseWriter, req *http.Request, params martini.Params) {
	c := appengine.NewContext(req)
	k := datastore.NewKey(c, "Post", params["name"], 0, nil)
	post := Post{}
	err := datastore.Get(c, k, &post)
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}

	postd, _ := base64.StdEncoding.DecodeString(post.Content)
	rw.Write(postd)
}

var PostTitleCache map[string]Post

func updatePostCache(Posts []Post) {
	for _, v := range Posts {
		PostTitleCache[v.Slug] = v
	}
}

func forceUpdatePostTitleCache(rw http.ResponseWriter, req *http.Request) {
	c := appengine.NewContext(req)
	q := datastore.NewQuery("Post").Order("-Date").Limit(100)
	posts := make([]Post, 0, 100)

	if _, err := q.GetAll(c, &posts); err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}
	updatePostCache(posts)
}

func ListPosts(rw http.ResponseWriter, req *http.Request) {
	c := appengine.NewContext(req)
	q := datastore.NewQuery("Post").Order("-Date").Limit(100)
	posts := make([]Post, 0, 100)

	if _, err := q.GetAll(c, &posts); err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}
	updatePostCache(posts)

	FormattedPosts := make([]PostFormatted, 0)

	for _, v := range posts {
		if !strings.HasPrefix(v.Slug, "DRAFT-") {
			newpost := PostFormatted{
				Author:  v.Author,
				Content: v.Content,
				Date:    v.Date.Format("2006-01-02 15:04:05"),
				Slug:    v.Slug,
				Title:   v.Title,
			}
			FormattedPosts = append(FormattedPosts, newpost)
		}
	}

	layoutData := struct {
		Posts []PostFormatted
	}{
		Posts: FormattedPosts,
	}

	err := HomeTemplate.Execute(rw, layoutData)
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
	}

}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net"
	"os"

	"github.com/concourse/atc/auth/provider"
	"github.com/concourse/fly/commands"
	"github.com/concourse/fly/rc"
	"github.com/concourse/fly/ui"
	"github.com/concourse/go-concourse/concourse"
	"github.com/jessevdk/go-flags"

	_ "github.com/concourse/atc/auth/genericoauth"
	_ "github.com/concourse/atc/auth/github"
	_ "github.com/concourse/atc/auth/uaa"
)

func main() {
	parser := flags.NewParser(&commands.Fly, flags.HelpFlag|flags.PassDoubleDash)
	parser.NamespaceDelimiter = "-"

	setTeamCommand := parser.Find("set-team")
	authConfigs := make(provider.AuthConfigs)

	for name, p := range provider.GetProviders() {
		authConfigs[name] = p.AddAuthGroup(setTeamCommand.Group)
	}

	commands.Fly.SetTeam.ProviderAuth = authConfigs

	_, err := parser.Parse()
	if err != nil {
		if err == concourse.ErrUnauthorized {
			fmt.Fprintln(ui.Stderr, "not authorized. run the following to log in:")
			fmt.Fprintln(ui.Stderr, "")
			fmt.Fprintln(ui.Stderr, "    "+ui.Embolden("fly -t %s login", commands.Fly.Target))
			fmt.Fprintln(ui.Stderr, "")
		} else if err == rc.ErrNoTargetSpecified {
			fmt.Fprintln(ui.Stderr, "no target specified. specify the target with "+ui.Embolden("-t")+" or log in like so:")
			fmt.Fprintln(ui.Stderr, "")
			fmt.Fprintln(ui.Stderr, "    "+ui.Embolden("fly -t (alias) login -c (concourse url)"))
			fmt.Fprintln(ui.Stderr, "")
		} else if versionErr, ok := err.(rc.ErrVersionMismatch); ok {
			fmt.Fprintln(ui.Stderr, versionErr.Error())
			fmt.Fprintln(ui.Stderr, ui.WarningColor("cowardly refusing to run due to significant version discrepancy"))
		} else if netErr, ok := err.(net.Error); ok {
			fmt.Fprintf(ui.Stderr, "could not reach the Concourse server called %s:\n", ui.Embolden("%s", commands.Fly.Target))

			fmt.Fprintln(ui.Stderr, "")
			fmt.Fprintln(ui.Stderr, "    "+ui.Embolden("%s", netErr))
			fmt.Fprintln(ui.Stderr, "")
			fmt.Fprintln(ui.Stderr, "is the targeted Concourse running? better go catch it lol")
		} else if err == commands.ErrShowHelpMessage {
			helpParser := flags.NewParser(&commands.Fly, flags.HelpFlag)
			helpParser.NamespaceDelimiter = "-"
			helpParser.ParseArgs([]string{"-h"})
			helpParser.WriteHelp(os.Stdout)
			os.Exit(0)
		} else {
			fmt.Fprintf(ui.Stderr, "error: %s\n", err)
		}

		os.Exit(1)
	}
}
<commit_msg>fix setting up auth flags<commit_after>package main

import (
	"fmt"
	"net"
	"os"

	"github.com/concourse/atc/auth/provider"
	"github.com/concourse/fly/commands"
	"github.com/concourse/fly/rc"
	"github.com/concourse/fly/ui"
	"github.com/concourse/go-concourse/concourse"
	"github.com/jessevdk/go-flags"

	_ "github.com/concourse/atc/auth/genericoauth"
	_ "github.com/concourse/atc/auth/github"
	_ "github.com/concourse/atc/auth/uaa"
)

func main() {
	parser := flags.NewParser(&commands.Fly, flags.HelpFlag|flags.PassDoubleDash)
	parser.NamespaceDelimiter = "-"

	setTeamCommand := parser.Find("set-team")
	authConfigs := make(provider.AuthConfigs)

	for name, p := range provider.GetProviders() {
		authGroup := p.AuthGroup()

		group, err := setTeamCommand.Group.AddGroup(authGroup.Name(), "", authGroup.AuthConfig())
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}

		group.Namespace = authGroup.Namespace()

		authConfigs[name] = authGroup.AuthConfig()
	}

	commands.Fly.SetTeam.ProviderAuth = authConfigs

	_, err := parser.Parse()
	if err != nil {
		if err == concourse.ErrUnauthorized {
			fmt.Fprintln(ui.Stderr, "not authorized. run the following to log in:")
			fmt.Fprintln(ui.Stderr, "")
			fmt.Fprintln(ui.Stderr, "    "+ui.Embolden("fly -t %s login", commands.Fly.Target))
			fmt.Fprintln(ui.Stderr, "")
		} else if err == rc.ErrNoTargetSpecified {
			fmt.Fprintln(ui.Stderr, "no target specified. specify the target with "+ui.Embolden("-t")+" or log in like so:")
			fmt.Fprintln(ui.Stderr, "")
			fmt.Fprintln(ui.Stderr, "    "+ui.Embolden("fly -t (alias) login -c (concourse url)"))
			fmt.Fprintln(ui.Stderr, "")
		} else if versionErr, ok := err.(rc.ErrVersionMismatch); ok {
			fmt.Fprintln(ui.Stderr, versionErr.Error())
			fmt.Fprintln(ui.Stderr, ui.WarningColor("cowardly refusing to run due to significant version discrepancy"))
		} else if netErr, ok := err.(net.Error); ok {
			fmt.Fprintf(ui.Stderr, "could not reach the Concourse server called %s:\n", ui.Embolden("%s", commands.Fly.Target))

			fmt.Fprintln(ui.Stderr, "")
			fmt.Fprintln(ui.Stderr, "    "+ui.Embolden("%s", netErr))
			fmt.Fprintln(ui.Stderr, "")
			fmt.Fprintln(ui.Stderr, "is the targeted Concourse running? better go catch it lol")
		} else if err == commands.ErrShowHelpMessage {
			helpParser := flags.NewParser(&commands.Fly, flags.HelpFlag)
			helpParser.NamespaceDelimiter = "-"
			helpParser.ParseArgs([]string{"-h"})
			helpParser.WriteHelp(os.Stdout)
			os.Exit(0)
		} else {
			fmt.Fprintf(ui.Stderr, "error: %s\n", err)
		}

		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"time"

	"github.com/MasteryConnect/skrape/lib/config"
	"github.com/MasteryConnect/skrape/lib/mysqlutils"
	"github.com/MasteryConnect/skrape/lib/setup"
	"github.com/MasteryConnect/skrape/lib/skrape"
	"github.com/MasteryConnect/skrape/lib/utility"
	"github.com/apex/log"
	"github.com/apex/log/handlers/level"
	"github.com/apex/log/handlers/text"
	"github.com/codegangsta/cli"
)

const Concurrency = 10

// cli flag vars
var (
	mysqlDumpPath         string
	host                  string
	port                  string
	user                  string
	database              string
	table                 string
	dest                  string
	pool                  int
	matchTables           bool
	skrapePwd             bool
	priority              cli.StringSlice
	exclude               cli.StringSlice
	kinesisStreamName     string
	kinesisStreamEndpoint string
	kinesisShardCount     int
	awsRegion             string
)

func init() {
}

func main() {
	log.SetHandler(level.New(text.New(os.Stdout), log.DebugLevel))

	app := cli.NewApp()
	app.Name = "skrape"
	app.Usage = "export MySQL RDBMS tables"
	app.Version = "1.0"
	// Global flags used by every command
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:        "u, username",
			Usage:       "username for mysql server",
			Value:       "root",
			Destination: &user,
		},
		cli.StringFlag{
			Name:        "P, port",
			Usage:       "host name for mysql server",
			Value:       "3306",
			Destination: &port,
		},
		cli.StringFlag{
			Name:        "H, hostname",
			Usage:       "host name for mysql server",
			Value:       "127.0.0.1",
			Destination: &host,
		},
		cli.StringFlag{
			Name:        "D, database",
			Usage:       "targeted database",
			Value:       "",
			Destination: &database,
		},
		cli.StringFlag{
			Name:        "t, table",
			Usage:       "name of the table to be exported",
			Value:       "",
			Destination: &table,
		},
		cli.StringFlag{
			Name:        "mysqldump-path",
			Usage:       "set the path for the location of the mysqldump binary (defaults to typical locations for mac/linux)",
			Value:       "",
			Destination: &mysqlDumpPath,
		},
		cli.StringFlag{
			Name:        "e, export-path",
			Usage:       "set the path where you wish to export the CSV files",
			Value:       "./",
			Destination: &dest,
		},
		cli.IntFlag{
			Name:        "C, concurrency",
			Usage:       "number of concurrent goroutines to be spawned (use with caution, this consumes resources)",
			Value:       10,
			Destination: &pool,
		},
		cli.StringSliceFlag{
			Name:  "priority",
			Usage: "declare larger tables as priority (will start these tables exporting first)",
			Value: &priority,
		},
		cli.StringSliceFlag{
			Name:  "exclude",
			Usage: "exclude tables from the export",
			Value: &exclude,
		},
		cli.BoolFlag{
			Name:        "M, match-table-count",
			Usage:       "set concurrency level to the number of tables being exported",
			Destination: &matchTables,
		},
		cli.BoolFlag{
			Name:        "p, skip-pass",
			Usage:       "do not prompt for password, instead use the env var",
			Destination: &skrapePwd,
			EnvVar:      "SKRAPE_PWD",
		},
	}
	// Commands available. Defaults to s3 if no command given
	app.Commands = []cli.Command{
		{
			Name:    "s3",
			Aliases: []string{"s"},
			Usage:   "export to csv files that are uploaded to s3",
			Action: func(c *cli.Context) error {
				return action(c, "s3")
			},
		},
		{
			Name:    "csv",
			Aliases: []string{"c"},
			Usage:   "export to csv files",
			Action: func(c *cli.Context) error {
				return action(c, "csv")
			},
		},
		{
			Name:    "kinesis",
			Aliases: []string{"k"},
			Usage:   "export to an AWS Kinesis stream",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:        "s, stream-name",
					Usage:       "Kinesis stream name. If the stream name includes the string {TABLE_NAME}, it will be replaced by the table name, allowing for 1 stream per table",
					Destination: &kinesisStreamName,
				},
				cli.StringFlag{
					Name:        "e, stream-endpoint",
					Usage:       "Kinesis stream URL endpoint",
					Destination: &kinesisStreamEndpoint,
				},
				cli.StringFlag{
					Name:        "r, region",
					Usage:       "AWS region",
					Destination: &awsRegion,
					EnvVar:      "AWS_REGION",
				},
				cli.IntFlag{
					Name:        "c, shard-count",
					Usage:       "number of shards for this stream",
					Value:       1,
					Destination: &kinesisShardCount,
				},
			},
			Action: func(c *cli.Context) error {
				return action(c, "kinesis")
			},
		},
	}
	// Default action if no command specified
	app.Action = func(c *cli.Context) error {
		return action(c, "s3")
	}
	app.Run(os.Args)
}

func action(c *cli.Context, sinkType string) error {
	start := time.Now()
	defer utility.Cleanup(setup.DefaultFile)
	defer func(start time.Time) { // Displays duration of run time
		log.WithFields(log.Fields{
			"Duration": time.Since(start).String(),
		}).Info("Export Completed")
	}(start)

	mysqlutils.VerifyMysqldump(mysqlDumpPath)                                                      // make sure that mysqldump is installed
	connect := setup.NewConnection(host, user, port, database, dest, pool, matchTables, skrapePwd) // new connection struct
	extract := skrape.NewExtract(
		sinkType,
		config.NewConfig(
			connect,
			awsRegion,
			kinesisStreamEndpoint,
			kinesisStreamName,
			kinesisShardCount,
		),
	)
	if !connect.Missing() {
		log.Error("Missing credentials for database connection")
		os.Exit(1)
	}
	setup.MysqlDefaults(skrapePwd) // set up defaults file in /tmp to store DB credentials

	if table != "" {
		log.Infof("Performing single table extract for: %s", table)
		chn := make(chan bool)

		go extract.Perform(chn, table)
		chn <- true

	} else {
		extract.TableHandler(priority, exclude)
	}

	return nil
}
<commit_msg>updated cli import to pin it to v1 and point to new repo<commit_after>package main

import (
	"os"
	"time"

	"github.com/MasteryConnect/skrape/lib/config"
	"github.com/MasteryConnect/skrape/lib/mysqlutils"
	"github.com/MasteryConnect/skrape/lib/setup"
	"github.com/MasteryConnect/skrape/lib/skrape"
	"github.com/MasteryConnect/skrape/lib/utility"
	"github.com/apex/log"
	"github.com/apex/log/handlers/level"
	"github.com/apex/log/handlers/text"
	"gopkg.in/urfave/cli.v1"
)

const Concurrency = 10

// cli flag vars
var (
	mysqlDumpPath         string
	host                  string
	port                  string
	user                  string
	database              string
	table                 string
	dest                  string
	pool                  int
	matchTables           bool
	skrapePwd             bool
	priority              cli.StringSlice
	exclude               cli.StringSlice
	kinesisStreamName     string
	kinesisStreamEndpoint string
	kinesisShardCount     int
	awsRegion             string
)

func init() {
}

func main() {
	log.SetHandler(level.New(text.New(os.Stdout), log.DebugLevel))

	app := cli.NewApp()
	app.Name = "skrape"
	app.Usage = "export MySQL RDBMS tables"
	app.Version = "1.0"
	// Global flags used by every command
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:        "u, username",
			Usage:       "username for mysql server",
			Value:       "root",
			Destination: &user,
		},
		cli.StringFlag{
			Name:        "P, port",
			Usage:       "host name for mysql server",
			Value:       "3306",
			Destination: &port,
		},
		cli.StringFlag{
			Name:        "H, hostname",
			Usage:       "host name for mysql server",
			Value:       "127.0.0.1",
			Destination: &host,
		},
		cli.StringFlag{
			Name:        "D, database",
			Usage:       "targeted database",
			Value:       "",
			Destination: &database,
		},
		cli.StringFlag{
			Name:        "t, table",
			Usage:       "name of the table to be exported",
			Value:       "",
			Destination: &table,
		},
		cli.StringFlag{
			Name:        "mysqldump-path",
			Usage:       "set the path for the location of the mysqldump binary (defaults to typical locations for mac/linux)",
			Value:       "",
			Destination: &mysqlDumpPath,
		},
		cli.StringFlag{
			Name:        "e, export-path",
			Usage:       "set the path where you wish to export the CSV files",
			Value:       "./",
			Destination: &dest,
		},
		cli.IntFlag{
			Name:        "C, concurrency",
			Usage:       "number of concurrent goroutines to be spawned (use with caution, this consumes resources)",
			Value:       10,
			Destination: &pool,
		},
		cli.StringSliceFlag{
			Name:  "priority",
			Usage: "declare larger tables as priority (will start these tables exporting first)",
			Value: &priority,
		},
		cli.StringSliceFlag{
			Name:  "exclude",
			Usage: "exclude tables from the export",
			Value: &exclude,
		},
		cli.BoolFlag{
			Name:        "M, match-table-count",
			Usage:       "set concurrency level to the number of tables being exported",
			Destination: &matchTables,
		},
		cli.BoolFlag{
			Name:        "p, skip-pass",
			Usage:       "do not prompt for password, instead use the env var",
			Destination: &skrapePwd,
			EnvVar:      "SKRAPE_PWD",
		},
	}
	// Commands available. Defaults to s3 if no command given
	app.Commands = []cli.Command{
		{
			Name:    "s3",
			Aliases: []string{"s"},
			Usage:   "export to csv files that are uploaded to s3",
			Action: func(c *cli.Context) error {
				return action(c, "s3")
			},
		},
		{
			Name:    "csv",
			Aliases: []string{"c"},
			Usage:   "export to csv files",
			Action: func(c *cli.Context) error {
				return action(c, "csv")
			},
		},
		{
			Name:    "kinesis",
			Aliases: []string{"k"},
			Usage:   "export to an AWS Kinesis stream",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:        "s, stream-name",
					Usage:       "Kinesis stream name. If the stream name includes the string {TABLE_NAME}, it will be replaced by the table name, allowing for 1 stream per table",
					Destination: &kinesisStreamName,
				},
				cli.StringFlag{
					Name:        "e, stream-endpoint",
					Usage:       "Kinesis stream URL endpoint",
					Destination: &kinesisStreamEndpoint,
				},
				cli.StringFlag{
					Name:        "r, region",
					Usage:       "AWS region",
					Destination: &awsRegion,
					EnvVar:      "AWS_REGION",
				},
				cli.IntFlag{
					Name:        "c, shard-count",
					Usage:       "number of shards for this stream",
					Value:       1,
					Destination: &kinesisShardCount,
				},
			},
			Action: func(c *cli.Context) error {
				return action(c, "kinesis")
			},
		},
	}
	// Default action if no command specified
	app.Action = func(c *cli.Context) error {
		return action(c, "s3")
	}
	app.Run(os.Args)
}

func action(c *cli.Context, sinkType string) error {
	start := time.Now()
	defer utility.Cleanup(setup.DefaultFile)
	defer func(start time.Time) { // Displays duration of run time
		log.WithFields(log.Fields{
			"Duration": time.Since(start).String(),
		}).Info("Export Completed")
	}(start)

	mysqlutils.VerifyMysqldump(mysqlDumpPath)                                                      // make sure that mysqldump is installed
	connect := setup.NewConnection(host, user, port, database, dest, pool, matchTables, skrapePwd) // new connection struct
	extract := skrape.NewExtract(
		sinkType,
		config.NewConfig(
			connect,
			awsRegion,
			kinesisStreamEndpoint,
			kinesisStreamName,
			kinesisShardCount,
		),
	)
	if !connect.Missing() {
		log.Error("Missing credentials for database connection")
		os.Exit(1)
	}
	setup.MysqlDefaults(skrapePwd) // set up defaults file in /tmp to store DB credentials

	if table != "" {
		log.Infof("Performing single table extract for: %s", table)
		chn := make(chan bool)

		go extract.Perform(chn, table)
		chn <- true

	} else {
		extract.TableHandler(priority, exclude)
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
    "flag"
    "runtime"
    "os"
    "fmt"
    "time"
    "log"
)

// Version of boom
const BoomVersion = "1.0"

var (
    // -cpu: The cpu to use
    cpuToUse int
    // -help: Show help message then exit
    showHelpUsage bool
    // -l: Enable log output
    showLogs bool
    // -V: Show version of boom then exit
    showVersion bool
)

// Options of boom
type BoomOptions struct {
    // -A: Supply BASIC Authentication credentials to the server.
    // The username and password are separated by a single : .
    authentication             string

    // -C: Add a Cookie: line to the request like: cookie-name=value
    requestCookies             string

    // -c: Content-type header to use for POST/PUT data,
    // eg. application/x-www-form-urlencoded. Default is text/plain.
    requestPostDataContentType string

    // -D: File or just a string containing data to POST. Remember to also set -c.
    requestPostData            string

    // -g: Number of threads(goroutines) to perform for the test.
    requestGoroutines          int

    // -H: Append extra headers to the request like: head-type:value
    requestHeaders             string

    // -k: Enable the HTTP KeepAlive feature
    enableKeepAlive            bool

    // -la: local address
    localAddress               string

    // -m: Custom HTTP method for the requests.
    requestMethod              string

    // -t: Duration of this test.
    requestDuration            time.Duration

    // -u： The url to request
    requestUrl                 string
    // -o: Output the reports in specified location
    resultOutput               string

    // -R: Generate reports in [text, json, plot]
    generateReports            string
    // -r: Number of requests to perform at one sec.
    requestPerSec              int
    // -P: Specify SSL/TLS protocol .
    requestProtocol            string
    // -s: Maximum number of seconds to wait before a request times out.
    requestTimeout             time.Duration
}

// Parse command line args
func parseArgs() (*BoomOptions, error) {
    boomOpts := &BoomOptions{
        requestGoroutines:1000,
        requestDuration:1,
        enableKeepAlive:false,
        requestTimeout:30,
    }
    flag.StringVar(&boomOpts.authentication, "A", "", "Supply BASIC Authentication credentials to the server. " +
        "The username and password are separated by a single : .")
    flag.StringVar(&boomOpts.requestCookies, "C", "", "Add a Cookie: line to the request like: cookie-name=value")
    flag.IntVar(&cpuToUse, "cpu", 1, "The cpu to use when sending requests")
    flag.StringVar(&boomOpts.requestPostDataContentType, "c", "", "Content-type header to use for POST/PUT data, " +
        "eg. application/x-www-form-urlencoded. Default is text/plain.")
    flag.StringVar(&boomOpts.requestPostData, "D", "", "File or just a string containing data to POST. Remember to also set -c.")
    flag.IntVar(&boomOpts.requestGoroutines, "g", 100, " Number of threads(goroutines) to perform for the test.")
    flag.StringVar(&boomOpts.requestHeaders, "H", "", "Append extra headers to the request like: head-type:value")
    flag.BoolVar(&boomOpts.enableKeepAlive, "k", false, "Enable the HTTP KeepAlive feature")
    flag.BoolVar(&showLogs, "l", false, "Enable log output")
    flag.StringVar(&boomOpts.localAddress, "la", "", "Local address")
    flag.StringVar(&boomOpts.requestMethod, "m", "GET", "Custom HTTP method for the requests.")
    flag.DurationVar(&boomOpts.requestDuration, "t", time.Second, "Duration of this test.")
    flag.StringVar(&boomOpts.requestUrl, "u", "", "The url to request")
    flag.StringVar(&boomOpts.resultOutput, "o", "", "Output the reports in specified location")
    flag.StringVar(&boomOpts.requestProtocol, "P", "HTTP", "Specify SSL/TLS protocol .")
    flag.DurationVar(&boomOpts.requestTimeout, "s", 30 * time.Second, "Maximum number of seconds to wait before a request times out.")
    flag.IntVar(&boomOpts.requestPerSec, "r", 50, "Number of requests to perform at one sec.")
    flag.StringVar(&boomOpts.generateReports, "R", "", "Generate reports in [text, json, plot]")
    flag.BoolVar(&showVersion, "V", false, " Show version of boom then exit")
    flag.Parse()
    return boomOpts, nil
}

// Show usage information
func usage() {
    usage := `aaaa`
    fmt.Print(usage)
    os.Exit(0)
}

// Main entrance
func main() {
    boomOpts, err := parseArgs()
    if err != nil {
        log.Fatal("Error accured when parsing command line args:" + err.Error())
        os.Exit(1)
    }
    if showVersion {
        fmt.Println(BoomVersion)
        os.Exit(0)
    }
    if showHelpUsage {
        usage()
        os.Exit(0)
    }
    // set GOMAXPROCS
    runtime.GOMAXPROCS(cpuToUse)

    // start boom
    boom(boomOpts)
}
<commit_msg>Complete usage()<commit_after>package main

import (
    "flag"
    "runtime"
    "os"
    "fmt"
    "time"
    "log"
    "io/ioutil"
)

// Version of boom
const BoomVersion = "boom version 1.0"

var (
    // -cpu: The cpu to use
    cpuToUse int
    // -h: Show help message then exit
    showHelpUsage bool
    // -l: Enable log output
    showLogs bool
    // -V: Show version of boom then exit
    showVersion bool
)

// Options of boom
type BoomOptions struct {
    // -A: Supply BASIC Authentication credentials to the server.
    // The username and password are separated by a single : .
    authentication             string

    // -C: Add a Cookie: line to the request like: cookie-name=value
    requestCookies             string

    // -c: Content-type header to use for POST/PUT data,
    // eg. application/x-www-form-urlencoded. Default is text/plain.
    requestPostDataContentType string

    // -D: File or just a string containing data to POST. Remember to also set -c.
    requestPostData            string

    // -g: Number of threads(goroutines) to perform for the test.
    requestGoroutines          int

    // -H: Append extra headers to the request like: head-type:value
    requestHeaders             string

    // -k: Enable the HTTP KeepAlive feature
    enableKeepAlive            bool

    // -la: local address
    localAddress               string

    // -m: Custom HTTP method for the requests.
    requestMethod              string

    // -t: Duration of this test.
    requestDuration            time.Duration

    // -u： The url to request
    requestUrl                 string
    // -o: Output the reports in specified location
    resultOutput               string

    // -R: Generate reports in [text, json, plot]
    generateReports            string
    // -r: Number of requests to perform at one sec.
    requestPerSec              int
    // -P: Specify SSL/TLS protocol .
    requestProtocol            string
    // -s: Maximum number of seconds to wait before a request times out.
    requestTimeout             time.Duration
}

// Parse command line args
func parseArgs() *BoomOptions {
    boomOpts := &BoomOptions{
        requestMethod:"GET",
        requestPerSec:1000,
        requestGoroutines:100,
        requestDuration: 1 * time.Second,
        enableKeepAlive:false,
        requestTimeout: 30 * time.Second,
    }
    flag.StringVar(&boomOpts.authentication, "A", "", "Supply BASIC Authentication credentials to the server. " +
        "The username and password are separated by a single : .")
    flag.StringVar(&boomOpts.requestCookies, "C", "", "Add a Cookie: line to the request like: cookie-name=value")
    flag.IntVar(&cpuToUse, "cpu", 1, "The cpu to use when sending requests")
    flag.StringVar(&boomOpts.requestPostDataContentType, "c", "", "Content-type header to use for POST/PUT data, " +
        "eg. application/x-www-form-urlencoded. Default is text/plain.")
    flag.StringVar(&boomOpts.requestPostData, "D", "", "File or just a string containing data to POST. Remember to also set -c." +
        "When using a file for input, remember add '@@' prefix to the file path. eg. @@/home/work/a.json")
    flag.IntVar(&boomOpts.requestGoroutines, "g", 100, " Number of threads(goroutines) to perform for the test.")
    flag.StringVar(&boomOpts.requestHeaders, "H", "", "Append extra headers to the request like: head-type:value")
    flag.BoolVar(&boomOpts.enableKeepAlive, "k", false, "Enable the HTTP KeepAlive feature")
    flag.BoolVar(&showLogs, "l", false, "Enable log output")
    flag.StringVar(&boomOpts.localAddress, "la", "", "Local address  to bind to when making outgoing connections.")
    flag.StringVar(&boomOpts.requestMethod, "m", "GET", "Custom HTTP method for the requests.")
    flag.DurationVar(&boomOpts.requestDuration, "t", time.Second, "Duration of this test.")
    flag.StringVar(&boomOpts.requestUrl, "u", "", "The url to request")
    flag.StringVar(&boomOpts.resultOutput, "o", "", "Output the reports in specified location")
    flag.StringVar(&boomOpts.requestProtocol, "P", "HTTP", "Specify SSL/TLS protocol .")
    flag.DurationVar(&boomOpts.requestTimeout, "s", 30 * time.Second, "Maximum number of seconds to wait before a request times out.")
    flag.IntVar(&boomOpts.requestPerSec, "r", 50, "Number of requests to perform at one sec.")
    flag.StringVar(&boomOpts.generateReports, "R", "", "Generate reports in [text, json, plot]")
    flag.BoolVar(&showVersion, "V", false, " Show version of boom then exit")
    flag.Parse()

    return boomOpts
}

// Show usage information
func usage() {
    usage := ` -A string
        Supply BASIC Authentication credentials to the server. The username and password are separated by a single : .
  -C string
        Add a Cookie: line to the request like: cookie-name=value
  -D string
        File or just a string containing data to POST. Remember to also set -c.When using a file for input, remember add '@@' prefix to the file path. eg. @@/home/work/a.json
  -H string
        Append extra headers to the request like: head-type:value
  -P string
        Specify SSL/TLS protocol . (default "HTTP")
  -R string
        Generate reports in [text, json, plot]
  -V     Show version of boom then exit
  -c string
        Content-type header to use for POST/PUT data, eg. application/x-www-form-urlencoded. Default is text/plain.
  -cpu int
        The cpu to use when sending requests (default 1)
  -g int
         Number of threads(goroutines) to perform for the test. (default 100)
  -k    Enable the HTTP KeepAlive feature
  -l    Enable log output
  -la string
        Local address  to bind to when making outgoing connections.
  -m string
        Custom HTTP method for the requests. (default "GET")
  -o string
        Output the reports in specified location
  -r int
        Number of requests to perform at one sec. (default 50)
  -s duration
        Maximum number of seconds to wait before a request times out. (default 30s)
  -t duration
        Duration of this test. (default 1s)
  -u string
        The url to request
`
    fmt.Print(usage)
    os.Exit(0)
}

// Main entrance
func main() {
    boomOpts := parseArgs()

    if showVersion {
        fmt.Println(BoomVersion)
        os.Exit(0)
    }
    if len(os.Args[1:]) == 0 {
        usage()
        os.Exit(0)
    }
    if !showLogs {
        log.SetOutput(ioutil.Discard)
    } else {
        log.SetOutput(os.Stdout)
    }

    // set GOMAXPROCS
    runtime.GOMAXPROCS(cpuToUse)

    log.Printf("Starting boom ...")
    // start boom
    boom(boomOpts)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"log"
	"math/rand"
	"sync"

	"github.com/AdRoll/goamz/aws"
	"github.com/AdRoll/goamz/sqs"
)

//QuestionQ : name of the questions SQS Queue
const QuestionQ = "demo-questions"

//AnswerQ : name of the answers SQS Queue
const AnswerQ = "demo-answers"

//SQS instace of the sqs var from goamz
var SQS *sqs.SQS

var wg sync.WaitGroup

//Question : struct that holds the data for the math problem.
type Question struct {
	Num1 int
	Num2 int
}

var sem = make(chan int, 250)

func main() {

}

func Serve(queue chan []sqs.Message, sqsQ sqs.Queue) {
	for q := range queue {
		sem <- 1
		go func() {
			q := q
			sem <- 1
			go func() {
				addToQuestionQ(q, sqsQ)
				<-sem
			}()
		}()
	}
}

func makeRun(public string, secret string, maxworkers int) error {

	auth := aws.Auth{AccessKey: public, SecretKey: secret}
	region := aws.Region{}
	region.Name = "us-west-2"
	region.SQSEndpoint = "http://sqs.us-west-2.amazonaws.com"
	SQS = sqs.New(auth, region)
	if SQS == nil {
		return fmt.Errorf("Can't get sqs reference for %v %v", auth, region)
	}
	questionq, getErr := SQS.GetQueue(QuestionQ)
	if getErr != nil {
		fmt.Println(QuestionQ)
		return getErr
	}

	msgSlice := []sqs.Message{}
	for i := 0; i < 100000; i++ {
		num1 := rand.Intn(9) + 1
		num2 := rand.Intn(9 + 1)
		q := Question{num1, num2}
		jsonQ, _ := json.Marshal(&q)
		msg := sqs.Message{}
		body := base64.StdEncoding.EncodeToString(jsonQ)
		msg.Body = body
		msgSlice = append(msgSlice, msg)
		if len(msgSlice) == 10 {
			//the slice is ready, time to process it
			//I guess we pass it to the Serve function
		}
	}

	for i := 0; i < 250; i++ {
		var msgList []sqs.Message
		for c := 0; c < 10; c++ {
			num1 := rand.Intn(9) + 1
			num2 := rand.Intn(9 + 1)
			q := Question{num1, num2}
			jsonQ, _ := json.Marshal(&q)
			msg := sqs.Message{}
			body := base64.StdEncoding.EncodeToString(jsonQ)
			msg.Body = body
			msgList = append(msgList, msg)
		}
		wg.Add(1)
		go addToQuestionQ(msgList, *questionq)
	}
	wg.Wait()
	return nil
}

func addToQuestionQ(msgList []sqs.Message, sqsQ sqs.Queue) {
	_, respErr := sqsQ.SendMessageBatch(msgList)
	if respErr != nil {
		log.Println("ERROR:", respErr)
	}
	wg.Done()
}
<commit_msg>this doesn't work, but I'm closer<commit_after>package main

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"log"
	"math/rand"

	"github.com/AdRoll/goamz/aws"
	"github.com/AdRoll/goamz/sqs"
)

//QuestionQName : name of the questions SQS Queue
const QuestionQName = "demo-questions"

//AnswerQName : name of the answers SQS Queue
const AnswerQName = "demo-answers"

//SQS instance of the sqs var from goamz
var SQS *sqs.SQS

//Question : struct that holds the data for the math problem.
type Question struct {
	Num1 int
	Num2 int
}

var sem = make(chan int, 250)

func main() {

}

func Serve(queue chan []sqs.Message, sqsQ *sqs.Queue) {
	log.Println("In Serve")
	for q := range queue {
		log.Println("In range")
		sem <- 1
		go func() {
			q := q
			addToQuestionQ(q, *sqsQ)
			<-sem
		}()
	}
	log.Println("Done")
}

func makeRun(public string, secret string, maxworkers int) error {

	auth := aws.Auth{AccessKey: public, SecretKey: secret}
	region := aws.Region{}
	region.Name = "us-west-2"
	region.SQSEndpoint = "http://sqs.us-west-2.amazonaws.com"
	SQS = sqs.New(auth, region)
	if SQS == nil {
		return fmt.Errorf("Can't get sqs reference for %v %v", auth, region)
	}
	questionq, getErr := SQS.GetQueue(QuestionQName)
	if getErr != nil {
		fmt.Println(QuestionQName)
		return getErr
	}

	msgSlice := []sqs.Message{}
	var queue chan []sqs.Message
	log.Println("Before Serve")
	Serve(queue, questionq)
	log.Println("After Serve")

	for i := 0; i < 100000; i++ {
		num1 := rand.Intn(9) + 1
		num2 := rand.Intn(9 + 1)
		q := Question{num1, num2}
		jsonQ, _ := json.Marshal(&q)
		msg := sqs.Message{}
		body := base64.StdEncoding.EncodeToString(jsonQ)
		msg.Body = body
		msgSlice = append(msgSlice, msg)
		if len(msgSlice) == 10 {
			//the slice is ready, time to process it
			//I guess we pass it to the Serve function
			log.Println("Send to Channel")
			queue <- msgSlice
			msgSlice = []sqs.Message{}
		}
	}

	return nil
}

func addToQuestionQ(msgList []sqs.Message, sqsQ sqs.Queue) {
	_, respErr := sqsQ.SendMessageBatch(msgList)
	if respErr != nil {
		log.Println("ERROR:", respErr)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"io/ioutil"
	"log"
	"net/http"
	"strconv"
	"strings"
)

var (
	warpAddr      = flag.String("warp.addr", "127.0.0.1:4242", "Address of sensision")
	listenAddress = flag.String("web.listen-address", ":9121", "Address to listen on for web interface and telemetry.")
	metricPath    = flag.String("web.telemetry-path", "/metrics", "Path under which to expose metrics.")
	debug         = flag.Bool("debug", false, "Debug mode displays what is fetch.")
)

type warp struct {
	warpAddr string
	metrics  map[string]warpMetric
}

type warpMetric struct {
	desc    *prometheus.Desc
	valType prometheus.ValueType
}

func parseFloatOrZero(s string) float64 {
	res, err := strconv.ParseFloat(s, 64)
	if err != nil {
		log.Printf("Cannot parse %s\n", s)
		return 0.0
	}
	return res
}

func NewWarpExporter(addr string) *warp {
	e := warp{
		warpAddr: addr,
		metrics: map[string]warpMetric{
			"warp_ingress_update_requests": {
				desc:    prometheus.NewDesc("warp_ingress_update_requests", "Number of request", nil, nil),
				valType: prometheus.CounterValue,
			},
			"warp_ingress_update_parseerrors": {
				desc:    prometheus.NewDesc("warp_ingress_update_parseerrors", "Number of parse error", nil, nil),
				valType: prometheus.CounterValue,
			},
			"warp_ingress_update_invalidtoken": {
				desc:    prometheus.NewDesc("warp_ingress_update_invalidtoken", "Number of invalid token", nil, nil),
				valType: prometheus.CounterValue,
			},
			"warp_directory_streaming_requests": {
				desc: prometheus.NewDesc("warp_directory_streaming_requests", "Number of directory request", nil, nil),
				valType: prometheus.CounterValue,
			},
			"warp_directory_metadata_cache_hits": {
				desc: prometheus.NewDesc("warp_directory_metadata_cache_hits", "Number of directory cache hit", nil, nil),
				valType: prometheus.GaugeValue,
			},
			"warp_directory_hbase_puts": {
				desc: prometheus.NewDesc("warp_directory_hbase_puts", "Number of directory hbase puts", nil, nil),
				valType: prometheus.GaugeValue,
			},
			"warp_directory_kafka_faileddecrypts" : {
				desc: prometheus.NewDesc("warp_directory_kafka_faileddecrypts", "Fail to descrypt kafka message", nil, nil),
				valType: prometheus.CounterValue,
			},
			"warp_directory_gts" : {
				desc: prometheus.NewDesc("warp_directory_gts", "Number of GTS metadata managed by Directory", nil, nil),
				valType: prometheus.GaugeValue,
			},
			"warp_store_hbase_puts_committed": {
				desc: prometheus.NewDesc("warp_store_hbase_puts_committed", "Number of HBase Puts committed in Store", nil, nil),
			},
		},
	}
	return &e
}

func (w *warp) scrapeSensisionMetrics(ch chan<- prometheus.Metric) {
	resp, err := http.Get("http://" + w.warpAddr + "/metrics")
	if err == nil {
		defer resp.Body.Close()
		body, _ := ioutil.ReadAll(resp.Body)
		for _, line := range strings.Split(string(body), "\n") {
			tokens := strings.Split(strings.Trim(line, "\t\r\n"), " ")
			if len(tokens) == 3 {
				metric := strings.Replace(tokens[1], ".", "_", -1)
				metric = metric[:strings.IndexRune(metric, '{')]
				if val, ok := w.metrics[metric]; ok {
					value := parseFloatOrZero(tokens[2])
					if *debug {
						log.Printf("Sensision output %s\n", line)
						log.Printf("Metric name %s, value %f\n", metric, value)
					}
					ch <- prometheus.MustNewConstMetric(val.desc, val.valType, value)
				}
			}
		}
	} else {
		log.Printf("Cannot fetch sensision output\n")
	}
}

func (w *warp) Describe(ch chan<- *prometheus.Desc) {
	for _, i := range w.metrics {
		ch <- i.desc
	}
}

func (w *warp) Collect(ch chan<- prometheus.Metric) {
	if *debug {
		log.Printf("Start collecting")
	}
	w.scrapeSensisionMetrics(ch)
	if *debug {
		log.Printf("Stop collecting")
	}
}

func main() {
	flag.Parse()
	e := NewWarpExporter(*warpAddr)
	prometheus.MustRegister(e)
	http.Handle(*metricPath, promhttp.Handler())
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`
		       <title>Warp10 exporter</title>

Warp10 exporter

<a href='` + *metricPath + `'>Metrics

		       `))
	})
	log.Printf("providing metrics at %s%s", *listenAddress, *metricPath)
	log.Fatal(http.ListenAndServe(*listenAddress, nil))
}
<commit_msg>Running golint and govet<commit_after>package main

import (
	"flag"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"io/ioutil"
	"log"
	"net/http"
	"strconv"
	"strings"
)

var (
	warpAddr      = flag.String("warp.addr", "127.0.0.1:4242", "Address of sensision")
	listenAddress = flag.String("web.listen-address", ":9121", "Address to listen on for web interface and telemetry.")
	metricPath    = flag.String("web.telemetry-path", "/metrics", "Path under which to expose metrics.")
	debug         = flag.Bool("debug", false, "Debug mode displays what is fetch.")
)

type warp struct {
	warpAddr string
	metrics  map[string]warpMetric
}

type warpMetric struct {
	desc    *prometheus.Desc
	valType prometheus.ValueType
}

func parseFloatOrZero(s string) float64 {
	res, err := strconv.ParseFloat(s, 64)
	if err != nil {
		log.Printf("Cannot parse %s\n", s)
		return 0.0
	}
	return res
}

func newWarpExporter(addr string) *warp {
	e := warp{
		warpAddr: addr,
		metrics: map[string]warpMetric{
			"warp_ingress_update_requests": {
				desc:    prometheus.NewDesc("warp_ingress_update_requests", "Number of request", nil, nil),
				valType: prometheus.CounterValue,
			},
			"warp_ingress_update_parseerrors": {
				desc:    prometheus.NewDesc("warp_ingress_update_parseerrors", "Number of parse error", nil, nil),
				valType: prometheus.CounterValue,
			},
			"warp_ingress_update_invalidtoken": {
				desc:    prometheus.NewDesc("warp_ingress_update_invalidtoken", "Number of invalid token", nil, nil),
				valType: prometheus.CounterValue,
			},
			"warp_directory_streaming_requests": {
				desc:    prometheus.NewDesc("warp_directory_streaming_requests", "Number of directory request", nil, nil),
				valType: prometheus.CounterValue,
			},
			"warp_directory_metadata_cache_hits": {
				desc:    prometheus.NewDesc("warp_directory_metadata_cache_hits", "Number of directory cache hit", nil, nil),
				valType: prometheus.GaugeValue,
			},
			"warp_directory_hbase_puts": {
				desc:    prometheus.NewDesc("warp_directory_hbase_puts", "Number of directory hbase puts", nil, nil),
				valType: prometheus.GaugeValue,
			},
			"warp_directory_kafka_faileddecrypts": {
				desc:    prometheus.NewDesc("warp_directory_kafka_faileddecrypts", "Fail to descrypt kafka message", nil, nil),
				valType: prometheus.CounterValue,
			},
			"warp_directory_gts": {
				desc:    prometheus.NewDesc("warp_directory_gts", "Number of GTS metadata managed by Directory", nil, nil),
				valType: prometheus.GaugeValue,
			},
			"warp_store_hbase_puts_committed": {
				desc: prometheus.NewDesc("warp_store_hbase_puts_committed", "Number of HBase Puts committed in Store", nil, nil),
			},
		},
	}
	return &e
}

func (w *warp) scrapeSensisionMetrics(ch chan<- prometheus.Metric) {
	resp, err := http.Get("http://" + w.warpAddr + "/metrics")
	if err == nil {
		defer resp.Body.Close()
		body, _ := ioutil.ReadAll(resp.Body)
		for _, line := range strings.Split(string(body), "\n") {
			tokens := strings.Split(strings.Trim(line, "\t\r\n"), " ")
			if len(tokens) == 3 {
				metric := strings.Replace(tokens[1], ".", "_", -1)
				metric = metric[:strings.IndexRune(metric, '{')]
				if val, ok := w.metrics[metric]; ok {
					value := parseFloatOrZero(tokens[2])
					if *debug {
						log.Printf("Sensision output %s\n", line)
						log.Printf("Metric name %s, value %f\n", metric, value)
					}
					ch <- prometheus.MustNewConstMetric(val.desc, val.valType, value)
				}
			}
		}
	} else {
		log.Printf("Cannot fetch sensision output\n")
	}
}

func (w *warp) Describe(ch chan<- *prometheus.Desc) {
	for _, i := range w.metrics {
		ch <- i.desc
	}
}

func (w *warp) Collect(ch chan<- prometheus.Metric) {
	if *debug {
		log.Printf("Start collecting")
	}
	w.scrapeSensisionMetrics(ch)
	if *debug {
		log.Printf("Stop collecting")
	}
}

func main() {
	flag.Parse()
	e := NewWarpExporter(*warpAddr)
	prometheus.MustRegister(e)
	http.Handle(*metricPath, promhttp.Handler())
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`
		       <title>Warp10 exporter</title>

Warp10 exporter

<a href='` + *metricPath + `'>Metrics

		       `))
	})
	log.Printf("providing metrics at %s%s", *listenAddress, *metricPath)
	log.Fatal(http.ListenAndServe(*listenAddress, nil))
}
<|endoftext|>


<commit_before>// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bufio"
	"flag"
	"os"
	"strings"

	"github.com/briandowns/logcolor/renderers"
)

var textChan = make(chan string)         // channel to pass log lines for processing
var formattedChan = make(chan string)    // channel to pass formatted text back
var signalChan = make(chan os.Signal, 1) // channel to catch ctrl-c

// TODO: replace code below with strings.Contains() and strings.Replace()
func process() {
	for {
		select {
		case t := <-textChan:
			brokenLine := strings.Split(t, " ")
			for i, s := range brokenLine {
				select {
				case renderers.HTTP.ExistsInBadLines(s):
					var formatted string
					brokenLine[i] = renderers.ColorBad(s)
					formattedChan <- stirngs.Join(brokenLine, " ")
				case renderers.HTTP.ExistsInWarnWords(s):
					brokenLine[i] = renderers.ColorBad(s)
					formattedChan <- stirngs.Join(brokenLine, " ")
				case renderers.HTTP.ExistsInGoodWords(s):
					brokenLine[i] = renderers.ColorBad(s)
					formattedChan <- stirngs.Join(brokenLine, " ")
				default:
					formattedChan <- s
				}
			}
		}
	}
}

// Pointers to hold the contents of the flag args.
var (
	templateFlag = flag.String("t", "", "template to use for log parsing")
)

func main() {
	flag.Parse()

	signal.Notify(signalChan, os.Interrupt)
	// setup go routine to catch a ctrl-c
	go func() {
		for range signalChan {
			os.Exit(1)
		}
	}()
	go process()
	for {
		reader := bufio.NewReader(os.Stdin)
		text, err := reader.ReadString('\n')
		if err != nil {
			fmt.Println("unable to read line")
			os.Exit(1)
		}
		textChan <- text
	}
	os.Exit(0)
}
<commit_msg>usage statement<commit_after>// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bufio"
	"flag"
	"os"
	"strings"

	"github.com/briandowns/logcolor/renderers"
)

var textChan = make(chan string)         // channel to pass log lines for processing
var formattedChan = make(chan string)    // channel to pass formatted text back
var signalChan = make(chan os.Signal, 1) // channel to catch ctrl-c

// TODO: replace code below with strings.Contains() and strings.Replace()
func process() {
	for {
		select {
		case t := <-textChan:
			brokenLine := strings.Split(t, " ")
			for i, s := range brokenLine {
				select {
				case renderers.HTTP.ExistsInBadLines(s):
					var formatted string
					brokenLine[i] = renderers.ColorBad(s)
					formattedChan <- stirngs.Join(brokenLine, " ")
				case renderers.HTTP.ExistsInWarnWords(s):
					brokenLine[i] = renderers.ColorBad(s)
					formattedChan <- stirngs.Join(brokenLine, " ")
				case renderers.HTTP.ExistsInGoodWords(s):
					brokenLine[i] = renderers.ColorBad(s)
					formattedChan <- stirngs.Join(brokenLine, " ")
				default:
					formattedChan <- s
				}
			}
		}
	}
}

// Pointers to hold the contents of the flag args.
var (
	templateFlag = flag.String("t", "", "template to use for log parsing")
)

const USAGE = `Usage: logcolor -t template [-h]`

func main() {
	flag.Parse()

	signal.Notify(signalChan, os.Interrupt)
	// setup go routine to catch a ctrl-c
	go func() {
		for range signalChan {
			os.Exit(1)
		}
	}()

	if len(*templateFlag) != 1 {
		fmt.Println(USAGE)
		os.Exit(1)
	}

	go process()
	for {
		reader := bufio.NewReader(os.Stdin)
		text, err := reader.ReadString('\n')
		if err != nil {
			fmt.Println("unable to read line")
			os.Exit(1)
		}
		textChan <- text
	}
	os.Exit(0)
}
<|endoftext|>


<commit_before>/*
Copyright (c) 2016, UPMC Enterprises
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in the
      documentation and/or other materials provided with the distribution.
    * Neither the name UPMC Enterprises nor the
      names of its contributors may be used to endorse or promote products
      derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL UPMC ENTERPRISES BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
*/

package main

import (
	"fmt"
	"log"
	"os"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/ecr"
	flag "github.com/spf13/pflag"
	"golang.org/x/net/context"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/client/restclient"
	"k8s.io/kubernetes/pkg/client/unversioned"
	kubectl_util "k8s.io/kubernetes/pkg/kubectl/cmd/util"
)

const (
	dockerCfgTemplate  = `{"%s":{"username":"oauth2accesstoken","password":"%s","email":"none"}}`
	dockerJSONTemplate = `{"auths":{"%s":{"auth":"%s","email":"none"}}}`
)

var (
	flags               = flag.NewFlagSet("", flag.ContinueOnError)
	cluster             = flags.Bool("use-kubernetes-cluster-service", true, `If true, use the built in kubernetes cluster for creating the client`)
	argKubecfgFile      = flags.String("kubecfg-file", "", `Location of kubecfg file for access to kubernetes master service; --kube_master_url overrides the URL part of this; if neither this nor --kube_master_url are provided, defaults to service account tokens`)
	argKubeMasterURL    = flags.String("kube-master-url", "", `URL to reach kubernetes master. Env variables in this flag will be expanded.`)
	argAWSSecretName    = flags.String("aws-secret-name", "awsecr-cred", `Default aws secret name`)
	argGCRSecretName    = flags.String("gcr-secret-name", "gcr-secret", `Default gcr secret name`)
	argDefaultNamespace = flags.String("default-namespace", "default", `Default namespace`)
	argGCRURL           = flags.String("gcr-url", "https://gcr.io", `Default GCR URL`)
	argAWSRegion        = flags.String("aws-region", "us-east-1", `Default AWS region`)
	argRefreshMinutes   = flags.Int("refresh-mins", 60, `Default time to wait before refreshing (60 minutes)`)
)

var (
	awsAccountID string
)

type controller struct {
	kubeClient kubeInterface
	ecrClient  ecrInterface
	gcrClient  gcrInterface
}

type kubeInterface interface {
	Secrets(namespace string) unversioned.SecretsInterface
	Namespaces() unversioned.NamespaceInterface
	ServiceAccounts(namespace string) unversioned.ServiceAccountsInterface
}

type ecrInterface interface {
	GetAuthorizationToken(input *ecr.GetAuthorizationTokenInput) (*ecr.GetAuthorizationTokenOutput, error)
}

type gcrInterface interface {
	DefaultTokenSource(ctx context.Context, scope ...string) (oauth2.TokenSource, error)
}

func newEcrClient() ecrInterface {
	return ecr.New(session.New(), aws.NewConfig().WithRegion(*argAWSRegion))
}

type gcrClient struct{}

func (gcr gcrClient) DefaultTokenSource(ctx context.Context, scope ...string) (oauth2.TokenSource, error) {
	return google.DefaultTokenSource(ctx, scope...)
}

func newGcrClient() gcrInterface {
	return gcrClient{}
}

func newKubeClient() kubeInterface {
	var kubeClient *unversioned.Client
	var config *restclient.Config
	var err error

	clientConfig := kubectl_util.DefaultClientConfig(flags)

	if *cluster {
		if kubeClient, err = unversioned.NewInCluster(); err != nil {
			log.Fatalf("Failed to create client: %v", err)
		}
	} else {
		config, err = clientConfig.ClientConfig()
		if err != nil {
			log.Fatalf("error connecting to the client: %v", err)
		}
		kubeClient, err = unversioned.New(config)

		if err != nil {
			log.Fatalf("Failed to create client: %v", err)
		}
	}

	return kubeClient
}

func (c *controller) getGCRAuthorizationKey() (AuthToken, error) {
	ts, err := c.gcrClient.DefaultTokenSource(context.TODO(), "https://www.googleapis.com/auth/cloud-platform")
	if err != nil {
		return AuthToken{}, err
	}

	token, err := ts.Token()
	if err != nil {
		return AuthToken{}, err
	}

	if !token.Valid() {
		return AuthToken{}, fmt.Errorf("token was invalid")
	}

	if token.Type() != "Bearer" {
		return AuthToken{}, fmt.Errorf(fmt.Sprintf("expected token type \"Bearer\" but got \"%s\"", token.Type()))
	}

	return AuthToken{
		AccessToken: token.AccessToken,
		Endpoint:    *argGCRURL}, nil
}

func (c *controller) getECRAuthorizationKey() (AuthToken, error) {
	params := &ecr.GetAuthorizationTokenInput{
		RegistryIds: []*string{
			aws.String(awsAccountID),
		},
	}

	resp, err := c.ecrClient.GetAuthorizationToken(params)

	if err != nil {
		// Print the error, cast err to awserr.Error to get the Code and
		// Message from an error.
		fmt.Println(err.Error())
		return AuthToken{}, err
	}

	token := resp.AuthorizationData[0]

	return AuthToken{
		AccessToken: *token.AuthorizationToken,
		Endpoint:    *token.ProxyEndpoint}, err
}

func generateSecretObj(token string, endpoint string, isJSONCfg bool, secretName string) *api.Secret {
	secret := &api.Secret{
		ObjectMeta: api.ObjectMeta{
			Name: secretName,
		},
	}
	if isJSONCfg {
		secret.Data = map[string][]byte{
			".dockerconfigjson": []byte(fmt.Sprintf(dockerJSONTemplate, endpoint, token))}
		secret.Type = "kubernetes.io/dockerconfigjson"
	} else {
		secret.Data = map[string][]byte{
			".dockercfg": []byte(fmt.Sprintf(dockerCfgTemplate, endpoint, token))}
		secret.Type = "kubernetes.io/dockercfg"
	}
	return secret
}

type AuthToken struct {
	AccessToken string
	Endpoint    string
}

type SecretGenerator struct {
	TokenGenFxn func() (AuthToken, error)
	IsJSONCfg   bool
	SecretName  string
}

func (c *controller) process() error {
	secretGenerators := []SecretGenerator{
		SecretGenerator{
			TokenGenFxn: c.getGCRAuthorizationKey,
			IsJSONCfg:   false,
			SecretName:  *argGCRSecretName,
		},
		SecretGenerator{
			TokenGenFxn: c.getECRAuthorizationKey,
			IsJSONCfg:   true,
			SecretName:  *argAWSSecretName,
		},
	}
	for _, secretGenerator := range secretGenerators {
		newToken, err := secretGenerator.TokenGenFxn()
		if err != nil {
			return err
		}
		newSecret := generateSecretObj(newToken.AccessToken, newToken.Endpoint, secretGenerator.IsJSONCfg, secretGenerator.SecretName)

		// Get all namespaces
		namespaces, err := c.kubeClient.Namespaces().List(api.ListOptions{})
		if err != nil {
			return err
		}

		for _, namespace := range namespaces.Items {

			if namespace.GetName() == "kube-system" {
				continue
			}

			// Check if the secret exists for the namespace
			_, err := c.kubeClient.Secrets(namespace.GetName()).Get(secretGenerator.SecretName)

			if err != nil {
				// Secret not found, create
				_, err := c.kubeClient.Secrets(namespace.GetName()).Create(newSecret)
				if err != nil {
					return err
				}
			} else {
				// Existing secret needs updated
				_, err := c.kubeClient.Secrets(namespace.GetName()).Update(newSecret)
				if err != nil {
					return err
				}
			}

			// Check if ServiceAccount exists
			serviceAccount, err := c.kubeClient.ServiceAccounts(namespace.GetName()).Get("default")

			if err != nil {
				return err
			}

			// Update existing one if image pull secrets already exists for aws ecr token
			imagePullSecretFound := false
			for i, imagePullSecret := range serviceAccount.ImagePullSecrets {
				if imagePullSecret.Name == secretGenerator.SecretName {
					serviceAccount.ImagePullSecrets[i] = api.LocalObjectReference{Name: secretGenerator.SecretName}
					imagePullSecretFound = true
					break
				}
			}

			// Append to list of existing service accounts if there isn't one already
			if !imagePullSecretFound {
				serviceAccount.ImagePullSecrets = append(serviceAccount.ImagePullSecrets, api.LocalObjectReference{Name: secretGenerator.SecretName})
			}

			_, err = c.kubeClient.ServiceAccounts(namespace.GetName()).Update(serviceAccount)
			if err != nil {
				return err
			}
		}
		log.Print("Finished processing secret for: ", secretGenerator.SecretName)
	}

	return nil
}

func validateParams() {
	awsAccountID = os.Getenv("awsaccount")
	if len(awsAccountID) == 0 {
		log.Print("Missing awsaccount env variable, assuming GCR usage")
	}

	awsRegionEnv := os.Getenv("awsregion")

	if len(awsRegionEnv) > 0 {
		argAWSRegion = &awsRegionEnv
	}
}

func main() {
	log.Print("Starting up...")
	flags.Parse(os.Args)

	validateParams()

	log.Print("Using AWS Account: ", awsAccountID)
	log.Printf("Using AWS Region: %s", *argAWSRegion)
	log.Print("Refresh Interval (minutes): ", *argRefreshMinutes)

	kubeClient := newKubeClient()
	ecrClient := newEcrClient()
	gcrClient := newGcrClient()
	c := &controller{kubeClient, ecrClient, gcrClient}

	tick := time.Tick(time.Duration(*argRefreshMinutes) * time.Minute)

	// Process once now, then wait for tick
	c.process()

	for {
		select {
		case <-tick:
			log.Print("Refreshing credentials...")
			if err := c.process(); err != nil {
				log.Fatalf("Failed to load ecr credentials: %v", err)
			}
		}
	}

}
<commit_msg>add a flag to toggle skipping kube-system<commit_after>/*
Copyright (c) 2016, UPMC Enterprises
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in the
      documentation and/or other materials provided with the distribution.
    * Neither the name UPMC Enterprises nor the
      names of its contributors may be used to endorse or promote products
      derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL UPMC ENTERPRISES BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
*/

package main

import (
	"fmt"
	"log"
	"os"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/ecr"
	flag "github.com/spf13/pflag"
	"golang.org/x/net/context"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/client/restclient"
	"k8s.io/kubernetes/pkg/client/unversioned"
	kubectl_util "k8s.io/kubernetes/pkg/kubectl/cmd/util"
)

const (
	dockerCfgTemplate  = `{"%s":{"username":"oauth2accesstoken","password":"%s","email":"none"}}`
	dockerJSONTemplate = `{"auths":{"%s":{"auth":"%s","email":"none"}}}`
)

var (
	flags               = flag.NewFlagSet("", flag.ContinueOnError)
	cluster             = flags.Bool("use-kubernetes-cluster-service", true, `If true, use the built in kubernetes cluster for creating the client`)
	argKubecfgFile      = flags.String("kubecfg-file", "", `Location of kubecfg file for access to kubernetes master service; --kube_master_url overrides the URL part of this; if neither this nor --kube_master_url are provided, defaults to service account tokens`)
	argKubeMasterURL    = flags.String("kube-master-url", "", `URL to reach kubernetes master. Env variables in this flag will be expanded.`)
	argAWSSecretName    = flags.String("aws-secret-name", "awsecr-cred", `Default aws secret name`)
	argGCRSecretName    = flags.String("gcr-secret-name", "gcr-secret", `Default gcr secret name`)
	argDefaultNamespace = flags.String("default-namespace", "default", `Default namespace`)
	argGCRURL           = flags.String("gcr-url", "https://gcr.io", `Default GCR URL`)
	argAWSRegion        = flags.String("aws-region", "us-east-1", `Default AWS region`)
	argRefreshMinutes   = flags.Int("refresh-mins", 60, `Default time to wait before refreshing (60 minutes)`)
	argSkipKubeSystem   = flags.Bool("skip-kube-system", true, `If true, will not attempt to set ImagePullSecrets on the kube-system namespace`)
)

var (
	awsAccountID string
)

type controller struct {
	kubeClient kubeInterface
	ecrClient  ecrInterface
	gcrClient  gcrInterface
}

type kubeInterface interface {
	Secrets(namespace string) unversioned.SecretsInterface
	Namespaces() unversioned.NamespaceInterface
	ServiceAccounts(namespace string) unversioned.ServiceAccountsInterface
}

type ecrInterface interface {
	GetAuthorizationToken(input *ecr.GetAuthorizationTokenInput) (*ecr.GetAuthorizationTokenOutput, error)
}

type gcrInterface interface {
	DefaultTokenSource(ctx context.Context, scope ...string) (oauth2.TokenSource, error)
}

func newEcrClient() ecrInterface {
	return ecr.New(session.New(), aws.NewConfig().WithRegion(*argAWSRegion))
}

type gcrClient struct{}

func (gcr gcrClient) DefaultTokenSource(ctx context.Context, scope ...string) (oauth2.TokenSource, error) {
	return google.DefaultTokenSource(ctx, scope...)
}

func newGcrClient() gcrInterface {
	return gcrClient{}
}

func newKubeClient() kubeInterface {
	var kubeClient *unversioned.Client
	var config *restclient.Config
	var err error

	clientConfig := kubectl_util.DefaultClientConfig(flags)

	if *cluster {
		if kubeClient, err = unversioned.NewInCluster(); err != nil {
			log.Fatalf("Failed to create client: %v", err)
		}
	} else {
		config, err = clientConfig.ClientConfig()
		if err != nil {
			log.Fatalf("error connecting to the client: %v", err)
		}
		kubeClient, err = unversioned.New(config)

		if err != nil {
			log.Fatalf("Failed to create client: %v", err)
		}
	}

	return kubeClient
}

func (c *controller) getGCRAuthorizationKey() (AuthToken, error) {
	ts, err := c.gcrClient.DefaultTokenSource(context.TODO(), "https://www.googleapis.com/auth/cloud-platform")
	if err != nil {
		return AuthToken{}, err
	}

	token, err := ts.Token()
	if err != nil {
		return AuthToken{}, err
	}

	if !token.Valid() {
		return AuthToken{}, fmt.Errorf("token was invalid")
	}

	if token.Type() != "Bearer" {
		return AuthToken{}, fmt.Errorf(fmt.Sprintf("expected token type \"Bearer\" but got \"%s\"", token.Type()))
	}

	return AuthToken{
		AccessToken: token.AccessToken,
		Endpoint:    *argGCRURL}, nil
}

func (c *controller) getECRAuthorizationKey() (AuthToken, error) {
	params := &ecr.GetAuthorizationTokenInput{
		RegistryIds: []*string{
			aws.String(awsAccountID),
		},
	}

	resp, err := c.ecrClient.GetAuthorizationToken(params)

	if err != nil {
		// Print the error, cast err to awserr.Error to get the Code and
		// Message from an error.
		fmt.Println(err.Error())
		return AuthToken{}, err
	}

	token := resp.AuthorizationData[0]

	return AuthToken{
		AccessToken: *token.AuthorizationToken,
		Endpoint:    *token.ProxyEndpoint}, err
}

func generateSecretObj(token string, endpoint string, isJSONCfg bool, secretName string) *api.Secret {
	secret := &api.Secret{
		ObjectMeta: api.ObjectMeta{
			Name: secretName,
		},
	}
	if isJSONCfg {
		secret.Data = map[string][]byte{
			".dockerconfigjson": []byte(fmt.Sprintf(dockerJSONTemplate, endpoint, token))}
		secret.Type = "kubernetes.io/dockerconfigjson"
	} else {
		secret.Data = map[string][]byte{
			".dockercfg": []byte(fmt.Sprintf(dockerCfgTemplate, endpoint, token))}
		secret.Type = "kubernetes.io/dockercfg"
	}
	return secret
}

type AuthToken struct {
	AccessToken string
	Endpoint    string
}

type SecretGenerator struct {
	TokenGenFxn func() (AuthToken, error)
	IsJSONCfg   bool
	SecretName  string
}

func (c *controller) process() error {
	secretGenerators := []SecretGenerator{
		SecretGenerator{
			TokenGenFxn: c.getGCRAuthorizationKey,
			IsJSONCfg:   false,
			SecretName:  *argGCRSecretName,
		},
		SecretGenerator{
			TokenGenFxn: c.getECRAuthorizationKey,
			IsJSONCfg:   true,
			SecretName:  *argAWSSecretName,
		},
	}
	for _, secretGenerator := range secretGenerators {
		newToken, err := secretGenerator.TokenGenFxn()
		if err != nil {
			return err
		}
		newSecret := generateSecretObj(newToken.AccessToken, newToken.Endpoint, secretGenerator.IsJSONCfg, secretGenerator.SecretName)

		// Get all namespaces
		namespaces, err := c.kubeClient.Namespaces().List(api.ListOptions{})
		if err != nil {
			return err
		}

		for _, namespace := range namespaces.Items {

			if *argSkipKubeSystem && namespace.GetName() == "kube-system" {
				continue
			}

			// Check if the secret exists for the namespace
			_, err := c.kubeClient.Secrets(namespace.GetName()).Get(secretGenerator.SecretName)

			if err != nil {
				// Secret not found, create
				_, err := c.kubeClient.Secrets(namespace.GetName()).Create(newSecret)
				if err != nil {
					return err
				}
			} else {
				// Existing secret needs updated
				_, err := c.kubeClient.Secrets(namespace.GetName()).Update(newSecret)
				if err != nil {
					return err
				}
			}

			// Check if ServiceAccount exists
			serviceAccount, err := c.kubeClient.ServiceAccounts(namespace.GetName()).Get("default")

			if err != nil {
				return err
			}

			// Update existing one if image pull secrets already exists for aws ecr token
			imagePullSecretFound := false
			for i, imagePullSecret := range serviceAccount.ImagePullSecrets {
				if imagePullSecret.Name == secretGenerator.SecretName {
					serviceAccount.ImagePullSecrets[i] = api.LocalObjectReference{Name: secretGenerator.SecretName}
					imagePullSecretFound = true
					break
				}
			}

			// Append to list of existing service accounts if there isn't one already
			if !imagePullSecretFound {
				serviceAccount.ImagePullSecrets = append(serviceAccount.ImagePullSecrets, api.LocalObjectReference{Name: secretGenerator.SecretName})
			}

			_, err = c.kubeClient.ServiceAccounts(namespace.GetName()).Update(serviceAccount)
			if err != nil {
				return err
			}
		}
		log.Print("Finished processing secret for: ", secretGenerator.SecretName)
	}

	return nil
}

func validateParams() {
	awsAccountID = os.Getenv("awsaccount")
	if len(awsAccountID) == 0 {
		log.Print("Missing awsaccount env variable, assuming GCR usage")
	}

	awsRegionEnv := os.Getenv("awsregion")

	if len(awsRegionEnv) > 0 {
		argAWSRegion = &awsRegionEnv
	}
}

func main() {
	log.Print("Starting up...")
	flags.Parse(os.Args)

	validateParams()

	log.Print("Using AWS Account: ", awsAccountID)
	log.Printf("Using AWS Region: %s", *argAWSRegion)
	log.Print("Refresh Interval (minutes): ", *argRefreshMinutes)

	kubeClient := newKubeClient()
	ecrClient := newEcrClient()
	gcrClient := newGcrClient()
	c := &controller{kubeClient, ecrClient, gcrClient}

	tick := time.Tick(time.Duration(*argRefreshMinutes) * time.Minute)

	// Process once now, then wait for tick
	c.process()

	for {
		select {
		case <-tick:
			log.Print("Refreshing credentials...")
			if err := c.process(); err != nil {
				log.Fatalf("Failed to load ecr credentials: %v", err)
			}
		}
	}

}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"

	// database import
	"database/sql"
	_ "github.com/lib/pq"
)

type Test struct {
	Message string
}

func handler(w http.ResponseWriter, r *http.Request) {

	db, err := sql.Open("postgres", "user=pqgotest dbname=pqgotest sslmode=verify-full")
	if err != nil {
		log.Fatal(err)
	}

	test := Test{
		fmt.Sprintf("Hi there, I love %s!", r.URL.Path[1:]),
	}

	b, err := json.Marshal(test)
	if err != nil {
		log.Fatal(err)
	}

	fmt.Fprintf(w, "%s", b)
}

func main() {
	http.HandleFunc("/", handler)
	http.ListenAndServe(":8080", nil)
}
<commit_msg>added pq postgresql library<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"

	// database import
	"database/sql"
	_ "github.com/lib/pq"
)

type Test struct {
	Message string
}

func handler(w http.ResponseWriter, r *http.Request) {

	db, err := sql.Open("postgres", "user=pqgotest dbname=pqgotest sslmode=verify-full")
	if err != nil {
		log.Fatal(err)
	}

	rows, err := db.Query(`SELECT name FROM users WHERE favorite_fruit = $1
	OR age BETWEEN $2 AND $2 + 3`, "orange", 64)
	if err != nil {
		log.Fatal(err)
	}
	defer rows.Close()

	test := Test{
		fmt.Sprintf("Hi there, I love %s!", r.URL.Path[1:]),
	}

	b, err := json.Marshal(test)
	if err != nil {
		log.Fatal(err)
	}

	fmt.Fprintf(w, "%s", b)
}

func main() {
	http.HandleFunc("/", handler)
	http.ListenAndServe(":8080", nil)
}
<|endoftext|>