<commit_before>package main

import (
	"crypto/sha1"
	"database/sql"
	"encoding/base64"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"html"
	"io"
	"log"
	"net/http"

	"github.com/dchest/uniuri"
	"github.com/ewhal/pygments"
	_ "github.com/go-sql-driver/mysql"
	"github.com/gorilla/mux"
)

const (
	ADDRESS  = "http://localhost:9900"
	LENGTH   = 6
	PORT     = ":9900"
	USERNAME = ""
	PASS     = ""
	NAME     = ""
	DATABASE = USERNAME + ":" + PASS + "@/" + NAME + "?charset=utf8"
)

type Response struct {
	ID     string `json:"id"`
	HASH   string `json:"hash"`
	URL    string `json:"url"`
	SIZE   int    `json:"size"`
	DELKEY string `json:"delkey"`
}

func check(err error) {
	if err != nil {
		fmt.Println(err)
	}
}

func generateName() string {
	s := uniuri.NewLen(LENGTH)
	db, err := sql.Open("mysql", DATABASE)
	check(err)

	query, err := db.Query("select id from pastebin")
	for query.Next() {
		var id string
		err := query.Scan(&id)
		if err != nil {

		}
		if id == s {
			generateName()
		}
	}
	db.Close()

	return s

}
func hash(paste string) string {
	hasher := sha1.New()

	hasher.Write([]byte(paste))
	sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil))
	return sha
}

func save(raw string, lang string) []string {
	db, err := sql.Open("mysql", DATABASE)
	check(err)

	sha := hash(raw)
	query, err := db.Query("select id, hash, data, delkey from pastebin")
	for query.Next() {
		var id, hash, paste, delkey string
		err := query.Scan(&id, &hash, &paste, &delkey)
		check(err)
		if hash == sha {
			url := ADDRESS + "/p/" + id
			return []string{id, hash, url, paste, delkey}
		}
	}
	id := generateName()
	var url string
	if lang == "" {
		url = ADDRESS + "/p/" + id
	} else {
		url = ADDRESS + "/p/" + id + "/" + lang
	}
	delKey := uniuri.NewLen(40)
	paste := html.EscapeString(raw)

	stmt, err := db.Prepare("INSERT INTO pastebin(id, hash, data, delkey) values(?,?,?,?)")
	check(err)
	_, err = stmt.Exec(id, sha, paste, delKey)
	check(err)
	db.Close()
	return []string{id, sha, url, paste, delKey}
}

func delHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	delkey := vars["delKey"]

	db, err := sql.Open("mysql", DATABASE)
	check(err)

	stmt, err := db.Prepare("delete from pastebin where delkey=?")
	check(err)

	res, err := stmt.Exec(html.EscapeString(delkey))
	check(err)

	affect, err := res.RowsAffected()
	if err == sql.ErrNoRows {
		io.WriteString(w, "Error invalid paste")
	} else {
		io.WriteString(w, paste+" deleted")
	}
	db.Close()

}
func saveHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	output := vars["output"]
	switch r.Method {
	case "POST":
		paste := r.FormValue("p")
		lang := r.FormValue("lang")
		if paste == "" {
			http.Error(w, "Empty paste", 500)
		}
		values := save(paste, lang)
		b := &Response{
			ID:     values[0],
			HASH:   values[1],
			URL:    values[2],
			SIZE:   len(values[3]),
			DELKEY: values[4],
		}

		switch output {
		case "json":

			w.Header().Set("Content-Type", "application/json")
			err := json.NewEncoder(w).Encode(b)

			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
		case "xml":
			x, err := xml.MarshalIndent(b, "", "  ")
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}

			w.Header().Set("Content-Type", "application/xml")
			w.Write(x)
		case "html":
			w.Header().Set("Content-Type", "text/html")
			io.WriteString(w, "
URL: "+b.URL+"
")
			io.WriteString(w, "
Delete Key: "+b.DELKEY+"
")

		default:
			io.WriteString(w, b.URL+"\n")
			io.WriteString(w, "delete key: "+b.DELKEY+"\n")
		}
	}

}

func langHandler(w http.ResponseWriter, r *http.Request) {

	vars := mux.Vars(r)
	paste := vars["pasteId"]
	lang := vars["lang"]
	s := getPaste(paste)
	highlight, err := pygments.Highlight(html.UnescapeString(s), html.EscapeString(lang), "html", "full, style=autumn,linenos=True, lineanchors=True,anchorlinenos=True,", "utf-8")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	io.WriteString(w, highlight)

}

func getPaste(paste string) string {
	param1 := html.EscapeString(paste)
	db, err := sql.Open("mysql", DATABASE)
	var s string
	err = db.QueryRow("select data from pastebin where id=?", param1).Scan(&s)
	db.Close()
	check(err)

	if err == sql.ErrNoRows {
		return "Error invalid paste"
	} else {
		return html.UnescapeString(s)
	}

}
func pasteHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	s := getPaste(paste)
	io.WriteString(w, s)

}

func main() {
	router := mux.NewRouter()
	router.HandleFunc("/p/{pasteId}", pasteHandler)
	router.HandleFunc("/p/{pasteId}/{lang}", langHandler)
	router.HandleFunc("/save", saveHandler)
	router.HandleFunc("/save/{output}", saveHandler)
	router.HandleFunc("/del/{pasteId}/{delKey}", delHandler)
	router.PathPrefix("/").Handler(http.StripPrefix("/", http.FileServer(http.Dir("assets/"))))
	err := http.ListenAndServe(PORT, router)
	if err != nil {
		log.Fatal(err)
	}

}
<commit_msg>Remove unused variable<commit_after>package main

import (
	"crypto/sha1"
	"database/sql"
	"encoding/base64"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"html"
	"io"
	"log"
	"net/http"

	"github.com/dchest/uniuri"
	"github.com/ewhal/pygments"
	_ "github.com/go-sql-driver/mysql"
	"github.com/gorilla/mux"
)

const (
	ADDRESS  = "http://localhost:9900"
	LENGTH   = 6
	PORT     = ":9900"
	USERNAME = ""
	PASS     = ""
	NAME     = ""
	DATABASE = USERNAME + ":" + PASS + "@/" + NAME + "?charset=utf8"
)

type Response struct {
	ID     string `json:"id"`
	HASH   string `json:"hash"`
	URL    string `json:"url"`
	SIZE   int    `json:"size"`
	DELKEY string `json:"delkey"`
}

func check(err error) {
	if err != nil {
		fmt.Println(err)
	}
}

func generateName() string {
	s := uniuri.NewLen(LENGTH)
	db, err := sql.Open("mysql", DATABASE)
	check(err)

	query, err := db.Query("select id from pastebin")
	for query.Next() {
		var id string
		err := query.Scan(&id)
		if err != nil {

		}
		if id == s {
			generateName()
		}
	}
	db.Close()

	return s

}
func hash(paste string) string {
	hasher := sha1.New()

	hasher.Write([]byte(paste))
	sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil))
	return sha
}

func save(raw string, lang string) []string {
	db, err := sql.Open("mysql", DATABASE)
	check(err)

	sha := hash(raw)
	query, err := db.Query("select id, hash, data, delkey from pastebin")
	for query.Next() {
		var id, hash, paste, delkey string
		err := query.Scan(&id, &hash, &paste, &delkey)
		check(err)
		if hash == sha {
			url := ADDRESS + "/p/" + id
			return []string{id, hash, url, paste, delkey}
		}
	}
	id := generateName()
	var url string
	if lang == "" {
		url = ADDRESS + "/p/" + id
	} else {
		url = ADDRESS + "/p/" + id + "/" + lang
	}
	delKey := uniuri.NewLen(40)
	paste := html.EscapeString(raw)

	stmt, err := db.Prepare("INSERT INTO pastebin(id, hash, data, delkey) values(?,?,?,?)")
	check(err)
	_, err = stmt.Exec(id, sha, paste, delKey)
	check(err)
	db.Close()
	return []string{id, sha, url, paste, delKey}
}

func delHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	delkey := vars["delKey"]

	db, err := sql.Open("mysql", DATABASE)
	check(err)

	stmt, err := db.Prepare("delete from pastebin where delkey=?")
	check(err)

	res, err := stmt.Exec(html.EscapeString(delkey))
	check(err)

	_, err := res.RowsAffected()
	if err == sql.ErrNoRows {
		io.WriteString(w, "Error invalid paste")
	} else {
		io.WriteString(w, paste+" deleted")
	}
	db.Close()

}
func saveHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	output := vars["output"]
	switch r.Method {
	case "POST":
		paste := r.FormValue("p")
		lang := r.FormValue("lang")
		if paste == "" {
			http.Error(w, "Empty paste", 500)
		}
		values := save(paste, lang)
		b := &Response{
			ID:     values[0],
			HASH:   values[1],
			URL:    values[2],
			SIZE:   len(values[3]),
			DELKEY: values[4],
		}

		switch output {
		case "json":

			w.Header().Set("Content-Type", "application/json")
			err := json.NewEncoder(w).Encode(b)

			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
		case "xml":
			x, err := xml.MarshalIndent(b, "", "  ")
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}

			w.Header().Set("Content-Type", "application/xml")
			w.Write(x)
		case "html":
			w.Header().Set("Content-Type", "text/html")
			io.WriteString(w, "
URL: "+b.URL+"
")
			io.WriteString(w, "
Delete Key: "+b.DELKEY+"
")

		default:
			io.WriteString(w, b.URL+"\n")
			io.WriteString(w, "delete key: "+b.DELKEY+"\n")
		}
	}

}

func langHandler(w http.ResponseWriter, r *http.Request) {

	vars := mux.Vars(r)
	paste := vars["pasteId"]
	lang := vars["lang"]
	s := getPaste(paste)
	highlight, err := pygments.Highlight(html.UnescapeString(s), html.EscapeString(lang), "html", "full, style=autumn,linenos=True, lineanchors=True,anchorlinenos=True,", "utf-8")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	io.WriteString(w, highlight)

}

func getPaste(paste string) string {
	param1 := html.EscapeString(paste)
	db, err := sql.Open("mysql", DATABASE)
	var s string
	err = db.QueryRow("select data from pastebin where id=?", param1).Scan(&s)
	db.Close()
	check(err)

	if err == sql.ErrNoRows {
		return "Error invalid paste"
	} else {
		return html.UnescapeString(s)
	}

}
func pasteHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	s := getPaste(paste)
	io.WriteString(w, s)

}

func main() {
	router := mux.NewRouter()
	router.HandleFunc("/p/{pasteId}", pasteHandler)
	router.HandleFunc("/p/{pasteId}/{lang}", langHandler)
	router.HandleFunc("/save", saveHandler)
	router.HandleFunc("/save/{output}", saveHandler)
	router.HandleFunc("/del/{pasteId}/{delKey}", delHandler)
	router.PathPrefix("/").Handler(http.StripPrefix("/", http.FileServer(http.Dir("assets/"))))
	err := http.ListenAndServe(PORT, router)
	if err != nil {
		log.Fatal(err)
	}

}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"strconv"

	"github.com/gizak/termui"
	"github.com/kolo/xmlrpc"
	"github.com/spf13/viper"
)

func main() {
	// handle configurations for server
	viper.SetConfigName("bapu")           // no need to include file extension
	viper.AddConfigPath("/usr/local/etc") // set the path of your config file
	viper.AddConfigPath("../bapu")        // set the path of your config file

	err := viper.ReadInConfig()
	if err != nil {
		log.Fatal(err)
	}

	var apiKey string
	var api *xmlrpc.Client

	production := viper.GetBool("production.enabled")
	if production {
		apiKey = viper.GetString("production.apiKey")
		api, err = xmlrpc.NewClient("https://rpc.gandi.net/xmlrpc/", nil)
		if err != nil {
			log.Fatal(err)
		}
	}

	development := viper.GetBool("development.enabled")
	if development {
		log.Println("Development Config found")
		api, err = xmlrpc.NewClient("https://rpc.ote.gandi.net/xmlrpc/", nil)
		if err != nil {
			log.Fatal(err)
		}
		apiKey = viper.GetString("development.apiKey")
	}

	if api == nil {
		log.Fatal("neither production nor development environment enabled in config")
	}

	// initialize termui
	err = termui.Init()
	if err != nil {
		panic(err)
	}
	defer termui.Close()

	// Title
	title := termui.NewPar("Bapu")
	title.Border = false
	title.Height = 1
	title.TextFgColor = termui.ColorMagenta
	title.Width = 10
	title.X = 1
	title.Y = 1

	// List
	strs := []string{
		"[0] github.com/gizak/termui",
		"[1] [你好，世界](fg-blue)",
		"[2] [こんにちは世界](fg-red)",
		"[3] [color output](fg-white,bg-green)",
		"[4] output.go",
		"[5] random_out.go",
		"[6] dashboard.go",
		"[7] nsf/termbox-go"}

	ls := termui.NewList()
	ls.Items = strs
	ls.ItemFgColor = termui.ColorYellow
	ls.BorderLabel = "Servers"
	ls.Height = 20
	ls.Width = 80
	ls.Y = 4

	termui.Render(title, ls)

	// Count number of instances
	var iaasCount *int
	err = api.Call("hosting.vm.count", apiKey, &iaasCount)
	if err != nil {
		log.Fatal(err)
	}

	count := termui.NewPar("# instances: " + strconv.Itoa(*iaasCount))
	count.Border = false
	count.Height = 1
	count.TextFgColor = termui.ColorMagenta
	count.Width = 20
	count.X = 1
	count.Y = 2

	termui.Render(count)

	// List instances
	//	var iaasList *int
	//	err = api.Call("iaas.count", apiKey, &iaasCount)
	//	if err != nil {
	//		log.Fatal(err)
	//	}

	// Quit with q
	termui.Handle("/sys/kbd/q", func(termui.Event) {
		termui.StopLoop()
	})

	termui.Handle("/sys/kbd/<up>", func(termui.Event) {
		termui.Render(ls)
	})

	termui.Loop()
}
<commit_msg>display server list<commit_after>package main

import (
	"log"
	"strconv"
	"time"

	"github.com/gizak/termui"
	"github.com/kolo/xmlrpc"
	"github.com/spf13/viper"
)

func main() {
	// handle configurations for server
	viper.SetConfigName("bapu")           // no need to include file extension
	viper.AddConfigPath("/usr/local/etc") // set the path of your config file
	viper.AddConfigPath("../bapu")        // set the path of your config file

	err := viper.ReadInConfig()
	if err != nil {
		log.Fatal(err)
	}

	var apiKey string
	var api *xmlrpc.Client

	production := viper.GetBool("production.enabled")
	if production {
		apiKey = viper.GetString("production.apiKey")
		api, err = xmlrpc.NewClient("https://rpc.gandi.net/xmlrpc/", nil)
		if err != nil {
			log.Fatal(err)
		}
	}

	development := viper.GetBool("development.enabled")
	if development {
		log.Println("Development Config found")
		api, err = xmlrpc.NewClient("https://rpc.ote.gandi.net/xmlrpc/", nil)
		if err != nil {
			log.Fatal(err)
		}
		apiKey = viper.GetString("development.apiKey")
	}

	if api == nil {
		log.Fatal("neither production nor development environment enabled in config")
	}

	// initialize termui
	err = termui.Init()
	if err != nil {
		panic(err)
	}
	defer termui.Close()

	// Title
	title := termui.NewPar("Bapu")
	title.Border = false
	title.Height = 1
	title.TextFgColor = termui.ColorMagenta
	title.Width = 10
	title.X = 1
	title.Y = 1

	termui.Render(title)

	// Count number of instances
	var hostingVMCount *int
	err = api.Call("hosting.vm.count", apiKey, &hostingVMCount)
	if err != nil {
		log.Fatal(err)
	}

	count := termui.NewPar("VM #: " + strconv.Itoa(*hostingVMCount))
	count.Border = false
	count.Height = 1
	count.TextFgColor = termui.ColorWhite
	count.Width = 20
	count.X = 1
	count.Y = 3

	termui.Render(count)

	// List instances
	type VMReturn struct {
		AiActive     int       `xmlrpc:"ai_active"`
		Console      int       `xmlrpc:"console"`
		Cores        int       `xmlrpc:"cores"`
		DatacenterID int       `xmlrpc:"datacenter_id"`
		DateCreated  time.Time `xmlrpc:"date_created"`
		DateUpdated  time.Time `xmlrpc:"date_updated"`
		Description  string    `xmlrpc:"description"`
		Hostname     string    `xmlrpc:"hostname"`
	}

	var hostingVMList *[]VMReturn
	err = api.Call("hosting.vm.list", apiKey, &hostingVMList)
	if err != nil {
		log.Fatal(err)
	}

	var strs []string
	list := *hostingVMList
	for _, val := range list {
		strs = append(strs, val.Hostname)
	}

	ls := termui.NewList()
	ls.Items = strs
	ls.ItemFgColor = termui.ColorYellow
	ls.BorderLabel = "Servers"
	ls.Height = 20
	ls.Width = 80
	ls.Y = 5

	termui.Render(ls)

	// Quit with q
	termui.Handle("/sys/kbd/q", func(termui.Event) {
		termui.StopLoop()
	})

	termui.Handle("/sys/kbd/<up>", func(termui.Event) {
		termui.Render(ls)
	})

	termui.Loop()
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"runtime"
	"strings"
	"time"

	"github.com/jmoiron/jsonq"
)

func main() {
	var (
		gaugePaths, counterPaths stringList
		metricsURL, source       string
		email, token             string
		period                   time.Duration
	)
	flag.StringVar(&metricsURL, "url", "", "URL of the service's metrics")
	flag.StringVar(&source, "source", "", "an optional source to use instead of the URL's host")
	flag.Var(&gaugePaths, "gauge", "the JSON path to a gauges's value")
	flag.Var(&counterPaths, "counter", "the JSON path to a counter's value")
	flag.StringVar(&email, "email", "", "Librato account email")
	flag.StringVar(&token, "token", "", "Librato account token")
	flag.DurationVar(&period, "period", 0, "send data periodically (0 for just once)")
	flag.Parse()

	if metricsURL == "" {
		fmt.Fprintln(os.Stderr, "No URL provided")
		flag.Usage()
		os.Exit(1)
	}

	if source == "" {
		u, err := url.Parse(metricsURL)
		if err != nil {
			panic(err)
		}
		source = u.Host
	}

	for _ = range ticker(period) {
		log.Printf("collecting %s", metricsURL)
		n := collect(metricsURL, source, email, token, gaugePaths, counterPaths)
		log.Printf("sent %d metrics", n)
	}
}

func collect(url, source, email, token string, gaugePaths, counterPaths stringList) int {
	defer func() {
		e := recover()
		if e != nil {
			log.Printf("panic: %v\n", e)
			for skip := 1; ; skip++ {
				pc, file, line, ok := runtime.Caller(skip)
				if !ok {
					break
				}
				if file[len(file)-1] == 'c' {
					continue
				}
				f := runtime.FuncForPC(pc)
				log.Printf("%s:%d %s()\n", file, line, f.Name())
			}
		}
	}()

	metrics := fetchMetrics(url)
	batch := batchMetrics(metrics, source, gaugePaths, counterPaths)
	postBatch(batch, email, token)

	return len(batch.Counters) + len(batch.Gauges)
}

func ticker(period time.Duration) <-chan time.Time {
	// if we're not doing periodic collections, return a closed channel with a
	// single time in it
	if period == 0 {
		c := make(chan time.Time, 1)
		c <- time.Now()
		close(c)
		return c
	}
	return time.Tick(period)
}

func postBatch(batch batch, email, token string) {
	j, err := json.Marshal(batch)
	if err != nil {
		panic(err)
	}

	r := bytes.NewReader(j)
	req, err := http.NewRequest("POST", "https://metrics-api.librato.com/v1/metrics", r)
	if err != nil {
		panic(err)
	}
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("Authorization", basicAuth(email, token))

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		panic(err)
	}
	defer func() {
		_, _ = io.Copy(ioutil.Discard, resp.Body)
		_ = resp.Body.Close()
	}()

	if resp.StatusCode != 200 {
		body := bytes.NewBuffer(nil)
		if _, err := io.Copy(body, resp.Body); err != nil {
			panic(err)
		}

		panic(fmt.Sprintf("received %s\n\n%s\n", resp.Status, body.String()))
	}
}

func basicAuth(u, p string) string {
	creds := base64.URLEncoding.EncodeToString([]byte(u + ":" + p))
	return fmt.Sprintf("Basic %s", creds)
}

type batch struct {
	Gauges   map[string]gauge   `json:"gauges"`
	Counters map[string]counter `json:"counters"`
	Source   string             `json:"source"`
}

type gauge struct {
	Value float64 `json:"value"`
}

type counter struct {
	Value int `json:"value"`
}

func batchMetrics(jq *jsonq.JsonQuery, source string, gauges, counters []string) batch {
	b := batch{
		Gauges:   make(map[string]gauge),
		Counters: make(map[string]counter),
		Source:   source,
	}

	for _, path := range gauges {
		v, err := jq.Float(strings.Split(path, ".")...)
		if err != nil {
			panic(err)
		}
		b.Gauges[path] = gauge{Value: v}
	}

	for _, path := range counters {
		v, err := jq.Int(strings.Split(path, ".")...)
		if err != nil {
			panic(err)
		}
		b.Counters[path] = counter{Value: v}
	}

	return b
}

func fetchMetrics(url string) *jsonq.JsonQuery {
	resp, err := http.Get(url)
	if err != nil {
		panic(err)
	}
	defer func() {
		_, _ = io.Copy(ioutil.Discard, resp.Body)
		_ = resp.Body.Close()
	}()

	if resp.StatusCode != 200 {
		panic("received a " + resp.Status + " response")
	}

	var metrics map[string]interface{}
	if err := json.NewDecoder(resp.Body).Decode(&metrics); err != nil {
		panic(err)
	}

	return jsonq.NewQuery(metrics)
}

type stringList []string

func (l *stringList) Set(v string) error {
	*l = append(*l, v)
	return nil
}

func (l *stringList) String() string {
	return strings.Join(*l, ",")
}
<commit_msg>Log collected metrics.<commit_after>package main

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"runtime"
	"strings"
	"time"

	"github.com/jmoiron/jsonq"
)

func main() {
	var (
		gaugePaths, counterPaths stringList
		metricsURL, source       string
		email, token             string
		period                   time.Duration
	)
	flag.StringVar(&metricsURL, "url", "", "URL of the service's metrics")
	flag.StringVar(&source, "source", "", "an optional source to use instead of the URL's host")
	flag.Var(&gaugePaths, "gauge", "the JSON path to a gauges's value")
	flag.Var(&counterPaths, "counter", "the JSON path to a counter's value")
	flag.StringVar(&email, "email", "", "Librato account email")
	flag.StringVar(&token, "token", "", "Librato account token")
	flag.DurationVar(&period, "period", 0, "send data periodically (0 for just once)")
	flag.Parse()

	if metricsURL == "" {
		fmt.Fprintln(os.Stderr, "No URL provided")
		flag.Usage()
		os.Exit(1)
	}

	if source == "" {
		u, err := url.Parse(metricsURL)
		if err != nil {
			panic(err)
		}
		source = u.Host
	}

	for _ = range ticker(period) {
		log.Printf("collecting %s", metricsURL)
		n := collect(metricsURL, source, email, token, gaugePaths, counterPaths)
		log.Printf("sent %d metrics", n)
	}
}

func collect(url, source, email, token string, gaugePaths, counterPaths stringList) int {
	defer func() {
		e := recover()
		if e != nil {
			log.Printf("panic: %v\n", e)
			for skip := 1; ; skip++ {
				pc, file, line, ok := runtime.Caller(skip)
				if !ok {
					break
				}
				if file[len(file)-1] == 'c' {
					continue
				}
				f := runtime.FuncForPC(pc)
				log.Printf("%s:%d %s()\n", file, line, f.Name())
			}
		}
	}()

	metrics := fetchMetrics(url)
	batch := batchMetrics(metrics, source, gaugePaths, counterPaths)
	postBatch(batch, email, token)

	return len(batch.Counters) + len(batch.Gauges)
}

func ticker(period time.Duration) <-chan time.Time {
	// if we're not doing periodic collections, return a closed channel with a
	// single time in it
	if period == 0 {
		c := make(chan time.Time, 1)
		c <- time.Now()
		close(c)
		return c
	}
	return time.Tick(period)
}

func postBatch(batch batch, email, token string) {
	j, err := json.Marshal(batch)
	if err != nil {
		panic(err)
	}

	r := bytes.NewReader(j)
	req, err := http.NewRequest("POST", "https://metrics-api.librato.com/v1/metrics", r)
	if err != nil {
		panic(err)
	}
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("Authorization", basicAuth(email, token))

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		panic(err)
	}
	defer func() {
		_, _ = io.Copy(ioutil.Discard, resp.Body)
		_ = resp.Body.Close()
	}()

	if resp.StatusCode != 200 {
		body := bytes.NewBuffer(nil)
		if _, err := io.Copy(body, resp.Body); err != nil {
			panic(err)
		}

		panic(fmt.Sprintf("received %s\n\n%s\n", resp.Status, body.String()))
	}
}

func basicAuth(u, p string) string {
	creds := base64.URLEncoding.EncodeToString([]byte(u + ":" + p))
	return fmt.Sprintf("Basic %s", creds)
}

type batch struct {
	Gauges   map[string]gauge   `json:"gauges"`
	Counters map[string]counter `json:"counters"`
	Source   string             `json:"source"`
}

type gauge struct {
	Value float64 `json:"value"`
}

type counter struct {
	Value int `json:"value"`
}

func batchMetrics(jq *jsonq.JsonQuery, source string, gauges, counters []string) batch {
	b := batch{
		Gauges:   make(map[string]gauge),
		Counters: make(map[string]counter),
		Source:   source,
	}

	for _, path := range gauges {
		v, err := jq.Float(strings.Split(path, ".")...)
		if err != nil {
			panic(err)
		}
		log.Printf("  %s=%v", path, v)
		b.Gauges[path] = gauge{Value: v}
	}

	for _, path := range counters {
		v, err := jq.Int(strings.Split(path, ".")...)
		if err != nil {
			panic(err)
		}
		log.Printf("  %s=%v", path, v)
		b.Counters[path] = counter{Value: v}
	}

	return b
}

func fetchMetrics(url string) *jsonq.JsonQuery {
	resp, err := http.Get(url)
	if err != nil {
		panic(err)
	}
	defer func() {
		_, _ = io.Copy(ioutil.Discard, resp.Body)
		_ = resp.Body.Close()
	}()

	if resp.StatusCode != 200 {
		panic("received a " + resp.Status + " response")
	}

	var metrics map[string]interface{}
	if err := json.NewDecoder(resp.Body).Decode(&metrics); err != nil {
		panic(err)
	}

	return jsonq.NewQuery(metrics)
}

type stringList []string

func (l *stringList) Set(v string) error {
	*l = append(*l, v)
	return nil
}

func (l *stringList) String() string {
	return strings.Join(*l, ",")
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/sha1"
	"database/sql"
	"encoding/base64"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"html"
	"io"
	"io/ioutil"
	"log"
	"net/http"

	"github.com/dchest/uniuri"
	"github.com/ewhal/pygments"
	_ "github.com/go-sql-driver/mysql"
	"github.com/gorilla/mux"
)

const (
	ADDRESS   = "http://localhost:9900"
	LENGTH    = 6
	DELETE    = ADDRESS + "/{PASTE}/{DELKEY}"
	PASTEARGS = ADDRESS + "/p/{PASTE}/(python|language)\n"
	URLARGS   = ADDRESS + "/save/XML|JSON\n"
	SOURCE    = "Source: https://github.com/ewhal/Pastebin\n"
	TEXT      = "$ <command> | curl -F 'p=<-' " + ADDRESS + "\n" + PASTEARGS + URLARGS + DELETE + SOURCE
	PORT      = ":9900"
	USERNAME  = ""
	PASS      = ""
	NAME      = ""
	DATABASE  = USERNAME + ":" + PASS + "@/" + NAME + "?charset=utf8"
)

type Response struct {
	ID     string `json:"id"`
	HASH   string `json:"hash"`
	URL    string `json:"url"`
	SIZE   int    `json:"size"`
	DELKEY string `json:"delkey"`
}

func check(err error) {
	if err != nil {
		fmt.Println(err)
	}
}

func generateName() string {
	s := uniuri.NewLen(LENGTH)
	db, err := sql.Open("mysql", DATABASE)
	check(err)

	query, err := db.Query("select id from pastebin")
	for query.Next() {
		var id string
		err := query.Scan(&id)
		if err != nil {

		}
		if id == s {
			generateName()
		}
	}
	db.Close()

	return s

}
func hash(paste []byte) string {
	hasher := sha1.New()
	hasher.Write(paste)
	sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil))
	return sha
}

func save(raw []byte) []string {
	p := raw[86 : len(raw)-46]

	db, err := sql.Open("mysql", DATABASE)
	check(err)

	sha := hash(p)
	query, err := db.Query("select id, hash, data, delkey from pastebin")
	for query.Next() {
		var id, hash, paste, delkey string
		err := query.Scan(&id, &hash, &paste, &delkey)
		check(err)
		if hash == sha {
			url := ADDRESS + "/p/" + id
			return []string{id, hash, url, paste, delkey}
		}
	}
	id := generateName()
	url := ADDRESS + "/p/" + id
	delKey := uniuri.NewLen(40)
	paste := html.EscapeString(string(p))

	stmt, err := db.Prepare("INSERT INTO pastebin(id, hash, data, delkey) values(?,?,?,?)")
	check(err)
	_, err = stmt.Exec(id, sha, paste, delKey)
	check(err)
	db.Close()
	return []string{id, sha, url, paste, delKey}

}

func delHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	delkey := vars["delKey"]

	db, err := sql.Open("mysql", DATABASE)
	check(err)

	stmt, err := db.Prepare("delete from pastebin where delkey=? id=?")
	check(err)

	res, err := stmt.Exec(html.EscapeString(delkey), html.EscapeString(paste))
	check(err)

	affect, err := res.RowsAffected()
	check(err)

	io.WriteString(w, string(affect))

	db.Close()

}
func saveHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	output := vars["output"]
	switch r.Method {
	case "POST":
		buf, err := ioutil.ReadAll(r.Body)
		if err != nil {
			http.Error(w, err.Error(), 500)
		}

		values := save(buf)
		b := &Response{
			ID:     values[0],
			HASH:   values[1],
			URL:    values[2],
			SIZE:   len(values[3]),
			DELKEY: values[4],
		}

		switch output {
		case "json":

			w.Header().Set("Content-Type", "application/json")
			err := json.NewEncoder(w).Encode(b)

			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
		case "xml":
			x, err := xml.MarshalIndent(b, "", "  ")
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}

			w.Header().Set("Content-Type", "application/xml")
			w.Write(x)

		default:
			io.WriteString(w, b.URL+"\n")
			io.WriteString(w, "delete key: "+b.DELKEY+"\n")
		}
	}

}

func rootHandler(w http.ResponseWriter, r *http.Request) {
	io.WriteString(w, TEXT)
}
func langHandler(w http.ResponseWriter, r *http.Request) {

	vars := mux.Vars(r)
	paste := vars["pasteId"]
	lang := vars["lang"]
	s := getPaste(paste)
	highlight := pygments.Highlight(html.UnescapeString(s), html.EscapeString(lang), "html", "full, style=autumn,linenos=True, lineanchors=True,anchorlinenos=True,", "utf-8")
	io.WriteString(w, highlight)

}

func getPaste(paste string) string {
	param1 := html.EscapeString(paste)
	db, err := sql.Open("mysql", DATABASE)
	var s string
	err = db.QueryRow("select data from pastebin where id=?", param1).Scan(&s)
	db.Close()
	check(err)

	if err == sql.ErrNoRows {
		return "Error invalid paste"
	} else {
		return html.UnescapeString(s)
	}

}
func pasteHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	s := getPaste(paste)
	io.WriteString(w, s)

}

func main() {
	router := mux.NewRouter().StrictSlash(true)
	router.HandleFunc("/", rootHandler)
	router.HandleFunc("/p/{pasteId}", pasteHandler)
	router.HandleFunc("/p/{pasteId}/{lang}", langHandler)
	router.HandleFunc("/save", saveHandler)
	router.HandleFunc("/save/{output}", saveHandler)
	router.HandleFunc("/del/{pasteId}/{delKey}", delHandler)
	err := http.ListenAndServe(PORT, router)
	if err != nil {
		log.Fatal(err)
	}

}
<commit_msg>Add newline to delete reference<commit_after>package main

import (
	"crypto/sha1"
	"database/sql"
	"encoding/base64"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"html"
	"io"
	"io/ioutil"
	"log"
	"net/http"

	"github.com/dchest/uniuri"
	"github.com/ewhal/pygments"
	_ "github.com/go-sql-driver/mysql"
	"github.com/gorilla/mux"
)

const (
	ADDRESS   = "http://localhost:9900"
	LENGTH    = 6
	DELETE    = ADDRESS + "/{PASTE}/{DELKEY}\n"
	PASTEARGS = ADDRESS + "/p/{PASTE}/(python|language)\n"
	URLARGS   = ADDRESS + "/save/XML|JSON\n"
	SOURCE    = "Source: https://github.com/ewhal/Pastebin\n"
	TEXT      = "$ <command> | curl -F 'p=<-' " + ADDRESS + "\n" + PASTEARGS + URLARGS + DELETE + SOURCE
	PORT      = ":9900"
	USERNAME  = ""
	PASS      = ""
	NAME      = ""
	DATABASE  = USERNAME + ":" + PASS + "@/" + NAME + "?charset=utf8"
)

type Response struct {
	ID     string `json:"id"`
	HASH   string `json:"hash"`
	URL    string `json:"url"`
	SIZE   int    `json:"size"`
	DELKEY string `json:"delkey"`
}

func check(err error) {
	if err != nil {
		fmt.Println(err)
	}
}

func generateName() string {
	s := uniuri.NewLen(LENGTH)
	db, err := sql.Open("mysql", DATABASE)
	check(err)

	query, err := db.Query("select id from pastebin")
	for query.Next() {
		var id string
		err := query.Scan(&id)
		if err != nil {

		}
		if id == s {
			generateName()
		}
	}
	db.Close()

	return s

}
func hash(paste []byte) string {
	hasher := sha1.New()
	hasher.Write(paste)
	sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil))
	return sha
}

func save(raw []byte) []string {
	p := raw[86 : len(raw)-46]

	db, err := sql.Open("mysql", DATABASE)
	check(err)

	sha := hash(p)
	query, err := db.Query("select id, hash, data, delkey from pastebin")
	for query.Next() {
		var id, hash, paste, delkey string
		err := query.Scan(&id, &hash, &paste, &delkey)
		check(err)
		if hash == sha {
			url := ADDRESS + "/p/" + id
			return []string{id, hash, url, paste, delkey}
		}
	}
	id := generateName()
	url := ADDRESS + "/p/" + id
	delKey := uniuri.NewLen(40)
	paste := html.EscapeString(string(p))

	stmt, err := db.Prepare("INSERT INTO pastebin(id, hash, data, delkey) values(?,?,?,?)")
	check(err)
	_, err = stmt.Exec(id, sha, paste, delKey)
	check(err)
	db.Close()
	return []string{id, sha, url, paste, delKey}

}

func delHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	delkey := vars["delKey"]

	db, err := sql.Open("mysql", DATABASE)
	check(err)

	stmt, err := db.Prepare("delete from pastebin where delkey=? id=?")
	check(err)

	res, err := stmt.Exec(html.EscapeString(delkey), html.EscapeString(paste))
	check(err)

	affect, err := res.RowsAffected()
	check(err)

	io.WriteString(w, string(affect))

	db.Close()

}
func saveHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	output := vars["output"]
	switch r.Method {
	case "POST":
		buf, err := ioutil.ReadAll(r.Body)
		if err != nil {
			http.Error(w, err.Error(), 500)
		}

		values := save(buf)
		b := &Response{
			ID:     values[0],
			HASH:   values[1],
			URL:    values[2],
			SIZE:   len(values[3]),
			DELKEY: values[4],
		}

		switch output {
		case "json":

			w.Header().Set("Content-Type", "application/json")
			err := json.NewEncoder(w).Encode(b)

			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
		case "xml":
			x, err := xml.MarshalIndent(b, "", "  ")
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}

			w.Header().Set("Content-Type", "application/xml")
			w.Write(x)

		default:
			io.WriteString(w, b.URL+"\n")
			io.WriteString(w, "delete key: "+b.DELKEY+"\n")
		}
	}

}

func rootHandler(w http.ResponseWriter, r *http.Request) {
	io.WriteString(w, TEXT)
}
func langHandler(w http.ResponseWriter, r *http.Request) {

	vars := mux.Vars(r)
	paste := vars["pasteId"]
	lang := vars["lang"]
	s := getPaste(paste)
	highlight := pygments.Highlight(html.UnescapeString(s), html.EscapeString(lang), "html", "full, style=autumn,linenos=True, lineanchors=True,anchorlinenos=True,", "utf-8")
	io.WriteString(w, highlight)

}

func getPaste(paste string) string {
	param1 := html.EscapeString(paste)
	db, err := sql.Open("mysql", DATABASE)
	var s string
	err = db.QueryRow("select data from pastebin where id=?", param1).Scan(&s)
	db.Close()
	check(err)

	if err == sql.ErrNoRows {
		return "Error invalid paste"
	} else {
		return html.UnescapeString(s)
	}

}
func pasteHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	paste := vars["pasteId"]
	s := getPaste(paste)
	io.WriteString(w, s)

}

func main() {
	router := mux.NewRouter().StrictSlash(true)
	router.HandleFunc("/", rootHandler)
	router.HandleFunc("/p/{pasteId}", pasteHandler)
	router.HandleFunc("/p/{pasteId}/{lang}", langHandler)
	router.HandleFunc("/save", saveHandler)
	router.HandleFunc("/save/{output}", saveHandler)
	router.HandleFunc("/del/{pasteId}/{delKey}", delHandler)
	err := http.ListenAndServe(PORT, router)
	if err != nil {
		log.Fatal(err)
	}

}
<|endoftext|>


<commit_before>package main

import (
	"net/http"
	"os"
)

func hello(resp http.ResponseWriter, req *http.Request) {
	resp.Write([]byte("
Hello world!
"))
}

func main() {
	http.HandleFunc("/", hello)
	err := http.ListenAndServe(":"+os.Getenv("PORT"), nil)
	if err != nil {
		panic(err)
	}
}
<commit_msg>Experimentation.<commit_after>package main

import (
	"fmt"
	"net/http"
	"os"
)

func hello(resp http.ResponseWriter, req *http.Request) {
	fmt.Println("
")
	resp.Write([]byte("Hello world!"))
	fmt.Println("
")
}

func main() {
	http.HandleFunc("/", hello)
	err := http.ListenAndServe(":"+os.Getenv("PORT"), nil)
	if err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"database/sql"
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	_ "github.com/mattn/go-sqlite3"
	"github.com/pkg/errors"
)

type standard struct {
	tagName    string
	fileID     int
	lineNumber int
	lineImage  string
}

func (s standard) String() string {
	return fmt.Sprintf("%d @n %d %s", s.fileID, s.lineNumber, s.lineImage)
}

type compact struct {
	fileID      int
	lineNumbers []int
}

func (c compact) String() string {
	continueCounter := 0
	output := fmt.Sprintf("%d", c.lineNumbers[0]) // [0] must be exist
	for l := 1; l < len(c.lineNumbers); l++ {
		diff := c.lineNumbers[l] - c.lineNumbers[l-1]
		if continueCounter == 0 {
			if diff == 1 {
				output += "-"
				continueCounter++
			} else {
				output += fmt.Sprintf(",%d", diff)
			}
		} else {
			if diff == 1 {
				continueCounter++
			} else {
				output += fmt.Sprintf("%d,%d", continueCounter, diff)
				continueCounter = 0
			}
		}
	}
	if continueCounter != 0 {
		output += fmt.Sprintf("%d", continueCounter)
	}
	return fmt.Sprintf("%d @n %s", c.fileID, output)
}

type global struct {
	fileID     int
	gtagsData  []standard
	grtagsData map[string]*compact
	db         map[string]*sql.DB
	// lineImageScanner
	basePath       string
	currentFile    *os.File
	currentRelPath string
	currentLine    int
	scanner        *bufio.Scanner
	fset           *token.FileSet
}

func newGlobal(fset *token.FileSet, basePath string) (*global, error) {
	g := &global{
		fileID:      0,
		gtagsData:   make([]standard, 0),
		grtagsData:  make(map[string]*compact),
		db:          make(map[string]*sql.DB),
		basePath:    basePath,
		currentFile: nil,
		currentLine: 0,
		scanner:     nil,
		fset:        fset,
	}

	dbfiles := []string{
		"GTAGS",
		"GRTAGS",
		"GPATH",
	}

	for _, file := range dbfiles {
		var err error
		os.Remove("./" + file)
		g.db[file], err = sql.Open("sqlite3", file)
		if err != nil {
			return nil, err
		}
		_, err = g.db[file].Exec(`create table db (key text primary key, dat text, extra text)`)
		if err != nil {
			return nil, err
		}
	}

	stmt, err := g.db["GTAGS"].Prepare(`insert into db (key, dat) values (?, ?)`)
	if err != nil {
		return nil, err
	}
	stmt.Exec(" __.COMPRESS", " __.COMPRESS ddefine ttypedef")
	stmt.Exec(" __.COMPNAME", " __.COMPNAME")
	stmt.Exec(" __.VERSION", " __.VERSION 6")

	stmt, err = g.db["GRTAGS"].Prepare(`insert into db (key, dat) values (?, ?)`)
	if err != nil {
		return nil, err
	}
	stmt.Exec(" __.COMPACT", " __.COMPACT")
	stmt.Exec(" __.COMPLINE", " __.COMPLINE")
	stmt.Exec(" __.COMPNAME", " __.COMPNAME")
	stmt.Exec(" __.VERSION", " __.VERSION 6")

	stmt, err = g.db["GPATH"].Prepare(`insert into db (key, dat) values (?, ?)`)
	if err != nil {
		return nil, err
	}
	stmt.Exec(" __.VERSION", " __.VERSION 2")
	stmt.Exec(" __.NEXTKEY", "1")

	return g, nil
}

func (g *global) dump() {
	if g.fileID == 0 {
		return
	}
	stmt, err := g.db["GTAGS"].Prepare(`insert into db (key, dat, extra) values (?, ?, ?)`)
	if err != nil {
		log.Fatal(err)
	}
	for _, s := range g.gtagsData {
		stmt.Exec(s.tagName, s.String(), strconv.Itoa(s.fileID))
	}
	stmt, err = g.db["GRTAGS"].Prepare(`insert into db (key, dat, extra) values (?, ?, ?)`)
	if err != nil {
		log.Fatal(err)
	}
	for tagName, compact := range g.grtagsData {
		stmt.Exec(tagName, compact.String(), strconv.Itoa(compact.fileID))
	}
	stmt, err = g.db["GPATH"].Prepare(`insert into db (key, dat) values (?, ?)`)
	if err != nil {
		log.Fatal(err)
	}

	filepath, _ := filepath.Rel(g.basePath, g.currentFile.Name())
	filepath = "./" + filepath
	log.Println(filepath)
	stmt.Exec(filepath, g.fileID)
	stmt.Exec(g.fileID, filepath)
	stmt.Exec(" __.NEXTKEY", strconv.Itoa(g.fileID+1))
}

func (g *global) finalize() error {
	if g.currentFile != nil {
		err := g.currentFile.Close()
		if err != nil {
			return err
		}
	}
	g.dump()
	return nil
}

func (g *global) switchFile(abspath string) (err error) {
	// Close and Setup Scanner
	if g.currentFile != nil {
		err := g.currentFile.Close()
		if err != nil {
			return errors.Wrapf(err, "failed to close current file, current: %s abspath: %s", g.currentFile.Name(), abspath)
		}
	}
	g.currentFile, err = os.Open(abspath)
	if err != nil {
		return errors.Wrap(err, "failed to open next file ")
	}
	g.scanner = bufio.NewScanner(g.currentFile)
	g.currentLine = 0

	// Reset parsed data
	g.gtagsData = make([]standard, 0)
	g.grtagsData = make(map[string]*compact)
	g.fileID++

	return nil
}

func (g *global) addFuncDecl(node *ast.FuncDecl) {
	ident := node.Name
	pos := g.fset.Position(node.Pos())
	for ; g.currentLine < pos.Line; g.currentLine++ {
		g.scanner.Scan()
	}
	lineImage := strings.Replace(strings.TrimSpace(g.scanner.Text()), ident.Name, "@n", -1)

	g.gtagsData = append(g.gtagsData, standard{
		tagName:    ident.Name,
		fileID:     g.fileID,
		lineNumber: pos.Line,
		lineImage:  lineImage,
	})
}

func (g *global) addIdent(ident *ast.Ident) {
	pos := g.fset.Position(ident.Pos())
	r, found := g.grtagsData[ident.Name]
	if found {
		r.lineNumbers = append(r.lineNumbers, pos.Line)
	} else {
		g.grtagsData[ident.Name] = &compact{
			fileID:      g.fileID,
			lineNumbers: []int{pos.Line},
		}
	}
}

func (g *global) parse(node ast.Node) bool {
	if node == nil {
		return false
	}
	if _, ok := node.(*ast.Package); ok {
		return true
	}
	pos := g.fset.Position(node.Pos())
	absPath, err := filepath.Abs(pos.Filename)
	if err != nil {
		log.Fatal("failed to get absolute path: ", err)
	}
	if g.currentFile == nil || g.currentFile.Name() != absPath {
		g.dump()

		err = g.switchFile(absPath)
		if err != nil {
			log.Fatal("failed to switch file: ", err)
		}
	}

	switch node.(type) {
	case *ast.FuncDecl:
		g.addFuncDecl(node.(*ast.FuncDecl))
	case *ast.Ident:
		g.addIdent(node.(*ast.Ident))
	}
	return true
}

func main() {
	basePath, err := filepath.Abs(".")
	if err != nil {
		log.Fatal("failed to get absolute path: ", err)
	}

	fset := token.NewFileSet() // positions are relative to fset
	g, err := newGlobal(fset, basePath)
	if err != nil {
		log.Fatal(err)
	}

	err = filepath.Walk(basePath, func(path string, info os.FileInfo, err error) error {
		if info.IsDir() {
			pkgs, err := parser.ParseDir(fset, path, nil, 0)
			if err != nil {
				log.Fatal(err)
			}
			for _, p := range pkgs {
				ast.Inspect(p, g.parse)
			}
		}
		return nil
	})
	if err != nil {
		log.Fatal(err)
	}

	err = g.finalize()
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Fix error handling<commit_after>package main

import (
	"bufio"
	"database/sql"
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	_ "github.com/mattn/go-sqlite3"
	"github.com/pkg/errors"
)

type tagType int

const (
	GTAGS tagType = iota
	GRTAGS
	GPATH
)

func (t tagType) String() string {
	switch t {
	case GTAGS:
		return "GTAGS"
	case GRTAGS:
		return "GRTAGS"
	case GPATH:
		return "GPATH"
	}
	panic("invalid tagType")
}

type standard struct {
	tagName    string
	fileID     int
	lineNumber int
	lineImage  string
}

func (s standard) String() string {
	return fmt.Sprintf("%d @n %d %s", s.fileID, s.lineNumber, s.lineImage)
}

type compact struct {
	fileID      int
	lineNumbers []int
}

func (c compact) String() string {
	continueCounter := 0
	output := fmt.Sprintf("%d", c.lineNumbers[0]) // [0] must be exist
	for l := 1; l < len(c.lineNumbers); l++ {
		diff := c.lineNumbers[l] - c.lineNumbers[l-1]
		if continueCounter == 0 {
			if diff == 1 {
				output += "-"
				continueCounter++
			} else {
				output += fmt.Sprintf(",%d", diff)
			}
		} else {
			if diff == 1 {
				continueCounter++
			} else {
				output += fmt.Sprintf("%d,%d", continueCounter, diff)
				continueCounter = 0
			}
		}
	}
	if continueCounter != 0 {
		output += fmt.Sprintf("%d", continueCounter)
	}
	return fmt.Sprintf("%d @n %s", c.fileID, output)
}

type global struct {
	fileID     int
	gtagsData  []standard
	grtagsData map[string]*compact
	db         map[tagType]*sql.DB
	// lineImageScanner
	basePath       string
	currentFile    *os.File
	currentRelPath string
	currentLine    int
	scanner        *bufio.Scanner
	fset           *token.FileSet
}

func newGlobal(fset *token.FileSet, basePath string) (*global, error) {
	g := &global{
		fileID:      0,
		gtagsData:   make([]standard, 0),
		grtagsData:  make(map[string]*compact),
		db:          make(map[tagType]*sql.DB),
		basePath:    basePath,
		currentFile: nil,
		currentLine: 0,
		scanner:     nil,
		fset:        fset,
	}

	dbfiles := []tagType{
		GTAGS,
		GRTAGS,
		GPATH,
	}

	var err error
	for _, file := range dbfiles {
		os.Remove("./" + file.String())
		g.db[file], err = sql.Open("sqlite3", file.String())
		if err != nil {
			return nil, err
		}
		_, err = g.db[file].Exec(`create table db (key text, dat text, extra text)`)
		if err != nil {
			return nil, err
		}
	}

	g.insertEntry(GTAGS, " __.COMPRESS", " __.COMPRESS ddefine ttypedef", nil)
	g.insertEntry(GTAGS, " __.COMPNAME", " __.COMPNAME", nil)
	g.insertEntry(GTAGS, " __.VERSION", " __.VERSION 6", nil)

	g.insertEntry(GRTAGS, " __.COMPACT", " __.COMPACT", nil)
	g.insertEntry(GRTAGS, " __.COMPLINE", " __.COMPLINE", nil)
	g.insertEntry(GRTAGS, " __.COMPNAME", " __.COMPNAME", nil)
	g.insertEntry(GRTAGS, " __.VERSION", " __.VERSION 6", nil)

	g.insertEntry(GPATH, " __.VERSION", " __.VERSION 2", nil)
	g.insertEntry(GPATH, " __.NEXTKEY", "1", nil)

	return g, nil
}

func (g *global) insertEntry(tag tagType, key, dat, extra interface{}) {
	stmt, err := g.db[tag].Prepare(`insert into db (key, dat, extra) values (?, ?, ?)`)
	if err != nil {
		log.Println(err, "tag:", tag, "|key:", key, "|dat:", dat, "|extra:", extra)
		panic("failed to prepare")
	}
	_, err = stmt.Exec(key, dat, extra)
	if err != nil {
		log.Println(err, "tag:", tag, "|key:", key, "|dat:", dat, "|extra:", extra)
		panic("failed to exec")
	}
}

func (g *global) dump() {
	if g.fileID == 0 {
		return
	}

	for _, s := range g.gtagsData {
		g.insertEntry(GTAGS, s.tagName, s.String(), strconv.Itoa(s.fileID))
	}
	for tagName, compact := range g.grtagsData {
		g.insertEntry(GRTAGS, tagName, compact.String(), strconv.Itoa(compact.fileID))
	}

	filepath, _ := filepath.Rel(g.basePath, g.currentFile.Name())
	filepath = "./" + filepath
	log.Println(filepath)

	g.insertEntry(GPATH, filepath, g.fileID, nil)
	g.insertEntry(GPATH, g.fileID, filepath, nil)
	g.insertEntry(GPATH, " __.NEXTKEY", strconv.Itoa(g.fileID+1), nil)
}

func (g *global) finalize() error {
	if g.currentFile != nil {
		err := g.currentFile.Close()
		if err != nil {
			return err
		}
	}
	g.dump()
	return nil
}

func (g *global) switchFile(abspath string) (err error) {
	// Close and Setup Scanner
	if g.currentFile != nil {
		err := g.currentFile.Close()
		if err != nil {
			return errors.Wrapf(err, "failed to close current file, current: %s abspath: %s", g.currentFile.Name(), abspath)
		}
	}
	g.currentFile, err = os.Open(abspath)
	if err != nil {
		return errors.Wrap(err, "failed to open next file ")
	}
	g.scanner = bufio.NewScanner(g.currentFile)
	g.currentLine = 0

	// Reset parsed data
	g.gtagsData = make([]standard, 0)
	g.grtagsData = make(map[string]*compact)
	g.fileID++

	return nil
}

func (g *global) addFuncDecl(node *ast.FuncDecl) {
	ident := node.Name
	pos := g.fset.Position(node.Pos())
	for ; g.currentLine < pos.Line; g.currentLine++ {
		g.scanner.Scan()
	}
	lineImage := strings.Replace(strings.TrimSpace(g.scanner.Text()), ident.Name, "@n", -1)

	g.gtagsData = append(g.gtagsData, standard{
		tagName:    ident.Name,
		fileID:     g.fileID,
		lineNumber: pos.Line,
		lineImage:  lineImage,
	})
}

func (g *global) addIdent(ident *ast.Ident) {
	pos := g.fset.Position(ident.Pos())
	r, found := g.grtagsData[ident.Name]
	if found {
		r.lineNumbers = append(r.lineNumbers, pos.Line)
	} else {
		g.grtagsData[ident.Name] = &compact{
			fileID:      g.fileID,
			lineNumbers: []int{pos.Line},
		}
	}
}

func (g *global) parse(node ast.Node) bool {
	if node == nil {
		return false
	}
	if _, ok := node.(*ast.Package); ok {
		return true
	}
	pos := g.fset.Position(node.Pos())
	absPath, err := filepath.Abs(pos.Filename)
	if err != nil {
		log.Fatal("failed to get absolute path: ", err)
	}
	if g.currentFile == nil || g.currentFile.Name() != absPath {
		g.dump()

		err = g.switchFile(absPath)
		if err != nil {
			log.Fatal("failed to switch file: ", err)
		}
	}

	switch node.(type) {
	case *ast.FuncDecl:
		g.addFuncDecl(node.(*ast.FuncDecl))
	case *ast.Ident:
		g.addIdent(node.(*ast.Ident))
	}
	return true
}

func main() {
	basePath, err := filepath.Abs(".")
	if err != nil {
		log.Fatal("failed to get absolute path: ", err)
	}

	fset := token.NewFileSet() // positions are relative to fset
	g, err := newGlobal(fset, basePath)
	if err != nil {
		log.Fatal(err)
	}

	err = filepath.Walk(basePath, func(path string, info os.FileInfo, err error) error {
		if info.IsDir() {
			pkgs, err := parser.ParseDir(fset, path, nil, 0)
			if err != nil {
				log.Fatal(err)
			}
			for _, p := range pkgs {
				ast.Inspect(p, g.parse)
			}
		}
		return nil
	})
	if err != nil {
		log.Fatal(err)
	}

	err = g.finalize()
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"os"
	"os/signal"

	cointip "github.com/Bullpeen/cointip/quadlek"
	log "github.com/Sirupsen/logrus"
	"github.com/jirwin/quadlek/plugins/echo"
	"github.com/jirwin/quadlek/plugins/karma"
	"github.com/jirwin/quadlek/plugins/nextep"
	"github.com/jirwin/quadlek/plugins/random"
	"github.com/jirwin/quadlek/plugins/spotify"
	"github.com/jirwin/quadlek/plugins/twitter"
	"github.com/jirwin/quadlek/quadlek"

	"github.com/Bullpeen/infobot"
	"github.com/urfave/cli"
)

const Version = "0.0.1"

func run(c *cli.Context) error {
	var apiToken string
	if c.IsSet("api-key") {
		apiToken = c.String("api-key")
	} else {
		cli.ShowAppHelp(c)
		return cli.NewExitError("Missing --api-key arg.", 1)
	}

	var verificationToken string
	if c.IsSet("verification-token") {
		verificationToken = c.String("verification-token")
	} else {
		cli.ShowAppHelp(c)
		return cli.NewExitError("Missing --verification-token arg.", 1)
	}

	dbPath := c.String("db-path")

	bot, err := quadlek.NewBot(context.Background(), apiToken, verificationToken, dbPath)
	if err != nil {
		log.WithFields(log.Fields{
			"err": err,
		}).Error("error creating bot")
		return nil
	}

	err = bot.RegisterPlugin(echo.Register())
	if err != nil {
		fmt.Printf("error registering echo plugin: %s", err.Error())
		return nil
	}

	err = bot.RegisterPlugin(karma.Register())
	if err != nil {
		fmt.Printf("error registering karma plugin: %s", err.Error())
		return nil
	}

	err = bot.RegisterPlugin(random.Register())
	if err != nil {
		fmt.Printf("error registering random plugin: %s", err.Error())
		return nil
	}

	err = bot.RegisterPlugin(spotify.Register())
	if err != nil {
		fmt.Printf("error registering spotify plugin: %s", err.Error())
		return nil
	}

	if c.IsSet("tvdb-key") {
		tvdbKey := c.String("tvdb-key")

		err = bot.RegisterPlugin(nextep.Register(tvdbKey))
		if err != nil {
			fmt.Printf("error registering nextep plugin: %s", err.Error())
			return nil
		}
	}

	err = bot.RegisterPlugin(infobot.Register())
	if err != nil {
		fmt.Printf("error registering infobot plugin: %s", err.Error())
		return nil
	}

	err = bot.RegisterPlugin(twitter.Register(
		c.String("twitter-consumer-key"),
		c.String("twitter-consumer-secret"),
		c.String("twitter-access-token"),
		c.String("twitter-access-secret"),
		// These must be twitter user ids, not names. https://tweeterid.com/ for easy conversion between the two.
		map[string]string{
			"25073877":            "politics", // @realDonaldTrump
			"822215679726100480":  "politics", // @POTUS
			"778682":              "general",  // @jirwin
			"26786244":            "general",  // @schonstal
			"2317524115":          "general",  // @PHP_CEO
		},
	))

	coinbasePlugin := cointip.Register(
		c.String("coinbase-key"),
		c.String("coinbase-secret"),
		c.String("coinbase-account"),
	)
	if coinbasePlugin != nil {
		err = bot.RegisterPlugin(coinbasePlugin)
		if err != nil {
			panic(err)
		}
	}

	signals := make(chan os.Signal, 1)
	signal.Notify(signals, os.Interrupt)

	bot.Start()
	<-signals
	bot.Stop()

	return nil
}

func main() {
	app := cli.NewApp()
	app.Name = "quadlek"
	app.Version = Version
	app.Usage = "a slack bot"
	app.Action = run
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "api-key",
			Usage:  "The slack api token for the bot",
			EnvVar: "QUADLEK_API_TOKEN",
		},
		cli.StringFlag{
			Name:   "verification-token",
			Usage:  "The slack webhook verification token.",
			EnvVar: "QUADLEK_VERIFICATION_TOKEN",
		},
		cli.StringFlag{
			Name:   "db-path",
			Usage:  "The path where the database is stored.",
			Value:  "quadlek.db",
			EnvVar: "QUADLEK_DB_PATH",
		},
		cli.StringFlag{
			Name:   "tvdb-key",
			Usage:  "The TVDB api key for the bot, used by the nextep command",
			EnvVar: "QUADLEK_TVDB_TOKEN",
		},
		cli.StringFlag{
			Name:   "twitter-consumer-key",
			Usage:  "The consumer key for the twitter api",
			EnvVar: "QUADLEK_TWITTER_CONSUMER_KEY",
		},
		cli.StringFlag{
			Name:   "twitter-consumer-secret",
			Usage:  "The consumer secret for the twitter api",
			EnvVar: "QUADLEK_TWITTER_CONSUMER_SECRET",
		},
		cli.StringFlag{
			Name:   "twitter-access-token",
			Usage:  "The access key for the twitter api",
			EnvVar: "QUADLEK_TWITTER_ACCESS_TOKEN",
		},
		cli.StringFlag{
			Name:   "twitter-access-secret",
			Usage:  "The access secret for the twitter api",
			EnvVar: "QUADLEK_TWITTER_ACCESS_SECRET",
		},
		cli.StringFlag{
			Name:   "coinbase-key",
			Usage:  "The access key for the coinbase api",
			EnvVar: "QUADLEK_COINBASE_KEY",
		},
		cli.StringFlag{
			Name:   "coinbase-secret",
			Usage:  "The access secret for the coinbase api",
			EnvVar: "QUADLEK_COINBASE_SECRET",
		},
		cli.StringFlag{
			Name:   "coinbase-account",
			Usage:  "The bank account for the coinbase api",
			EnvVar: "QUADLEK_COINBASE_BANK_ACCOUNT",
		},
	}

	app.Run(os.Args)
}
<commit_msg>Added PresVillian to #politics<commit_after>package main

import (
	"context"
	"fmt"
	"os"
	"os/signal"

	cointip "github.com/Bullpeen/cointip/quadlek"
	log "github.com/Sirupsen/logrus"
	"github.com/jirwin/quadlek/plugins/echo"
	"github.com/jirwin/quadlek/plugins/karma"
	"github.com/jirwin/quadlek/plugins/nextep"
	"github.com/jirwin/quadlek/plugins/random"
	"github.com/jirwin/quadlek/plugins/spotify"
	"github.com/jirwin/quadlek/plugins/twitter"
	"github.com/jirwin/quadlek/quadlek"

	"github.com/Bullpeen/infobot"
	"github.com/urfave/cli"
)

const Version = "0.0.1"

func run(c *cli.Context) error {
	var apiToken string
	if c.IsSet("api-key") {
		apiToken = c.String("api-key")
	} else {
		cli.ShowAppHelp(c)
		return cli.NewExitError("Missing --api-key arg.", 1)
	}

	var verificationToken string
	if c.IsSet("verification-token") {
		verificationToken = c.String("verification-token")
	} else {
		cli.ShowAppHelp(c)
		return cli.NewExitError("Missing --verification-token arg.", 1)
	}

	dbPath := c.String("db-path")

	bot, err := quadlek.NewBot(context.Background(), apiToken, verificationToken, dbPath)
	if err != nil {
		log.WithFields(log.Fields{
			"err": err,
		}).Error("error creating bot")
		return nil
	}

	err = bot.RegisterPlugin(echo.Register())
	if err != nil {
		fmt.Printf("error registering echo plugin: %s", err.Error())
		return nil
	}

	err = bot.RegisterPlugin(karma.Register())
	if err != nil {
		fmt.Printf("error registering karma plugin: %s", err.Error())
		return nil
	}

	err = bot.RegisterPlugin(random.Register())
	if err != nil {
		fmt.Printf("error registering random plugin: %s", err.Error())
		return nil
	}

	err = bot.RegisterPlugin(spotify.Register())
	if err != nil {
		fmt.Printf("error registering spotify plugin: %s", err.Error())
		return nil
	}

	if c.IsSet("tvdb-key") {
		tvdbKey := c.String("tvdb-key")

		err = bot.RegisterPlugin(nextep.Register(tvdbKey))
		if err != nil {
			fmt.Printf("error registering nextep plugin: %s", err.Error())
			return nil
		}
	}

	err = bot.RegisterPlugin(infobot.Register())
	if err != nil {
		fmt.Printf("error registering infobot plugin: %s", err.Error())
		return nil
	}

	err = bot.RegisterPlugin(twitter.Register(
		c.String("twitter-consumer-key"),
		c.String("twitter-consumer-secret"),
		c.String("twitter-access-token"),
		c.String("twitter-access-secret"),
		// These must be twitter user ids, not names. https://tweeterid.com/ for easy conversion between the two.
		map[string]string{
			"25073877":            "politics", // @realDonaldTrump
			"822215679726100480":  "politics", // @POTUS
			"830896623689547776":  "politics", // @PresVillain
			"778682":              "general",  // @jirwin
			"26786244":            "general",  // @schonstal
			"2317524115":          "general",  // @PHP_CEO
		},
	))

	coinbasePlugin := cointip.Register(
		c.String("coinbase-key"),
		c.String("coinbase-secret"),
		c.String("coinbase-account"),
	)
	if coinbasePlugin != nil {
		err = bot.RegisterPlugin(coinbasePlugin)
		if err != nil {
			panic(err)
		}
	}

	signals := make(chan os.Signal, 1)
	signal.Notify(signals, os.Interrupt)

	bot.Start()
	<-signals
	bot.Stop()

	return nil
}

func main() {
	app := cli.NewApp()
	app.Name = "quadlek"
	app.Version = Version
	app.Usage = "a slack bot"
	app.Action = run
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "api-key",
			Usage:  "The slack api token for the bot",
			EnvVar: "QUADLEK_API_TOKEN",
		},
		cli.StringFlag{
			Name:   "verification-token",
			Usage:  "The slack webhook verification token.",
			EnvVar: "QUADLEK_VERIFICATION_TOKEN",
		},
		cli.StringFlag{
			Name:   "db-path",
			Usage:  "The path where the database is stored.",
			Value:  "quadlek.db",
			EnvVar: "QUADLEK_DB_PATH",
		},
		cli.StringFlag{
			Name:   "tvdb-key",
			Usage:  "The TVDB api key for the bot, used by the nextep command",
			EnvVar: "QUADLEK_TVDB_TOKEN",
		},
		cli.StringFlag{
			Name:   "twitter-consumer-key",
			Usage:  "The consumer key for the twitter api",
			EnvVar: "QUADLEK_TWITTER_CONSUMER_KEY",
		},
		cli.StringFlag{
			Name:   "twitter-consumer-secret",
			Usage:  "The consumer secret for the twitter api",
			EnvVar: "QUADLEK_TWITTER_CONSUMER_SECRET",
		},
		cli.StringFlag{
			Name:   "twitter-access-token",
			Usage:  "The access key for the twitter api",
			EnvVar: "QUADLEK_TWITTER_ACCESS_TOKEN",
		},
		cli.StringFlag{
			Name:   "twitter-access-secret",
			Usage:  "The access secret for the twitter api",
			EnvVar: "QUADLEK_TWITTER_ACCESS_SECRET",
		},
		cli.StringFlag{
			Name:   "coinbase-key",
			Usage:  "The access key for the coinbase api",
			EnvVar: "QUADLEK_COINBASE_KEY",
		},
		cli.StringFlag{
			Name:   "coinbase-secret",
			Usage:  "The access secret for the coinbase api",
			EnvVar: "QUADLEK_COINBASE_SECRET",
		},
		cli.StringFlag{
			Name:   "coinbase-account",
			Usage:  "The bank account for the coinbase api",
			EnvVar: "QUADLEK_COINBASE_BANK_ACCOUNT",
		},
	}

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package main

import (
	"./format"
	"bufio"
	"errors"
	"flag"
	"fmt"
	"log"
	"net"
	"net/url"
	"strconv"
	"strings"
	"time"
)

type Tick struct {
	Size    int64
	Latency time.Duration
}

type Config struct {
	Url         string
	Duration    time.Duration
	Threads     int
	Connections int
	addr        *net.TCPAddr
	url         *url.URL
}

type Errors struct {
	connect uint64
	read    uint64
	write   uint64
	status  uint64
	timeout uint64
}

type Thread struct {
	url            *url.URL
	addr           *net.TCPAddr
	conn           *net.TCPConn
	complete       uint64
	requests       uint64
	bytes          uint64
	start          time.Time
	latency        time.Duration
	connectLatency time.Duration
	errors         Errors
	quit           chan bool
}

func usage() {
	fmt.Println("Usage: http-bench <options> <url>\n  Options:\n    -c, --connections <N>  Connections to keep open\n    -d, --duration    <T>  Duration of test")
}

func main() {
	config := Config{}
	flag.DurationVar(&config.Duration, "d", 10, "Duration of test")
	flag.IntVar(&config.Threads, "t", 10, "Number of threads to use")
	flag.Parse()

	if len(flag.Args()) != 1 {
		usage()
		return
	}

	config.Url = flag.Args()[0]
	if config.Url == "" {
		usage()
		return
	}

	config.Connections = config.Threads

	start(config)
}

func start(config Config) {
	fmt.Printf("Running %v test @ %v\n", config.Duration, config.Url)
	fmt.Printf("  %v threads and %v connections\n", config.Threads, config.Connections)

	resolveAddr(&config)

	threads := make([]*Thread, 0)
	for i := 0; i < config.Threads; i++ {
		thread := Thread{addr: config.addr, url: config.url}
		startWorker(&config, &thread)
		threads = append(threads, &thread)
	}

	for i := 0; i < config.Threads; i++ {
		<-threads[i].quit
	}

	results := mergeResults(threads)
	outputResult(results)
}

func mergeResults(threads []*Thread) *Thread {
	return threads[0]
}

func outputResult(t *Thread) {
	var avg time.Duration
	var reqps, bytesps float64
	if t.complete > 0 {
		avg = (time.Duration)(int64(t.latency) / int64(t.complete))
		reqps = float64(time.Second) / float64(avg)
		bytesps = float64(t.bytes) / float64(float64(t.latency)/float64(time.Second))
	}

	fmt.Println("Latency:", format.Duration(avg))
	fmt.Printf("%v requests in %v, %v read\n", t.complete, format.Duration(t.latency), format.Bytes(float64(t.bytes)))
	fmt.Printf("Requests/sec: %v\n", format.Reqps(reqps))
	fmt.Printf("Transfer/sec: %v\n", format.Bytes(bytesps))
	fmt.Printf("Errors: %v connect, %v write, %v read, %v status, %v timeout\n", t.errors.connect, t.errors.write, t.errors.read, t.errors.status, t.errors.timeout)
	fmt.Printf("%v\n", t)
}

func startWorker(config *Config, thread *Thread) {
	thread.quit = make(chan bool)

	go func() {
		timeout_at := time.After(config.Duration)
		for {
			select {
			case <-timeout_at:
				thread.quit <- true
				return
			default:
				connect(thread)
			}
		}
	}()
}

func resolveAddr(config *Config) {
	var err error
	config.url, err = url.Parse(config.Url)
	if err != nil {
		log.Fatalln(err.Error())
	}

	if config.url.Path == "" {
		config.url.Path = "/"
	}

	host := config.url.Host
	if !strings.Contains(host, ":") {
		switch config.url.Scheme {
		case "https":
			host = fmt.Sprintf("%s:%v", host, 443)
		case "http":
			host = fmt.Sprintf("%s:%v", host, 80)
		default:
			errMsg := fmt.Sprintf("Unknown scheme %v", config.url.Scheme)
			log.Fatalln(errors.New(errMsg))
		}
	}

	config.addr, err = net.ResolveTCPAddr("tcp", host)
	if err != nil {
		log.Fatalln(err.Error())
	}
}

func connect(t *Thread) {
	var err error
	t.start = time.Now()

	if t.conn != nil {
		post_request(t)
		return
	}

	fmt.Println("Dialing tcp connection")
	t.conn, err = net.DialTCP("tcp", nil, t.addr)
	if err == nil {
		post_request(t)
	} else {
		t.errors.connect++
	}
}

func post_request(t *Thread) {
	req := fmt.Sprintf("GET %v HTTP/1.1\r\nHost: %v\r\nConnection: Keep-Alive\r\n\r\n", t.url.Path, t.url.Host)
	_, err := fmt.Fprintf(t.conn, "%s", req)
	if err != nil {
		t.errors.write++
		t.conn.Close()
		t.conn = nil
	}

	r := bufio.NewReader(t.conn)
	status, err := r.ReadString('\n')

	// Read status
	if err != nil {
		t.errors.read++
		t.conn.Close()
		t.conn = nil
		return
	}

	ok := strings.Index(status, "HTTP/1.1 200") == 0
	if !ok {
		t.errors.status++
	}

	t.bytes += uint64(len(status))

	// Read headers
	var s string
	var content_length uint64
	keep_alive := true

	for {
		s, err = r.ReadString('\n')
		if err != nil {
			t.errors.read++
			t.conn.Close()
			t.conn = nil
			return
		}

		t.bytes += uint64(len(s))

		headers := strings.SplitN(s, ":", 2)
		switch strings.ToLower(headers[0]) {
		case "content-length":
			trimmed := strings.Trim(headers[1], " \r\n")
			content_length, err = strconv.ParseUint(trimmed, 10, 64)
			if err != nil {
				log.Fatalln(err.Error())
			}
		case "connection":
			keep_alive = strings.Contains(strings.ToLower(headers[1]), "keep-alive")
		}

		if s == "\r\n" {
			break
		}
	}

	// Ready body.
	buf := make([]byte, 1024)
	var n int
	for {
		n, err = r.Read(buf)
		if err != nil {
			t.errors.read++
			t.conn.Close()
			t.conn = nil
			return
		}
		if n > 0 {
			content_length -= uint64(n)
		}
		if content_length == 0 {
			break
		}
	}

	if n > 0 {
		t.bytes += uint64(n)
	}

	// Completed.
	t.complete++
	t.latency += time.Since(t.start)

	if !keep_alive {
		t.conn.Close()
		t.conn = nil
	}
}
<commit_msg>UPD formatting<commit_after>package main

import (
	"./format"
	"bufio"
	"errors"
	"flag"
	"fmt"
	"log"
	"net"
	"net/url"
	"strconv"
	"strings"
	"time"
)

type Tick struct {
	Size    int64
	Latency time.Duration
}

type Config struct {
	Url         string
	Duration    time.Duration
	Threads     int
	Connections int
	addr        *net.TCPAddr
	url         *url.URL
}

type Errors struct {
	connect uint64
	read    uint64
	write   uint64
	status  uint64
	timeout uint64
}

type Thread struct {
	url            *url.URL
	addr           *net.TCPAddr
	conn           *net.TCPConn
	complete       uint64
	requests       uint64
	bytes          uint64
	start          time.Time
	latency        time.Duration
	connectLatency time.Duration
	errors         Errors
	quit           chan bool
}

func usage() {
	fmt.Println("Usage: http-bench <options> <url>\n  Options:\n    -c, --connections <N>  Connections to keep open\n    -d, --duration    <T>  Duration of test")
}

func main() {
	config := Config{}
	flag.DurationVar(&config.Duration, "d", 10, "Duration of test")
	flag.IntVar(&config.Threads, "t", 10, "Number of threads to use")
	flag.Parse()

	if len(flag.Args()) != 1 {
		usage()
		return
	}

	config.Url = flag.Args()[0]
	if config.Url == "" {
		usage()
		return
	}

	config.Connections = config.Threads

	start(config)
}

func start(config Config) {
	fmt.Printf("Running %v test @ %v\n", config.Duration, config.Url)
	fmt.Printf("  %v threads and %v connections\n", config.Threads, config.Connections)

	resolveAddr(&config)

	threads := make([]*Thread, 0)
	for i := 0; i < config.Threads; i++ {
		thread := Thread{addr: config.addr, url: config.url}
		startWorker(&config, &thread)
		threads = append(threads, &thread)
	}

	for i := 0; i < config.Threads; i++ {
		<-threads[i].quit
	}

	results := mergeResults(threads)
	outputResult(results)
}

func mergeResults(threads []*Thread) *Thread {
	return threads[0]
}

func outputErrors(errors uint64, name string) {
	if errors > 0 {
		fmt.Println(name, "errors:", errors)
	}
}

func outputResult(t *Thread) {
	var avg time.Duration
	var reqps, bytesps float64
	if t.complete > 0 {
		avg = (time.Duration)(int64(t.latency) / int64(t.complete))
		reqps = float64(time.Second) / float64(avg)
		bytesps = float64(t.bytes) / float64(float64(t.latency)/float64(time.Second))
	}

	fmt.Println("Latency:", format.Duration(avg))
	fmt.Printf("%v requests in %v, %v read\n", t.complete, format.Duration(t.latency), format.Bytes(float64(t.bytes)))
	fmt.Printf("Requests/sec: %v\n", format.Reqps(reqps))
	fmt.Printf("Transfer/sec: %v\n", format.Bytes(bytesps))

	outputErrors(t.errors.connect, "connect")
	outputErrors(t.errors.write, "write")
	outputErrors(t.errors.read, "read")
	outputErrors(t.errors.status, "status")
	outputErrors(t.errors.timeout, "timeout")
}

func startWorker(config *Config, thread *Thread) {
	thread.quit = make(chan bool)

	go func() {
		timeout_at := time.After(config.Duration)
		for {
			select {
			case <-timeout_at:
				thread.quit <- true
				return
			default:
				connect(thread)
			}
		}
	}()
}

func resolveAddr(config *Config) {
	var err error
	config.url, err = url.Parse(config.Url)
	if err != nil {
		log.Fatalln(err.Error())
	}

	if config.url.Path == "" {
		config.url.Path = "/"
	}

	host := config.url.Host
	if !strings.Contains(host, ":") {
		switch config.url.Scheme {
		case "https":
			host = fmt.Sprintf("%s:%v", host, 443)
		case "http":
			host = fmt.Sprintf("%s:%v", host, 80)
		default:
			errMsg := fmt.Sprintf("Unknown scheme %v", config.url.Scheme)
			log.Fatalln(errors.New(errMsg))
		}
	}

	config.addr, err = net.ResolveTCPAddr("tcp", host)
	if err != nil {
		log.Fatalln(err.Error())
	}
}

func connect(t *Thread) {
	var err error
	t.start = time.Now()

	if t.conn != nil {
		post_request(t)
		return
	}

	t.conn, err = net.DialTCP("tcp", nil, t.addr)
	if err == nil {
		post_request(t)
	} else {
		t.errors.connect++
	}
}

func post_request(t *Thread) {
	req := fmt.Sprint("GET ", t.url.Path, " HTTP/1.1\r\nHost: ", t.url.Host, "\r\n\r\n")

	_, err := fmt.Fprintf(t.conn, "%s", req)
	if err != nil {
		t.errors.write++
		t.conn.Close()
		t.conn = nil
	}

	read_response(t)
}

func read_response(t *Thread) {
	r := bufio.NewReader(t.conn)
	status, err := r.ReadString('\n')

	// Read status
	if err != nil {
		t.errors.read++
		t.conn.Close()
		t.conn = nil
		return
	}

	ok := strings.Index(status, "HTTP/1.1 200") == 0
	if !ok {
		t.errors.status++
	}

	t.bytes += uint64(len(status))

	// Read headers
	var s string
	var content_length uint64
	keep_alive := true

	for {
		s, err = r.ReadString('\n')
		if err != nil {
			t.errors.read++
			t.conn.Close()
			t.conn = nil
			return
		}

		t.bytes += uint64(len(s))

		headers := strings.SplitN(s, ":", 2)
		switch strings.ToLower(headers[0]) {
		case "content-length":
			trimmed := strings.Trim(headers[1], " \r\n")
			content_length, err = strconv.ParseUint(trimmed, 10, 64)
			if err != nil {
				log.Fatalln(err.Error())
			}
		case "connection":
			keep_alive = strings.Contains(strings.ToLower(headers[1]), "keep-alive")
		}

		if s == "\r\n" {
			break
		}
	}

	// Read body.
	buf := make([]byte, content_length)
	remains := content_length
	for {
		var n int
		n, err = r.Read(buf)
		if err != nil {
			t.errors.read++
			t.conn.Close()
			t.conn = nil
			return
		}

		if n > 0 {
			remains -= uint64(n)
		}

		if remains == 0 {
			break
		}
	}

	t.bytes += uint64(content_length)

	// Completed
	t.complete++
	t.latency += time.Since(t.start)

	if !keep_alive {
		t.conn.Close()
		t.conn = nil
	}
}
<|endoftext|>


<commit_before>package main

// netbackup - Consistent multi-method backup tool
//
// See instructions in the README.md file that accompanies this program.
//
// (C) 2014 by Marco Paganini <paganini AT paganini DOT net>

import (
	"flag"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"time"

	"github.com/marcopaganini/logger"
	"github.com/marcopaganini/netbackup/config"
	"github.com/marcopaganini/netbackup/transports"
)

const (
	// DEBUG
	defaultLogDir      = "/tmp/log/netbackup"
	defaultLogDirMode  = 0770
	defaultLogFileMode = 0660
)

var (
	// Generic logging object
	log *logger.Logger
)

// Transport lists the transport agent (rclone/rbackup/rdiff-backup)
// used to make a particular backup.
type Transport interface {
	Run() error
	SetLogFile(io.Writer) error
}

// usage prints an error message and program usage to stderr, exiting after
// that.
func usage(err error) {
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error: %v\n\n", err)
	}
	fmt.Fprintf(os.Stderr, "Usage%s:\n", os.Args[0])
	flag.PrintDefaults()
	os.Exit(2)
}

// createOutputLog creates a new output logfile. If logFile is set, it is used
// unchanged. If not, a new log is created based on the default log directory,
// the configuration name, and the system date. Intermediate directories are
// created as needed. Returns a *os.File, the file created, error.
func createOutputLog(logFile string, configName string) (*os.File, string, error) {
	path := logFile
	if path == "" {
		dir := filepath.Join(defaultLogDir, configName)
		if err := os.MkdirAll(dir, defaultLogDirMode); err != nil {
			return nil, "", fmt.Errorf("Error trying to crete dir tree %q: %v", dir, err)
		}
		ymd := time.Now().Format("2006-01-02")
		path = filepath.Join(dir, configName+"-"+ymd+".log")
	}

	w, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY|os.O_CREATE, defaultLogFileMode)
	if err != nil {
		return nil, path, fmt.Errorf("Error opening %q: %v", path, err)
	}
	return w, path, err
}

func main() {
	// Set verbose level
	log = logger.New("")

	// Parse command line flags and read config file.
	if err := parseFlags(); err != nil {
		log.Fatalf("Error: %v", err)
	}
	// Set verbose level
	if opt.verbose > 0 {
		log.SetVerboseLevel(int(opt.verbose))
	}
	if opt.dryrun {
		log.Verbosef(2, "Warning: Dry-Run mode. Won't execute any commands.")
	}

	cfg, err := os.Open(opt.config)
	if err != nil {
		log.Fatalf("Unable to open config file: %v", err)
	}

	//config, err := parseConfig(cfg)
	config, err := config.ParseConfig(cfg)
	if err != nil {
		log.Fatalf("Configuration error in %q: %v", opt.config, err)
	}

	if config.Transport == "rclone" {
		outLog, outPath, err := createOutputLog(config.Logfile, config.Name)
		if err != nil {
			log.Fatalf("Error opening output logfile: %q: %v", outPath, err)
		}
		defer outLog.Close()

		// new rclone instance
		t, err := transports.NewRcloneTransport(config, nil, outLog, int(opt.verbose), opt.dryrun)
		if err != nil {
			log.Fatalf("Error creating rclone transport: %v", err)
		}
		if err := t.Run(); err != nil {
			log.Fatalln(err)
		}
	} else {
		log.Fatalf("Only rclone supported for now")
	}
}
<commit_msg>Small cleanups in main.go<commit_after>// netbackup - Consistent multi-method backup tool
//
// See instructions in the README.md file that accompanies this program.
//
// (C) 2015 by Marco Paganini <paganini AT paganini DOT net>

package main

import (
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"time"

	"github.com/marcopaganini/logger"
	"github.com/marcopaganini/netbackup/config"
	"github.com/marcopaganini/netbackup/transports"
)

const (
	// DEBUG
	defaultLogDir      = "/tmp/log/netbackup"
	defaultLogDirMode  = 0770
	defaultLogFileMode = 0660
)

var (
	// Generic logging object
	log *logger.Logger
)

// usage prints an error message and program usage to stderr, exiting after
// that.
func usage(err error) {
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error: %v\n\n", err)
	}
	fmt.Fprintf(os.Stderr, "Usage%s:\n", os.Args[0])
	flag.PrintDefaults()
	os.Exit(2)
}

// createOutputLog creates a new output logfile. If logFile is set, it is used
// unchanged. If not, a new log is created based on the default log directory,
// the configuration name, and the system date. Intermediate directories are
// created as needed. Returns a *os.File, the file created, error.
func createOutputLog(logFile string, configName string) (*os.File, string, error) {
	path := logFile
	if path == "" {
		dir := filepath.Join(defaultLogDir, configName)
		if err := os.MkdirAll(dir, defaultLogDirMode); err != nil {
			return nil, "", fmt.Errorf("Error trying to crete dir tree %q: %v", dir, err)
		}
		ymd := time.Now().Format("2006-01-02")
		path = filepath.Join(dir, configName+"-"+ymd+".log")
	}

	w, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY|os.O_CREATE, defaultLogFileMode)
	if err != nil {
		return nil, path, fmt.Errorf("Error opening %q: %v", path, err)
	}
	return w, path, err
}

func main() {
	log = logger.New("")

	// Parse command line flags and read config file.
	if err := parseFlags(); err != nil {
		log.Fatalf("Error: %v", err)
	}
	// Set verbose level
	if opt.verbose > 0 {
		log.SetVerboseLevel(int(opt.verbose))
	}
	if opt.dryrun {
		log.Verbosef(2, "Warning: Dry-Run mode. Won't execute any commands.")
	}

	cfg, err := os.Open(opt.config)
	if err != nil {
		log.Fatalf("Unable to open config file: %v", err)
	}

	//config, err := parseConfig(cfg)
	config, err := config.ParseConfig(cfg)
	if err != nil {
		log.Fatalf("Configuration error in %q: %v", opt.config, err)
	}

	if config.Transport == "rclone" {
		outLog, outPath, err := createOutputLog(config.Logfile, config.Name)
		if err != nil {
			log.Fatalf("Error opening output logfile: %q: %v", outPath, err)
		}
		defer outLog.Close()

		// new rclone instance
		t, err := transports.NewRcloneTransport(config, nil, outLog, int(opt.verbose), opt.dryrun)
		if err != nil {
			log.Fatalf("Error creating rclone transport: %v", err)
		}
		if err := t.Run(); err != nil {
			log.Fatalln(err)
		}
	} else {
		log.Fatalf("Only rclone supported for now")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"html"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"sync"

	"github.com/PuerkitoBio/goquery"
	"github.com/vincent-petithory/dataurl"
)

const yuyuteiURL = "http://yuyu-tei.jp/"
const wsDeckURL = "https://wsdecks.com"
const hoTcURL = "http://www.heartofthecards.com/code/cardlist.html?card=WS_"
const yuyuteiBase = "http://yuyu-tei.jp/game_ws"

// Prox struct
type Prox struct {
	// target url of reverse proxy
	target *url.URL
	// instance of Go ReverseProxy thatwill do the job for us
	proxy *httputil.ReverseProxy
}

type siteConfig struct {
	Name   string
	Filter string
}

type deckConfig struct {
	Dir  string
	Site siteConfig
}

type cardStruc struct {
	ID          string
	Translation string
}

type cardDeckInfo struct {
	ID     string
	Amount int
}

// New proxy
func New(target string) *Prox {
	url, _ := url.Parse(target)
	// you should handle error on parsing
	return &Prox{target: url, proxy: httputil.NewSingleHostReverseProxy(url)}
}

func (p *Prox) handle(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("X-GoProxy", "GoProxy")
	// call to magic method from ReverseProxy object
	p.proxy.ServeHTTP(w, r)
}
func lowCostSystemToURL(syspath string) string {
	return strings.Replace(syspath, "\\", "/", -1)
}

func convertToJpg(filePath string) {
	// convert -density 150 -trim to_love-ru_darkness_2nd_trial_deck.pdf -quality 100 -sharpen 0x1.0 love.jpg
	cmd := exec.Command("convert", "-density", "150", "-trim", filePath, "-quality", "100", "-sharpen", "0x1.0", filePath+".jpg")
	err := cmd.Start()
	if err != nil {
		fmt.Println(err)
	}
	err = cmd.Wait()
}

func createCardsCodeFile(dirPath string, cardsID []cardDeckInfo) (string, error) {
	//TODO Do nothing if file exists
	dirPath += "/"
	out, err := os.Create(dirPath + "codes.txt")
	defer out.Close()
	if err != nil {
		fmt.Println(err)
		return "", err
	}
	b, errMarshal := json.Marshal(cardsID)
	if errMarshal != nil {
		fmt.Println(errMarshal)
	}
	out.WriteString(string(b))
	return out.Name(), nil
}

func getTranslationHotC(codesPath string) []cardStruc {
	translations := []cardStruc{}
	file, err := os.Open(codesPath + "/codes.txt")
	scanner := bufio.NewScanner(file)
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println("getTranslationHotC")
	for scanner.Scan() {
		// fmt.Println(scanner.Text())
		cardsDeck := []cardDeckInfo{}
		json.Unmarshal(scanner.Bytes(), &cardsDeck)
		for _, cardDeck := range cardsDeck {
			url := hoTcURL + cardDeck.ID
			fmt.Println(url)
			doc, err := goquery.NewDocument(url)
			if err != nil {
				fmt.Println(err)
			}
			textHTML, err := doc.Find(".cards3").Slice(2, 3).Html()
			if err != nil {
				fmt.Println(err)
			}
			textHTML = strings.Replace(textHTML, "
", "
", -1)
			card := cardStruc{ID: cardDeck.ID, Translation: html.UnescapeString(textHTML)}

			translations = append(translations, card)
		}
	}
	return translations
}

func getDeckConfig(link string) (deckConfig, error) {
	uid := ""
	site := siteConfig{}

	if strings.Contains(link, yuyuteiURL) {
		site.Name = "yuyutei"
		site.Filter = ".card_list_box" + " .image img"
		parsedURL, _ := url.Parse(link)
		values, _ := url.ParseQuery(parsedURL.RawQuery)
		uid = values.Get("ver")
	} else if strings.Contains(link, wsDeckURL) {
		site.Name = "wsdeck"
		site.Filter = ".wscard" + " img"
		uid = filepath.Base(link)
	}
	dir := filepath.Join("static", site.Name, uid)
	deckConfig := deckConfig{Dir: dir, Site: site}
	if site.Filter == "" {
		return deckConfig, fmt.Errorf("Url is not supported %v", link)
	}

	return deckConfig, nil
}

func yytImages(w http.ResponseWriter, r *http.Request) {
	fmt.Println("yytImage")
	out, err := os.Create(filepath.Join("static", "yyt_image_urls.json"))
	var buffer bytes.Buffer
	defer out.Close()
	cardMap := map[string]string{}
	filter := "ul[data-class=sell] .item_single_card .nav_list_second .nav_list_third a"
	doc, err := goquery.NewDocument(yuyuteiBase)

	if err != nil {
		fmt.Println("Error in get yyt urls")
	}

	doc.Find(filter).Each(func(i int, s *goquery.Selection) {
		url, has := s.Attr("href")
		fmt.Println(url)
		if has {
			images, errCard := goquery.NewDocument(yuyuteiURL + url)
			images.Find(".card_unit").Each(func(cardI int, cardS *goquery.Selection) {
				cardURL, _ := cardS.Find(".image img").Attr("src")
				cardURL = strings.Replace(cardURL, "90_126", "front", 1)
				cardMap[strings.TrimSpace(cardS.Find(".id").Text())] = cardURL
			})
			if errCard != nil {
				fmt.Println(errCard)
			}
		}
	})
	b, errMarshal := json.Marshal(cardMap)
	if errMarshal != nil {
		fmt.Println(errMarshal)
	}
	json.Indent(&buffer, b, "", "\t")
	buffer.WriteTo(out)
	fmt.Println("finish")

}

func cardimages(w http.ResponseWriter, r *http.Request) {
	link := r.PostFormValue("url")
	cardIDs := []cardDeckInfo{}
	result := []string{}

	if link != "" {
	}
	doc, err := goquery.NewDocument(link)
	if err != nil {
		fmt.Println(err)
	}
	deckConfig, err := getDeckConfig(link)
	if err != nil {
		fmt.Println(err)
	}

	doc.Find("div .wscard").Each(func(i int, s *goquery.Selection) {
		card := cardDeckInfo{}
		cardID, exists := s.Attr("data-cardid")
		if exists {
			card.ID = cardID
		}

		cardAmount, exists := s.Attr("data-amount")
		if exists {
			card.Amount, err = strconv.Atoi(cardAmount)
			if err != nil {
				fmt.Println(err)
			}
		}
		cardIDs = append(cardIDs, card)
	})
	os.MkdirAll(deckConfig.Dir, 0744)
	createCardsCodeFile(deckConfig.Dir, cardIDs)
	yytImages, yytErr := ioutil.ReadFile(filepath.Join("static", "yyt_image_urls.json"))
	if yytErr != nil {
		fmt.Println(yytErr)
	}
	var yytMap = map[string]string{}
	json.Unmarshal(yytImages, &yytMap)
	for _, card := range cardIDs {
		cardURL, has := yytMap[card.ID]
		if has {
			urlPath := yuyuteiURL + cardURL
			result = append(result, urlPath)
		}
	}
	b, err := json.Marshal(result)

	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	w.Write(b)

}

func main() {
	proxy := New("http://localhost:8080")
	os.MkdirAll(filepath.Join("static", "yuyutei"), 0744)
	os.MkdirAll(filepath.Join("static", "wsdeck"), 0744)
	// static := http.FileServer(http.Dir("./"))
	http.HandleFunc("/", proxy.handle)

	http.HandleFunc("/translationimages", func(w http.ResponseWriter, r *http.Request) {
		link := r.PostFormValue("url")
		deckConfig, err := getDeckConfig(link)
		if err != nil {
			fmt.Println(err)
		}
		result := getTranslationHotC(deckConfig.Dir)
		b, err := json.Marshal(result)
		if err != nil {
			fmt.Println(err)
		}
		w.Write(b)
	})

	http.HandleFunc("/translationimages_old", func(w http.ResponseWriter, r *http.Request) {
		file := r.PostFormValue("file")
		filename := r.PostFormValue("filename")
		uid := strings.Replace(filename, filepath.Ext(filename), "", 1)
		dir := filepath.Join("static", uid)
		filePath := filepath.Join(dir, filename)

		data, err := dataurl.DecodeString(file)
		if err != nil {
			fmt.Println(err)
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		// fmt.Println(dataURL.Data)
		os.MkdirAll(dir, 0777)
		ioutil.WriteFile(filePath, data.Data, 0644)
		if _, err := os.Stat(dir); os.IsNotExist(err) {
			convertToJpg(filePath)
		}
		if err != nil {
			fmt.Println(err)
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		listJpg, err := filepath.Glob(filePath + "*.jpg")
		if err != nil {
			fmt.Println(err)
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		for i, jpgfile := range listJpg {
			listJpg[i] = lowCostSystemToURL(jpgfile)
		}

		b, err := json.Marshal(listJpg)

		if err != nil {
			fmt.Println(err)
			http.Error(w, err.Error(), 500)
			return
		}

		w.Write(b)
	})

	http.HandleFunc("/cardimages", cardimages)
	http.HandleFunc("/cardimages_old", func(w http.ResponseWriter, r *http.Request) {
		var wg sync.WaitGroup

		result := []string{}
		link := r.PostFormValue("url")

		if link != "" {
			doc, err := goquery.NewDocument(link)
			imageURL := ""
			deckConfig, err := getDeckConfig(link)

			if err != nil {
				fmt.Println(err)
			}
			if _, err := os.Stat(deckConfig.Dir); os.IsNotExist(err) {
				os.MkdirAll(deckConfig.Dir, 0744)
				doc.Find(deckConfig.Site.Filter).Each(func(i int, s *goquery.Selection) {
					wg.Add(1)
					val, _ := s.Attr("src")
					if deckConfig.Site.Name == "yuyutei" {
						big := strings.Replace(val, "90_126", "front", 1)
						imageURL = yuyuteiURL + big
					} else if deckConfig.Site.Name == "wsdeck" {
						imageURL = wsDeckURL + val
					}

					go func(url string) {
						defer wg.Done()
						// fmt.Println("dir : ", dir)
						fileName := filepath.Join(deckConfig.Dir, path.Base(url))
						fileName = strings.Split(fileName, "?")[0]
						out, err := os.Create(fileName)
						if err != nil {
							fmt.Println(err)
						}
						defer out.Close()
						reps, err := http.Get(url)

						if err != nil {
							fmt.Println(err)
						}

						file, err := io.Copy(out, reps.Body)
						if err != nil {
							fmt.Println(err)
						}

						fmt.Println("File", file)
						// fmt.Printf("Link: n-%d __ %v%v\n", i, imageURL, uid)
						defer reps.Body.Close()
						// fmt.Println("image url: ", strings.Replace(fileName, "\\", "/", 1))

						result = append(result, lowCostSystemToURL(fileName))
					}(imageURL)
				})
			} else {
				files, err := ioutil.ReadDir(deckConfig.Dir)
				if err != nil {
					fmt.Println(err)
				}
				yytImages, yytErr := ioutil.ReadFile(filepath.Join("static", "yyt_image_urls.json"))
				if yytErr != nil {
					fmt.Println(yytErr)
				}
				var yytMap = map[string]string{}
				json.Unmarshal(yytImages, &yytMap)
				for _, file := range files {
					cardID := strings.Replace(file.Name(), "_", "/", 1)
					cardID = strings.Replace(cardID, "_", "-", 1)
					cardID = strings.Split(cardID, ".")[0]
					fmt.Println(cardID)
					cardURL, has := yytMap[cardID]
					if has {
						urlPath := yuyuteiURL + cardURL
						result = append(result, urlPath)
					}
				}

			}
			wg.Wait()
			fmt.Printf("Finish")
			// createCardsCodeFile(deckConfig.Dir)
		}

		b, err := json.Marshal(result)

		if err != nil {
			http.Error(w, err.Error(), 500)
			return
		}
		w.Write(b)
	})

	http.HandleFunc("/update_yyt_images", yytImages)

	http.HandleFunc("/static/", func(w http.ResponseWriter, r *http.Request) {
		// fmt.Println("static", r.URL.Path[1:])
		http.ServeFile(w, r, r.URL.Path[1:])
	})

	http.ListenAndServe(":8010", nil)
}
<commit_msg>Add price info<commit_after>package main

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"html"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"sync"

	"github.com/PuerkitoBio/goquery"
	"github.com/vincent-petithory/dataurl"
)

const yuyuteiURL = "http://yuyu-tei.jp/"
const wsDeckURL = "https://wsdecks.com"
const hoTcURL = "http://www.heartofthecards.com/code/cardlist.html?card=WS_"
const yuyuteiBase = "http://yuyu-tei.jp/game_ws"

// Prox struct
type Prox struct {
	// target url of reverse proxy
	target *url.URL
	// instance of Go ReverseProxy thatwill do the job for us
	proxy *httputil.ReverseProxy
}

type siteConfig struct {
	Name   string
	Filter string
}

type deckConfig struct {
	Dir  string
	Site siteConfig
}

type cardStruc struct {
	ID          string
	Translation string
}

type cardDeckInfo struct {
	ID     string
	Amount int
}

type YytInfo struct {
	URL   string
	Price int
}

// New proxy
func New(target string) *Prox {
	url, _ := url.Parse(target)
	// you should handle error on parsing
	return &Prox{target: url, proxy: httputil.NewSingleHostReverseProxy(url)}
}

func (p *Prox) handle(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("X-GoProxy", "GoProxy")
	// call to magic method from ReverseProxy object
	p.proxy.ServeHTTP(w, r)
}
func lowCostSystemToURL(syspath string) string {
	return strings.Replace(syspath, "\\", "/", -1)
}

func convertToJpg(filePath string) {
	// convert -density 150 -trim to_love-ru_darkness_2nd_trial_deck.pdf -quality 100 -sharpen 0x1.0 love.jpg
	cmd := exec.Command("convert", "-density", "150", "-trim", filePath, "-quality", "100", "-sharpen", "0x1.0", filePath+".jpg")
	err := cmd.Start()
	if err != nil {
		fmt.Println(err)
	}
	err = cmd.Wait()
}

func createCardsCodeFile(dirPath string, cardsID []cardDeckInfo) (string, error) {
	//TODO Do nothing if file exists
	dirPath += "/"
	out, err := os.Create(dirPath + "codes.txt")
	defer out.Close()
	if err != nil {
		fmt.Println(err)
		return "", err
	}
	b, errMarshal := json.Marshal(cardsID)
	if errMarshal != nil {
		fmt.Println(errMarshal)
	}
	out.WriteString(string(b))
	return out.Name(), nil
}

func getTranslationHotC(codesPath string) []cardStruc {
	translations := []cardStruc{}
	file, err := os.Open(codesPath + "/codes.txt")
	scanner := bufio.NewScanner(file)
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println("getTranslationHotC")
	for scanner.Scan() {
		// fmt.Println(scanner.Text())
		cardsDeck := []cardDeckInfo{}
		json.Unmarshal(scanner.Bytes(), &cardsDeck)
		for _, cardDeck := range cardsDeck {
			url := hoTcURL + cardDeck.ID
			fmt.Println(url)
			doc, err := goquery.NewDocument(url)
			if err != nil {
				fmt.Println(err)
			}
			textHTML, err := doc.Find(".cards3").Slice(2, 3).Html()
			if err != nil {
				fmt.Println(err)
			}
			textHTML = strings.Replace(textHTML, "
", "
", -1)
			card := cardStruc{ID: cardDeck.ID, Translation: html.UnescapeString(textHTML)}

			translations = append(translations, card)
		}
	}
	return translations
}

func getDeckConfig(link string) (deckConfig, error) {
	uid := ""
	site := siteConfig{}

	if strings.Contains(link, yuyuteiURL) {
		site.Name = "yuyutei"
		site.Filter = ".card_list_box" + " .image img"
		parsedURL, _ := url.Parse(link)
		values, _ := url.ParseQuery(parsedURL.RawQuery)
		uid = values.Get("ver")
	} else if strings.Contains(link, wsDeckURL) {
		site.Name = "wsdeck"
		site.Filter = ".wscard" + " img"
		uid = filepath.Base(link)
	}
	dir := filepath.Join("static", site.Name, uid)
	deckConfig := deckConfig{Dir: dir, Site: site}
	if site.Filter == "" {
		return deckConfig, fmt.Errorf("Url is not supported %v", link)
	}

	return deckConfig, nil
}

func yytInfos(w http.ResponseWriter, r *http.Request) {
	fmt.Println("yytImage")
	out, err := os.Create(filepath.Join("static", "yyt_infos.json"))
	var buffer bytes.Buffer
	defer out.Close()
	cardMap := map[string]YytInfo{}
	filter := "ul[data-class=sell] .item_single_card .nav_list_second .nav_list_third a"
	doc, err := goquery.NewDocument(yuyuteiBase)

	if err != nil {
		fmt.Println("Error in get yyt urls")
	}

	doc.Find(filter).Each(func(i int, s *goquery.Selection) {
		url, has := s.Attr("href")
		fmt.Println(url)
		if has {
			images, errCard := goquery.NewDocument(yuyuteiURL + url)
			images.Find(".card_unit").Each(func(cardI int, cardS *goquery.Selection) {
				var price string
				price = cardS.Find(".price .sale").Text()
				if price == "" {
					price = strings.TrimSpace(cardS.Find(".price").Text())
				}
				cardPrice, errAtoi := strconv.Atoi(strings.TrimSuffix(price, "円"))
				if errAtoi != nil {
					fmt.Println(errAtoi)
				}
				cardURL, _ := cardS.Find(".image img").Attr("src")
				cardURL = strings.Replace(cardURL, "90_126", "front", 1)
				yytInfo := YytInfo{URL: cardURL, Price: cardPrice}
				cardMap[strings.TrimSpace(cardS.Find(".id").Text())] = yytInfo
			})
			if errCard != nil {
				fmt.Println(errCard)
			}
		}
	})
	b, errMarshal := json.Marshal(cardMap)
	if errMarshal != nil {
		fmt.Println(errMarshal)
	}
	json.Indent(&buffer, b, "", "\t")
	buffer.WriteTo(out)
	fmt.Println("finish")

}

func cardimages(w http.ResponseWriter, r *http.Request) {
	link := r.PostFormValue("url")
	cardIDs := []cardDeckInfo{}
	result := []string{}

	if link != "" {
	}
	doc, err := goquery.NewDocument(link)
	if err != nil {
		fmt.Println(err)
	}
	deckConfig, err := getDeckConfig(link)
	if err != nil {
		fmt.Println(err)
	}

	doc.Find("div .wscard").Each(func(i int, s *goquery.Selection) {
		card := cardDeckInfo{}
		cardID, exists := s.Attr("data-cardid")
		if exists {
			card.ID = cardID
		}

		cardAmount, exists := s.Attr("data-amount")
		if exists {
			card.Amount, err = strconv.Atoi(cardAmount)
			if err != nil {
				fmt.Println(err)
			}
		}
		cardIDs = append(cardIDs, card)
	})
	os.MkdirAll(deckConfig.Dir, 0744)
	createCardsCodeFile(deckConfig.Dir, cardIDs)
	yytInfos, yytErr := ioutil.ReadFile(filepath.Join("static", "yyt_infos.json"))
	if yytErr != nil {
		fmt.Println(yytErr)
	}
	var yytMap = map[string]string{}
	json.Unmarshal(yytInfos, &yytMap)
	for _, card := range cardIDs {
		cardURL, has := yytMap[card.ID]
		if has {
			urlPath := yuyuteiURL + cardURL
			result = append(result, urlPath)
		}
	}
	b, err := json.Marshal(result)

	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	w.Write(b)

}

func main() {
	proxy := New("http://localhost:8080")
	os.MkdirAll(filepath.Join("static", "yuyutei"), 0744)
	os.MkdirAll(filepath.Join("static", "wsdeck"), 0744)
	// static := http.FileServer(http.Dir("./"))
	http.HandleFunc("/", proxy.handle)

	http.HandleFunc("/translationimages", func(w http.ResponseWriter, r *http.Request) {
		link := r.PostFormValue("url")
		deckConfig, err := getDeckConfig(link)
		if err != nil {
			fmt.Println(err)
		}
		result := getTranslationHotC(deckConfig.Dir)
		b, err := json.Marshal(result)
		if err != nil {
			fmt.Println(err)
		}
		w.Write(b)
	})

	http.HandleFunc("/translationimages_old", func(w http.ResponseWriter, r *http.Request) {
		file := r.PostFormValue("file")
		filename := r.PostFormValue("filename")
		uid := strings.Replace(filename, filepath.Ext(filename), "", 1)
		dir := filepath.Join("static", uid)
		filePath := filepath.Join(dir, filename)

		data, err := dataurl.DecodeString(file)
		if err != nil {
			fmt.Println(err)
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		// fmt.Println(dataURL.Data)
		os.MkdirAll(dir, 0777)
		ioutil.WriteFile(filePath, data.Data, 0644)
		if _, err := os.Stat(dir); os.IsNotExist(err) {
			convertToJpg(filePath)
		}
		if err != nil {
			fmt.Println(err)
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		listJpg, err := filepath.Glob(filePath + "*.jpg")
		if err != nil {
			fmt.Println(err)
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		for i, jpgfile := range listJpg {
			listJpg[i] = lowCostSystemToURL(jpgfile)
		}

		b, err := json.Marshal(listJpg)

		if err != nil {
			fmt.Println(err)
			http.Error(w, err.Error(), 500)
			return
		}

		w.Write(b)
	})

	http.HandleFunc("/cardimages", cardimages)
	http.HandleFunc("/cardimages_old", func(w http.ResponseWriter, r *http.Request) {
		var wg sync.WaitGroup

		result := []string{}
		link := r.PostFormValue("url")

		if link != "" {
			doc, err := goquery.NewDocument(link)
			imageURL := ""
			deckConfig, err := getDeckConfig(link)

			if err != nil {
				fmt.Println(err)
			}
			if _, err := os.Stat(deckConfig.Dir); os.IsNotExist(err) {
				os.MkdirAll(deckConfig.Dir, 0744)
				doc.Find(deckConfig.Site.Filter).Each(func(i int, s *goquery.Selection) {
					wg.Add(1)
					val, _ := s.Attr("src")
					if deckConfig.Site.Name == "yuyutei" {
						big := strings.Replace(val, "90_126", "front", 1)
						imageURL = yuyuteiURL + big
					} else if deckConfig.Site.Name == "wsdeck" {
						imageURL = wsDeckURL + val
					}

					go func(url string) {
						defer wg.Done()
						// fmt.Println("dir : ", dir)
						fileName := filepath.Join(deckConfig.Dir, path.Base(url))
						fileName = strings.Split(fileName, "?")[0]
						out, err := os.Create(fileName)
						if err != nil {
							fmt.Println(err)
						}
						defer out.Close()
						reps, err := http.Get(url)

						if err != nil {
							fmt.Println(err)
						}

						file, err := io.Copy(out, reps.Body)
						if err != nil {
							fmt.Println(err)
						}

						fmt.Println("File", file)
						// fmt.Printf("Link: n-%d __ %v%v\n", i, imageURL, uid)
						defer reps.Body.Close()
						// fmt.Println("image url: ", strings.Replace(fileName, "\\", "/", 1))

						result = append(result, lowCostSystemToURL(fileName))
					}(imageURL)
				})
			} else {
				files, err := ioutil.ReadDir(deckConfig.Dir)
				if err != nil {
					fmt.Println(err)
				}
				yytInfos, yytErr := ioutil.ReadFile(filepath.Join("static", "yyt_infos.json"))
				if yytErr != nil {
					fmt.Println(yytErr)
				}
				var yytMap = map[string]string{}
				json.Unmarshal(yytInfos, &yytMap)
				for _, file := range files {
					cardID := strings.Replace(file.Name(), "_", "/", 1)
					cardID = strings.Replace(cardID, "_", "-", 1)
					cardID = strings.Split(cardID, ".")[0]
					fmt.Println(cardID)
					cardURL, has := yytMap[cardID]
					if has {
						urlPath := yuyuteiURL + cardURL
						result = append(result, urlPath)
					}
				}

			}
			wg.Wait()
			fmt.Printf("Finish")
			// createCardsCodeFile(deckConfig.Dir)
		}

		b, err := json.Marshal(result)

		if err != nil {
			http.Error(w, err.Error(), 500)
			return
		}
		w.Write(b)
	})

	http.HandleFunc("/update_yyt_infos", yytInfos)

	http.HandleFunc("/static/", func(w http.ResponseWriter, r *http.Request) {
		// fmt.Println("static", r.URL.Path[1:])
		http.ServeFile(w, r, r.URL.Path[1:])
	})

	http.ListenAndServe(":8010", nil)
}
<|endoftext|>