<commit_before>package main

import (
	"crypto/tls"
	"encoding/json"
	"flag"
	"html/template"
	"log"
	"net/http"
	"os"
	"os/exec"
	"sort"
	"strconv"
	"strings"
)

var (
	port           *int
	interval       *int
	insecure       *bool
	configFileName *string
	config         map[string]string
)

func main() {
	parseFlagsAndArgs()
	parseConfigFile()
	registerUI()
	registerAPI()
	listenAndServe()
}

// parseConfigFile decodes the configuration JSON
// file into a map.
func parseConfigFile() {
	configFile, err := os.Open(*configFileName)
	if err != nil {
		panic(err.Error())
	}

	err = json.NewDecoder(configFile).Decode(&config)
	if err != nil {
		panic(err.Error())
	}
}

func parseFlagsAndArgs() {
	port = flag.Int("port", 8080, "The port on which to access the results.")
	interval = flag.Int("interval", 60, "The interval between each UI refresh in seconds.")
	insecure = flag.Bool("insecure", false, "If set, will not verify the servers' certificate chain and host name.")
	configFileName = flag.String("config", "config.json", "The name of the configuration file.")
	flag.Parse()
}

// registerUI registers an HTTP handler function that presents the results
// in a web user interface
func registerUI() {
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		results := testNodes()

		tpl, err := template.ParseFiles("index.html")
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
		tpl.Execute(w, &ViewModel{*interval, results})
	})
}

// registerAPI registers an HTTP handler function that provides  the results
// in JSON format
func registerAPI() {
	http.HandleFunc("/json", func(w http.ResponseWriter, r *http.Request) {
		results := testNodes()
		w.Header().Set("Content-Type", "application/json")
		json.NewEncoder(w).Encode(results)
	})
}

// testNodes calls the getHTTPStatus function for
// all addresses in the config file that have a "http"
// prefix, and calls the ping function for all other
// provided addresses.
func testNodes() *TestResults {
	results := TestResults{}

	for name, url := range config {
		result := TestResult{}
		result.Name = name
		result.URL = url

		if strings.HasPrefix(result.URL, "http") {
			result.Method = "HTTP/S"

			code, err := getHTTPStatus(result.URL)
			if err != nil {
				result.Note = err.Error()
			}

			if code >= 200 && code < 300 {
				result.IsOK = true
			}

		} else {
			result.Method = "Ping"

			err := ping(result.URL)
			if err != nil {
				result.Note = err.Error()
			} else {
				result.IsOK = true
			}
		}

		results = append(results, result)
	}
	sort.Sort(&results)
	return &results
}

// getHTTPStatus issues an HTTP GET call to the specified URL
// and returns the HTTP status code. 0 is returned along with
// an error if the HTTP call could not be completed successfully.
func getHTTPStatus(url string) (code int, err error) {
	res, err := httpClient().Get(url)
	if err != nil {
		return 0, err
	}
	defer res.Body.Close()
	return res.StatusCode, nil
}

// ping returns nil if the specified IP address
// is responsive to pings, and returns an error
// otherwise
//
// NOTE: Currently only supported on Linux
func ping(ipAddr string) (err error) {
	cmd := exec.Command("ping", "-c", "1", "-w", "1", ipAddr)
	if err := cmd.Start(); err != nil {
		return err
	}
	if err := cmd.Wait(); err != nil {
		return err
	}
	return nil
}

func httpClient() *http.Client {

	if *insecure {
		transCfg := &http.Transport{
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: true,
			},
		}
		return &http.Client{Transport: transCfg}
	}
	return http.DefaultClient
}

func listenAndServe() {
	log.Println("Listening on port " + strconv.Itoa(*port))
	http.ListenAndServe(":"+strconv.Itoa(*port), nil)
}

// TestResult ...
type TestResult struct {
	// The name of the service
	Name string
	// The URL under test
	URL string
	// True if the status code is 2xx
	IsOK bool
	// Notes such as error messages
	Note string
	// The method that was used to test
	// the service's availability
	// (ex: ping or http)
	Method string
}

// TestResults ...
type TestResults []TestResult

func (results TestResults) Len() int {
	return len(results)
}

func (results TestResults) Less(i, j int) bool {
	return results[i].Name < results[j].Name
}

func (results TestResults) Swap(i, j int) {
	results[i], results[j] = results[j], results[i]
}

// ViewModel is the data structure for the UI template.
type ViewModel struct {
	// Polling interval in seconds
	PollingInterval int
	Results         *TestResults
}
<commit_msg>Add doc comment to httpClient function<commit_after>package main

import (
	"crypto/tls"
	"encoding/json"
	"flag"
	"html/template"
	"log"
	"net/http"
	"os"
	"os/exec"
	"sort"
	"strconv"
	"strings"
)

var (
	port           *int
	interval       *int
	insecure       *bool
	configFileName *string
	config         map[string]string
)

func main() {
	parseFlagsAndArgs()
	parseConfigFile()
	registerUI()
	registerAPI()
	listenAndServe()
}

// parseConfigFile decodes the configuration JSON
// file into a map.
func parseConfigFile() {
	configFile, err := os.Open(*configFileName)
	if err != nil {
		panic(err.Error())
	}

	err = json.NewDecoder(configFile).Decode(&config)
	if err != nil {
		panic(err.Error())
	}
}

func parseFlagsAndArgs() {
	port = flag.Int("port", 8080, "The port on which to access the results.")
	interval = flag.Int("interval", 60, "The interval between each UI refresh in seconds.")
	insecure = flag.Bool("insecure", false, "If set, will not verify the servers' certificate chain and host name.")
	configFileName = flag.String("config", "config.json", "The name of the configuration file.")
	flag.Parse()
}

// registerUI registers an HTTP handler function that presents the results
// in a web user interface
func registerUI() {
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		results := testNodes()

		tpl, err := template.ParseFiles("index.html")
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
		tpl.Execute(w, &ViewModel{*interval, results})
	})
}

// registerAPI registers an HTTP handler function that provides  the results
// in JSON format
func registerAPI() {
	http.HandleFunc("/json", func(w http.ResponseWriter, r *http.Request) {
		results := testNodes()
		w.Header().Set("Content-Type", "application/json")
		json.NewEncoder(w).Encode(results)
	})
}

// testNodes calls the getHTTPStatus function for
// all addresses in the config file that have a "http"
// prefix, and calls the ping function for all other
// provided addresses.
func testNodes() *TestResults {
	results := TestResults{}

	for name, url := range config {
		result := TestResult{}
		result.Name = name
		result.URL = url

		if strings.HasPrefix(result.URL, "http") {
			result.Method = "HTTP/S"

			code, err := getHTTPStatus(result.URL)
			if err != nil {
				result.Note = err.Error()
			}

			if code >= 200 && code < 300 {
				result.IsOK = true
			}

		} else {
			result.Method = "Ping"

			err := ping(result.URL)
			if err != nil {
				result.Note = err.Error()
			} else {
				result.IsOK = true
			}
		}

		results = append(results, result)
	}
	sort.Sort(&results)
	return &results
}

// getHTTPStatus issues an HTTP GET call to the specified URL
// and returns the HTTP status code. 0 is returned along with
// an error if the HTTP call could not be completed successfully.
func getHTTPStatus(url string) (code int, err error) {
	res, err := httpClient().Get(url)
	if err != nil {
		return 0, err
	}
	defer res.Body.Close()
	return res.StatusCode, nil
}

// ping returns nil if the specified IP address
// is responsive to pings, and returns an error
// otherwise
//
// NOTE: Currently only supported on Linux
func ping(ipAddr string) (err error) {
	cmd := exec.Command("ping", "-c", "1", "-w", "1", ipAddr)
	if err := cmd.Start(); err != nil {
		return err
	}
	if err := cmd.Wait(); err != nil {
		return err
	}
	return nil
}

// httpClient returns a HTTP client that does not
// verify a server's certificate chain and host name
// if the "insecure" command-line flag in set to true.
func httpClient() *http.Client {

	if *insecure {
		transCfg := &http.Transport{
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: true,
			},
		}
		return &http.Client{Transport: transCfg}
	}
	return http.DefaultClient
}

func listenAndServe() {
	log.Println("Listening on port " + strconv.Itoa(*port))
	http.ListenAndServe(":"+strconv.Itoa(*port), nil)
}

// TestResult ...
type TestResult struct {
	// The name of the service
	Name string
	// The URL under test
	URL string
	// True if the status code is 2xx
	IsOK bool
	// Notes such as error messages
	Note string
	// The method that was used to test
	// the service's availability
	// (ex: ping or http)
	Method string
}

// TestResults ...
type TestResults []TestResult

func (results TestResults) Len() int {
	return len(results)
}

func (results TestResults) Less(i, j int) bool {
	return results[i].Name < results[j].Name
}

func (results TestResults) Swap(i, j int) {
	results[i], results[j] = results[j], results[i]
}

// ViewModel is the data structure for the UI template.
type ViewModel struct {
	// Polling interval in seconds
	PollingInterval int
	Results         *TestResults
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/aquilax/go-dirble"
	"github.com/codegangsta/cli"
	"net/http"
	"os"
)

const (
	appName    = "dirble-cli"
	appVersion = "0.0.1"
	defaultInt = -1
)

func getDirble(token string) *dirble.Dirble {
	tr := http.Transport{}
	return dirble.New(&tr, token)
}

func processResult(d interface{}, err error) {
	var res []byte
	if err != nil {
		panic(err)
	}
	if res, err = json.MarshalIndent(d, "", "	"); err != nil {
		panic(err)
	}
	fmt.Printf("%s\n", res)
}

func intToParam(c *cli.Context, name string) *int {
	if c.IsSet(name) {
		result := c.Int(name)
		return &result
	}
	return nil
}

func main() {
	app := cli.NewApp()
	app.Name = appName
	app.Version = appVersion
	app.Usage = "Fetches information from dirble.com"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "token, t",
			Value:  "",
			Usage:  "API Token",
			EnvVar: "DIRBLE_API_TOKEN",
		},
	}
	app.Commands = []cli.Command{
		{
			Name:    "stations",
			Aliases: []string{"st"},
			Usage:   "Get List of stations",
			Flags: []cli.Flag{
				cli.IntFlag{
					Name:  "page",
					Usage: "page to fetch",
				},
				cli.IntFlag{
					Name:  "ipp",
					Usage: "items per page",
				},
				cli.IntFlag{
					Name:  "offset",
					Usage: "offset",
				},
			},
			Action: func(c *cli.Context) {
				processResult(getDirble(c.GlobalString("token")).Stations(intToParam(c, "page"),
					intToParam(c, "ipp"), intToParam(c, "offset")))
			},
		},
	}
	app.Run(os.Args)
}
<commit_msg>Added station command<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/aquilax/go-dirble"
	"github.com/codegangsta/cli"
	"net/http"
	"os"
	"strconv"
)

const (
	appName    = "dirble-cli"
	appVersion = "0.0.1"
	defaultInt = -1
)

func getDirble(token string) *dirble.Dirble {
	tr := http.Transport{}
	return dirble.New(&tr, token)
}

func processResult(d interface{}, err error) {
	var res []byte
	if err != nil {
		panic(err)
	}
	if res, err = json.MarshalIndent(d, "", "	"); err != nil {
		panic(err)
	}
	fmt.Printf("%s\n", res)
}

func intToParam(c *cli.Context, name string) *int {
	if c.IsSet(name) {
		result := c.Int(name)
		return &result
	}
	return nil
}

func main() {
	app := cli.NewApp()
	app.Name = appName
	app.Version = appVersion
	app.Usage = "Fetches information from dirble.com"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "token, t",
			Value:  "",
			Usage:  "API Token",
			EnvVar: "DIRBLE_API_TOKEN",
		},
	}
	app.Commands = []cli.Command{
		{
			Name:    "stations",
			Aliases: []string{"st"},
			Usage:   "Get List of stations",
			Flags: []cli.Flag{
				cli.IntFlag{
					Name:  "page",
					Usage: "page to fetch",
				},
				cli.IntFlag{
					Name:  "ipp",
					Usage: "items per page",
				},
				cli.IntFlag{
					Name:  "offset",
					Usage: "offset",
				},
			},
			Action: func(c *cli.Context) {
				processResult(getDirble(c.GlobalString("token")).Stations(intToParam(c, "page"),
					intToParam(c, "ipp"), intToParam(c, "offset")))
			},
		},
		{
			Name:  "station",
			Usage: "Get information about single station",
			Action: func(c *cli.Context) {
				if len(c.Args()) > 0 {
					id, err := strconv.Atoi(c.Args()[0])
					if err != nil {
						panic(err)
					}
					processResult(getDirble(c.GlobalString("token")).Station(id))
				}
			},
		},
	}
	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
)

func userCreateHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Println("Hello!")
	fmt.Println("The received method is... ", r.Method)
	switch r.Method {
	case "POST":
		r.ParseForm()

		steamID, _ := resolveVanityURL(r.PostFormValue("steamname"))
		fmt.Println("Your SteamID is...", steamID)

		http.Redirect(w, r, "/", http.StatusFound)
	default:
		http.NotFound(w, r)
	}
}

type ResolveVanityURLResponse struct {
	Response struct {
		SteamID string `json:"steamid"`
	}
}

func resolveVanityURL(steamName string) (string, error) {
	values := url.Values{}
	values.Add("vanityurl", url.QueryEscape(steamName))

	resolveVanityURLEndpoint := generateSteamAPIURL("ISteamUser/ResolveVanityURL/v0001", values, true)

	resp, err := http.Get(resolveVanityURLEndpoint.String())
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	var body []byte
	body, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}

	fmt.Println(string(body))

	structuredResponse := &ResolveVanityURLResponse{}
	err = json.Unmarshal(body, structuredResponse)
	if err != nil {
		return "", err
	}

	return structuredResponse.Response.SteamID, nil
}

func generateSteamAPIURL(apiPath string, values url.Values, withKey bool) *url.URL {
	generatedURL := &url.URL{Scheme: "http", Host: "api.steampowered.com", Path: apiPath}

	if withKey {
		values.Add("key", os.Getenv("STEAM_API_KEY"))
	}
	generatedURL.RawQuery = values.Encode()

	fmt.Println("the URL is...", generatedURL.String())
	return generatedURL
}

func main() {
	http.Handle("/", http.FileServer(http.Dir("./staticfiles")))
	http.HandleFunc("/user/create", userCreateHandler)

	http.ListenAndServe(":8080", nil)
}
<commit_msg>Fetch steam user's game library<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
)

func userCreateHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Println("Hello!")
	fmt.Println("The received method is... ", r.Method)
	switch r.Method {
	case "POST":
		r.ParseForm()

		steamID, _ := resolveVanityURL(r.PostFormValue("steamname"))
		fmt.Println("Your SteamID is...", steamID)

		gamesList, _ := getOwnedGames(steamID)
		fmt.Println("These are the games you own... ", gamesList)
		fmt.Printf("You own %d games\n", len(gamesList))

		http.Redirect(w, r, "/", http.StatusFound)
	default:
		http.NotFound(w, r)
	}
}

type ResolveVanityURLResponse struct {
	Response struct {
		SteamID string `json:"steamid"`
	}
}

func resolveVanityURL(steamName string) (string, error) {
	values := url.Values{}
	values.Add("vanityurl", url.QueryEscape(steamName))

	resolveVanityURLEndpoint := generateSteamAPIURL("ISteamUser/ResolveVanityURL/v0001", values, true)

	resp, err := http.Get(resolveVanityURLEndpoint.String())
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	var body []byte
	body, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}

	fmt.Println(string(body))

	structuredResponse := &ResolveVanityURLResponse{}
	err = json.Unmarshal(body, structuredResponse)
	if err != nil {
		return "", err
	}

	return structuredResponse.Response.SteamID, nil
}

type Game struct {
	Name     string
	AppID    int
	Playtime int `json:"playtime_forever"`
}

type GetOwnedGamesResponse struct {
	Response struct {
		Games []Game
	}
}

func getOwnedGames(steamID string) ([]Game, error) {
	values := url.Values{}
	values.Add("steamid", url.QueryEscape(steamID))
	values.Add("include_appinfo", "1")

	getOwnedGamesEndpoint := generateSteamAPIURL("IPlayerService/GetOwnedGames/v0001", values, true)

	resp, err := http.Get(getOwnedGamesEndpoint.String())
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	var body []byte
	body, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	fmt.Println(string(body))

	structuredResponse := &GetOwnedGamesResponse{}
	err = json.Unmarshal(body, structuredResponse)
	if err != nil {
		return nil, err
	}

	return structuredResponse.Response.Games, nil
}

func generateSteamAPIURL(apiPath string, values url.Values, withKey bool) *url.URL {
	generatedURL := &url.URL{Scheme: "http", Host: "api.steampowered.com", Path: apiPath}

	if withKey {
		values.Add("key", os.Getenv("STEAM_API_KEY"))
	}
	generatedURL.RawQuery = values.Encode()

	fmt.Println("the URL is...", generatedURL.String())
	return generatedURL
}

func main() {
	http.Handle("/", http.FileServer(http.Dir("./staticfiles")))
	http.HandleFunc("/user/create", userCreateHandler)

	http.ListenAndServe(":8080", nil)
}
<|endoftext|>


<commit_before>// ## golit

// **golit** generates literate-programming-style HTML documentation
// from a Go source file. It produces HTML with comments alongside your
// code. Comments are parsed through [Markdown](http://daringfireball.net/projects/markdown/syntax)
// and code highlighted with [Pygments](http://pygments.org/).

// golit is based on [docco](http://jashkenas.github.com/docco/)
// and [shocco](http://rtomayko.github.com/shocco/), two earlier
// programs in the same style.

// This page is the result of running golit against its own source
// file.

package main

import (
    "fmt"
    "io/ioutil"
    "os"
    "os/exec"
    "regexp"
    "strings"
)

// ### Usage

// golit takes exactly one argument: the path to a Go source file.
// It writes the compiled HTML on stdout.
var usage = "usage: golit input.go title > output.html"

// ### Helpers

// Panic on non-nil errors. We'll call this after error-returning
// functions.
func check(err error) {
    if err != nil {
        panic(err)
    }
}

// We'll implement Markdown rendering and Pygments syntax highlighting
// by piping the source data through external programs. This is a
// general helper for handling both cases.
func pipe(bin string, arg []string, src string) string {
    cmd := exec.Command(bin, arg...)
    in, _ := cmd.StdinPipe()
    out, _ := cmd.StdoutPipe()
    cmd.Start()
    in.Write([]byte(src))
    in.Close()
    bytes, _ := ioutil.ReadAll(out)
    err := cmd.Wait()
    check(err)
    return string(bytes)
}

// ### Rendering

// Recognize doc lines, extract their comment prefixes.
var docsPat = regexp.MustCompile("^\\s*\\/\\/\\s")

// Recognize header comment lines specially.
var headerPat = regexp.MustCompile("^\\/\\/\\s#+\\s")

// We'll break the code into `{docs, code}` pairs, and then render
// those text segments before including them in the HTML doc.
type seg struct {
    docs, code, docsRendered, codeRendered string
}

func main() {
    // Accept exactly 2 argument, the source path and page title.
    if len(os.Args) != 3 {
        fmt.Fprintln(os.Stderr, usage)
        os.Exit(1)
    }
    sourcePath := os.Args[1]
    title := os.Args[2]

    // Ensure that we have `markdown` and `pygmentize` binaries,
    // remember their paths.
    markdownPath, err := exec.LookPath("markdown")
    check(err)
    pygmentizePath, err := exec.LookPath("pygmentize")
    check(err)

    // Read the source file in, split into lines.
    srcBytes, err := ioutil.ReadFile(sourcePath)
    check(err)
    lines := strings.Split(string(srcBytes), "\n")

    // Group lines into docs/code segments.
    segs := []*seg{}
    segs = append(segs, &seg{code: "", docs: ""})
    lastSeen := ""
    for _, line := range lines {
        headerMatch := headerPat.MatchString(line)
        docsMatch := docsPat.MatchString(line)
        emptyMatch := line == ""
        lastSeg := segs[len(segs)-1]
        lastHeader := lastSeen == "header"
        lastDocs := lastSeen == "docs"
        newHeader := (lastSeen != "header")
        newDocs := (lastSeen != "docs") && lastSeg.docs != ""
        newCode := (lastSeen != "code") && lastSeg.code != ""
        // Header line - strip out comment indicator and ensure a
        // dedicated segment for the header, indpendent of potential
        // surrounding docs.
        if headerMatch || (emptyMatch && lastHeader) {
            trimmed := docsPat.ReplaceAllString(line, "")
            if newHeader {
                newSeg := seg{docs: trimmed, code: ""}
                segs = append(segs, &newSeg)
            } else {
                lastSeg.docs = lastSeg.docs + "\n" + trimmed
            }
            // Docs line - strip out comment indicator.
        } else if docsMatch || (emptyMatch && lastDocs) {
            trimmed := docsPat.ReplaceAllString(line, "")
            if newDocs {
                newSeg := seg{docs: trimmed, code: ""}
                segs = append(segs, &newSeg)
            } else {
                lastSeg.docs = lastSeg.docs + "\n" + trimmed
            }
            lastSeen = "docs"
            // Code line - preserve all whitespace.
        } else {
            if newCode {
                newSeg := seg{docs: "", code: line}
                segs = append(segs, &newSeg)
            } else {
                lastSeg.code = lastSeg.code + "\n" + line
            }
            lastSeen = "code"
        }
    }

    // Render docs via `markdown` and code via `pygmentize` in each
    // segment.
    for _, seg := range segs {
        seg.docsRendered = pipe(markdownPath, []string{}, seg.docs)
        seg.codeRendered = pipe(pygmentizePath, []string{"-l", "go", "-f", "html"}, seg.code+"  ")
    }

    // Print HTML header.
    fmt.Printf(`
<!DOCTYPE html>

    <meta http-eqiv="content-type" content="text/html;charset=utf-8">
    <title>%s</title>
    <link rel=stylesheet href="http://jashkenas.github.com/docco/resources/docco.css">

`, title)

    // Print HTML docs/code segments.
    for _, seg := range segs {
        fmt.Printf(
            `

             %s
             %s

`, seg.docsRendered, seg.codeRendered)
    }

    // Print HTML footer.
    fmt.Print(`

     `)
}
<commit_msg>more comments<commit_after>// ## golit

// **golit** generates literate-programming-style HTML documentation
// from a Go source file. It produces HTML with comments alongside your
// code. Comments are parsed through [Markdown](http://daringfireball.net/projects/markdown/syntax)
// and code highlighted with [Pygments](http://pygments.org/).

// golit is based on [docco](http://jashkenas.github.com/docco/)
// and [shocco](http://rtomayko.github.com/shocco/), two earlier
// programs in the same style.

// This page is the result of running golit against its own source
// file.

package main

import (
    "fmt"
    "io/ioutil"
    "os"
    "os/exec"
    "regexp"
    "strings"
)

// ### Usage

// golit takes exactly one argument: the path to a Go source file.
// It writes the compiled HTML on stdout.
var usage = "usage: golit input.go title > output.html"

// ### Helpers

// Panic on non-nil errors. We'll call this after error-returning
// functions.
func check(err error) {
    if err != nil {
        panic(err)
    }
}

// We'll implement Markdown rendering and Pygments syntax highlighting
// by piping the source data through external programs. This is a
// general helper for handling both cases.
func pipe(bin string, arg []string, src string) string {
    cmd := exec.Command(bin, arg...)
    in, _ := cmd.StdinPipe()
    out, _ := cmd.StdoutPipe()
    cmd.Start()
    in.Write([]byte(src))
    in.Close()
    bytes, _ := ioutil.ReadAll(out)
    err := cmd.Wait()
    check(err)
    return string(bytes)
}

// ### Processing

// Recognize doc lines, extract their comment prefixes.
var docsPat = regexp.MustCompile("^\\s*\\/\\/\\s")

// Recognize header comment lines specially.
var headerPat = regexp.MustCompile("^\\/\\/\\s#+\\s")

// We'll break the code into `{docs, code}` pairs, and then render
// those text segments before including them in the HTML doc.
type seg struct {
    docs, code, docsRendered, codeRendered string
}

func main() {
    // Accept exactly 2 argument, the source path and page title.
    if len(os.Args) != 3 {
        fmt.Fprintln(os.Stderr, usage)
        os.Exit(1)
    }
    sourcePath := os.Args[1]
    title := os.Args[2]

    // Ensure that we have `markdown` and `pygmentize` binaries,
    // remember their paths.
    markdownPath, err := exec.LookPath("markdown")
    check(err)
    pygmentizePath, err := exec.LookPath("pygmentize")
    check(err)

    // Read the source file in, split into lines.
    srcBytes, err := ioutil.ReadFile(sourcePath)
    check(err)
    lines := strings.Split(string(srcBytes), "\n")

    // Group lines into docs/code segments. There are two tricky
    // aspects to this. First, we want to treat header comments
    // sepcially so that they are always in their own segment and
    // therefore never directly adjacent to any code. Second, we need
    // to correctly start new segments on certain code/doc boundries
    // but not on others. In order to handle this later aspect we'll
    // refer to some state about the previous line and segment in
    // deciding to handle the one being processed.
    segs := []*seg{}
    segs = append(segs, &seg{code: "", docs: ""})
    lastSeen := ""
    for _, line := range lines {
        headerMatch := headerPat.MatchString(line)
        docsMatch := docsPat.MatchString(line)
        emptyMatch := line == ""
        lastSeg := segs[len(segs)-1]
        lastHeader := lastSeen == "header"
        lastDocs := lastSeen == "docs"
        newHeader := (lastSeen != "header")
        newDocs := (lastSeen != "docs") && lastSeg.docs != ""
        newCode := (lastSeen != "code") && lastSeg.code != ""
        // Header line - strip out comment indicator and ensure a
        // dedicated segment for the header, indpendent of potential
        // surrounding docs. Note that here an in the other cases
        // below we coalesced empty lines into the type of the previous
        // line.
        if headerMatch || (emptyMatch && lastHeader) {
            trimmed := docsPat.ReplaceAllString(line, "")
            if newHeader {
                newSeg := seg{docs: trimmed, code: ""}
                segs = append(segs, &newSeg)
            } else {
                lastSeg.docs = lastSeg.docs + "\n" + trimmed
            }
        // Docs line - strip out comment indicator.
        } else if docsMatch || (emptyMatch && lastDocs) {
            trimmed := docsPat.ReplaceAllString(line, "")
            if newDocs {
                newSeg := seg{docs: trimmed, code: ""}
                segs = append(segs, &newSeg)
            } else {
                lastSeg.docs = lastSeg.docs + "\n" + trimmed
            }
            lastSeen = "docs"
        // Code line - preserve all whitespace.
        } else {
            if newCode {
                newSeg := seg{docs: "", code: line}
                segs = append(segs, &newSeg)
            } else {
                lastSeg.code = lastSeg.code + "\n" + line
            }
            lastSeen = "code"
        }
    }

    // Render docs via `markdown` and code via `pygmentize` in each
    // segment, using our `pipe` helper.
    for _, seg := range segs {
        seg.docsRendered = pipe(markdownPath, []string{}, seg.docs)
        seg.codeRendered = pipe(pygmentizePath, []string{"-l", "go", "-f", "html"}, seg.code+"  ")
    }

// ### Rendering

    // Print HTML header.
    fmt.Printf(`
<!DOCTYPE html>

    <meta http-eqiv="content-type" content="text/html;charset=utf-8">
    <title>%s</title>
    <link rel=stylesheet href="http://jashkenas.github.com/docco/resources/docco.css">

`, title)

    // Print HTML docs/code segments.
    for _, seg := range segs {
        fmt.Printf(
            `

             %s
             %s

`, seg.docsRendered, seg.codeRendered)
    }

    // Print HTML footer.
    fmt.Print(`

     `)
}
<|endoftext|>


<commit_before>package main

import (
  "fmt"
  "os"
  "log"
  "strconv"
  "path/filepath"
  tag "github.com/dhowden/tag"
)

func getNewFileName(m tag.Metadata) string {
  artist := m.AlbumArtist()

  if artist == "" {
    artist = m.Artist()
  }

  year := strconv.Itoa(m.Year())
  album := m.Album()

  folder := artist + " - " + year + " - " + album

  return folder
}

func main () {
  // Get the folders passed as arguments
  folders := os.Args[1:]

  for _,folder := range folders {
    d, err := os.Open(folder)

    if err != nil {
      fmt.Println(err)
      os.Exit(1)
    }

    defer d.Close()

    files, err := d.Readdir(-1)
    if err != nil {
      fmt.Println(err)
      os.Exit(1)
    }

    for _, file := range files {
      if file.Mode().IsRegular() && filepath.Ext(file.Name()) == ".mp3" {
        var f, err = os.Open(folder + "/" + file.Name())

        if err != nil {
          log.Fatal(err)
        }

        defer f.Close()

        m, err := tag.ReadFrom(f)

        if err != nil {
          log.Fatal(err)
        }

        var newFolderName = getNewFileName(m)
        fmt.Println(newFolderName);

        os.Rename(folder, newFolderName)

        break
      }
    }
  }
}
<commit_msg>Colored output<commit_after>package main

import (
  "fmt"
  "os"
  "log"
  "strconv"
  "path/filepath"
  tag "github.com/dhowden/tag"
  color "github.com/fatih/color"
)

func getNewFileName(m tag.Metadata) string {
  artist := m.AlbumArtist()

  if artist == "" {
    artist = m.Artist()
  }

  year := strconv.Itoa(m.Year())
  album := m.Album()

  folder := artist + " - " + year + " - " + album

  return folder
}

func main () {
  // Get the folders passed as arguments
  folders := os.Args[1:]

  green := color.New(color.FgGreen).SprintFunc()

  for _,folder := range folders {
    d, err := os.Open(folder)

    if err != nil {
      fmt.Println(err)
      os.Exit(1)
    }

    defer d.Close()

    files, err := d.Readdir(-1)
    if err != nil {
      fmt.Println(err)
      os.Exit(1)
    }

    for _, file := range files {
      if file.Mode().IsRegular() && filepath.Ext(file.Name()) == ".mp3" {
        var f, err = os.Open(folder + "/" + file.Name())

        if err != nil {
          log.Fatal(err)
        }

        defer f.Close()

        m, err := tag.ReadFrom(f)

        if err != nil {
          log.Fatal(err)
        }

        var newFolderName = getNewFileName(m)
        fmt.Printf("Renaming \"" + folder + "\" to \"" + newFolderName + "\" ... ")
        fmt.Printf("%s\n", green("Success ✔"))

        os.Rename(folder, newFolderName)

        break
      }
    }
  }
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"

	"github.com/bcicen/ctop/config"
	"github.com/bcicen/ctop/cwidgets/compact"
	"github.com/bcicen/ctop/logging"
	"github.com/bcicen/ctop/widgets"
	ui "github.com/gizak/termui"
)

var (
	build   = "none"
	version = "dev-build"

	log    *logging.CTopLogger
	cursor *GridCursor
	cGrid  *compact.CompactGrid
	header *widgets.CTopHeader
)

func main() {
	defer panicExit()

	// init global config
	config.Init()

	// parse command line arguments
	var versionFlag = flag.Bool("v", false, "output version information and exit")
	var helpFlag = flag.Bool("h", false, "display this help dialog")
	var filterFlag = flag.String("f", "", "filter containers")
	var activeOnlyFlag = flag.Bool("a", false, "show active containers only")
	var sortFieldFlag = flag.String("s", "", "select container sort field")
	var reverseSortFlag = flag.Bool("r", false, "reverse container sort order")
	flag.Parse()

	if *versionFlag == true {
		printVersion()
		os.Exit(0)
	}

	if *helpFlag == true {
		printHelp()
		os.Exit(0)
	}

	// init ui
	ui.ColorMap = ColorMap // override default colormap
	if err := ui.Init(); err != nil {
		panic(err)
	}
	defer ui.Close()

	// override default config values with command line flags
	if *filterFlag != "" {
		config.Update("filterStr", *filterFlag)
	}

	if *activeOnlyFlag == true {
		config.Toggle("allContainers")
	}

	if *sortFieldFlag != "" {
		config.Update("sortField", *sortFieldFlag)
	}

	if *reverseSortFlag == true {
		config.Toggle("sortReversed")
	}

	// init logger
	log = logging.Init()
	if config.GetSwitchVal("loggingEnabled") {
		logging.StartServer()
	}

	// init grid, cursor, header
	cursor = NewGridCursor()
	cGrid = compact.NewCompactGrid()
	header = widgets.NewCTopHeader()

	for {
		exit := Display()
		if exit {
			log.Notice("shutting down")
			log.Exit()
			return
		}
	}
}

func panicExit() {
	if r := recover(); r != nil {
		ui.Clear()
		fmt.Printf("panic: %s\n", r)
		os.Exit(1)
	}
}

var helpMsg = `ctop - container metric viewer

usage: ctop [options]

options:
`

func printHelp() {
	fmt.Println(helpMsg)
	flag.PrintDefaults()
}

func printVersion() {
	fmt.Printf("ctop version %v, build %v\n", version, build)
}
<commit_msg>add validation to sort field option<commit_after>package main

import (
	"flag"
	"fmt"
	"os"

	"github.com/bcicen/ctop/config"
	"github.com/bcicen/ctop/cwidgets/compact"
	"github.com/bcicen/ctop/logging"
	"github.com/bcicen/ctop/widgets"
	ui "github.com/gizak/termui"
)

var (
	build   = "none"
	version = "dev-build"

	log    *logging.CTopLogger
	cursor *GridCursor
	cGrid  *compact.CompactGrid
	header *widgets.CTopHeader
)

func main() {
	defer panicExit()

	// init global config
	config.Init()

	// parse command line arguments
	var versionFlag = flag.Bool("v", false, "output version information and exit")
	var helpFlag = flag.Bool("h", false, "display this help dialog")
	var filterFlag = flag.String("f", "", "filter containers")
	var activeOnlyFlag = flag.Bool("a", false, "show active containers only")
	var sortFieldFlag = flag.String("s", "", "select container sort field")
	var reverseSortFlag = flag.Bool("r", false, "reverse container sort order")
	flag.Parse()

	if *versionFlag == true {
		printVersion()
		os.Exit(0)
	}

	if *helpFlag == true {
		printHelp()
		os.Exit(0)
	}

	// override default config values with command line flags
	if *filterFlag != "" {
		config.Update("filterStr", *filterFlag)
	}

	if *activeOnlyFlag == true {
		config.Toggle("allContainers")
	}

	if *sortFieldFlag != "" {
		validSort(*sortFieldFlag)
		config.Update("sortField", *sortFieldFlag)
	}

	if *reverseSortFlag == true {
		config.Toggle("sortReversed")
	}

	// init logger
	log = logging.Init()
	if config.GetSwitchVal("loggingEnabled") {
		logging.StartServer()
	}

	// init ui
	ui.ColorMap = ColorMap // override default colormap
	if err := ui.Init(); err != nil {
		panic(err)
	}
	defer ui.Close()

	// init grid, cursor, header
	cursor = NewGridCursor()
	cGrid = compact.NewCompactGrid()
	header = widgets.NewCTopHeader()

	for {
		exit := Display()
		if exit {
			log.Notice("shutting down")
			log.Exit()
			return
		}
	}
}

// ensure a given sort field is valid
func validSort(s string) {
	if _, ok := Sorters[s]; !ok {
		fmt.Printf("invalid sort field: %s\n", s)
		os.Exit(1)
	}
}

func panicExit() {
	if r := recover(); r != nil {
		ui.Clear()
		fmt.Printf("panic: %s\n", r)
		os.Exit(1)
	}
}

var helpMsg = `ctop - container metric viewer

usage: ctop [options]

options:
`

func printHelp() {
	fmt.Println(helpMsg)
	flag.PrintDefaults()
}

func printVersion() {
	fmt.Printf("ctop version %v, build %v\n", version, build)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"errors"
	"golang.org/x/net/html/charset"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path"
	"regexp"
	"syscall"
)

type songInfo struct {
	artist, title string
}

func main() {
	home := os.Getenv("HOME")
	if home == "" {
		log.Fatal("HOME not found")
	}

	songinfo, err := getSongInfo()
	if err != nil {
		log.Fatal(err)
	}

	// TODO
	// artist and title can contain slashes
	dotDir := path.Join(home, ".show-lyrics")
	cacheDir := path.Join(dotDir, "cache")
	cacheArtistDir := path.Join(cacheDir, songinfo.artist)
	songFile := path.Join(cacheArtistDir, songinfo.title + ".txt")

	for _, dir := range []string{dotDir, cacheDir, cacheArtistDir} {
		err := mkdirUnlessExists(dir)
		if err != nil {
			log.Fatal(err)
		}
	}

	client := &http.Client{}

	lyrics, err := fetchLyrics(client, songinfo)
	if err != nil {
		log.Fatal(err)
	}

	// TODO: add pretty title
	// TODO: add newline

	err = ioutil.WriteFile(songFile, lyrics, 0644)
	if err != nil {
		log.Fatal(err)
	}

	lessBin, err := exec.LookPath("less")
	if err != nil {
		log.Fatal(err)
	}
	err = syscall.Exec(lessBin, []string{"-c", songFile}, os.Environ())
	if err != nil {
		log.Fatal(err)
	}
}

func getSongInfo() (*songInfo, error) {
	cmusStatus, err := getCmusStatus()
	if err != nil {
		return nil, err
	}
	songinfo, err := parseCmusStatus(cmusStatus)
	if err != nil {
		return nil, err
	}
	return songinfo, nil
}

func mkdirUnlessExists(dir string) error {
	_, err := os.Stat(dir)
	if err != nil {
		err = os.Mkdir(dir, 0755)
		if err != nil {
			return err
		}
	}
	return nil
}

func getCmusStatus() ([]byte, error) {
	cmd := exec.Command("cmus-remote", "-Q")
	return cmd.Output()
}

var artistRe = regexp.MustCompile(`(?m)^tag\s+artist\s+(.+)\s*$`)
var titleRe = regexp.MustCompile(`(?m)^tag\s+title\s+(.+)\s*$`)

func regexpMatch(re *regexp.Regexp, buf []byte) []byte {
	match := re.FindAllSubmatch(buf, 1)
	if len(match) > 0 {
		return match[0][1]
	}
	return nil
}

func parseCmusStatus(cmusStatus []byte) (*songInfo, error) {
	artist := regexpMatch(artistRe, cmusStatus)
	title := regexpMatch(titleRe, cmusStatus)

	if artist == nil || title == nil {
		return nil, errors.New("Failed to parse cmus status")
	}

	si := songInfo{
		artist: string(artist),
		title:  string(title),
	}

	return &si, nil
}

func makeURL(si *songInfo) string {
	artist := []byte(si.artist)
	title := []byte(si.title)

	theRe := regexp.MustCompile(`(?i)^the `)
	weirdRe := regexp.MustCompile(`(?i)[^a-z0-9]`)

	artist = theRe.ReplaceAll(artist, []byte{})

	artist = bytes.ToLower(artist)
	title = bytes.ToLower(title)

	for _, str := range []*[]byte{&artist, &title} {
		*str = bytes.ToLower(*str)
		*str = weirdRe.ReplaceAll(*str, []byte{})
	}

	url := "https://www.azlyrics.com/lyrics/"
	url += string(artist) + "/" + string(title) + ".html"

	return url
}

func fetchLyrics(client *http.Client, si *songInfo) ([]byte, error) {
	reqUrl := makeURL(si)

	req, err := http.NewRequest("GET", reqUrl, nil)
	if err != nil {
		return []byte{}, err
	}

	resp, err := client.Do(req)
	if err != nil {
		return []byte{}, err
	}

	if resp.StatusCode != 200 {
		return []byte{}, errors.New(resp.Status)
	}

	utf8, err := charset.NewReader(resp.Body, resp.Header.Get("Content-Type"))
	if err != nil {
		return []byte{}, err
	}
	body, err := ioutil.ReadAll(utf8)
	if err != nil {
		return []byte{}, err
	}

	lyrics, err := parseLyrics(body)
	if err != nil {
		return []byte{}, err
	}

	return lyrics, nil
}

func htmlStrip(html []byte) []byte {
	commentsRe := regexp.MustCompile(`(?s)<!--.*?-->`)
	brRe := regexp.MustCompile(`<br/?>`)

	html = commentsRe.ReplaceAll(html, []byte{})
	html = brRe.ReplaceAll(html, []byte{})
	html = bytes.TrimSpace(html)

	return html
}

func parseLyrics(lyricsHtml []byte) ([]byte, error) {
	re := regexp.MustCompile(
		`(?s)<div[^<>]*?class="lyricsh"[^<>]*?>.*?</div>\s*?` +
			`<div[^<>]*?>.*?</div>\s*` +
			`.*?` +
			`<div[^<>]*?>(.*?)</div>`)

	match := re.FindAllSubmatch(lyricsHtml, 1)
	if match == nil {
		return []byte{}, errors.New("Failed to parse html")
	}

	lyrics := htmlStrip(match[0][1])
	return lyrics, nil
}
<commit_msg>Add func execLess<commit_after>package main

import (
	"bytes"
	"errors"
	"golang.org/x/net/html/charset"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path"
	"regexp"
	"syscall"
)

type songInfo struct {
	artist, title string
}

func main() {
	home := os.Getenv("HOME")
	if home == "" {
		log.Fatal("HOME not found")
	}

	songinfo, err := getSongInfo()
	if err != nil {
		log.Fatal(err)
	}

	// TODO
	// artist and title can contain slashes
	dotDir := path.Join(home, ".show-lyrics")
	cacheDir := path.Join(dotDir, "cache")
	cacheArtistDir := path.Join(cacheDir, songinfo.artist)
	songFile := path.Join(cacheArtistDir, songinfo.title + ".txt")

	for _, dir := range []string{dotDir, cacheDir, cacheArtistDir} {
		err := mkdirUnlessExists(dir)
		if err != nil {
			log.Fatal(err)
		}
	}

	client := &http.Client{}

	lyrics, err := fetchLyrics(client, songinfo)
	if err != nil {
		log.Fatal(err)
	}

	// TODO: add pretty title
	// TODO: add newline

	err = ioutil.WriteFile(songFile, lyrics, 0644)
	if err != nil {
		log.Fatal(err)
	}

	err = execLess(songFile)
	if err != nil {
		log.Fatal(err)
	}
}

func execLess(file string) error {
	lessBin, err := exec.LookPath("less")
	if err != nil {
		return err
	}
	err = syscall.Exec(lessBin, []string{"-c", file}, os.Environ())
	if err != nil {
		return err
	}
	return nil
}

func getSongInfo() (*songInfo, error) {
	cmusStatus, err := getCmusStatus()
	if err != nil {
		return nil, err
	}
	songinfo, err := parseCmusStatus(cmusStatus)
	if err != nil {
		return nil, err
	}
	return songinfo, nil
}

func mkdirUnlessExists(dir string) error {
	_, err := os.Stat(dir)
	if err != nil {
		err = os.Mkdir(dir, 0755)
		if err != nil {
			return err
		}
	}
	return nil
}

func getCmusStatus() ([]byte, error) {
	cmd := exec.Command("cmus-remote", "-Q")
	return cmd.Output()
}

var artistRe = regexp.MustCompile(`(?m)^tag\s+artist\s+(.+)\s*$`)
var titleRe = regexp.MustCompile(`(?m)^tag\s+title\s+(.+)\s*$`)

func regexpMatch(re *regexp.Regexp, buf []byte) []byte {
	match := re.FindAllSubmatch(buf, 1)
	if len(match) > 0 {
		return match[0][1]
	}
	return nil
}

func parseCmusStatus(cmusStatus []byte) (*songInfo, error) {
	artist := regexpMatch(artistRe, cmusStatus)
	title := regexpMatch(titleRe, cmusStatus)

	if artist == nil || title == nil {
		return nil, errors.New("Failed to parse cmus status")
	}

	si := songInfo{
		artist: string(artist),
		title:  string(title),
	}

	return &si, nil
}

func makeURL(si *songInfo) string {
	artist := []byte(si.artist)
	title := []byte(si.title)

	theRe := regexp.MustCompile(`(?i)^the `)
	weirdRe := regexp.MustCompile(`(?i)[^a-z0-9]`)

	artist = theRe.ReplaceAll(artist, []byte{})

	artist = bytes.ToLower(artist)
	title = bytes.ToLower(title)

	for _, str := range []*[]byte{&artist, &title} {
		*str = bytes.ToLower(*str)
		*str = weirdRe.ReplaceAll(*str, []byte{})
	}

	url := "https://www.azlyrics.com/lyrics/"
	url += string(artist) + "/" + string(title) + ".html"

	return url
}

func fetchLyrics(client *http.Client, si *songInfo) ([]byte, error) {
	reqUrl := makeURL(si)

	req, err := http.NewRequest("GET", reqUrl, nil)
	if err != nil {
		return []byte{}, err
	}

	resp, err := client.Do(req)
	if err != nil {
		return []byte{}, err
	}

	if resp.StatusCode != 200 {
		return []byte{}, errors.New(resp.Status)
	}

	utf8, err := charset.NewReader(resp.Body, resp.Header.Get("Content-Type"))
	if err != nil {
		return []byte{}, err
	}
	body, err := ioutil.ReadAll(utf8)
	if err != nil {
		return []byte{}, err
	}

	lyrics, err := parseLyrics(body)
	if err != nil {
		return []byte{}, err
	}

	return lyrics, nil
}

func htmlStrip(html []byte) []byte {
	commentsRe := regexp.MustCompile(`(?s)<!--.*?-->`)
	brRe := regexp.MustCompile(`<br/?>`)

	html = commentsRe.ReplaceAll(html, []byte{})
	html = brRe.ReplaceAll(html, []byte{})
	html = bytes.TrimSpace(html)

	return html
}

func parseLyrics(lyricsHtml []byte) ([]byte, error) {
	re := regexp.MustCompile(
		`(?s)<div[^<>]*?class="lyricsh"[^<>]*?>.*?</div>\s*?` +
			`<div[^<>]*?>.*?</div>\s*` +
			`.*?` +
			`<div[^<>]*?>(.*?)</div>`)

	match := re.FindAllSubmatch(lyricsHtml, 1)
	if match == nil {
		return []byte{}, errors.New("Failed to parse html")
	}

	lyrics := htmlStrip(match[0][1])
	return lyrics, nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"
	"text/template"
	"time"

	"github.com/austinkelmore/catarang/job"
	"golang.org/x/net/websocket"
)

// Config All of the run time data for the Catarang server
type Config struct {
	Jobs  []job.Job
	Conns []*websocket.Conn
}

var config Config
var configFileName = "catarang_config.json"

func addJob(w http.ResponseWriter, r *http.Request) {
	job := job.NewJob(r.FormValue("name"), r.FormValue("repo"), r.FormValue("build_config"))
	config.Jobs = append(config.Jobs, job)
	saveConfig()

	renderWebpage(w, r)
}

func deleteJob(w http.ResponseWriter, r *http.Request) {
	renderWebpage(w, r)
}

func pollJobs() {
	for {
		// todo: akelmore - figure out if this is safe to poll like this if
		// we're inserting/deleting from it or if there needs to be a lock of some sort
		for index := range config.Jobs {
			if config.Jobs[index].NeedsRunning() {
				config.Jobs[index].Run()
				saveConfig()
			}
		}
		time.Sleep(time.Second * 10)
	}
}

func renderWebpage(w http.ResponseWriter, r *http.Request) {
	root, err := template.ParseFiles("web/root.html")
	if err != nil {
		log.Println("Can't parse root.html file.")
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	root.Execute(w, config)
}

// todo: akelmore - fix threading with the reading/writing of the config
func readInConfig() {
	data, err := ioutil.ReadFile(configFileName)
	if err == nil {
		if err = json.Unmarshal(data, &config); err != nil {
			log.Println("Error reading in", configFileName)
			log.Println(err.Error())
		}
		return
	}

	// create a new config and save it out
	log.Println("No Catarang config detected, creating new one.")
	saveConfig()
}

func saveConfig() {
	data, err := json.MarshalIndent(&config, "", "\t")
	if err != nil {
		log.Println("Error marshaling save data:", err.Error())
		return
	}

	err = ioutil.WriteFile(configFileName, []byte(data), 0644)
	if err != nil {
		log.Println("Error writing config file", configFileName)
		log.Println(err.Error())
	}
}

func handleConsoleText(ws *websocket.Conn) {
	config.Conns = append(config.Conns, ws)
	type inOut struct {
		err int
		out int
	}
	var sent []inOut

	for {
		if len(config.Jobs) > 0 && len(config.Jobs[0].History) > 0 {
			for index := range config.Jobs[0].History[0].Log {
				if index >= len(sent) {
					log.Printf("Index = %v\n", index)
					if index > len(sent)-1 {
						sent = append(sent, inOut{err: 0, out: 0})
					}

					logger := &config.Jobs[0].History[0].Log[index]
					splitErr := strings.Split(string(logger.Err.Bytes()), "\n")
					for i := sent[index].err; i < len(splitErr); i++ {
						log.Printf("Err - Index: %v, Num: %v, Val: %v", index, i, splitErr[i])
						websocket.Message.Send(ws, splitErr[i])
					}
					splitOut := strings.Split(string(logger.Out.Bytes()), "\n")
					for i := sent[index].out; i < len(splitOut); i++ {
						log.Printf("Out - Index: %v, Num: %v, Val: %v", index, i, splitOut[i])
						websocket.Message.Send(ws, splitOut[i])
					}

					sent[index].err = len(splitErr)
					sent[index].out = len(splitOut)
				}
			}
		}
		time.Sleep(time.Second * 2)
	}
}

func main() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)
	log.Println("Running Catarang!")
	readInConfig()

	go pollJobs()

	str, _ := os.Getwd()
	log.Println("Current working dir: " + str)

	log.Println("Web dir: " + http.Dir("./web/static/"))

	http.HandleFunc("/", renderWebpage)
	http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("web/static/"))))
	http.HandleFunc("/addjob", addJob)
	http.HandleFunc("/deletejob", deleteJob)

	http.Handle("/ws", websocket.Handler(handleConsoleText))

	log.Fatal(http.ListenAndServe(":8080", nil))
}
<commit_msg>Removing debugging info and making websockets handle errors correctly.<commit_after>package main

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
	"text/template"
	"time"

	"github.com/austinkelmore/catarang/job"
	"golang.org/x/net/websocket"
)

// Config All of the run time data for the Catarang server
type Config struct {
	Jobs  []job.Job
	Conns []*websocket.Conn
}

var config Config
var configFileName = "catarang_config.json"

func addJob(w http.ResponseWriter, r *http.Request) {
	job := job.NewJob(r.FormValue("name"), r.FormValue("repo"), r.FormValue("build_config"))
	config.Jobs = append(config.Jobs, job)
	saveConfig()

	renderWebpage(w, r)
}

func deleteJob(w http.ResponseWriter, r *http.Request) {
	renderWebpage(w, r)
}

func pollJobs() {
	for {
		// todo: akelmore - figure out if this is safe to poll like this if
		// we're inserting/deleting from it or if there needs to be a lock of some sort
		for index := range config.Jobs {
			if config.Jobs[index].NeedsRunning() {
				config.Jobs[index].Run()
				saveConfig()
			}
		}
		time.Sleep(time.Second * 10)
	}
}

func renderWebpage(w http.ResponseWriter, r *http.Request) {
	root, err := template.ParseFiles("web/root.html")
	if err != nil {
		log.Println("Can't parse root.html file.")
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	root.Execute(w, config)
}

// todo: akelmore - fix threading with the reading/writing of the config
func readInConfig() {
	data, err := ioutil.ReadFile(configFileName)
	if err == nil {
		if err = json.Unmarshal(data, &config); err != nil {
			log.Println("Error reading in", configFileName)
			log.Println(err.Error())
		}
		return
	}

	// create a new config and save it out
	log.Println("No Catarang config detected, creating new one.")
	saveConfig()
}

func saveConfig() {
	data, err := json.MarshalIndent(&config, "", "\t")
	if err != nil {
		log.Println("Error marshaling save data:", err.Error())
		return
	}

	err = ioutil.WriteFile(configFileName, []byte(data), 0644)
	if err != nil {
		log.Println("Error writing config file", configFileName)
		log.Println(err.Error())
	}
}

func handleConsoleText(ws *websocket.Conn) {
	config.Conns = append(config.Conns, ws)
	type inOut struct {
		err int
		out int
	}
	var sent []inOut

	for {
		if len(config.Jobs) > 0 && len(config.Jobs[0].History) > 0 {
			for index := range config.Jobs[0].History[0].Log {
				if index >= len(sent) {
					log.Printf("Index = %v\n", index)
					if index > len(sent)-1 {
						sent = append(sent, inOut{err: 0, out: 0})
					}

					logger := &config.Jobs[0].History[0].Log[index]
					splitErr := strings.Split(string(logger.Err.Bytes()), "\n")
					for i := sent[index].err; i < len(splitErr); i++ {
						if err := websocket.Message.Send(ws, splitErr[i]); err != nil {
							log.Printf("Error sending websocket: %s\n", err.Error())
						}

					}
					splitOut := strings.Split(string(logger.Out.Bytes()), "\n")
					for i := sent[index].out; i < len(splitOut); i++ {
						if err := websocket.Message.Send(ws, splitOut[i]); err != nil {
							log.Printf("Error sending websocket: %s\n", err.Error())
						}
					}

					sent[index].err = len(splitErr)
					sent[index].out = len(splitOut)
				}
			}
		}
		time.Sleep(time.Second * 2)
	}
}

func main() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)
	log.Println("Running Catarang!")
	readInConfig()

	go pollJobs()

	http.HandleFunc("/", renderWebpage)
	http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("web/static/"))))
	http.HandleFunc("/addjob", addJob)
	http.HandleFunc("/deletejob", deleteJob)

	http.Handle("/ws", websocket.Handler(handleConsoleText))

	log.Fatal(http.ListenAndServe(":8080", nil))
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os/exec"
	"time"

	"github.com/c9s/goprocinfo/linux"
)

type Metricer interface {
	Metric() (float32, error)
}

type MetricFunc func() (float32, error)

func (f MetricFunc) Metric() (float32, error) {
	return f()
}

var MemoryMetric Metricer = MetricFunc(func() (percent float32, err error) {
	var mi *linux.MemInfo
	if mi, err = linux.ReadMemInfo("/proc/meminfo"); err != nil {
		return
	}

	free := mi.MemFree + mi.Cached + mi.Buffers
	percent = (float32(free) / float32(mi.MemTotal)) * 100.0
	return
})

func getStats() (*linux.CPUStat, error) {
	if stat, err := linux.ReadStat("/proc/stat"); err != nil {
		return nil, err
	} else {
		return &stat.CPUStatAll, nil
	}
}

type CPUMetric struct {
	sampleTime time.Duration
}

func NewCPUMetric(sampleTime time.Duration) *CPUMetric {
	return &CPUMetric{sampleTime}
}

func (u *CPUMetric) Metric() (percent float32, err error) {
	var pstat, cstat *linux.CPUStat

	if pstat, err = getStats(); err != nil {
		return
	}

	time.Sleep(u.sampleTime)

	if cstat, err = getStats(); err != nil {
		return
	}

	// Calculate the percentage total CPU usage (adapted from
	// <http://stackoverflow.com/questions/23367857/accurate-calculation-of-cpu-usage-given-in-percentage-in-linux>).
	prevIdle := pstat.Idle + pstat.IOWait
	idle := cstat.Idle + cstat.IOWait

	prevNonIdle := pstat.User + pstat.Nice + pstat.System + pstat.IRQ + pstat.SoftIRQ + pstat.Steal
	nonIdle := cstat.User + cstat.Nice + cstat.System + cstat.IRQ + cstat.SoftIRQ + cstat.Steal

	prevTotal := prevIdle + prevNonIdle
	total := idle + nonIdle

	// Differentiate: actual values minus the previous one
	totald := total - prevTotal
	idled := idle - prevIdle

	percent = (float32(totald-idled) / float32(totald)) * 100
	return
}

type Threshold struct {
	LastValue float32  // Last usage value
	Threshold float32  // Percentage threshold
	Metric    Metricer // Source to be checked
	Err       error    // Errors encountered
}

func NewThreshold(metric Metricer, threshold float32) *Threshold {
	return &Threshold{
		Threshold: threshold,
		Metric:    metric,
	}
}

func (t *Threshold) Exceeded() bool {
	u, err := t.Metric.Metric()
	if err != nil {
		t.Err = err
		return true
	}

	t.LastValue = u
	return t.LastValue > t.Threshold
}

/*type Check struct {
	Thresholds []
}*/

func main() {
	thresholds := []*Threshold{
		NewThreshold(NewCPUMetric(1*time.Second), 75),
		NewThreshold(MemoryMetric, 60),
	}

	for i := 0; i < 20; i++ {

	Loop:
		for {
			for j, t := range thresholds {
				if t.Exceeded() {
					log.Printf("%d usage = %f%%: waiting", j, t.LastValue)
					continue Loop
				}
			}

			break Loop // No thresholds are exceeded.
		}

		log.Printf("Command %d executing", i)
		go func(index int) {
			cmd := exec.Command("sh", "-c", "timeout 10s yes > /dev/null")
			if err := cmd.Run(); err != nil {
				log.Printf("Command %d failed: %s", index, err)
			} else {
				log.Printf("Command %d succeeded", index)
			}
		}(i)
	}

	// Wait for goroutines to finish.
	select {}
}
<commit_msg>Update: replace Metrics with the concepts of samplers and thresholds.<commit_after>package main

import (
	"bufio"
	"context"
	"errors"
	"log"
	"os"
	"os/exec"
	"sync"
	"time"

	"github.com/c9s/goprocinfo/linux"
)

type SampleHandler func(metric float32, err error)

type Sampler interface {
	Sample(ctx context.Context, interval time.Duration, cb SampleHandler)
}

type SampleFunc func() (float32, error)

func (f SampleFunc) Sample(ctx context.Context, interval time.Duration, cb SampleHandler) {
	t := time.NewTicker(interval)
	defer t.Stop()

	for {
		select {
		case <-t.C:
			cb(f())
		case <-ctx.Done():
			// fmt.Println(ctx.Err()) // prints "context deadline exceeded"
			return
		}
	}

	return
}

var (
	MemorySampler Sampler = SampleFunc(func() (percent float32, err error) {
		var mi *linux.MemInfo
		if mi, err = linux.ReadMemInfo("/proc/meminfo"); err != nil {
			return
		}

		free := mi.MemFree + mi.Cached + mi.Buffers
		percent = (float32(free) / float32(mi.MemTotal)) * 100.0
		return
	})

	LoadAvg1MinSampler Sampler = SampleFunc(func() (load float32, err error) {
		var l *linux.LoadAvg
		if l, err = linux.ReadLoadAvg("/proc/loadavg"); err != nil {
			return
		}

		load = float32(l.Last1Min)
		return
	})
)

func getStats() (*linux.CPUStat, error) {
	if stat, err := linux.ReadStat("/proc/stat"); err != nil {
		return nil, err
	} else {
		return &stat.CPUStatAll, nil
	}
}

type CPUSampler struct {
	prevStat *linux.CPUStat
}

func NewCPUSampler() *CPUSampler {
	return &CPUSampler{}
}

func (s *CPUSampler) Init() (err error) {
	s.prevStat, err = getStats()
	return
}

func (s *CPUSampler) Measure() (percent float32, err error) {
	var curStat *linux.CPUStat
	curStat, err = getStats()
	if err != nil {
		s.prevStat = nil
		return
	}

	defer func() {
		s.prevStat = curStat // Update the previous value with the current one.
	}()

	if s.prevStat == nil {
		err = errors.New("no previous cpu statistics available")
		return
	}

	// Calculate the percentage total CPU usage (adapted from
	// <http://stackoverflow.com/questions/23367857/accurate-calculation-of-cpu-usage-given-in-percentage-in-linux>).
	prevIdle := s.prevStat.Idle + s.prevStat.IOWait
	idle := curStat.Idle + curStat.IOWait

	prevNonIdle := s.prevStat.User + s.prevStat.Nice + s.prevStat.System + s.prevStat.IRQ + s.prevStat.SoftIRQ + s.prevStat.Steal
	nonIdle := curStat.User + curStat.Nice + curStat.System + curStat.IRQ + curStat.SoftIRQ + curStat.Steal

	prevTotal := prevIdle + prevNonIdle
	total := idle + nonIdle

	// Differentiate: actual values minus the previous one
	totald := total - prevTotal
	idled := idle - prevIdle

	percent = (float32(totald-idled) / float32(totald)) * 100
	return
}

func (s *CPUSampler) Sample(ctx context.Context, interval time.Duration, cb SampleHandler) {
	t := time.NewTicker(interval)
	defer t.Stop()

	if err := s.Init(); err != nil {
		cb(0, err)
	}

	for {
		select {
		case <-t.C:
			cb(s.Measure())
		case <-ctx.Done():
			// fmt.Println(ctx.Err()) // prints "context deadline exceeded"
			return
		}
	}

	return
}

type Threshold struct {
	Name      string  // The identifier for this threshold.
	LastValue float32 // Last usage value
	Threshold float32 // Percentage threshold
	Sampler   Sampler // Sample source.
}

type AlertHandler func(name string, value float32, exceeded bool)

type ErrorHandler func(err error)

func NewThreshold(name string, sampler Sampler, threshold float32) *Threshold {
	return &Threshold{
		Name:      name,
		Sampler:   sampler,
		Threshold: threshold,
	}
}

func (t *Threshold) Poll(ctx context.Context, interval time.Duration, alert AlertHandler, eh ErrorHandler) {
	handler := func(metric float32, err error) {
		if err != nil {
			eh(err)
			return
		}

		// See if we need to send an alert.
		if t.LastValue < t.Threshold && metric >= t.Threshold {
			alert(t.Name, metric, true)
		} else if t.LastValue >= t.Threshold && metric < t.Threshold {
			alert(t.Name, metric, false)
		}
		t.LastValue = metric
	}

	// Set the initial value as the threshold.
	handler(t.Threshold, nil)

	go t.Sampler.Sample(ctx, interval, handler)
	return
}

type ThresholdGroup struct {
	sync.RWMutex
	Thresholds []*Threshold
	exceeded   uint8
	wait       chan struct{}
}

func NewThresholdGroup(thresholds ...*Threshold) *ThresholdGroup {
	return &ThresholdGroup{Thresholds: thresholds}
}

func (t *ThresholdGroup) updateExceeded(exceeded bool) {
	t.Lock()
	defer t.Unlock()

	if exceeded {
		t.exceeded++
		if t.exceeded == 1 {
			t.wait = make(chan struct{})
		}
	} else {
		t.exceeded--
		if t.exceeded == 0 {
			close(t.wait)
			t.wait = nil
		}
	}

	return
}

func (t *ThresholdGroup) Exceeded() bool {
	t.RLock()
	defer t.RUnlock()
	return t.exceeded != 0
}

func (t *ThresholdGroup) Wait() {
	select {
	case _, ok := <-t.wait:
		if !ok {
			return
		}
	}
}

func (t *ThresholdGroup) Poll(ctx context.Context, interval time.Duration, errh ErrorHandler) {
	alert := func(name string, value float32, exceeded bool) {
		log.Printf("%s %v %v", name, value, exceeded)
		t.updateExceeded(exceeded)
	}

	for _, ct := range t.Thresholds {
		if ct == nil {
			continue
		}

		ct.Poll(ctx, interval, alert, errh)
	}
}

func main() {
	interval := time.Second * 1
	t := NewThresholdGroup(
		NewThreshold("cpu", NewCPUSampler(), 90),
		NewThreshold("ram", MemorySampler, 90),
		NewThreshold("load", LoadAvg1MinSampler, 5),
	)

	t.Poll(context.Background(), interval, func(err error) {
		log.Println(err)
	})

	var wg sync.WaitGroup

	scanner := bufio.NewScanner(os.Stdin)
	i := 0
	for scanner.Scan() {
		i++
		wg.Add(1)
		time.Sleep(interval)
		cmdText := scanner.Text()

		if t.Exceeded() {
			log.Printf("%d: waiting", i)
			t.Wait()
		}

		log.Printf("Command %d executing: %s", i, cmdText)
		go func(index int) {
			defer wg.Done()
			cmd := exec.Command("sh", "-c", cmdText)
			if err := cmd.Run(); err != nil {
				log.Printf("Command %d failed: %s", index, err)
			} else {
				log.Printf("Command %d succeeded", index)
			}
		}(i)
	}
	if err := scanner.Err(); err != nil {
		log.Println("reading standard input:", err)
	}

	wg.Wait()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/CenturyLinkLabs/docker-reg-client/registry"
	"github.com/docker/docker/pkg/units"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"strings"
)

var (
	client      *registry.Client
	registryURL string
)

type imageNode struct {
	id       string
	size     int64
	tags     []string
	children []*imageNode
}

func init() {
	client = registry.NewClient()
}

func getRepos() *registry.SearchResults {
	log.Print("Fetching repos...")
	results, _ := client.Search.Query("", 0, 0)
	log.Printf("%v repo(s) fetched", results.NumResults)
	return results
}

func getTags(name string) registry.TagMap {
	log.Printf("Fetching tags for %s ...", name)
	tags, _ := client.Repository.ListTags(name, registry.NilAuth{})
	log.Printf("%v tags fetched for repo %s", len(tags), name)
	fqTags := make(registry.TagMap)
	for tag, id := range tags {
		fqTags[fqTag(name, tag)] = id
	}
	return fqTags
}

func getAncestry(id string) []string {
	log.Printf("Fetching ancestry for %s ...", id)
	ancestry, _ := client.Image.GetAncestry(id, registry.NilAuth{})
	log.Printf("%v ancestors fetched for tag %s", len(ancestry), id)
	return ancestry
}

func getMetadata(id string) *registry.ImageMetadata {
	log.Printf("Fetching metadata for %s ...", id)
	metadata, _ := client.Image.GetMetadata(id, registry.NilAuth{})
	log.Printf("Metadata fetched for tag %s", id)
	return metadata
}

func fqTag(name string, t string) string {
	canonicalName := strings.TrimPrefix(name, "library/")
	return canonicalName + ":" + t
}

func printTree(root *imageNode, level int, cumsize int64) {
	cumsize = cumsize + root.size
	if len(root.tags) > 0 || len(root.children) > 1 {
		fmt.Printf("%s %s%v %s\n", root.id, strings.Repeat("  ", level), root.tags, units.HumanSize(float64(cumsize)))
		level = level + 1
		cumsize = 0
	}
	for _, child := range root.children {
		printTree(child, level, cumsize)
	}
}

func main() {
	var (
		remaining   int                                  // how many more responses are we waiting from the goroutine?
		tagsCh      = make(chan registry.TagMap)         // tags fetcher/consumer channel
		tagsByImage = make(map[string][]string)          // image ids grouped by tags
		ancestryCh  = make(chan []string)                // ancestries fetcher/consumer channel
		images      = make(map[string]*imageNode)        // already processed nodes as we are building up the trees
		metadataCh  = make(chan *registry.ImageMetadata) // metadata fetcher/consumer channel
		roots       []*imageNode                         // roots as we are building up the threes
	)
	if len(registryURL) == 0 {
		registryURL = os.Getenv("REGISTRY_URL")
	}
	if len(registryURL) == 0 {
		log.Fatal("No registry URL provided, use the environment variable REGISTRY_URL to set it")
	}
	if len(os.Getenv("REGISTREE_DEBUG")) > 0 {
		log.SetOutput(os.Stderr)
	} else {
		log.SetOutput(ioutil.Discard)
	}
	client.BaseURL, _ = url.Parse(registryURL + "/v1/")
	// get tags in parallel
	for _, repo := range getRepos().Results {
		remaining = remaining + 1
		go func(name string) { tagsCh <- getTags(name) }(repo.Name)
	}
	// group them as they are fetched
	for remaining != 0 {
		for tag, id := range <-tagsCh {
			tags, _ := tagsByImage[id]
			tagsByImage[id] = append(tags, tag)
		}
		remaining = remaining - 1
	}
	// get ancestries in parallel
	for imageId := range tagsByImage {
		go func(id string) { ancestryCh <- getAncestry(id) }(imageId)
	}
	// process them as they arrive until all tagged images have been used
	for len(tagsByImage) != 0 {
		var (
			ancestry     = <-ancestryCh
			previousNode *imageNode
		)
		for _, id := range ancestry {
			if node, ok := images[id]; ok {
				// we already went up the hierarchy from there, just append a new child
				if previousNode != nil {
					node.children = append(node.children, previousNode)
				}
				previousNode = nil
				break
			}
			// retrieve layer metadata async
			remaining = remaining + 1
			go func(id string) { metadataCh <- getMetadata(id) }(id)
			// register the node in the tree
			node := &imageNode{id: id}
			if tags, ok := tagsByImage[id]; ok {
				node.tags = tags
				// don't wait for that image's ancestry, we already are going up that one
				delete(tagsByImage, id)
			}
			if previousNode != nil {
				// this is not a leaf in the tree, so attach its child
				node.children = []*imageNode{previousNode}
			}
			images[id] = node
			previousNode = node
		}
		if previousNode != nil {
			// the previous loop didn't break out, so the last node considered is a root
			roots = append(roots, previousNode)
		}
	}
	// store metadata about all images as they get back
	for remaining != 0 {
		metadata := <-metadataCh
		images[metadata.ID].size = metadata.Size
		remaining = remaining - 1
	}
	// dump all the trees
	for _, root := range roots {
		printTree(root, 0, 0)
	}

}
<commit_msg>limit number of concurrent requests to the registry<commit_after>package main

import (
	"fmt"
	"github.com/CenturyLinkLabs/docker-reg-client/registry"
	"github.com/docker/docker/pkg/units"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"strings"
)

var (
	client      *registry.Client
	registryURL string
)

type imageNode struct {
	id       string
	size     int64
	tags     []string
	children []*imageNode
}

func init() {
	client = registry.NewClient()
}

func getRepos() *registry.SearchResults {
	log.Print("Fetching repos...")
	results, _ := client.Search.Query("", 0, 0)
	log.Printf("%v repo(s) fetched", results.NumResults)
	return results
}

func getTags(name string) registry.TagMap {
	log.Printf("Fetching tags for %s ...", name)
	tags, _ := client.Repository.ListTags(name, registry.NilAuth{})
	log.Printf("%v tags fetched for repo %s", len(tags), name)
	fqTags := make(registry.TagMap)
	for tag, id := range tags {
		fqTags[fqTag(name, tag)] = id
	}
	return fqTags
}

func getAncestry(id string) []string {
	log.Printf("Fetching ancestry for %s ...", id)
	ancestry, _ := client.Image.GetAncestry(id, registry.NilAuth{})
	log.Printf("%v ancestors fetched for tag %s", len(ancestry), id)
	return ancestry
}

func getMetadata(id string) *registry.ImageMetadata {
	log.Printf("Fetching metadata for %s ...", id)
	metadata, _ := client.Image.GetMetadata(id, registry.NilAuth{})
	log.Printf("Metadata fetched for tag %s", id)
	return metadata
}

func fqTag(name string, t string) string {
	canonicalName := strings.TrimPrefix(name, "library/")
	return canonicalName + ":" + t
}

func printTree(root *imageNode, level int, cumsize int64) {
	cumsize = cumsize + root.size
	if len(root.tags) > 0 || len(root.children) > 1 {
		fmt.Printf("%s %s%v %s\n", root.id, strings.Repeat("  ", level), root.tags, units.HumanSize(float64(cumsize)))
		level = level + 1
		cumsize = 0
	}
	for _, child := range root.children {
		printTree(child, level, cumsize)
	}
}

func main() {
	var (
		remaining   int                                  // how many more responses are we waiting from the goroutine?
		throttleCh  = make(chan struct{}, 10)            // helper to limit concurrency
		tagsCh      = make(chan registry.TagMap)         // tags fetcher/consumer channel
		tagsByImage = make(map[string][]string)          // image ids grouped by tags
		ancestryCh  = make(chan []string)                // ancestries fetcher/consumer channel
		images      = make(map[string]*imageNode)        // already processed nodes as we are building up the trees
		metadataCh  = make(chan *registry.ImageMetadata) // metadata fetcher/consumer channel
		roots       []*imageNode                         // roots as we are building up the threes
	)
	if len(registryURL) == 0 {
		registryURL = os.Getenv("REGISTRY_URL")
	}
	if len(registryURL) == 0 {
		log.Fatal("No registry URL provided, use the environment variable REGISTRY_URL to set it")
	}
	if len(os.Getenv("REGISTREE_DEBUG")) > 0 {
		log.SetOutput(os.Stderr)
	} else {
		log.SetOutput(ioutil.Discard)
	}
	client.BaseURL, _ = url.Parse(registryURL + "/v1/")
	// get tags in parallel
	for _, repo := range getRepos().Results {
		remaining = remaining + 1
		go func(name string) {
			throttleCh <- struct{}{}
			tagsCh <- getTags(name)
			<-throttleCh
		}(repo.Name)
	}
	// group them as they are fetched
	for remaining != 0 {
		for tag, id := range <-tagsCh {
			tags, _ := tagsByImage[id]
			tagsByImage[id] = append(tags, tag)
		}
		remaining = remaining - 1
	}
	// get ancestries in parallel
	for imageId := range tagsByImage {
		go func(id string) {
			throttleCh <- struct{}{}
			ancestryCh <- getAncestry(id)
			<-throttleCh
		}(imageId)
	}
	// process them as they arrive until all tagged images have been used
	for len(tagsByImage) != 0 {
		var (
			ancestry     = <-ancestryCh
			previousNode *imageNode
		)
		for _, id := range ancestry {
			if node, ok := images[id]; ok {
				// we already went up the hierarchy from there, just append a new child
				if previousNode != nil {
					node.children = append(node.children, previousNode)
				}
				previousNode = nil
				break
			}
			// retrieve layer metadata async
			remaining = remaining + 1
			go func(id string) {
				throttleCh <- struct{}{}
				metadataCh <- getMetadata(id)
				<-throttleCh
			}(id)
			// register the node in the tree
			node := &imageNode{id: id}
			if tags, ok := tagsByImage[id]; ok {
				node.tags = tags
				// don't wait for that image's ancestry, we already are going up that one
				delete(tagsByImage, id)
			}
			if previousNode != nil {
				// this is not a leaf in the tree, so attach its child
				node.children = []*imageNode{previousNode}
			}
			images[id] = node
			previousNode = node
		}
		if previousNode != nil {
			// the previous loop didn't break out, so the last node considered is a root
			roots = append(roots, previousNode)
		}
	}
	// store metadata about all images as they get back
	for remaining != 0 {
		metadata := <-metadataCh
		images[metadata.ID].size = metadata.Size
		remaining = remaining - 1
	}
	// dump all the trees
	for _, root := range roots {
		printTree(root, 0, 0)
	}

}
<|endoftext|>


<commit_before>package main

import "fmt"
import "io/ioutil"
import "io"
import "log"
import "net/url"
import "os"
import "os/exec"
import "path/filepath"
import "strings"

func cleanWcRoot(wcPath string) (err error) {
	infos, err := ioutil.ReadDir(wcPath)
	if nil != err {
		return
	}
	for _, inf := range infos {
		if ".svn" == inf.Name() {
			continue
		}
		fullPath := filepath.Join(wcPath, inf.Name())
		err = os.RemoveAll(fullPath)
		if nil != err {
			return
		}
	}
	return nil
}

func execPiped(name string, arg ...string) error {
	fmt.Println(name + " " + strings.Join(arg, " "))
	cmd := exec.Command(name, arg...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin
	return cmd.Run()
}

func copyFile(src, dst string) (err error) {
	s, err := os.Open(src)
	if err != nil {
		return
	}
	defer func() {
		closeErr := s.Close()
		if nil == err {
			err = closeErr
		}
	}()
	d, err := os.Create(dst)
	if err != nil {
		return
	}
	_, err = io.Copy(d, s)
	if nil != err {
		d.Close()
		return
	}
	return d.Close()
}

func copyRecursive(srcDir, dstDir string) (err error) {
	err = os.MkdirAll(dstDir, perm)
	if nil != err {
		return
	}
	infs, err := ioutil.ReadDir(srcDir)
	if nil != err {
		return
	}
	for _, inf := range infs {
		src := filepath.Join(srcDir, inf.Name())
		dst := filepath.Join(dstDir, inf.Name())
		if inf.IsDir() {
			err = copyRecursive(src, dst)
			if nil != err {
				return
			}
			continue
		}
		err = copyFile(src, dst)
		if nil != err {
			return
		}
	}
	return nil
}

func execPrint(name string, arg ...string) ([]byte, error) {
	fmt.Println(name + " " + strings.Join(arg, " "))
	return exec.Command(name, arg...).Output()
}

func svnDiffCommit(srcPath string, wcPath string, repos *url.URL) (err error) {
	err = execPiped("svn", "checkout", repos.String(), wcPath)
	if nil != err {
		return
	}
	err = cleanWcRoot(wcPath)
	if nil != err {
		return
	}
	err = copyRecursive(srcPath, wcPath)
	if nil != err {
		return
	}
	err = execPiped("svn", "add", wcPath, "--force")
	if nil != err {
		return
	}
	out, err := execPrint("svn", "status", wcPath)
	if err != nil {
		return
	}
	statusOut := string(out)
	fmt.Println(statusOut)
	// svn remove all missing files
	// svn commit
	return nil
}

func createRepos(reposPath string) (repos *url.URL, err error) {
	err = execPiped("svnadmin", "create", reposPath)
	if nil != err {
		return
	}
	absReposPath, err := filepath.Abs(reposPath)
	if nil != err {
		return
	}
	absReposPath = "file://" + absReposPath
	repos, err = url.Parse(absReposPath)
	return
}

type testData struct {
	Path    string
	IsDir   bool
	Content string
}

func makeTestData() []testData {
	result := []testData{
		{"1.txt", false, "data1"},
		{"2.txt", false, "data2"},
		{"subdir1", true, ""},
		{filepath.Join("subdir1", "1.txt"), false, "subdata1"},
		{"subdir2", true, ""}}
	return result
}

const perm = 0755

func createTestSourceFiles(basePath string) (err error) {
	err = os.Mkdir(basePath, perm)
	if nil != err {
		return
	}
	testDatas := makeTestData()
	for _, td := range testDatas {
		path := filepath.Join(basePath, td.Path)
		if td.IsDir {
			err = os.Mkdir(path, perm)
			if nil != err {
				return
			}
			continue
		}
		err = ioutil.WriteFile(path, []byte(td.Content), perm)
		if nil != err {
			return
		}
	}
	return nil
}

func setupTest(testPath string) (repos *url.URL, srcPath string, err error) {
	err = os.Mkdir(testPath, perm)
	if nil != err {
		return
	}
	srcPath = filepath.Join(testPath, "src")
	err = createTestSourceFiles(srcPath)
	if nil != err {
		return
	}
	reposPath := filepath.Join(testPath, "repos")
	repos, err = createRepos(reposPath)
	return
}

func teardownTest(testPath string) {
	err := os.RemoveAll(testPath)
	if nil != err {
		log.Println("ERROR: ", err)
	}
}

func runSelfTest() (err error) {
	fmt.Print("\n\nSelf test --> Start...\n\n\n")
	testPath := filepath.Join(".", "self_test")
	reposUrl, srcPath, err := setupTest(testPath)
	if nil != err {
		return
	}
	defer teardownTest(testPath)
	wcPath := filepath.Join(testPath, "wc")
	err = svnDiffCommit(srcPath, wcPath, reposUrl)
	if nil != err {
		return
	}
	fmt.Print("\n\nSelf test --> Success.\n\n\n")
	return nil
}

func main() {
	err := runSelfTest()
	if nil != err {
		log.Fatal(err)
	}
}
<commit_msg>Work in progress.<commit_after>package main

import "fmt"
import "io/ioutil"
import "io"
import "log"
import "net/url"
import "os"
import "os/exec"
import "path/filepath"
import "strings"

func cleanWcRoot(wcPath string) (err error) {
	infos, err := ioutil.ReadDir(wcPath)
	if nil != err {
		return
	}
	for _, inf := range infos {
		if ".svn" == inf.Name() {
			continue
		}
		fullPath := filepath.Join(wcPath, inf.Name())
		err = os.RemoveAll(fullPath)
		if nil != err {
			return
		}
	}
	return nil
}

func execPiped(name string, arg ...string) error {
	fmt.Println(name + " " + strings.Join(arg, " "))
	cmd := exec.Command(name, arg...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin
	return cmd.Run()
}

func copyFile(src, dst string) (err error) {
	s, err := os.Open(src)
	if err != nil {
		return
	}
	defer func() {
		closeErr := s.Close()
		if nil == err {
			err = closeErr
		}
	}()
	d, err := os.Create(dst)
	if err != nil {
		return
	}
	_, err = io.Copy(d, s)
	if nil != err {
		d.Close()
		return
	}
	return d.Close()
}

func copyRecursive(srcDir, dstDir string) (err error) {
	err = os.MkdirAll(dstDir, perm)
	if nil != err {
		return
	}
	infs, err := ioutil.ReadDir(srcDir)
	if nil != err {
		return
	}
	for _, inf := range infs {
		src := filepath.Join(srcDir, inf.Name())
		dst := filepath.Join(dstDir, inf.Name())
		if inf.IsDir() {
			err = copyRecursive(src, dst)
			if nil != err {
				return
			}
			continue
		}
		err = copyFile(src, dst)
		if nil != err {
			return
		}
	}
	return nil
}

func execPrint(name string, arg ...string) ([]byte, error) {
	fmt.Println(name + " " + strings.Join(arg, " "))
	return exec.Command(name, arg...).Output()
}

type svnOptions struct {
	Username *string // --username ARG
	Password *string // --password ARG
	NoAuthCache bool // --no-auth-cache
	//REQUIRED NonInteractive bool // --non-ineractive
	TrustServerCertFailures *string // --trust-server-cert-failures ARG 'unknown-ca' 'cn-mismatch' 'expired' 'not-yet-valid' 'other'
	ConfigDir *string // config-dir ARG
	ConfigOption *string // --config-options ARG
	CommitMessage string // --message ARG
}

func makeGlobalArgs(opts svnOptions) []string {
	return []string{}
}

func svnDiffCommit(
		srcPath string,
		wcPath string,
		repos *url.URL,
		opts svnOptions) (err error) {
	err = execPiped("svn", "checkout", repos.String(), wcPath)
	if nil != err {
		return
	}
	err = cleanWcRoot(wcPath)
	if nil != err {
		return
	}
	err = copyRecursive(srcPath, wcPath)
	if nil != err {
		return
	}
	err = execPiped("svn", "add", wcPath, "--force")
	if nil != err {
		return
	}
	out, err := execPrint("svn", "status", wcPath)
	if nil != err {
		return
	}
	statusLines := strings.Split(string(out), "\n")
	for _, line := range statusLines {
		// svn remove all missing files
		fmt.Println(line) // TODO: avoid printing an extra line
	}
	commitArgs := []string{"commit", wcPath}
	commitArgs = append(commitArgs, "--message", "hej hej :D")
	return execPiped("svn", commitArgs...)
}

func createRepos(reposPath string) (repos *url.URL, err error) {
	err = execPiped("svnadmin", "create", reposPath)
	if nil != err {
		return
	}
	absReposPath, err := filepath.Abs(reposPath)
	if nil != err {
		return
	}
	absReposPath = "file://" + absReposPath
	repos, err = url.Parse(absReposPath)
	return
}

type testData struct {
	Path    string
	IsDir   bool
	Content string
}

func makeTestData() []testData {
	result := []testData{
		{"1.txt", false, "data1"},
		{"2.txt", false, "data2"},
		{"subdir1", true, ""},
		{filepath.Join("subdir1", "1.txt"), false, "subdata1"},
		{"subdir2", true, ""}}
	return result
}

const perm = 0755

func createTestSourceFiles(basePath string) (err error) {
	err = os.Mkdir(basePath, perm)
	if nil != err {
		return
	}
	testDatas := makeTestData()
	for _, td := range testDatas {
		path := filepath.Join(basePath, td.Path)
		if td.IsDir {
			err = os.Mkdir(path, perm)
			if nil != err {
				return
			}
			continue
		}
		err = ioutil.WriteFile(path, []byte(td.Content), perm)
		if nil != err {
			return
		}
	}
	return nil
}

func setupTest(testPath string) (repos *url.URL, srcPath string, err error) {
	err = os.Mkdir(testPath, perm)
	if nil != err {
		return
	}
	srcPath = filepath.Join(testPath, "src")
	err = createTestSourceFiles(srcPath)
	if nil != err {
		return
	}
	reposPath := filepath.Join(testPath, "repos")
	repos, err = createRepos(reposPath)
	return
}

func teardownTest(testPath string) {
	err := os.RemoveAll(testPath)
	if nil != err {
		log.Println("ERROR: ", err)
	}
}

func runSelfTest() (err error) {
	fmt.Print("\n\nSelf test --> Start...\n\n\n")
	testPath := filepath.Join(".", "self_test")
	reposUrl, srcPath, err := setupTest(testPath)
	if nil != err {
		return
	}
	defer teardownTest(testPath)
	wcPath := filepath.Join(testPath, "wc")
	var opts svnOptions
	err = svnDiffCommit(srcPath, wcPath, reposUrl, opts)
	if nil != err {
		return
	}
	fmt.Print("\n\nSelf test --> Success.\n\n\n")
	return nil
}

func main() {
	err := runSelfTest()
	if nil != err {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"log"
	"os"
	"strings"
	"unicode/utf8"
)

const (
	delimiter = ','
	fn        = "things.csv"
)

func main() {
	var columnNum int
	var columnCounts = make(map[int]int)
	var lines []string
	f, err := os.Open(fn)
	if err != nil {
		log.Fatalln(err)
	}
	defer f.Close()
	out, err := os.Create("out" + fn)
	if err != nil {
		log.Fatalln(err)
	}
	defer out.Close()
	scanner := bufio.NewScanner(f)

	for scanner.Scan() {
		temp := 0
		columnNum = 0
		line := scanner.Text()
		for i, v := range line {
			temp += utf8.RuneLen(v)
			if v != delimiter && i < len(line)-1 {
				continue
			}
			if temp > columnCounts[columnNum] {
				columnCounts[columnNum] = temp
			}
			columnNum++
			temp = 0
		}
		lines = append(lines, line)
	}

	w := bufio.NewWriter(out)
	// w := bufio.NewWriter(os.Stdout)
	for _, line := range lines {
		words := strings.Split(line, string(delimiter))
		columnNum = 0
		for _, word := range words {
			for len(word) < columnCounts[columnNum] {
				word += " "
			}
			rCount, wordLen := utf8.RuneCountInString(word), len(word)
			if rCount < wordLen {
				for i := 0; i < wordLen-rCount; i++ {
					word += " "
					// if i == 10 {
					// 	break
					// }
				}
			}
			columnNum++
			if _, ok := columnCounts[columnNum]; ok {
				w.WriteString(word + string(delimiter))
				continue
			}
			w.WriteString(word)
		}
		w.WriteByte('\n')
	}
	w.Flush()
}
<commit_msg>remove comment<commit_after>package main

import (
	"bufio"
	"log"
	"os"
	"strings"
	"unicode/utf8"
)

const (
	delimiter = ','
	fn        = "things.csv"
)

func main() {
	var columnNum int
	var columnCounts = make(map[int]int)
	var lines []string
	f, err := os.Open(fn)
	if err != nil {
		log.Fatalln(err)
	}
	defer f.Close()
	out, err := os.Create("out" + fn)
	if err != nil {
		log.Fatalln(err)
	}
	defer out.Close()
	scanner := bufio.NewScanner(f)

	for scanner.Scan() {
		temp := 0
		columnNum = 0
		line := scanner.Text()
		for i, v := range line {
			temp += utf8.RuneLen(v)
			if v != delimiter && i < len(line)-1 {
				continue
			}
			if temp > columnCounts[columnNum] {
				columnCounts[columnNum] = temp
			}
			columnNum++
			temp = 0
		}
		lines = append(lines, line)
	}

	w := bufio.NewWriter(out)
	// w := bufio.NewWriter(os.Stdout)
	for _, line := range lines {
		words := strings.Split(line, string(delimiter))
		columnNum = 0
		for _, word := range words {
			for len(word) < columnCounts[columnNum] {
				word += " "
			}
			rCount, wordLen := utf8.RuneCountInString(word), len(word)
			if rCount < wordLen {
				for i := 0; i < wordLen-rCount; i++ {
					word += " "
				}
			}
			columnNum++
			// since columnNum was just incremented, do not add a comma to the last field
			if _, ok := columnCounts[columnNum]; ok {
				w.WriteString(word + string(delimiter))
				continue
			}
			w.WriteString(word)
		}
		w.WriteByte('\n')
	}
	w.Flush()
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"github.com/goraft/raft"
	"github.com/rcrowley/go-metrics"
	"github.com/rcrowley/go-metrics/stathat"
	"log"
	"net"
	"os"
	"strings"
	"time"
)

var (
	join, ldns, lhttp, dataDir, domain string
	rtimeout, wtimeout                 time.Duration
	discover                           bool
	metricsToStdErr                    bool
	graphiteServer, stathatUser        string
)

func init() {
	flag.StringVar(&join, "join", "", "Member of SkyDNS cluster to join can be comma separated list")
	flag.BoolVar(&discover, "discover", false, "Auto discover SkyDNS cluster. Performs an NS lookup on the -domain to find SkyDNS members")
	flag.StringVar(&domain, "domain", "skydns.local", "Domain to anchor requests to")
	flag.StringVar(&ldns, "dns", "127.0.0.1:53", "IP:Port to bind to for DNS")
	flag.StringVar(&lhttp, "http", "127.0.0.1:8080", "IP:Port to bind to for HTTP")
	flag.StringVar(&dataDir, "data", "./data", "SkyDNS data directory")
	flag.DurationVar(&rtimeout, "rtimeout", 2*time.Second, "Read timeout")
	flag.DurationVar(&wtimeout, "wtimeout", 2*time.Second, "Write timeout")
	flag.BoolVar(&metricsToStdErr, "metricsToStdErr", false, "Write metrics to stderr periodically")
	flag.StringVar(&graphiteServer, "graphiteServer", "", "Graphite Server connection string e.g. 127.0.0.1:2003")
	flag.StringVar(&stathatUser, "stathatUser", "", "StatHat account for metrics")
}

func main() {
	members := make([]string, 0)

	raft.SetLogLevel(0)

	flag.Parse()

	if discover {
		ns, err := net.LookupNS(domain)

		if err != nil {
			log.Fatal(err)
			return
		}

		if len(ns) < 1 {
			log.Fatal("No NS records found for ", domain)
			return
		}

		for _, n := range ns {
			members = append(members, n.Host)
		}
	} else if join != "" {
		members = strings.Split(join, ",")
	}

	s := NewServer(members, domain, ldns, lhttp, dataDir, rtimeout, wtimeout)

	// Set up metrics if specified on the command line
	if metricsToStdErr {
		go metrics.Log(metrics.DefaultRegistry, 60e9, log.New(os.Stderr, "metrics: ", log.Lmicroseconds))
	}

	if len(graphiteServer) > 1 {
		addr, err := net.ResolveTCPAddr("tcp", graphiteServer)
		if err != nil {
			go metrics.Graphite(metrics.DefaultRegistry, 10e9, "skydns", addr)
		}
	}

	if len(stathatUser) > 1 {
		go stathat.Stathat(metrics.DefaultRegistry, 10e9, stathatUser)
	}

	waiter := s.Start()
	waiter.Wait()
}
<commit_msg>fix bug with -discover. Domains returned from NS lookup will have a trailing .<commit_after>package main

import (
	"flag"
	"github.com/goraft/raft"
	"github.com/rcrowley/go-metrics"
	"github.com/rcrowley/go-metrics/stathat"
	"log"
	"net"
	"os"
	"strings"
	"time"
)

var (
	join, ldns, lhttp, dataDir, domain string
	rtimeout, wtimeout                 time.Duration
	discover                           bool
	metricsToStdErr                    bool
	graphiteServer, stathatUser        string
)

func init() {
	flag.StringVar(&join, "join", "", "Member of SkyDNS cluster to join can be comma separated list")
	flag.BoolVar(&discover, "discover", false, "Auto discover SkyDNS cluster. Performs an NS lookup on the -domain to find SkyDNS members")
	flag.StringVar(&domain, "domain", "skydns.local", "Domain to anchor requests to")
	flag.StringVar(&ldns, "dns", "127.0.0.1:53", "IP:Port to bind to for DNS")
	flag.StringVar(&lhttp, "http", "127.0.0.1:8080", "IP:Port to bind to for HTTP")
	flag.StringVar(&dataDir, "data", "./data", "SkyDNS data directory")
	flag.DurationVar(&rtimeout, "rtimeout", 2*time.Second, "Read timeout")
	flag.DurationVar(&wtimeout, "wtimeout", 2*time.Second, "Write timeout")
	flag.BoolVar(&metricsToStdErr, "metricsToStdErr", false, "Write metrics to stderr periodically")
	flag.StringVar(&graphiteServer, "graphiteServer", "", "Graphite Server connection string e.g. 127.0.0.1:2003")
	flag.StringVar(&stathatUser, "stathatUser", "", "StatHat account for metrics")
}

func main() {
	members := make([]string, 0)

	raft.SetLogLevel(0)

	flag.Parse()

	if discover {
		ns, err := net.LookupNS(domain)

		if err != nil {
			log.Fatal(err)
			return
		}

		if len(ns) < 1 {
			log.Fatal("No NS records found for ", domain)
			return
		}

		for _, n := range ns {
			members = append(members, strings.TrimPrefix(n.Host, "."))
		}
	} else if join != "" {
		members = strings.Split(join, ",")
	}

	s := NewServer(members, domain, ldns, lhttp, dataDir, rtimeout, wtimeout)

	// Set up metrics if specified on the command line
	if metricsToStdErr {
		go metrics.Log(metrics.DefaultRegistry, 60e9, log.New(os.Stderr, "metrics: ", log.Lmicroseconds))
	}

	if len(graphiteServer) > 1 {
		addr, err := net.ResolveTCPAddr("tcp", graphiteServer)
		if err != nil {
			go metrics.Graphite(metrics.DefaultRegistry, 10e9, "skydns", addr)
		}
	}

	if len(stathatUser) > 1 {
		go stathat.Stathat(metrics.DefaultRegistry, 10e9, stathatUser)
	}

	waiter := s.Start()
	waiter.Wait()
}
<|endoftext|>