<commit_before>package main

import (
	"errors"
	"io/ioutil"
	"os"

	"github.com/mithrandie/csvq/lib/action"
	"github.com/mithrandie/csvq/lib/cmd"
	"github.com/mithrandie/csvq/lib/query"

	"github.com/urfave/cli"
)

var version = "v0.6.3"

func main() {
	cli.AppHelpTemplate = appHHelpTemplate
	cli.CommandHelpTemplate = commandHelpTemplate

	app := cli.NewApp()

	app.Name = "csvq"
	app.Usage = "SQL like query language for csv"
	app.ArgsUsage = "[\"query\"|\"statements\"|argument]"
	app.Version = version

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "delimiter, d",
			Usage: "field delimiter. Default is \",\" for csv files, \"\\t\" for tsv files.",
		},
		cli.StringFlag{
			Name:  "encoding, e",
			Value: "UTF8",
			Usage: "file encoding. one of: UTF8|SJIS",
		},
		cli.StringFlag{
			Name:  "line-break, l",
			Value: "LF",
			Usage: "line break. one of: CRLF|LF|CR",
		},
		cli.StringFlag{
			Name:  "timezone, z",
			Value: "Local",
			Usage: "default timezone. \"Local\", \"UTC\" or a timezone name(e.g. \"America/Los_Angeles\")",
		},
		cli.StringFlag{
			Name:  "repository, r",
			Usage: "directory path where files are located",
		},
		cli.StringFlag{
			Name:  "source, s",
			Usage: "load query from `FILE`",
		},
		cli.StringFlag{
			Name:  "datetime-format, t",
			Usage: "set datetime format to parse strings",
		},
		cli.BoolFlag{
			Name:  "no-header, n",
			Usage: "import the first line as a record",
		},
		cli.BoolFlag{
			Name:  "without-null, a",
			Usage: "parse empty fields as empty strings",
		},
		cli.StringFlag{
			Name:  "write-encoding, E",
			Value: "UTF8",
			Usage: "file encoding. one of: UTF8|SJIS",
		},
		cli.StringFlag{
			Name:  "out, o",
			Usage: "write output to `FILE`",
		},
		cli.StringFlag{
			Name:  "format, f",
			Usage: "output format. one of: CSV|TSV|JSON|TEXT",
		},
		cli.StringFlag{
			Name:  "write-delimiter, D",
			Usage: "field delimiter for CSV",
		},
		cli.BoolFlag{
			Name:  "without-header, N",
			Usage: "when the file format is specified as CSV or TSV, write without the header line",
		},
		cli.IntFlag{
			Name:  "cpu, p",
			Value: 1,
			Usage: "number of cpu cores to be used. 1 - number of cpu cores",
		},
		cli.BoolFlag{
			Name:  "stats, x",
			Usage: "show execution time and memory statistics",
		},
	}

	app.Commands = []cli.Command{
		{
			Name:      "fields",
			Usage:     "Show fields in a file",
			ArgsUsage: "CSV_FILE_PATH",
			Action: func(c *cli.Context) error {
				if c.NArg() != 1 {
					cli.ShowCommandHelp(c, "fields")
					return cli.NewExitError("table is not specified", 1)
				}

				table := c.Args().First()

				err := action.ShowFields(table)
				if err != nil {
					return cli.NewExitError(err.Error(), 1)
				}

				return nil
			},
		},
		{
			Name:      "calc",
			Usage:     "Calculate a value from stdin",
			ArgsUsage: "\"expression\"",
			Action: func(c *cli.Context) error {
				if c.NArg() != 1 {
					cli.ShowCommandHelp(c, "calc")
					return cli.NewExitError("expression is empty", 1)
				}

				expr := c.Args().First()
				err := action.Calc(expr)
				if err != nil {
					return cli.NewExitError(err.Error(), 1)
				}

				return nil
			},
		},
	}

	app.Before = func(c *cli.Context) error {
		return setFlags(c)
	}

	app.Action = func(c *cli.Context) error {
		queryString, err := readQuery(c)
		if err != nil {
			cli.ShowAppHelp(c)
			return cli.NewExitError(err.Error(), 1)
		}

		if err = action.Run(queryString, cmd.GetFlags().Source); err != nil {
			code := 1
			if apperr, ok := err.(query.AppError); ok {
				code = apperr.GetCode()
			}
			return cli.NewExitError(err.Error(), code)
		}

		return nil
	}

	app.Run(os.Args)
}

func readQuery(c *cli.Context) (string, error) {
	var queryString string

	flags := cmd.GetFlags()
	if 0 < len(flags.Source) {
		fp, err := os.Open(flags.Source)
		if err != nil {
			return queryString, err
		}
		defer fp.Close()

		buf, err := ioutil.ReadAll(fp)
		if err != nil {
			return queryString, err
		}
		queryString = string(buf)

	} else {
		if c.NArg() != 1 {
			return queryString, errors.New("query is empty")
		}
		queryString = c.Args().First()
	}

	return queryString, nil
}

func setFlags(c *cli.Context) error {
	if err := cmd.SetDelimiter(c.GlobalString("delimiter")); err != nil {
		return err
	}
	if err := cmd.SetEncoding(c.GlobalString("encoding")); err != nil {
		return err
	}
	if err := cmd.SetLineBreak(c.String("line-break")); err != nil {
		return err
	}
	if err := cmd.SetLocation(c.String("timezone")); err != nil {
		return err
	}
	if err := cmd.SetRepository(c.GlobalString("repository")); err != nil {
		return err
	}
	if err := cmd.SetSource(c.GlobalString("source")); err != nil {
		return err
	}
	cmd.SetDatetimeFormat(c.GlobalString("datetime-format"))
	cmd.SetNoHeader(c.GlobalBool("no-header"))
	cmd.SetWithoutNull(c.GlobalBool("without-null"))

	if err := cmd.SetWriteEncoding(c.GlobalString("write-encoding")); err != nil {
		return err
	}
	if err := cmd.SetOut(c.GlobalString("out")); err != nil {
		return err
	}
	if err := cmd.SetFormat(c.GlobalString("format")); err != nil {
		return err
	}
	if err := cmd.SetWriteDelimiter(c.GlobalString("write-delimiter")); err != nil {
		return err
	}
	cmd.SetWithoutHeader(c.GlobalBool("without-header"))

	cmd.SetCPU(c.GlobalInt("cpu"))
	cmd.SetStats(c.GlobalBool("stats"))

	return nil
}
<commit_msg>Update version for Release v0.6.4<commit_after>package main

import (
	"errors"
	"io/ioutil"
	"os"

	"github.com/mithrandie/csvq/lib/action"
	"github.com/mithrandie/csvq/lib/cmd"
	"github.com/mithrandie/csvq/lib/query"

	"github.com/urfave/cli"
)

var version = "v0.6.4"

func main() {
	cli.AppHelpTemplate = appHHelpTemplate
	cli.CommandHelpTemplate = commandHelpTemplate

	app := cli.NewApp()

	app.Name = "csvq"
	app.Usage = "SQL like query language for csv"
	app.ArgsUsage = "[\"query\"|\"statements\"|argument]"
	app.Version = version

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "delimiter, d",
			Usage: "field delimiter. Default is \",\" for csv files, \"\\t\" for tsv files.",
		},
		cli.StringFlag{
			Name:  "encoding, e",
			Value: "UTF8",
			Usage: "file encoding. one of: UTF8|SJIS",
		},
		cli.StringFlag{
			Name:  "line-break, l",
			Value: "LF",
			Usage: "line break. one of: CRLF|LF|CR",
		},
		cli.StringFlag{
			Name:  "timezone, z",
			Value: "Local",
			Usage: "default timezone. \"Local\", \"UTC\" or a timezone name(e.g. \"America/Los_Angeles\")",
		},
		cli.StringFlag{
			Name:  "repository, r",
			Usage: "directory path where files are located",
		},
		cli.StringFlag{
			Name:  "source, s",
			Usage: "load query from `FILE`",
		},
		cli.StringFlag{
			Name:  "datetime-format, t",
			Usage: "set datetime format to parse strings",
		},
		cli.BoolFlag{
			Name:  "no-header, n",
			Usage: "import the first line as a record",
		},
		cli.BoolFlag{
			Name:  "without-null, a",
			Usage: "parse empty fields as empty strings",
		},
		cli.StringFlag{
			Name:  "write-encoding, E",
			Value: "UTF8",
			Usage: "file encoding. one of: UTF8|SJIS",
		},
		cli.StringFlag{
			Name:  "out, o",
			Usage: "write output to `FILE`",
		},
		cli.StringFlag{
			Name:  "format, f",
			Usage: "output format. one of: CSV|TSV|JSON|TEXT",
		},
		cli.StringFlag{
			Name:  "write-delimiter, D",
			Usage: "field delimiter for CSV",
		},
		cli.BoolFlag{
			Name:  "without-header, N",
			Usage: "when the file format is specified as CSV or TSV, write without the header line",
		},
		cli.IntFlag{
			Name:  "cpu, p",
			Value: 1,
			Usage: "number of cpu cores to be used. 1 - number of cpu cores",
		},
		cli.BoolFlag{
			Name:  "stats, x",
			Usage: "show execution time and memory statistics",
		},
	}

	app.Commands = []cli.Command{
		{
			Name:      "fields",
			Usage:     "Show fields in a file",
			ArgsUsage: "CSV_FILE_PATH",
			Action: func(c *cli.Context) error {
				if c.NArg() != 1 {
					cli.ShowCommandHelp(c, "fields")
					return cli.NewExitError("table is not specified", 1)
				}

				table := c.Args().First()

				err := action.ShowFields(table)
				if err != nil {
					return cli.NewExitError(err.Error(), 1)
				}

				return nil
			},
		},
		{
			Name:      "calc",
			Usage:     "Calculate a value from stdin",
			ArgsUsage: "\"expression\"",
			Action: func(c *cli.Context) error {
				if c.NArg() != 1 {
					cli.ShowCommandHelp(c, "calc")
					return cli.NewExitError("expression is empty", 1)
				}

				expr := c.Args().First()
				err := action.Calc(expr)
				if err != nil {
					return cli.NewExitError(err.Error(), 1)
				}

				return nil
			},
		},
	}

	app.Before = func(c *cli.Context) error {
		return setFlags(c)
	}

	app.Action = func(c *cli.Context) error {
		queryString, err := readQuery(c)
		if err != nil {
			cli.ShowAppHelp(c)
			return cli.NewExitError(err.Error(), 1)
		}

		if err = action.Run(queryString, cmd.GetFlags().Source); err != nil {
			code := 1
			if apperr, ok := err.(query.AppError); ok {
				code = apperr.GetCode()
			}
			return cli.NewExitError(err.Error(), code)
		}

		return nil
	}

	app.Run(os.Args)
}

func readQuery(c *cli.Context) (string, error) {
	var queryString string

	flags := cmd.GetFlags()
	if 0 < len(flags.Source) {
		fp, err := os.Open(flags.Source)
		if err != nil {
			return queryString, err
		}
		defer fp.Close()

		buf, err := ioutil.ReadAll(fp)
		if err != nil {
			return queryString, err
		}
		queryString = string(buf)

	} else {
		if c.NArg() != 1 {
			return queryString, errors.New("query is empty")
		}
		queryString = c.Args().First()
	}

	return queryString, nil
}

func setFlags(c *cli.Context) error {
	if err := cmd.SetDelimiter(c.GlobalString("delimiter")); err != nil {
		return err
	}
	if err := cmd.SetEncoding(c.GlobalString("encoding")); err != nil {
		return err
	}
	if err := cmd.SetLineBreak(c.String("line-break")); err != nil {
		return err
	}
	if err := cmd.SetLocation(c.String("timezone")); err != nil {
		return err
	}
	if err := cmd.SetRepository(c.GlobalString("repository")); err != nil {
		return err
	}
	if err := cmd.SetSource(c.GlobalString("source")); err != nil {
		return err
	}
	cmd.SetDatetimeFormat(c.GlobalString("datetime-format"))
	cmd.SetNoHeader(c.GlobalBool("no-header"))
	cmd.SetWithoutNull(c.GlobalBool("without-null"))

	if err := cmd.SetWriteEncoding(c.GlobalString("write-encoding")); err != nil {
		return err
	}
	if err := cmd.SetOut(c.GlobalString("out")); err != nil {
		return err
	}
	if err := cmd.SetFormat(c.GlobalString("format")); err != nil {
		return err
	}
	if err := cmd.SetWriteDelimiter(c.GlobalString("write-delimiter")); err != nil {
		return err
	}
	cmd.SetWithoutHeader(c.GlobalBool("without-header"))

	cmd.SetCPU(c.GlobalInt("cpu"))
	cmd.SetStats(c.GlobalBool("stats"))

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/httputil"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
)

var (
	GithubToken       string
	GithubUser        string
	GithubRepo        string
	GithubAPIEndpoint string
	DEBUG             bool
)

type Release struct {
	UploadURL  string `json:"upload_url,omitempty"`
	TagName    string `json:"tag_name"`
	Branch     string `json:"target_commitish"`
	Name       string `json:"name"`
	Body       string `json:"body"`
	Draft      bool   `json:"draft"`
	Prerelease bool   `json:"prerelease"`
}

func init() {
	log.SetFlags(0)

	DEBUG, _ = strconv.ParseBool(os.Getenv("DEBUG"))

	GithubToken = os.Getenv("GITHUB_TOKEN")
	GithubUser = os.Getenv("GITHUB_USER")
	GithubRepo = os.Getenv("GITHUB_REPO")
	GithubAPIEndpoint = os.Getenv("GITHUB_API")
}

var usage string = `Github release tool.
Usage:
	gh-release <user/repo> <tag> <branch> <description> <files>

<files> can be specified using glob patterns.
`

func main() {
	if len(os.Args) < 6 {
		log.Fatal(usage)
	}

	userRepo := strings.Split(os.Args[1], "/")
	if len(userRepo) != 2 {
		log.Printf("Error: Invalid format used for username and repository: %s\n\n", os.Args[1])
		log.Fatal(usage)
	}

	if GithubToken == "" {
		log.Fatal(`Error: GITHUB_TOKEN environment variable is not set.
Please refer to https://help.github.com/articles/creating-an-access-token-for-command-line-use/ for more help\n`)
	}

	GithubUser = userRepo[0]
	GithubRepo = userRepo[1]
	GithubAPIEndpoint = fmt.Sprintf("https://api.github.com/repos/%s/%s", GithubUser, GithubRepo)

	filepaths, err := filepath.Glob(os.Args[5])
	if err != nil {
		log.Fatalf("Error: Invalid glob pattern: %s\n", os.Args[3])
	}

	tag := os.Args[2]
	branch := os.Args[3]
	desc := os.Args[4]

	CreateRelease(tag, branch, desc, filepaths)
	log.Println("Done")
}

func uploadFile(uploadURL, path string) {
	file, err := os.Open(path)
	if err != nil {
		log.Printf("Error: %s\n", err.Error())
		return
	}
	defer file.Close()

	size, err := fileSize(file)
	if err != nil {
		log.Printf("Error: %s\n", err.Error())
		return
	}

	body, err := doRequest("POST", uploadURL+"?name="+file.Name(), "application/octet-stream", file, size)
	if err != nil {
		log.Printf("Error: %s\n", err.Error())
	}

	if DEBUG {
		log.Println("========= UPLOAD RESPONSE ===========")
		log.Println(string(body[:]))
	}
}

// Creates a Github Release, attaching the given files as release assets
// If a release already exist, up in Github, this function will attempt to attach the given files to it
func CreateRelease(tag, branch, desc string, filepaths []string) {
	endpoint := fmt.Sprintf("%s/releases", GithubAPIEndpoint)

	release := Release{
		TagName:    tag,
		Name:       tag,
		Prerelease: false,
		Draft:      false,
		Branch:     branch,
		Body:       desc,
	}

	releaseData, err := json.Marshal(release)
	if err != nil {
		log.Fatalln(err)
	}

	releaseBuffer := bytes.NewBuffer(releaseData)

	data, err := doRequest("POST", endpoint, "application/json", releaseBuffer, int64(releaseBuffer.Len()))
	if err != nil {
		log.Fatalln(err)
	}

	// Gets the release Upload URL from the returned JSON data
	err = json.Unmarshal(data, &release)
	if err != nil {
		log.Fatalln(err)
	}

	// Upload URL comes like this https://uploads.github.com/repos/octocat/Hello-World/releases/1/assets{?name}
	// So we need to remove the {?name} part
	uploadURL := strings.Split(release.UploadURL, "{")[0]

	var wg sync.WaitGroup
	for i := range filepaths {
		wg.Add(1)
		func(index int) {
			uploadFile(uploadURL, filepaths[index])
			wg.Done()
		}(i)
	}
	wg.Wait()
}

func fileSize(file *os.File) (int64, error) {
	stat, err := file.Stat()
	if err != nil {
		return 0, err
	}
	return stat.Size(), nil
}

// Sends HTTP request to Github API
func doRequest(method, url, contentType string, reqBody io.Reader, bodySize int64) ([]byte, error) {
	req, err := http.NewRequest(method, url, reqBody)
	if err != nil {
		return nil, err
	}

	req.Header.Set("Authorization", fmt.Sprintf("token %s", GithubToken))
	req.Header.Set("Content-type", contentType)
	req.Header.Set("Accept", "application/vnd.github.v3+json")
	req.ContentLength = bodySize

	if DEBUG {
		log.Println("================ REQUEST DUMP ==================")
		dump, err := httputil.DumpRequestOut(req, true)
		if err != nil {
			log.Println(err.Error())
		}
		log.Println(string(dump[:]))
	}

	resp, err := http.DefaultClient.Do(req)

	if DEBUG {
		log.Println("================ RESPONSE DUMP ==================")
		dump, err := httputil.DumpResponse(resp, true)
		if err != nil {
			log.Println(err.Error())
		}
		log.Println(string(dump[:]))
	}

	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()

	respBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated {
		return nil, fmt.Errorf("Github returned an error:\n Code: %s. \n Body: %s", resp.Status, respBody)
	}

	return respBody, nil
}
<commit_msg>Adds version flag<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/httputil"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
)

var (
	GithubToken       string
	GithubUser        string
	GithubRepo        string
	GithubAPIEndpoint string
	Version           string
	Debug             bool
)

type Release struct {
	UploadURL  string `json:"upload_url,omitempty"`
	TagName    string `json:"tag_name"`
	Branch     string `json:"target_commitish"`
	Name       string `json:"name"`
	Body       string `json:"body"`
	Draft      bool   `json:"draft"`
	Prerelease bool   `json:"prerelease"`
}

var verFlag bool

func init() {
	log.SetFlags(0)

	Debug, _ = strconv.ParseBool(os.Getenv("DEBUG"))

	GithubToken = os.Getenv("GITHUB_TOKEN")
	GithubUser = os.Getenv("GITHUB_USER")
	GithubRepo = os.Getenv("GITHUB_REPO")
	GithubAPIEndpoint = os.Getenv("GITHUB_API")

	flag.BoolVar(&verFlag, "version", false, "-version")
	flag.Parse()
}

var usage string = `Github command line release tool.
Usage:
	gh-release <user/repo> <tag> <branch> <description> <files>

	<files> can be specified using glob patterns.

Options:
	-version: Displays version
`

func main() {
	if verFlag {
		log.Println(Version)
		return
	}

	if len(os.Args) < 6 {
		log.Fatal(usage)
	}

	userRepo := strings.Split(os.Args[1], "/")
	if len(userRepo) != 2 {
		log.Printf("Error: Invalid format used for username and repository: %s\n\n", os.Args[1])
		log.Fatal(usage)
	}

	if GithubToken == "" {
		log.Fatal(`Error: GITHUB_TOKEN environment variable is not set.
Please refer to https://help.github.com/articles/creating-an-access-token-for-command-line-use/ for more help\n`)
	}

	GithubUser = userRepo[0]
	GithubRepo = userRepo[1]
	GithubAPIEndpoint = fmt.Sprintf("https://api.github.com/repos/%s/%s", GithubUser, GithubRepo)

	if Debug {
		log.Println("Glob pattern received: ")
		log.Println(os.Args[5])
	}

	filepaths, err := filepath.Glob(os.Args[5])
	if err != nil {
		log.Fatalf("Error: Invalid glob pattern: %s\n", os.Args[5])
	}

	if Debug {
		log.Println("Expanded glob pattern: ")
		log.Printf("%v\n", filepaths)
	}

	tag := os.Args[2]
	branch := os.Args[3]
	desc := os.Args[4]

	CreateRelease(tag, branch, desc, filepaths)
	log.Println("Done")
}

func uploadFile(uploadURL, path string) {
	file, err := os.Open(path)
	if err != nil {
		log.Printf("Error: %s\n", err.Error())
		return
	}
	defer file.Close()

	size, err := fileSize(file)
	if err != nil {
		log.Printf("Error: %s\n", err.Error())
		return
	}

	body, err := doRequest("POST", uploadURL+"?name="+file.Name(), "application/octet-stream", file, size)
	if err != nil {
		log.Printf("Error: %s\n", err.Error())
	}

	if Debug {
		log.Println("========= UPLOAD RESPONSE ===========")
		log.Println(string(body[:]))
	}
}

// Creates a Github Release, attaching the given files as release assets
// If a release already exist, up in Github, this function will attempt to attach the given files to it
func CreateRelease(tag, branch, desc string, filepaths []string) {
	endpoint := fmt.Sprintf("%s/releases", GithubAPIEndpoint)

	release := Release{
		TagName:    tag,
		Name:       tag,
		Prerelease: false,
		Draft:      false,
		Branch:     branch,
		Body:       desc,
	}

	releaseData, err := json.Marshal(release)
	if err != nil {
		log.Fatalln(err)
	}

	releaseBuffer := bytes.NewBuffer(releaseData)

	data, err := doRequest("POST", endpoint, "application/json", releaseBuffer, int64(releaseBuffer.Len()))
	if err != nil {
		log.Fatalln(err)
	}

	// Gets the release Upload URL from the returned JSON data
	err = json.Unmarshal(data, &release)
	if err != nil {
		log.Fatalln(err)
	}

	// Upload URL comes like this https://uploads.github.com/repos/octocat/Hello-World/releases/1/assets{?name}
	// So we need to remove the {?name} part
	uploadURL := strings.Split(release.UploadURL, "{")[0]

	var wg sync.WaitGroup
	for i := range filepaths {
		wg.Add(1)
		func(index int) {
			uploadFile(uploadURL, filepaths[index])
			wg.Done()
		}(i)
	}
	wg.Wait()
}

func fileSize(file *os.File) (int64, error) {
	stat, err := file.Stat()
	if err != nil {
		return 0, err
	}
	return stat.Size(), nil
}

// Sends HTTP request to Github API
func doRequest(method, url, contentType string, reqBody io.Reader, bodySize int64) ([]byte, error) {
	req, err := http.NewRequest(method, url, reqBody)
	if err != nil {
		return nil, err
	}

	req.Header.Set("Authorization", fmt.Sprintf("token %s", GithubToken))
	req.Header.Set("Content-type", contentType)
	req.Header.Set("Accept", "application/vnd.github.v3+json")
	req.ContentLength = bodySize

	if Debug {
		log.Println("================ REQUEST DUMP ==================")
		dump, err := httputil.DumpRequestOut(req, true)
		if err != nil {
			log.Println(err.Error())
		}
		log.Println(string(dump[:]))
	}

	resp, err := http.DefaultClient.Do(req)

	if Debug {
		log.Println("================ RESPONSE DUMP ==================")
		dump, err := httputil.DumpResponse(resp, true)
		if err != nil {
			log.Println(err.Error())
		}
		log.Println(string(dump[:]))
	}

	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()

	respBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated {
		return nil, fmt.Errorf("Github returned an error:\n Code: %s. \n Body: %s", resp.Status, respBody)
	}

	return respBody, nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"github.com/fujiwara/go-zabbix-get/zabbix"
)

var (
	Version = "0.0.2"
)

func main() {
	var (
		port int
		server string
		key string
		timeout int
		showVersion bool
	)
	flag.IntVar(&port, "p", 10050, "port")
	flag.StringVar(&server, "s", "127.0.0.1", "hostname or IP")
	flag.StringVar(&key, "k", "", "key")
	flag.IntVar(&timeout, "t", 30, "timeout")
	flag.BoolVar(&showVersion, "V", false, "show Version")
	flag.Parse()

	if showVersion {
		fmt.Printf("go-zabbix-get version %s (revision %s)\n", Version, Revision)
		os.Exit(255)
	}

	if key == "" {
		flag.PrintDefaults()
		os.Exit(255)
	}

	address := fmt.Sprintf("%s:%d", server, port)
	value, err := zabbix.Get(address, key, timeout)
	if err != nil {
		log.Fatalf("error: %s", err)
	}
	fmt.Printf("%s\n", value)
}
<commit_msg>add -f (format) option. output as sensu plugin format.<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/fujiwara/go-zabbix-get/zabbix"
	"log"
	"os"
	"time"
)

var (
	Version = "0.0.2"
)

func main() {
	var (
		port        int
		server      string
		key         string
		timeout     int
		showVersion bool
		format      string
	)
	flag.IntVar(&port, "p", 10050, "port")
	flag.StringVar(&server, "s", "127.0.0.1", "hostname or IP")
	flag.StringVar(&key, "k", "", "key")
	flag.IntVar(&timeout, "t", 30, "timeout")
	flag.BoolVar(&showVersion, "V", false, "show Version")
	flag.StringVar(&format, "f", "zabbix", "output format (zabbix or sensu)")
	flag.Parse()

	if showVersion {
		fmt.Printf("go-zabbix-get version %s (revision %s)\n", Version, Revision)
		os.Exit(255)
	}

	if key == "" {
		flag.PrintDefaults()
		os.Exit(255)
	}

	address := fmt.Sprintf("%s:%d", server, port)
	value, err := zabbix.Get(address, key, timeout)
	if err != nil {
		log.Fatalf("error: %s", err)
	}

	switch format {
	case "sensu":
		printFormatSensu(key, value)
	default:
		printFormatZabbix(value)
	}
}

func printFormatZabbix(value []byte) {
	fmt.Printf("%s\n", value)
}

func printFormatSensu(key string, value []byte) {
	fmt.Printf("%s\t%s\t%d\n", key, value, time.Now().Unix())
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/hashicorp/consul/api"
	"gopkg.in/alecthomas/kingpin.v2"
	"math/rand"
	"net/url"
	"os"
	"os/exec"
	"syscall"
)

var (
	app = kingpin.New("consult", "Query Consul catalog for service")
)

type appOpts struct {
	dcs            []string
	allDCs         bool
	JsonFormat     bool
	DetailedOutput bool
	serverURL      *url.URL
	ConsulConfigs  []*api.Config
}

type Command struct {
	opts *appOpts
}

type sshCommand struct {
	QueryCommand
	user string
}

func main() {
	app.Version("0.0.2")
	opts := &appOpts{}

	app.Flag("dc", "Consul datacenter").StringsVar(&opts.dcs)
	app.Flag("all-dcs", "Query all datacenters").BoolVar(&opts.allDCs)
	app.Flag("server", "Consul URL; can also be provided using the CONSUL_URL environment variable").Default("http://127.0.0.1:8500/").Envar("CONSUL_URL").URLVar(&opts.serverURL)
	app.Flag("json", "JSON query output").Short('j').BoolVar(&opts.JsonFormat)
	app.Flag("detailed", "Detailed output (ignored if --json given)").Short('d').BoolVar(&opts.DetailedOutput)
	app.HelpFlag.Short('h')

	listRegisterCli(app, opts)
	httpRegisterCli(app, opts)
	sshRegisterCli(app, opts)
	queryRegisterCli(app, opts)
	kingpin.MustParse(app.Parse(os.Args[1:]))
}

func (q *QueryCommand) registerCli(cmd *kingpin.CmdClause) {
	cmd.Flag("tag", "Consul tag").Short('t').StringsVar(&q.tags)
	cmd.Flag("service", "Consul service").Required().Short('s').StringVar(&q.service)
	cmd.Flag("tags-mode", "Find nodes with *all* or *any* of the tags").Short('m').Default("all").EnumVar(&q.tagsMerge, "all", "any")
}

func sshRegisterCli(app *kingpin.Application, opts *appOpts) {
	s := &sshCommand{}
	s.IQuery = s
	s.opts = opts
	sshCmd := app.Command("ssh", "ssh into server using Consul query").Action(s.run)
	sshCmd.Flag("username", "ssh user name").Short('u').StringVar(&s.user)
	s.registerCli(sshCmd)
}

func (s *sshCommand) run(c *kingpin.ParseContext) error {
	results, err := s.queryServicesGeneric()
	if err != nil {
		return err
	}
	ssh(selectRandomSvc(results).Node, s.user)
	return nil
}

func printJsonResults(results []*api.CatalogService) {
	if b, err := json.MarshalIndent(results, "", "    "); err != nil {
		kingpin.Fatalf("Failed to convert results to json, %s\n", err.Error())
	} else {
		fmt.Println(string(b))
	}
}

func selectRandomSvc(services []*api.CatalogService) *api.CatalogService {
	return services[rand.Intn(len(services))]
}

func ssh(address string, user string) {
	bin, err := exec.LookPath("ssh")
	if err != nil {
		kingpin.Fatalf("Failed to find ssh binary: %s\n", err.Error())
	}

	ssh_args := make([]string, 2, 3)
	ssh_args[0] = "ssh"
	ssh_args[1] = address
	if user != "" {
		ssh_args = append(ssh_args, "-l "+user)
	}

	syscall.Exec(bin, ssh_args, os.Environ())
}

func getCurrentDC(c *api.Client) (string, error) {
	if config, err := c.Agent().Self(); err != nil {
		return "", err
	} else {
		return config["Config"]["Datacenter"].(string), nil
	}
}

func (o *Command) GetConsulClient() (*api.Client, error) {
	config := api.DefaultConfig()
	config.Address = o.opts.serverURL.Host
	config.Scheme = o.opts.serverURL.Scheme
	return api.NewClient(config)
}

func (o *Command) GetConsulClients() (map[string]*api.Client, error) {
	clients := make(map[string]*api.Client, len(o.opts.dcs))

	if o.opts.allDCs {
		l := &listCommand{Command: *o}
		if dcs, err := l.listDCs(); err != nil {
			return nil, err
		} else {
			o.opts.dcs = dcs
		}
	}

	if len(o.opts.dcs) == 0 {
		if client, err := o.GetConsulClient(); err != nil {
			return nil, err
		} else if dc, err := getCurrentDC(client); err != nil {
			return nil, err
		} else {
			clients[dc] = client
			return clients, nil
		}
	}

	for _, dc := range o.opts.dcs {
		config := api.DefaultConfig()
		config.Address = o.opts.serverURL.Host
		config.Scheme = o.opts.serverURL.Scheme
		config.Datacenter = dc

		if client, err := api.NewClient(config); err != nil {
			return nil, err
		} else {
			clients[dc] = client
		}
	}
	return clients, nil
}

func (o *Command) Output(data interface{}, simpleLong []string, simpleShort []string) {
	if o.opts.JsonFormat {
		if b, err := json.MarshalIndent(data, "", "    "); err != nil {
			kingpin.Fatalf("Failed to convert results to json, %s\n", err.Error())
		} else {
			fmt.Println(string(b))
		}
	} else if o.opts.DetailedOutput {
		for _, line := range simpleLong {
			fmt.Println(line)
		}
	} else {
		for _, line := range simpleShort {
			fmt.Println(line)
		}
	}
}
<commit_msg>move dc selection logic to function<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/hashicorp/consul/api"
	"gopkg.in/alecthomas/kingpin.v2"
	"math/rand"
	"net/url"
	"os"
	"os/exec"
	"syscall"
)

var (
	app = kingpin.New("consult", "Query Consul catalog for service")
)

type appOpts struct {
	dcs            []string
	allDCs         bool
	JsonFormat     bool
	DetailedOutput bool
	serverURL      *url.URL
	ConsulConfigs  []*api.Config
}

type Command struct {
	opts *appOpts
}

type sshCommand struct {
	QueryCommand
	user string
}

func main() {
	app.Version("0.0.2")
	opts := &appOpts{}

	app.Flag("dc", "Consul datacenter").StringsVar(&opts.dcs)
	app.Flag("all-dcs", "Query all datacenters").BoolVar(&opts.allDCs)
	app.Flag("server", "Consul URL; can also be provided using the CONSUL_URL environment variable").Default("http://127.0.0.1:8500/").Envar("CONSUL_URL").URLVar(&opts.serverURL)
	app.Flag("json", "JSON query output").Short('j').BoolVar(&opts.JsonFormat)
	app.Flag("detailed", "Detailed output (ignored if --json given)").Short('d').BoolVar(&opts.DetailedOutput)
	app.HelpFlag.Short('h')

	listRegisterCli(app, opts)
	httpRegisterCli(app, opts)
	sshRegisterCli(app, opts)
	queryRegisterCli(app, opts)
	kingpin.MustParse(app.Parse(os.Args[1:]))
}

func (q *QueryCommand) registerCli(cmd *kingpin.CmdClause) {
	cmd.Flag("tag", "Consul tag").Short('t').StringsVar(&q.tags)
	cmd.Flag("service", "Consul service").Required().Short('s').StringVar(&q.service)
	cmd.Flag("tags-mode", "Find nodes with *all* or *any* of the tags").Short('m').Default("all").EnumVar(&q.tagsMerge, "all", "any")
}

func sshRegisterCli(app *kingpin.Application, opts *appOpts) {
	s := &sshCommand{}
	s.IQuery = s
	s.opts = opts
	sshCmd := app.Command("ssh", "ssh into server using Consul query").Action(s.run)
	sshCmd.Flag("username", "ssh user name").Short('u').StringVar(&s.user)
	s.registerCli(sshCmd)
}

func (s *sshCommand) run(c *kingpin.ParseContext) error {
	results, err := s.queryServicesGeneric()
	if err != nil {
		return err
	}
	ssh(selectRandomSvc(results).Node, s.user)
	return nil
}

func printJsonResults(results []*api.CatalogService) {
	if b, err := json.MarshalIndent(results, "", "    "); err != nil {
		kingpin.Fatalf("Failed to convert results to json, %s\n", err.Error())
	} else {
		fmt.Println(string(b))
	}
}

func selectRandomSvc(services []*api.CatalogService) *api.CatalogService {
	return services[rand.Intn(len(services))]
}

func ssh(address string, user string) {
	bin, err := exec.LookPath("ssh")
	if err != nil {
		kingpin.Fatalf("Failed to find ssh binary: %s\n", err.Error())
	}

	ssh_args := make([]string, 2, 3)
	ssh_args[0] = "ssh"
	ssh_args[1] = address
	if user != "" {
		ssh_args = append(ssh_args, "-l "+user)
	}

	syscall.Exec(bin, ssh_args, os.Environ())
}

func getCurrentDC(c *api.Client) (string, error) {
	if config, err := c.Agent().Self(); err != nil {
		return "", err
	} else {
		return config["Config"]["Datacenter"].(string), nil
	}
}

func (o *Command) GetConsulClient() (*api.Client, error) {
	config := api.DefaultConfig()
	config.Address = o.opts.serverURL.Host
	config.Scheme = o.opts.serverURL.Scheme
	return api.NewClient(config)
}

func (o *Command) GetDCs() ([]string, error) {
	if o.opts.allDCs {
		l := &listCommand{Command: *o}
		if dcs, err := l.listDCs(); err != nil {
			return nil, err
		} else {
			return dcs, nil
		}
	} else if len(o.opts.dcs) == 0 {
		if client, err := o.GetConsulClient(); err != nil {
			return nil, err
		} else if dc, err := getCurrentDC(client); err != nil {
			return nil, err
		} else {
			return []string{dc}, nil
		}
	} else {
		return o.opts.dcs, nil
	}
}

func (o *Command) GetConsulClients() (map[string]*api.Client, error) {
	clients := make(map[string]*api.Client, len(o.opts.dcs))
	if dcs, err := o.GetDCs(); err != nil {
		return nil, err
	} else {
		for _, dc := range dcs {
			config := api.DefaultConfig()
			config.Address = o.opts.serverURL.Host
			config.Scheme = o.opts.serverURL.Scheme
			config.Datacenter = dc

			if client, err := api.NewClient(config); err != nil {
				return nil, err
			} else {
				clients[dc] = client
			}
		}
		return clients, nil
	}
}

func (o *Command) Output(data interface{}, simpleLong []string, simpleShort []string) {
	if o.opts.JsonFormat {
		if b, err := json.MarshalIndent(data, "", "    "); err != nil {
			kingpin.Fatalf("Failed to convert results to json, %s\n", err.Error())
		} else {
			fmt.Println(string(b))
		}
	} else if o.opts.DetailedOutput {
		for _, line := range simpleLong {
			fmt.Println(line)
		}
	} else {
		for _, line := range simpleShort {
			fmt.Println(line)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"encoding/csv"
	"flag"
	"fmt"
	"io/ioutil"
	"regexp"

	_ "github.com/mattn/go-sqlite3"

	"bytes"
	"encoding/hex"
	"io"
	"log"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"strconv"
	"strings"
	"time"
	"unicode/utf8"
)

func main() {
	// Parse command line opts
	commands := flag.String("sql", "", "SQL Command(s) to run on the data")
	source_text := flag.String("source", "stdin", "Source file to load, or defaults to stdin")
	delimiter := flag.String("dlm", ",", "Delimiter between fields -dlm=tab for tab, -dlm=0x## to specify a character code in hex")
	header := flag.Bool("header", false, "Treat file as having the first row as a header row")
	outputHeader := flag.Bool("output-header", false, "Display column names in output")
	tableName := flag.String("table-name", "tbl", "Override the default table name (tbl)")
	save_to := flag.String("save-to", "", "If set, sqlite3 db is left on disk at this path")
	console := flag.Bool("console", false, "After all commands are run, open sqlite3 console with this data")
	verbose := flag.Bool("verbose", false, "Enable verbose logging")
	flag.Parse()

	if *console && (*source_text == "stdin") {
		log.Fatalln("Can not open console with pipe input, read a file instead")
	}

	separator := determineSeparator(delimiter)

	// Open db, in memory if possible
	db, openPath := openDB(save_to, console)

	// Open the input source
	var fp *os.File
	fp = openFileOrStdin(source_text)
	defer fp.Close()

	// Init a structured text reader
	reader := csv.NewReader(fp)
	reader.FieldsPerRecord = 0
	reader.Comma = separator

	// Read the first row
	first_row, read_err := reader.Read()

	if read_err != nil {
		log.Fatalln(read_err)
	}

	var headerRow []string

	if *header {
		headerRow = first_row
		first_row, read_err = reader.Read()

		if read_err != nil {
			log.Fatalln(read_err)
		}
	} else {
		headerRow = make([]string, len(first_row))
		for i := 0; i < len(first_row); i++ {
			headerRow[i] = "c" + strconv.Itoa(i)
		}
	}

	// Create the table to load to
	createTable(tableName, &headerRow, db, verbose)

	// Start the clock for importing
	t0 := time.Now()

	// Create transaction
	tx, tx_err := db.Begin()

	if tx_err != nil {
		log.Fatalln(tx_err)
	}

	// Load first row
	stmt := createLoadStmt(tableName, &headerRow, tx)
	loadRow(tableName, &first_row, tx, stmt, verbose)

	// Read the data
	for {
		row, file_err := reader.Read()
		if file_err == io.EOF {
			break
		} else if file_err != nil {
			log.Println(file_err)
		} else {
			loadRow(tableName, &row, tx, stmt, verbose)
		}
	}
	stmt.Close()
	tx.Commit()

	t1 := time.Now()

	if *verbose {
		fmt.Fprintf(os.Stderr, "Data loaded in: %v\n", t1.Sub(t0))
	}

	// Determine what sql to execute
	sqls_to_execute := strings.Split(*commands, ";")

	t0 = time.Now()

	// Execute given SQL
	for _, sql_cmd := range sqls_to_execute {
		if strings.Trim(sql_cmd, " ") != "" {
			result, err := db.Query(sql_cmd)
			if err != nil {
				log.Fatalln(err)
			}
			displayResult(result, outputHeader)
		}
	}

	t1 = time.Now()

	if *verbose {
		fmt.Fprintf(os.Stderr, "Queries run in: %v\n", t1.Sub(t0))
	}

	// Open console
	if *console {
		db.Close()
		args := []string{*openPath}
		if *outputHeader {
			args = append(args, "-header")
		}
		cmd := exec.Command("sqlite3", args...)

		cmd.Stdin = os.Stdin
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		cmd_err := cmd.Run()
		if cmd.Process != nil {
			cmd.Process.Release()
		}

		if len(*save_to) == 0 {
			os.RemoveAll(filepath.Dir(*openPath))
		}

		if cmd_err != nil {
			log.Fatalln(cmd_err)
		}
	} else if len(*save_to) == 0 {
		db.Close()
		os.Remove(*openPath)
	} else {
		db.Close()
	}
}

func createTable(tableName *string, columnNames *[]string, db *sql.DB, verbose *bool) error {
	var buffer bytes.Buffer

	buffer.WriteString("CREATE TABLE IF NOT EXISTS " + (*tableName) + " (")

	for i, col := range *columnNames {
		var col_name string

		reg := regexp.MustCompile(`[^a-zA-Z0-9]`)

		col_name = reg.ReplaceAllString(col, "_")
		if *verbose && col_name != col {
			fmt.Fprintf(os.Stderr, "Column %x renamed to %s\n", col, col_name)
		}

		buffer.WriteString(col_name + " TEXT")

		if i != len(*columnNames)-1 {
			buffer.WriteString(", ")
		}
	}

	buffer.WriteString(");")

	_, err := db.Exec(buffer.String())

	if err != nil {
		log.Fatalln(err)
	}

	return err
}

func createLoadStmt(tableName *string, values *[]string, db *sql.Tx) *sql.Stmt {
	if len(*values) == 0 {
		log.Fatalln("Nothing to build insert with!")
	}
	var buffer bytes.Buffer

	buffer.WriteString("INSERT INTO " + (*tableName) + " VALUES (")
	for i := range *values {
		buffer.WriteString("?")
		if i != len(*values)-1 {
			buffer.WriteString(", ")
		}
	}
	buffer.WriteString(");")
	stmt, err := db.Prepare(buffer.String())
	if err != nil {
		log.Fatalln(err)
	}
	return stmt
}

func loadRow(tableName *string, values *[]string, db *sql.Tx, stmt *sql.Stmt, verbose *bool) error {
	if len(*values) == 0 {
		return nil
	}
	vals := make([]interface{}, 0)
	for _, val := range *values {
		vals = append(vals, val)
	}
	_, err := stmt.Exec(vals...)
	if err != nil && *verbose {
		fmt.Fprintln(os.Stderr, "Bad row: ", err)
	}
	return err
}

func displayResult(rows *sql.Rows, outputHeader *bool) {
	cols, cols_err := rows.Columns()

	if cols_err != nil {
		log.Fatalln(cols_err)
	}

	if *outputHeader {
		for i, c := range cols {
			fmt.Printf("%s", c)
			if i != len(cols)-1 {
				fmt.Printf(", ")
			}
		}
		fmt.Printf("\n")
	}

	rawResult := make([][]byte, len(cols))
	result := make([]string, len(cols))

	dest := make([]interface{}, len(cols))
	for i := range cols {
		dest[i] = &rawResult[i]
	}

	for rows.Next() {
		rows.Scan(dest...)

		for i, raw := range rawResult {
			result[i] = string(raw)
		}

		for j, v := range result {
			fmt.Printf("%s", v)
			if j != len(result)-1 {
				fmt.Printf(", ")
			}
		}
		fmt.Printf("\n")
	}
}

func openFileOrStdin(path *string) *os.File {
	var fp *os.File
	var err error
	if (*path) == "stdin" {
		fp = os.Stdin
		err = nil
	} else {
		fp, err = os.Open(*cleanPath(path))
	}

	if err != nil {
		log.Fatalln(err)
	}

	return fp
}

func cleanPath(path *string) *string {
	var result string
	usr, err := user.Current()
	if err != nil {
		log.Fatalln(err)
	}

	if (*path)[:2] == "~/" {
		dir := usr.HomeDir + "/"
		result = strings.Replace(*path, "~/", dir, 1)
	} else {
		result = (*path)
	}

	abs_result, abs_err := filepath.Abs(result)
	if abs_err != nil {
		log.Fatalln(err)
	}

	clean_result := filepath.Clean(abs_result)

	return &clean_result
}

func openDB(path *string, no_memory *bool) (*sql.DB, *string) {
	openPath := ":memory:"
	if len(*path) != 0 {
		openPath = *cleanPath(path)
	} else if *no_memory {
		outDir, err := ioutil.TempDir(os.TempDir(), "textql")
		if err != nil {
			log.Fatalln(err)
		}
		openPath = filepath.Join(outDir, "textql.db")
	}

	db, err := sql.Open("sqlite3", openPath)

	if err != nil {
		log.Fatalln(err)
	}
	return db, &openPath
}

func determineSeparator(delimiter *string) rune {
	var separator rune

	if (*delimiter) == "tab" {
		separator = '\t'
	} else if strings.Index((*delimiter), "0x") == 0 {
		dlm, hex_err := hex.DecodeString((*delimiter)[2:])

		if hex_err != nil {
			log.Fatalln(hex_err)
		}

		separator, _ = utf8.DecodeRuneInString(string(dlm))
	} else {
		separator, _ = utf8.DecodeRuneInString(*delimiter)
	}
	return separator
}
<commit_msg>Expose LazyQuotes option.<commit_after>package main

import (
	"database/sql"
	"encoding/csv"
	"flag"
	"fmt"
	"io/ioutil"
	"regexp"

	_ "github.com/mattn/go-sqlite3"

	"bytes"
	"encoding/hex"
	"io"
	"log"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"strconv"
	"strings"
	"time"
	"unicode/utf8"
)

func main() {
	// Parse command line opts
	commands := flag.String("sql", "", "SQL Command(s) to run on the data")
	source_text := flag.String("source", "stdin", "Source file to load, or defaults to stdin")
	delimiter := flag.String("dlm", ",", "Delimiter between fields -dlm=tab for tab, -dlm=0x## to specify a character code in hex")
	lazyQuotes := flag.Bool("lazy-quotes", false, "Enable LazyQuotes in the csv parser")
	header := flag.Bool("header", false, "Treat file as having the first row as a header row")
	outputHeader := flag.Bool("output-header", false, "Display column names in output")
	tableName := flag.String("table-name", "tbl", "Override the default table name (tbl)")
	save_to := flag.String("save-to", "", "If set, sqlite3 db is left on disk at this path")
	console := flag.Bool("console", false, "After all commands are run, open sqlite3 console with this data")
	verbose := flag.Bool("verbose", false, "Enable verbose logging")
	flag.Parse()

	if *console && (*source_text == "stdin") {
		log.Fatalln("Can not open console with pipe input, read a file instead")
	}

	separator := determineSeparator(delimiter)

	// Open db, in memory if possible
	db, openPath := openDB(save_to, console)

	// Open the input source
	var fp *os.File
	fp = openFileOrStdin(source_text)
	defer fp.Close()

	// Init a structured text reader
	reader := csv.NewReader(fp)
	reader.FieldsPerRecord = 0
	reader.Comma = separator
	reader.LazyQuotes = *lazyQuotes

	// Read the first row
	first_row, read_err := reader.Read()

	if read_err != nil {
		log.Fatalln(read_err)
	}

	var headerRow []string

	if *header {
		headerRow = first_row
		first_row, read_err = reader.Read()

		if read_err != nil {
			log.Fatalln(read_err)
		}
	} else {
		headerRow = make([]string, len(first_row))
		for i := 0; i < len(first_row); i++ {
			headerRow[i] = "c" + strconv.Itoa(i)
		}
	}

	// Create the table to load to
	createTable(tableName, &headerRow, db, verbose)

	// Start the clock for importing
	t0 := time.Now()

	// Create transaction
	tx, tx_err := db.Begin()

	if tx_err != nil {
		log.Fatalln(tx_err)
	}

	// Load first row
	stmt := createLoadStmt(tableName, &headerRow, tx)
	loadRow(tableName, &first_row, tx, stmt, verbose)

	// Read the data
	for {
		row, file_err := reader.Read()
		if file_err == io.EOF {
			break
		} else if file_err != nil {
			log.Println(file_err)
		} else {
			loadRow(tableName, &row, tx, stmt, verbose)
		}
	}
	stmt.Close()
	tx.Commit()

	t1 := time.Now()

	if *verbose {
		fmt.Fprintf(os.Stderr, "Data loaded in: %v\n", t1.Sub(t0))
	}

	// Determine what sql to execute
	sqls_to_execute := strings.Split(*commands, ";")

	t0 = time.Now()

	// Execute given SQL
	for _, sql_cmd := range sqls_to_execute {
		if strings.Trim(sql_cmd, " ") != "" {
			result, err := db.Query(sql_cmd)
			if err != nil {
				log.Fatalln(err)
			}
			displayResult(result, outputHeader)
		}
	}

	t1 = time.Now()

	if *verbose {
		fmt.Fprintf(os.Stderr, "Queries run in: %v\n", t1.Sub(t0))
	}

	// Open console
	if *console {
		db.Close()
		args := []string{*openPath}
		if *outputHeader {
			args = append(args, "-header")
		}
		cmd := exec.Command("sqlite3", args...)

		cmd.Stdin = os.Stdin
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		cmd_err := cmd.Run()
		if cmd.Process != nil {
			cmd.Process.Release()
		}

		if len(*save_to) == 0 {
			os.RemoveAll(filepath.Dir(*openPath))
		}

		if cmd_err != nil {
			log.Fatalln(cmd_err)
		}
	} else if len(*save_to) == 0 {
		db.Close()
		os.Remove(*openPath)
	} else {
		db.Close()
	}
}

func createTable(tableName *string, columnNames *[]string, db *sql.DB, verbose *bool) error {
	var buffer bytes.Buffer

	buffer.WriteString("CREATE TABLE IF NOT EXISTS " + (*tableName) + " (")

	for i, col := range *columnNames {
		var col_name string

		reg := regexp.MustCompile(`[^a-zA-Z0-9]`)

		col_name = reg.ReplaceAllString(col, "_")
		if *verbose && col_name != col {
			fmt.Fprintf(os.Stderr, "Column %x renamed to %s\n", col, col_name)
		}

		buffer.WriteString(col_name + " TEXT")

		if i != len(*columnNames)-1 {
			buffer.WriteString(", ")
		}
	}

	buffer.WriteString(");")

	_, err := db.Exec(buffer.String())

	if err != nil {
		log.Fatalln(err)
	}

	return err
}

func createLoadStmt(tableName *string, values *[]string, db *sql.Tx) *sql.Stmt {
	if len(*values) == 0 {
		log.Fatalln("Nothing to build insert with!")
	}
	var buffer bytes.Buffer

	buffer.WriteString("INSERT INTO " + (*tableName) + " VALUES (")
	for i := range *values {
		buffer.WriteString("?")
		if i != len(*values)-1 {
			buffer.WriteString(", ")
		}
	}
	buffer.WriteString(");")
	stmt, err := db.Prepare(buffer.String())
	if err != nil {
		log.Fatalln(err)
	}
	return stmt
}

func loadRow(tableName *string, values *[]string, db *sql.Tx, stmt *sql.Stmt, verbose *bool) error {
	if len(*values) == 0 {
		return nil
	}
	vals := make([]interface{}, 0)
	for _, val := range *values {
		vals = append(vals, val)
	}
	_, err := stmt.Exec(vals...)
	if err != nil && *verbose {
		fmt.Fprintln(os.Stderr, "Bad row: ", err)
	}
	return err
}

func displayResult(rows *sql.Rows, outputHeader *bool) {
	cols, cols_err := rows.Columns()

	if cols_err != nil {
		log.Fatalln(cols_err)
	}

	if *outputHeader {
		for i, c := range cols {
			fmt.Printf("%s", c)
			if i != len(cols)-1 {
				fmt.Printf(", ")
			}
		}
		fmt.Printf("\n")
	}

	rawResult := make([][]byte, len(cols))
	result := make([]string, len(cols))

	dest := make([]interface{}, len(cols))
	for i := range cols {
		dest[i] = &rawResult[i]
	}

	for rows.Next() {
		rows.Scan(dest...)

		for i, raw := range rawResult {
			result[i] = string(raw)
		}

		for j, v := range result {
			fmt.Printf("%s", v)
			if j != len(result)-1 {
				fmt.Printf(", ")
			}
		}
		fmt.Printf("\n")
	}
}

func openFileOrStdin(path *string) *os.File {
	var fp *os.File
	var err error
	if (*path) == "stdin" {
		fp = os.Stdin
		err = nil
	} else {
		fp, err = os.Open(*cleanPath(path))
	}

	if err != nil {
		log.Fatalln(err)
	}

	return fp
}

func cleanPath(path *string) *string {
	var result string
	usr, err := user.Current()
	if err != nil {
		log.Fatalln(err)
	}

	if (*path)[:2] == "~/" {
		dir := usr.HomeDir + "/"
		result = strings.Replace(*path, "~/", dir, 1)
	} else {
		result = (*path)
	}

	abs_result, abs_err := filepath.Abs(result)
	if abs_err != nil {
		log.Fatalln(err)
	}

	clean_result := filepath.Clean(abs_result)

	return &clean_result
}

func openDB(path *string, no_memory *bool) (*sql.DB, *string) {
	openPath := ":memory:"
	if len(*path) != 0 {
		openPath = *cleanPath(path)
	} else if *no_memory {
		outDir, err := ioutil.TempDir(os.TempDir(), "textql")
		if err != nil {
			log.Fatalln(err)
		}
		openPath = filepath.Join(outDir, "textql.db")
	}

	db, err := sql.Open("sqlite3", openPath)

	if err != nil {
		log.Fatalln(err)
	}
	return db, &openPath
}

func determineSeparator(delimiter *string) rune {
	var separator rune

	if (*delimiter) == "tab" {
		separator = '\t'
	} else if strings.Index((*delimiter), "0x") == 0 {
		dlm, hex_err := hex.DecodeString((*delimiter)[2:])

		if hex_err != nil {
			log.Fatalln(hex_err)
		}

		separator, _ = utf8.DecodeRuneInString(string(dlm))
	} else {
		separator, _ = utf8.DecodeRuneInString(*delimiter)
	}
	return separator
}
<|endoftext|>


<commit_before>// © 2014, Robert Hülle

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
)

var (
	// KindleDir is path to kindle user directory.
	KindleDir string

	// KindleCollectionFile is name of file where collections are stored.
	KindleCollectionFile string
	PreserveLastAccess   bool
)

func init() {
	flag.StringVar(&KindleDir, "dir", "/mnt/us", "where kindle is mounted")
	flag.StringVar(&KindleCollectionFile, "col", "system/collections.json", "where is collection file")
	flag.BoolVar(&PreserveLastAccess, "la", true, "preserve last access in collections.json")
}

func main() {
	flag.Parse()
	var oc map[string]Collection
	if PreserveLastAccess {
		oc = loadOldCollections(KindleDir + "/" + KindleCollectionFile)
	}
	f := FindFiles(KindleDir)
	c := BuildCollection(f, oc)
	b, err := EncodeCollections(c)
	if err != nil {
		fmt.Fprintf(os.Stderr, "can not encode collections: %v\n", err)
		return
	}
	file, err := os.Create(KindleDir + "/" + KindleCollectionFile)
	if err != nil {
		fmt.Fprintf(os.Stderr, "con not open collection file: %v\n", err)
		return
	}
	defer file.Close()
	_, err = file.Write(b)
	if err != nil {
		fmt.Fprintf(os.Stderr, "can not write to collection file: %v\n", err)
	}
}

// loadOldCollections reads collection file to preserve lastAccess.
func loadOldCollections(path string) map[string]Collection {
	data, err := ioutil.ReadFile(path)
	if os.IsNotExist(err) {
		return nil
	}
	if err != nil {
		fmt.Fprintf(os.Stderr, "can not read collections.json: %v\n", err)
		return nil
	}
	col, err := DecodeCollections(data)
	if err != nil {
		fmt.Fprintf(os.Stderr, "can not decode json file: %v\n", err)
		return nil
	}
	return col
}
<commit_msg>typo in error message<commit_after>// © 2014, Robert Hülle

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
)

var (
	// KindleDir is path to kindle user directory.
	KindleDir string

	// KindleCollectionFile is name of file where collections are stored.
	KindleCollectionFile string
	PreserveLastAccess   bool
)

func init() {
	flag.StringVar(&KindleDir, "dir", "/mnt/us", "where kindle is mounted")
	flag.StringVar(&KindleCollectionFile, "col", "system/collections.json", "where is collection file")
	flag.BoolVar(&PreserveLastAccess, "la", true, "preserve last access in collections.json")
}

func main() {
	flag.Parse()
	var oc map[string]Collection
	if PreserveLastAccess {
		oc = loadOldCollections(KindleDir + "/" + KindleCollectionFile)
	}
	f := FindFiles(KindleDir)
	c := BuildCollection(f, oc)
	b, err := EncodeCollections(c)
	if err != nil {
		fmt.Fprintf(os.Stderr, "can not encode collections: %v\n", err)
		return
	}
	file, err := os.Create(KindleDir + "/" + KindleCollectionFile)
	if err != nil {
		fmt.Fprintf(os.Stderr, "can not open collection file: %v\n", err)
		return
	}
	defer file.Close()
	_, err = file.Write(b)
	if err != nil {
		fmt.Fprintf(os.Stderr, "can not write to collection file: %v\n", err)
	}
}

// loadOldCollections reads collection file to preserve lastAccess.
func loadOldCollections(path string) map[string]Collection {
	data, err := ioutil.ReadFile(path)
	if os.IsNotExist(err) {
		return nil
	}
	if err != nil {
		fmt.Fprintf(os.Stderr, "can not read collections.json: %v\n", err)
		return nil
	}
	col, err := DecodeCollections(data)
	if err != nil {
		fmt.Fprintf(os.Stderr, "can not decode json file: %v\n", err)
		return nil
	}
	return col
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"

	"github.com/drone-plugins/drone-git-push/repo"
	"github.com/drone/drone-go/drone"
	"github.com/drone/drone-plugin-go/plugin"
)

var (
	buildCommit string
)

var netrcFile = `
machine %s
login %s
password %s
`

func main() {
	fmt.Printf("Drone GitHub Pages plugin built from %s\n", buildCommit)

	v := new(Params)
	r := new(drone.Repo)
	b := new(drone.Build)
	w := new(drone.Workspace)
	plugin.Param("repo", r)
	plugin.Param("build", b)
	plugin.Param("workspace", w)
	plugin.Param("vargs", &v)
	plugin.MustParse()
	err := publishDocs(r, b, w, v)
	if err != nil {
		fmt.Printf("%s\n", err)
		os.Exit(1)
	}
}

func publishDocs(r *drone.Repo, b *drone.Build, w *drone.Workspace, v *Params) error {
	if v.UpstreamName == "" {
		v.UpstreamName = "origin"
	}
	if v.TargetBranch == "" {
		v.TargetBranch = "gh-pages"
	}
	if v.TemporaryBase == "" {
		v.TemporaryBase = ".tmp"
	}
	if v.PagesDirectory == "" {
		v.PagesDirectory = "docs"
	}

	temporaryBaseDirectory := ""
	if filepath.IsAbs(v.TemporaryBase) {
		temporaryBaseDirectory = v.TemporaryBase
	} else {
		temporaryBaseDirectory = filepath.Join(w.Path, v.TemporaryBase)
	}
	temporaryPagesDirectory := filepath.Join(temporaryBaseDirectory, v.PagesDirectory)

	fullPagesDirectory := filepath.Join(w.Path, v.PagesDirectory)

	// generate the .netrc file
	if err := writeNetrc(w); err != nil {
		fmt.Fprintln(os.Stderr, err)
		return err
	}

	// write the rsa private key if provided
	if err := repo.WriteKey(w); err != nil {
		fmt.Fprintln(os.Stderr, err)
		return err
	}

	err := repo.GlobalUser(b).Run()
	if err != nil {
		return err
	}

	err = repo.GlobalName(b).Run()
	if err != nil {
		return err
	}

	err = os.MkdirAll(temporaryPagesDirectory, 0777)
	if err != nil {
		return err
	}
	defer os.RemoveAll(temporaryPagesDirectory)

	err = runPublishSteps(w.Path, temporaryBaseDirectory, temporaryPagesDirectory, fullPagesDirectory, v.TargetBranch, v.UpstreamName, r.Clone)
	if err != nil {
		return err
	}
	return nil
}

func runCommand(cmd *exec.Cmd, out *bytes.Buffer) error {
	if out == nil {
		if cmd.Stdout == nil {
			cmd.Stdout = os.Stdout
		}
	} else {
		cmd.Stdout = out
	}
	cmd.Stderr = os.Stderr

	err := cmd.Run()
	if err != nil {
		fmt.Printf("Error: %+v %s\n", cmd, err)
		return err
	}
	return nil
}

func runPublishSteps(
	workspacePath string,
	temporaryBaseDirectory string,
	temporaryPagesDirectory string,
	fullPagesDirectory string,
	vargsTargetBranch string,
	vargsUpstreamName string,
	cloneUrl string) error {

	// Get the commit message
	msgRaw, err := commitMessage(workspacePath).Output()
	if err != nil {
		return err
	}
	msg := fmt.Sprintf("%s", msgRaw)
	fmt.Printf("%s\n", msg)

	// Set up git config (rsa, etc)
	fmt.Printf("about to clone\n")
	err = runCommand(cloneTarget(workspacePath, vargsTargetBranch, cloneUrl, temporaryPagesDirectory), nil)
	if err != nil {
		return err
	}

	fmt.Printf("about to rsync\n")
	err = runCommand(rsyncPages(workspacePath, fullPagesDirectory, temporaryBaseDirectory), nil)
	if err != nil {
		return err
	}

	fmt.Printf("about to add to clone\n")
	var addResult bytes.Buffer
	err = runCommand(addTemporaryFilesToClone(temporaryPagesDirectory), &addResult)
	if err != nil {
		fmt.Printf("Error on add: %s", addResult.String())
		return err
	}

	// For now, if the add resulted in a success, with output, we are assuming that there are
	// changes to commit and push
	if addResult.Len() > 0 {
		fmt.Printf("about to commit\n")
		err = runCommand(commitTemporaryFilesToClone(temporaryPagesDirectory, msg), nil)
		if err != nil {
			return err
		}

		fmt.Printf("about to push\n")
		err = runCommand(pushTemporaryClone(temporaryPagesDirectory, vargsUpstreamName, vargsTargetBranch), nil)
		if err != nil {
			return err
		}
	}

	return nil
}

// Returns command to get the commit message for the gh-pages
// commit based on the last commit in the build
func commitMessage(workspacePath string) *exec.Cmd {
	cmd := exec.Command(
		"git",
		"show",
		"-q",
	)
	cmd.Dir = workspacePath
	return cmd
}

// Returns command to clone gh-pages into our temporary location.
// git clone -b gh-pages --single-branch repo .tmp/[pages directory]
func cloneTarget(workspacePath string, targetBranch string, repo string, temporaryPagesDirectory string) *exec.Cmd {
	cmd := exec.Command(
		"git",
		"clone",
		"-b",
		targetBranch,
		"--single-branch",
		repo,
		temporaryPagesDirectory,
	)
	cmd.Dir = workspacePath
	return cmd
}

// Copy the pages content to the temporary location
// rsync --delete --exclude .git -r docs .tmp
func rsyncPages(workspacePath string, pagesDirectory string, temporaryBaseDirectory string) *exec.Cmd {
	cmd := exec.Command(
		"rsync",
		"--delete",
		"--exclude",
		".git",
		"-r",
		pagesDirectory,
		temporaryBaseDirectory,
	)
	cmd.Dir = workspacePath
	return cmd
}

// Add the files in the temporary directory to the commit
// we want to make in the clone.
func addTemporaryFilesToClone(temporaryPagesDirectory string) *exec.Cmd {
	cmd := exec.Command(
		"git",
		"add",
		"--verbose",
		".",
	)
	cmd.Dir = temporaryPagesDirectory
	return cmd
}

// Commit the working version of the pages content to
// our clone
func commitTemporaryFilesToClone(temporaryPagesDirectory string, message string) *exec.Cmd {
	cmd := exec.Command(
		"git",
		"commit",
		"-m",
		message,
	)
	cmd.Dir = temporaryPagesDirectory
	return cmd
}

// Push our clone to the upstream
func pushTemporaryClone(temporaryPagesDirectory string, upstreamName string, targetBranch string) *exec.Cmd {
	cmd := repo.RemotePush(upstreamName, targetBranch, false)
	cmd.Dir = temporaryPagesDirectory
	return cmd
}

// Writes the netrc file.
func writeNetrc(in *drone.Workspace) error {
	if in.Netrc == nil || len(in.Netrc.Machine) == 0 {
		return nil
	}
	out := fmt.Sprintf(
		netrcFile,
		in.Netrc.Machine,
		in.Netrc.Login,
		in.Netrc.Password,
	)
	home := "/root"
	u, err := user.Current()
	if err == nil {
		home = u.HomeDir
	}
	path := filepath.Join(home, ".netrc")
	return ioutil.WriteFile(path, []byte(out), 0600)
}
<commit_msg>Get rid of the deprecated drone plugin<commit_after>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"

	"github.com/drone/drone-go/drone"
	"github.com/drone/drone-go/plugin"
	"github.com/drone-plugins/drone-git-push/repo"
)

var (
	buildCommit string
)

var netrcFile = `
machine %s
login %s
password %s
`

func main() {
	fmt.Printf("Drone GitHub Pages plugin built from %s\n", buildCommit)

	workspace := drone.Workspace{}
	repo := drone.Repo{}
	build := drone.Build{}
	vargs := Params{}

	plugin.Param("workspace", &workspace)
	plugin.Param("repo", &repo)
	plugin.Param("build", &build)
	plugin.Param("vargs", &vargs)
	plugin.MustParse()

	err := publishDocs(&repo, &build, &workspace, &vargs)

	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}

func publishDocs(r *drone.Repo, b *drone.Build, w *drone.Workspace, v *Params) error {
	if v.UpstreamName == "" {
		v.UpstreamName = "origin"
	}

	if v.TargetBranch == "" {
		v.TargetBranch = "gh-pages"
	}

	if v.TemporaryBase == "" {
		v.TemporaryBase = ".tmp"
	}

	if v.PagesDirectory == "" {
		v.PagesDirectory = "docs"
	}

	temporaryBaseDirectory := ""

	if filepath.IsAbs(v.TemporaryBase) {
		temporaryBaseDirectory = v.TemporaryBase
	} else {
		temporaryBaseDirectory = filepath.Join(w.Path, v.TemporaryBase)
	}

	temporaryPagesDirectory := filepath.Join(temporaryBaseDirectory, v.PagesDirectory)
	fullPagesDirectory := filepath.Join(w.Path, v.PagesDirectory)

	// generate the .netrc file
	if err := writeNetrc(w); err != nil {
		fmt.Fprintln(os.Stderr, err)
		return err
	}

	// write the rsa private key if provided
	if err := repo.WriteKey(w); err != nil {
		fmt.Fprintln(os.Stderr, err)
		return err
	}

	if err := repo.GlobalUser(b).Run(); err != nil {
		return err
	}

	if err := repo.GlobalName(b).Run(); err != nil {
		return err
	}

	if err := os.MkdirAll(temporaryPagesDirectory, 0777); err != nil {
		return err
	}

	defer os.RemoveAll(temporaryPagesDirectory)

	err := runPublishSteps(
		w.Path,
		temporaryBaseDirectory,
		temporaryPagesDirectory,
		fullPagesDirectory,
		v.TargetBranch,
		v.UpstreamName,
		r.Clone,
	)

	if err != nil {
		return err
	}

	return nil
}

func runCommand(cmd *exec.Cmd, out *bytes.Buffer) error {
	if out == nil {
		if cmd.Stdout == nil {
			cmd.Stdout = os.Stdout
		}
	} else {
		cmd.Stdout = out
	}

	cmd.Stderr = os.Stderr

	if err := cmd.Run(); err != nil {
		fmt.Printf("Error: %+v %s\n", cmd, err)
		return err
	}

	return nil
}

func runPublishSteps(
	workspacePath string,
	temporaryBaseDirectory string,
	temporaryPagesDirectory string,
	fullPagesDirectory string,
	vargsTargetBranch string,
	vargsUpstreamName string,
	cloneUrl string) error {

	// Get the commit message
	msgRaw, err := commitMessage(workspacePath).Output()

	if err != nil {
		return err
	}

	msg := fmt.Sprintf("%s", msgRaw)
	fmt.Printf("%s\n", msg)

	// Set up git config (rsa, etc)
	fmt.Printf("about to clone\n")
	err = runCommand(cloneTarget(workspacePath, vargsTargetBranch, cloneUrl, temporaryPagesDirectory), nil)
	if err != nil {
		return err
	}

	fmt.Printf("about to rsync\n")
	err = runCommand(rsyncPages(workspacePath, fullPagesDirectory, temporaryBaseDirectory), nil)
	if err != nil {
		return err
	}

	fmt.Printf("about to add to clone\n")
	var addResult bytes.Buffer
	err = runCommand(addTemporaryFilesToClone(temporaryPagesDirectory), &addResult)
	if err != nil {
		fmt.Printf("Error on add: %s", addResult.String())
		return err
	}

	// For now, if the add resulted in a success, with output, we are assuming that there are
	// changes to commit and push
	if addResult.Len() > 0 {
		fmt.Printf("about to commit\n")
		err = runCommand(commitTemporaryFilesToClone(temporaryPagesDirectory, msg), nil)
		if err != nil {
			return err
		}

		fmt.Printf("about to push\n")
		err = runCommand(pushTemporaryClone(temporaryPagesDirectory, vargsUpstreamName, vargsTargetBranch), nil)
		if err != nil {
			return err
		}
	}

	return nil
}

// Returns command to get the commit message for the gh-pages
// commit based on the last commit in the build
func commitMessage(workspacePath string) *exec.Cmd {
	cmd := exec.Command(
		"git",
		"show",
		"-q",
	)
	cmd.Dir = workspacePath
	return cmd
}

// Returns command to clone gh-pages into our temporary location.
// git clone -b gh-pages --single-branch repo .tmp/[pages directory]
func cloneTarget(workspacePath string, targetBranch string, repo string, temporaryPagesDirectory string) *exec.Cmd {
	cmd := exec.Command(
		"git",
		"clone",
		"-b",
		targetBranch,
		"--single-branch",
		repo,
		temporaryPagesDirectory,
	)
	cmd.Dir = workspacePath
	return cmd
}

// Copy the pages content to the temporary location
// rsync --delete --exclude .git -r docs .tmp
func rsyncPages(workspacePath string, pagesDirectory string, temporaryBaseDirectory string) *exec.Cmd {
	cmd := exec.Command(
		"rsync",
		"--delete",
		"--exclude",
		".git",
		"-r",
		pagesDirectory,
		temporaryBaseDirectory,
	)
	cmd.Dir = workspacePath
	return cmd
}

// Add the files in the temporary directory to the commit
// we want to make in the clone.
func addTemporaryFilesToClone(temporaryPagesDirectory string) *exec.Cmd {
	cmd := exec.Command(
		"git",
		"add",
		"--verbose",
		".",
	)
	cmd.Dir = temporaryPagesDirectory
	return cmd
}

// Commit the working version of the pages content to
// our clone
func commitTemporaryFilesToClone(temporaryPagesDirectory string, message string) *exec.Cmd {
	cmd := exec.Command(
		"git",
		"commit",
		"-m",
		message,
	)
	cmd.Dir = temporaryPagesDirectory
	return cmd
}

// Push our clone to the upstream
func pushTemporaryClone(temporaryPagesDirectory string, upstreamName string, targetBranch string) *exec.Cmd {
	cmd := repo.RemotePush(upstreamName, targetBranch, false)
	cmd.Dir = temporaryPagesDirectory
	return cmd
}

// Writes the netrc file.
func writeNetrc(in *drone.Workspace) error {
	if in.Netrc == nil || len(in.Netrc.Machine) == 0 {
		return nil
	}
	out := fmt.Sprintf(
		netrcFile,
		in.Netrc.Machine,
		in.Netrc.Login,
		in.Netrc.Password,
	)
	home := "/root"
	u, err := user.Current()
	if err == nil {
		home = u.HomeDir
	}
	path := filepath.Join(home, ".netrc")
	return ioutil.WriteFile(path, []byte(out), 0600)
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"flag"
	"io/ioutil"
	"log"
	"os"
	"path"
	"regexp"

	"fmt"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/client"
)

var cniPath = flag.String("cni.path", "/var/lib/cni/networks/podnet", "Path to lookup CNI IP Assigments")

func main() {
	flag.Parse()

	cli, err := client.NewEnvClient()
	if err != nil {
		log.Fatal("Unable to connect to docker agent", err.Error())
	}

	containers, err := cli.ContainerList(context.Background(), types.ContainerListOptions{Quiet: true})
	if err != nil {
		log.Fatal("Unable to list Containers", err.Error())
	}

	assignmentFiles, err := ioutil.ReadDir(*cniPath)
	if err != nil {
		log.Fatal("Unable to list assigned IPs", err.Error())
	}

	for _, assignment := range assignmentFiles {

		assignmentFile := assignment.Name()

		assignmentFileContents, err := ioutil.ReadFile(path.Join(*cniPath, assignmentFile))
		if err != nil {
			log.Fatal("Unable to open assignment file", err.Error())
		}

		isIPAddress, err := regexp.MatchString(`\d+[.]\d+[.]\d+[.]\d+`, assignmentFile)

		if !isIPAddress {
			continue
		}

		containerID := string(assignmentFileContents)

		if isRunningContainer(containerID, containers) {
			fmt.Println("Container found. Assignment active")

		} else {
			fmt.Println("Container " + containerID + " not found. Removing assigned IP")
			os.Remove(path.Join(*cniPath, assignmentFile))
		}
	}
}

func isRunningContainer(containerID string, containers []types.Container) bool {

	for _, container := range containers {

		if container.ID == containerID {
			return true
		}
	}

	return false
}
<commit_msg>Use 1.12.6 level apis<commit_after>package main

import (
	"context"
	"flag"
	"io/ioutil"
	"log"
	"os"
	"path"
	"regexp"

	"fmt"

	"github.com/docker/engine-api/client"
	"github.com/docker/engine-api/types"
)

var cniPath = flag.String("cni.path", "/var/lib/cni/networks/podnet", "Path to lookup CNI IP Assigments")

func main() {
	flag.Parse()

	cli, err := client.NewEnvClient()
	if err != nil {
		log.Fatal("Unable to connect to docker agent", err.Error())
	}

	containers, err := cli.ContainerList(context.Background(), types.ContainerListOptions{Quiet: true})
	if err != nil {
		log.Fatal("Unable to list Containers", err.Error())
	}

	assignmentFiles, err := ioutil.ReadDir(*cniPath)
	if err != nil {
		log.Fatal("Unable to list assigned IPs", err.Error())
	}

	for _, assignment := range assignmentFiles {

		assignmentFile := assignment.Name()

		assignmentFileContents, err := ioutil.ReadFile(path.Join(*cniPath, assignmentFile))
		if err != nil {
			log.Fatal("Unable to open assignment file", err.Error())
		}

		isIPAddress, err := regexp.MatchString(`\d+[.]\d+[.]\d+[.]\d+`, assignmentFile)

		if !isIPAddress {
			continue
		}

		containerID := string(assignmentFileContents)

		if isRunningContainer(containerID, containers) {
			fmt.Println("Container found. Assignment active")

		} else {
			fmt.Println("Container " + containerID + " not found. Removing assigned IP")
			os.Remove(path.Join(*cniPath, assignmentFile))
		}
	}
}

func isRunningContainer(containerID string, containers []types.Container) bool {

	for _, container := range containers {

		if container.ID == containerID {
			return true
		}
	}

	return false
}
<|endoftext|>


<commit_before>// The MIT License (MIT)
//
// Copyright (c) 2016 Maciek Borzecki <maciek.borzecki@gmail.com>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"os/signal"
	"strconv"
	"syscall"
	"time"

	log "github.com/sirupsen/logrus"
	log_syslog "github.com/sirupsen/logrus/hooks/syslog"
	"log/syslog"
)

var (
	optDebug      = flag.Bool("debug", false, "Enable debug logging")
	optCacheRoot  = flag.String("cache-root", "./tmp", "Cache directory path")
	optListenAddr = flag.String("listen", ":8080", "Listen address")
	optMirrors    = flag.String("mirrors", "", "Mirror list file")
	optTimeout    = flag.Duration("client-timeout", 15*time.Second, "Forward request timeout")
	optVersion    = flag.Bool("version", false, "Show version")
	optSyslog     = flag.Bool("syslog", false, "Enable logging to syslog")
	optPidfile    = flag.String("pidfile", "", "Write self PID to this file")

	Version = "(unknown)"
)

func main() {

	flag.Parse()

	if *optVersion {
		fmt.Println(Version)
		return
	}

	if *optSyslog {
		h, err := log_syslog.NewSyslogHook("", "", syslog.LOG_INFO, "viadown")
		if err != nil {
			log.Errorf("failed to connect to syslog: %v", err)
		} else {
			log.AddHook(h)
		}
	}

	if *optDebug {
		log.SetLevel(log.DebugLevel)
		log.Debugf("debug logging enabled")
	}

	if *optMirrors == "" {
		log.Errorf("no mirrors, cannot continue")
		os.Exit(1)
	}

	pid := os.Getpid()
	log.Infof("viadown version %v starting... PID: %v", Version, pid)

	if *optPidfile != "" {
		err := ioutil.WriteFile(*optPidfile, []byte(strconv.Itoa(pid)),
			0600)
		if err != nil {
			log.Fatalf("failed to write pid to %s: %v",
				*optPidfile, err)
		}
	}

	m := Mirrors{}
	if err := m.LoadFile(*optMirrors); err != nil {
		log.Errorf("failed to load mirrors from %v: %v",
			*optMirrors, err)
		os.Exit(1)
	}

	log.Infof("cache root: %v", *optCacheRoot)
	cache := Cache{
		Dir: *optCacheRoot,
	}

	addr := *optListenAddr
	server := http.Server{
		Addr:    addr,
		Handler: NewViaDownloadServer(&m, &cache, *optTimeout),
	}
	log.Infof("listen on %v", addr)

	listenerrchan := make(chan error)
	sigchan := make(chan os.Signal)

	// wait for SIGINT, SIGTERM, SIGQUIT
	signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM,
		syscall.SIGQUIT)

	go func() {
		listenerrchan <- server.ListenAndServe()
	}()

	select {
	case fail := <-listenerrchan:
		log.Fatalf("listen failed: %v", fail)
	case sig := <-sigchan:
		log.Infof("exiting on signal... %s", sig)
	}
}
<commit_msg>main: make signal channel bufferred<commit_after>// The MIT License (MIT)
//
// Copyright (c) 2016 Maciek Borzecki <maciek.borzecki@gmail.com>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"os/signal"
	"strconv"
	"syscall"
	"time"

	log "github.com/sirupsen/logrus"
	log_syslog "github.com/sirupsen/logrus/hooks/syslog"
	"log/syslog"
)

var (
	optDebug      = flag.Bool("debug", false, "Enable debug logging")
	optCacheRoot  = flag.String("cache-root", "./tmp", "Cache directory path")
	optListenAddr = flag.String("listen", ":8080", "Listen address")
	optMirrors    = flag.String("mirrors", "", "Mirror list file")
	optTimeout    = flag.Duration("client-timeout", 15*time.Second, "Forward request timeout")
	optVersion    = flag.Bool("version", false, "Show version")
	optSyslog     = flag.Bool("syslog", false, "Enable logging to syslog")
	optPidfile    = flag.String("pidfile", "", "Write self PID to this file")

	Version = "(unknown)"
)

func main() {

	flag.Parse()

	if *optVersion {
		fmt.Println(Version)
		return
	}

	if *optSyslog {
		h, err := log_syslog.NewSyslogHook("", "", syslog.LOG_INFO, "viadown")
		if err != nil {
			log.Errorf("failed to connect to syslog: %v", err)
		} else {
			log.AddHook(h)
		}
	}

	if *optDebug {
		log.SetLevel(log.DebugLevel)
		log.Debugf("debug logging enabled")
	}

	if *optMirrors == "" {
		log.Errorf("no mirrors, cannot continue")
		os.Exit(1)
	}

	pid := os.Getpid()
	log.Infof("viadown version %v starting... PID: %v", Version, pid)

	if *optPidfile != "" {
		err := ioutil.WriteFile(*optPidfile, []byte(strconv.Itoa(pid)),
			0600)
		if err != nil {
			log.Fatalf("failed to write pid to %s: %v",
				*optPidfile, err)
		}
	}

	m := Mirrors{}
	if err := m.LoadFile(*optMirrors); err != nil {
		log.Errorf("failed to load mirrors from %v: %v",
			*optMirrors, err)
		os.Exit(1)
	}

	log.Infof("cache root: %v", *optCacheRoot)
	cache := Cache{
		Dir: *optCacheRoot,
	}

	addr := *optListenAddr
	server := http.Server{
		Addr:    addr,
		Handler: NewViaDownloadServer(&m, &cache, *optTimeout),
	}
	log.Infof("listen on %v", addr)

	listenerrchan := make(chan error)
	sigchan := make(chan os.Signal, 3)

	// wait for SIGINT, SIGTERM, SIGQUIT
	signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM,
		syscall.SIGQUIT)

	go func() {
		listenerrchan <- server.ListenAndServe()
	}()

	select {
	case fail := <-listenerrchan:
		log.Fatalf("listen failed: %v", fail)
	case sig := <-sigchan:
		log.Infof("exiting on signal... %s", sig)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/spf13/viper"
	"log"
	"os"
)

var (
	olog *log.Logger
	elog *log.Logger
)

func init() {
	loadConfig()
	olog = log.New(os.Stdout, "", 0)
	elog = log.New(os.Stderr, "", 0)
}

func main() {
	r := new(rsync)
	r.sync(r.generateOptions())
	olog.Println("Sycing succesful!")
}

func loadConfig() {
	viper.SetConfigName("gorsync")
	configDirectory, _ := getCurrentDirectory()
	viper.AddConfigPath(configDirectory)
	viper.ReadInConfig()
}
<commit_msg>Add check if config file is present<commit_after>package main

import (
	"github.com/spf13/viper"
	"log"
	"os"
)

var (
	olog *log.Logger
	elog *log.Logger
)

func init() {
	olog = log.New(os.Stdout, "", 0)
	elog = log.New(os.Stderr, "", 0)
	loadConfig()
}

func main() {
	r := new(rsync)
	r.sync(r.generateOptions())
	olog.Println("Sycing succesful!")
}

func loadConfig() {
	viper.SetConfigName("gorsync")
	configDirectory, _ := getCurrentDirectory()
	viper.AddConfigPath(configDirectory)
	error := viper.ReadInConfig()
	if error != nil {
		elog.Fatal("No ./gorsync.yml|json|toml config file found!")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/md5"
	"database/sql"
	"flag"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"os"
	"strings"

	"golang.org/x/net/context"

	"github.com/kusubooru/shimmie"
	"github.com/kusubooru/shimmie/store"
)

//go:generate go run generate/version.go

var (
	httpAddr = flag.String("http", "localhost:8080", "HTTP listen address")
	dbDriver = flag.String("dbdriver", "mysql", "database driver")
	dbConfig = flag.String("dbconfig", "", "username:password@(host:port)/database?parseTime=true")
	loginURL = flag.String("loginurl", "/kanri/login", "login URL path to redirect to")
	//version  = flag.Bool("v", false, "print program version")
	certFile = flag.String("tlscert", "", "TLS public key in PEM format.  Must be used together with -tlskey")
	keyFile  = flag.String("tlskey", "", "TLS private key in PEM format.  Must be used together with -tlscert")
	// Set after flag parsing based on certFile & keyFile.
	useTLS bool
)

const (
	description = `Usage: kanri [options]
  Management tools for shimmie.
Options:
`
)

func usage() {
	fmt.Fprintf(os.Stderr, description)
	flag.PrintDefaults()
	fmt.Fprintf(os.Stderr, "\n")
}

func main() {
	flag.Usage = usage
	flag.Parse()
	useTLS = *certFile != "" && *keyFile != ""

	//if *version {
	//	// TheVersion is generated by go generate.
	//	fmt.Printf("teian%v\n", TheVersion)
	//	return
	//}

	// open store with new database connection and create new Shimmie
	shim := shimmie.New(store.Open(*dbDriver, *dbConfig))

	// get common conf
	common, cerr := shim.Store.GetCommon()
	if cerr != nil {
		log.Fatalln("could not get common conf:", cerr)
	}

	// app with Shimmie and common conf
	app := App{Shimmie: shim, Common: common}

	// new context
	ctx := context.Background()

	http.Handle("/kanri", shim.Auth(ctx, app.serveIndex, *loginURL))
	http.Handle("/kanri/safe", shim.Auth(ctx, app.serveSafe, *loginURL))
	http.Handle("/kanri/_image/", shim.Auth(ctx, app.serveImage, *loginURL))
	http.Handle("/kanri/_thumb/", shim.Auth(ctx, app.serveThumb, *loginURL))
	http.Handle("/kanri/login", newHandler(ctx, app.serveLogin))
	http.Handle("/kanri/login/submit", newHandler(ctx, app.handleLogin))
	http.Handle("/kanri/logout", http.HandlerFunc(handleLogout))

	if useTLS {
		if err := http.ListenAndServeTLS(*httpAddr, *certFile, *keyFile, nil); err != nil {
			log.Fatalf("Could not start listening (TLS) on %v: %v", *httpAddr, err)
		}
	} else {
		if err := http.ListenAndServe(*httpAddr, nil); err != nil {
			log.Fatalf("Could not start listening on %v: %v", *httpAddr, err)
		}
	}
}

type ctxHandlerFunc func(context.Context, http.ResponseWriter, *http.Request)

func newHandler(ctx context.Context, fn ctxHandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		fn(ctx, w, r)
	}
}

type App struct {
	Shimmie *shimmie.Shimmie
	Common  *shimmie.Common
}

func (app *App) serveIndex(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	app.render(w, indexTmpl, nil)
}

func (app *App) serveSafe(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	images, err := app.Shimmie.GetSafeBustedImages("kusubooru")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	app.render(w, safeTmpl, images)
}

func (app *App) serveImage(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	app.serveImageFile(ctx, w, r, app.Shimmie.ImagePath)
}

func (app *App) serveThumb(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	app.serveImageFile(ctx, w, r, app.Shimmie.ThumbPath)
}

func (app *App) serveImageFile(ctx context.Context, w http.ResponseWriter, r *http.Request, path string) {
	hash := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:]

	err := app.Shimmie.WriteImageFile(w, path, hash)
	if err != nil {
		http.Error(w, fmt.Sprintf("could not write image file: %v", err), http.StatusInternalServerError)
		return
	}
}

func (app *App) serveLogin(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	app.render(w, loginTmpl, nil)
}

func (app *App) handleLogin(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	// only accept POST method
	if r.Method != "POST" {
		http.Error(w, fmt.Sprintf("%v method not allowed", r.Method), http.StatusMethodNotAllowed)
		return
	}
	username := r.PostFormValue("username")
	password := r.PostFormValue("password")
	user, err := app.Shimmie.GetUser(username)
	if err != nil {
		if err == sql.ErrNoRows {
			app.render(w, loginTmpl, "User does not exist.")
			return
		}
		msg := fmt.Sprintf("get user %q failed: %v", username, err.Error())
		log.Print(msg)
		http.Error(w, msg, http.StatusInternalServerError)
		return
	}
	hash := md5.Sum([]byte(username + password))
	passwordHash := fmt.Sprintf("%x", hash)
	if user.Pass != passwordHash {
		app.render(w, loginTmpl, "Username and password do not match.")
		return
	}
	addr := strings.Split(r.RemoteAddr, ":")[0]
	cookieValue := shimmie.CookieValue(passwordHash, addr)
	shimmie.SetCookie(w, "shm_user", username)
	shimmie.SetCookie(w, "shm_session", cookieValue)
	http.Redirect(w, r, "/kanri", http.StatusFound)
}

func handleLogout(w http.ResponseWriter, r *http.Request) {
	shimmie.SetCookie(w, "shm_user", "")
	shimmie.SetCookie(w, "shm_session", "")
	http.Redirect(w, r, "/kanri", http.StatusFound)
}

func render(w http.ResponseWriter, t *template.Template, data interface{}) {
	if err := t.Execute(w, data); err != nil {
		msg := fmt.Sprintln("could not render template:", err)
		log.Print(msg)
		http.Error(w, msg, http.StatusInternalServerError)
		return
	}
}

func (app *App) render(w http.ResponseWriter, t *template.Template, data interface{}) {
	render(w, t, struct {
		Data   interface{}
		Common *shimmie.Common
	}{
		Data:   data,
		Common: app.Common,
	})
}
<commit_msg>Change serveSafe to receive user from context<commit_after>package main

import (
	"crypto/md5"
	"database/sql"
	"flag"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"os"
	"strings"

	"golang.org/x/net/context"

	"github.com/kusubooru/shimmie"
	"github.com/kusubooru/shimmie/store"
)

//go:generate go run generate/version.go

var (
	httpAddr = flag.String("http", "localhost:8080", "HTTP listen address")
	dbDriver = flag.String("dbdriver", "mysql", "database driver")
	dbConfig = flag.String("dbconfig", "", "username:password@(host:port)/database?parseTime=true")
	loginURL = flag.String("loginurl", "/kanri/login", "login URL path to redirect to")
	//version  = flag.Bool("v", false, "print program version")
	certFile = flag.String("tlscert", "", "TLS public key in PEM format.  Must be used together with -tlskey")
	keyFile  = flag.String("tlskey", "", "TLS private key in PEM format.  Must be used together with -tlscert")
	// Set after flag parsing based on certFile & keyFile.
	useTLS bool
)

const (
	description = `Usage: kanri [options]
  Management tools for shimmie.
Options:
`
)

func usage() {
	fmt.Fprintf(os.Stderr, description)
	flag.PrintDefaults()
	fmt.Fprintf(os.Stderr, "\n")
}

func main() {
	flag.Usage = usage
	flag.Parse()
	useTLS = *certFile != "" && *keyFile != ""

	//if *version {
	//	// TheVersion is generated by go generate.
	//	fmt.Printf("teian%v\n", TheVersion)
	//	return
	//}

	// open store with new database connection and create new Shimmie
	shim := shimmie.New(store.Open(*dbDriver, *dbConfig))

	// get common conf
	common, cerr := shim.Store.GetCommon()
	if cerr != nil {
		log.Fatalln("could not get common conf:", cerr)
	}

	// app with Shimmie and common conf
	app := App{Shimmie: shim, Common: common}

	// new context
	ctx := context.Background()

	http.Handle("/kanri", shim.Auth(ctx, app.serveIndex, *loginURL))
	http.Handle("/kanri/safe", shim.Auth(ctx, app.serveSafe, *loginURL))
	http.Handle("/kanri/_image/", shim.Auth(ctx, app.serveImage, *loginURL))
	http.Handle("/kanri/_thumb/", shim.Auth(ctx, app.serveThumb, *loginURL))
	http.Handle("/kanri/login", newHandler(ctx, app.serveLogin))
	http.Handle("/kanri/login/submit", newHandler(ctx, app.handleLogin))
	http.Handle("/kanri/logout", http.HandlerFunc(handleLogout))

	if useTLS {
		if err := http.ListenAndServeTLS(*httpAddr, *certFile, *keyFile, nil); err != nil {
			log.Fatalf("Could not start listening (TLS) on %v: %v", *httpAddr, err)
		}
	} else {
		if err := http.ListenAndServe(*httpAddr, nil); err != nil {
			log.Fatalf("Could not start listening on %v: %v", *httpAddr, err)
		}
	}
}

type ctxHandlerFunc func(context.Context, http.ResponseWriter, *http.Request)

func newHandler(ctx context.Context, fn ctxHandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		fn(ctx, w, r)
	}
}

type App struct {
	Shimmie *shimmie.Shimmie
	Common  *shimmie.Common
}

func (app *App) serveIndex(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	app.render(w, indexTmpl, nil)
}

func (app *App) serveSafe(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	user, ok := ctx.Value("user").(*shimmie.User)
	if !ok || user.Admin != "Y" {
		http.Error(w, "You are not authorized to view this page.", http.StatusUnauthorized)
		return
	}
	images, err := app.Shimmie.GetRatedImages(user.Name)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	app.render(w, safeTmpl, images)
}

func (app *App) serveImage(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	app.serveImageFile(ctx, w, r, app.Shimmie.ImagePath)
}

func (app *App) serveThumb(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	app.serveImageFile(ctx, w, r, app.Shimmie.ThumbPath)
}

func (app *App) serveImageFile(ctx context.Context, w http.ResponseWriter, r *http.Request, path string) {
	hash := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:]

	err := app.Shimmie.WriteImageFile(w, path, hash)
	if err != nil {
		http.Error(w, fmt.Sprintf("could not write image file: %v", err), http.StatusInternalServerError)
		return
	}
}

func (app *App) serveLogin(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	app.render(w, loginTmpl, nil)
}

func (app *App) handleLogin(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	// only accept POST method
	if r.Method != "POST" {
		http.Error(w, fmt.Sprintf("%v method not allowed", r.Method), http.StatusMethodNotAllowed)
		return
	}
	username := r.PostFormValue("username")
	password := r.PostFormValue("password")
	user, err := app.Shimmie.GetUser(username)
	if err != nil {
		if err == sql.ErrNoRows {
			app.render(w, loginTmpl, "User does not exist.")
			return
		}
		msg := fmt.Sprintf("get user %q failed: %v", username, err.Error())
		log.Print(msg)
		http.Error(w, msg, http.StatusInternalServerError)
		return
	}
	hash := md5.Sum([]byte(username + password))
	passwordHash := fmt.Sprintf("%x", hash)
	if user.Pass != passwordHash {
		app.render(w, loginTmpl, "Username and password do not match.")
		return
	}
	addr := strings.Split(r.RemoteAddr, ":")[0]
	cookieValue := shimmie.CookieValue(passwordHash, addr)
	shimmie.SetCookie(w, "shm_user", username)
	shimmie.SetCookie(w, "shm_session", cookieValue)
	http.Redirect(w, r, "/kanri", http.StatusFound)
}

func handleLogout(w http.ResponseWriter, r *http.Request) {
	shimmie.SetCookie(w, "shm_user", "")
	shimmie.SetCookie(w, "shm_session", "")
	http.Redirect(w, r, "/kanri", http.StatusFound)
}

func render(w http.ResponseWriter, t *template.Template, data interface{}) {
	if err := t.Execute(w, data); err != nil {
		msg := fmt.Sprintln("could not render template:", err)
		log.Print(msg)
		http.Error(w, msg, http.StatusInternalServerError)
		return
	}
}

func (app *App) render(w http.ResponseWriter, t *template.Template, data interface{}) {
	render(w, t, struct {
		Data   interface{}
		Common *shimmie.Common
	}{
		Data:   data,
		Common: app.Common,
	})
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/kisielk/whisper-go/whisper"

	"net/http"
	"encoding/json"
	"path/filepath"
	"strings"
	"strconv"
	"fmt"
)

var config = struct {
	WhisperData	string
}{
	WhisperData: "/var/lib/carbon/whisper",
}

type WhisperFetchResponse struct {
	Name		string		`json:"name"`
	StartTime	uint32		`json:"startTime"`
	StopTime	uint32		`json:"stopTime"`
	StepTime	uint32		`json:"stepTime"`
	Values		[]float64	`json:"values"`
}

type WhisperGlobResponse struct {
	Name		string		`json:"name"`
	Paths		[]string	`json:"paths"`
}

func findHandler(wr http.ResponseWriter, req *http.Request) {
//	GET /metrics/find/?local=1&format=pickle&query=general.hadoop.lhr4.ha201jobtracker-01.jobtracker.NonHeapMemoryUsage.committed HTTP/1.1
//	http://localhost:8080/metrics/glob/?query=test
	req.ParseForm()
	glob := req.FormValue("query")

	/* things to glob:
	 * - carbon.relays  -> carbon.relays
	 * - carbon.re      -> carbon.relays, carbon.rewhatever
	 * - implicit * at the end of each query
	 * - match is either dir or .wsp file
	 */
	path := config.WhisperData + "/" + strings.Replace(glob, ".", "/", -1) + "*"
	files, err := filepath.Glob(path)
	if err != nil {
		files = make([]string, 0)
	}

	response := WhisperGlobResponse {
		Name:		glob,
	}
	for _, p := range files {
		p = p[len(config.WhisperData + "/"):]
		if strings.HasSuffix(p, ".wsp") {
			p = p[:len(p) - 4]
		}
		response.Paths = append(response.Paths, strings.Replace(p, "/", ".", -1))
	}

	b, err := json.Marshal(response)
	if err != nil {
		fmt.Printf("failed to create JSON data for %s: %s\n", glob, err)
		return
	}
	wr.Write(b)
	fmt.Printf("served %d points\n", len(response.Paths))
	return
}

func fetchHandler(wr http.ResponseWriter, req *http.Request) {
//	GET /render/?target=general.me.1.percent_time_active.pfnredis&format=pickle&from=1396008021&until=1396022421 HTTP/1.1
//	http://localhost:8080/metrics/fetch/?target=testmetric&format=json&from=1395961200&until=1395961800
	req.ParseForm()
	metric := req.FormValue("target")
	format := req.FormValue("format")
	from := req.FormValue("from")
	until := req.FormValue("until")

	if format != "json" {
		fmt.Printf("dropping invalid uri (format): %s\n", req.URL.RequestURI())
		return
	}

	path := config.WhisperData + "/" + strings.Replace(metric, ".", "/", -1) + ".wsp"
	w, err := whisper.Open(path)
	if err != nil {
		fmt.Printf("failed to open %s: %s\n", path, err)
		return
	}
	defer w.Close()

	i, err := strconv.Atoi(from)
	if err != nil {
		fmt.Printf("fromTime (%s) invalid: %s\n", from, err)
	}
	fromTime := uint32(i)
	i, err = strconv.Atoi(until)
	if err != nil {
		fmt.Printf("untilTime (%s) invalid: %s\n", from, err)
	}
	untilTime := uint32(i)

	interval, points, err := w.FetchUntil(fromTime, untilTime)
	if err != nil {
		fmt.Printf("failed to getch points from %s: %s\n", path, err)
		return
	}

	response := WhisperFetchResponse {
		Name:		metric,
		StartTime:	interval.FromTimestamp,
		StopTime:	interval.UntilTimestamp,
		StepTime:	interval.Step,
	}
	for _, p := range points {
		response.Values = append(response.Values, p.Value)
	}

	b, err := json.Marshal(response)
	if err != nil {
		fmt.Printf("failed to create JSON data for %s: %s\n", path, err)
		return
	}
	wr.Write(b)
	fmt.Printf("served %d points\n", len(response.Values))
	return
}

func main() {
	http.HandleFunc("/metrics/glob/", findHandler)
	http.HandleFunc("/metrics/fetch/", fetchHandler)

	http.ListenAndServe(":8080", nil)
}
<commit_msg>return empty list if nothing found<commit_after>package main

import (
	"github.com/kisielk/whisper-go/whisper"

	"net/http"
	"encoding/json"
	"path/filepath"
	"strings"
	"strconv"
	"fmt"
)

var config = struct {
	WhisperData	string
}{
	WhisperData: "/var/lib/carbon/whisper",
}

type WhisperFetchResponse struct {
	Name		string		`json:"name"`
	StartTime	uint32		`json:"startTime"`
	StopTime	uint32		`json:"stopTime"`
	StepTime	uint32		`json:"stepTime"`
	Values		[]float64	`json:"values"`
}

type WhisperGlobResponse struct {
	Name		string		`json:"name"`
	Paths		[]string	`json:"paths"`
}

func findHandler(wr http.ResponseWriter, req *http.Request) {
//	GET /metrics/find/?local=1&format=pickle&query=general.hadoop.lhr4.ha201jobtracker-01.jobtracker.NonHeapMemoryUsage.committed HTTP/1.1
//	http://localhost:8080/metrics/glob/?query=test
	req.ParseForm()
	glob := req.FormValue("query")

	/* things to glob:
	 * - carbon.relays  -> carbon.relays
	 * - carbon.re      -> carbon.relays, carbon.rewhatever
	 * - implicit * at the end of each query
	 * - match is either dir or .wsp file
	 */
	path := config.WhisperData + "/" + strings.Replace(glob, ".", "/", -1) + "*"
	files, err := filepath.Glob(path)
	if err != nil {
		files = make([]string, 0)
	}

	response := WhisperGlobResponse {
		Name:		glob,
		Paths:		make([]string, 0),
	}
	for _, p := range files {
		p = p[len(config.WhisperData + "/"):]
		if strings.HasSuffix(p, ".wsp") {
			p = p[:len(p) - 4]
		}
		response.Paths = append(response.Paths, strings.Replace(p, "/", ".", -1))
	}

	b, err := json.Marshal(response)
	if err != nil {
		fmt.Printf("failed to create JSON data for %s: %s\n", glob, err)
		return
	}
	wr.Write(b)
	fmt.Printf("served %d points\n", len(response.Paths))
	return
}

func fetchHandler(wr http.ResponseWriter, req *http.Request) {
//	GET /render/?target=general.me.1.percent_time_active.pfnredis&format=pickle&from=1396008021&until=1396022421 HTTP/1.1
//	http://localhost:8080/metrics/fetch/?target=testmetric&format=json&from=1395961200&until=1395961800
	req.ParseForm()
	metric := req.FormValue("target")
	format := req.FormValue("format")
	from := req.FormValue("from")
	until := req.FormValue("until")

	if format != "json" {
		fmt.Printf("dropping invalid uri (format): %s\n", req.URL.RequestURI())
		return
	}

	path := config.WhisperData + "/" + strings.Replace(metric, ".", "/", -1) + ".wsp"
	w, err := whisper.Open(path)
	if err != nil {
		fmt.Printf("failed to open %s: %s\n", path, err)
		return
	}
	defer w.Close()

	i, err := strconv.Atoi(from)
	if err != nil {
		fmt.Printf("fromTime (%s) invalid: %s\n", from, err)
	}
	fromTime := uint32(i)
	i, err = strconv.Atoi(until)
	if err != nil {
		fmt.Printf("untilTime (%s) invalid: %s\n", from, err)
	}
	untilTime := uint32(i)

	interval, points, err := w.FetchUntil(fromTime, untilTime)
	if err != nil {
		fmt.Printf("failed to getch points from %s: %s\n", path, err)
		return
	}

	response := WhisperFetchResponse {
		Name:		metric,
		StartTime:	interval.FromTimestamp,
		StopTime:	interval.UntilTimestamp,
		StepTime:	interval.Step,
	}
	for _, p := range points {
		response.Values = append(response.Values, p.Value)
	}

	b, err := json.Marshal(response)
	if err != nil {
		fmt.Printf("failed to create JSON data for %s: %s\n", path, err)
		return
	}
	wr.Write(b)
	fmt.Printf("served %d points\n", len(response.Values))
	return
}

func main() {
	http.HandleFunc("/metrics/glob/", findHandler)
	http.HandleFunc("/metrics/fetch/", fetchHandler)

	http.ListenAndServe(":8080", nil)
}
<|endoftext|>