<commit_before>package main

import (
	"fmt"
	// "github.com/eaciit/toolkit"
	//. "github.com/frezadev/hdc/hive"
	. "github.com/eaciit/hdc/hive"
	// "reflect"
	// "os"
)

var h *Hive

type Sample7 struct {
	Code        string `tag_name:"code"`
	Description string `tag_name:"description"`
	Total_emp   string `tag_name:"total_emp"`
	Salary      string `tag_name:"salary"`
}

func main() {
	var e error
	h = HiveConfig("192.168.0.223:10000", "default", "developer", "b1gD@T@")
	q := "select * from sample_07 limit 5;"

	/*fmt.Println("---------------------- EXEC ----------------")
	result, e := h.Exec(q)

	fmt.Printf("error: \n%v\n", e)

	for _, res := range result {
		tmp := Sample7{}
		h.ParseOutput(res, &tmp)
		fmt.Println(tmp)
	}*/

	fmt.Println("---------------------- EXEC LINE ----------------")

	//to execute query and read the result per line and then process its result

	var DoSomething = func(res string) {
		tmp := Sample7{}
		h.ParseOutput(res, &tmp)
		fmt.Println(tmp)
	}

	e = h.ExecLine(q, DoSomething)
	fmt.Printf("error: \n%v\n", e)

	// test := "00-0000,All Occupations,134354250,40690"

	/*var x = Sample7{}
	var z interface{}
	z = x
	s := reflect.ValueOf(&z).Elem()
	typeOfT := s.Type()
	fmt.Println(reflect.ValueOf(&z).Interface())
	for i := 0; i < s.NumField(); i++ {
		f := s.Field(i)
		tag := s.Type().Field(i).Tag
		fmt.Printf("%d: %s %s = %v | tag %s \n", i, typeOfT.Field(i).Name, f.Type(), f.Interface(), tag.Get("tag_name"))

	}*/

	/*h = HiveConfig("192.168.0.223:10000", "default", "developer", "b1gD@T@")
	h.Header = []string{"code", "description", "total_emp", "salary"}
	qTest := "00-0000,All Occupations,134354250,40690"
	var result = Sample7{}
	h.ParseOutputX(qTest, &result)
	fmt.Printf("result: %s\n", result)*/
}
<commit_msg>main test<commit_after>package main

import (
	"fmt"
	// "github.com/eaciit/toolkit"
	//. "github.com/frezadev/hdc/hive"
	. "github.com/hdc/yanda15/hdc/hive"
	// "reflect"
	// "os"
)

var h *Hive

type Sample7 struct {
	Code        string `tag_name:"code"`
	Description string `tag_name:"description"`
	Total_emp   string `tag_name:"total_emp"`
	Salary      string `tag_name:"salary"`
}

func main() {
	// var e error
	// h = HiveConfig("192.168.0.223:10000", "default", "developer", "b1gD@T@")
	// q := "select * from sample_07 limit 5;"

	/*fmt.Println("---------------------- EXEC ----------------")
	result, e := h.Exec(q)

	fmt.Printf("error: \n%v\n", e)

	for _, res := range result {
		tmp := Sample7{}
		h.ParseOutput(res, &tmp)
		fmt.Println(tmp)
	}*/

	// fmt.Println("---------------------- EXEC LINE ----------------")

	//to execute query and read the result per line and then process its result

	// var DoSomething = func(res string) {
	// 	tmp := Sample7{}
	// 	h.ParseOutput(res, &tmp)
	// 	fmt.Println(tmp)
	// }

	// e = h.ExecLine(q, DoSomething)
	// fmt.Printf("error: \n%v\n", e)

	// test := "00-0000,All Occupations,134354250,40690"

	/*var x = Sample7{}
	var z interface{}
	z = x
	s := reflect.ValueOf(&z).Elem()
	typeOfT := s.Type()
	fmt.Println(reflect.ValueOf(&z).Interface())
	for i := 0; i < s.NumField(); i++ {
		f := s.Field(i)
		tag := s.Type().Field(i).Tag
		fmt.Printf("%d: %s %s = %v | tag %s \n", i, typeOfT.Field(i).Name, f.Type(), f.Interface(), tag.Get("tag_name"))

	}*/

	/*h = HiveConfig("192.168.0.223:10000", "default", "developer", "b1gD@T@")
	h.Header = []string{"code", "description", "total_emp", "salary"}
	qTest := "00-0000,All Occupations,134354250,40690"
	var result = Sample7{}
	h.ParseOutputX(qTest, &result)
	fmt.Printf("result: %s\n", result)*/
}

func DoSomething(res string) {
		tmp := Sample7{}
		h.ParseOutput(res, &tmp)
		fmt.Println(tmp)
}

func TestExec() {
	// var e error
	h = HiveConfig("192.168.0.223:10000", "default", "developer", "b1gD@T@")
	q := "select * from sample_07 limit 5;"
	res, e := h.Exec(q)

	if e !=nil{
		fmt.Printf("error: \n%v\n", e)
	}else{
		fmt.Println(res)
	}
}

func TestExecPerLine() {
	var e error
	h = HiveConfig("192.168.0.223:10000", "default", "developer", "b1gD@T@")
	q := "select * from sample_07 limit 5;"
	e = h.ExecLine(q, DoSomething)

	if e !=nil{
			fmt.Printf("error: \n%v\n", e)
	}
}

func TestParseOutput() {
	h.Header = []string{"code","sample","description","total_emp","salary"}
	res := "00-0000,All Occupations,134354250,40690"
	tmp := Sample7{}
	h.ParseOutput(res, &tmp)
	fmt.Println(tmp)
}

<|endoftext|>


<commit_before>/*
   Copyright 2011-2014 gtalent2@gmail.com

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/
package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
)

const (
	DEFAULT_LICENSE_FILE = ".liccor"
	VERSION              = "liccor 1.7 (go1)"
	// list of file extensions
	SUFFIX_GO   = ".go"
	SUFFIX_C    = ".c"
	SUFFIX_CPP  = ".cpp"
	SUFFIX_CXX  = ".cxx"
	SUFFIX_H    = ".h"
	SUFFIX_HPP  = ".hpp"
	SUFFIX_JAVA = ".java"
	SUFFIX_JS   = ".js"
)

var (
	flagLicenseFile string
	flagVerbose     bool
	showVersion     bool
)

func verboseLog(msg string) {
	if flagVerbose {
		fmt.Println(msg)
	}
}

func findLicense(dir string) (string, error) {
	verboseLog("Search for '" + flagLicenseFile + "' file at directory '" + dir + "'")

	d, err := ioutil.ReadDir(dir)
	if err != nil {
		return "", fmt.Errorf("Could not find " + flagLicenseFile + " file")
	}
	for _, v := range d {
		if v.Name() == flagLicenseFile {
			licenseData, err := ioutil.ReadFile(dir + "/" + v.Name())
			if err != nil {
				err = fmt.Errorf("Could not access " + flagLicenseFile + " file")
			}
			return string(licenseData), err
		}
	}

	return findLicense(dir + "./.")
}

func findSrcFiles(dir string) ([]string, error) {
	verboseLog("Search source files at '" + dir + "'")

	l, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil, err
	}
	output := make([]string, 0)
	for _, v := range l {
		if v.IsDir() {
			// ignore .git dir
			if v.Name() != ".git" {
				files, err := findSrcFiles(dir + "/" + v.Name())
				if err != nil {
					return output, err
				}
				for _, v2 := range files {
					output = append(output, v2)
				}
			}
		} else {
			pt := strings.LastIndex(v.Name(), ".")
			if pt == -1 {
				continue
			}
			switch v.Name()[pt:] {
			case SUFFIX_GO, SUFFIX_C, SUFFIX_CPP, SUFFIX_CXX, SUFFIX_H, SUFFIX_HPP, SUFFIX_JAVA, SUFFIX_JS:
				srcPath := dir + "/" + v.Name()
				output = append(output, srcPath)
				verboseLog("Found source '" + srcPath + "'")
			}
		}
	}
	return output, err
}

func hasLicense(file string) (bool, int) {
	for i, c := range file {
		switch c {
		case ' ', '\t', '\n':
			continue
		case '/':
			i++
			if len(file) > i && file[i] == '*' {
				return true, i
			}
		default:
			return false, -1
		}
	}
	return false, -1
}

func correct(path, license string) (bool, error) {
	input, err := ioutil.ReadFile(path)
	if err != nil {
		return false, err
	}
	file := string(input)
	orig := file
	if hasLicense, licenseStart := hasLicense(file); hasLicense {
		//remove old license
		for i := licenseStart; i < len(file); i++ {
			if file[i] == '*' && file[i+1] == '/' {
				i += 2
				if file[i] == '\n' {
					i += 1
				}
				file = file[i:len(file)]
				break
			}
		}
	}
	file = license + file
	output := []byte(file)
	if file != orig {
		err = ioutil.WriteFile(path, output, 0)
		return true, err
	}
	return false, nil
}

func version() {
	if showVersion {
		println(VERSION)
		os.Exit(0)
	}
}

func init() {
	flag.StringVar(&flagLicenseFile, "license", DEFAULT_LICENSE_FILE, "the name of the license file")
	flag.StringVar(&flagLicenseFile, "l", DEFAULT_LICENSE_FILE, "shortcut for license")
	flag.BoolVar(&flagVerbose, "verbose", false, "print verbose output")
	flag.BoolVar(&flagVerbose, "v", false, "shortcut for verbose")
	flag.BoolVar(&showVersion, "version", false, "version of liccor")
	flag.Usage = func() {
		fmt.Print("\n")
		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
		fmt.Println("\nOptions:")
		flag.PrintDefaults()
		fmt.Println("\nExample usage:")
		fmt.Println("  ./liccor -verbose")
		fmt.Print("\n\n")
	}
}

func main() {
	flag.Parse()
	version()

	licenseData, err := findLicense(".")
	if err != nil {
		fmt.Println(err)
		return
	}
	licenseData = licenseData[0 : len(licenseData)-1]
	lics := make(map[string]string)
	lics["c-like"] = "/*\n * " + strings.Replace(string(licenseData), "\n", "\n * ", -1) + "\n */\n"
	lics["go"] = func() string {
		golic := "/*\n   " + strings.Replace(string(licenseData), "\n", "\n   ", -1) + "\n*/\n"
		golic = strings.Replace(golic, "\n   \n", "\n\n", -1)
		return golic
	}()

	files, err := findSrcFiles(".")
	if err != nil {
		return
	}

	allSuccess := true
	for i := 0; i < len(files); i++ {
		pt := strings.LastIndex(files[i], ".")
		lic := ""
		//determine how to format the license
		switch files[i][pt:] {
		case SUFFIX_GO:
			lic = lics["go"]
		case SUFFIX_C, SUFFIX_CPP, SUFFIX_CXX, SUFFIX_H, SUFFIX_HPP, SUFFIX_JAVA, SUFFIX_JS:
			lic = lics["c-like"]
		}
		changed, err := correct(files[i], lic)
		if changed {
			if err != nil {
				fmt.Println("Correcting '" + files[i][2:] + "'... Failure!")
				allSuccess = false
			} else {
				fmt.Println("Correcting '" + files[i][2:] + "'... Success!")
			}
		}
	}
	if allSuccess {
		fmt.Println("All files up to date!")
	}
}
<commit_msg>Add some more license files as search fallback<commit_after>/*
   Copyright 2011-2014 gtalent2@gmail.com

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/
package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
)

const (
	DEFAULT_LICENSE_FILE = ".liccor"
	VERSION              = "liccor 1.8 (go1)"
	// list of file extensions
	SUFFIX_GO   = ".go"
	SUFFIX_C    = ".c"
	SUFFIX_CPP  = ".cpp"
	SUFFIX_CXX  = ".cxx"
	SUFFIX_H    = ".h"
	SUFFIX_HPP  = ".hpp"
	SUFFIX_JAVA = ".java"
	SUFFIX_JS   = ".js"
)

var (
	flagLicenseFile string
	flagVerbose     bool
	showVersion     bool
)

func verboseLog(msg string) {
	if flagVerbose {
		fmt.Println(msg)
	}
}

func findLicense(dir string) (string, error) {
	verboseLog("Search for a license file at directory '" + dir + "'")

	d, err := ioutil.ReadDir(dir)
	if err != nil {
		return "", fmt.Errorf("Could not find license file")
	}
	for _, v := range d {
		filename := v.Name()
		// search the license file
		if filename == flagLicenseFile || filename == DEFAULT_LICENSE_FILE || filename == "LICENSE" || filename == "LICENSE.txt" {
			licenseData, err := ioutil.ReadFile(dir + "/" + v.Name())
			if err != nil {
				err = fmt.Errorf("Could not access " + filename + " file")
			}
			verboseLog("License file '" + filename + "' found...")
			return string(licenseData), err
		}
	}

	return findLicense(dir + "./.")
}

func findSrcFiles(dir string) ([]string, error) {
	verboseLog("Search source files at '" + dir + "'")

	l, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil, err
	}
	output := make([]string, 0)
	for _, v := range l {
		if v.IsDir() {
			// ignore .git dir
			if v.Name() != ".git" {
				files, err := findSrcFiles(dir + "/" + v.Name())
				if err != nil {
					return output, err
				}
				for _, v2 := range files {
					output = append(output, v2)
				}
			}
		} else {
			pt := strings.LastIndex(v.Name(), ".")
			if pt == -1 {
				continue
			}
			switch v.Name()[pt:] {
			case SUFFIX_GO, SUFFIX_C, SUFFIX_CPP, SUFFIX_CXX, SUFFIX_H, SUFFIX_HPP, SUFFIX_JAVA, SUFFIX_JS:
				srcPath := dir + "/" + v.Name()
				output = append(output, srcPath)
				verboseLog("Found source '" + srcPath + "'")
			}
		}
	}
	return output, err
}

func hasLicense(file string) (bool, int) {
	for i, c := range file {
		switch c {
		case ' ', '\t', '\n':
			continue
		case '/':
			i++
			if len(file) > i && file[i] == '*' {
				return true, i
			}
		default:
			return false, -1
		}
	}
	return false, -1
}

func correct(path, license string) (bool, error) {
	input, err := ioutil.ReadFile(path)
	if err != nil {
		return false, err
	}
	file := string(input)
	orig := file
	if hasLicense, licenseStart := hasLicense(file); hasLicense {
		//remove old license
		for i := licenseStart; i < len(file); i++ {
			if file[i] == '*' && file[i+1] == '/' {
				i += 2
				if file[i] == '\n' {
					i += 1
				}
				file = file[i:len(file)]
				break
			}
		}
	}
	file = license + file
	output := []byte(file)
	if file != orig {
		err = ioutil.WriteFile(path, output, 0)
		return true, err
	}
	return false, nil
}

func version() {
	if showVersion {
		println(VERSION)
		os.Exit(0)
	}
}

func init() {
	flag.StringVar(&flagLicenseFile, "license", DEFAULT_LICENSE_FILE, "the name of the license file")
	flag.StringVar(&flagLicenseFile, "l", DEFAULT_LICENSE_FILE, "shortcut for license")
	flag.BoolVar(&flagVerbose, "verbose", false, "print verbose output")
	flag.BoolVar(&flagVerbose, "v", false, "shortcut for verbose")
	flag.BoolVar(&showVersion, "version", false, "version of liccor")
	flag.Usage = func() {
		fmt.Print("\n")
		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
		fmt.Println("\nOptions:")
		flag.PrintDefaults()
		fmt.Println("\nExample usage:")
		fmt.Println("  ./liccor -verbose")
		fmt.Print("\n\n")
	}
}

func main() {
	flag.Parse()
	version()

	licenseData, err := findLicense(".")
	if err != nil {
		fmt.Println(err)
		return
	}
	licenseData = licenseData[0 : len(licenseData)-1]
	lics := make(map[string]string)
	lics["c-like"] = "/*\n * " + strings.Replace(string(licenseData), "\n", "\n * ", -1) + "\n */\n"
	lics["go"] = func() string {
		golic := "/*\n   " + strings.Replace(string(licenseData), "\n", "\n   ", -1) + "\n*/\n"
		golic = strings.Replace(golic, "\n   \n", "\n\n", -1)
		return golic
	}()

	files, err := findSrcFiles(".")
	if err != nil {
		return
	}

	allSuccess := true
	for i := 0; i < len(files); i++ {
		pt := strings.LastIndex(files[i], ".")
		lic := ""
		//determine how to format the license
		switch files[i][pt:] {
		case SUFFIX_GO:
			lic = lics["go"]
		case SUFFIX_C, SUFFIX_CPP, SUFFIX_CXX, SUFFIX_H, SUFFIX_HPP, SUFFIX_JAVA, SUFFIX_JS:
			lic = lics["c-like"]
		}
		changed, err := correct(files[i], lic)
		if changed {
			if err != nil {
				fmt.Println("Correcting '" + files[i][2:] + "'... Failure!")
				allSuccess = false
			} else {
				fmt.Println("Correcting '" + files[i][2:] + "'... Success!")
			}
		}
	}
	if allSuccess {
		fmt.Println("All files up to date!")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"
)

var (
	client      *http.Client
	registryURL string
)

type repos struct {
	Query      string `json:"query"`
	NumResults int    `json:"num_results"`
	Results    []repo `json:"results"`
}

type repo struct {
	Name        string `json:"name"`
	Description string `json:"description"`
}

type tag string

type image string

type tags map[tag]image

type imageNode struct {
	tags     []tag
	children []*imageNode
}

func init() {
	transport := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}
	client = &http.Client{Transport: transport}
}

func doGet(path string) []byte {
	res, err := client.Get(registryURL + "/v1/" + path)
	if err != nil {
		panic(err.Error())
	}
	if res.StatusCode != 200 {
		return []byte{}
	}
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		panic(err.Error())
	}
	res.Body.Close()
	return body
}

func getRepos() repos {
	var repos repos
	json.Unmarshal(doGet("search"), &repos)
	return repos
}

func getTags(name string) map[tag]image {
	var tags tags
	json.Unmarshal(doGet("repositories/"+name+"/tags"), &tags)
	return tags
}

func getAncestry(id image) []image {
	var ancestry []image
	json.Unmarshal(doGet("images/"+string(id)+"/ancestry"), &ancestry)
	return ancestry
}

func fqTag(name string, t tag) tag {
	canonicalName := strings.TrimPrefix(name, "library/")
	return tag(canonicalName + ":" + string(t))
}

func printTree(root *imageNode, level int) {
	if len(root.tags) > 0 || len(root.children) > 1 {
		fmt.Printf("%s%v\n", strings.Repeat(" ", level), root.tags)
		level = level +1
	}
	for _, child := range root.children {
		printTree(child, level)
	}
}

func main() {
	var (
		tagsByImage = make(map[image][]tag)
		images      = make(map[image]*imageNode)
		roots       []*imageNode
	)
	if len(registryURL) == 0 {
		registryURL = os.Getenv("REGISTRY_URL")
	}
	if len(registryURL) == 0 {
		log.Fatal("No registry URL provided, use the environment variable REGISTRY_URL to set it")
	}
	for _, repo := range getRepos().Results {
		name := repo.Name
		for t, id := range getTags(name) {
			tags, _ := tagsByImage[id]
			tagsByImage[id] = append(tags, fqTag(name, t))
		}
	}
	for imageId := range tagsByImage {
		var previousNode *imageNode
		for _, ancestryId := range getAncestry(imageId) {
			if node, ok := images[ancestryId]; ok {
				if previousNode != nil {
					node.children = append(node.children, previousNode)
				}
				previousNode = nil
				break
			}
			node := &imageNode{}
			if tags, ok := tagsByImage[ancestryId]; ok {
				node.tags = tags
			}
			if previousNode != nil {
				node.children = []*imageNode{previousNode}
			}
			images[ancestryId] = node
			previousNode = node
		}
		if previousNode != nil {
			roots = append(roots, previousNode)
		}
	}
	for _, root := range roots {
		printTree(root, 0)
	}

}
<commit_msg>verbose debug output configurable via env variable<commit_after>package main

import (
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"
)

var (
	client      *http.Client
	registryURL string
)

type repos struct {
	Query      string `json:"query"`
	NumResults int    `json:"num_results"`
	Results    []repo `json:"results"`
}

type repo struct {
	Name        string `json:"name"`
	Description string `json:"description"`
}

type tag string

type image string

type tags map[tag]image

type imageNode struct {
	tags     []tag
	children []*imageNode
}

func init() {
	transport := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}
	client = &http.Client{Transport: transport}
}

func doGet(path string) []byte {
	res, err := client.Get(registryURL + "/v1/" + path)
	if err != nil {
		panic(err.Error())
	}
	if res.StatusCode != 200 {
		return []byte{}
	}
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		panic(err.Error())
	}
	res.Body.Close()
	return body
}

func getRepos() repos {
	log.Print("Fetching repos...")
	var repos repos
	json.Unmarshal(doGet("search"), &repos)
	log.Printf("%v repo(s) fetched", repos.NumResults)
	return repos
}

func getTags(name string) map[tag]image {
	log.Printf("Fetching tags for %s ...", name)
	var tags tags
	json.Unmarshal(doGet("repositories/"+name+"/tags"), &tags)
	log.Printf("%v tags fetched for repo %s", len(tags), name)
	return tags
}

func getAncestry(id image) []image {
	log.Printf("Fetching ancestry for %s ...", id)
	var ancestry []image
	json.Unmarshal(doGet("images/"+string(id)+"/ancestry"), &ancestry)
	log.Printf("%v ancestors fetched for repo %s", len(ancestry), id)
	return ancestry
}

func fqTag(name string, t tag) tag {
	canonicalName := strings.TrimPrefix(name, "library/")
	return tag(canonicalName + ":" + string(t))
}

func printTree(root *imageNode, level int) {
	if len(root.tags) > 0 || len(root.children) > 1 {
		fmt.Printf("%s%v\n", strings.Repeat(" ", level), root.tags)
		level = level +1
	}
	for _, child := range root.children {
		printTree(child, level)
	}
}

func main() {
	var (
		tagsByImage = make(map[image][]tag)
		images      = make(map[image]*imageNode)
		roots       []*imageNode
	)
	if len(registryURL) == 0 {
		registryURL = os.Getenv("REGISTRY_URL")
	}
	if len(registryURL) == 0 {
		log.Fatal("No registry URL provided, use the environment variable REGISTRY_URL to set it")
	}
	if len(os.Getenv("REGISTREE_DEBUG")) > 0 {
		log.SetOutput(os.Stderr)
	} else {
		log.SetOutput(ioutil.Discard)
	}
	for _, repo := range getRepos().Results {
		name := repo.Name
		for t, id := range getTags(name) {
			tags, _ := tagsByImage[id]
			tagsByImage[id] = append(tags, fqTag(name, t))
		}
	}
	for imageId := range tagsByImage {
		var previousNode *imageNode
		for _, ancestryId := range getAncestry(imageId) {
			if node, ok := images[ancestryId]; ok {
				if previousNode != nil {
					node.children = append(node.children, previousNode)
				}
				previousNode = nil
				break
			}
			node := &imageNode{}
			if tags, ok := tagsByImage[ancestryId]; ok {
				node.tags = tags
			}
			if previousNode != nil {
				node.children = []*imageNode{previousNode}
			}
			images[ancestryId] = node
			previousNode = node
		}
		if previousNode != nil {
			roots = append(roots, previousNode)
		}
	}
	for _, root := range roots {
		printTree(root, 0)
	}

}
<|endoftext|>


<commit_before>// Copyright 2016 Mender Software AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.
package main

import "errors"
import "flag"
import "github.com/mendersoftware/log"
import "io/ioutil"
import "os"
import "strings"
import "crypto/tls"
import "crypto/x509"

type runOptionsType struct {
	imageFile  string
	committing bool
	daemon     bool
	// Cert+privkey that authenticates this client
	clientCert tls.Certificate
	// Trusted server certificates
	trustedCerts x509.CertPool
	// hostname or address to bootstrap to
	bootstrap string
}

var errMsgNoArgumentsGiven error = errors.New("Must give either -rootfs or -commit or -bootstrap")
var errMsgIncompatibleLogOptions error = errors.New("One or more " +
	"incompatible log log options specified.")
var errMsgDaemonMixedWithOtherOptions error = errors.New("Daemon option can not " +
	"be mixed with other options.")

func CertPoolAppendCertsFromFile(s *x509.CertPool, f string) bool {
	cacert, err := ioutil.ReadFile(f)
	if err != nil {
		log.Warnln("Error reading file", f, err)
		return false
	}

	ret := s.AppendCertsFromPEM(cacert)
	return ret
}

func argsParse(args []string) (runOptionsType, error) {
	var runOptions runOptionsType

	parsing := flag.NewFlagSet("mender", flag.ContinueOnError)

	// FLAGS ---------------------------------------------------------------

	committing := parsing.Bool("commit", false, "Commit current update.")

	debug := parsing.Bool("debug", false, "Debug log level. This is a "+
		"shorthand for '-l debug'.")

	info := parsing.Bool("info", false, "Info log level. This is a "+
		"shorthand for '-l info'.")

	imageFile := parsing.String("rootfs", "",
		"Root filesystem URI to use for update. Can be either a local "+
			"file or a URL.")

	logLevel := parsing.String("log-level", "", "Log level, which can be "+
		"'debug', 'info', 'warning', 'error', 'fatal' or 'panic'. "+
		"Earlier log levels will also log the subsequent levels (so "+
		"'debug' will log everything). The default log level is "+
		"'warning'.")

	logModules := parsing.String("log-modules", "", "Filter logging by "+
		"module. This is a comma separated list of modules to log, "+
		"other modules will be omitted. To see which modules are "+
		"available, take a look at a non-filtered log and select "+
		"the modules appropriate for you.")

	noSyslog := parsing.Bool("no-syslog", false, "Disable logging to "+
		"syslog. Note that debug message are never logged to syslog.")

	logFile := parsing.String("log-file", "", "File to log to.")

	daemon := parsing.Bool("daemon", false, "Run as a daemon.")

	bootstrap := parsing.String("bootstrap", "",
		"Server to bootstrap to")

	certFile := parsing.String("certificate", "",
		"Client certificate")
	certKey := parsing.String("cert-key", "",
		"Client certificate's private key")
	serverCert := parsing.String("trusted-certs", "",
		"Trusted server certificates")

	// PARSING -------------------------------------------------------------

	if err := parsing.Parse(args); err != nil {
		return runOptions, err
	}

	// FLAG LOGIC ----------------------------------------------------------

	var logOptCount int = 0

	if *logLevel != "" {
		level, err := log.ParseLevel(*logLevel)
		if err != nil {
			return runOptions, err
		}
		log.SetLevel(level)
		logOptCount += 1
	}

	if *info {
		log.SetLevel(log.InfoLevel)
		logOptCount += 1
	}

	if *debug {
		log.SetLevel(log.DebugLevel)
		logOptCount += 1
	}

	if logOptCount > 1 {
		return runOptions, errMsgIncompatibleLogOptions
	} else if logOptCount == 0 {
		// Default log level.
		log.SetLevel(log.WarnLevel)
	}

	if *logFile != "" {
		fd, err := os.Create(*logFile)
		if err != nil {
			return runOptions, err
		}
		log.SetOutput(fd)
	}

	if *logModules != "" {
		modules := strings.Split(*logModules, ",")
		log.SetModuleFilter(modules)
	}

	if !*noSyslog {
		if err := log.AddSyslogHook(); err != nil {
			log.Warnf("Could not connect to syslog daemon: %s. "+
				"(use -no-syslog to disable completely)",
				err.Error())
		}
	}

	if *daemon && (*committing || *imageFile != "") {
		// Make sure that daemon switch is not passing together with
		// commit ot rootfs
		return runOptions, errMsgDaemonMixedWithOtherOptions
	}

	if *imageFile == "" && !*committing && !*daemon {
		return runOptions, errMsgNoArgumentsGiven
	}

	runOptions.imageFile = *imageFile
	runOptions.committing = *committing
	runOptions.daemon = *daemon
	runOptions.bootstrap = *bootstrap

	runOptions.trustedCerts = *x509.NewCertPool()
	if *serverCert != "" {
		CertPoolAppendCertsFromFile(&runOptions.trustedCerts, *serverCert)
	}

	numTrusted := len(runOptions.trustedCerts.Subjects())
	if *bootstrap != "" && numTrusted == 0 {
		log.Warnln("No server certificate is trusted," +
			" use -trusted-certs with a proper certificate")
	}

	var haveCert bool = false
	clientCert, err := tls.LoadX509KeyPair(*certFile, *certKey)
	if err != nil {
		log.Warnln("Failed to load certificate and key from files:",
			*certFile, *certKey)
	} else {
		runOptions.clientCert = clientCert
		haveCert = true
	}

	if *bootstrap != "" && !haveCert {
		log.Warnln("No client certificate is provided," +
			"use options -certificate and -cert-key")
	}

	return runOptions, nil
}

func doMain(args []string) error {
	runOptions, err := argsParse(args)
	if err != nil {
		return err
	}

	// run as a daemon
	if runOptions.daemon {
		if err := runAsDemon(); err != nil {
			return err
		}
	}

	if runOptions.imageFile != "" {
		if err := doRootfs(runOptions.imageFile); err != nil {
			return err
		}
	}
	if runOptions.committing {
		if err := doCommitRootfs(); err != nil {
			return err
		}
	}
	if runOptions.bootstrap != "" {
		err := doBootstrap(runOptions.bootstrap,
			runOptions.trustedCerts,
			runOptions.clientCert)
		return err
	}

	return nil
}

func runAsDemon() error {
	for {

	}
}

func main() {
	if err := doMain(os.Args[1:]); err != nil && err != flag.ErrHelp {
		log.Errorln(err.Error())
		os.Exit(1)
	}
}
<commit_msg>Do not mix authentication credentials and command line arguments.<commit_after>// Copyright 2016 Mender Software AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.
package main

import "errors"
import "flag"
import "github.com/mendersoftware/log"
import "io/ioutil"
import "os"
import "strings"
import "crypto/tls"
import "crypto/x509"

type authCredsType struct {
	// hostname or address to bootstrap to
	bootstrap string
	// Cert+privkey that authenticates this client
	clientCert tls.Certificate
	// Trusted server certificates
	trustedCerts x509.CertPool
}

type runOptionsType struct {
	imageFile  string
	committing bool
	daemon     bool
	auth       authCredsType
}

var errMsgNoArgumentsGiven error = errors.New("Must give either -rootfs or -commit or -bootstrap")
var errMsgIncompatibleLogOptions error = errors.New("One or more " +
	"incompatible log log options specified.")
var errMsgDaemonMixedWithOtherOptions error = errors.New("Daemon option can not " +
	"be mixed with other options.")

func CertPoolAppendCertsFromFile(s *x509.CertPool, f string) bool {
	cacert, err := ioutil.ReadFile(f)
	if err != nil {
		log.Warnln("Error reading file", f, err)
		return false
	}

	ret := s.AppendCertsFromPEM(cacert)
	return ret
}

func argsParse(args []string) (runOptionsType, error) {
	var runOptions runOptionsType
	var authCreds authCredsType

	parsing := flag.NewFlagSet("mender", flag.ContinueOnError)

	// FLAGS ---------------------------------------------------------------

	committing := parsing.Bool("commit", false, "Commit current update.")

	debug := parsing.Bool("debug", false, "Debug log level. This is a "+
		"shorthand for '-l debug'.")

	info := parsing.Bool("info", false, "Info log level. This is a "+
		"shorthand for '-l info'.")

	imageFile := parsing.String("rootfs", "",
		"Root filesystem URI to use for update. Can be either a local "+
			"file or a URL.")

	logLevel := parsing.String("log-level", "", "Log level, which can be "+
		"'debug', 'info', 'warning', 'error', 'fatal' or 'panic'. "+
		"Earlier log levels will also log the subsequent levels (so "+
		"'debug' will log everything). The default log level is "+
		"'warning'.")

	logModules := parsing.String("log-modules", "", "Filter logging by "+
		"module. This is a comma separated list of modules to log, "+
		"other modules will be omitted. To see which modules are "+
		"available, take a look at a non-filtered log and select "+
		"the modules appropriate for you.")

	noSyslog := parsing.Bool("no-syslog", false, "Disable logging to "+
		"syslog. Note that debug message are never logged to syslog.")

	logFile := parsing.String("log-file", "", "File to log to.")

	daemon := parsing.Bool("daemon", false, "Run as a daemon.")

	bootstrap := parsing.String("bootstrap", "",
		"Server to bootstrap to")

	certFile := parsing.String("certificate", "",
		"Client certificate")
	certKey := parsing.String("cert-key", "",
		"Client certificate's private key")
	serverCert := parsing.String("trusted-certs", "",
		"Trusted server certificates")

	// PARSING -------------------------------------------------------------

	if err := parsing.Parse(args); err != nil {
		return runOptions, err
	}

	// FLAG LOGIC ----------------------------------------------------------

	var logOptCount int = 0

	if *logLevel != "" {
		level, err := log.ParseLevel(*logLevel)
		if err != nil {
			return runOptions, err
		}
		log.SetLevel(level)
		logOptCount += 1
	}

	if *info {
		log.SetLevel(log.InfoLevel)
		logOptCount += 1
	}

	if *debug {
		log.SetLevel(log.DebugLevel)
		logOptCount += 1
	}

	if logOptCount > 1 {
		return runOptions, errMsgIncompatibleLogOptions
	} else if logOptCount == 0 {
		// Default log level.
		log.SetLevel(log.WarnLevel)
	}

	if *logFile != "" {
		fd, err := os.Create(*logFile)
		if err != nil {
			return runOptions, err
		}
		log.SetOutput(fd)
	}

	if *logModules != "" {
		modules := strings.Split(*logModules, ",")
		log.SetModuleFilter(modules)
	}

	if !*noSyslog {
		if err := log.AddSyslogHook(); err != nil {
			log.Warnf("Could not connect to syslog daemon: %s. "+
				"(use -no-syslog to disable completely)",
				err.Error())
		}
	}

	if *daemon && (*committing || *imageFile != "") {
		// Make sure that daemon switch is not passing together with
		// commit ot rootfs
		return runOptions, errMsgDaemonMixedWithOtherOptions
	}

	if *imageFile == "" && !*committing && !*daemon {
		return runOptions, errMsgNoArgumentsGiven
	}

	runOptions.imageFile = *imageFile
	runOptions.committing = *committing
	runOptions.daemon = *daemon
	authCreds.bootstrap = *bootstrap

	authCreds.trustedCerts = *x509.NewCertPool()
	if *serverCert != "" {
		CertPoolAppendCertsFromFile(&authCreds.trustedCerts, *serverCert)
	}

	numTrusted := len(authCreds.trustedCerts.Subjects())
	if *bootstrap != "" && numTrusted == 0 {
		log.Warnln("No server certificate is trusted," +
			" use -trusted-certs with a proper certificate")
	}

	var haveCert bool = false
	clientCert, err := tls.LoadX509KeyPair(*certFile, *certKey)
	if err != nil {
		log.Warnln("Failed to load certificate and key from files:",
			*certFile, *certKey)
	} else {
		authCreds.clientCert = clientCert
		haveCert = true
	}

	if *bootstrap != "" && !haveCert {
		log.Warnln("No client certificate is provided," +
			"use options -certificate and -cert-key")
	}

	return runOptions, nil
}

func doMain(args []string) error {
	runOptions, err := argsParse(args)
	if err != nil {
		return err
	}

	// run as a daemon
	if runOptions.daemon {
		if err := runAsDemon(); err != nil {
			return err
		}
	}

	if runOptions.imageFile != "" {
		if err := doRootfs(runOptions.imageFile); err != nil {
			return err
		}
	}
	if runOptions.committing {
		if err := doCommitRootfs(); err != nil {
			return err
		}
	}
	if runOptions.auth.bootstrap != "" {
		err := doBootstrap(runOptions.auth.bootstrap,
			runOptions.auth.trustedCerts,
			runOptions.auth.clientCert)
		return err
	}

	return nil
}

func runAsDemon() error {
	for {

	}
}

func main() {
	if err := doMain(os.Args[1:]); err != nil && err != flag.ErrHelp {
		log.Errorln(err.Error())
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/md5"
	"crypto/sha1"
	"encoding/hex"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"os"
	"strings"
)

func main() {
	var (
		input  string
		datset string
	)

	flag.StringVar(&input, "f", "", "input file")
	flag.StringVar(&datset, "d", "", "datset")
	flag.Parse()

	if input == "" || datset == "" {
		flag.Usage()
		os.Exit(1)
	}

	var err error

	in, err := os.Open(input)
	if err != nil {
		panic(err)
	}
	defer in.Close()

	h1, h2 := md5.New(), sha1.New()

	io.Copy(io.MultiWriter(h1, h2), in)

	md5hash := strings.ToUpper(hex.EncodeToString(h1.Sum(nil)))
	sha1hash := strings.ToUpper(hex.EncodeToString(h2.Sum(nil)))

	f, err := os.Open(datset)
	if err != nil {
		panic(err)
	}
	defer f.Close()

	p := NewParser(f)

	col, err := p.Parse()
	if err != nil {
		panic(err)
	}

	for _, g := range col.Games {
		if g.ROM.MD5 != md5hash && g.ROM.SHA1 != sha1hash {
			continue
		}

		b, err := json.MarshalIndent(g, "", "  ")
		if err != nil {
			panic(err)
		}

		fmt.Printf("Found matching ROM:\n%s\n", b)
	}
}
<commit_msg>Clean up output<commit_after>package main

import (
	"crypto/md5"
	"crypto/sha1"
	"encoding/hex"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"os"
	"strings"
)

func main() {
	var (
		input  string
		datset string
	)

	flag.StringVar(&input, "f", "", "input file")
	flag.StringVar(&datset, "d", "", "datset")
	flag.Parse()

	if input == "" || datset == "" {
		flag.Usage()
		os.Exit(1)
	}

	var err error

	in, err := os.Open(input)
	if err != nil {
		panic(err)
	}
	defer in.Close()

	h1, h2 := md5.New(), sha1.New()

	io.Copy(io.MultiWriter(h1, h2), in)

	md5hash := strings.ToUpper(hex.EncodeToString(h1.Sum(nil)))
	sha1hash := strings.ToUpper(hex.EncodeToString(h2.Sum(nil)))

	f, err := os.Open(datset)
	if err != nil {
		panic(err)
	}
	defer f.Close()

	p := NewParser(f)

	col, err := p.Parse()
	if err != nil {
		panic(err)
	}

	for _, g := range col.Games {
		if g.ROM.MD5 != md5hash && g.ROM.SHA1 != sha1hash {
			continue
		}

		b, err := json.MarshalIndent(g, "", "  ")
		if err != nil {
			panic(err)
		}

		fmt.Println(string(b))
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"

	"github.com/jessevdk/go-flags"
	"github.com/yuya-takeyama/argf"
)

func usage() {
	os.Stderr.WriteString(`
Usage: cspv OPTION... [FILE]...
Print selected parts of CSV from each FILE to standard output.

Options:
  -i, --indexes=LIST       select only these indexes
  -h, --headers=LIST       select only these headers
      --help               display this help text and exit
      --version            display version information and exit
`[1:])
}

func version() {
	os.Stderr.WriteString(`
v0.2.0
`[1:])
}

type Option struct {
	IndexesList string `short:"i" long:"indexes"`
	HeadersList string `short:"h" long:"headers"`
	IsHelp      bool   `          long:"help"`
	IsVersion   bool   `          long:"version"`
	Files       []string
}

func parseOption(args []string) (opt *Option, err error) {
	opt = &Option{}
	flag := flags.NewParser(opt, flags.PassDoubleDash)

	opt.Files, err = flag.ParseArgs(args)
	if err != nil && !opt.IsHelp && !opt.IsVersion {
		return nil, err
	}
	return opt, nil
}

func newCSVScannerFromOption(opt *Option) (c *CSVScanner, err error) {
	var selector Selector
	switch {
	case opt.IndexesList == "" && opt.HeadersList == "":
		return nil, fmt.Errorf("you must specify a list of indexes or headers")
	case opt.IndexesList != "" && opt.HeadersList != "":
		return nil, fmt.Errorf("only one type of list may be specified")
	case opt.IndexesList != "":
		indexes, err := parseIndexesList(opt.IndexesList)
		if err != nil {
			return nil, err
		}
		selector = NewIndexes(indexes)
	case opt.HeadersList != "":
		headers, err := parseHeadersList(opt.HeadersList)
		if err != nil {
			return nil, err
		}
		selector = NewHeaders(headers)
	}
	reader, err := argf.From(opt.Files)
	if err != nil {
		return nil, err
	}
	return NewCSVScanner(selector, reader), nil
}

func do(c *CSVScanner) error {
	for c.Scan() {
		fmt.Println(c.Text())
	}
	return c.Err()
}

func printErr(err error) {
	fmt.Fprintln(os.Stderr, "cspv:", err)
}

func guideToHelp() {
	os.Stderr.WriteString(`
Try 'cspv --help' for more information.
`[1:])
}

func _main() int {
	opt, err := parseOption(os.Args[1:])
	if err != nil {
		printErr(err)
		guideToHelp()
		return 2
	}
	switch {
	case opt.IsHelp:
		usage()
		return 0
	case opt.IsVersion:
		version()
		return 0
	}

	c, err := newCSVScannerFromOption(opt)
	if err != nil {
		printErr(err)
		guideToHelp()
		return 2
	}
	if err = do(c); err != nil {
		printErr(err)
		return 1
	}
	return 0
}

func main() {
	e := _main()
	os.Exit(e)
}
<commit_msg>Support --delimiter<commit_after>package main

import (
	"fmt"
	"os"

	"github.com/jessevdk/go-flags"
	"github.com/yuya-takeyama/argf"
)

func usage() {
	os.Stderr.WriteString(`
Usage: cspv OPTION... [FILE]...
Print selected parts of CSV from each FILE to standard output.

Options:
  -i, --indexes=LIST       select only these indexes
  -h, --headers=LIST       select only these headers
  -d, --delimiter=STRING   use STRING as the output delimiter (default: \t)
      --help               display this help text and exit
      --version            display version information and exit
`[1:])
}

func version() {
	os.Stderr.WriteString(`
v0.2.0
`[1:])
}

type Option struct {
	IndexesList string `short:"i" long:"indexes"`
	HeadersList string `short:"h" long:"headers"`
	Delimiter   string `short:"d" long:"delimiter" default:"\t"`
	IsHelp      bool   `          long:"help"`
	IsVersion   bool   `          long:"version"`
	Files       []string
}

func parseOption(args []string) (opt *Option, err error) {
	opt = &Option{}
	flag := flags.NewParser(opt, flags.PassDoubleDash)

	opt.Files, err = flag.ParseArgs(args)
	if err != nil && !opt.IsHelp && !opt.IsVersion {
		return nil, err
	}
	return opt, nil
}

func newCSVScannerFromOption(opt *Option) (c *CSVScanner, err error) {
	var selector Selector
	switch {
	case opt.IndexesList == "" && opt.HeadersList == "":
		return nil, fmt.Errorf("you must specify a list of indexes or headers")
	case opt.IndexesList != "" && opt.HeadersList != "":
		return nil, fmt.Errorf("only one type of list may be specified")
	case opt.IndexesList != "":
		indexes, err := parseIndexesList(opt.IndexesList)
		if err != nil {
			return nil, err
		}
		selector = NewIndexes(indexes)
	case opt.HeadersList != "":
		headers, err := parseHeadersList(opt.HeadersList)
		if err != nil {
			return nil, err
		}
		selector = NewHeaders(headers)
	}
	reader, err := argf.From(opt.Files)
	if err != nil {
		return nil, err
	}
	c = NewCSVScanner(selector, reader)
	c.Delimiter = opt.Delimiter
	return c, nil
}

func do(c *CSVScanner) error {
	for c.Scan() {
		fmt.Println(c.Text())
	}
	return c.Err()
}

func printErr(err error) {
	fmt.Fprintln(os.Stderr, "cspv:", err)
}

func guideToHelp() {
	os.Stderr.WriteString(`
Try 'cspv --help' for more information.
`[1:])
}

func _main() int {
	opt, err := parseOption(os.Args[1:])
	if err != nil {
		printErr(err)
		guideToHelp()
		return 2
	}
	switch {
	case opt.IsHelp:
		usage()
		return 0
	case opt.IsVersion:
		version()
		return 0
	}

	c, err := newCSVScannerFromOption(opt)
	if err != nil {
		printErr(err)
		guideToHelp()
		return 2
	}
	if err = do(c); err != nil {
		printErr(err)
		return 1
	}
	return 0
}

func main() {
	e := _main()
	os.Exit(e)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"runtime"

	"github.com/alphagov/govuk_crawler_worker/queue"
	"github.com/alphagov/govuk_crawler_worker/ttl_hash_set"
	"github.com/streadway/amqp"
)

type CrawlerMessageItem struct {
	amqp.Delivery
	HTMLBody []byte
}

func NewCrawlerMessageItem(delivery amqp.Delivery) *CrawlerMessageItem {
	return &CrawlerMessageItem{Delivery: delivery}
}

var (
	amqpAddr       = getEnvDefault("AMQP_ADDRESS", "amqp://guest:guest@localhost:5672/")
	exchangeName   = getEnvDefault("AMQP_EXCHANGE", "govuk_crawler_exchange")
	queueName      = getEnvDefault("AMQP_MESSAGE_QUEUE", "govuk_crawler_queue")
	redisAddr      = getEnvDefault("REDIS_ADDRESS", "127.0.0.1:6379")
	redisKeyPrefix = getEnvDefault("REDIS_KEY_PREFIX", "govuk_crawler_worker")
	rootURL        = getEnvDefault("ROOT_URL", "https://www.gov.uk/")
)

func main() {
	if os.Getenv("GOMAXPROCS") == "" {
		// Use all available cores if not otherwise specified
		runtime.GOMAXPROCS(runtime.NumCPU())
	}
	log.Println(fmt.Sprintf("using GOMAXPROCS value of %d", runtime.NumCPU()))

	ttlHashSet, err := ttl_hash_set.NewTTLHashSet(redisKeyPrefix, redisAddr)
	if err != nil {
		log.Fatal(err)
	}
	defer ttlHashSet.Close()
	log.Println("Connected to Redis service:", ttlHashSet)

	queueManager, err := queue.NewQueueManager(amqpAddr, exchangeName, queueName)
	if err != nil {
		log.Fatal(err)
	}
	defer queueManager.Close()
	log.Println("Connected to AMQP service:", queueManager)

	log.Fatal("Nothing to see here yet.")
}

func getEnvDefault(key string, defaultVal string) string {
	val := os.Getenv(key)
	if val == "" {
		return defaultVal
	}

	return val
}
<commit_msg>Move CrawlerMessageItem out of main.go into it's own file<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"runtime"

	"github.com/alphagov/govuk_crawler_worker/http_crawler"
	"github.com/alphagov/govuk_crawler_worker/queue"
	"github.com/alphagov/govuk_crawler_worker/ttl_hash_set"
	"github.com/streadway/amqp"
)

var (
	amqpAddr       = getEnvDefault("AMQP_ADDRESS", "amqp://guest:guest@localhost:5672/")
	exchangeName   = getEnvDefault("AMQP_EXCHANGE", "govuk_crawler_exchange")
	queueName      = getEnvDefault("AMQP_MESSAGE_QUEUE", "govuk_crawler_queue")
	redisAddr      = getEnvDefault("REDIS_ADDRESS", "127.0.0.1:6379")
	redisKeyPrefix = getEnvDefault("REDIS_KEY_PREFIX", "govuk_crawler_worker")
	rootURL        = getEnvDefault("ROOT_URL", "https://www.gov.uk/")
)

func main() {
	if os.Getenv("GOMAXPROCS") == "" {
		// Use all available cores if not otherwise specified
		runtime.GOMAXPROCS(runtime.NumCPU())
	}
	log.Println(fmt.Sprintf("using GOMAXPROCS value of %d", runtime.NumCPU()))

	ttlHashSet, err := ttl_hash_set.NewTTLHashSet(redisKeyPrefix, redisAddr)
	if err != nil {
		log.Fatal(err)
	}
	defer ttlHashSet.Close()
	log.Println("Connected to Redis service:", ttlHashSet)

	queueManager, err := queue.NewQueueManager(amqpAddr, exchangeName, queueName)
	if err != nil {
		log.Fatal(err)
	}
	defer queueManager.Close()
	log.Println("Connected to AMQP service:", queueManager)

	log.Fatal("Nothing to see here yet.")
}

func getEnvDefault(key string, defaultVal string) string {
	val := os.Getenv(key)
	if val == "" {
		return defaultVal
	}

	return val
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/dokterbob/ipfs-search/crawler"
	"github.com/dokterbob/ipfs-search/indexer"
	"github.com/dokterbob/ipfs-search/queue"
	"gopkg.in/ipfs/go-ipfs-api.v1"
	"gopkg.in/olivere/elastic.v3"
	"gopkg.in/urfave/cli.v1"
	"log"
	"os"
	"time"
)

const (
	IPFS_API     = "localhost:5001"
	HASH_WORKERS = 40
	FILE_WORKERS = 0
	TIMEOUT      = 60 * time.Duration(time.Second)
	HASH_WAIT    = time.Duration(time.Second)
	FILE_WAIT    = HASH_WAIT
)

func main() {
	// Prefix logging with filename and line number: "d.go:23"
	// log.SetFlags(log.Lshortfile)

	// Logging w/o prefix
	log.SetFlags(0)

	app := cli.NewApp()
	app.Name = "ipfs-search"
	app.Usage = "IPFS search engine."

	app.Commands = []cli.Command{
		{
			Name:    "add",
			Aliases: []string{"a"},
			Usage:   "add `HASH` to crawler queue",
			Action:  add,
		},
		{
			Name:    "crawl",
			Aliases: []string{"c"},
			Usage:   "start crawler",
			Action:  crawl,
		},
	}

	app.Run(os.Args)
}

func get_elastic() (*elastic.Client, error) {
	el, err := elastic.NewClient()
	if err != nil {
		return nil, err
	}
	exists, err := el.IndexExists("ipfs").Do()
	if err != nil {
		return nil, err
	}
	if !exists {
		// Index does not exist yet, create
		el.CreateIndex("ipfs")
	}

	return el, nil
}

func add(c *cli.Context) error {
	if c.NArg() != 1 {
		return cli.NewExitError("Please supply one hash as argument.", 1)
	}

	hash := c.Args().Get(0)

	fmt.Printf("Adding hash '%s' to queue\n", hash)

	ch, err := queue.NewChannel()
	if err != nil {
		return cli.NewExitError(err.Error(), 1)
	}
	defer ch.Close()

	queue, err := queue.NewTaskQueue(ch, "hashes")
	if err != nil {
		return cli.NewExitError(err.Error(), 1)
	}

	err = queue.AddTask(map[string]interface{}{
		"hash": hash,
	})

	if err != nil {
		return cli.NewExitError(err.Error(), 1)
	}

	return nil
}

func crawl(c *cli.Context) error {
	// For now, assume gateway running on default host:port
	sh := shell.NewShell(IPFS_API)

	// Set 1 minute timeout on IPFS requests
	sh.SetTimeout(TIMEOUT)

	el, err := get_elastic()
	if err != nil {
		return cli.NewExitError(err.Error(), 1)
	}

	add_ch, err := queue.NewChannel()
	if err != nil {
		return cli.NewExitError(err.Error(), 1)
	}
	defer add_ch.Close()

	hq, err := queue.NewTaskQueue(add_ch, "hashes")
	if err != nil {
		return cli.NewExitError(err.Error(), 1)
	}

	fq, err := queue.NewTaskQueue(add_ch, "files")
	if err != nil {
		return cli.NewExitError(err.Error(), 1)
	}

	id := indexer.NewIndexer(el)

	crawli := crawler.NewCrawler(sh, id, fq, hq)

	errc := make(chan error, 1)

	for i := 0; i < HASH_WORKERS; i++ {
		// Now create queues and channel for workers
		ch, err := queue.NewChannel()
		if err != nil {
			return cli.NewExitError(err.Error(), 1)
		}
		defer ch.Close()

		hq, err := queue.NewTaskQueue(ch, "hashes")
		if err != nil {
			return cli.NewExitError(err.Error(), 1)
		}

		hq.StartConsumer(func(params interface{}) error {
			args := params.(*crawler.CrawlerArgs)

			return crawli.CrawlHash(
				args.Hash,
				args.Name,
				args.ParentHash,
				args.ParentName,
			)
		}, &crawler.CrawlerArgs{}, errc, true, add_ch)

		// Start workers timeout/hash time apart
		time.Sleep(HASH_WAIT)
	}

	for i := 0; i < FILE_WORKERS; i++ {
		ch, err := queue.NewChannel()
		if err != nil {
			return cli.NewExitError(err.Error(), 1)
		}
		defer ch.Close()

		fq, err := queue.NewTaskQueue(ch, "files")
		if err != nil {
			return cli.NewExitError(err.Error(), 1)
		}

		fq.StartConsumer(func(params interface{}) error {
			args := params.(*crawler.CrawlerArgs)

			return crawli.CrawlFile(
				args.Hash,
				args.Name,
				args.ParentHash,
				args.ParentName,
				args.Size,
			)
		}, &crawler.CrawlerArgs{}, errc, true, add_ch)

		// Start workers timeout/hash time apart
		time.Sleep(FILE_WAIT)
	}

	// sigs := make(chan os.Signal, 1)
	// signal.Notify(sigs, syscall.SIGQUIT)

	log.Printf(" [*] Waiting for messages. To exit press CTRL+C")

	for {
		select {
		case err = <-errc:
			log.Printf("%T: %v", err, err)
		}
	}

	// No error
	return nil
}
<commit_msg>IPFS 0.4.3rc4 much faster<commit_after>package main

import (
	"fmt"
	"github.com/dokterbob/ipfs-search/crawler"
	"github.com/dokterbob/ipfs-search/indexer"
	"github.com/dokterbob/ipfs-search/queue"
	"gopkg.in/ipfs/go-ipfs-api.v1"
	"gopkg.in/olivere/elastic.v3"
	"gopkg.in/urfave/cli.v1"
	"log"
	"os"
	"time"
)

const (
	IPFS_API     = "localhost:5001"
	HASH_WORKERS = 200
	FILE_WORKERS = 0
	TIMEOUT      = 120 * time.Duration(time.Second)
	HASH_WAIT    = time.Duration(time.Second)
	FILE_WAIT    = HASH_WAIT
)

func main() {
	// Prefix logging with filename and line number: "d.go:23"
	// log.SetFlags(log.Lshortfile)

	// Logging w/o prefix
	log.SetFlags(0)

	app := cli.NewApp()
	app.Name = "ipfs-search"
	app.Usage = "IPFS search engine."

	app.Commands = []cli.Command{
		{
			Name:    "add",
			Aliases: []string{"a"},
			Usage:   "add `HASH` to crawler queue",
			Action:  add,
		},
		{
			Name:    "crawl",
			Aliases: []string{"c"},
			Usage:   "start crawler",
			Action:  crawl,
		},
	}

	app.Run(os.Args)
}

func get_elastic() (*elastic.Client, error) {
	el, err := elastic.NewClient()
	if err != nil {
		return nil, err
	}
	exists, err := el.IndexExists("ipfs").Do()
	if err != nil {
		return nil, err
	}
	if !exists {
		// Index does not exist yet, create
		el.CreateIndex("ipfs")
	}

	return el, nil
}

func add(c *cli.Context) error {
	if c.NArg() != 1 {
		return cli.NewExitError("Please supply one hash as argument.", 1)
	}

	hash := c.Args().Get(0)

	fmt.Printf("Adding hash '%s' to queue\n", hash)

	ch, err := queue.NewChannel()
	if err != nil {
		return cli.NewExitError(err.Error(), 1)
	}
	defer ch.Close()

	queue, err := queue.NewTaskQueue(ch, "hashes")
	if err != nil {
		return cli.NewExitError(err.Error(), 1)
	}

	err = queue.AddTask(map[string]interface{}{
		"hash": hash,
	})

	if err != nil {
		return cli.NewExitError(err.Error(), 1)
	}

	return nil
}

func crawl(c *cli.Context) error {
	// For now, assume gateway running on default host:port
	sh := shell.NewShell(IPFS_API)

	// Set 1 minute timeout on IPFS requests
	sh.SetTimeout(TIMEOUT)

	el, err := get_elastic()
	if err != nil {
		return cli.NewExitError(err.Error(), 1)
	}

	add_ch, err := queue.NewChannel()
	if err != nil {
		return cli.NewExitError(err.Error(), 1)
	}
	defer add_ch.Close()

	hq, err := queue.NewTaskQueue(add_ch, "hashes")
	if err != nil {
		return cli.NewExitError(err.Error(), 1)
	}

	fq, err := queue.NewTaskQueue(add_ch, "files")
	if err != nil {
		return cli.NewExitError(err.Error(), 1)
	}

	id := indexer.NewIndexer(el)

	crawli := crawler.NewCrawler(sh, id, fq, hq)

	errc := make(chan error, 1)

	for i := 0; i < HASH_WORKERS; i++ {
		// Now create queues and channel for workers
		ch, err := queue.NewChannel()
		if err != nil {
			return cli.NewExitError(err.Error(), 1)
		}
		defer ch.Close()

		hq, err := queue.NewTaskQueue(ch, "hashes")
		if err != nil {
			return cli.NewExitError(err.Error(), 1)
		}

		hq.StartConsumer(func(params interface{}) error {
			args := params.(*crawler.CrawlerArgs)

			return crawli.CrawlHash(
				args.Hash,
				args.Name,
				args.ParentHash,
				args.ParentName,
			)
		}, &crawler.CrawlerArgs{}, errc, true, add_ch)

		// Start workers timeout/hash time apart
		time.Sleep(HASH_WAIT)
	}

	for i := 0; i < FILE_WORKERS; i++ {
		ch, err := queue.NewChannel()
		if err != nil {
			return cli.NewExitError(err.Error(), 1)
		}
		defer ch.Close()

		fq, err := queue.NewTaskQueue(ch, "files")
		if err != nil {
			return cli.NewExitError(err.Error(), 1)
		}

		fq.StartConsumer(func(params interface{}) error {
			args := params.(*crawler.CrawlerArgs)

			return crawli.CrawlFile(
				args.Hash,
				args.Name,
				args.ParentHash,
				args.ParentName,
				args.Size,
			)
		}, &crawler.CrawlerArgs{}, errc, true, add_ch)

		// Start workers timeout/hash time apart
		time.Sleep(FILE_WAIT)
	}

	// sigs := make(chan os.Signal, 1)
	// signal.Notify(sigs, syscall.SIGQUIT)

	log.Printf(" [*] Waiting for messages. To exit press CTRL+C")

	for {
		select {
		case err = <-errc:
			log.Printf("%T: %v", err, err)
		}
	}

	// No error
	return nil
}
<|endoftext|>


<commit_before>package main

import (
    "bufio"
    "flag"
    "fmt"
    "log"
    "milo/utils"
    "os"
    "path/filepath"
    "regexp"
    "runtime"
    "strings"
)

var root string
var extensions []string
var pattern *regexp.Regexp

// getNames creates a filepath.WalkFunc suitable for passing to
// filepath.Walk which passes the filenames found into a channel.
func getNames(c chan string) filepath.WalkFunc {
    return func(path string, info os.FileInfo, err error) error {
        if !info.Mode().IsRegular() {
            return nil
        }
        if len(extensions) > 0 {
            for _, ext := range extensions {
                if filepath.Ext(path) == ext {
                    c <- path
                    return nil
                }
            }
            return nil
        }
        c <- path
        return nil
    }
}

// init parses the command-line arguments into the values
// used to execute. There should be a pattern at the very
// least. Optionally, a path (defaulting to "."), and
// file extensions to search may be provided.
func init() {
    flag.Parse()
    args := flag.Args()
    if len(args) == 0 {
        log.Fatalf("No arguments passed.")
    }
    args = getExts(args)
    args = getRoot(args)
    if len(args) != 1 {
        log.Fatalf("Unable to find pattern.\n")
    }
    p, err := regexp.Compile(args[0])
    if err != nil {
        log.Fatal(err)
    }
    pattern = p
    runtime.GOMAXPROCS(runtime.NumCPU())
}

// getExts sets the extensions global variable,
// removes any extension arguments from args,
// and returns args for further processing.
func getExts(args []string) []string {
    var unused []string
    for _, val := range args {
        if strings.HasPrefix(val, "--") {
            if len(val) < 3 {
                log.Fatalf("Invalid extension: '%s'\n", val)
            }
            extensions = append(extensions, "."+val[2:])
        } else {
            unused = append(unused, val)
        }
    }
    return unused
}

// getRoot finds a valid directory in the command-line
// args, sets it to the global "root" variable, and
// returns the remaining arguments.
func getRoot(args []string) []string {
    var unused []string
    for _, val := range args {
        if utils.IsDir(val) {
            if root != "" {
                log.Fatalf("Too many directory arguments\n")
            } else {
                root = val
            }
        } else {
            unused = append(unused, val)
        }
    }
    if root == "" {
        root = "."
    }
    return unused
}

func main() {
    filenames := make(chan string, 3333)
    results := make(chan bool, 3333)
    f := getNames(filenames)
    go func() {
        filepath.Walk(root, f)
        close(filenames)
    }()
    count := 0
    for filename := range filenames {
        count += 1
        go checkFile(filename, results)
    }
    for count > 0 {
        <-results
        count--
    }
}

// checkFile takes a filename and reads the file to determine
// whether the file contains the regex in the global pattern.
func checkFile(filename string, done chan bool) {
    file, err := os.Open(filename)
    if err != nil {
        log.Println(err)
        done <- true
        return
    }
    defer file.Close()
    scanner := bufio.NewScanner(file)
    line := 0
    for scanner.Scan() {
        line += 1
        found := pattern.FindIndex(scanner.Bytes())
        if found != nil {
            fmt.Printf("%s:%d: %s\n", filename, line, scanner.Text())
        }
    }
    done <- true
}
<commit_msg>stop opening too many files at once<commit_after>package main

import (
    "bufio"
    "flag"
    "fmt"
    "log"
    "milo/utils"
    "os"
    "path/filepath"
    "regexp"
    "runtime"
    "strings"
)

const buffer = 500

var root string
var extensions []string
var pattern *regexp.Regexp
var cpus int

// getNames creates a filepath.WalkFunc suitable for passing to
// filepath.Walk which passes the filenames found into a channel.
func getNames(c chan string) filepath.WalkFunc {
    return func(path string, info os.FileInfo, err error) error {
        if !info.Mode().IsRegular() {
            return nil
        }
        if len(extensions) > 0 {
            for _, ext := range extensions {
                if filepath.Ext(path) == ext {
                    c <- path
                    return nil
                }
            }
            return nil
        }
        c <- path
        return nil
    }
}

// init parses the command-line arguments into the values
// used to execute. There should be a pattern at the very
// least. Optionally, a path (defaulting to "."), and
// file extensions to search may be provided.
func init() {
    flag.Parse()
    args := flag.Args()
    if len(args) == 0 {
        log.Fatalf("No arguments passed.")
    }
    args = getExts(args)
    args = getRoot(args)
    if len(args) != 1 {
        log.Fatalf("Unable to find pattern.\n")
    }
    p, err := regexp.Compile(args[0])
    if err != nil {
        log.Fatal(err)
    }
    pattern = p
    cpus = runtime.NumCPU()
    runtime.GOMAXPROCS(cpus)
}

// getExts sets the extensions global variable,
// removes any extension arguments from args,
// and returns args for further processing.
func getExts(args []string) []string {
    var unused []string
    for _, val := range args {
        if strings.HasPrefix(val, "--") {
            if len(val) < 3 {
                log.Fatalf("Invalid extension: '%s'\n", val)
            }
            extensions = append(extensions, "."+val[2:])
        } else {
            unused = append(unused, val)
        }
    }
    return unused
}

// getRoot finds a valid directory in the command-line
// args, sets it to the global "root" variable, and
// returns the remaining arguments.
func getRoot(args []string) []string {
    var unused []string
    for _, val := range args {
        if utils.IsDir(val) {
            if root != "" {
                log.Fatalf("Too many directory arguments\n")
            } else {
                root = val
            }
        } else {
            unused = append(unused, val)
        }
    }
    if root == "" {
        root = "."
    }
    return unused
}

func main() {
    filenames := make(chan string, buffer)
    toProcess := make(chan string, buffer)
    results := make(chan bool, buffer)
    walkFunc := getNames(filenames)
    go func() {
        filepath.Walk(root, walkFunc)
        close(filenames)
    }()
    for i := 0; i < cpus; i++ {
        go feedCheckFile(toProcess, results)
    }
    go func() {
        for filename := range filenames {
            toProcess <- filename
        }
        close(toProcess)
    }()
    for i := 0; i < cpus; i++ {
        <-results
    }
}

// feedCheckFile receives a channel of strings and a results
// channel (of bool) and calls checkFile in a loop. It would be
// easier to rewrite checkFile to just add a simple loop, but this
// allows the checkFile code to remain simpler in case I decide
// to use it differently later.
func feedCheckFile(filenames chan string, results chan bool) {
    for val := range(filenames) {
        checkFile(val)
    }
    results <- true
}

// checkFile takes a filename and reads the file to determine
// whether the file contains the regex in the global pattern.
func checkFile(filename string) {
    file, err := os.Open(filename)
    if err != nil {
        log.Println(err)
        return
    }
    defer file.Close()
    scanner := bufio.NewScanner(file)
    line := 0
    for scanner.Scan() {
        line += 1
        found := pattern.FindIndex(scanner.Bytes())
        if found != nil {
            fmt.Printf("%s:%d: %s\n", filename, line, scanner.Text())
        }
    }
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"encoding/base64"
	"io/ioutil"
	"log"
	"net/http"
	"time"

	"github.com/caarlos0/env"
	sqlite "github.com/mattn/go-sqlite3"
)

const (
	pixelRaw   = "R0lGODlhAQABAIAAANvf7wAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw=="
	pixelRoute = "/orkka.gif"
	jsRoute    = "/a.js"
)

type config struct {
	Port                  string `env:"PORT"             envDefault:"8080"`
	DBFile                string `env:"DB_FILE"          envDefault:"events.db"`
	EventQueueSize        int    `env:"MAX_CONNECTIONS"  envDefault:"4096"`
	WriteQueueDefaultSize int    `env:"WRITE_QUEUE_SIZE" envDefault:"1024"`
	WriteFrequencyMillis  int    `env:"WRITE_FREQUENCY"  envDefault:"100"`
	JSFile                string `env:"JSFile"           envDefault:"analytics.js"`
}

func main() {
	var config config
	if err := env.Parse(&config); err != nil {
		log.Fatal(err)
	}

	eventQueue := make(chan *pageEvent, config.EventQueueSize)

	go processEventWriteQueue(&config, eventQueue)
	http.HandleFunc(pixelRoute, makeHandlePixel(eventQueue))
	http.HandleFunc(jsRoute, makeHandleScript(config.JSFile))

	log.Fatal(http.ListenAndServe(":"+config.Port, nil))
}

func makeHandleScript(file string) http.HandlerFunc {
	javascript, err := ioutil.ReadFile(file)
	if err != nil {
		log.Fatal("Failed to read javascript file.", err)
	}

	return func(out http.ResponseWriter, req *http.Request) {
		out.Header().Set("Content-Type", "text/javascript")
		out.Header().Set("Cache-Control", "public, max-age=86400")
		out.WriteHeader(http.StatusOK)
		out.Write(javascript)
	}
}

func makeHandlePixel(eventQueue chan *pageEvent) http.HandlerFunc {
	pixel, err := base64.StdEncoding.DecodeString(pixelRaw)
	if err != nil {
		log.Fatalf("failed to encode pixel: %v", err)
	}

	return func(out http.ResponseWriter, req *http.Request) {
		eventQueue <- pageEventFromRequest(req)
		out.Header().Set("Content-Type", "image/gif")
		out.WriteHeader(http.StatusOK)
		out.Write(pixel)
	}
}

func processEventWriteQueue(config *config, eventQueue chan *pageEvent) {
	db := initDB(config)
	defer db.Close()

	writeFrequency := time.Duration(time.Duration(config.WriteFrequencyMillis) * time.Millisecond)
	writeTimer := time.NewTimer(writeFrequency)

	writeQueue := make([]*pageEvent, 0, config.WriteQueueDefaultSize)

	for {
		select {
		case ev := <-eventQueue:
			writeQueue = append(writeQueue, ev)

		case <-writeTimer.C:
			if len(writeQueue) > 0 {
				writeTimer.Stop()
				writeEvents(db, writeQueue)
				writeQueue = writeQueue[:0]
			}
			writeTimer.Reset(writeFrequency)
		}
	}
}

func writeEvents(db *sql.DB, events []*pageEvent) {
	var ev *pageEvent
	i := -1
	db.Exec("BEGIN TRANSACTION;")
	for i, ev = range events {
		writePageEventToDB(db, ev)
	}
	db.Exec("END TRANSACTION;")

	log.Println("wrote", i+1, "records.")
}

func initDB(config *config) *sql.DB {
	sql.Register("sqlite3_custom", &sqlite.SQLiteDriver{
		ConnectHook: registerSQLiteExtensions,
	})

	db, err := sql.Open("sqlite3_custom", config.DBFile)

	if err != nil {
		log.Fatal(err)
	}
	if db == nil {
		log.Fatal("database is null")
	}

	if _, err = db.Exec(sqlCreatePageEventTable); err != nil {
		log.Fatal(err)
	}

	return db
}

const (
	pageEventSchemaVersion  = 1
	sqlCreatePageEventTable = `
		CREATE TABLE IF NOT EXISTS events (
			script_version   integer,
			datetime         datetime NOT NULL,
			server           text,
			remote_addr      text,
			user_agent       text,
			request_referrer text,
			title            text,
			referrer         text,
			url              text,
			event_type       text,
			session_token    text,
			user_token       text
		);

		CREATE INDEX IF NOT EXISTS idx1 ON events(url, referrer);
		CREATE INDEX IF NOT EXISTS idx2 ON events(user_token, session_token, event_type);

		PRAGMA journal_mode = WAL;
	`
	sqlInsertPageEvent = `
		INSERT INTO events(
			script_version,
			datetime,
			server,
			remote_addr,
			user_agent,
			request_referrer,
			title,
			referrer,
			url,
			event_type,
			session_token,
			user_token
		)
		VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);
	`
)

type pageEvent struct {
	ScriptVersion   string    `json:"script_version"`
	Time            time.Time `json:"datetime"`
	Host            string    `json:"server"`
	RemoteAddr      string    `json:"remote_addr"`
	UserAgent       string    `json:"user_agent"`
	RequestReferrer string    `json:"request_referrer"`
	Title           string    `json:"title"`
	PageReferrer    string    `json:"referrer"`
	URL             string    `json:"url"`
	EventType       string    `json:"event_type"`
	SessionToken    string    `json:"session_token"`
	UserToken       string    `json:"user_token"`
	ViewIndex       string    `json:"view_index"`
}

func pageEventFromRequest(req *http.Request) *pageEvent {
	params := req.URL.Query()

	return &pageEvent{
		Host:            req.Host,
		RemoteAddr:      req.RemoteAddr,
		UserAgent:       req.UserAgent(),
		RequestReferrer: req.Referer(),
		Time:            time.Now(),
		Title:           params.Get("t"),
		PageReferrer:    params.Get("ref"),
		URL:             params.Get("url"),
		ScriptVersion:   params.Get("ver"),
		EventType:       params.Get("evt"),
		SessionToken:    params.Get("st"),
		UserToken:       params.Get("ut"),
		ViewIndex:       params.Get("vwix"),
	}
}

func writePageEventToDB(db *sql.DB, ev *pageEvent) {
	stmt, err := db.Prepare(sqlInsertPageEvent)
	if err != nil {
		log.Fatal(err)
	}
	defer stmt.Close()

	_, err = stmt.Exec(
		ev.ScriptVersion,
		ev.Time,
		ev.Host,
		ev.RemoteAddr,
		ev.UserAgent,
		ev.RequestReferrer,
		ev.Title,
		ev.PageReferrer,
		ev.URL,
		ev.EventType,
		ev.SessionToken,
		ev.UserToken,
	)

	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Fix enviroment variable<commit_after>package main

import (
	"database/sql"
	"encoding/base64"
	"io/ioutil"
	"log"
	"net/http"
	"time"

	"github.com/caarlos0/env"
	sqlite "github.com/mattn/go-sqlite3"
)

const (
	pixelRaw   = "R0lGODlhAQABAIAAANvf7wAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw=="
	pixelRoute = "/orkka.gif"
	jsRoute    = "/a.js"
)

type config struct {
	Port                  string `env:"PORT"             envDefault:"8080"`
	DBFile                string `env:"DB_FILE"          envDefault:"events.db"`
	EventQueueSize        int    `env:"MAX_CONNECTIONS"  envDefault:"4096"`
	WriteQueueDefaultSize int    `env:"WRITE_QUEUE_SIZE" envDefault:"1024"`
	WriteFrequencyMillis  int    `env:"WRITE_FREQUENCY"  envDefault:"100"`
	JSFile                string `env:"JSFILE"           envDefault:"analytics.js"`
}

func main() {
	var config config
	if err := env.Parse(&config); err != nil {
		log.Fatal(err)
	}

	eventQueue := make(chan *pageEvent, config.EventQueueSize)

	go processEventWriteQueue(&config, eventQueue)
	http.HandleFunc(pixelRoute, makeHandlePixel(eventQueue))
	http.HandleFunc(jsRoute, makeHandleScript(config.JSFile))

	log.Fatal(http.ListenAndServe(":"+config.Port, nil))
}

func makeHandleScript(file string) http.HandlerFunc {
	javascript, err := ioutil.ReadFile(file)
	if err != nil {
		log.Fatal("Failed to read javascript file.", err)
	}

	return func(out http.ResponseWriter, req *http.Request) {
		out.Header().Set("Content-Type", "text/javascript")
		out.Header().Set("Cache-Control", "public, max-age=86400")
		out.WriteHeader(http.StatusOK)
		out.Write(javascript)
	}
}

func makeHandlePixel(eventQueue chan *pageEvent) http.HandlerFunc {
	pixel, err := base64.StdEncoding.DecodeString(pixelRaw)
	if err != nil {
		log.Fatalf("failed to encode pixel: %v", err)
	}

	return func(out http.ResponseWriter, req *http.Request) {
		eventQueue <- pageEventFromRequest(req)
		out.Header().Set("Content-Type", "image/gif")
		out.WriteHeader(http.StatusOK)
		out.Write(pixel)
	}
}

func processEventWriteQueue(config *config, eventQueue chan *pageEvent) {
	db := initDB(config)
	defer db.Close()

	writeFrequency := time.Duration(time.Duration(config.WriteFrequencyMillis) * time.Millisecond)
	writeTimer := time.NewTimer(writeFrequency)

	writeQueue := make([]*pageEvent, 0, config.WriteQueueDefaultSize)

	for {
		select {
		case ev := <-eventQueue:
			writeQueue = append(writeQueue, ev)

		case <-writeTimer.C:
			if len(writeQueue) > 0 {
				writeTimer.Stop()
				writeEvents(db, writeQueue)
				writeQueue = writeQueue[:0]
			}
			writeTimer.Reset(writeFrequency)
		}
	}
}

func writeEvents(db *sql.DB, events []*pageEvent) {
	var ev *pageEvent
	i := -1
	db.Exec("BEGIN TRANSACTION;")
	for i, ev = range events {
		writePageEventToDB(db, ev)
	}
	db.Exec("END TRANSACTION;")

	log.Println("wrote", i+1, "records.")
}

func initDB(config *config) *sql.DB {
	sql.Register("sqlite3_custom", &sqlite.SQLiteDriver{
		ConnectHook: registerSQLiteExtensions,
	})

	db, err := sql.Open("sqlite3_custom", config.DBFile)

	if err != nil {
		log.Fatal(err)
	}
	if db == nil {
		log.Fatal("database is null")
	}

	if _, err = db.Exec(sqlCreatePageEventTable); err != nil {
		log.Fatal(err)
	}

	return db
}

const (
	pageEventSchemaVersion  = 1
	sqlCreatePageEventTable = `
		CREATE TABLE IF NOT EXISTS events (
			script_version   integer,
			datetime         datetime NOT NULL,
			server           text,
			remote_addr      text,
			user_agent       text,
			request_referrer text,
			title            text,
			referrer         text,
			url              text,
			event_type       text,
			session_token    text,
			user_token       text
		);

		CREATE INDEX IF NOT EXISTS idx1 ON events(url, referrer);
		CREATE INDEX IF NOT EXISTS idx2 ON events(user_token, session_token, event_type);

		PRAGMA journal_mode = WAL;
	`
	sqlInsertPageEvent = `
		INSERT INTO events(
			script_version,
			datetime,
			server,
			remote_addr,
			user_agent,
			request_referrer,
			title,
			referrer,
			url,
			event_type,
			session_token,
			user_token
		)
		VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);
	`
)

type pageEvent struct {
	ScriptVersion   string    `json:"script_version"`
	Time            time.Time `json:"datetime"`
	Host            string    `json:"server"`
	RemoteAddr      string    `json:"remote_addr"`
	UserAgent       string    `json:"user_agent"`
	RequestReferrer string    `json:"request_referrer"`
	Title           string    `json:"title"`
	PageReferrer    string    `json:"referrer"`
	URL             string    `json:"url"`
	EventType       string    `json:"event_type"`
	SessionToken    string    `json:"session_token"`
	UserToken       string    `json:"user_token"`
	ViewIndex       string    `json:"view_index"`
}

func pageEventFromRequest(req *http.Request) *pageEvent {
	params := req.URL.Query()

	return &pageEvent{
		Host:            req.Host,
		RemoteAddr:      req.RemoteAddr,
		UserAgent:       req.UserAgent(),
		RequestReferrer: req.Referer(),
		Time:            time.Now(),
		Title:           params.Get("t"),
		PageReferrer:    params.Get("ref"),
		URL:             params.Get("url"),
		ScriptVersion:   params.Get("ver"),
		EventType:       params.Get("evt"),
		SessionToken:    params.Get("st"),
		UserToken:       params.Get("ut"),
		ViewIndex:       params.Get("vwix"),
	}
}

func writePageEventToDB(db *sql.DB, ev *pageEvent) {
	stmt, err := db.Prepare(sqlInsertPageEvent)
	if err != nil {
		log.Fatal(err)
	}
	defer stmt.Close()

	_, err = stmt.Exec(
		ev.ScriptVersion,
		ev.Time,
		ev.Host,
		ev.RemoteAddr,
		ev.UserAgent,
		ev.RequestReferrer,
		ev.Title,
		ev.PageReferrer,
		ev.URL,
		ev.EventType,
		ev.SessionToken,
		ev.UserToken,
	)

	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"compress/lzw"
	"crypto/md5"
	"crypto/tls"
	"encoding/gob"
	"encoding/hex"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"

	"github.com/hashicorp/consul-template/dependency"
)

type KeyPair []struct {
	LockIndex   uint64
	Key         string
	Flags       uint64
	Value       []byte
	CreateIndex uint64
	ModifyIndex uint64
}

type templateData struct {
	Data map[string]interface{}
}

type Config struct {
	addr string
	file string
	hash string
}

func getFlags(config *Config) *Config {
	const (
		consulDefault = "localhost:8500"
		consulDesc    = "Consul address"
		fileDefault   = ""
		fileDesc      = "Consul-template file; must end in .ctmpl"
	)

	flag.StringVar(&config.addr, "consul", consulDefault, consulDesc)
	flag.StringVar(&config.file, "file", fileDefault, fileDesc)
	flag.Parse()
	return config
}

func getMd5(config *Config) string {
	contents, err := ioutil.ReadFile(config.file)
	if err != nil {
		log.Fatal(err)
	}

	hash := md5.Sum(contents)
	return hex.EncodeToString(hash[:])
}

func getValue(config *Config) []byte {
	url := fmt.Sprintf(
		"https://%s/v1/kv/consul-template/dedup/%s/data", config.addr, config.hash)

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		log.Fatal(err)
	}

	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}
	c := http.Client{Transport: tr}
	resp, err := c.Do(req)
	if err != nil {
		log.Fatal(err)
	}

	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal(err)
	}

	if resp.StatusCode != 200 {
		log.Fatal(err)
	}

	var keyPair KeyPair
	if err := json.Unmarshal(body, &keyPair); err != nil {
		log.Fatal(err)
	}

	return keyPair[0].Value
}

func decodeValue(value []byte) {
	r := bytes.NewReader(value)
	decompress := lzw.NewReader(r, lzw.LSB, 8)
	defer decompress.Close()
	dec := gob.NewDecoder(decompress)

	var td templateData
	if err := dec.Decode(&td); err != nil {
		log.Fatal(err)
	}

	// spew.Dump(td.Data)
	for k, v := range td.Data {
		serviceName := strings.Replace(k, "HealthServices|", "", -1)
		fmt.Println(serviceName)
		for _, j := range v.([]*dependency.HealthService) {
			// spew.Dump(j)
			fmt.Printf("  %s\n", j.Node)
		}
		fmt.Println()
	}
}

func main() {
	var config *Config = &Config{}
	config = getFlags(config)

	if len(os.Args) <= 1 {
		fmt.Printf("Usage of %s:\n", os.Args[0])
		flag.PrintDefaults()
		os.Exit(1)
	}

	config.hash = getMd5(config)
	fmt.Printf("key hash: %s\n\n", config.hash)

	value := getValue(config)
	decodeValue(value)
}
<commit_msg>make sure file extension ends in .ctmpl<commit_after>package main

import (
	"bytes"
	"compress/lzw"
	"crypto/md5"
	"crypto/tls"
	"encoding/gob"
	"encoding/hex"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strings"

	"github.com/hashicorp/consul-template/dependency"
)

type KeyPair []struct {
	LockIndex   uint64
	Key         string
	Flags       uint64
	Value       []byte
	CreateIndex uint64
	ModifyIndex uint64
}

type templateData struct {
	Data map[string]interface{}
}

type Config struct {
	addr string
	file string
	hash string
}

func getFlags(config *Config) *Config {
	const (
		consulDefault = "localhost:8500"
		consulDesc    = "Consul address"
		fileDefault   = ""
		fileDesc      = "Consul-template file; must end in .ctmpl"
	)

	flag.StringVar(&config.addr, "consul", consulDefault, consulDesc)
	flag.StringVar(&config.file, "file", fileDefault, fileDesc)
	flag.Parse()
	return config
}

func getMd5(config *Config) string {
	contents, err := ioutil.ReadFile(config.file)
	if err != nil {
		log.Fatal(err)
	}

	hash := md5.Sum(contents)
	return hex.EncodeToString(hash[:])
}

func getValue(config *Config) []byte {
	url := fmt.Sprintf(
		"https://%s/v1/kv/consul-template/dedup/%s/data", config.addr, config.hash)

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		log.Fatal(err)
	}

	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}
	c := http.Client{Transport: tr}
	resp, err := c.Do(req)
	if err != nil {
		log.Fatal(err)
	}

	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal(err)
	}

	if resp.StatusCode != 200 {
		log.Fatal(err)
	}

	var keyPair KeyPair
	if err := json.Unmarshal(body, &keyPair); err != nil {
		log.Fatal(err)
	}

	return keyPair[0].Value
}

func decodeValue(value []byte) {
	r := bytes.NewReader(value)
	decompress := lzw.NewReader(r, lzw.LSB, 8)
	defer decompress.Close()
	dec := gob.NewDecoder(decompress)

	var td templateData
	if err := dec.Decode(&td); err != nil {
		log.Fatal(err)
	}

	// spew.Dump(td.Data)
	for k, v := range td.Data {
		serviceName := strings.Replace(k, "HealthServices|", "", -1)
		fmt.Println(serviceName)
		for _, j := range v.([]*dependency.HealthService) {
			// spew.Dump(j)
			fmt.Printf("  %s\n", j.Node)
		}
		fmt.Println()
	}
}

func main() {
	var config *Config = &Config{}
	config = getFlags(config)

	if len(os.Args) <= 1 {
		fmt.Printf("Usage of %s:\n", os.Args[0])
		flag.PrintDefaults()
		os.Exit(1)
	}

	if err := filepath.Ext(config.file); err != ".ctmpl" {
		fmt.Println("File extension needs to end in .ctmpl.")
		os.Exit(1)
	}

	config.hash = getMd5(config)
	fmt.Printf("key hash: %s\n\n", config.hash)

	value := getValue(config)
	decodeValue(value)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"os"
	"strconv"
	"strings"
)

type FileName struct {
	Oldname string
	Newname string
	Modify  bool
}

func usage() {
	fmt.Println("Usage: go-renamer filepath")
}

func renameAll(f string) {
	if !strings.HasSuffix(f, "/") {
		f += "/"
	}
	files, err := ioutil.ReadDir(f)
	if err != nil {
		fmt.Println(err)
		return
	}
	filename := make([]FileName, len(files))
	for i := 0; i < len(files); i++ {
		filename[i] = FileName{Oldname: files[i].Name(), Newname: "", Modify: false}
	}

	setName(filename, 0)

	for {
		showChangeList(filename)
		fmt.Print("Really change files name or modify?(y/n/m): ")
		reader := bufio.NewReader(os.Stdin)
		c, err := reader.ReadByte()
		if err != nil {
			fmt.Println(err)
			return
		}
		if c == 'y' {
			for i := 0; i < len(filename); i++ {
				os.Rename(f+filename[i].Oldname, f+filename[i].Newname)
			}
			break
		} else if c == 'm' {
			modifyName(filename)
			setName(filename, 0)
		} else {
			break
		}
	}
}

func modifyName(filename []FileName) {
	fmt.Print("Input modify number -->")
	reader := bufio.NewReader(os.Stdin)
	c, _, err := reader.ReadLine()
	if err != nil {
		fmt.Println(err)
		return
	}
	num, err := strconv.Atoi(string(c))
	if err != nil {
		fmt.Println(err)
		return
	}
	fmt.Print("Input new file name -->")
	c, _, err = reader.ReadLine()
	if checkName(filename, string(c)) {
		fmt.Println("Already exists the same name file")
		return
	}
	filename[num].Newname = string(c)
	filename[num].Modify = true
}

func checkName(filename []FileName, f string) bool {
	for _, fn := range filename {
		if fn.Newname == f {
			return true
		}
	}
	return false
}

func setName(filename []FileName, start int) {
	index := 0
	for i := start; i < len(filename); i++ {
		if filename[i].Modify {
			continue
		}
		if filename[i].Oldname[0] == '.' {
			filename[i].Newname = filename[i].Oldname
		} else if strings.Index(filename[i].Oldname, ".") == -1 {
			tmp := fmt.Sprintf("%03d", index)
			filename[i].Newname = tmp
			index++
		} else {
			tmp := strings.Split(filename[i].Oldname, ".")
			suffix := tmp[len(tmp)-1]
			fname := fmt.Sprintf("%03d.%s", index, suffix)
			filename[i].Newname = fname
			index++
		}
	}
}

func showChangeList(filename []FileName) {
	for i := 0; i < len(filename); i++ {
		fmt.Println(strconv.Itoa(i) + ": " + filename[i].Oldname + " --> " + filename[i].Newname)
	}
}

func main() {
	if len(os.Args) < 2 {
		usage()
		os.Exit(1)
	}

	f, err := os.Stat(os.Args[1])
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	if f.IsDir() {
		renameAll(os.Args[1])
	} else {
		scanner := bufio.NewScanner(os.Stdin)
		fmt.Print(f.Name() + "--> ")
		for scanner.Scan() {
			nf := scanner.Text()
			if nf == "" {
				fmt.Print(f.Name() + "--> ")
				continue
			}
			os.Rename(os.Args[1], nf)
			break
		}
	}
}
<commit_msg>ignore directry<commit_after>package main

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"os"
	"strconv"
	"strings"
)

type FileName struct {
	Oldname string
	Newname string
	Modify  bool
}

func usage() {
	fmt.Println("Usage: go-renamer filepath")
}

func renameAll(f string) {
	if !strings.HasSuffix(f, "/") {
		f += "/"
	}
	files, err := ioutil.ReadDir(f)
	if err != nil {
		fmt.Println(err)
		return
	}
	filename := make([]FileName, 0)
	for i := 0; i < len(files); i++ {
		if !files[i].IsDir() {
			filename = append(filename, FileName{Oldname: files[i].Name(), Newname: "", Modify: false})
		}
	}

	setName(filename, 0)

	for {
		showChangeList(filename)
		fmt.Print("Really change files name or modify?(y/n/m): ")
		reader := bufio.NewReader(os.Stdin)
		c, err := reader.ReadByte()
		if err != nil {
			fmt.Println(err)
			return
		}
		if c == 'y' {
			for i := 0; i < len(filename); i++ {
				os.Rename(f+filename[i].Oldname, f+filename[i].Newname)
			}
			break
		} else if c == 'm' {
			modifyName(filename)
			setName(filename, 0)
		} else {
			break
		}
	}
}

func modifyName(filename []FileName) {
	fmt.Print("Input modify number -->")
	reader := bufio.NewReader(os.Stdin)
	c, _, err := reader.ReadLine()
	if err != nil {
		fmt.Println(err)
		return
	}
	num, err := strconv.Atoi(string(c))
	if err != nil {
		fmt.Println(err)
		return
	}
	fmt.Print("Input new file name -->")
	c, _, err = reader.ReadLine()
	if checkName(filename, string(c)) {
		fmt.Println("Already exists the same name file")
		return
	}
	filename[num].Newname = string(c)
	filename[num].Modify = true
}

func checkName(filename []FileName, f string) bool {
	for _, fn := range filename {
		if fn.Newname == f {
			return true
		}
	}
	return false
}

func setName(filename []FileName, start int) {
	index := 0
	for i := start; i < len(filename); i++ {
		if filename[i].Modify {
			continue
		}
		if filename[i].Oldname[0] == '.' {
			filename[i].Newname = filename[i].Oldname
		} else if strings.Index(filename[i].Oldname, ".") == -1 {
			tmp := fmt.Sprintf("%03d", index)
			filename[i].Newname = tmp
			index++
		} else {
			tmp := strings.Split(filename[i].Oldname, ".")
			suffix := tmp[len(tmp)-1]
			fname := fmt.Sprintf("%03d.%s", index, suffix)
			filename[i].Newname = fname
			index++
		}
	}
}

func showChangeList(filename []FileName) {
	for i := 0; i < len(filename); i++ {
		fmt.Println(strconv.Itoa(i) + ": " + filename[i].Oldname + " --> " + filename[i].Newname)
	}
}

func main() {
	if len(os.Args) < 2 {
		usage()
		os.Exit(1)
	}

	f, err := os.Stat(os.Args[1])
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	if f.IsDir() {
		renameAll(os.Args[1])
	} else {
		scanner := bufio.NewScanner(os.Stdin)
		fmt.Print(f.Name() + "--> ")
		for scanner.Scan() {
			nf := scanner.Text()
			if nf == "" {
				fmt.Print(f.Name() + "--> ")
				continue
			}
			os.Rename(os.Args[1], nf)
			break
		}
	}
}
<|endoftext|>