<commit_before>/*
Copyright 2015 Home Office All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"math/rand"
	"os"
	"path"
	"strings"
	"time"

	"os/exec"
	"path/filepath"

	"github.com/golang/glog"
	"gopkg.in/yaml.v2"
)

func init() {
	rand.Seed(int64(time.Now().Nanosecond()))
}

// showUsage prints the command usage and exits
//	message		: an error message to display if exiting with an error
func showUsage(message string, args ...interface{}) {
	flag.PrintDefaults()
	if message != "" {
		fmt.Printf("\n[error] "+message+"\n", args...)
		os.Exit(1)
	}

	os.Exit(0)
}

// hasKey checks to see if a key is present
//	key			: the key we are looking for
//	data		: a map of strings to something we are looking at
func hasKey(key string, data map[string]interface{}) bool {
	_, found := data[key]
	return found
}

// getKeys retrieves a list of keys from the map
// 	data		: the map which you wish to extract the keys from
func getKeys(data map[string]interface{}) []string {
	var list []string
	for key := range data {
		list = append(list, key)
	}
	return list
}

// readConfigFile read in a configuration file
//	filename		: the path to the file
func readConfigFile(filename, fileFormat string) (*vaultAuthOptions, error) {
	// step: check the file exists
	if exists, err := fileExists(filename); !exists {
		return nil, fmt.Errorf("the file: %s does not exist", filename)
	} else if err != nil {
		return nil, err
	}
	// step: we only read in json or yaml formats
	suffix := path.Ext(filename)
	switch suffix {
	case ".yaml":
		fallthrough
	case ".yml":
		return readYAMLFile(filename)
	default:
		return readJSONFile(filename, fileFormat)
	}
}

// readJsonFile read in and unmarshall the data into a map
//	filename	: the path to the file container the json data
func readJSONFile(filename, format string) (*vaultAuthOptions, error) {
	opts := &vaultAuthOptions{}

	content, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	// unmarshall the data
	err = json.Unmarshal(content, &opts)
	if err != nil && format == "default" {
		return nil, err
	}
	if err != nil {
		return nil, err
	}
	if format == "kubernetes-vault" && opts.ClientToken != "" {
		opts.Method = "token"
		opts.Token = opts.ClientToken
	}

	return opts, nil
}

// readYAMLFile read in and unmarshall the data into a map
//	filename	: the path to the file container the yaml data
func readYAMLFile(filename string) (*vaultAuthOptions, error) {
	o := &vaultAuthOptions{}
	content, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	err = yaml.Unmarshal(content, o)
	if err != nil {
		return nil, err
	}

	return o, nil
}

// getDurationWithin generate a random integer between min and max
//	min			: the smallest number we can accept
//	max			: the largest number we can accept
func getDurationWithin(min, max int) time.Duration {
	duration := rand.Intn(max-min) + min
	return time.Duration(duration) * time.Second
}

// getEnv checks to see if an environment variable exists otherwise uses the default
//	env			: the name of the environment variable you are checking for
//	value		: the default value to return if the value is not there
func getEnv(env, value string) string {
	if v := os.Getenv(env); v != "" {
		return v
	}

	return value
}

// fileExists checks to see if a file exists
//	filename		: the full path to the file you are checking for
func fileExists(filename string) (bool, error) {
	if _, err := os.Stat(filename); err != nil {
		if os.IsNotExist(err) {
			return false, nil
		}
		return false, err
	}

	return true, nil
}

// processResource is responsible for generating the specific content from the resource
// 	rn		: a point to the vault resource
//	data		: a map of the related secret associated to the resource
func processResource(rn *VaultResource, data map[string]interface{}) (err error) {
	// step: determine the resource path
	filename := rn.GetFilename()
	if !strings.HasPrefix(filename, "/") {
		filename = fmt.Sprintf("%s/%s", options.outputDir, filepath.Base(filename))
	}
	// step: format and write the file
	switch rn.format {
	case "yaml":
		fallthrough
	case "yml":
		err = writeYAMLFile(filename, data, rn.fileMode)
	case "json":
		err = writeJSONFile(filename, data, rn.fileMode)
	case "ini":
		err = writeIniFile(filename, data, rn.fileMode)
	case "csv":
		err = writeCSVFile(filename, data, rn.fileMode)
	case "env":
		err = writeEnvFile(filename, data, rn.fileMode)
	case "cert":
		err = writeCertificateFile(filename, data, rn.fileMode)
	case "txt":
		err = writeTxtFile(filename, data, rn.fileMode)
	case "bundle":
		err = writeCertificateBundleFile(filename, data, rn.fileMode)
	default:
		return fmt.Errorf("unknown output format: %s", rn.format)
	}
	// step: check for an error
	if err != nil {
		return err
	}

	// step: check if we need to execute a command
	if rn.execPath != "" {
		glog.V(10).Infof("executing the command: %s for resource: %s", rn.execPath, filename)
		cmd := exec.Command(rn.execPath, filename)
		cmd.Start()
		timer := time.AfterFunc(options.execTimeout, func() {
			if err = cmd.Process.Kill(); err != nil {
				glog.Errorf("failed to kill the command, pid: %d, error: %s", cmd.Process.Pid, err)
			}
		})
		// step: wait for the command to finish
		err = cmd.Wait()
		timer.Stop()
	}

	return err
}
<commit_msg>utils: split exec args on space<commit_after>/*
Copyright 2015 Home Office All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"math/rand"
	"os"
	"path"
	"strings"
	"time"

	"os/exec"
	"path/filepath"

	"github.com/golang/glog"
	"gopkg.in/yaml.v2"
)

func init() {
	rand.Seed(int64(time.Now().Nanosecond()))
}

// showUsage prints the command usage and exits
//	message		: an error message to display if exiting with an error
func showUsage(message string, args ...interface{}) {
	flag.PrintDefaults()
	if message != "" {
		fmt.Printf("\n[error] "+message+"\n", args...)
		os.Exit(1)
	}

	os.Exit(0)
}

// hasKey checks to see if a key is present
//	key			: the key we are looking for
//	data		: a map of strings to something we are looking at
func hasKey(key string, data map[string]interface{}) bool {
	_, found := data[key]
	return found
}

// getKeys retrieves a list of keys from the map
// 	data		: the map which you wish to extract the keys from
func getKeys(data map[string]interface{}) []string {
	var list []string
	for key := range data {
		list = append(list, key)
	}
	return list
}

// readConfigFile read in a configuration file
//	filename		: the path to the file
func readConfigFile(filename, fileFormat string) (*vaultAuthOptions, error) {
	// step: check the file exists
	if exists, err := fileExists(filename); !exists {
		return nil, fmt.Errorf("the file: %s does not exist", filename)
	} else if err != nil {
		return nil, err
	}
	// step: we only read in json or yaml formats
	suffix := path.Ext(filename)
	switch suffix {
	case ".yaml":
		fallthrough
	case ".yml":
		return readYAMLFile(filename)
	default:
		return readJSONFile(filename, fileFormat)
	}
}

// readJsonFile read in and unmarshall the data into a map
//	filename	: the path to the file container the json data
func readJSONFile(filename, format string) (*vaultAuthOptions, error) {
	opts := &vaultAuthOptions{}

	content, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	// unmarshall the data
	err = json.Unmarshal(content, &opts)
	if err != nil && format == "default" {
		return nil, err
	}
	if err != nil {
		return nil, err
	}
	if format == "kubernetes-vault" && opts.ClientToken != "" {
		opts.Method = "token"
		opts.Token = opts.ClientToken
	}

	return opts, nil
}

// readYAMLFile read in and unmarshall the data into a map
//	filename	: the path to the file container the yaml data
func readYAMLFile(filename string) (*vaultAuthOptions, error) {
	o := &vaultAuthOptions{}
	content, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	err = yaml.Unmarshal(content, o)
	if err != nil {
		return nil, err
	}

	return o, nil
}

// getDurationWithin generate a random integer between min and max
//	min			: the smallest number we can accept
//	max			: the largest number we can accept
func getDurationWithin(min, max int) time.Duration {
	duration := rand.Intn(max-min) + min
	return time.Duration(duration) * time.Second
}

// getEnv checks to see if an environment variable exists otherwise uses the default
//	env			: the name of the environment variable you are checking for
//	value		: the default value to return if the value is not there
func getEnv(env, value string) string {
	if v := os.Getenv(env); v != "" {
		return v
	}

	return value
}

// fileExists checks to see if a file exists
//	filename		: the full path to the file you are checking for
func fileExists(filename string) (bool, error) {
	if _, err := os.Stat(filename); err != nil {
		if os.IsNotExist(err) {
			return false, nil
		}
		return false, err
	}

	return true, nil
}

// processResource is responsible for generating the specific content from the resource
// 	rn		: a point to the vault resource
//	data		: a map of the related secret associated to the resource
func processResource(rn *VaultResource, data map[string]interface{}) (err error) {
	// step: determine the resource path
	filename := rn.GetFilename()
	if !strings.HasPrefix(filename, "/") {
		filename = fmt.Sprintf("%s/%s", options.outputDir, filepath.Base(filename))
	}
	// step: format and write the file
	switch rn.format {
	case "yaml":
		fallthrough
	case "yml":
		err = writeYAMLFile(filename, data, rn.fileMode)
	case "json":
		err = writeJSONFile(filename, data, rn.fileMode)
	case "ini":
		err = writeIniFile(filename, data, rn.fileMode)
	case "csv":
		err = writeCSVFile(filename, data, rn.fileMode)
	case "env":
		err = writeEnvFile(filename, data, rn.fileMode)
	case "cert":
		err = writeCertificateFile(filename, data, rn.fileMode)
	case "txt":
		err = writeTxtFile(filename, data, rn.fileMode)
	case "bundle":
		err = writeCertificateBundleFile(filename, data, rn.fileMode)
	default:
		return fmt.Errorf("unknown output format: %s", rn.format)
	}
	// step: check for an error
	if err != nil {
		return err
	}

	// step: check if we need to execute a command
	if rn.execPath != "" {
		glog.V(10).Infof("executing the command: %s for resource: %s", rn.execPath, filename)
		parts := strings.Split(rn.execPath, " ")
		var args []string
		if len(parts) > 1 {
			args = parts[1:]
		} else {
			args = []string{filename}
		}

		cmd := exec.Command(parts[0], args...)
		cmd.Start()
		timer := time.AfterFunc(options.execTimeout, func() {
			if err = cmd.Process.Kill(); err != nil {
				glog.Errorf("failed to kill the command, pid: %d, error: %s", cmd.Process.Pid, err)
			}
		})
		// step: wait for the command to finish
		err = cmd.Wait()
		timer.Stop()
	}

	return err
}
<|endoftext|>


<commit_before>package osm

import (
	"database/sql"
	"log"
	"reflect"
	"strings"
	"time"
)

const (
	formatDate     = "2006-01-02"
	formatDateTime = "2006-01-02 15:04:05"
)

// format time string
func timeFormat(t time.Time, format string) string {
	return t.Format(format)
}

// // camel string, xx_yy to XxYy
// func toGoName(s string) string {
// 	data := make([]byte, 0, len(s))
// 	j := false
// 	k := false
// 	num := len(s) - 1
// 	for i := 0; i <= num; i++ {
// 		d := s[i]
// 		if k == false && d >= 'A' && d <= 'Z' {
// 			k = true
// 		}
// 		if d >= 'a' && d <= 'z' && (j || k == false) {
// 			d = d - 32
// 			j = false
// 			k = true
// 		}
// 		if k && d == '_' && num > i /**&& s[i+1] >= 'a' && s[i+1] <= 'z'**/ {
// 			j = true
// 			continue
// 		}
// 		data = append(data, d)
// 	}
// 	return string(data[:len(data)])
// }

// func toGoName(name string) string {
// 	names := strings.Split(name, "_")

// 	newName := ""
// 	for _, ntemp := range names {
// 		newName += strings.ToUpper(ntemp[0:1]) + strings.ToLower(ntemp[1:len(ntemp)])
// 	}
// 	return newName
// }

// camel string, xx_yy to XxYy, 特列字符 ID
func toGoName(name string) string {
	num := len(name)
	data := make([]byte, len(name))
	j := 0
	k := true
	for i := 0; i < num; i++ {
		d := name[i]
		if d == '_' {
			k = true
			if j >= 2 && data[j-2] == 'I' && data[j-1] == 'd' {
				data[j-1] = 'D'
			}
		} else {
			if k {
				if d >= 'a' && d <= 'z' {
					d = d - 32
				}
			} else {
				if d >= 'A' && d <= 'Z' {
					d = d + 32
				}
			}
			data[j] = d
			j++
			k = false
		}
	}
	if j > 1 && data[j-1] == 'd' && data[j-2] == 'I' {
		data[j-1] = 'D'
	}
	return string(data[:j])
}

// scanRow 从sql.Rows中读一行数据
func scanRow(
	rows *sql.Rows,
	isPtrs []bool,
	elementTypes []reflect.Type,
	values []reflect.Value,
) error {
	lenContainers := len(isPtrs)
	var refs []interface{}
	isTimes := make([]bool, lenContainers)
	for i, isPtr := range isPtrs {
		if isPtr {
			if elementTypes[i].Elem().String() == "time.Time" {
				isTimes[i] = true
				refs = append(refs, new(string))
			} else {
				values[i].Set(reflect.New(elementTypes[i].Elem()))
				refs = append(refs, values[i].Interface())
			}
		} else {
			if elementTypes[i].String() == "time.Time" {
				isTimes[i] = true
				refs = append(refs, new(string))
			} else {
				refs = append(refs, values[i].Addr().Interface())
			}
		}
	}

	err := rows.Scan(refs...)
	if err != nil {
		return err
	}

	for i, isPtr := range isPtrs {
		if isTimes[i] {
			strPtr, ok := refs[i].(*string)
			if ok && strPtr != nil {
				str := *strPtr
				if str != "" {
					var t time.Time
					if len(str) >= 19 {
						if strings.Contains(str, "T") {
							t, err = time.Parse(time.RFC3339Nano, str)
						} else {
							str = str[:19]
							t, err = time.Parse(formatDateTime, str)
						}
					} else if len(str) >= 10 {
						str = str[:10]
						t, err = time.Parse(formatDate, str)
					}
					if err == nil {
						t = t.Local()
						if isPtr {
							values[i].Set(reflect.ValueOf(&t))
						} else {
							values[i].Set(reflect.ValueOf(t))
						}
					} else {
						log.Println(err)
					}
				}
			}
		}
	}
	return nil
}
<commit_msg>string null<commit_after>package osm

import (
	"database/sql"
	"log"
	"reflect"
	"strings"
	"time"
)

const (
	formatDate     = "2006-01-02"
	formatDateTime = "2006-01-02 15:04:05"
)

// format time string
func timeFormat(t time.Time, format string) string {
	return t.Format(format)
}

// // camel string, xx_yy to XxYy
// func toGoName(s string) string {
// 	data := make([]byte, 0, len(s))
// 	j := false
// 	k := false
// 	num := len(s) - 1
// 	for i := 0; i <= num; i++ {
// 		d := s[i]
// 		if k == false && d >= 'A' && d <= 'Z' {
// 			k = true
// 		}
// 		if d >= 'a' && d <= 'z' && (j || k == false) {
// 			d = d - 32
// 			j = false
// 			k = true
// 		}
// 		if k && d == '_' && num > i /**&& s[i+1] >= 'a' && s[i+1] <= 'z'**/ {
// 			j = true
// 			continue
// 		}
// 		data = append(data, d)
// 	}
// 	return string(data[:len(data)])
// }

// func toGoName(name string) string {
// 	names := strings.Split(name, "_")

// 	newName := ""
// 	for _, ntemp := range names {
// 		newName += strings.ToUpper(ntemp[0:1]) + strings.ToLower(ntemp[1:len(ntemp)])
// 	}
// 	return newName
// }

// camel string, xx_yy to XxYy, 特列字符 ID
func toGoName(name string) string {
	num := len(name)
	data := make([]byte, len(name))
	j := 0
	k := true
	for i := 0; i < num; i++ {
		d := name[i]
		if d == '_' {
			k = true
			if j >= 2 && data[j-2] == 'I' && data[j-1] == 'd' {
				data[j-1] = 'D'
			}
		} else {
			if k {
				if d >= 'a' && d <= 'z' {
					d = d - 32
				}
			} else {
				if d >= 'A' && d <= 'Z' {
					d = d + 32
				}
			}
			data[j] = d
			j++
			k = false
		}
	}
	if j > 1 && data[j-1] == 'd' && data[j-2] == 'I' {
		data[j-1] = 'D'
	}
	return string(data[:j])
}

// scanRow 从sql.Rows中读一行数据
func scanRow(
	rows *sql.Rows,
	isPtrs []bool,
	elementTypes []reflect.Type,
	values []reflect.Value,
) error {
	lenContainers := len(isPtrs)
	var refs []interface{}
	types := make([]byte, lenContainers)
	for i, isPtr := range isPtrs {
		t := elementTypes[i]
		if isPtr {
			t = elementTypes[i].Elem()
		}
		if t.String() == "time.Time" {
			types[i] = 1 // time.Time
			refs = append(refs, new(string))
		} else if t.Kind() == reflect.String {
			types[i] = 2 // string
			refs = append(refs, new([]byte))
		} else if isPtr {
			values[i].Set(reflect.New(t))
			refs = append(refs, values[i].Interface())
		} else {
			refs = append(refs, values[i].Addr().Interface())
		}
	}

	err := rows.Scan(refs...)
	if err != nil {
		return err
	}

	for i, isPtr := range isPtrs {
		if refs[i] == nil {
			continue
		}
		if types[i] == 1 {
			strPtr, _ := refs[i].(*string)
			str := *strPtr
			if str != "" {
				var t time.Time
				if len(str) >= 19 {
					if strings.Contains(str, "T") {
						t, err = time.Parse(time.RFC3339Nano, str)
					} else {
						str = str[:19]
						t, err = time.Parse(formatDateTime, str)
					}
				} else if len(str) >= 10 {
					str = str[:10]
					t, err = time.Parse(formatDate, str)
				}
				if err == nil {
					t = t.Local()
					if isPtr {
						values[i].Set(reflect.ValueOf(&t))
					} else {
						values[i].Set(reflect.ValueOf(t))
					}
				} else {
					log.Println(err)
				}
			}
		} else if types[i] == 2 {
			bytePtr, _ := refs[i].(*[]byte)
			str := string(*bytePtr)
			if isPtr {
				values[i].Set(reflect.ValueOf(&str))
			} else {
				values[i].Set(reflect.ValueOf(str))
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>/*
Package Hasty provides very simple and fast Multiplexer for Go
Package is OpenSource at https://github.com/harshvladha/hasty
Developed by Harsh Vardhan Ladha
Email: harsh.ladha@gmail.com
*/

package hasty

import "net/http"

type ErrorStatusCode struct {
	HttpStatus int
}

func (mux *Mux) ListenAndServe(port string) error {
	return http.ListenAndServe(":"+port, mux)
}

func (mux *Mux) validate(rw http.ResponseWriter, req *http.Request) (bool, ErrorStatusCode) {
	pathLen := len(req.URL.Path)
	if pathLen > 1 && req.URL.Path[pathLen-1:] == "/" {
		cleanURL(&req.URL.Path)
		rw.Header().Set("Location", req.URL.String())
		rw.WriteHeader(http.StatusFound)
		return true, nil
	}
	return mux.parse(rw, req)
}

func cleanURL(url *string) {
	urlLen := len(*url)
	if urlLen > 1 {
		if (*url)[urlLen-1:] == "/" {
			*url = (*url)[:urlLen-1]
			cleanURL(url)
		}
	}
}

func (mux *Mux) parse(rw http.ResponseWriter, req *http.Request) (bool, ErrorStatusCode) {
	if mux.Routes[req.URL.Path] == nil {
		return false, ErrorStatusCode{HttpStatus: http.StatusNotFound}
	}
	requestMethod := req.Method
	// for HEAD method, default to GET
	if requestMethod == http.MethodHead {
		requestMethod = http.MethodGet
	}
	// check if Method is allowed for the given route
	if (methods[requestMethod])&(mux.Routes[req.URL.Path].methods) == 0 {
		return false, ErrorStatusCode{HttpStatus: http.StatusMethodNotAllowed}
	}
	mux.Routes[req.URL.Path].Handler.ServeHTTP(rw, req)
	return true, nil
}
<commit_msg>Mux.validate func returns pointer of ErrorStatusCode to handle nil returns for it<commit_after>/*
Package Hasty provides very simple and fast Multiplexer for Go
Package is OpenSource at https://github.com/harshvladha/hasty
Developed by Harsh Vardhan Ladha
Email: harsh.ladha@gmail.com
*/

package hasty

import "net/http"

type ErrorStatusCode struct {
	HttpStatus int
}

func (mux *Mux) ListenAndServe(port string) error {
	return http.ListenAndServe(":"+port, mux)
}

func (mux *Mux) validate(rw http.ResponseWriter, req *http.Request) (bool, *ErrorStatusCode) {
	pathLen := len(req.URL.Path)
	if pathLen > 1 && req.URL.Path[pathLen-1:] == "/" {
		cleanURL(&req.URL.Path)
		rw.Header().Set("Location", req.URL.String())
		rw.WriteHeader(http.StatusFound)
		return true, nil
	}
	return mux.parse(rw, req)
}

func cleanURL(url *string) {
	urlLen := len(*url)
	if urlLen > 1 {
		if (*url)[urlLen-1:] == "/" {
			*url = (*url)[:urlLen-1]
			cleanURL(url)
		}
	}
}

func (mux *Mux) parse(rw http.ResponseWriter, req *http.Request) (bool, *ErrorStatusCode) {
	if mux.Routes[req.URL.Path] == nil {
		return false, &ErrorStatusCode{HttpStatus: http.StatusNotFound}
	}
	requestMethod := req.Method
	// for HEAD method, default to GET
	if requestMethod == http.MethodHead {
		requestMethod = http.MethodGet
	}
	// check if Method is allowed for the given route
	if (methods[requestMethod])&(mux.Routes[req.URL.Path].methods) == 0 {
		return false, &ErrorStatusCode{HttpStatus: http.StatusMethodNotAllowed}
	}
	mux.Routes[req.URL.Path].Handler.ServeHTTP(rw, req)
	return true, nil
}
<|endoftext|>


<commit_before>package wsl

import (
	"errors"
	"fmt"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"unicode"
)

func Hook() {
	sigs := make(chan os.Signal, 1)
	done := make(chan bool, 1)
	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)

	go func() {
		for {
			select {
			case sig := <-sigs:
				fmt.Println(sig)
				// cleanup code here
				done <- true
			}
		}
	}()

	<-done
	fmt.Println("Bye!")
}

func extractParamsFromMap(m map[string]interface{}) []interface{} {
	if params, ok := m["params"].([]interface{}); ok {
		return params
	}
	ret := []interface{}{}
	for i := 0; ; i++ {
		if val, ok := m[fmt.Sprint("_", i)]; ok {
			ret = append(ret, val)
		} else {
			break
		}
	}
	return ret
}

func ExtractScriptParamsFromMap(m map[string]interface{}) map[string]interface{} {
	ret := map[string]interface{}{}
	for k, v := range m {
		if strings.HasPrefix(k, "__") {
			vs := v.(string)
			sqlSafe(&vs)
			ret[k] = v
		}
	}
	return ret
}

func valuesToMap(keyLowerCase bool, values ...map[string][]string) map[string]interface{} {
	ret := map[string]interface{}{}
	for _, vs := range values {
		for k, v := range vs {
			if keyLowerCase {
				ret[strings.ToLower(k)] = v[0]
			} else {
				ret[k] = v[0]
			}
		}
	}
	return ret
}

// true if the first character is uppercase, false otherwise
func ShouldExport(sql string) bool {
	if !unicode.IsLetter([]rune(sql)[0]) {
		return false
	}
	return strings.ToUpper(sql[0:1]) == sql[0:1]
}

// returns whether export the result of this sql statement or not
func SqlNormalize(sql *string) {
	*sql = strings.TrimSpace(*sql)
	var ret string
	lines := strings.Split(*sql, "\n")
	for _, line := range lines {
		lineTrimmed := strings.TrimSpace(line)
		if lineTrimmed != "" && !strings.HasPrefix(lineTrimmed, "-- ") {
			ret += line + "\n"
		}
	}
	*sql = ret
}

func SplitSqlLable(sql string) (label string, s string) {
	sql = strings.TrimSpace(sql)
	if strings.HasPrefix(sql, "#") {
		ss := strings.Fields(sql)
		lenSS := len(ss)
		if lenSS == 0 {
			return "", ""
		} else if lenSS == 1 {
			return ss[0][1:], ""
		}
		return ss[0][1:], strings.TrimSpace(sql[len(ss[0]):])
	}
	return "", sql
}

func sqlSafe(s *string) {
	*s = strings.Replace(*s, "'", "''", -1)
	*s = strings.Replace(*s, "--", "", -1)
}

func IsQuery(sql string) bool {
	sqlUpper := strings.ToUpper(strings.TrimSpace(sql))
	if strings.HasPrefix(sqlUpper, "SELECT") ||
		strings.HasPrefix(sqlUpper, "SHOW") ||
		strings.HasPrefix(sqlUpper, "DESCRIBE") ||
		strings.HasPrefix(sqlUpper, "EXPLAIN") {
		return true
	}
	return false
}

var ConvertStringArrayToInterfaceArray = func(arrayOfStrings []string) []interface{} {
	ret := []interface{}{}
	for _, v := range arrayOfStrings {
		ret = append(ret, v)
	}
	return ret
}

var ConvertInterfaceArrayToStringArray = func(arrayOfInterfaces []interface{}) ([]string, error) {
	ret := []string{}
	for _, v := range arrayOfInterfaces {
		if s, ok := v.(string); ok {
			ret = append(ret, s)
		} else {
			return nil, errors.New("Failed to convert.")
		}
	}
	return ret, nil
}

var ConvertMapOfInterfacesToMapOfStrings = func(data map[string]interface{}) (map[string]string, error) {
	if data == nil {
		return nil, errors.New("Cannot convert nil.")
	}
	ret := map[string]string{}
	for k, v := range data {
		if v == nil {
			return nil, errors.New("Data contains nil.")
		}
		ret[k] = v.(string)
	}
	return ret, nil
}

var ConvertMapOfStringsToMapOfInterfaces = func(data map[string]string) (map[string]interface{}, error) {
	if data == nil {
		return nil, errors.New("Cannot convert nil.")
	}
	ret := map[string]interface{}{}
	for k, v := range data {
		ret[k] = v
	}
	return ret, nil
}
<commit_msg>Trivial change to cause sum to change.<commit_after>package wsl

import (
	"errors"
	"fmt"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"unicode"
)

func Hook() {
	sigs := make(chan os.Signal, 1)
	done := make(chan bool, 1)
	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)

	go func() {
		for {
			select {
			case sig := <-sigs:
				fmt.Println(sig)
				// cleanup code here
				done <- true
			}
		}
	}()

	<-done
	fmt.Println("Bye!")
}

func extractParamsFromMap(m map[string]interface{}) []interface{} {
	if params, ok := m["params"].([]interface{}); ok {
		return params
	}
	ret := []interface{}{}
	for i := 0; ; i++ {
		if val, ok := m[fmt.Sprint("_", i)]; ok {
			ret = append(ret, val)
		} else {
			break
		}
	}
	return ret
}

func ExtractScriptParamsFromMap(m map[string]interface{}) map[string]interface{} {
	ret := map[string]interface{}{}
	for k, v := range m {
		if strings.HasPrefix(k, "__") {
			vs := v.(string)
			sqlSafe(&vs)
			ret[k] = v
		}
	}
	return ret
}

func valuesToMap(keyLowerCase bool, values ...map[string][]string) map[string]interface{} {
	ret := map[string]interface{}{}
	for _, vs := range values {
		for k, v := range vs {
			if keyLowerCase {
				ret[strings.ToLower(k)] = v[0]
			} else {
				ret[k] = v[0]
			}
		}
	}
	return ret
}

// true if the first character is uppercase, false otherwise
func ShouldExport(sql string) bool {
	if !unicode.IsLetter([]rune(sql)[0]) {
		return false
	}
	return strings.ToUpper(sql[0:1]) == sql[0:1]
}

// returns whether to export the result of this sql statement or not
func SqlNormalize(sql *string) {
	*sql = strings.TrimSpace(*sql)
	var ret string
	lines := strings.Split(*sql, "\n")
	for _, line := range lines {
		lineTrimmed := strings.TrimSpace(line)
		if lineTrimmed != "" && !strings.HasPrefix(lineTrimmed, "-- ") {
			ret += line + "\n"
		}
	}
	*sql = ret
}

func SplitSqlLable(sql string) (label string, s string) {
	sql = strings.TrimSpace(sql)
	if strings.HasPrefix(sql, "#") {
		ss := strings.Fields(sql)
		lenSS := len(ss)
		if lenSS == 0 {
			return "", ""
		} else if lenSS == 1 {
			return ss[0][1:], ""
		}
		return ss[0][1:], strings.TrimSpace(sql[len(ss[0]):])
	}
	return "", sql
}

func sqlSafe(s *string) {
	*s = strings.Replace(*s, "'", "''", -1)
	*s = strings.Replace(*s, "--", "", -1)
}

func IsQuery(sql string) bool {
	sqlUpper := strings.ToUpper(strings.TrimSpace(sql))
	if strings.HasPrefix(sqlUpper, "SELECT") ||
		strings.HasPrefix(sqlUpper, "SHOW") ||
		strings.HasPrefix(sqlUpper, "DESCRIBE") ||
		strings.HasPrefix(sqlUpper, "EXPLAIN") {
		return true
	}
	return false
}

var ConvertStringArrayToInterfaceArray = func(arrayOfStrings []string) []interface{} {
	ret := []interface{}{}
	for _, v := range arrayOfStrings {
		ret = append(ret, v)
	}
	return ret
}

var ConvertInterfaceArrayToStringArray = func(arrayOfInterfaces []interface{}) ([]string, error) {
	ret := []string{}
	for _, v := range arrayOfInterfaces {
		if s, ok := v.(string); ok {
			ret = append(ret, s)
		} else {
			return nil, errors.New("Failed to convert.")
		}
	}
	return ret, nil
}

var ConvertMapOfInterfacesToMapOfStrings = func(data map[string]interface{}) (map[string]string, error) {
	if data == nil {
		return nil, errors.New("Cannot convert nil.")
	}
	ret := map[string]string{}
	for k, v := range data {
		if v == nil {
			return nil, errors.New("Data contains nil.")
		}
		ret[k] = v.(string)
	}
	return ret, nil
}

var ConvertMapOfStringsToMapOfInterfaces = func(data map[string]string) (map[string]interface{}, error) {
	if data == nil {
		return nil, errors.New("Cannot convert nil.")
	}
	ret := map[string]interface{}{}
	for k, v := range data {
		ret[k] = v
	}
	return ret, nil
}
<|endoftext|>


<commit_before>package gowebdav

import (
	"bytes"
	"encoding/xml"
	"errors"
	"fmt"
	"io"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"
)

func Error(r *http.Response) error {
	return errors.New(fmt.Sprintf("%s - %s %s", r.Status, r.Request.Method, r.Request.URL.String()))
}

func newPathError(op string, path string, statusCode int) error {
	return &os.PathError{op, path, errors.New(fmt.Sprintf("%d", statusCode))}
}

func FixSlash(s string) string {
	if !strings.HasSuffix(s, "/") {
		s += "/"
	}
	return s
}

func FixSlashes(s string) string {
	if s[0] != '/' {
		s = "/" + s
	}
	return FixSlash(s)
}

func Join(path0 string, path1 string) string {
	return strings.TrimSuffix(path0, "/") + "/" + strings.TrimPrefix(path1, "/")
}

func String(r io.Reader) string {
	buf := new(bytes.Buffer)
	buf.ReadFrom(r)
	return buf.String()
}

func parseUint(s *string) uint {
	if n, e := strconv.ParseUint(*s, 10, 32); e == nil {
		return uint(n)
	}
	return 0
}

func parseInt64(s *string) int64 {
	if n, e := strconv.ParseInt(*s, 10, 64); e == nil {
		return n
	}
	return 0
}

func parseModified(s *string) time.Time {
	if t, e := time.Parse(time.RFC1123, *s); e == nil {
		return t
	}
	return time.Unix(0, 0)
}

func parseXML(data io.Reader, resp interface{}, parse func(resp interface{})) {
	decoder := xml.NewDecoder(data)
	for t, _ := decoder.Token(); t != nil; t, _ = decoder.Token() {
		switch se := t.(type) {
		case xml.StartElement:
			if se.Name.Local == "response" {
				if e := decoder.DecodeElement(resp, &se); e == nil {
					parse(resp)
				}
			}
		}
	}
}
<commit_msg>add utils.log<commit_after>package gowebdav

import (
	"bytes"
	"encoding/xml"
	"errors"
	"fmt"
	"io"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"
)

func log(msg interface{}) {
	fmt.Println(msg)
}

func Error(r *http.Response) error {
	return errors.New(fmt.Sprintf("%s - %s %s", r.Status, r.Request.Method, r.Request.URL.String()))
}

func newPathError(op string, path string, statusCode int) error {
	return &os.PathError{op, path, errors.New(fmt.Sprintf("%d", statusCode))}
}

func FixSlash(s string) string {
	if !strings.HasSuffix(s, "/") {
		s += "/"
	}
	return s
}

func FixSlashes(s string) string {
	if s[0] != '/' {
		s = "/" + s
	}
	return FixSlash(s)
}

func Join(path0 string, path1 string) string {
	return strings.TrimSuffix(path0, "/") + "/" + strings.TrimPrefix(path1, "/")
}

func String(r io.Reader) string {
	buf := new(bytes.Buffer)
	buf.ReadFrom(r)
	return buf.String()
}

func parseUint(s *string) uint {
	if n, e := strconv.ParseUint(*s, 10, 32); e == nil {
		return uint(n)
	}
	return 0
}

func parseInt64(s *string) int64 {
	if n, e := strconv.ParseInt(*s, 10, 64); e == nil {
		return n
	}
	return 0
}

func parseModified(s *string) time.Time {
	if t, e := time.Parse(time.RFC1123, *s); e == nil {
		return t
	}
	return time.Unix(0, 0)
}

func parseXML(data io.Reader, resp interface{}, parse func(resp interface{})) {
	decoder := xml.NewDecoder(data)
	for t, _ := decoder.Token(); t != nil; t, _ = decoder.Token() {
		switch se := t.(type) {
		case xml.StartElement:
			if se.Name.Local == "response" {
				if e := decoder.DecodeElement(resp, &se); e == nil {
					parse(resp)
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package flash

import (
	"compress/gzip"
	"encoding/json"
	"io"
	"log"
	"net/http"
	"path"
	"unicode"
)

func extractJSONPayload(data io.Reader, v interface{}) error {
	return json.NewDecoder(data).Decode(&v)
}

func capitalize(s string) string {
	if len(s) == 0 {
		return s
	}
	a := []rune(s)
	a[0] = unicode.ToUpper(a[0])
	return string(a)
}

// RenderJSONError common function to render error to client in JSON format
func RenderJSONError(w http.ResponseWriter, code int, s string) {
	RenderJSON(w, code, JSON{"errors": JSON{"message": []string{s}}})
}

// RenderJSON common function to render JSON to client
func RenderJSON(w http.ResponseWriter, code int, s JSON) {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	w.WriteHeader(code)
	if err := json.NewEncoder(w).Encode(s); err != nil {
		log.Println("JSON Encoding error:", err)
	}
}

// RenderJSONgzip common function to render gzipped JSON to client
func RenderJSONgzip(w http.ResponseWriter, code int, s JSON) {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	w.Header().Set("Content-Encoding", "gzip")
	w.WriteHeader(code)
	gz := gzip.NewWriter(w)
	defer gz.Close()
	if err := json.NewEncoder(gz).Encode(s); err != nil {
		log.Println("JSON Encoding error:", err)
	}
}

// cleanPath returns the canonical path for p, eliminating . and .. elements.
// Borrowed from the net/http package.
func cleanPath(p string) string {
	if p == "" {
		return "/"
	}
	if p[0] != '/' {
		p = "/" + p
	}
	np := path.Clean(p)
	// path.Clean removes trailing slash except for root;
	// put the trailing slash back if necessary.
	if p[len(p)-1] == '/' && np != "/" {
		np += "/"
	}
	return np
}
<commit_msg>improve capitalize func<commit_after>package flash

import (
	"compress/gzip"
	"encoding/json"
	"io"
	"log"
	"net/http"
	"path"
	"unicode"
	"unicode/utf8"
)

func extractJSONPayload(data io.Reader, v interface{}) error {
	return json.NewDecoder(data).Decode(&v)
}

func capitalize(s string) string {
	if s == "" {
		return s
	}
	r, n := utf8.DecodeRuneInString(s)
	return string(unicode.ToUpper(r)) + s[n:]
}

// RenderJSONError common function to render error to client in JSON format
func RenderJSONError(w http.ResponseWriter, code int, s string) {
	RenderJSON(w, code, JSON{"errors": JSON{"message": []string{s}}})
}

// RenderJSON common function to render JSON to client
func RenderJSON(w http.ResponseWriter, code int, s JSON) {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	w.WriteHeader(code)
	if err := json.NewEncoder(w).Encode(s); err != nil {
		log.Println("JSON Encoding error:", err)
	}
}

// RenderJSONgzip common function to render gzipped JSON to client
func RenderJSONgzip(w http.ResponseWriter, code int, s JSON) {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	w.Header().Set("Content-Encoding", "gzip")
	w.WriteHeader(code)
	gz := gzip.NewWriter(w)
	defer gz.Close()
	if err := json.NewEncoder(gz).Encode(s); err != nil {
		log.Println("JSON Encoding error:", err)
	}
}

// cleanPath returns the canonical path for p, eliminating . and .. elements.
// Borrowed from the net/http package.
func cleanPath(p string) string {
	if p == "" {
		return "/"
	}
	if p[0] != '/' {
		p = "/" + p
	}
	np := path.Clean(p)
	// path.Clean removes trailing slash except for root;
	// put the trailing slash back if necessary.
	if p[len(p)-1] == '/' && np != "/" {
		np += "/"
	}
	return np
}
<|endoftext|>


<commit_before>package main // import "eriol.xyz/piken"

import (
	"encoding/csv"
	"io"
	"net/http"
	"os"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/mitchellh/go-homedir"
)

// Get Last-Modified for resources with the given URL.
func checkLastModified(url string) (time.Time, error) {
	r, err := http.Head(url)
	if err != nil {
		return time.Unix(0, 0), err
	}
	defer r.Body.Close()

	t, err := time.Parse(time.RFC1123, r.Header["Last-Modified"][0])
	if err != nil {
		return time.Unix(0, 0), err
	}

	return t, nil

}

func download(url, output string) error {

	r, err := http.Get(url)
	if err != nil {
		return err
	}
	defer r.Body.Close()

	out, err := os.Create(output)
	if err != nil {
		return err
	}
	defer out.Close()

	// io.copyBuffer, the actual implementation of io.Copy, reads maximum 32 KB
	// from input, writes to output and then repeats. No need to worry about
	// the size of file to download.
	_, err = io.Copy(out, r.Body)
	if err != nil {
		return err
	}

	return nil
}

// Get user home directory or exit with a fatal error.
func getHome() string {

	homeDir, err := homedir.Dir()
	if err != nil {
		logrus.Fatal(err)
	}

	return homeDir
}

// Read a CSV file and return a slice of slice.
func readCsvFile(filepath string) (records [][]string, err error) {

	file, err := os.Open(filepath)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	reader := csv.NewReader(file)
	reader.Comma = ';'
	records, err = reader.ReadAll()
	if err != nil {
		return nil, err
	}

	return records, nil
}
<commit_msg>Show a progress bar during download<commit_after>package main // import "eriol.xyz/piken"

import (
	"encoding/csv"
	"io"
	"net/http"
	"os"
	"strconv"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/cheggaaa/pb"
	"github.com/mitchellh/go-homedir"
)

// Get Last-Modified for resources with the given URL.
func checkLastModified(url string) (time.Time, error) {
	r, err := http.Head(url)
	if err != nil {
		return time.Unix(0, 0), err
	}
	defer r.Body.Close()

	t, err := time.Parse(time.RFC1123, r.Header["Last-Modified"][0])
	if err != nil {
		return time.Unix(0, 0), err
	}

	return t, nil

}

// Download a file from URL `url` and save to `output`.
// A progress bar is showed during the download.
func download(url, output string) error {

	r, err := http.Get(url)
	if err != nil {
		return err
	}
	defer r.Body.Close()

	out, err := os.Create(output)
	if err != nil {
		return err
	}
	defer out.Close()

	size, _ := strconv.Atoi(r.Header.Get("Content-Length"))

	bar := pb.New(size).SetUnits(pb.U_BYTES).SetRefreshRate(time.Millisecond * 10)
	bar.ShowSpeed = true
	bar.Start()

	writer := io.MultiWriter(out, bar)

	// io.copyBuffer, the actual implementation of io.Copy, reads maximum 32 KB
	// from input, writes to output and then repeats. No need to worry about
	// the size of file to download.
	_, err = io.Copy(writer, r.Body)
	if err != nil {
		return err
	}

	bar.Finish()
	return nil
}

// Get user home directory or exit with a fatal error.
func getHome() string {

	homeDir, err := homedir.Dir()
	if err != nil {
		logrus.Fatal(err)
	}

	return homeDir
}

// Read a CSV file and return a slice of slice.
func readCsvFile(filepath string) (records [][]string, err error) {

	file, err := os.Open(filepath)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	reader := csv.NewReader(file)
	reader.Comma = ';'
	records, err = reader.ReadAll()
	if err != nil {
		return nil, err
	}

	return records, nil
}
<|endoftext|>


<commit_before>package zfs

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"os/exec"
	"regexp"
	"runtime"
	"strconv"
	"strings"

	"github.com/pborman/uuid"
)

type command struct {
	Command string
	Stdin   io.Reader
	Stdout  io.Writer
}

func (c *command) Run(arg ...string) ([][]string, error) {

	cmd := exec.Command(c.Command, arg...)

	var stdout, stderr bytes.Buffer

	if c.Stdout == nil {
		cmd.Stdout = &stdout
	} else {
		cmd.Stdout = c.Stdout
	}

	if c.Stdin != nil {
		cmd.Stdin = c.Stdin

	}
	cmd.Stderr = &stderr

	id := uuid.New()
	joinedArgs := strings.Join(cmd.Args, " ")

	logger.Log([]string{"ID:" + id, "START", joinedArgs})
	err := cmd.Run()
	logger.Log([]string{"ID:" + id, "FINISH"})

	if err != nil {
		return nil, &Error{
			Err:    err,
			Debug:  strings.Join([]string{cmd.Path, joinedArgs}, " "),
			Stderr: stderr.String(),
		}
	}

	// assume if you passed in something for stdout, that you know what to do with it
	if c.Stdout != nil {
		return nil, nil
	}

	lines := strings.Split(stdout.String(), "\n")

	//last line is always blank
	lines = lines[0 : len(lines)-1]
	output := make([][]string, len(lines))

	for i, l := range lines {
		output[i] = strings.Fields(l)
	}

	return output, nil
}

func setString(field *string, value string) {
	v := ""
	if value != "-" {
		v = value
	}
	*field = v
}

func setUint(field *uint64, value string) error {
	var v uint64
	if value != "-" {
		var err error
		v, err = strconv.ParseUint(value, 10, 64)
		if err != nil {
			return err
		}
	}
	*field = v
	return nil
}

func (ds *Dataset) parseLine(line []string) error {
	var err error

	if len(line) != len(dsPropList) {
		return errors.New("Output does not match what is expected on this platform")
	}
	setString(&ds.Name, line[0])
	setString(&ds.Origin, line[1])

	if err = setUint(&ds.Used, line[2]); err != nil {
		return err
	}
	if err = setUint(&ds.Avail, line[3]); err != nil {
		return err
	}

	setString(&ds.Mountpoint, line[4])
	setString(&ds.Compression, line[5])
	setString(&ds.Type, line[6])

	if err = setUint(&ds.Volsize, line[7]); err != nil {
		return err
	}
	if err = setUint(&ds.Quota, line[8]); err != nil {
		return err
	}
	if err = setUint(&ds.Referenced, line[9]); err != nil {
		return err
	}

	if runtime.GOOS == "solaris" {
		return nil
	}

	if err = setUint(&ds.Written, line[10]); err != nil {
		return err
	}
	if err = setUint(&ds.Logicalused, line[11]); err != nil {
		return err
	}
	if err = setUint(&ds.Usedbydataset, line[12]); err != nil {
		return err
	}

	return nil
}

/*
 * from zfs diff`s escape function:
 *
 * Prints a file name out a character at a time.  If the character is
 * not in the range of what we consider "printable" ASCII, display it
 * as an escaped 3-digit octal value.  ASCII values less than a space
 * are all control characters and we declare the upper end as the
 * DELete character.  This also is the last 7-bit ASCII character.
 * We choose to treat all 8-bit ASCII as not printable for this
 * application.
 */
func unescapeFilepath(path string) (string, error) {
	buf := make([]byte, 0, len(path))
	llen := len(path)
	for i := 0; i < llen; {
		if path[i] == '\\' {
			if llen < i+4 {
				return "", fmt.Errorf("Invalid octal code: too short")
			}
			octalCode := path[(i + 1):(i + 4)]
			val, err := strconv.ParseUint(octalCode, 8, 8)
			if err != nil {
				return "", fmt.Errorf("Invalid octal code: %v", err)
			}
			buf = append(buf, byte(val))
			i += 4
		} else {
			buf = append(buf, path[i])
			i++
		}
	}
	return string(buf), nil
}

var changeTypeMap = map[string]ChangeType{
	"-": Removed,
	"+": Created,
	"M": Modified,
	"R": Renamed,
}
var inodeTypeMap = map[string]InodeType{
	"B": BlockDevice,
	"C": CharacterDevice,
	"/": Directory,
	">": Door,
	"|": NamedPipe,
	"@": SymbolicLink,
	"P": EventPort,
	"=": Socket,
	"F": File,
}

// matches (+1) or (-1)
var referenceCountRegex = regexp.MustCompile("\\(([+-]\\d+?)\\)")

func parseReferenceCount(field string) (int, error) {
	matches := referenceCountRegex.FindStringSubmatch(field)
	if matches == nil {
		return 0, fmt.Errorf("Regexp does not match")
	}
	return strconv.Atoi(matches[1])
}

func parseInodeChange(line []string) (*InodeChange, error) {
	llen := len(line)
	if llen < 1 {
		return nil, fmt.Errorf("Empty line passed")
	}

	changeType := changeTypeMap[line[0]]
	if changeType == 0 {
		return nil, fmt.Errorf("Unknown change type '%s'", line[0])
	}

	switch changeType {
	case Renamed:
		if llen != 4 {
			return nil, fmt.Errorf("Mismatching number of fields: expect 4, got: %d", llen)
		}
	case Modified:
		if llen != 4 && llen != 3 {
			return nil, fmt.Errorf("Mismatching number of fields: expect 3..4, got: %d", llen)
		}
	default:
		if llen != 3 {
			return nil, fmt.Errorf("Mismatching number of fields: expect 3, got: %d", llen)
		}
	}

	inodeType := inodeTypeMap[line[1]]
	if inodeType == 0 {
		return nil, fmt.Errorf("Unknown inode type '%s'", line[1])
	}

	path, err := unescapeFilepath(line[2])
	if err != nil {
		return nil, fmt.Errorf("Failed to parse filename: %v", err)
	}

	var newPath string
	var referenceCount int
	switch changeType {
	case Renamed:
		newPath, err = unescapeFilepath(line[3])
		if err != nil {
			return nil, fmt.Errorf("Failed to parse filename: %v", err)
		}
	case Modified:
		if llen == 4 {
			referenceCount, err = parseReferenceCount(line[3])
			if err != nil {
				return nil, fmt.Errorf("Failed to parse reference count: %v", err)
			}
		}
	default:
		newPath = ""
	}

	return &InodeChange{
		Change:               changeType,
		Type:                 inodeType,
		Path:                 path,
		NewPath:              newPath,
		ReferenceCountChange: referenceCount,
	}, nil
}

// example input
//M       /       /testpool/bar/
//+       F       /testpool/bar/hello.txt
//M       /       /testpool/bar/hello.txt (+1)
//M       /       /testpool/bar/hello-hardlink
func parseInodeChanges(lines [][]string) ([]*InodeChange, error) {
	changes := make([]*InodeChange, len(lines))

	for i, line := range lines {
		c, err := parseInodeChange(line)
		if err != nil {
			return nil, fmt.Errorf("Failed to parse line %d of zfs diff: %v, got: '%s'", i, err, line)
		}
		changes[i] = c
	}
	return changes, nil
}

func listByType(t, filter string) ([]*Dataset, error) {
	args := []string{"list", "-rHp", "-t", t, "-o", dsPropListOptions}

	if filter != "" {
		args = append(args, filter)
	}
	out, err := zfs(args...)
	if err != nil {
		return nil, err
	}

	var datasets []*Dataset

	name := ""
	var ds *Dataset
	for _, line := range out {
		if name != line[0] {
			name = line[0]
			ds = &Dataset{Name: name}
			datasets = append(datasets, ds)
		}
		if err := ds.parseLine(line); err != nil {
			return nil, err
		}
	}

	return datasets, nil
}

func propsSlice(properties map[string]string) []string {
	args := make([]string, 0, len(properties)*3)
	for k, v := range properties {
		args = append(args, "-o")
		args = append(args, fmt.Sprintf("%s=%s", k, v))
	}
	return args
}

func (z *Zpool) parseLine(line []string) error {
	prop := line[1]
	val := line[2]

	var err error

	switch prop {
	case "name":
		setString(&z.Name, val)
	case "health":
		setString(&z.Health, val)
	case "allocated":
		err = setUint(&z.Allocated, val)
	case "size":
		err = setUint(&z.Size, val)
	case "free":
		err = setUint(&z.Free, val)
	case "fragmentation":
		// Trim trailing "%" before parsing uint
		i := strings.Index(val, "%")
		if i < 0 {
			i = len(val)
		}
		err = setUint(&z.Fragmentation, val[:i])
	case "readonly":
		z.ReadOnly = val == "on"
	case "freeing":
		err = setUint(&z.Freeing, val)
	case "leaked":
		err = setUint(&z.Leaked, val)
	case "dedupratio":
		// Trim trailing "x" before parsing float64
		z.DedupRatio, err = strconv.ParseFloat(val[:len(val)-1], 64)
	}
	return err
}
<commit_msg>remove command name from joined args<commit_after>package zfs

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"os/exec"
	"regexp"
	"runtime"
	"strconv"
	"strings"

	"github.com/pborman/uuid"
)

type command struct {
	Command string
	Stdin   io.Reader
	Stdout  io.Writer
}

func (c *command) Run(arg ...string) ([][]string, error) {

	cmd := exec.Command(c.Command, arg...)

	var stdout, stderr bytes.Buffer

	if c.Stdout == nil {
		cmd.Stdout = &stdout
	} else {
		cmd.Stdout = c.Stdout
	}

	if c.Stdin != nil {
		cmd.Stdin = c.Stdin

	}
	cmd.Stderr = &stderr

	id := uuid.New()
	joinedArgs := strings.Join(cmd.Args, " ")

	logger.Log([]string{"ID:" + id, "START", joinedArgs})
	err := cmd.Run()
	logger.Log([]string{"ID:" + id, "FINISH"})

	if err != nil {
		return nil, &Error{
			Err:    err,
			Debug:  strings.Join([]string{cmd.Path, joinedArgs[1:]}, " "),
			Stderr: stderr.String(),
		}
	}

	// assume if you passed in something for stdout, that you know what to do with it
	if c.Stdout != nil {
		return nil, nil
	}

	lines := strings.Split(stdout.String(), "\n")

	//last line is always blank
	lines = lines[0 : len(lines)-1]
	output := make([][]string, len(lines))

	for i, l := range lines {
		output[i] = strings.Fields(l)
	}

	return output, nil
}

func setString(field *string, value string) {
	v := ""
	if value != "-" {
		v = value
	}
	*field = v
}

func setUint(field *uint64, value string) error {
	var v uint64
	if value != "-" {
		var err error
		v, err = strconv.ParseUint(value, 10, 64)
		if err != nil {
			return err
		}
	}
	*field = v
	return nil
}

func (ds *Dataset) parseLine(line []string) error {
	var err error

	if len(line) != len(dsPropList) {
		return errors.New("Output does not match what is expected on this platform")
	}
	setString(&ds.Name, line[0])
	setString(&ds.Origin, line[1])

	if err = setUint(&ds.Used, line[2]); err != nil {
		return err
	}
	if err = setUint(&ds.Avail, line[3]); err != nil {
		return err
	}

	setString(&ds.Mountpoint, line[4])
	setString(&ds.Compression, line[5])
	setString(&ds.Type, line[6])

	if err = setUint(&ds.Volsize, line[7]); err != nil {
		return err
	}
	if err = setUint(&ds.Quota, line[8]); err != nil {
		return err
	}
	if err = setUint(&ds.Referenced, line[9]); err != nil {
		return err
	}

	if runtime.GOOS == "solaris" {
		return nil
	}

	if err = setUint(&ds.Written, line[10]); err != nil {
		return err
	}
	if err = setUint(&ds.Logicalused, line[11]); err != nil {
		return err
	}
	if err = setUint(&ds.Usedbydataset, line[12]); err != nil {
		return err
	}

	return nil
}

/*
 * from zfs diff`s escape function:
 *
 * Prints a file name out a character at a time.  If the character is
 * not in the range of what we consider "printable" ASCII, display it
 * as an escaped 3-digit octal value.  ASCII values less than a space
 * are all control characters and we declare the upper end as the
 * DELete character.  This also is the last 7-bit ASCII character.
 * We choose to treat all 8-bit ASCII as not printable for this
 * application.
 */
func unescapeFilepath(path string) (string, error) {
	buf := make([]byte, 0, len(path))
	llen := len(path)
	for i := 0; i < llen; {
		if path[i] == '\\' {
			if llen < i+4 {
				return "", fmt.Errorf("Invalid octal code: too short")
			}
			octalCode := path[(i + 1):(i + 4)]
			val, err := strconv.ParseUint(octalCode, 8, 8)
			if err != nil {
				return "", fmt.Errorf("Invalid octal code: %v", err)
			}
			buf = append(buf, byte(val))
			i += 4
		} else {
			buf = append(buf, path[i])
			i++
		}
	}
	return string(buf), nil
}

var changeTypeMap = map[string]ChangeType{
	"-": Removed,
	"+": Created,
	"M": Modified,
	"R": Renamed,
}
var inodeTypeMap = map[string]InodeType{
	"B": BlockDevice,
	"C": CharacterDevice,
	"/": Directory,
	">": Door,
	"|": NamedPipe,
	"@": SymbolicLink,
	"P": EventPort,
	"=": Socket,
	"F": File,
}

// matches (+1) or (-1)
var referenceCountRegex = regexp.MustCompile("\\(([+-]\\d+?)\\)")

func parseReferenceCount(field string) (int, error) {
	matches := referenceCountRegex.FindStringSubmatch(field)
	if matches == nil {
		return 0, fmt.Errorf("Regexp does not match")
	}
	return strconv.Atoi(matches[1])
}

func parseInodeChange(line []string) (*InodeChange, error) {
	llen := len(line)
	if llen < 1 {
		return nil, fmt.Errorf("Empty line passed")
	}

	changeType := changeTypeMap[line[0]]
	if changeType == 0 {
		return nil, fmt.Errorf("Unknown change type '%s'", line[0])
	}

	switch changeType {
	case Renamed:
		if llen != 4 {
			return nil, fmt.Errorf("Mismatching number of fields: expect 4, got: %d", llen)
		}
	case Modified:
		if llen != 4 && llen != 3 {
			return nil, fmt.Errorf("Mismatching number of fields: expect 3..4, got: %d", llen)
		}
	default:
		if llen != 3 {
			return nil, fmt.Errorf("Mismatching number of fields: expect 3, got: %d", llen)
		}
	}

	inodeType := inodeTypeMap[line[1]]
	if inodeType == 0 {
		return nil, fmt.Errorf("Unknown inode type '%s'", line[1])
	}

	path, err := unescapeFilepath(line[2])
	if err != nil {
		return nil, fmt.Errorf("Failed to parse filename: %v", err)
	}

	var newPath string
	var referenceCount int
	switch changeType {
	case Renamed:
		newPath, err = unescapeFilepath(line[3])
		if err != nil {
			return nil, fmt.Errorf("Failed to parse filename: %v", err)
		}
	case Modified:
		if llen == 4 {
			referenceCount, err = parseReferenceCount(line[3])
			if err != nil {
				return nil, fmt.Errorf("Failed to parse reference count: %v", err)
			}
		}
	default:
		newPath = ""
	}

	return &InodeChange{
		Change:               changeType,
		Type:                 inodeType,
		Path:                 path,
		NewPath:              newPath,
		ReferenceCountChange: referenceCount,
	}, nil
}

// example input
//M       /       /testpool/bar/
//+       F       /testpool/bar/hello.txt
//M       /       /testpool/bar/hello.txt (+1)
//M       /       /testpool/bar/hello-hardlink
func parseInodeChanges(lines [][]string) ([]*InodeChange, error) {
	changes := make([]*InodeChange, len(lines))

	for i, line := range lines {
		c, err := parseInodeChange(line)
		if err != nil {
			return nil, fmt.Errorf("Failed to parse line %d of zfs diff: %v, got: '%s'", i, err, line)
		}
		changes[i] = c
	}
	return changes, nil
}

func listByType(t, filter string) ([]*Dataset, error) {
	args := []string{"list", "-rHp", "-t", t, "-o", dsPropListOptions}

	if filter != "" {
		args = append(args, filter)
	}
	out, err := zfs(args...)
	if err != nil {
		return nil, err
	}

	var datasets []*Dataset

	name := ""
	var ds *Dataset
	for _, line := range out {
		if name != line[0] {
			name = line[0]
			ds = &Dataset{Name: name}
			datasets = append(datasets, ds)
		}
		if err := ds.parseLine(line); err != nil {
			return nil, err
		}
	}

	return datasets, nil
}

func propsSlice(properties map[string]string) []string {
	args := make([]string, 0, len(properties)*3)
	for k, v := range properties {
		args = append(args, "-o")
		args = append(args, fmt.Sprintf("%s=%s", k, v))
	}
	return args
}

func (z *Zpool) parseLine(line []string) error {
	prop := line[1]
	val := line[2]

	var err error

	switch prop {
	case "name":
		setString(&z.Name, val)
	case "health":
		setString(&z.Health, val)
	case "allocated":
		err = setUint(&z.Allocated, val)
	case "size":
		err = setUint(&z.Size, val)
	case "free":
		err = setUint(&z.Free, val)
	case "fragmentation":
		// Trim trailing "%" before parsing uint
		i := strings.Index(val, "%")
		if i < 0 {
			i = len(val)
		}
		err = setUint(&z.Fragmentation, val[:i])
	case "readonly":
		z.ReadOnly = val == "on"
	case "freeing":
		err = setUint(&z.Freeing, val)
	case "leaked":
		err = setUint(&z.Leaked, val)
	case "dedupratio":
		// Trim trailing "x" before parsing float64
		z.DedupRatio, err = strconv.ParseFloat(val[:len(val)-1], 64)
	}
	return err
}
<|endoftext|>


<commit_before>package nlp

import "gonum.org/v1/gonum/mat"

// ColDo executes fn for each column j in m
func ColDo(m mat.Matrix, fn func(j int, vec mat.Vector)) {
	if v, isOk := m.(mat.Vector); isOk {
		fn(0, v)
		return
	}

	if cv, isOk := m.(mat.ColViewer); isOk {
		_, c := m.Dims()
		for j := 0; j < c; j++ {
			fn(j, cv.ColView(j))
		}
		return
	}

	if cv, isOk := m.(mat.RawColViewer); isOk {
		r, c := m.Dims()
		for j := 0; j < c; j++ {
			fn(j, mat.NewVecDense(r, cv.RawColView(j)))
		}
		return
	}

	r, c := m.Dims()
	for j := 0; j < c; j++ {
		fn(j, mat.NewVecDense(r, mat.Col(nil, j, m)))
	}
}

// ColNonZeroElemDo executes fn for each element in column j of matrix m.
// If m implements mat.ColNonZeroDoer then only non-zero elements
// will be visited.
func ColNonZeroElemDo(m mat.Matrix, j int, fn func(i, j int, v float64)) {
	colNonZeroDoer, isSparse := m.(mat.ColNonZeroDoer)
	r, _ := m.Dims()

	if isSparse {
		colNonZeroDoer.DoColNonZero(j, fn)
	} else {
		for i := 0; i < r; i++ {
			v := m.At(i, j)
			if v != 0 {
				fn(i, j, v)
			}
		}
	}
}
<commit_msg>updated documentation<commit_after>package nlp

import "gonum.org/v1/gonum/mat"

// ColDo executes fn for each column j in m
func ColDo(m mat.Matrix, fn func(j int, vec mat.Vector)) {
	if v, isOk := m.(mat.Vector); isOk {
		fn(0, v)
		return
	}

	if cv, isOk := m.(mat.ColViewer); isOk {
		_, c := m.Dims()
		for j := 0; j < c; j++ {
			fn(j, cv.ColView(j))
		}
		return
	}

	if cv, isOk := m.(mat.RawColViewer); isOk {
		r, c := m.Dims()
		for j := 0; j < c; j++ {
			fn(j, mat.NewVecDense(r, cv.RawColView(j)))
		}
		return
	}

	r, c := m.Dims()
	for j := 0; j < c; j++ {
		fn(j, mat.NewVecDense(r, mat.Col(nil, j, m)))
	}
}

// ColNonZeroElemDo executes fn for each non-zero element in column j of matrix m.
// If m implements mat.ColNonZeroDoer then this interface will be used to perform
// the iteration.
func ColNonZeroElemDo(m mat.Matrix, j int, fn func(i, j int, v float64)) {
	colNonZeroDoer, isSparse := m.(mat.ColNonZeroDoer)
	r, _ := m.Dims()

	if isSparse {
		colNonZeroDoer.DoColNonZero(j, fn)
	} else {
		for i := 0; i < r; i++ {
			v := m.At(i, j)
			if v != 0 {
				fn(i, j, v)
			}
		}
	}
}
<|endoftext|>


<commit_before>package gitbackup

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
)

type repository struct {
	name     string
	cloneURL string
}

// BackupTarget backs up an entity that holds one or more git repositories and
// has an interface to retrieve that list of repositories.
// Examples of entities include:
//   - A GitHub user.
//   - A BitBucket user.
//   - A GitHub organization.
func BackupTarget(target Target, backupDirectory string) error {
	fmt.Printf("########## Backing up target %s ##########\n\n", target.Name)

	// Retrieve a list of all the git repositories available from the target.
	var repoList []repository
	var err error
	switch target.Source {
	case "github":
		repoList, err = getGitHubRepoList(target, backupDirectory)
	case "bitbucket":
		repoList, err = getBitBucketRepoList(target, backupDirectory)
	default:
		err = fmt.Errorf(`"%s" is not a recognized source type`, target.Source)
	}
	if err != nil {
		return err
	}

	// Back up each repository found.
	backupSuccesses := 0
	for _, repo := range repoList {
		fmt.Println(fmt.Sprintf("#> %s", repo.name))
		if includeRepository(repo.name, target) {
			backupSuccess := backupRepository(
				target.Name,
				repo.name,
				repo.cloneURL,
				backupDirectory,
			)
			if backupSuccess {
				backupSuccesses++
			}
			fmt.Println("")
		} else {
			fmt.Print("Skipped.\n\n")
		}
	}

	// Print out summary of the backup actions performed for this target.
	fmt.Printf(
		"Backed up %d/%d repositories successfully.\n\n\n",
		backupSuccesses,
		len(repoList),
	)

	return nil
}

// getGitHubRepoList finds all the repositories belonging to a given user or
// organization on GitHub.
func getGitHubRepoList(target Target, backupDirectory string) ([]repository, error) {
	// Create URL to request list of repos.
	requestURL := fmt.Sprintf(
		"https://api.github.com/%s/%s/repos?access_token=%s&per_page=200",
		target.Type,
		target.Entity,
		target.Token,
	)

	// Retrieve list of repositories.
	response, err := http.Get(requestURL)
	if err != nil {
		return nil, fmt.Errorf("Failed to connect with the source to retrieve the list of repositories: %s", err)
	}
	defer response.Body.Close()
	contents, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, fmt.Errorf("Failed to retrieve the list of repositories: %s", err)
	}

	// Parse JSON response.
	var data []map[string]interface{}
	if err := json.Unmarshal(contents, &data); err != nil {
		return nil, fmt.Errorf("Failed to parse JSON: %s", err)
	}

	// Make a list of repositories.
	var repoList []repository
	for _, repo := range data {
		repoName, _ := repo["name"].(string)
		cloneURL, _ := repo["clone_url"].(string)
		cloneURL = strings.Replace(
			cloneURL,
			"https://",
			fmt.Sprintf("https://%s:%s@", target.Entity, target.Token),
			1,
		)
		repo := repository{name: repoName, cloneURL: cloneURL}
		repoList = append(repoList, repo)
	}

	// No errors.
	return repoList, nil
}

// getBitBucketRepoList finds all the repositories belonging to a given user on
// BitBucket.
func getBitBucketRepoList(target Target, backupDirectory string) ([]repository, error) {
	// Create URL to request list of repos.
	// TODO: support pagination.
	requestURL := fmt.Sprintf(
		"https://%s:%s@bitbucket.org/api/2.0/repositories/%s?page=1&pagelen=100",
		target.Entity,
		target.Password,
		target.Entity,
	)

	// Retrieve list of repositories.
	response, err := http.Get(requestURL)
	if err != nil {
		return nil, fmt.Errorf("Failed to connect with the source to retrieve the list of repositories: %s", err)
	}
	defer response.Body.Close()
	contents, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, fmt.Errorf("Failed to retrieve the list of repositories: %s", err)
	}

	// Parse JSON response.
	var metadata map[string]json.RawMessage
	if err := json.Unmarshal(contents, &metadata); err != nil {
		return nil, fmt.Errorf("Failed to parse JSON: %s", err)
	}
	var data []map[string]json.RawMessage
	if err := json.Unmarshal(metadata["values"], &data); err != nil {
		return nil, fmt.Errorf("Failed to parse JSON: %s", err)
	}

	// Make a list of repositories.
	repoList := make([]repository, len(data))
	for i, repo := range data {
		// Parse the remaining JSON message that pertains to this repository.
		var repoName string
		if err := json.Unmarshal(repo["name"], &repoName); err != nil {
			return nil, fmt.Errorf("Failed to parse JSON: %s", err)
		}
		var links map[string]json.RawMessage
		if err := json.Unmarshal(repo["links"], &links); err != nil {
			return nil, fmt.Errorf("Failed to parse JSON: %s", err)
		}
		var cloneLinks []map[string]string
		if err := json.Unmarshal(links["clone"], &cloneLinks); err != nil {
			return nil, fmt.Errorf("Failed to parse JSON: %s", err)
		}

		// Find the https URL to use for cloning.
		var cloneURL string
		for _, link := range cloneLinks {
			if link["name"] == "https" {
				cloneURL = link["href"]
			}
		}
		if cloneURL == "" {
			return nil, fmt.Errorf("Could not determine HTTPS cloning URL: %s", cloneLinks)
		}

		// Determine URL for cloning.
		cloneURL = strings.Replace(
			cloneURL,
			fmt.Sprintf("https://%s@", target.Entity),
			fmt.Sprintf("https://%s:%s@", target.Entity, target.Password),
			1,
		)

		repoList[i] = repository{name: repoName, cloneURL: cloneURL}
	}

	// No errors.
	return repoList, nil
}

// backupRepository takes a remote git repository and backs it up locally.
// Note that this makes a mirror repository - in other words, the backup only
// contains the content of a normal .git repository but no working directory,
// which saves space. You can always get a normal repository from the backup by
// doing a normal git clone of the backup itself.
func backupRepository(targetName string, repoName string, cloneURL string, backupDirectory string) bool {
	cloneDirectory := filepath.Join(backupDirectory, targetName, repoName)

	if _, err := os.Stat(cloneDirectory); os.IsNotExist(err) {
		// The repo doesn't exist locally, clone it.
		cmd := exec.Command("git", "clone", "--mirror", cloneURL, cloneDirectory)
		cmdOut, err := cmd.CombinedOutput()
		if err != nil {
			fmt.Println("Error cloning the repository:", err)
			return false
		}
		if len(cmdOut) > 0 {
			fmt.Printf(string(cmdOut))
		}
		fmt.Println("Cloned repository.")
	} else {
		// The repo already exists.
		// Set the origin again in case the credentials have changed.
		cmd := exec.Command("git", "remote", "set-url", "origin", cloneURL)
		cmd.Dir = cloneDirectory
		cmdOut, err := cmd.CombinedOutput()
		if err != nil {
			fmt.Println("Error setting remote URL:", err)
			return false
		}

		// Pull updates.
		cmd = exec.Command("git", "remote", "update", "-p")
		cmd.Dir = cloneDirectory
		cmdOut, err = cmd.CombinedOutput()
		if err != nil {
			fmt.Println("Error pulling in the repository:", err)
			return false
		}

		// Display pulled information.
		if len(cmdOut) > 0 {
			fmt.Printf(string(cmdOut))
		}
		fmt.Println("Pulled latest updates in the repository.")
	}

	return true
}

// includeRepository takes a repository name and the information about the
// target it is part of, and determines whether the repository should be backed
// up or not.
func includeRepository(repoName string, target Target) bool {
	if target.Skip != "" {
		r, err := regexp.Compile(target.Skip)
		if err != nil {
			log.Fatalf(`"skip" does not specify a valid regular expression: %s`, err)
		}
		return r.MatchString(repoName) == false
	}
	if target.Only != "" {
		r, err := regexp.Compile(target.Only)
		if err != nil {
			log.Fatalf(`"only" does not specify a valid regular expression: %s`, err)
		}
		return r.MatchString(repoName) == true
	}

	return true
}
<commit_msg>Improve backup result reporting for each target.<commit_after>package gitbackup

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
)

type repository struct {
	name     string
	cloneURL string
}

// BackupTarget backs up an entity that holds one or more git repositories and
// has an interface to retrieve that list of repositories.
// Examples of entities include:
//   - A GitHub user.
//   - A BitBucket user.
//   - A GitHub organization.
func BackupTarget(target Target, backupDirectory string) error {
	fmt.Printf("########## Backing up target %s ##########\n\n", target.Name)

	// Retrieve a list of all the git repositories available from the target.
	var repoList []repository
	var err error
	switch target.Source {
	case "github":
		repoList, err = getGitHubRepoList(target, backupDirectory)
	case "bitbucket":
		repoList, err = getBitBucketRepoList(target, backupDirectory)
	default:
		err = fmt.Errorf(`"%s" is not a recognized source type`, target.Source)
	}
	if err != nil {
		return err
	}

	// Back up each repository found.
	stats := map[string]int{
		"success": 0,
		"skipped": 0,
		"failure": 0,
	}
	for _, repo := range repoList {
		fmt.Println(fmt.Sprintf("#> %s", repo.name))
		if includeRepository(repo.name, target) {
			backupSuccess := backupRepository(
				target.Name,
				repo.name,
				repo.cloneURL,
				backupDirectory,
			)
			if backupSuccess {
				stats["success"]++
			} else {
				stats["failure"]++
			}
			fmt.Println("")
		} else {
			fmt.Print("Skipped.\n\n")
			stats["skipped"]++
		}
	}

	// Print out summary of the backup actions performed for this target.
	fmt.Printf(
		"%d success(es) + %d skipped + %d failure(s) = %d repositories for this target.\n\n\n",
		stats["success"],
		stats["skipped"],
		stats["failure"],
		len(repoList),
	)

	return nil
}

// getGitHubRepoList finds all the repositories belonging to a given user or
// organization on GitHub.
func getGitHubRepoList(target Target, backupDirectory string) ([]repository, error) {
	// Create URL to request list of repos.
	requestURL := fmt.Sprintf(
		"https://api.github.com/%s/%s/repos?access_token=%s&per_page=200",
		target.Type,
		target.Entity,
		target.Token,
	)

	// Retrieve list of repositories.
	response, err := http.Get(requestURL)
	if err != nil {
		return nil, fmt.Errorf("Failed to connect with the source to retrieve the list of repositories: %s", err)
	}
	defer response.Body.Close()
	contents, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, fmt.Errorf("Failed to retrieve the list of repositories: %s", err)
	}

	// Parse JSON response.
	var data []map[string]interface{}
	if err := json.Unmarshal(contents, &data); err != nil {
		return nil, fmt.Errorf("Failed to parse JSON: %s", err)
	}

	// Make a list of repositories.
	var repoList []repository
	for _, repo := range data {
		repoName, _ := repo["name"].(string)
		cloneURL, _ := repo["clone_url"].(string)
		cloneURL = strings.Replace(
			cloneURL,
			"https://",
			fmt.Sprintf("https://%s:%s@", target.Entity, target.Token),
			1,
		)
		repo := repository{name: repoName, cloneURL: cloneURL}
		repoList = append(repoList, repo)
	}

	// No errors.
	return repoList, nil
}

// getBitBucketRepoList finds all the repositories belonging to a given user on
// BitBucket.
func getBitBucketRepoList(target Target, backupDirectory string) ([]repository, error) {
	// Create URL to request list of repos.
	// TODO: support pagination.
	requestURL := fmt.Sprintf(
		"https://%s:%s@bitbucket.org/api/2.0/repositories/%s?page=1&pagelen=100",
		target.Entity,
		target.Password,
		target.Entity,
	)

	// Retrieve list of repositories.
	response, err := http.Get(requestURL)
	if err != nil {
		return nil, fmt.Errorf("Failed to connect with the source to retrieve the list of repositories: %s", err)
	}
	defer response.Body.Close()
	contents, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, fmt.Errorf("Failed to retrieve the list of repositories: %s", err)
	}

	// Parse JSON response.
	var metadata map[string]json.RawMessage
	if err := json.Unmarshal(contents, &metadata); err != nil {
		return nil, fmt.Errorf("Failed to parse JSON: %s", err)
	}
	var data []map[string]json.RawMessage
	if err := json.Unmarshal(metadata["values"], &data); err != nil {
		return nil, fmt.Errorf("Failed to parse JSON: %s", err)
	}

	// Make a list of repositories.
	repoList := make([]repository, len(data))
	for i, repo := range data {
		// Parse the remaining JSON message that pertains to this repository.
		var repoName string
		if err := json.Unmarshal(repo["name"], &repoName); err != nil {
			return nil, fmt.Errorf("Failed to parse JSON: %s", err)
		}
		var links map[string]json.RawMessage
		if err := json.Unmarshal(repo["links"], &links); err != nil {
			return nil, fmt.Errorf("Failed to parse JSON: %s", err)
		}
		var cloneLinks []map[string]string
		if err := json.Unmarshal(links["clone"], &cloneLinks); err != nil {
			return nil, fmt.Errorf("Failed to parse JSON: %s", err)
		}

		// Find the https URL to use for cloning.
		var cloneURL string
		for _, link := range cloneLinks {
			if link["name"] == "https" {
				cloneURL = link["href"]
			}
		}
		if cloneURL == "" {
			return nil, fmt.Errorf("Could not determine HTTPS cloning URL: %s", cloneLinks)
		}

		// Determine URL for cloning.
		cloneURL = strings.Replace(
			cloneURL,
			fmt.Sprintf("https://%s@", target.Entity),
			fmt.Sprintf("https://%s:%s@", target.Entity, target.Password),
			1,
		)

		repoList[i] = repository{name: repoName, cloneURL: cloneURL}
	}

	// No errors.
	return repoList, nil
}

// backupRepository takes a remote git repository and backs it up locally.
// Note that this makes a mirror repository - in other words, the backup only
// contains the content of a normal .git repository but no working directory,
// which saves space. You can always get a normal repository from the backup by
// doing a normal git clone of the backup itself.
func backupRepository(targetName string, repoName string, cloneURL string, backupDirectory string) bool {
	cloneDirectory := filepath.Join(backupDirectory, targetName, repoName)

	if _, err := os.Stat(cloneDirectory); os.IsNotExist(err) {
		// The repo doesn't exist locally, clone it.
		cmd := exec.Command("git", "clone", "--mirror", cloneURL, cloneDirectory)
		cmdOut, err := cmd.CombinedOutput()
		if err != nil {
			fmt.Println("Error cloning the repository:", err)
			return false
		}
		if len(cmdOut) > 0 {
			fmt.Printf(string(cmdOut))
		}
		fmt.Println("Cloned repository.")
	} else {
		// The repo already exists.
		// Set the origin again in case the credentials have changed.
		cmd := exec.Command("git", "remote", "set-url", "origin", cloneURL)
		cmd.Dir = cloneDirectory
		cmdOut, err := cmd.CombinedOutput()
		if err != nil {
			fmt.Println("Error setting remote URL:", err)
			return false
		}

		// Pull updates.
		cmd = exec.Command("git", "remote", "update", "-p")
		cmd.Dir = cloneDirectory
		cmdOut, err = cmd.CombinedOutput()
		if err != nil {
			fmt.Println("Error pulling in the repository:", err)
			return false
		}

		// Display pulled information.
		if len(cmdOut) > 0 {
			fmt.Printf(string(cmdOut))
		}
		fmt.Println("Pulled latest updates in the repository.")
	}

	return true
}

// includeRepository takes a repository name and the information about the
// target it is part of, and determines whether the repository should be backed
// up or not.
func includeRepository(repoName string, target Target) bool {
	if target.Skip != "" {
		r, err := regexp.Compile(target.Skip)
		if err != nil {
			log.Fatalf(`"skip" does not specify a valid regular expression: %s`, err)
		}
		return r.MatchString(repoName) == false
	}
	if target.Only != "" {
		r, err := regexp.Compile(target.Only)
		if err != nil {
			log.Fatalf(`"only" does not specify a valid regular expression: %s`, err)
		}
		return r.MatchString(repoName) == true
	}

	return true
}
<|endoftext|>


<commit_before>// Copyright 2015 ISRG.  All rights reserved
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"

	"github.com/codegangsta/cli"
	"github.com/jcjones/github-pr-status/api"
)

var authenticationConfig api.AuthConfig

func showError(c *cli.Context, err error) error {
	fmt.Println("ERROR: ")
	fmt.Println(err.Error())
	fmt.Println("")
	cli.ShowCommandHelp(c, c.Command.Name)
	return err
}

func actionSetStatus(c *cli.Context) {
	var err error

	if !c.IsSet("state") {
		err = fmt.Errorf("A state must be specified")
		showError(c, err)
		return
	}
	state := c.String("state")

	switch state {
	case "pending":
		fallthrough
	case "success":
		fallthrough
	case "error":
		fallthrough
	case "failure":
		// OK
		break
	default:
		showError(c, fmt.Errorf("Not a valid state: %s", c.String("state")))
		return
	}

	if !c.IsSet("context") {
		err = fmt.Errorf("A context must be specified")
		showError(c, err)
		return
	}
	context := c.String("context")

	if !c.IsSet("sha") {
		err = fmt.Errorf("A commit ID must be specified")
		showError(c, err)
		return
	}
	sha := c.String("sha")

	// Optionals
	url := c.String("url")
	description := c.String("description")

	// Accept description from stdin
	if description == "-" {
		data, err := ioutil.ReadAll(os.Stdin)
		if err != nil {
			showError(c, err)
			return
		}

		description = string(data)
	}

	// Checked at the global scope
	owner := c.GlobalString("owner")
	repo := c.GlobalString("repo")
	debug := c.GlobalBool("debug")

	fmt.Printf("Status ID=%s State=%s Context=%s URL=%s Description=%s\n", sha, state, context, url, description)
	err = api.StatusSet(authenticationConfig, owner, repo, sha, state, context, url, description, debug)
	if err != nil {
		fmt.Printf("Could not post: %s\n", err.Error())
	}
}

func actionAddComment(c *cli.Context) {
	var err error

	if !c.IsSet("body") {
		err = fmt.Errorf("A comment body must be specified")
		showError(c, err)
		return
	}
	body := c.String("body")

	// Accept body from stdin
	if body == "-" {
		data, err := ioutil.ReadAll(os.Stdin)
		if err != nil {
			showError(c, err)
			return
		}

		body = string(data)
	}

	if !c.IsSet("pr") {
		err = fmt.Errorf("A PR must be specified")
		showError(c, err)
		return
	}

	pr := c.Int("pr")
	if pr < 1 {
		err = fmt.Errorf("PRs must be >0")
		showError(c, err)
		return
	}

	fmt.Printf("Comment PR=#%d, Body=%s\n", pr, body)

	// Checked at the global scope
	owner := c.GlobalString("owner")
	repo := c.GlobalString("repo")
	debug := c.GlobalBool("debug")

	err = api.IssueAddComment(authenticationConfig, owner, repo, body, pr, debug)
	if err != nil {
		fmt.Printf("Could not post: %s\n", err.Error())
	}

}

func loadAuthDetailsAndConfigureLogging(c *cli.Context) (err error) {
	if !c.GlobalIsSet("authfile") {
		err = fmt.Errorf("You must specifiy a path to the authfile.")
		return showError(c, err)
	}

	path := c.GlobalString("authfile")
	configJSON, err := ioutil.ReadFile(path)
	if err != nil {
		err = fmt.Errorf("Could not read authentication details. Path=%s Err=%s", path, err)
		return showError(c, err)
	}

	err = json.Unmarshal(configJSON, &authenticationConfig)
	if err != nil {
		err = fmt.Errorf("Could not parse authentication details. Err=%s", err)
		return showError(c, err)
	}

	if authenticationConfig.Type == nil || *authenticationConfig.Type != "basic" {
		err = fmt.Errorf("Only authentication type supported is 'basic'.")
		return showError(c, err)
	}

	if authenticationConfig.Username == nil {
		err = fmt.Errorf("You must provide a username in the authfile.")
		return showError(c, err)
	}

	if authenticationConfig.Password == nil {
		err = fmt.Errorf("You must provide a password or token in the authfile.")
		return showError(c, err)
	}

	if !c.GlobalIsSet("owner") {
		err = fmt.Errorf("You must provide a repo owner.")
		return showError(c, err)
	}

	if !c.GlobalIsSet("repo") {
		err = fmt.Errorf("You must provide a repo to operate on.")
		return showError(c, err)
	}

	return
}

func main() {
	app := cli.NewApp()
	app.Usage = "GitHub CI PR Tools"
	app.Version = "0.4.0"
	app.Copyright = "Copyright 2015 ISRG.  All rights reserved."
	app.Authors = []cli.Author{
		cli.Author{Name: "James 'J.C.' Jones", Email: "jcjones@letsencrypt.org"},
	}
	app.Before = loadAuthDetailsAndConfigureLogging

	app.Flags = []cli.Flag{
		cli.BoolFlag{Name: "debug, D", Usage: "Enable debugging output"},
		cli.StringFlag{Name: "owner, o", Usage: "Owner segment of GitHub Path"},
		cli.StringFlag{Name: "repo, r", Usage: "Repo segment of GitHub Path"},
		cli.StringFlag{Name: "authfile, f", Usage: "Authentication details"},
	}

	statusCommand := cli.Command{
		Name:   "status",
		Usage:  "Set or update a status on a commit",
		Action: actionSetStatus,
		Flags: []cli.Flag{
			cli.StringFlag{Name: "sha, i", Usage: "Commit ID"},
			cli.StringFlag{Name: "state, s", Usage: "One of {pending, success, error, failure}"},
			cli.StringFlag{Name: "url, l", Usage: "URL to link from the status"},
			cli.StringFlag{Name: "context, c", Usage: "Status context, such as 'integration/gofmt'"},
			cli.StringFlag{Name: "description, d", Usage: "Extra descriptive text, or '-' to read from stdin"},
		},
	}

	commentCommand := cli.Command{
		Name:   "comment",
		Usage:  "Add a comment to a PR",
		Action: actionAddComment,
		Flags: []cli.Flag{
			cli.IntFlag{Name: "pr, n", Usage: "Pull Request #"},
			cli.StringFlag{Name: "body, b", Usage: "Body of the comment, or '-' to read from stdin"},
		},
	}

	app.Commands = []cli.Command{
		statusCommand, commentCommand,
	}

	app.Run(os.Args)
}
<commit_msg>Don't echo body when submitting.<commit_after>// Copyright 2015 ISRG.  All rights reserved
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"

	"github.com/codegangsta/cli"
	"github.com/jcjones/github-pr-status/api"
)

var authenticationConfig api.AuthConfig

func showError(c *cli.Context, err error) error {
	fmt.Println("ERROR: ")
	fmt.Println(err.Error())
	fmt.Println("")
	cli.ShowCommandHelp(c, c.Command.Name)
	return err
}

func actionSetStatus(c *cli.Context) {
	var err error

	if !c.IsSet("state") {
		err = fmt.Errorf("A state must be specified")
		showError(c, err)
		return
	}
	state := c.String("state")

	switch state {
	case "pending":
		fallthrough
	case "success":
		fallthrough
	case "error":
		fallthrough
	case "failure":
		// OK
		break
	default:
		showError(c, fmt.Errorf("Not a valid state: %s", c.String("state")))
		return
	}

	if !c.IsSet("context") {
		err = fmt.Errorf("A context must be specified")
		showError(c, err)
		return
	}
	context := c.String("context")

	if !c.IsSet("sha") {
		err = fmt.Errorf("A commit ID must be specified")
		showError(c, err)
		return
	}
	sha := c.String("sha")

	// Optionals
	url := c.String("url")
	description := c.String("description")

	// Accept description from stdin
	if description == "-" {
		data, err := ioutil.ReadAll(os.Stdin)
		if err != nil {
			showError(c, err)
			return
		}

		description = string(data)
	}

	// Checked at the global scope
	owner := c.GlobalString("owner")
	repo := c.GlobalString("repo")
	debug := c.GlobalBool("debug")

	fmt.Printf("Status ID=%s State=%s Context=%s URL=%s Description=%s\n", sha, state, context, url, description)
	err = api.StatusSet(authenticationConfig, owner, repo, sha, state, context, url, description, debug)
	if err != nil {
		fmt.Printf("Could not post: %s\n", err.Error())
	}
}

func actionAddComment(c *cli.Context) {
	var err error

	if !c.IsSet("body") {
		err = fmt.Errorf("A comment body must be specified")
		showError(c, err)
		return
	}
	body := c.String("body")

	// Accept body from stdin
	if body == "-" {
		data, err := ioutil.ReadAll(os.Stdin)
		if err != nil {
			showError(c, err)
			return
		}

		body = string(data)
	}

	if !c.IsSet("pr") {
		err = fmt.Errorf("A PR must be specified")
		showError(c, err)
		return
	}

	pr := c.Int("pr")
	if pr < 1 {
		err = fmt.Errorf("PRs must be >0")
		showError(c, err)
		return
	}

	fmt.Printf("Comment PR=#%d\n", pr)

	// Checked at the global scope
	owner := c.GlobalString("owner")
	repo := c.GlobalString("repo")
	debug := c.GlobalBool("debug")

	err = api.IssueAddComment(authenticationConfig, owner, repo, body, pr, debug)
	if err != nil {
		fmt.Printf("Could not post: %s\n", err.Error())
	}

}

func loadAuthDetailsAndConfigureLogging(c *cli.Context) (err error) {
	if !c.GlobalIsSet("authfile") {
		err = fmt.Errorf("You must specifiy a path to the authfile.")
		return showError(c, err)
	}

	path := c.GlobalString("authfile")
	configJSON, err := ioutil.ReadFile(path)
	if err != nil {
		err = fmt.Errorf("Could not read authentication details. Path=%s Err=%s", path, err)
		return showError(c, err)
	}

	err = json.Unmarshal(configJSON, &authenticationConfig)
	if err != nil {
		err = fmt.Errorf("Could not parse authentication details. Err=%s", err)
		return showError(c, err)
	}

	if authenticationConfig.Type == nil || *authenticationConfig.Type != "basic" {
		err = fmt.Errorf("Only authentication type supported is 'basic'.")
		return showError(c, err)
	}

	if authenticationConfig.Username == nil {
		err = fmt.Errorf("You must provide a username in the authfile.")
		return showError(c, err)
	}

	if authenticationConfig.Password == nil {
		err = fmt.Errorf("You must provide a password or token in the authfile.")
		return showError(c, err)
	}

	if !c.GlobalIsSet("owner") {
		err = fmt.Errorf("You must provide a repo owner.")
		return showError(c, err)
	}

	if !c.GlobalIsSet("repo") {
		err = fmt.Errorf("You must provide a repo to operate on.")
		return showError(c, err)
	}

	return
}

func main() {
	app := cli.NewApp()
	app.Usage = "GitHub CI PR Tools"
	app.Version = "0.4.0"
	app.Copyright = "Copyright 2015 ISRG.  All rights reserved."
	app.Authors = []cli.Author{
		cli.Author{Name: "James 'J.C.' Jones", Email: "jcjones@letsencrypt.org"},
	}
	app.Before = loadAuthDetailsAndConfigureLogging

	app.Flags = []cli.Flag{
		cli.BoolFlag{Name: "debug, D", Usage: "Enable debugging output"},
		cli.StringFlag{Name: "owner, o", Usage: "Owner segment of GitHub Path"},
		cli.StringFlag{Name: "repo, r", Usage: "Repo segment of GitHub Path"},
		cli.StringFlag{Name: "authfile, f", Usage: "Authentication details"},
	}

	statusCommand := cli.Command{
		Name:   "status",
		Usage:  "Set or update a status on a commit",
		Action: actionSetStatus,
		Flags: []cli.Flag{
			cli.StringFlag{Name: "sha, i", Usage: "Commit ID"},
			cli.StringFlag{Name: "state, s", Usage: "One of {pending, success, error, failure}"},
			cli.StringFlag{Name: "url, l", Usage: "URL to link from the status"},
			cli.StringFlag{Name: "context, c", Usage: "Status context, such as 'integration/gofmt'"},
			cli.StringFlag{Name: "description, d", Usage: "Extra descriptive text, or '-' to read from stdin"},
		},
	}

	commentCommand := cli.Command{
		Name:   "comment",
		Usage:  "Add a comment to a PR",
		Action: actionAddComment,
		Flags: []cli.Flag{
			cli.IntFlag{Name: "pr, n", Usage: "Pull Request #"},
			cli.StringFlag{Name: "body, b", Usage: "Body of the comment, or '-' to read from stdin"},
		},
	}

	app.Commands = []cli.Command{
		statusCommand, commentCommand,
	}

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package functions

// FuncNames defines the set of available function names available to the CLI
var FuncNames = []string{
	"BroadbandProductType",
	"DataPumpVersion",
	"DataReceived",
	"DataSent",
	"DhcpPoolStart",
	"DhcpPoolEnd",
	"DhcpSubnetMask",
	"DownstreamSyncSpeed",
	"EnableDebug",
	"EventLog",
	"HardwareVersion",
	"InternetConnectionStatus",
	"LightStatus",
	"LocalTime",
	"MaintenaceFirmwareVersion",
	"PublicIPAddress",
	"PublicSubnetMask",
	"Reboot",
	"SambaIP",
	"SambaHost",
	"SerialNumber",
	"SoftwareVersion",
	"UpstreamSyncSpeed",
	"Version",
	"WiFiSecurityMode",
	"WiFiSSID"}
<commit_msg>Add bandwidth monitor support<commit_after>package functions

// FuncNames defines the set of available function names available to the CLI
var FuncNames = []string{
        "BandwidthMonitor",
	"BroadbandProductType",
	"DataPumpVersion",
	"DataReceived",
	"DataSent",
	"DhcpPoolStart",
	"DhcpPoolEnd",
	"DhcpSubnetMask",
	"DownstreamSyncSpeed",
	"EnableDebug",
	"EventLog",
	"HardwareVersion",
	"InternetConnectionStatus",
	"LightStatus",
	"LocalTime",
	"MaintenaceFirmwareVersion",
	"PublicIPAddress",
	"PublicSubnetMask",
	"Reboot",
	"SambaIP",
	"SambaHost",
	"SerialNumber",
	"SoftwareVersion",
	"UpstreamSyncSpeed",
	"Version",
	"WiFiSecurityMode",
	"WiFiSSID"}
<|endoftext|>