<commit_before>package main

import (
	"bytes"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/pem"
	"fmt"
	"io"
	"io/ioutil"
)

// returns a "normalized" variant of the pkix.Name
// which might be used as a file on disk
func clientIdByName(name *pkix.Name) string {

	nameBytes := typeValsToBytes(name.Names, true)
	return string(nameBytes)
}

func printClientIdTo(w io.Writer, certFile string) error {
	var (
		cert          *x509.Certificate
		block         *pem.Block
		rawBytes, err = ioutil.ReadFile(certFile)
	)

	if err != nil {
		return err
	}

	for {
		block, rawBytes = pem.Decode(rawBytes)
		if block == nil {
			break
		}
		if block.Type != "CERTIFICATE" || len(block.Headers) != 0 {
			continue
		}

		if cert, err = x509.ParseCertificate(block.Bytes); err != nil {
			return nil
		}

		break
	}

	if cert == nil {
		return fmt.Errorf("cert file %q does not contain a certicate", certFile)
	}

	clientId := clientIdByName(&cert.Subject)
	fmt.Fprintf(w, "%s\n", clientId)

	return nil
}

// lookup table. it's a short table, we just scan it linear in pkixNameToBytes()
// the values are taken from http://golang.org/src/crypto/x509/pkix/pkix.go, see
// oidCountry, oidCommonName etc.
var oidToKeys = [...]struct {
	oid [4]int
	key string
}{
	{[4]int{2, 5, 4, 6}, "C"},   // country
	{[4]int{2, 5, 4, 10}, "O"},  // organization
	{[4]int{2, 5, 4, 11}, "OU"}, // organizational unit
	{[4]int{2, 5, 4, 3}, "CN"},  // common name
	{[4]int{2, 5, 4, 5}, "SN"},  // serial number
	{[4]int{2, 5, 4, 7}, "L"},   // locality
	{[4]int{2, 5, 4, 8}, "P"},   // province
	{[4]int{2, 5, 4, 9}, "S"},   // street // TODO: check correct key
	{[4]int{2, 5, 4, 17}, "PC"}, // postal code // TODO: check correct key
}

// returns a serialized version of 'name' for each known
// asn1.object-identifier ("key") in the form:
//  key1=value1,key2=value2,key3=value3...
//
func typeValsToBytes(names []pkix.AttributeTypeAndValue, cleanValues bool) []byte {

	buf := bytes.NewBuffer(nil)
	for i := range names {

		entry := &names[i]
		key := ""
		for j := 0; j < len(oidToKeys); j++ {
			if len(entry.Type) == 4 &&
				entry.Type[0] == oidToKeys[j].oid[0] &&
				entry.Type[1] == oidToKeys[j].oid[1] &&
				entry.Type[2] == oidToKeys[j].oid[2] &&
				entry.Type[3] == oidToKeys[j].oid[3] {

				key = oidToKeys[j].key
				break
			}
		}

		if key == "" {
			continue
		}

		if buf.Len() > 0 {
			io.WriteString(buf, ",")
		}

		oldLen := len(buf.Bytes())
		fmt.Fprintf(buf, "%s=%s", key, entry.Value)
		if cleanValues {
			cleanPkixNameBytes(buf.Bytes()[oldLen+len(key)+1:])
		}
	}

	return buf.Bytes()
}

// replace any [^a-zA-Z0-9_-] by a '_'
func cleanPkixNameBytes(in []byte) {
	for i := range in {
		c := in[i]
		switch {
		default:
			in[i] = '_'
		case '0' <= c && c <= '9':
		case 'a' <= c && c <= 'z':
		case 'A' <= c && c <= 'Z':
		case c == '_', c == '-':
		}
	}
}
<commit_msg>fix typo<commit_after>package main

import (
	"bytes"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/pem"
	"fmt"
	"io"
	"io/ioutil"
)

// returns a "normalized" variant of the pkix.Name
// which might be used as a file on disk
func clientIdByName(name *pkix.Name) string {

	nameBytes := typeValsToBytes(name.Names, true)
	return string(nameBytes)
}

func printClientIdTo(w io.Writer, certFile string) error {
	var (
		cert          *x509.Certificate
		block         *pem.Block
		rawBytes, err = ioutil.ReadFile(certFile)
	)

	if err != nil {
		return err
	}

	for {
		block, rawBytes = pem.Decode(rawBytes)
		if block == nil {
			break
		}
		if block.Type != "CERTIFICATE" || len(block.Headers) != 0 {
			continue
		}

		if cert, err = x509.ParseCertificate(block.Bytes); err != nil {
			return nil
		}

		break
	}

	if cert == nil {
		return fmt.Errorf("cert file %q does not contain a certificate", certFile)
	}

	clientId := clientIdByName(&cert.Subject)
	fmt.Fprintf(w, "%s\n", clientId)

	return nil
}

// lookup table. it's a short table, we just scan it linear in pkixNameToBytes()
// the values are taken from http://golang.org/src/crypto/x509/pkix/pkix.go, see
// oidCountry, oidCommonName etc.
var oidToKeys = [...]struct {
	oid [4]int
	key string
}{
	{[4]int{2, 5, 4, 6}, "C"},   // country
	{[4]int{2, 5, 4, 10}, "O"},  // organization
	{[4]int{2, 5, 4, 11}, "OU"}, // organizational unit
	{[4]int{2, 5, 4, 3}, "CN"},  // common name
	{[4]int{2, 5, 4, 5}, "SN"},  // serial number
	{[4]int{2, 5, 4, 7}, "L"},   // locality
	{[4]int{2, 5, 4, 8}, "P"},   // province
	{[4]int{2, 5, 4, 9}, "S"},   // street // TODO: check correct key
	{[4]int{2, 5, 4, 17}, "PC"}, // postal code // TODO: check correct key
}

// returns a serialized version of 'name' for each known
// asn1.object-identifier ("key") in the form:
//  key1=value1,key2=value2,key3=value3...
//
func typeValsToBytes(names []pkix.AttributeTypeAndValue, cleanValues bool) []byte {

	buf := bytes.NewBuffer(nil)
	for i := range names {

		entry := &names[i]
		key := ""
		for j := 0; j < len(oidToKeys); j++ {
			if len(entry.Type) == 4 &&
				entry.Type[0] == oidToKeys[j].oid[0] &&
				entry.Type[1] == oidToKeys[j].oid[1] &&
				entry.Type[2] == oidToKeys[j].oid[2] &&
				entry.Type[3] == oidToKeys[j].oid[3] {

				key = oidToKeys[j].key
				break
			}
		}

		if key == "" {
			continue
		}

		if buf.Len() > 0 {
			io.WriteString(buf, ",")
		}

		oldLen := len(buf.Bytes())
		fmt.Fprintf(buf, "%s=%s", key, entry.Value)
		if cleanValues {
			cleanPkixNameBytes(buf.Bytes()[oldLen+len(key)+1:])
		}
	}

	return buf.Bytes()
}

// replace any [^a-zA-Z0-9_-] by a '_'
func cleanPkixNameBytes(in []byte) {
	for i := range in {
		c := in[i]
		switch {
		default:
			in[i] = '_'
		case '0' <= c && c <= '9':
		case 'a' <= c && c <= 'z':
		case 'A' <= c && c <= 'Z':
		case c == '_', c == '-':
		}
	}
}
<|endoftext|>


<commit_before>package parse

import (
	"go/token"
	"go/parser"
	"go/ast"
	"io/ioutil"
	"github.com/thisisfineio/implement"
	"strings"
	//"reflect"
	"fmt"
)

// Parse returns an *ast.File, and an error
func Parse(data []byte, filename string) (*ast.File, []byte, error) {
	fset := token.NewFileSet()
	f, err := parser.ParseFile(fset, filename, data, parser.ParseComments)
	return f, data, err
}

func File(filename string) (*ast.File, []byte, error) {
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, nil, err
	}
	return Parse(data, filename)
}

// Inspect visits all nodes in the *ast.File (recursively)
// data is necessary for us to determine interface names
func Inspect(f *ast.File, data []byte) map[string][]*implement.FunctionSignature {
	signatures := make(map[string][]*implement.FunctionSignature)
	var lastIdent string
	ast.Inspect(f, func(n ast.Node) bool {
		//fmt.Println(n)
		switch  t := n.(type) {
		case *ast.Ident:
			lastIdent = t.Name
		// top level interface definition
		case *ast.InterfaceType:
			sigs := make([]*implement.FunctionSignature, 0)
			for _, f := range t.Methods.List {
				name := getFunctionName(f.Pos(), f.End(), data)
				sig := FunctionSignature(f.Type)

				sig.Name = name
				sigs = append(sigs, sig)
			}
			signatures[lastIdent] = sigs
		}
		return true
	})
	return signatures
}

func getFunctionName(start, end token.Pos, data []byte) string {
	return strings.Split(string(data[start - 1: end -1]), "(")[0]
}

func FunctionSignature(expr ast.Expr) (*implement.FunctionSignature) {
	signature := &implement.FunctionSignature{}
	switch n := expr.(type) {
	// the top level function
	case *ast.FuncType:
		if n.Params != nil {
			//letterMap := make(map[string]int)
			for _, p := range n.Params.List {
				s := getTypeIdentifier(p.Type)
				for _, n := range p.Names {
					param := &implement.Parameter{Name: n.Name, Type: s}
					signature.Parameters = append(signature.Parameters, param)
				}

				if len(p.Names) == 0 {
					param := &implement.Parameter{Type: s}
					signature.Parameters = append(signature.Parameters, param)
				}

			}
		}

		if n.Results != nil {
			for _, r := range n.Results.List {
				s := getTypeIdentifier(r.Type)
				for _, n := range r.Names {
					result := &implement.ReturnValue{Name: n.Name, Type: s}
					signature.ReturnValues = append(signature.ReturnValues, result)
				}

				if len(r.Names) == 0 {
					result := &implement.ReturnValue{Type: s}
					signature.ReturnValues = append(signature.ReturnValues, result)
				}
			}
		}
	}
	return signature
}

func lowerFirstLetterOfVar(s string) string {
	split := strings.Split(s, "*")
	var firstLetter string
	if len(split) > 1 {
		firstLetter = strings.ToLower(split[1])
	} else {
		firstLetter = strings.ToLower(split[0])
	}
	return firstLetter
}

// TODO - refactor to return *implement.Type instead of string :(
func getTypeIdentifier(expr ast.Expr) string {

	switch t := expr.(type) {
	case *ast.Ident:
		return t.String()
	case *ast.StarExpr:
		s := getTypeIdentifier(t.X)
		return "*" + s
	case *ast.SelectorExpr:
		s := getTypeIdentifier(t.X)
		return s + "." + t.Sel.String()
	case *ast.FuncType:
		var typ = "func("
		if t.Params != nil {
			for _, p := range t.Params.List {
				s := getTypeIdentifier(p.Type)
				if len(p.Names) > 0 {
					for i := range p.Names {
						typ += s
						if i != len(p.Names) - 1 {
							typ += ","
						}
					}

				} else {
					typ += s
				}
			}
		}
		typ += ")"
		if t.Results != nil {
			if len(t.Results.List) > 1 {
				typ += " ("
			}

			for i, r := range t.Results.List {
				s := getTypeIdentifier(r.Type)
				typ += s
				if len(t.Results.List) > 1 && i != len(t.Results.List) - 1 {
					typ += ", "
				}
			}

			if len(t.Results.List) > 1 {
				typ += ")"
			}
		}
		return typ

	case *ast.InterfaceType:
		return "interface{}"
	}
	return ""
}<commit_msg>[ch132] removes fmt from parse package<commit_after>package parse

import (
	"go/token"
	"go/parser"
	"go/ast"
	"io/ioutil"
	"github.com/thisisfineio/implement"
	"strings"
	//"reflect"
)

// Parse returns an *ast.File, and an error
func Parse(data []byte, filename string) (*ast.File, []byte, error) {
	fset := token.NewFileSet()
	f, err := parser.ParseFile(fset, filename, data, parser.ParseComments)
	return f, data, err
}

func File(filename string) (*ast.File, []byte, error) {
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, nil, err
	}
	return Parse(data, filename)
}

// Inspect visits all nodes in the *ast.File (recursively)
// data is necessary for us to determine interface names
func Inspect(f *ast.File, data []byte) map[string][]*implement.FunctionSignature {
	signatures := make(map[string][]*implement.FunctionSignature)
	var lastIdent string
	ast.Inspect(f, func(n ast.Node) bool {
		//fmt.Println(n)
		switch  t := n.(type) {
		case *ast.Ident:
			lastIdent = t.Name
		// top level interface definition
		case *ast.InterfaceType:
			sigs := make([]*implement.FunctionSignature, 0)
			for _, f := range t.Methods.List {
				name := getFunctionName(f.Pos(), f.End(), data)
				sig := FunctionSignature(f.Type)

				sig.Name = name
				sigs = append(sigs, sig)
			}
			signatures[lastIdent] = sigs
		}
		return true
	})
	return signatures
}

func getFunctionName(start, end token.Pos, data []byte) string {
	return strings.Split(string(data[start - 1: end -1]), "(")[0]
}

func FunctionSignature(expr ast.Expr) (*implement.FunctionSignature) {
	signature := &implement.FunctionSignature{}
	switch n := expr.(type) {
	// the top level function
	case *ast.FuncType:
		if n.Params != nil {
			//letterMap := make(map[string]int)
			for _, p := range n.Params.List {
				s := getTypeIdentifier(p.Type)
				for _, n := range p.Names {
					param := &implement.Parameter{Name: n.Name, Type: s}
					signature.Parameters = append(signature.Parameters, param)
				}

				if len(p.Names) == 0 {
					param := &implement.Parameter{Type: s}
					signature.Parameters = append(signature.Parameters, param)
				}

			}
		}

		if n.Results != nil {
			for _, r := range n.Results.List {
				s := getTypeIdentifier(r.Type)
				for _, n := range r.Names {
					result := &implement.ReturnValue{Name: n.Name, Type: s}
					signature.ReturnValues = append(signature.ReturnValues, result)
				}

				if len(r.Names) == 0 {
					result := &implement.ReturnValue{Type: s}
					signature.ReturnValues = append(signature.ReturnValues, result)
				}
			}
		}
	}
	return signature
}

func lowerFirstLetterOfVar(s string) string {
	split := strings.Split(s, "*")
	var firstLetter string
	if len(split) > 1 {
		firstLetter = strings.ToLower(split[1])
	} else {
		firstLetter = strings.ToLower(split[0])
	}
	return firstLetter
}

// TODO - refactor to return *implement.Type instead of string :(
func getTypeIdentifier(expr ast.Expr) string {

	switch t := expr.(type) {
	case *ast.Ident:
		return t.String()
	case *ast.StarExpr:
		s := getTypeIdentifier(t.X)
		return "*" + s
	case *ast.SelectorExpr:
		s := getTypeIdentifier(t.X)
		return s + "." + t.Sel.String()
	case *ast.FuncType:
		var typ = "func("
		if t.Params != nil {
			for _, p := range t.Params.List {
				s := getTypeIdentifier(p.Type)
				if len(p.Names) > 0 {
					for i := range p.Names {
						typ += s
						if i != len(p.Names) - 1 {
							typ += ","
						}
					}

				} else {
					typ += s
				}
			}
		}
		typ += ")"
		if t.Results != nil {
			if len(t.Results.List) > 1 {
				typ += " ("
			}

			for i, r := range t.Results.List {
				s := getTypeIdentifier(r.Type)
				typ += s
				if len(t.Results.List) > 1 && i != len(t.Results.List) - 1 {
					typ += ", "
				}
			}

			if len(t.Results.List) > 1 {
				typ += ")"
			}
		}
		return typ

	case *ast.InterfaceType:
		return "interface{}"
	}
	return ""
}<|endoftext|>


<commit_before>package main

import (
	"encoding/hex"
	"encoding/json"
	"fmt"
	"os"
	"strings"

	"github.com/bazooka-ci/bazooka/commons/matrix"

	log "github.com/Sirupsen/logrus"
	lib "github.com/bazooka-ci/bazooka/commons"
	bzklog "github.com/bazooka-ci/bazooka/commons/logs"
)

const (
	BazookaConfigFile       = ".bazooka.yml"
	TravisConfigFile        = ".travis.yml"
	MX_ENV_PREFIX           = "env::"
	BazookaEnvJobParameters = "BZK_JOB_PARAMETERS"
)

func init() {
	log.SetFormatter(&bzklog.BzkFormatter{})
	err := lib.LoadCryptoKeyFromFile("/bazooka-cryptokey")
	if err != nil {
		log.Fatal(err)
	}

}

func main() {
	log.Info("Starting Parsing Phase")
	// Find either .travis.yml or .bazooka.yml file in the project
	configFile, err := lib.ResolveConfigFile(paths.container.source)
	if err != nil {
		log.Fatal(err)
	}

	envParamString := os.Getenv(BazookaEnvJobParameters)
	var envParams []lib.BzkString
	err = json.Unmarshal([]byte(envParamString), &envParams)
	if err != nil {
		log.Fatal(err)
	}

	jobParameters := groupByName(envParams, MX_ENV_PREFIX)

	// parse the configuration
	config := &lib.Config{}
	err = lib.Parse(configFile, config)
	if err != nil {
		log.Fatal(err)
	}
	log.WithFields(log.Fields{
		"config": config,
	}).Debug("Configuration parsed")

	var variants []*variantData

	if len(config.Language) == 0 {
		if len(config.Image) == 0 {
			log.Fatal("One of 'language' or 'image' needs to be set")
		}
		variants = generateImageVariants(config)
	} else {
		// resolve the docker image corresponding to this particular language parser
		image, err := resolveLanguageParser(config.Language)
		if err != nil {
			log.Fatal(err)
		}

		// run the parser image
		langParser := &LanguageParser{
			Image: image,
		}
		variants, err = langParser.Parse()
		if err != nil {
			log.Fatal(err)
		}
	}

	log.Info("Starting Matrix generation")

	for _, variant := range variants {
		// create a matrix from the environment variables
		// a matrix has N variables (dimensions) where each variable has M values
		// config.Env is a list of key=value strings
		// groupByName transforms that into a map[string][]string
		// for example ["A=1", "A=2", B="3"]
		// when grouped by name gets transformed into
		// {A: [1, 2], B: [3]}
		// this matches the matrix layout, so we could store them directly into the matrix
		// but since we would like to be able to extract them later, and to avoid mixing them with a language specific variables (like jdk, go, etc.)
		// explode prefixes the env variables names with a prefix defined in the constant MX_ENV_PREFIX
		// Hence, our matrix is more like: {"env::A": [1, 2], "env::B": [3]}
		variantVariables := groupByName(variant.config.Env, MX_ENV_PREFIX)

		// insert or replace environment variables defined by the job parameters
		for k, v := range jobParameters {
			variantVariables[k] = v
		}

		// check if all environment variables are defined
		for k, v := range variantVariables {
			if len(v) == 1 && len(v[0]) == 0 {
				log.Fatalf("Missing required parameter %v", k)
			}
		}

		mx := matrix.Matrix(variantVariables)

		// and then add the new language specific variables parsed from the meta file to the build matrix (which already contains the env variables)
		err = feedMatrix(variant.meta, &mx)
		if err != nil {
			log.Fatal(err)
		}

		// we're not done yet: we need to also handle the matrix exclusions
		// we parse them into a list of matrices
		exclusions, err := exclusionsMatrices(variant.config.Matrix.Exclude)
		if err != nil {
			log.Fatal(err)
		}

		// and finally, we iterate over all the permutations of the build matrix
		// these permutations are the list of all the combinations of env variables and language specific variables, minux the exclusions
		mx.IterAll(func(permutation map[string]string, counter string) {
			// we get called for every non-excluded permutations with the different variables values for this permutations and a unique permutation counter
			// handlePermutation will start from the .bazooka.*.yml file, which should already contain a single language specific permutation
			// and enrich it with the env variables combination
			// the same goes for the meta file
			if err := handlePermutation(permutation, variant.config, variant.meta, counter, variant.counter); err != nil {
				log.Fatal(fmt.Errorf("Error while generating the permutations: %v", err))
			}
		}, exclusions)
	}
	log.Info("Matrix generated")

	log.Info("Starting generating Dockerfiles from Matrix")
	// Now we're left with the final build files
	files, err := lib.ListFilesWithPrefix(paths.container.output, ".bazooka")
	if err != nil {
		log.Fatal(fmt.Errorf("Error while listing .bazooka* files: %v", err))
	}

	for _, file := range files {
		config := &lib.Config{}
		err = lib.Parse(file, config)
		if err != nil {
			log.Fatal(fmt.Errorf("Error while parsing config file %s: %v", file, err))
		}

		// transform the .bazooka.x.yml file into a set of dockerfile + shell scripts who perform the actual build
		g := &Generator{
			Config:       config,
			OutputFolder: paths.container.output,
			Index:        parseCounter(file),
		}
		err = g.GenerateDockerfile()
		if err != nil {
			log.Fatal("Error while generating a dockerfile: %v", err)
		}
	}
	log.Info("Dockerfiles all created successfully")

}

func handlePermutation(permutation map[string]string, config *lib.Config, meta map[string]interface{}, counter, rootCounter string) error {
	// start from the language-spcecific permutation
	newConfig := *config

	// and replace its env variables with this unique permutation
	envMap := extractEnv(permutation, config.Env)

	// Insert BZK_BUILD_DIR if not present
	if _, ok := envMap["BZK_BUILD_DIR"]; !ok {
		envMap["BZK_BUILD_DIR"] = lib.BzkString{"BZK_BUILD_DIR", "/bazooka", false}
	}

	newConfig.Env = mapValues(envMap)
	if err := lib.Flush(newConfig, fmt.Sprintf("%s/.bazooka.%s%s.yml", paths.container.output, rootCounter, counter)); err != nil {
		return err
	}

	// do the same for the meta file
	// start from the language specific permutation meta file
	// and add this permutation's env map
	metaEnv, err:=generateEnvForMeta(newConfig.Env)
	if err != nil {
		return err
	}

	meta["env"] = metaEnv
	metaFile := fmt.Sprintf("%s/%s%s", paths.container.meta, rootCounter, counter)
	lib.Flush(meta, metaFile)

	return nil
}

func generateImageVariants(conf *lib.Config) []*variantData {
	res := make([]*variantData, len(conf.Image))
	for i, im := range conf.Image {
		imageConf := *conf
		imageConf.FromImage = im
		imageConf.Image = nil

		res[i] = &variantData{
			counter: fmt.Sprintf("%d", i),
			config:  &imageConf,
			meta: map[string]interface{}{
				"image": im,
			},
		}
	}
	return res
}

func feedMatrix(extra map[string]interface{}, mx *matrix.Matrix) error {
	for k, v := range extra {
		switch k {
		case "env":
			switch converted:=v.(type) {
			case []interface{}:
				envVars := []lib.BzkString{}
				for _, envVar := range converted {
					if strEnvVar, ok := envVar.(string); ok {
						n, v:=lib.SplitNameValue(strEnvVar)
						envVars = append(envVars, lib.BzkString{n, v, false})
					} else {
						return fmt.Errorf("Invalid config: env should contain a sequence of strings: found a non string value %v:%T", envVar, envVar)

					}
				}
				mx.Merge(groupByName(envVars, MX_ENV_PREFIX))
			case string:
						n, v:=lib.SplitNameValue(converted)
				mx.Merge(groupByName([]lib.BzkString{lib.BzkString{n, v, false}}, MX_ENV_PREFIX))
			default:
				return fmt.Errorf("Invalid config: env should contain a sequence of strings: %v:%T", v, v)
			}
		default:
			mx.AddVar(k, fmt.Sprintf("%v", v))
		}
	}
	return nil
}

func exclusionsMatrices(xs []map[string]interface{}) ([]*matrix.Matrix, error) {
	res := make([]*matrix.Matrix, len(xs))
	for i, x := range xs {
		mx := matrix.Matrix{}
		if err := feedMatrix(x, &mx); err != nil {
			return nil, err
		}
		res[i] = &mx
	}
	return res, nil
}

// parseCounter extract the * part from a .bazooka.*.yml file name
func parseCounter(filePath string) string {
	splits := strings.Split(filePath, "/")
	file := splits[len(splits)-1]
	return strings.Split(file, ".")[2]
}

// groupByName starts from a list of key=valye strings and stores them into a map
// it also handles repeated values, so ["A=1", "A=2", B="3"] gets transformed into {A: [1, 2], B: [3]}
func groupByName(props []lib.BzkString, keyPrefix string) map[string][]string {
	res := make(map[string][]string)
	for _, env := range props {
		res[keyPrefix+env.Name] = append(res[keyPrefix+env.Name], env.Value)
	}
	return res
}

// prefixMapKeys returns a new map where all keys are prefixed with prefix
func prefixMapKeys(m map[string][]string, prefix string) map[string][]string {
	res := make(map[string][]string)
	for k, v := range m {
		res[prefix+k] = v
	}
	return res
}

// extractEnv extracts back the env variables from the permutation values
func extractEnv(from map[string]string, originalEnv []lib.BzkString) map[string]lib.BzkString {
	res := make(map[string]lib.BzkString)
	for k, v := range from {
		if strings.HasPrefix(k, MX_ENV_PREFIX) {
			name:=strings.TrimPrefix(k, MX_ENV_PREFIX)
			res[name] = findOrCreateBzkString(name, v, originalEnv)
		}
	}
	return res
}

func findOrCreateBzkString(name, value string, env []lib.BzkString) lib.BzkString {
	for _, e:=range env {
		if name==e.Name && value==e.Value {
			return e
		}
	}
	return lib.BzkString{name, value, false}
}

func mapValues(m map[string]lib.BzkString) []lib.BzkString {
	res :=make([]lib.BzkString, 0, len(m))
	for _, v:=range m {
		res=append(res, v)
	}
	return res
}

func generateEnvForMeta(env []lib.BzkString) ([]string, error) {
	key, err:=lib.ReadCryptoKey(paths.container.cryptoKey)
	if err != nil {
		return nil, err
	}

	res := make([]string, 0, len(env))
	for _, e:=range env {
		value := e.Value
		if e.Secured {
			eValue, err:= lib.Encrypt(key, []byte(value))
			if err != nil {
				return nil, err
			}
			value = hex.EncodeToString(eValue)
		}
		res = append(res, fmt.Sprintf("%s=%s", e.Name, value))
	}
	return res, nil
}
func resolveLanguageParser(language string) (string, error) {
	parserMap := map[string]string{
		"golang":  "bazooka/parser-golang",
		"go":      "bazooka/parser-golang",
		"java":    "bazooka/parser-java",
		"python":  "bazooka/parser-python",
		"node_js": "bazooka/parser-nodejs",
		"nodejs":  "bazooka/parser-nodejs",
		"php":     "bazooka/parser-php",
	}
	if val, ok := parserMap[language]; ok {
		return val, nil
	}
	return "", fmt.Errorf("Unable to find Bazooka Docker Image for Language Parser %s\n", language)
}
<commit_msg>Add ruby language parser<commit_after>package main

import (
	"encoding/hex"
	"encoding/json"
	"fmt"
	"os"
	"strings"

	"github.com/bazooka-ci/bazooka/commons/matrix"

	log "github.com/Sirupsen/logrus"
	lib "github.com/bazooka-ci/bazooka/commons"
	bzklog "github.com/bazooka-ci/bazooka/commons/logs"
)

const (
	BazookaConfigFile       = ".bazooka.yml"
	TravisConfigFile        = ".travis.yml"
	MX_ENV_PREFIX           = "env::"
	BazookaEnvJobParameters = "BZK_JOB_PARAMETERS"
)

func init() {
	log.SetFormatter(&bzklog.BzkFormatter{})
	err := lib.LoadCryptoKeyFromFile("/bazooka-cryptokey")
	if err != nil {
		log.Fatal(err)
	}

}

func main() {
	log.Info("Starting Parsing Phase")
	// Find either .travis.yml or .bazooka.yml file in the project
	configFile, err := lib.ResolveConfigFile(paths.container.source)
	if err != nil {
		log.Fatal(err)
	}

	envParamString := os.Getenv(BazookaEnvJobParameters)
	var envParams []lib.BzkString
	err = json.Unmarshal([]byte(envParamString), &envParams)
	if err != nil {
		log.Fatal(err)
	}

	jobParameters := groupByName(envParams, MX_ENV_PREFIX)

	// parse the configuration
	config := &lib.Config{}
	err = lib.Parse(configFile, config)
	if err != nil {
		log.Fatal(err)
	}
	log.WithFields(log.Fields{
		"config": config,
	}).Debug("Configuration parsed")

	var variants []*variantData

	if len(config.Language) == 0 {
		if len(config.Image) == 0 {
			log.Fatal("One of 'language' or 'image' needs to be set")
		}
		variants = generateImageVariants(config)
	} else {
		// resolve the docker image corresponding to this particular language parser
		image, err := resolveLanguageParser(config.Language)
		if err != nil {
			log.Fatal(err)
		}

		// run the parser image
		langParser := &LanguageParser{
			Image: image,
		}
		variants, err = langParser.Parse()
		if err != nil {
			log.Fatal(err)
		}
	}

	log.Info("Starting Matrix generation")

	for _, variant := range variants {
		// create a matrix from the environment variables
		// a matrix has N variables (dimensions) where each variable has M values
		// config.Env is a list of key=value strings
		// groupByName transforms that into a map[string][]string
		// for example ["A=1", "A=2", B="3"]
		// when grouped by name gets transformed into
		// {A: [1, 2], B: [3]}
		// this matches the matrix layout, so we could store them directly into the matrix
		// but since we would like to be able to extract them later, and to avoid mixing them with a language specific variables (like jdk, go, etc.)
		// explode prefixes the env variables names with a prefix defined in the constant MX_ENV_PREFIX
		// Hence, our matrix is more like: {"env::A": [1, 2], "env::B": [3]}
		variantVariables := groupByName(variant.config.Env, MX_ENV_PREFIX)

		// insert or replace environment variables defined by the job parameters
		for k, v := range jobParameters {
			variantVariables[k] = v
		}

		// check if all environment variables are defined
		for k, v := range variantVariables {
			if len(v) == 1 && len(v[0]) == 0 {
				log.Fatalf("Missing required parameter %v", k)
			}
		}

		mx := matrix.Matrix(variantVariables)

		// and then add the new language specific variables parsed from the meta file to the build matrix (which already contains the env variables)
		err = feedMatrix(variant.meta, &mx)
		if err != nil {
			log.Fatal(err)
		}

		// we're not done yet: we need to also handle the matrix exclusions
		// we parse them into a list of matrices
		exclusions, err := exclusionsMatrices(variant.config.Matrix.Exclude)
		if err != nil {
			log.Fatal(err)
		}

		// and finally, we iterate over all the permutations of the build matrix
		// these permutations are the list of all the combinations of env variables and language specific variables, minux the exclusions
		mx.IterAll(func(permutation map[string]string, counter string) {
			// we get called for every non-excluded permutations with the different variables values for this permutations and a unique permutation counter
			// handlePermutation will start from the .bazooka.*.yml file, which should already contain a single language specific permutation
			// and enrich it with the env variables combination
			// the same goes for the meta file
			if err := handlePermutation(permutation, variant.config, variant.meta, counter, variant.counter); err != nil {
				log.Fatal(fmt.Errorf("Error while generating the permutations: %v", err))
			}
		}, exclusions)
	}
	log.Info("Matrix generated")

	log.Info("Starting generating Dockerfiles from Matrix")
	// Now we're left with the final build files
	files, err := lib.ListFilesWithPrefix(paths.container.output, ".bazooka")
	if err != nil {
		log.Fatal(fmt.Errorf("Error while listing .bazooka* files: %v", err))
	}

	for _, file := range files {
		config := &lib.Config{}
		err = lib.Parse(file, config)
		if err != nil {
			log.Fatal(fmt.Errorf("Error while parsing config file %s: %v", file, err))
		}

		// transform the .bazooka.x.yml file into a set of dockerfile + shell scripts who perform the actual build
		g := &Generator{
			Config:       config,
			OutputFolder: paths.container.output,
			Index:        parseCounter(file),
		}
		err = g.GenerateDockerfile()
		if err != nil {
			log.Fatal("Error while generating a dockerfile: %v", err)
		}
	}
	log.Info("Dockerfiles all created successfully")

}

func handlePermutation(permutation map[string]string, config *lib.Config, meta map[string]interface{}, counter, rootCounter string) error {
	// start from the language-spcecific permutation
	newConfig := *config

	// and replace its env variables with this unique permutation
	envMap := extractEnv(permutation, config.Env)

	// Insert BZK_BUILD_DIR if not present
	if _, ok := envMap["BZK_BUILD_DIR"]; !ok {
		envMap["BZK_BUILD_DIR"] = lib.BzkString{"BZK_BUILD_DIR", "/bazooka", false}
	}

	newConfig.Env = mapValues(envMap)
	if err := lib.Flush(newConfig, fmt.Sprintf("%s/.bazooka.%s%s.yml", paths.container.output, rootCounter, counter)); err != nil {
		return err
	}

	// do the same for the meta file
	// start from the language specific permutation meta file
	// and add this permutation's env map
	metaEnv, err:=generateEnvForMeta(newConfig.Env)
	if err != nil {
		return err
	}

	meta["env"] = metaEnv
	metaFile := fmt.Sprintf("%s/%s%s", paths.container.meta, rootCounter, counter)
	lib.Flush(meta, metaFile)

	return nil
}

func generateImageVariants(conf *lib.Config) []*variantData {
	res := make([]*variantData, len(conf.Image))
	for i, im := range conf.Image {
		imageConf := *conf
		imageConf.FromImage = im
		imageConf.Image = nil

		res[i] = &variantData{
			counter: fmt.Sprintf("%d", i),
			config:  &imageConf,
			meta: map[string]interface{}{
				"image": im,
			},
		}
	}
	return res
}

func feedMatrix(extra map[string]interface{}, mx *matrix.Matrix) error {
	for k, v := range extra {
		switch k {
		case "env":
			switch converted:=v.(type) {
			case []interface{}:
				envVars := []lib.BzkString{}
				for _, envVar := range converted {
					if strEnvVar, ok := envVar.(string); ok {
						n, v:=lib.SplitNameValue(strEnvVar)
						envVars = append(envVars, lib.BzkString{n, v, false})
					} else {
						return fmt.Errorf("Invalid config: env should contain a sequence of strings: found a non string value %v:%T", envVar, envVar)

					}
				}
				mx.Merge(groupByName(envVars, MX_ENV_PREFIX))
			case string:
						n, v:=lib.SplitNameValue(converted)
				mx.Merge(groupByName([]lib.BzkString{lib.BzkString{n, v, false}}, MX_ENV_PREFIX))
			default:
				return fmt.Errorf("Invalid config: env should contain a sequence of strings: %v:%T", v, v)
			}
		default:
			mx.AddVar(k, fmt.Sprintf("%v", v))
		}
	}
	return nil
}

func exclusionsMatrices(xs []map[string]interface{}) ([]*matrix.Matrix, error) {
	res := make([]*matrix.Matrix, len(xs))
	for i, x := range xs {
		mx := matrix.Matrix{}
		if err := feedMatrix(x, &mx); err != nil {
			return nil, err
		}
		res[i] = &mx
	}
	return res, nil
}

// parseCounter extract the * part from a .bazooka.*.yml file name
func parseCounter(filePath string) string {
	splits := strings.Split(filePath, "/")
	file := splits[len(splits)-1]
	return strings.Split(file, ".")[2]
}

// groupByName starts from a list of key=valye strings and stores them into a map
// it also handles repeated values, so ["A=1", "A=2", B="3"] gets transformed into {A: [1, 2], B: [3]}
func groupByName(props []lib.BzkString, keyPrefix string) map[string][]string {
	res := make(map[string][]string)
	for _, env := range props {
		res[keyPrefix+env.Name] = append(res[keyPrefix+env.Name], env.Value)
	}
	return res
}

// prefixMapKeys returns a new map where all keys are prefixed with prefix
func prefixMapKeys(m map[string][]string, prefix string) map[string][]string {
	res := make(map[string][]string)
	for k, v := range m {
		res[prefix+k] = v
	}
	return res
}

// extractEnv extracts back the env variables from the permutation values
func extractEnv(from map[string]string, originalEnv []lib.BzkString) map[string]lib.BzkString {
	res := make(map[string]lib.BzkString)
	for k, v := range from {
		if strings.HasPrefix(k, MX_ENV_PREFIX) {
			name:=strings.TrimPrefix(k, MX_ENV_PREFIX)
			res[name] = findOrCreateBzkString(name, v, originalEnv)
		}
	}
	return res
}

func findOrCreateBzkString(name, value string, env []lib.BzkString) lib.BzkString {
	for _, e:=range env {
		if name==e.Name && value==e.Value {
			return e
		}
	}
	return lib.BzkString{name, value, false}
}

func mapValues(m map[string]lib.BzkString) []lib.BzkString {
	res :=make([]lib.BzkString, 0, len(m))
	for _, v:=range m {
		res=append(res, v)
	}
	return res
}

func generateEnvForMeta(env []lib.BzkString) ([]string, error) {
	key, err:=lib.ReadCryptoKey(paths.container.cryptoKey)
	if err != nil {
		return nil, err
	}

	res := make([]string, 0, len(env))
	for _, e:=range env {
		value := e.Value
		if e.Secured {
			eValue, err:= lib.Encrypt(key, []byte(value))
			if err != nil {
				return nil, err
			}
			value = hex.EncodeToString(eValue)
		}
		res = append(res, fmt.Sprintf("%s=%s", e.Name, value))
	}
	return res, nil
}
func resolveLanguageParser(language string) (string, error) {
	parserMap := map[string]string{
		"golang":  "bazooka/parser-golang",
		"go":      "bazooka/parser-golang",
		"java":    "bazooka/parser-java",
		"python":  "bazooka/parser-python",
		"ruby":    "bazooka/parser-ruby",
		"node_js": "bazooka/parser-nodejs",
		"nodejs":  "bazooka/parser-nodejs",
		"php":     "bazooka/parser-php",
	}
	if val, ok := parserMap[language]; ok {
		return val, nil
	}
	return "", fmt.Errorf("Unable to find Bazooka Docker Image for Language Parser %s\n", language)
}
<|endoftext|>


<commit_before>package simpleSqlParser

import (
	"fmt"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestSelect(t *testing.T) {
	tests := []struct {
		s            string
		expectedStmt SelectStatement
	}{
		{
			s: `SELECT foo FROM k1.tbl1; `,
			expectedStmt: SelectStatement{
				Keyspace:  "k1",
				TableName: "tbl1",
				Columns:   []string{"foo"},
			},
		},
		{
			s: `Select * from k2.tbl;`,
			expectedStmt: SelectStatement{
				Keyspace:   "k2",
				TableName:  "tbl",
				AllColumns: true,
			},
		},
		{
			s: `Select (col1, col2  ,  col3 ) from k1.tbl;`,
			expectedStmt: SelectStatement{
				Keyspace:  "k1",
				TableName: "tbl",
				Columns:   []string{"col1", "col2", "col3"},
			},
		},
		{
			s: `Select * from k1.tbl where x=10;`,
			expectedStmt: SelectStatement{
				Keyspace:     "k1",
				TableName:    "tbl",
				AllColumns:   true,
				WhereColumns: []string{"x"},
				WhereValues:  []string{"10"},
				Operators:    []string{"="},
			},
		},
		{
			s: `Select * from k1.tbl where x>10 and y=abc;`,
			expectedStmt: SelectStatement{
				Keyspace:     "k1",
				TableName:    "tbl",
				AllColumns:   true,
				WhereColumns: []string{"x", "y"},
				WhereValues:  []string{"10", "abc"},
				Operators:    []string{">", "="},
			},
		},
		{
			s: `Select * from k1.tbl where x>=10 and y < abc and z= 3.122 limit 5 ;`,
			expectedStmt: SelectStatement{
				Keyspace:     "k1",
				TableName:    "tbl",
				AllColumns:   true,
				WhereColumns: []string{"x", "y", "z"},
				WhereValues:  []string{"10", "abc", "3.122"},
				Operators:    []string{">=", "<", "="},
				Limit:        int(5),
			},
		},
	}

	for i, eachTest := range tests {
		got := &SQL{Buffer: eachTest.s}
		got.Init()
		if err := got.Parse(); assert.Nil(t, err, fmt.Sprintf("Test Case %d", i)) {
			got.Execute()
			assert.Equal(t, eachTest.expectedStmt, got.SelectStatement, fmt.Sprintf("Test Case %d ", i))
		}

	}

}

func TestInsert(t *testing.T) {
	tests := []struct {
		s            string
		expectedStmt InsertStatement
	}{
		{
			s: `Insert into k2.instaTbl values ( 1, "a",1.123, "d"   );`,
			expectedStmt: InsertStatement{
				Keyspace:  "k2",
				TableName: "instaTbl",
				Values:    []string{"1", "a", "1.123", "d"},
			},
		},
		{
			s: `INSERT INTO k1.insertTable(a,b,c)values("a",23434, 23.533536);`,
			expectedStmt: InsertStatement{
				Keyspace:  "k1",
				TableName: "insertTable",
				Values:    []string{"a", "23434", "23.533536"},
				Columns:   []string{"a", "b", "c"},
			},
		},
		{
			s: `INSERT INTO k1.kaboo(a,b,c) VALUES (1);`,
		},
	}

	for i, eachTest := range tests {
		got := &SQL{Buffer: eachTest.s}
		got.Init()
		if err := got.Parse(); assert.Nil(t, err) {
			got.Execute()
			assert.Equal(t, eachTest.expectedStmt, got.InsertStatement, fmt.Sprintf("Test Case %d ", i))
		}
	}
}

func TestCreate(t *testing.T) {
	tests := []struct {
		s            string
		expectedStmt CreateStatement
	}{
		{
			s: ` Create Table keyspace.TableName( col1 , col2, col3 ); `,
			expectedStmt: CreateStatement{
				Keyspace:        "keyspace",
				TableName:       "TableName",
				Columns:         []string{"col1", "col2", "col3"},
				PartitioningKey: "col1",
			},
		},
	}

	for i, eachTest := range tests {
		got := &SQL{Buffer: eachTest.s}
		got.Init()
		if err := got.Parse(); assert.Nil(t, err) {
			got.Execute()
			assert.Equal(t, got.CreateStatement, eachTest.expectedStmt, fmt.Sprint("Test Case %d ", i))
		}
	}

}
<commit_msg>Add test case for drop statement<commit_after>package simpleSqlParser

import (
	"fmt"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestSelect(t *testing.T) {
	tests := []struct {
		s            string
		expectedStmt SelectStatement
	}{
		{
			s: `SELECT foo FROM k1.tbl1; `,
			expectedStmt: SelectStatement{
				Keyspace:  "k1",
				TableName: "tbl1",
				Columns:   []string{"foo"},
			},
		},
		{
			s: `Select * from k2.tbl;`,
			expectedStmt: SelectStatement{
				Keyspace:   "k2",
				TableName:  "tbl",
				AllColumns: true,
			},
		},
		{
			s: `Select (col1, col2  ,  col3 ) from k1.tbl;`,
			expectedStmt: SelectStatement{
				Keyspace:  "k1",
				TableName: "tbl",
				Columns:   []string{"col1", "col2", "col3"},
			},
		},
		{
			s: `Select * from k1.tbl where x=10;`,
			expectedStmt: SelectStatement{
				Keyspace:     "k1",
				TableName:    "tbl",
				AllColumns:   true,
				WhereColumns: []string{"x"},
				WhereValues:  []string{"10"},
				Operators:    []string{"="},
			},
		},
		{
			s: `Select * from k1.tbl where x>10 and y=abc;`,
			expectedStmt: SelectStatement{
				Keyspace:     "k1",
				TableName:    "tbl",
				AllColumns:   true,
				WhereColumns: []string{"x", "y"},
				WhereValues:  []string{"10", "abc"},
				Operators:    []string{">", "="},
			},
		},
		{
			s: `Select * from k1.tbl where x>=10 and y < abc and z= 3.122 limit 5 ;`,
			expectedStmt: SelectStatement{
				Keyspace:     "k1",
				TableName:    "tbl",
				AllColumns:   true,
				WhereColumns: []string{"x", "y", "z"},
				WhereValues:  []string{"10", "abc", "3.122"},
				Operators:    []string{">=", "<", "="},
				Limit:        int(5),
			},
		},
	}

	for i, eachTest := range tests {
		got := &SQL{Buffer: eachTest.s}
		got.Init()
		if err := got.Parse(); assert.Nil(t, err, fmt.Sprintf("Test Case %d", i)) {
			got.Execute()
			assert.Equal(t, eachTest.expectedStmt, got.SelectStatement, fmt.Sprintf("Test Case %d ", i))
		}

	}

}

func TestInsert(t *testing.T) {
	tests := []struct {
		s            string
		expectedStmt InsertStatement
	}{
		{
			s: `Insert into k2.instaTbl values ( 1, "a",1.123, "d"   );`,
			expectedStmt: InsertStatement{
				Keyspace:  "k2",
				TableName: "instaTbl",
				Values:    []string{"1", "a", "1.123", "d"},
			},
		},
		{
			s: `INSERT INTO k1.insertTable(a,b,c)values("a",23434, 23.533536);`,
			expectedStmt: InsertStatement{
				Keyspace:  "k1",
				TableName: "insertTable",
				Values:    []string{"a", "23434", "23.533536"},
				Columns:   []string{"a", "b", "c"},
			},
		},
		{
			s: `INSERT INTO k1.kaboo(a,b,c) VALUES (1);`,
		},
	}

	for i, eachTest := range tests {
		got := &SQL{Buffer: eachTest.s}
		got.Init()
		if err := got.Parse(); assert.Nil(t, err) {
			got.Execute()
			assert.Equal(t, eachTest.expectedStmt, got.InsertStatement, fmt.Sprintf("Test Case %d ", i))
		}
	}
}

func TestCreate(t *testing.T) {
	tests := []struct {
		s            string
		expectedStmt CreateStatement
	}{
		{
			s: ` Create Table keyspace.TableName( col1 , col2, col3 ); `,
			expectedStmt: CreateStatement{
				Keyspace:        "keyspace",
				TableName:       "TableName",
				Columns:         []string{"col1", "col2", "col3"},
				PartitioningKey: "col1",
			},
		},
	}

	for i, eachTest := range tests {
		got := &SQL{Buffer: eachTest.s}
		got.Init()
		if err := got.Parse(); assert.Nil(t, err) {
			got.Execute()
			assert.Equal(t, got.CreateStatement, eachTest.expectedStmt, fmt.Sprint("Test Case %d ", i))
		}
	}

}

func TestDrop(t *testing.T) {
	tests := []struct {
		s            string
		expectedStmt DropStatement
	}{
		{
			s: `Drop keyspace.Tbl;`,
			expectedStmt: DropStatement{
				Keyspace:  "keyspace",
				TableName: "Tbl",
			},
		},
	}

	for i, eachTest := range tests {
		got := &SQL{Buffer: eachTest.s}
		got.Init()
		if err := got.Parse(); assert.Nil(t, err) {
			got.Execute()
			assert.Equal(t, got.DropStatement, eachTest.expectedStmt, fmt.Sprint("Test Case %d ", i))
		}
	}

}
<|endoftext|>


<commit_before>// Implements Relatable for Bam

package parsers

import (
	"io"
	"log"

	"github.com/biogo/hts/bam"
	"github.com/biogo/hts/sam"
	"github.com/brentp/irelate/interfaces"
)

type Bam struct {
	*sam.Record
	source     uint32
	related    []interfaces.Relatable
	Chromosome string
	_end       uint32
}

func (a *Bam) Chrom() string {
	return a.Chromosome
}

// cast to 32 bits.
func (a *Bam) Start() uint32 {
	return uint32(a.Record.Start())
}

func (a *Bam) End() uint32 {
	if a._end != 0 {
		return a._end
	}
	a._end = uint32(a.Record.End())
	return a._end
}

func (a *Bam) Source() uint32 {
	return a.source
}

func (a *Bam) SetSource(src uint32) {
	a.source = src
}

func (a *Bam) AddRelated(b interfaces.Relatable) {
	if a.related == nil {
		a.related = make([]interfaces.Relatable, 1, 2)
		a.related[0] = b
	} else {
		a.related = append(a.related, b)
	}
}
func (a *Bam) Related() []interfaces.Relatable {
	return a.related
}

func (a *Bam) MapQ() int {
	return int(a.Record.MapQ)
}

func check(err error) {
	if err != nil {
		panic(err)
	}
}

func BamToRelatable(f io.Reader) (interfaces.RelatableChannel, error) {

	ch := make(chan interfaces.Relatable, 64)
	b, err := bam.NewReader(f, 0)
	if err != nil {
		return nil, err
	}

	go func() {
		for {
			rec, err := b.Read()
			if err != nil {
				if err == io.EOF {
					break
				} else {
					log.Println(err)
					break
				}
			}
			if rec.RefID() == -1 { // unmapped
				continue
			}
			// TODO: see if keeping the list of chrom names and using a ref is better.
			bam := Bam{Record: rec, Chromosome: rec.Ref.Name(), related: nil}
			ch <- &bam
		}
		close(ch)
		b.Close()
		f.(io.ReadCloser).Close()
	}()
	return ch, nil
}
<commit_msg>make bams Queryable<commit_after>// Implements Relatable for Bam

package parsers

import (
	"compress/gzip"
	"fmt"
	"io"
	"log"
	"os"
	"strings"

	"github.com/biogo/hts/bam"
	"github.com/biogo/hts/bgzf"
	"github.com/biogo/hts/bgzf/index"
	"github.com/biogo/hts/sam"
	"github.com/brentp/irelate/interfaces"
)

type Bam struct {
	*sam.Record
	source     uint32
	related    []interfaces.Relatable
	Chromosome string
	_end       uint32
}

func (a *Bam) Chrom() string {
	return a.Chromosome
}

// cast to 32 bits.
func (a *Bam) Start() uint32 {
	return uint32(a.Record.Start())
}

func (a *Bam) End() uint32 {
	if a._end != 0 {
		return a._end
	}
	a._end = uint32(a.Record.End())
	return a._end
}

func (a *Bam) Source() uint32 {
	return a.source
}

func (a *Bam) SetSource(src uint32) {
	a.source = src
}

func (a *Bam) AddRelated(b interfaces.Relatable) {
	if a.related == nil {
		a.related = make([]interfaces.Relatable, 1, 2)
		a.related[0] = b
	} else {
		a.related = append(a.related, b)
	}
}
func (a *Bam) Related() []interfaces.Relatable {
	return a.related
}

func (a *Bam) MapQ() int {
	return int(a.Record.MapQ)
}

func check(err error) {
	if err != nil {
		panic(err)
	}
}

func BamToRelatable(f io.Reader) (interfaces.RelatableChannel, error) {

	ch := make(chan interfaces.Relatable, 64)
	b, err := bam.NewReader(f, 0)
	if err != nil {
		return nil, err
	}

	go func() {
		for {
			rec, err := b.Read()
			if err != nil {
				if err == io.EOF {
					break
				} else {
					log.Println(err)
					break
				}
			}
			if rec.RefID() == -1 { // unmapped
				continue
			}
			// TODO: see if keeping the list of chrom names and using a ref is better.
			bam := Bam{Record: rec, Chromosome: rec.Ref.Name(), related: nil}
			ch <- &bam
		}
		close(ch)
		b.Close()
		f.(io.ReadCloser).Close()
	}()
	return ch, nil
}

type BamQueryable struct {
	idx  *bam.Index
	path string
	file io.Reader
	bgzf *bgzf.Reader
	refs map[string]*sam.Reference
}

func NewBamQueryable(path string, workers ...int) (*BamQueryable, error) {
	f, err := os.Open(path + ".bai")
	if err != nil {
		return nil, err
	}
	defer f.Close()
	gz, err := gzip.NewReader(f)
	if err != nil {
		return nil, err
	}
	defer gz.Close()
	idx, err := bam.ReadIndex(gz)

	b, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	n := 1
	if len(workers) > 0 {
		n = workers[0]
	}

	bgz, err := bgzf.NewReader(b, n)
	if err != nil {
		return nil, err
	}

	br, err := bam.NewReader(bgz, 1)
	if err != nil {
		return nil, err
	}
	hdr := br.Header()
	refs := make(map[string]*sam.Reference, 40)
	for _, r := range hdr.Refs() {
		refs[r.Name()] = r
	}
	br.Close()

	return &BamQueryable{idx: idx, path: path, file: b, bgzf: bgz, refs: refs}, nil

}

func newShort(old *BamQueryable) (*BamQueryable, error) {
	b := &BamQueryable{
		idx:  old.idx,
		path: old.path,
		refs: old.refs,
	}
	var err error
	b.file, err = os.Open(b.path)
	if err != nil {
		return nil, err
	}
	b.bgzf, err = bgzf.NewReader(b.file, 2)
	if err != nil {
		return nil, err
	}
	return b, nil
}

func (b *BamQueryable) Query(region interfaces.IPosition) (interfaces.RelatableIterator, error) {
	bn, err := newShort(b) // make a copy since we're messing with the file-pointer
	if err != nil {
		return nil, err
	}

	ref, ok := b.refs[region.Chrom()]
	if !ok {
		if !strings.HasPrefix(region.Chrom(), "chr") {
			ref, ok = b.refs["chr"+region.Chrom()]
		} else if strings.HasPrefix(region.Chrom(), "chr") {
			ref, ok = b.refs[region.Chrom()[3:]]
		}
	}
	if !ok {
		return nil, fmt.Errorf("%s not found in %s", region.Chrom(), bn.path)
	}

	ch := make(chan interfaces.Relatable, 100)
	go func() {
		chrom := ref.Name()
		chunks, err := bn.idx.Chunks(ref, int(region.Start()), int(region.End()))
		if err != nil {
			if err != io.EOF {
				log.Println(err)
			}
			close(ch)
			return
		}
		cr, err := index.NewChunkReader(bn.bgzf, chunks)
		if err != nil {
			if err != io.EOF {
				log.Println(err)
			}
			close(ch)
			return
		}
		br, err := bam.NewReader(cr, 2)
		if err != nil {
			if err != io.EOF {
				log.Println(err)
			}
			close(ch)
			return
		}
		for {
			rec, err := br.Read()
			if err != nil {
				if err != io.EOF {
					log.Println(err)
				}
				break
			}
			ch <- &Bam{Record: rec, Chromosome: chrom, related: nil}
		}
		br.Close()
		close(ch)
	}()
	return &BamIterator{ch, bn}, nil
}

func (b *BamQueryable) Close() error {
	b.bgzf.Close()
	if cr, ok := b.file.(io.ReadCloser); ok {
		cr.Close()
	}
	return nil
}

type BamIterator struct {
	ch interfaces.RelatableChannel
	b  *BamQueryable
}

func NewBamIterator(f string) (*BamIterator, error) {
	fh, err := os.Open(f)
	if err != nil {
		return nil, err
	}
	ch, err := BamToRelatable(fh)

	b := &BamIterator{ch: ch}

	return b, err
}

func (b *BamIterator) Close() error {
	if b.b != nil {
		return b.Close()
	}
	return nil
}

func (b *BamIterator) Next() (interfaces.Relatable, error) {
	rec, ok := <-b.ch
	if !ok {
		return nil, io.EOF
	}
	return rec, nil
}
<|endoftext|>


<commit_before>package tools

import (
	"math"
	"math/big"
	"sort"
	"strconv"
)

// Sum returns the sum of its argument.
func Sum(a ...int) int {
	var s int
	for _, v := range a {
		s += v
	}
	return s
}

// SqrtInt computes the square root of n and casts to an int.
func SqrtInt(n int) int {
	return int(math.Sqrt(float64(n)))
}

// Divisors returns all the proper divisors of n.
func Divisors(n int) []int {
	d := []int{1}
	for i := 2; i <= SqrtInt(n); i++ {
		if n%i == 0 {
			d = append(d, i)
			if other := n / i; other != i {
				d = append(d, other)
			}
		}
	}
	return d
}

// SumDivisors sums all the proper divisors of n.
func SumDivisors(n int) int {
	return Sum(Divisors(n)...)
}

// Perms return a channel of each permutation of a slice.
func Perms(slice []int) chan []int {
	c := make(chan []int)
	go func() {
		if len(slice) == 1 {
			c <- slice
		} else {
			for i, v := range slice {
				b := append([]int(nil), slice[:i]...)
				b = append(b, slice[i+1:]...)
				for p := range Perms(b) {
					c <- append([]int{v}, p...)
				}
			}
		}
		close(c)
	}()
	return c
}

// Divmod gives the quotient and remainder of integer division.
func Divmod(a, b int) (int, int) {
	return a / b, a % b
}

// PrimeFactors gives the prime factors of an int.
func PrimeFactors(n int) map[int]int {
	factors := make(map[int]int)
	d := 2
	for n > 1 {
		for n%d == 0 {
			n /= d
			factors[d]++
		}
		d++
	}
	return factors
}

// IsPrime determines if a given number is prime.
// https://en.wikipedia.org/wiki/Primality_test
func IsPrime(n int) bool {
	if n <= 1 {
		return false
	}
	if n <= 3 {
		return true
	}
	if n%2 == 0 || n%3 == 0 {
		return false
	}
	b := SqrtInt(n)
	for i := 5; i <= b; i += 6 {
		if n%i == 0 || n%(i+2) == 0 {
			return false
		}
	}
	return true
}

// SievePrime gives all the primes below n via the method of Eratosthenes.
func SievePrime(n int) []int {
	var primes []int
	comp := make([]bool, n+2)
	for i := 2; i <= int(math.Sqrt(float64(n))); i++ {
		for j := i * i; j < n; j += i {
			comp[j] = true
		}
	}
	for i, v := range comp {
		if i > 1 && !v && i < n {
			primes = append(primes, i)
		}
	}
	return primes
}

// Exp computes x to the power of y.
func Exp(x, y int) *big.Int {
	a := big.NewInt(1)
	b := big.NewInt(int64(x))
	for i := 0; i < y; i++ {
		a.Mul(a, b)
	}
	return a
}

// DigitSum sums the digit^p of a given number.
func DigitSum(n, p int) *big.Int {
	s := big.NewInt(0)
	for _, c := range strconv.Itoa(n) {
		i, _ := strconv.Atoi(string(c))
		s.Add(s, Exp(i, p))
	}
	return s
}

// JoinInts joins slice of ints and return it as an int.
func JoinInts(slice []int) int {
	var sum int
	len := len(slice) - 1
	for i, v := range slice {
		p := 1
		for j := 0; j < len-i; j++ {
			p *= 10
		}
		sum += p * v
	}
	return sum
}

// JoinIntsBig joins slice of ints into a big.Int.
func JoinIntsBig(slice []int) *big.Int {
	sum := big.NewInt(0)
	ten := big.NewInt(10)
	p := Exp(10, len(slice)-1)
	for _, v := range slice {
		z := big.NewInt(1)
		z.Mul(p, big.NewInt(int64(v)))
		sum.Add(sum, z)
		p.Div(p, ten)
	}
	return sum
}

// ProdInts computes the product of the slice of ints.
func ProdInts(slice []int) int {
	p := 1
	for _, v := range slice {
		p *= v
	}
	return p
}

// MinInt returns the smaller number of a and b.
func MinInt(a, b int) int {
	if a < b {
		return a
	}
	return b
}

// MaxInt returns the larger number of a and b.
func MaxInt(a, b int) int {
	if a > b {
		return a
	}
	return b
}

// SimplifyFraction simplifies a/b and return the (numerator, denominator).
func SimplifyFraction(a, b int) (int, int) {
	pa := PrimeFactors(a)
	pb := PrimeFactors(b)
	for k, v := range pa {
		v2, ok := pb[k]
		if !ok {
			continue
		}
		m := MinInt(v, v2)
		pa[k] = v - m
		pb[k] = v2 - m
	}
	sa := 1
	for k, v := range pa {
		for i := 0; i < v; i++ {
			sa *= k
		}
	}
	sb := 1
	for k, v := range pb {
		for i := 0; i < v; i++ {
			sb *= k
		}
	}
	return sa, sb
}

// Digits returns the individual digits as a slice of int.
func Digits(n int) []int {
	var d []int
	for n > 0 {
		d = append([]int{n % 10}, d...)
		n /= 10
	}
	return d
}

// DigitsBig returns the individual digits of a big.Int as a slice of int.
func DigitsBig(n *big.Int) []int {
	var d []int
	x := big.NewInt(0)
	x.Set(n)
	zero := big.NewInt(0)
	ten := big.NewInt(10)
	for x.Cmp(zero) > 0 {
		m := big.NewInt(1)
		x.DivMod(x, ten, m)
		d = append([]int{int(m.Int64())}, d...)
	}
	return d
}

// DigitsIth returns the i-th digit of a number n.
func DigitsIth(n, i int) int {
	d := Digits(n)
	if i < 0 || i >= len(d) {
		return -1
	}
	return Digits(n)[i]
}

// ReverseSliceInts reverses a slice of ints, and return the new slice.
func ReverseSliceInts(a []int) []int {
	s := len(a)
	r := make([]int, s)
	for i, v := range a {
		r[s-1-i] = v
	}
	return r
}

// ReverseInt reverses a given int.
func ReverseInt(n int) int {
	return JoinInts(ReverseSliceInts(Digits(n)))
}

// ReverseIntBig reverses a given big.Int.
func ReverseIntBig(n *big.Int) *big.Int {
	return JoinIntsBig(ReverseSliceInts(DigitsBig(n)))
}

// Factorial returns the fractorial of a number.
func Factorial(n int) *big.Int {
	a := big.NewInt(1)
	for i := 2; i <= n; i++ {
		a.Mul(a, big.NewInt(int64(i)))
	}
	return a
}

// IsPalindromeInt tells if a number is a palindrome.
func IsPalindromeInt(n int) bool {
	return IsPalindromeString(strconv.Itoa(n))
}

// IsPalindromeIntBig tells if a big.Int number is a palindrome.
func IsPalindromeIntBig(n *big.Int) bool {
	return IsPalindromeString(n.String())
}

// IsPandigital determines if a number is pandigital.
func IsPandigital(n int) bool {
	d := Digits(n)
	sort.Ints(d)
	ans := true
	for i, v := range d {
		if i+1 != v {
			ans = false
			break
		}
	}
	return ans
}

// IsPermuted determines if digits in the two given ints are permuted.
func IsPermuted(a, b int) bool {
	da := Digits(a)
	db := Digits(b)
	sort.Ints(da)
	sort.Ints(db)
	return JoinInts(da) == JoinInts(db)
}

// TriangleNumber returns the n-th triangle number.
func TriangleNumber(n int) int {
	return n * (n + 1) / 2
}

// IsTriangleNumber determines if a number is a triangle number.
func IsTriangleNumber(n int) bool {
	a := 2 * n
	b := SqrtInt(a)
	return b*(b+1) == a
}

// PentagonNumber returns the n-th pentagonal number.
func PentagonNumber(n int) int {
	return n * (3*n - 1) / 2
}

// IsPentagonNumber determines if a number is a pentagonal number.
func IsPentagonNumber(n int) bool {
	t := (math.Sqrt(24*float64(n)+1) + 1) / 6
	return t == math.Floor(t)
}

// HexagonNumber returns the n-th hexagonal number.
func HexagonNumber(n int) int {
	return n * (2*n - 1)
}

// IsHexagonNumber determines if a number is a hexagonal number.
func IsHexagonNumber(n int) bool {
	t := (math.Sqrt(8*float64(n)+1) + 1) / 4
	return t == math.Floor(t)
}

// IsPermute determines if all the given ints are permutation of each other.
func IsPermute(args ...int) bool {
	permute := true
	var compare int
	for i, v := range args {
		d := Digits(v)
		sort.Ints(d)
		j := JoinInts(d)
		if i == 0 {
			compare = j
		} else {
			if j != compare {
				permute = false
				break
			}
		}
	}
	return permute
}

// NCR computes the number nCr.
func NCR(n, r int) *big.Int {
	z := big.NewInt(0)
	return z.Binomial(int64(n), int64(r))
}
<commit_msg>+ add CombIndex and Comb<commit_after>package tools

import (
	"math"
	"math/big"
	"reflect"
	"sort"
	"strconv"
)

// Sum returns the sum of its argument.
func Sum(a ...int) int {
	var s int
	for _, v := range a {
		s += v
	}
	return s
}

// SqrtInt computes the square root of n and casts to an int.
func SqrtInt(n int) int {
	return int(math.Sqrt(float64(n)))
}

// Divisors returns all the proper divisors of n.
func Divisors(n int) []int {
	d := []int{1}
	for i := 2; i <= SqrtInt(n); i++ {
		if n%i == 0 {
			d = append(d, i)
			if other := n / i; other != i {
				d = append(d, other)
			}
		}
	}
	return d
}

// SumDivisors sums all the proper divisors of n.
func SumDivisors(n int) int {
	return Sum(Divisors(n)...)
}

// Perms return a channel of each permutation of a slice.
func Perms(slice []int) chan []int {
	c := make(chan []int)
	go func() {
		if len(slice) == 1 {
			c <- slice
		} else {
			for i, v := range slice {
				b := append([]int(nil), slice[:i]...)
				b = append(b, slice[i+1:]...)
				for p := range Perms(b) {
					c <- append([]int{v}, p...)
				}
			}
		}
		close(c)
	}()
	return c
}

// Divmod gives the quotient and remainder of integer division.
func Divmod(a, b int) (int, int) {
	return a / b, a % b
}

// PrimeFactors gives the prime factors of an int.
func PrimeFactors(n int) map[int]int {
	factors := make(map[int]int)
	d := 2
	for n > 1 {
		for n%d == 0 {
			n /= d
			factors[d]++
		}
		d++
	}
	return factors
}

// IsPrime determines if a given number is prime.
// https://en.wikipedia.org/wiki/Primality_test
func IsPrime(n int) bool {
	if n <= 1 {
		return false
	}
	if n <= 3 {
		return true
	}
	if n%2 == 0 || n%3 == 0 {
		return false
	}
	b := SqrtInt(n)
	for i := 5; i <= b; i += 6 {
		if n%i == 0 || n%(i+2) == 0 {
			return false
		}
	}
	return true
}

// SievePrime gives all the primes below n via the method of Eratosthenes.
func SievePrime(n int) []int {
	var primes []int
	comp := make([]bool, n+2)
	for i := 2; i <= int(math.Sqrt(float64(n))); i++ {
		for j := i * i; j < n; j += i {
			comp[j] = true
		}
	}
	for i, v := range comp {
		if i > 1 && !v && i < n {
			primes = append(primes, i)
		}
	}
	return primes
}

// Exp computes x to the power of y.
func Exp(x, y int) *big.Int {
	a := big.NewInt(1)
	b := big.NewInt(int64(x))
	for i := 0; i < y; i++ {
		a.Mul(a, b)
	}
	return a
}

// DigitSum sums the digit^p of a given number.
func DigitSum(n, p int) *big.Int {
	s := big.NewInt(0)
	for _, c := range strconv.Itoa(n) {
		i, _ := strconv.Atoi(string(c))
		s.Add(s, Exp(i, p))
	}
	return s
}

// JoinInts joins slice of ints and return it as an int.
func JoinInts(slice []int) int {
	var sum int
	len := len(slice) - 1
	for i, v := range slice {
		p := 1
		for j := 0; j < len-i; j++ {
			p *= 10
		}
		sum += p * v
	}
	return sum
}

// JoinIntsBig joins slice of ints into a big.Int.
func JoinIntsBig(slice []int) *big.Int {
	sum := big.NewInt(0)
	ten := big.NewInt(10)
	p := Exp(10, len(slice)-1)
	for _, v := range slice {
		z := big.NewInt(1)
		z.Mul(p, big.NewInt(int64(v)))
		sum.Add(sum, z)
		p.Div(p, ten)
	}
	return sum
}

// ProdInts computes the product of the slice of ints.
func ProdInts(slice []int) int {
	p := 1
	for _, v := range slice {
		p *= v
	}
	return p
}

// MinInt returns the smaller number of a and b.
func MinInt(a, b int) int {
	if a < b {
		return a
	}
	return b
}

// MaxInt returns the larger number of a and b.
func MaxInt(a, b int) int {
	if a > b {
		return a
	}
	return b
}

// SimplifyFraction simplifies a/b and return the (numerator, denominator).
func SimplifyFraction(a, b int) (int, int) {
	pa := PrimeFactors(a)
	pb := PrimeFactors(b)
	for k, v := range pa {
		v2, ok := pb[k]
		if !ok {
			continue
		}
		m := MinInt(v, v2)
		pa[k] = v - m
		pb[k] = v2 - m
	}
	sa := 1
	for k, v := range pa {
		for i := 0; i < v; i++ {
			sa *= k
		}
	}
	sb := 1
	for k, v := range pb {
		for i := 0; i < v; i++ {
			sb *= k
		}
	}
	return sa, sb
}

// Digits returns the individual digits as a slice of int.
func Digits(n int) []int {
	var d []int
	for n > 0 {
		d = append([]int{n % 10}, d...)
		n /= 10
	}
	return d
}

// DigitsBig returns the individual digits of a big.Int as a slice of int.
func DigitsBig(n *big.Int) []int {
	var d []int
	x := big.NewInt(0)
	x.Set(n)
	zero := big.NewInt(0)
	ten := big.NewInt(10)
	for x.Cmp(zero) > 0 {
		m := big.NewInt(1)
		x.DivMod(x, ten, m)
		d = append([]int{int(m.Int64())}, d...)
	}
	return d
}

// DigitsIth returns the i-th digit of a number n.
func DigitsIth(n, i int) int {
	d := Digits(n)
	if i < 0 || i >= len(d) {
		return -1
	}
	return Digits(n)[i]
}

// ReverseSliceInts reverses a slice of ints, and return the new slice.
func ReverseSliceInts(a []int) []int {
	s := len(a)
	r := make([]int, s)
	for i, v := range a {
		r[s-1-i] = v
	}
	return r
}

// ReverseInt reverses a given int.
func ReverseInt(n int) int {
	return JoinInts(ReverseSliceInts(Digits(n)))
}

// ReverseIntBig reverses a given big.Int.
func ReverseIntBig(n *big.Int) *big.Int {
	return JoinIntsBig(ReverseSliceInts(DigitsBig(n)))
}

// Factorial returns the fractorial of a number.
func Factorial(n int) *big.Int {
	a := big.NewInt(1)
	for i := 2; i <= n; i++ {
		a.Mul(a, big.NewInt(int64(i)))
	}
	return a
}

// IsPalindromeInt tells if a number is a palindrome.
func IsPalindromeInt(n int) bool {
	return IsPalindromeString(strconv.Itoa(n))
}

// IsPalindromeIntBig tells if a big.Int number is a palindrome.
func IsPalindromeIntBig(n *big.Int) bool {
	return IsPalindromeString(n.String())
}

// IsPandigital determines if a number is pandigital.
func IsPandigital(n int) bool {
	d := Digits(n)
	sort.Ints(d)
	ans := true
	for i, v := range d {
		if i+1 != v {
			ans = false
			break
		}
	}
	return ans
}

// IsPermuted determines if digits in the two given ints are permuted.
func IsPermuted(a, b int) bool {
	da := Digits(a)
	db := Digits(b)
	sort.Ints(da)
	sort.Ints(db)
	return JoinInts(da) == JoinInts(db)
}

// TriangleNumber returns the n-th triangle number.
func TriangleNumber(n int) int {
	return n * (n + 1) / 2
}

// IsTriangleNumber determines if a number is a triangle number.
func IsTriangleNumber(n int) bool {
	a := 2 * n
	b := SqrtInt(a)
	return b*(b+1) == a
}

// PentagonNumber returns the n-th pentagonal number.
func PentagonNumber(n int) int {
	return n * (3*n - 1) / 2
}

// IsPentagonNumber determines if a number is a pentagonal number.
func IsPentagonNumber(n int) bool {
	t := (math.Sqrt(24*float64(n)+1) + 1) / 6
	return t == math.Floor(t)
}

// HexagonNumber returns the n-th hexagonal number.
func HexagonNumber(n int) int {
	return n * (2*n - 1)
}

// IsHexagonNumber determines if a number is a hexagonal number.
func IsHexagonNumber(n int) bool {
	t := (math.Sqrt(8*float64(n)+1) + 1) / 4
	return t == math.Floor(t)
}

// IsPermute determines if all the given ints are permutation of each other.
func IsPermute(args ...int) bool {
	permute := true
	var compare int
	for i, v := range args {
		d := Digits(v)
		sort.Ints(d)
		j := JoinInts(d)
		if i == 0 {
			compare = j
		} else {
			if j != compare {
				permute = false
				break
			}
		}
	}
	return permute
}

// NCR computes the number nCr.
func NCR(n, r int) *big.Int {
	z := big.NewInt(0)
	return z.Binomial(int64(n), int64(r))
}

// CombIndex gives the combinations of selecting [0, k-1] from [0, n-1] in
// ascending order.
func CombIndex(n, k int) <-chan []int {
	c := make(chan []int)
	var comb []int
	for i := 0; i < k; i++ {
		comb = append(comb, i)
	}
	next := func() bool {
		i := k - 1
		comb[i]++
		for i > 0 && comb[i] >= n-k+1+i {
			i--
			comb[i]++
		}
		if comb[0] > n-k {
			return false
		}
		for i = i + 1; i < k; i++ {
			comb[i] = comb[i-1] + 1
		}
		return true
	}
	copy := func(a []int) []int {
		var b []int
		for _, v := range a {
			b = append(b, v)
		}
		return b
	}
	go func() {
		defer close(c)
		c <- copy(comb)
		for next() {
			c <- copy(comb)
		}
	}()
	return c
}

// Comb gives the combinations of selecting k items from the slice of collection.
func Comb(collection interface{}, k int) <-chan []interface{} {
	col := reflect.ValueOf(collection)
	c := make(chan []interface{})
	go func() {
		defer close(c)
		for indices := range CombIndex(col.Len(), k) {
			var chosen []interface{}
			for _, i := range indices {
				chosen = append(chosen, col.Index(i).Interface())
			}
			c <- chosen
		}
	}()
	return c
}
<|endoftext|>


<commit_before>package nodetasks

import (
	"fmt"
	"github.com/golang/glog"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/nodeup/cloudinit"
	"k8s.io/kops/upup/pkg/fi/nodeup/local"
	"k8s.io/kops/upup/pkg/fi/nodeup/tags"
	"os"
	"os/exec"
)

type UpdatePackages struct {
	// We can't be completely empty or we don't run
	Updated bool
}

var _ fi.HasDependencies = &UpdatePackages{}

func NewUpdatePackages() *UpdatePackages {
	return &UpdatePackages{Updated: true}
}

func (p *UpdatePackages) GetDependencies(tasks map[string]fi.Task) []fi.Task {
	return []fi.Task{}
}

func (p *UpdatePackages) String() string {
	return fmt.Sprintf("UpdatePackages")
}

func (e *UpdatePackages) Find(c *fi.Context) (*UpdatePackages, error) {
	return nil, nil
}

func (e *UpdatePackages) Run(c *fi.Context) error {
	return fi.DefaultDeltaRunMethod(e, c)
}

func (s *UpdatePackages) CheckChanges(a, e, changes *UpdatePackages) error {
	return nil
}

func (_ *UpdatePackages) RenderLocal(t *local.LocalTarget, a, e, changes *UpdatePackages) error {
	if os.Getenv("SKIP_PACKAGE_UPDATE") != "" {
		glog.Infof("SKIP_PACKAGE_UPDATE was set; skipping package update")
		return nil
	}
	var args []string
	if t.HasTag(tags.TagOSFamilyDebian) {
		args = []string{"apt-get", "update"}

	} else if t.HasTag(tags.TagOSFamilyRHEL) {
		// Probably not technically needed
		args = []string{"/usr/bin/yum", "check-update"}
	} else {
		return fmt.Errorf("unsupported package system")
	}
	glog.Infof("running command %s", args)
	cmd := exec.Command(args[0], args[1:]...)
	output, err := cmd.CombinedOutput()
	if err != nil {
		return fmt.Errorf("error update packages: %v: %s", err, string(output))
	}

	return nil
}

func (_ *UpdatePackages) RenderCloudInit(t *cloudinit.CloudInitTarget, a, e, changes *UpdatePackages) error {
	t.Config.PackageUpdate = true
	return nil
}
<commit_msg>Handle exit code 100 from 'yum check-update'<commit_after>package nodetasks

import (
	"fmt"
	"github.com/golang/glog"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/nodeup/cloudinit"
	"k8s.io/kops/upup/pkg/fi/nodeup/local"
	"k8s.io/kops/upup/pkg/fi/nodeup/tags"
	"os"
	"os/exec"
	"syscall"
)

type UpdatePackages struct {
	// We can't be completely empty or we don't run
	Updated bool
}

var _ fi.HasDependencies = &UpdatePackages{}

func NewUpdatePackages() *UpdatePackages {
	return &UpdatePackages{Updated: true}
}

func (p *UpdatePackages) GetDependencies(tasks map[string]fi.Task) []fi.Task {
	return []fi.Task{}
}

func (p *UpdatePackages) String() string {
	return fmt.Sprintf("UpdatePackages")
}

func (e *UpdatePackages) Find(c *fi.Context) (*UpdatePackages, error) {
	return nil, nil
}

func (e *UpdatePackages) Run(c *fi.Context) error {
	return fi.DefaultDeltaRunMethod(e, c)
}

func (s *UpdatePackages) CheckChanges(a, e, changes *UpdatePackages) error {
	return nil
}

func (_ *UpdatePackages) RenderLocal(t *local.LocalTarget, a, e, changes *UpdatePackages) error {
	if os.Getenv("SKIP_PACKAGE_UPDATE") != "" {
		glog.Infof("SKIP_PACKAGE_UPDATE was set; skipping package update")
		return nil
	}
	var args []string
	if t.HasTag(tags.TagOSFamilyDebian) {
		args = []string{"apt-get", "update"}

	} else if t.HasTag(tags.TagOSFamilyRHEL) {
		// Probably not technically needed
		args = []string{"/usr/bin/yum", "check-update"}
	} else {
		return fmt.Errorf("unsupported package system")
	}
	glog.Infof("running command %s", args)
	cmd := exec.Command(args[0], args[1:]...)
	output, err := cmd.CombinedOutput()
	// 'yum check-update' exits with 100 if it finds updates; treat it like a success
	if exitCode := cmd.ProcessState.Sys().(syscall.WaitStatus).ExitStatus(); err != nil && exitCode != 100 {
		return fmt.Errorf("error update packages: %v: %s", err, string(output))
	}

	return nil
}

func (_ *UpdatePackages) RenderCloudInit(t *cloudinit.CloudInitTarget, a, e, changes *UpdatePackages) error {
	t.Config.PackageUpdate = true
	return nil
}
<|endoftext|>


<commit_before>package grayt

import "math/rand"

func TracerImage(s Scene, acc Accumulator) {
	pxPitch := 2.0 / float64(acc.wide)
	for pxX := 0; pxX < acc.wide; pxX++ {
		for pxY := 0; pxY < acc.high; pxY++ {
			//if pxX != 140 || pxY != 40 {
			//	continue
			//}
			x := (float64(pxX-acc.wide/2) + rand.Float64()) * pxPitch
			y := (float64(pxY-acc.high/2) + rand.Float64()) * pxPitch * -1.0
			r := s.Camera.MakeRay(x, y)
			r.Dir = r.Dir.Unit()
			acc.add(pxX, pxY, tracePath(s.Entities, r))
		}
	}
}

func tracePath(entities []Entity, r Ray) Colour {

	intersection, hitEntity := closestHit(entities, r)
	if hitEntity == nil {
		return Colour{0, 0, 0}
	}

	// Calculate probability of emitting.
	const pEmit = 0.5

	// Handle emit case.
	if rand.Float64() < pEmit {
		return hitEntity.Material.Colour.
			Scale(1.0 / pEmit * hitEntity.Material.Emittance)
	}

	// Find where the ray hit. Reduce the intersection distance by a small
	// amount so that reflected rays don't intersect with it immediately.
	hitLoc := r.At(addULPs(intersection.Distance, -50))

	// Orient the unit normal towards the ray origin.
	if intersection.UnitNormal.Dot(r.Dir) > 0 {
		intersection.UnitNormal = intersection.UnitNormal.Extended(-1.0)
	}

	// Create a random vector on the hemisphere towards the normal.
	rnd := Vect{rand.NormFloat64(), rand.NormFloat64(), rand.NormFloat64()}
	rnd = rnd.Unit()
	if rnd.Dot(intersection.UnitNormal) < 0 {
		rnd = rnd.Extended(-1.0)
	}

	// Apply the BRDF (bidirectional reflection distribution function).
	brdf := rnd.Dot(intersection.UnitNormal)

	return tracePath(entities, Ray{Start: hitLoc, Dir: rnd}).
		Scale(1.0 / (1 - pEmit) * brdf).
		Mul(hitEntity.Material.Colour)
}

func closestHit(entities []Entity, r Ray) (Intersection, *Entity) {
	var closest struct {
		intersection Intersection
		entity       *Entity
	}
	for i := range entities {
		intersection, hit := entities[i].Surface.Intersect(r)
		if !hit {
			continue
		}
		if closest.entity == nil || intersection.Distance < closest.intersection.Distance {
			closest.intersection = intersection
			closest.entity = &entities[i]
		}
	}
	return closest.intersection, closest.entity
}
<commit_msg>Emit with probability 1.0 whet hitting a light, 0.1 otherwise<commit_after>package grayt

import "math/rand"

func TracerImage(s Scene, acc Accumulator) {
	pxPitch := 2.0 / float64(acc.wide)
	for pxX := 0; pxX < acc.wide; pxX++ {
		for pxY := 0; pxY < acc.high; pxY++ {
			//if pxX != 140 || pxY != 40 {
			//	continue
			//}
			x := (float64(pxX-acc.wide/2) + rand.Float64()) * pxPitch
			y := (float64(pxY-acc.high/2) + rand.Float64()) * pxPitch * -1.0
			r := s.Camera.MakeRay(x, y)
			r.Dir = r.Dir.Unit()
			acc.add(pxX, pxY, tracePath(s.Entities, r))
		}
	}
}

func tracePath(entities []Entity, r Ray) Colour {

	intersection, hitEntity := closestHit(entities, r)
	if hitEntity == nil {
		return Colour{0, 0, 0}
	}

	// Calculate probability of emitting.
	pEmit := 0.1
	if hitEntity.Material.Emittance != 0 {
		pEmit = 1.0
	}

	// Handle emit case.
	if rand.Float64() < pEmit {
		return hitEntity.Material.Colour.
			Scale(hitEntity.Material.Emittance / pEmit)
	}

	// Find where the ray hit. Reduce the intersection distance by a small
	// amount so that reflected rays don't intersect with it immediately.
	hitLoc := r.At(addULPs(intersection.Distance, -50))

	// Orient the unit normal towards the ray origin.
	if intersection.UnitNormal.Dot(r.Dir) > 0 {
		intersection.UnitNormal = intersection.UnitNormal.Extended(-1.0)
	}

	// Create a random vector on the hemisphere towards the normal.
	rnd := Vect{rand.NormFloat64(), rand.NormFloat64(), rand.NormFloat64()}
	rnd = rnd.Unit()
	if rnd.Dot(intersection.UnitNormal) < 0 {
		rnd = rnd.Extended(-1.0)
	}

	// Apply the BRDF (bidirectional reflection distribution function).
	brdf := rnd.Dot(intersection.UnitNormal)

	return tracePath(entities, Ray{Start: hitLoc, Dir: rnd}).
		Scale(brdf / (1 - pEmit)).
		Mul(hitEntity.Material.Colour)
}

func closestHit(entities []Entity, r Ray) (Intersection, *Entity) {
	var closest struct {
		intersection Intersection
		entity       *Entity
	}
	for i := range entities {
		intersection, hit := entities[i].Surface.Intersect(r)
		if !hit {
			continue
		}
		if closest.entity == nil || intersection.Distance < closest.intersection.Distance {
			closest.intersection = intersection
			closest.entity = &entities[i]
		}
	}
	return closest.intersection, closest.entity
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"io/ioutil"
	"reflect"
	"testing"
)

const headerEmail = "X-Auth-Email"
const headerKey = "X-Auth-Key"

func TestBuildingAnHTTPRequest(t *testing.T) {
	query := &CloudFlareQuery{
		RootURL:   "foo.com",
		AuthEmail: "user@example.com",
		AuthKey:   "abc123",
	}

	request, err := query.NewRequest("GET", "/zones")
	if err != nil {
		t.Fatalf("Should've built request without errors", err.Error())
	}

	if request.Method != "GET" {
		t.Fatal("Expected a GET request")
	}

	if request.URL.String() != "foo.com/zones" {
		t.Fatal("Not the zones path for CF")
	}

	if val := request.Header.Get(headerEmail); val != query.AuthEmail {
		t.Error("AuthEmail incorrect:", val)
	}

	if val := request.Header.Get(headerKey); val != query.AuthKey {
		t.Error("AuthKey incorrect:", val)
	}
}

func TestSettingARequestBody(t *testing.T) {
	query := &CloudFlareQuery{
		RootURL:   "foo.com",
		AuthEmail: "user@example.com",
		AuthKey:   "abc123",
	}

	request, err := query.NewRequestBody("POST", "/foo", bytes.NewBuffer([]byte(`{"a": 1}`)))
	if err != nil {
		t.Fatal("Should've built a request with a body with no errors", err)
	}

	defer request.Body.Close()
	body, err := ioutil.ReadAll(request.Body)
	if err != nil {
		t.Fatal("Couldn't read the response body without errors", err)
	}

	if !reflect.DeepEqual(body, []byte(`{"a": 1}`)) {
		t.Fatal("The body should've been set to some JSON we provided but was:", body)
	}
}
<commit_msg>Replace CloudFlareQuery tests with ginkgo<commit_after>package main_test

import (
	. "github.com/alphagov/cloudflare-configure"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"io/ioutil"
	"net/http"
	"strings"
)

const headerEmail = "X-Auth-Email"
const headerKey = "X-Auth-Key"

var _ = Describe("CloudFlareQuery", func() {
	var (
		authEmail = "user@example.com"
		authKey   = "abc123"
		query     CloudFlareQuery
	)

	BeforeEach(func() {
		query = CloudFlareQuery{
			RootURL:   "https://example.com/api",
			AuthEmail: authEmail,
			AuthKey:   authKey,
		}
	})

	Describe("MakeRequest", func() {
		var (
			req *http.Request
			err error
		)

		BeforeEach(func() {
			req, err = query.NewRequest("GET", "/zones")
		})

		It("should return request and no errors", func() {
			Expect(req).ToNot(BeNil())
			Expect(err).To(BeNil())
		})

		It("should set method and path", func() {
			Expect(req.Method).To(Equal("GET"))
			Expect(req.URL.String()).To(Equal("https://example.com/api/zones"))
		})

		It("should not set request body", func() {
			Expect(req.Body).To(BeNil())
		})

		It("should set authentication email and key header", func() {
			Expect(req.Header.Get(headerEmail)).To(Equal(authEmail))
			Expect(req.Header.Get(headerKey)).To(Equal(authKey))
		})
	})

	Describe("MakeRequestBody", func() {
		var (
			body = `{"foo": "bar"}`
			req  *http.Request
			err  error
		)

		BeforeEach(func() {
			req, err = query.NewRequestBody(
				"PATCH", "/settings", strings.NewReader(body),
			)
		})

		It("should return request and no errors", func() {
			Expect(req).ToNot(BeNil())
			Expect(err).To(BeNil())
		})

		It("should set method and URL", func() {
			Expect(req.Method).To(Equal("PATCH"))
			Expect(req.URL.String()).To(Equal("https://example.com/api/settings"))
		})

		It("should set request body", func() {
			buf, err := ioutil.ReadAll(req.Body)
			defer req.Body.Close()

			Expect(err).To(BeNil())
			Expect(buf).To(MatchJSON(body))
		})
	})
})
<|endoftext|>


<commit_before>package pbkdf2

import (
	"bytes"
	"crypto/sha1"
	"hash"
	"testing"
)

func tt(t *testing.T, p string, s string, h func()hash.Hash, i int, rl int, cres []byte) {
	pp := []byte(p)
	ss := []byte(s)
	res := WithHMAC(h, pp, ss, i, rl)
	if !bytes.Equal(cres, res) {
		t.Errorf("Invalid PBKDF2: expected %x, got %x", cres, res)
	}
}

func rt(h func()hash.Hash, t *testing.T) {
	tt(t, "foobar", "goo", h, 0, 0, []byte{})
	tt(t, "foobar", "goo", h, 0, 8, []byte{77, 143, 141, 64, 196, 163, 251, 183})
	tt(t, "foobar", "goo", h, 1, 8, []byte{77, 143, 141, 64, 196, 163, 251, 183})
	tt(t, "foobar", "goo", h, 0, 33, []byte{77, 143, 141, 64, 196, 163, 251, 183, 124, 86, 88, 214, 86, 145, 123, 6, 25, 119, 42, 183, 52, 232, 71, 60, 24, 220, 84, 176, 110, 220, 222, 25, 41})
	tt(t, "foobar", "goo", h, 1, 33, []byte{77, 143, 141, 64, 196, 163, 251, 183, 124, 86, 88, 214, 86, 145, 123, 6, 25, 119, 42, 183, 52, 232, 71, 60, 24, 220, 84, 176, 110, 220, 222, 25, 41})
	tt(t, "foobar", "goo", h, 99, 33, []byte{52, 231, 127, 102, 146, 5, 107, 205, 237, 136, 102, 199, 69, 221, 29, 209, 135, 238, 185, 250, 92, 114, 22, 73, 152, 233, 92, 190, 243, 86, 114, 207, 65})
}

func TestPbkdf2(t *testing.T) {
	rt(sha1.New, t)
}
<commit_msg>Use test vectors from RFC6070.<commit_after>package pbkdf2

import (
	"bytes"
	"crypto/sha1"
	"testing"
)

// Test vectors from http://tools.ietf.org/html/rfc6070
var testVectors = []struct {
	password string
	salt     string
	iter     int
	output   []byte
}{
	{
		"password",
		"salt",
		1,
		[]byte{0x0c, 0x60, 0xc8, 0x0f, 0x96, 0x1f, 0x0e, 0x71, 0xf3, 0xa9, 0xb5, 0x24, 0xaf, 0x60, 0x12, 0x06, 0x2f, 0xe0, 0x37, 0xa6},
	},
	{
		"password",
		"salt",
		2,
		[]byte{0xea, 0x6c, 0x01, 0x4d, 0xc7, 0x2d, 0x6f, 0x8c, 0xcd, 0x1e, 0xd9, 0x2a, 0xce, 0x1d, 0x41, 0xf0, 0xd8, 0xde, 0x89, 0x57},
	},
	{
		"password",
		"salt",
		4096,
		[]byte{0x4b, 0x00, 0x79, 0x01, 0xb7, 0x65, 0x48, 0x9a, 0xbe, 0xad, 0x49, 0xd9, 0x26, 0xf7, 0x21, 0xd0, 0x65, 0xa4, 0x29, 0xc1},
	},
	//// Takes too long:
	//	{
	//		"password",
	//		"salt",
	//		16777216,
	//		[]byte{0xee,0xfe,0x3d,0x61,0xcd,0x4d,0xa4,0xe4,0xe9,0x94,0x5b,0x3d,0x6b,0xa2,0x15,0x8c,0x26,0x34,0xe9,0x84},
	//	},
	{
		"passwordPASSWORDpassword",
		"saltSALTsaltSALTsaltSALTsaltSALTsalt",
		4096,
		[]byte{0x3d, 0x2e, 0xec, 0x4f, 0xe4, 0x1c, 0x84, 0x9b, 0x80, 0xc8, 0xd8, 0x36, 0x62, 0xc0, 0xe4, 0x4a, 0x8b, 0x29, 0x1a, 0x96, 0x4c, 0xf2, 0xf0, 0x70, 0x38},
	},
	{
		"pass\000word",
		"sa\000lt",
		4096,
		[]byte{0x56, 0xfa, 0x6a, 0xa7, 0x55, 0x48, 0x09, 0x9d, 0xcc, 0x37, 0xd7, 0xf0, 0x34, 0x25, 0xe0, 0xc3},
	},
}

func TestWithHMAC(t *testing.T) {
	for i, v := range testVectors {
		o := WithHMAC(sha1.New, []byte(v.password), []byte(v.salt), v.iter, len(v.output))
		if !bytes.Equal(o, v.output) {
			t.Errorf("%d: expected %x, got %x", i, v.output, o)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/buildboxhq/buildbox-agent/buildbox"
	"github.com/codegangsta/cli"
	"os"
	"path/filepath"
)

var AppHelpTemplate = `A utility to upload/download artifacts for jobs on Buildbox

Usage:

  {{.Name}} command [arguments]

The commands are:

  {{range .Commands}}{{.Name}}{{with .ShortName}}, {{.}}{{end}}{{ "\t" }}{{.Usage}}
  {{end}}
Use "buildbox-artifact help [command]" for more information about a command.

`

var CommandHelpTemplate = `Usage: buildbox-artifact {{.Name}} [command options] [arguments...]

{{.Description}}

Options:
   {{range .Flags}}{{.}}
   {{end}}
`

var UploadHelpDescription = `Uploads files to a job as artifacts.

You need to ensure that the paths are surrounded by quotes otherwise the
built-in shell path globbing will provide the files, which is currently not
supported.

Example:

buildbox-artifact upload "log/**/*.log" --job [job] \
                                        --agent-access-token [agent-access-token]

You can also upload directy to Amazon S3 if you'd like to host your own artifacts:

export AWS_SECRET_ACCESS_KEY=yyy
export AWS_ACCESS_KEY_ID=xxx
buildbox-artifact upload "log/**/*.log" s3://name-of-your-s3-bucket/$BUILDBOX_JOB_ID --job [job] \
                                                                                --agent-access-token [agent-access-token]`

var DownloadHelpDescription = `Downloads artifacts from Buildbox to the local machine.

You need to ensure that your search query is surrounded by quotes if
using a wild card as the built-in shell path globbing will provide files,
which will break the download.

Example:

buildbox-artifact download "pkg/*.tar.gz" . --build [build] \
                                            --agent-access-token [agent-access-token]

This will search across all the artifacts for the build with files that match that part.
The first argument is the search query, and the second argument is the download destination.

If you're trying to download a specific file, and there are multiple artifacts from different
jobs, you can target the paticular job you want to download the artifact from:

buildbox-artifact download "pkg/*.tar.gz" . --job "tests" \
                                            --build [build] \
                                            --agent-access-token [agent-access-token]

You can also use the job's id (provided by the environment variable $BUILDBOX_JOB_ID)`

var JobIdEnv = "BUILDBOX_JOB_ID"
var JobIdDefault = "$" + JobIdEnv

var BuildIdEnv = "BUILDBOX_BUILD_ID"
var BuildIdDefault = "$" + BuildIdEnv

var AgentAccessTokenEnv = "BUILDBOX_AGENT_ACCESS_TOKEN"
var AgentAccessTokenDefault = "$" + AgentAccessTokenEnv

func main() {
	cli.AppHelpTemplate = AppHelpTemplate
	cli.CommandHelpTemplate = CommandHelpTemplate

	app := cli.NewApp()
	app.Name = "buildbox-artifact"
	app.Version = buildbox.Version

	// Define the actions for our CLI
	app.Commands = []cli.Command{
		{
			Name:        "upload",
			Usage:       "Upload the following artifacts to the build",
			Description: UploadHelpDescription,
			Flags: []cli.Flag{
				cli.StringFlag{"job", JobIdDefault, "Which job should the artifacts be uploaded to"},
				cli.StringFlag{"agent-access-token", AgentAccessTokenDefault, "The access token used to identify the agent"},
				cli.StringFlag{"url", "https://agent.buildbox.io/v2", "The agent API endpoint"},
				cli.BoolFlag{"debug", "Enable debug mode"},
			},
			Action: func(c *cli.Context) {
				// Init debugging
				if c.Bool("debug") {
					buildbox.LoggerInitDebug()
				}

				agentAccessToken := c.String("agent-access-token")

				// Should we look to the environment for the agent access token?
				if agentAccessToken == AgentAccessTokenDefault {
					agentAccessToken = os.Getenv(AgentAccessTokenEnv)
				}

				if agentAccessToken == "" {
					fmt.Printf("%s: missing agent access token\nSee '%s help upload'\n", app.Name, app.Name)
					os.Exit(1)
				}

				jobId := c.String("job")

				// Should we look to the environment for the job id?
				if jobId == JobIdDefault {
					jobId = os.Getenv(JobIdEnv)
				}

				if jobId == "" {
					fmt.Printf("%s: missing job\nSee '%s help upload'\n", app.Name, app.Name)
					os.Exit(1)
				}

				// Grab the first argument and use as paths to download
				paths := c.Args().First()
				if paths == "" {
					fmt.Printf("%s: missing upload paths\nSee '%s help upload'\n", app.Name, app.Name)
					os.Exit(1)
				}

				// Do we have a custom destination
				destination := ""
				if len(c.Args()) > 1 {
					destination = c.Args()[1]
				}

				// Set the agent options
				var agent buildbox.Agent

				// Client specific options
				agent.Client.AuthorizationToken = agentAccessToken
				agent.Client.URL = c.String("url")

				// Setup the agent
				agent.Setup()

				// Find the actual job now
				job, err := agent.Client.JobFind(jobId)
				if err != nil {
					buildbox.Logger.Fatalf("Could not find job: %s", jobId)
				}

				// Create artifact structs for all the files we need to upload
				artifacts, err := buildbox.CollectArtifacts(job, paths)
				if err != nil {
					buildbox.Logger.Fatalf("Failed to collect artifacts: %s", err)
				}

				if len(artifacts) == 0 {
					buildbox.Logger.Infof("No files matched paths: %s", paths)
				} else {
					buildbox.Logger.Infof("Found %d files that match \"%s\"", len(artifacts), paths)

					err := buildbox.UploadArtifacts(agent.Client, job, artifacts, destination)
					if err != nil {
						buildbox.Logger.Fatalf("Failed to upload artifacts: %s", err)
					}
				}
			},
		},
		{
			Name:        "download",
			Usage:       "Download the following artifacts",
			Description: DownloadHelpDescription,
			Flags: []cli.Flag{
				cli.StringFlag{"job", "", "Which job should the artifacts be downloaded from"},
				cli.StringFlag{"build", BuildIdDefault, "Which build should the artifacts be downloaded from"},
				cli.StringFlag{"agent-access-token", AgentAccessTokenDefault, "The access token used to identify the agent"},
				cli.StringFlag{"url", "https://agent.buildbox.io/v2", "The agent API endpoint"},
				cli.BoolFlag{"debug", "Enable debug mode"},
			},
			Action: func(c *cli.Context) {
				// Init debugging
				if c.Bool("debug") {
					buildbox.LoggerInitDebug()
				}

				agentAccessToken := c.String("agent-access-token")

				// Should we look to the environment for the agent access token?
				if agentAccessToken == AgentAccessTokenDefault {
					agentAccessToken = os.Getenv(AgentAccessTokenEnv)
				}

				if agentAccessToken == "" {
					fmt.Printf("%s: missing agent access token\nSee '%s help download'\n", app.Name, app.Name)
					os.Exit(1)
				}

				if len(c.Args()) != 2 {
					fmt.Printf("%s: invalid usage\nSee '%s help download'\n", app.Name, app.Name)
					os.Exit(1)
				}

				// Find the build id
				buildId := c.String("build")
				if buildId == BuildIdDefault {
					buildId = os.Getenv(BuildIdEnv)
				}

				if buildId == "" {
					fmt.Printf("%s: missing build\nSee '%s help download'\n", app.Name, app.Name)
					os.Exit(1)
				}

				// Get our search query and download destination
				searchQuery := c.Args()[0]
				downloadDestination := c.Args()[1]
				jobQuery := c.String("job")

				// Turn the download destination into an absolute path and confirm it exists
				downloadDestination, _ = filepath.Abs(downloadDestination)
				fileInfo, err := os.Stat(downloadDestination)
				if err != nil {
					buildbox.Logger.Fatalf("Could not find information about destination: %s", downloadDestination)
				}
				if !fileInfo.IsDir() {
					buildbox.Logger.Fatalf("%s is not a directory", downloadDestination)
				}

				// Set the agent options
				var agent buildbox.Agent

				// Client specific options
				agent.Client.AgentAccessToken = agentAccessToken
				agent.Client.URL = c.String("url")

				// Setup the agent
				agent.Setup()

				if jobQuery == "" {
					buildbox.Logger.Infof("Searching for artifacts: \"%s\"", searchQuery)
				} else {
					buildbox.Logger.Infof("Searching for artifacts: \"%s\" within job: \"%s\"", searchQuery, jobQuery)
				}

				// Search for artifacts (only those that have finished uploaded) to download
				artifacts, err := agent.Client.SearchArtifacts(buildId, searchQuery, jobQuery, "finished")
				if err != nil {
					buildbox.Logger.Fatalf("Failed to find artifacts: %s", err)
				}

				if len(artifacts) == 0 {
					buildbox.Logger.Info("No artifacts found for downloading")
				} else {
					buildbox.Logger.Infof("Found %d artifacts. Starting to download to: %s", len(artifacts), downloadDestination)

					err := buildbox.DownloadArtifacts(artifacts, downloadDestination)
					if err != nil {
						buildbox.Logger.Fatalf("Failed to download artifacts: %s", err)
					}
				}
			},
		},
	}

	// Default the default action
	app.Action = func(c *cli.Context) {
		cli.ShowAppHelp(c)
		os.Exit(1)
	}

	app.Run(os.Args)
}
<commit_msg>Fixed a reference to the old AgentAccessToken property on client.<commit_after>package main

import (
	"fmt"
	"github.com/buildboxhq/buildbox-agent/buildbox"
	"github.com/codegangsta/cli"
	"os"
	"path/filepath"
)

var AppHelpTemplate = `A utility to upload/download artifacts for jobs on Buildbox

Usage:

  {{.Name}} command [arguments]

The commands are:

  {{range .Commands}}{{.Name}}{{with .ShortName}}, {{.}}{{end}}{{ "\t" }}{{.Usage}}
  {{end}}
Use "buildbox-artifact help [command]" for more information about a command.

`

var CommandHelpTemplate = `Usage: buildbox-artifact {{.Name}} [command options] [arguments...]

{{.Description}}

Options:
   {{range .Flags}}{{.}}
   {{end}}
`

var UploadHelpDescription = `Uploads files to a job as artifacts.

You need to ensure that the paths are surrounded by quotes otherwise the
built-in shell path globbing will provide the files, which is currently not
supported.

Example:

buildbox-artifact upload "log/**/*.log" --job [job] \
                                        --agent-access-token [agent-access-token]

You can also upload directy to Amazon S3 if you'd like to host your own artifacts:

export AWS_SECRET_ACCESS_KEY=yyy
export AWS_ACCESS_KEY_ID=xxx
buildbox-artifact upload "log/**/*.log" s3://name-of-your-s3-bucket/$BUILDBOX_JOB_ID --job [job] \
                                                                                --agent-access-token [agent-access-token]`

var DownloadHelpDescription = `Downloads artifacts from Buildbox to the local machine.

You need to ensure that your search query is surrounded by quotes if
using a wild card as the built-in shell path globbing will provide files,
which will break the download.

Example:

buildbox-artifact download "pkg/*.tar.gz" . --build [build] \
                                            --agent-access-token [agent-access-token]

This will search across all the artifacts for the build with files that match that part.
The first argument is the search query, and the second argument is the download destination.

If you're trying to download a specific file, and there are multiple artifacts from different
jobs, you can target the paticular job you want to download the artifact from:

buildbox-artifact download "pkg/*.tar.gz" . --job "tests" \
                                            --build [build] \
                                            --agent-access-token [agent-access-token]

You can also use the job's id (provided by the environment variable $BUILDBOX_JOB_ID)`

var JobIdEnv = "BUILDBOX_JOB_ID"
var JobIdDefault = "$" + JobIdEnv

var BuildIdEnv = "BUILDBOX_BUILD_ID"
var BuildIdDefault = "$" + BuildIdEnv

var AgentAccessTokenEnv = "BUILDBOX_AGENT_ACCESS_TOKEN"
var AgentAccessTokenDefault = "$" + AgentAccessTokenEnv

func main() {
	cli.AppHelpTemplate = AppHelpTemplate
	cli.CommandHelpTemplate = CommandHelpTemplate

	app := cli.NewApp()
	app.Name = "buildbox-artifact"
	app.Version = buildbox.Version

	// Define the actions for our CLI
	app.Commands = []cli.Command{
		{
			Name:        "upload",
			Usage:       "Upload the following artifacts to the build",
			Description: UploadHelpDescription,
			Flags: []cli.Flag{
				cli.StringFlag{"job", JobIdDefault, "Which job should the artifacts be uploaded to"},
				cli.StringFlag{"agent-access-token", AgentAccessTokenDefault, "The access token used to identify the agent"},
				cli.StringFlag{"url", "https://agent.buildbox.io/v2", "The agent API endpoint"},
				cli.BoolFlag{"debug", "Enable debug mode"},
			},
			Action: func(c *cli.Context) {
				// Init debugging
				if c.Bool("debug") {
					buildbox.LoggerInitDebug()
				}

				agentAccessToken := c.String("agent-access-token")

				// Should we look to the environment for the agent access token?
				if agentAccessToken == AgentAccessTokenDefault {
					agentAccessToken = os.Getenv(AgentAccessTokenEnv)
				}

				if agentAccessToken == "" {
					fmt.Printf("%s: missing agent access token\nSee '%s help upload'\n", app.Name, app.Name)
					os.Exit(1)
				}

				jobId := c.String("job")

				// Should we look to the environment for the job id?
				if jobId == JobIdDefault {
					jobId = os.Getenv(JobIdEnv)
				}

				if jobId == "" {
					fmt.Printf("%s: missing job\nSee '%s help upload'\n", app.Name, app.Name)
					os.Exit(1)
				}

				// Grab the first argument and use as paths to download
				paths := c.Args().First()
				if paths == "" {
					fmt.Printf("%s: missing upload paths\nSee '%s help upload'\n", app.Name, app.Name)
					os.Exit(1)
				}

				// Do we have a custom destination
				destination := ""
				if len(c.Args()) > 1 {
					destination = c.Args()[1]
				}

				// Set the agent options
				var agent buildbox.Agent

				// Client specific options
				agent.Client.AuthorizationToken = agentAccessToken
				agent.Client.URL = c.String("url")

				// Setup the agent
				agent.Setup()

				// Find the actual job now
				job, err := agent.Client.JobFind(jobId)
				if err != nil {
					buildbox.Logger.Fatalf("Could not find job: %s", jobId)
				}

				// Create artifact structs for all the files we need to upload
				artifacts, err := buildbox.CollectArtifacts(job, paths)
				if err != nil {
					buildbox.Logger.Fatalf("Failed to collect artifacts: %s", err)
				}

				if len(artifacts) == 0 {
					buildbox.Logger.Infof("No files matched paths: %s", paths)
				} else {
					buildbox.Logger.Infof("Found %d files that match \"%s\"", len(artifacts), paths)

					err := buildbox.UploadArtifacts(agent.Client, job, artifacts, destination)
					if err != nil {
						buildbox.Logger.Fatalf("Failed to upload artifacts: %s", err)
					}
				}
			},
		},
		{
			Name:        "download",
			Usage:       "Download the following artifacts",
			Description: DownloadHelpDescription,
			Flags: []cli.Flag{
				cli.StringFlag{"job", "", "Which job should the artifacts be downloaded from"},
				cli.StringFlag{"build", BuildIdDefault, "Which build should the artifacts be downloaded from"},
				cli.StringFlag{"agent-access-token", AgentAccessTokenDefault, "The access token used to identify the agent"},
				cli.StringFlag{"url", "https://agent.buildbox.io/v2", "The agent API endpoint"},
				cli.BoolFlag{"debug", "Enable debug mode"},
			},
			Action: func(c *cli.Context) {
				// Init debugging
				if c.Bool("debug") {
					buildbox.LoggerInitDebug()
				}

				agentAccessToken := c.String("agent-access-token")

				// Should we look to the environment for the agent access token?
				if agentAccessToken == AgentAccessTokenDefault {
					agentAccessToken = os.Getenv(AgentAccessTokenEnv)
				}

				if agentAccessToken == "" {
					fmt.Printf("%s: missing agent access token\nSee '%s help download'\n", app.Name, app.Name)
					os.Exit(1)
				}

				if len(c.Args()) != 2 {
					fmt.Printf("%s: invalid usage\nSee '%s help download'\n", app.Name, app.Name)
					os.Exit(1)
				}

				// Find the build id
				buildId := c.String("build")
				if buildId == BuildIdDefault {
					buildId = os.Getenv(BuildIdEnv)
				}

				if buildId == "" {
					fmt.Printf("%s: missing build\nSee '%s help download'\n", app.Name, app.Name)
					os.Exit(1)
				}

				// Get our search query and download destination
				searchQuery := c.Args()[0]
				downloadDestination := c.Args()[1]
				jobQuery := c.String("job")

				// Turn the download destination into an absolute path and confirm it exists
				downloadDestination, _ = filepath.Abs(downloadDestination)
				fileInfo, err := os.Stat(downloadDestination)
				if err != nil {
					buildbox.Logger.Fatalf("Could not find information about destination: %s", downloadDestination)
				}
				if !fileInfo.IsDir() {
					buildbox.Logger.Fatalf("%s is not a directory", downloadDestination)
				}

				// Set the agent options
				var agent buildbox.Agent

				// Client specific options
				agent.Client.AuthorizationToken = agentAccessToken
				agent.Client.URL = c.String("url")

				// Setup the agent
				agent.Setup()

				if jobQuery == "" {
					buildbox.Logger.Infof("Searching for artifacts: \"%s\"", searchQuery)
				} else {
					buildbox.Logger.Infof("Searching for artifacts: \"%s\" within job: \"%s\"", searchQuery, jobQuery)
				}

				// Search for artifacts (only those that have finished uploaded) to download
				artifacts, err := agent.Client.SearchArtifacts(buildId, searchQuery, jobQuery, "finished")
				if err != nil {
					buildbox.Logger.Fatalf("Failed to find artifacts: %s", err)
				}

				if len(artifacts) == 0 {
					buildbox.Logger.Info("No artifacts found for downloading")
				} else {
					buildbox.Logger.Infof("Found %d artifacts. Starting to download to: %s", len(artifacts), downloadDestination)

					err := buildbox.DownloadArtifacts(artifacts, downloadDestination)
					if err != nil {
						buildbox.Logger.Fatalf("Failed to download artifacts: %s", err)
					}
				}
			},
		},
	}

	// Default the default action
	app.Action = func(c *cli.Context) {
		cli.ShowAppHelp(c)
		os.Exit(1)
	}

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	log "github.com/Sirupsen/logrus"
	chttpd "github.com/nochso/colourl/http"
	"net/http"
)

var port int
var (
	Version   string
	BuildDate string
)

func init() {
	flag.IntVar(&port, "port", 9191, "HTTP listening port")
	flag.IntVar(&port, "p", 9191, "HTTP listening port (shorthand)")
}

func main() {
	flag.Parse()
	log.WithFields(log.Fields{
		"version":    Version,
		"build_date": BuildDate,
	}).Info("colourl-http")
	log.WithFields(log.Fields{
		"port":    port,
		"verbose": verbose,
	}).Info("Starting HTTP server")
	http.HandleFunc("/", chttpd.IndexHandler)
	http.HandleFunc("/svg", chttpd.SVGHandler)
	http.ListenAndServe(fmt.Sprintf(":%d", port), nil)
}
<commit_msg>Add verbose flag to colourl-http<commit_after>package main

import (
	"flag"
	"fmt"
	log "github.com/Sirupsen/logrus"
	chttpd "github.com/nochso/colourl/http"
	"net/http"
)

var (
	port    int
	verbose bool
)

var (
	Version   string
	BuildDate string
)

func init() {
	flag.IntVar(&port, "p", 9191, "HTTP listening port")
	flag.BoolVar(&verbose, "v", false, "Enable verbose / debug output")
}

func main() {
	flag.Parse()
	if verbose {
		log.SetLevel(log.DebugLevel)
	}
	log.WithFields(log.Fields{
		"version":    Version,
		"build_date": BuildDate,
	}).Info("colourl-http")
	log.WithFields(log.Fields{
		"port":    port,
		"verbose": verbose,
	}).Info("Starting HTTP server")
	http.HandleFunc("/", chttpd.IndexHandler)
	http.HandleFunc("/svg", chttpd.SVGHandler)
	http.ListenAndServe(fmt.Sprintf(":%d", port), nil)
}
<|endoftext|>