<commit_before>package main

// Glen Newton
// glen.newton@gmail.com

import (
	"flag"
	"fmt"
	"log"
	"math/rand"
	"os"
	"os/exec"
	"runtime"
	"sort"
	"strings"
	"time"
)

func init() {
	flag.StringVar(&outputFormat, "outputFormat", "txt", "Values: json|txt|latex|exhibit")
	flag.StringVar(&header, "header", "Installed Software", "gggg")
	flag.BoolVar(&doR, "R", false, "Find R packages")
	flag.StringVar(&outputLocation, "o", "rpmoutOut", "Base path and name for output file(s); only used by outputFormat=exhibit; all other outputs are to stdout")
}

var outputLocation string = ""
var outputFormat string
var header string
var doR = false

type PackageInfo struct {
	Name      string
	IndexName string
	IsR       bool
	Tags      map[string]string
}

type RpmWriter interface {
	output(string, string, string, []string, []string, map[string]*PackageInfo, map[string]bool, map[string]*Node) error
}

func handleParameters() bool {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "%s: Output rpm info for rpms that have install components in certain directories\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "\t %s <args> <rootDir0>...<rootDirN>\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "\t default <rootDir>: /\n")
		fmt.Fprintf(os.Stderr, "Args:\n")
		flag.PrintDefaults()
		fmt.Fprintf(os.Stderr, "\nExample:  %s -outputFormat=json /opt /usr/local\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "\nNote that the 'rpm' program (http://www.rpm.org/max-rpm/rpm.8.html) needs to be in your path\n\n")
	}

	flag.Parse()
	switch outputFormat {
	case "json":
		rpmWriter = new(JsonOut)

	case "txt":
		rpmWriter = new(TextOut)

	case "latex":
		rpmWriter = new(LaTeXOut)

	case "exhibit":
		exhibitOut := new(ExhibitOut)
		exhibitOut.OutputDir = outputLocation
		rpmWriter = exhibitOut

	default:
		log.Print("Unknown/unsupported output format: " + outputFormat)
		return false
	}
	return true
}

var allRpmsExec = []string{"rpm", "-qa"}

var numFindOfInterestRpmsWorkers = 6
var numFindRpmTagsWorkers = 2

var rpmWriter RpmWriter

func main() {

	runtime.GOMAXPROCS(8)

	_, err := exec.LookPath(allRpmsExec[0])

	if err != nil {
		log.Fatal("'", allRpmsExec[0], "' is not in your path: needed to run; it is usually in '/usr/bin/rpm'")
	}

	if !handleParameters() {
		log.Fatal("Bad parameters")
	}

	rand.Seed(time.Now().Unix())

	var dirsOfInterest []string
	if len(flag.Args()) == 0 {
		dirsOfInterest = []string{"/"}
	} else {
		dirsOfInterest = flag.Args()
	}

	//fmt.Println(dirsOfInterest)

	rpmListChannel, err := makeRpmList()
	if err != nil {
		log.Fatal("jjjj ", err)
	}

	ofInterestChannel, ofInterestDoneChannel, err := findOfInterestRpms(dirsOfInterest, rpmListChannel)

	tagInfoChannel, tagInfoDoneChannel, err := findRpmTags(ofInterestChannel)

	packageInfoMap := make(map[string]*PackageInfo, 200)
	//packageInfoMap2 := make(map[string] PackageInfo, 200)
	groupSet := make(map[string]bool)

	var nodes map[string]*Node
	nodes = make(map[string]*Node)

	addResultsDoneChannel := addResultsToMap(packageInfoMap, groupSet, tagInfoChannel, nodes)

	for i := 0; i < numFindOfInterestRpmsWorkers; i++ {
		_ = <-ofInterestDoneChannel
	}
	close(ofInterestChannel)

	for i := 0; i < numFindRpmTagsWorkers; i++ {
		_ = <-tagInfoDoneChannel
	}
	close(tagInfoChannel)

	_ = <-addResultsDoneChannel

	var rpackageInfoMap map[string]*PackageInfo = nil
	if doR {
		rpackageInfoMap = findRPackages()
	}

	add(packageInfoMap, rpackageInfoMap)
	rpmWriter.output(outputLocation, "", header, dirsOfInterest, sortStringKeyMap(packageInfoMap), packageInfoMap, groupSet, nodes)
}

func xlog(m string) {
	fmt.Println(m)
}

func makeRpmList() (chan *PackageInfo, error) {
	rpmListChannel := make(chan *PackageInfo, 200)

	stringListChannel := runExec(allRpmsExec)

	go func() {
		for stringInfo := range stringListChannel {
			//fmt.Println("makeRpmList: [",  stringInfo.val, "] ", &stringInfo.val," ", stringInfo.done)
			packageInfo := new(PackageInfo)
			packageInfo.Name = stringInfo.val
			packageInfo.Tags = make(map[string]string)
			rpmListChannel <- packageInfo
		}
		close(rpmListChannel)
	}()
	return rpmListChannel, nil
}

func findOfInterestRpms(dirsOfInterest []string, rpmListChannel chan *PackageInfo) (chan *PackageInfo, chan bool, error) {
	ofInterestChannel := make(chan *PackageInfo, 100)
	doneChannel := make(chan bool, numFindOfInterestRpmsWorkers)

	for i := 0; i < numFindOfInterestRpmsWorkers; i++ {
		go func() {
			for packageInfo := range rpmListChannel {
				//fmt.Println("findOfInterestRpms ", packageInfo.Name)
				cmd := exec.Command(allRpmsExec[0], "-ql", packageInfo.Name)
				out, err := cmd.Output()

				if err != nil {
					log.Fatal("jjjj ", err)
				}

				lines := strings.Split(string(out), "\n")
				done := false
				for _, line := range lines {
					if done {
						break
					}
					for _, dir := range dirsOfInterest {
						if strings.Index(line, dir) == 0 {
							ofInterestChannel <- packageInfo
							done = true
						}
					}
				}
			}
			doneChannel <- true
		}()
	}
	return ofInterestChannel, doneChannel, nil
}

func findRpmTags(ofInterestChannel chan *PackageInfo) (chan *PackageInfo, chan bool, error) {
	tagInfoChannel := make(chan *PackageInfo, 200)
	doneChannel := make(chan bool, numFindRpmTagsWorkers)

	for i := 0; i < numFindRpmTagsWorkers; i++ {
		go func() {
			numTags := random(100, 400)
			tagBuffer := make([]*PackageInfo, numTags)
			count := 0
			for packageInfo := range ofInterestChannel {
				//fmt.Println("\t findRpmTags ", packageInfo.name)
				//fmt.Println("--",count," ", packageInfo.name)
				if count >= numTags {
					out := runCommand(tagBuffer, count)
					parseAndSend(count, tagBuffer, out, tagInfoChannel)
					//fmt.Println("+++ ", packageInfo.name, " --- ", out)
					count = 0
					tagInfoChannel <- packageInfo

				}
				tagBuffer[count] = packageInfo
				count = count + 1
			}
			if count > 0 {
				out := runCommand(tagBuffer, count)
				parseAndSend(count, tagBuffer, out, tagInfoChannel)
				//_ = runCommand(tagBuffer, count)
				//fmt.Println("+++ ", " --- ", out)
			}
			doneChannel <- true
		}()
	}
	return tagInfoChannel, doneChannel, nil
}

const tagSeparator = "--==--"

const recordSeparator = "|||"

var rpmFormat = "name:%{NAME}" + tagSeparator + "os:%{OS}" + tagSeparator + "version:%{VERSION}" + tagSeparator + "release:%{RELEASE}" + tagSeparator + "arch:%{ARCH}" + tagSeparator + "installtime:%{INSTALLTIME:date}" + tagSeparator + "group:%{GROUP}" + tagSeparator + "size:%{SIZE}" + tagSeparator + "license:%{LICENSE}" + tagSeparator + "sourcerpm:%{SOURCERPM}" + tagSeparator + "buildtime:%{BUILDTIME}" + tagSeparator + "buildhost:%{BUILDHOST}" + tagSeparator + "packager:%{PACKAGER}" + tagSeparator + "vendor:%{VENDOR}" + tagSeparator + "url:%{URL}" + tagSeparator + "summary:%{SUMMARY}" + tagSeparator + "description:%{DESCRIPTION}" + tagSeparator + "" + tagSeparator + "distribution:%{DISTRIBUTION}" + tagSeparator + "packager:%{PACKAGER}" + tagSeparator + "patch:%{PATCH}" + recordSeparator

func parseAndSend(count int, tagInfoBuffer []*PackageInfo, out string, tagInfoChannel chan *PackageInfo) {
	records := strings.Split(out, recordSeparator)
	if len(records) != count+1 {
		log.Fatal("jiijjj ")
	}
	for i := 0; i < count; i++ {
		tags := strings.Split(records[i], tagSeparator)
		for j := 0; j < len(tags); j++ {
			//fmt.Println("\t", j, " ", tags[j])
			parts := strings.SplitN(tags[j], ":", 2)
			if len(parts) == 2 {
				parts[0] = strings.Trim(parts[0], " ")
				parts[1] = strings.Trim(parts[1], " ")
				if len(parts[1]) > 0 && parts[1] != "(none)" {
					tagInfoBuffer[i].Tags[parts[0]] = parts[1]
				}
			}
		}
		tagInfoChannel <- tagInfoBuffer[i]
	}
}

func runCommand(tagBuffer []*PackageInfo, count int) string {
	args := []string{}
	args = append(args, "--qf")
	args = append(args, rpmFormat)
	args = append(args, "-q")
	for j := 0; j < count; j++ {
		args = append(args, tagBuffer[j].Name)
	}
	cmd := exec.Command("rpm", args...)
	out, err := cmd.Output()
	if err != nil {
		log.Fatal("help", err)
	}
	return string(out)
}

func makeArgs(cmd *exec.Cmd, tagBuffer []*PackageInfo, count int) {
	cmd.Args = make([]string, 3+count)
	cmd.Args[0] = "--qf"
	cmd.Args[1] = rpmFormat
	cmd.Args[2] = "-q"
	for i := 0; i < count; i++ {
		cmd.Args[3+i] = tagBuffer[i].Name
	}
}

func addResultsToMap(rpmMap map[string]*PackageInfo, groupSet map[string]bool, tagInfoChannel chan *PackageInfo, nodes map[string]*Node) chan bool {
	doneChannel := make(chan bool)
	go func() {
		for packageInfo := range tagInfoChannel {
			rpmMap[strings.ToLower(packageInfo.Name)] = packageInfo
			groupSet[packageInfo.Tags["group"]] = true
			//fmt.Println("Adding to map ", packageInfo.Name)
			extractHierarchy(packageInfo, nodes)
		}
		doneChannel <- true
	}()
	return doneChannel
}

func random(min, max int) int {
	return rand.Intn(max-min) + min
}

func sortStringKeyMap2(m map[string]*struct{}) []string {
	sm := make([]string, len(m))
	return sm
}

//func sortStringKeyMap(m map[string]struct{}) []string{
func sortStringKeyMap(m map[string]*PackageInfo) []string {
	sm := make([]string, len(m))
	i := 0
	for k, _ := range m {
		sm[i] = k
		i++
	}
	sort.Strings(sm)
	return sm
}

func sortLicenseCountMap(licenseCount map[string]int) []string {
	sm := make([]string, len(licenseCount))
	i := 0
	for k, _ := range licenseCount {
		sm[i] = k
		i++
	}
	sort.Strings(sm)
	return sm
}

type Pair struct {
	Key   string
	Value int
}

// From: https://groups.google.com/forum/#!topic/golang-nuts/FT7cjmcL7gw
type PairList []Pair

func (p PairList) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
func (p PairList) Len() int           { return len(p) }
func (p PairList) Less(i, j int) bool { return p[i].Value < p[j].Value }

// A function to turn a map into a PairList, then sort and return it.
func sortMapByValue(m map[string]int) PairList {
	pp := make(PairList, len(m))
	i := 0
	for k, v := range m {
		pp[i] = Pair{k, v}
	}
	sort.Sort(pp)
	return pp
}

func sortStringKeyNodeMap(m map[string]*Node) []string {
	sm := make([]string, len(m))
	i := 0
	for k, _ := range m {
		sm[i] = k
		i++
	}
	sort.Strings(sm)
	return sm
}

// Add the R packages to the RPM packages
func add(rpms map[string]*PackageInfo, rpacks map[string]*PackageInfo) {
	for k, v := range rpacks {
		rpms[k] = v
	}
}
<commit_msg>Change runtime.GOMAXPROCS(8) to runtime.GOMAXPROCS(runtime.NumCPU()) in rpmout,go<commit_after>package main

// Glen Newton
// glen.newton@gmail.com

import (
	"flag"
	"fmt"
	"log"
	"math/rand"
	"os"
	"os/exec"
	"runtime"
	"sort"
	"strings"
	"time"
)

func init() {
	flag.StringVar(&outputFormat, "outputFormat", "txt", "Values: json|txt|latex|exhibit")
	flag.StringVar(&header, "header", "Installed Software", "gggg")
	flag.BoolVar(&doR, "R", false, "Find R packages")
	flag.StringVar(&outputLocation, "o", "rpmoutOut", "Base path and name for output file(s); only used by outputFormat=exhibit; all other outputs are to stdout")
}

var outputLocation string = ""
var outputFormat string
var header string
var doR = false

type PackageInfo struct {
	Name      string
	IndexName string
	IsR       bool
	Tags      map[string]string
}

type RpmWriter interface {
	output(string, string, string, []string, []string, map[string]*PackageInfo, map[string]bool, map[string]*Node) error
}

func handleParameters() bool {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "%s: Output rpm info for rpms that have install components in certain directories\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "\t %s <args> <rootDir0>...<rootDirN>\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "\t default <rootDir>: /\n")
		fmt.Fprintf(os.Stderr, "Args:\n")
		flag.PrintDefaults()
		fmt.Fprintf(os.Stderr, "\nExample:  %s -outputFormat=json /opt /usr/local\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "\nNote that the 'rpm' program (http://www.rpm.org/max-rpm/rpm.8.html) needs to be in your path\n\n")
	}

	flag.Parse()
	switch outputFormat {
	case "json":
		rpmWriter = new(JsonOut)

	case "txt":
		rpmWriter = new(TextOut)

	case "latex":
		rpmWriter = new(LaTeXOut)

	case "exhibit":
		exhibitOut := new(ExhibitOut)
		exhibitOut.OutputDir = outputLocation
		rpmWriter = exhibitOut

	default:
		log.Print("Unknown/unsupported output format: " + outputFormat)
		return false
	}
	return true
}

var allRpmsExec = []string{"rpm", "-qa"}

var numFindOfInterestRpmsWorkers = 6
var numFindRpmTagsWorkers = 2

var rpmWriter RpmWriter

func main() {

	runtime.GOMAXPROCS(runtime.NumCPU())

	_, err := exec.LookPath(allRpmsExec[0])

	if err != nil {
		log.Fatal("'", allRpmsExec[0], "' is not in your path: needed to run; it is usually in '/usr/bin/rpm'")
	}

	if !handleParameters() {
		log.Fatal("Bad parameters")
	}

	rand.Seed(time.Now().Unix())

	var dirsOfInterest []string
	if len(flag.Args()) == 0 {
		dirsOfInterest = []string{"/"}
	} else {
		dirsOfInterest = flag.Args()
	}

	//fmt.Println(dirsOfInterest)

	rpmListChannel, err := makeRpmList()
	if err != nil {
		log.Fatal("jjjj ", err)
	}

	ofInterestChannel, ofInterestDoneChannel, err := findOfInterestRpms(dirsOfInterest, rpmListChannel)

	tagInfoChannel, tagInfoDoneChannel, err := findRpmTags(ofInterestChannel)

	packageInfoMap := make(map[string]*PackageInfo, 200)
	//packageInfoMap2 := make(map[string] PackageInfo, 200)
	groupSet := make(map[string]bool)

	var nodes map[string]*Node
	nodes = make(map[string]*Node)

	addResultsDoneChannel := addResultsToMap(packageInfoMap, groupSet, tagInfoChannel, nodes)

	for i := 0; i < numFindOfInterestRpmsWorkers; i++ {
		_ = <-ofInterestDoneChannel
	}
	close(ofInterestChannel)

	for i := 0; i < numFindRpmTagsWorkers; i++ {
		_ = <-tagInfoDoneChannel
	}
	close(tagInfoChannel)

	_ = <-addResultsDoneChannel

	var rpackageInfoMap map[string]*PackageInfo = nil
	if doR {
		rpackageInfoMap = findRPackages()
	}

	add(packageInfoMap, rpackageInfoMap)
	rpmWriter.output(outputLocation, "", header, dirsOfInterest, sortStringKeyMap(packageInfoMap), packageInfoMap, groupSet, nodes)
}

func xlog(m string) {
	fmt.Println(m)
}

func makeRpmList() (chan *PackageInfo, error) {
	rpmListChannel := make(chan *PackageInfo, 200)

	stringListChannel := runExec(allRpmsExec)

	go func() {
		for stringInfo := range stringListChannel {
			//fmt.Println("makeRpmList: [",  stringInfo.val, "] ", &stringInfo.val," ", stringInfo.done)
			packageInfo := new(PackageInfo)
			packageInfo.Name = stringInfo.val
			packageInfo.Tags = make(map[string]string)
			rpmListChannel <- packageInfo
		}
		close(rpmListChannel)
	}()
	return rpmListChannel, nil
}

func findOfInterestRpms(dirsOfInterest []string, rpmListChannel chan *PackageInfo) (chan *PackageInfo, chan bool, error) {
	ofInterestChannel := make(chan *PackageInfo, 100)
	doneChannel := make(chan bool, numFindOfInterestRpmsWorkers)

	for i := 0; i < numFindOfInterestRpmsWorkers; i++ {
		go func() {
			for packageInfo := range rpmListChannel {
				//fmt.Println("findOfInterestRpms ", packageInfo.Name)
				cmd := exec.Command(allRpmsExec[0], "-ql", packageInfo.Name)
				out, err := cmd.Output()

				if err != nil {
					log.Fatal("jjjj ", err)
				}

				lines := strings.Split(string(out), "\n")
				done := false
				for _, line := range lines {
					if done {
						break
					}
					for _, dir := range dirsOfInterest {
						if strings.Index(line, dir) == 0 {
							ofInterestChannel <- packageInfo
							done = true
						}
					}
				}
			}
			doneChannel <- true
		}()
	}
	return ofInterestChannel, doneChannel, nil
}

func findRpmTags(ofInterestChannel chan *PackageInfo) (chan *PackageInfo, chan bool, error) {
	tagInfoChannel := make(chan *PackageInfo, 200)
	doneChannel := make(chan bool, numFindRpmTagsWorkers)

	for i := 0; i < numFindRpmTagsWorkers; i++ {
		go func() {
			numTags := random(100, 400)
			tagBuffer := make([]*PackageInfo, numTags)
			count := 0
			for packageInfo := range ofInterestChannel {
				//fmt.Println("\t findRpmTags ", packageInfo.name)
				//fmt.Println("--",count," ", packageInfo.name)
				if count >= numTags {
					out := runCommand(tagBuffer, count)
					parseAndSend(count, tagBuffer, out, tagInfoChannel)
					//fmt.Println("+++ ", packageInfo.name, " --- ", out)
					count = 0
					tagInfoChannel <- packageInfo

				}
				tagBuffer[count] = packageInfo
				count = count + 1
			}
			if count > 0 {
				out := runCommand(tagBuffer, count)
				parseAndSend(count, tagBuffer, out, tagInfoChannel)
				//_ = runCommand(tagBuffer, count)
				//fmt.Println("+++ ", " --- ", out)
			}
			doneChannel <- true
		}()
	}
	return tagInfoChannel, doneChannel, nil
}

const tagSeparator = "--==--"

const recordSeparator = "|||"

var rpmFormat = "name:%{NAME}" + tagSeparator + "os:%{OS}" + tagSeparator + "version:%{VERSION}" + tagSeparator + "release:%{RELEASE}" + tagSeparator + "arch:%{ARCH}" + tagSeparator + "installtime:%{INSTALLTIME:date}" + tagSeparator + "group:%{GROUP}" + tagSeparator + "size:%{SIZE}" + tagSeparator + "license:%{LICENSE}" + tagSeparator + "sourcerpm:%{SOURCERPM}" + tagSeparator + "buildtime:%{BUILDTIME}" + tagSeparator + "buildhost:%{BUILDHOST}" + tagSeparator + "packager:%{PACKAGER}" + tagSeparator + "vendor:%{VENDOR}" + tagSeparator + "url:%{URL}" + tagSeparator + "summary:%{SUMMARY}" + tagSeparator + "description:%{DESCRIPTION}" + tagSeparator + "" + tagSeparator + "distribution:%{DISTRIBUTION}" + tagSeparator + "packager:%{PACKAGER}" + tagSeparator + "patch:%{PATCH}" + recordSeparator

func parseAndSend(count int, tagInfoBuffer []*PackageInfo, out string, tagInfoChannel chan *PackageInfo) {
	records := strings.Split(out, recordSeparator)
	if len(records) != count+1 {
		log.Fatal("jiijjj ")
	}
	for i := 0; i < count; i++ {
		tags := strings.Split(records[i], tagSeparator)
		for j := 0; j < len(tags); j++ {
			//fmt.Println("\t", j, " ", tags[j])
			parts := strings.SplitN(tags[j], ":", 2)
			if len(parts) == 2 {
				parts[0] = strings.Trim(parts[0], " ")
				parts[1] = strings.Trim(parts[1], " ")
				if len(parts[1]) > 0 && parts[1] != "(none)" {
					tagInfoBuffer[i].Tags[parts[0]] = parts[1]
				}
			}
		}
		tagInfoChannel <- tagInfoBuffer[i]
	}
}

func runCommand(tagBuffer []*PackageInfo, count int) string {
	args := []string{}
	args = append(args, "--qf")
	args = append(args, rpmFormat)
	args = append(args, "-q")
	for j := 0; j < count; j++ {
		args = append(args, tagBuffer[j].Name)
	}
	cmd := exec.Command("rpm", args...)
	out, err := cmd.Output()
	if err != nil {
		log.Fatal("help", err)
	}
	return string(out)
}

func makeArgs(cmd *exec.Cmd, tagBuffer []*PackageInfo, count int) {
	cmd.Args = make([]string, 3+count)
	cmd.Args[0] = "--qf"
	cmd.Args[1] = rpmFormat
	cmd.Args[2] = "-q"
	for i := 0; i < count; i++ {
		cmd.Args[3+i] = tagBuffer[i].Name
	}
}

func addResultsToMap(rpmMap map[string]*PackageInfo, groupSet map[string]bool, tagInfoChannel chan *PackageInfo, nodes map[string]*Node) chan bool {
	doneChannel := make(chan bool)
	go func() {
		for packageInfo := range tagInfoChannel {
			rpmMap[strings.ToLower(packageInfo.Name)] = packageInfo
			groupSet[packageInfo.Tags["group"]] = true
			//fmt.Println("Adding to map ", packageInfo.Name)
			extractHierarchy(packageInfo, nodes)
		}
		doneChannel <- true
	}()
	return doneChannel
}

func random(min, max int) int {
	return rand.Intn(max-min) + min
}

func sortStringKeyMap2(m map[string]*struct{}) []string {
	sm := make([]string, len(m))
	return sm
}

//func sortStringKeyMap(m map[string]struct{}) []string{
func sortStringKeyMap(m map[string]*PackageInfo) []string {
	sm := make([]string, len(m))
	i := 0
	for k, _ := range m {
		sm[i] = k
		i++
	}
	sort.Strings(sm)
	return sm
}

func sortLicenseCountMap(licenseCount map[string]int) []string {
	sm := make([]string, len(licenseCount))
	i := 0
	for k, _ := range licenseCount {
		sm[i] = k
		i++
	}
	sort.Strings(sm)
	return sm
}

type Pair struct {
	Key   string
	Value int
}

// From: https://groups.google.com/forum/#!topic/golang-nuts/FT7cjmcL7gw
type PairList []Pair

func (p PairList) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
func (p PairList) Len() int           { return len(p) }
func (p PairList) Less(i, j int) bool { return p[i].Value < p[j].Value }

// A function to turn a map into a PairList, then sort and return it.
func sortMapByValue(m map[string]int) PairList {
	pp := make(PairList, len(m))
	i := 0
	for k, v := range m {
		pp[i] = Pair{k, v}
	}
	sort.Sort(pp)
	return pp
}

func sortStringKeyNodeMap(m map[string]*Node) []string {
	sm := make([]string, len(m))
	i := 0
	for k, _ := range m {
		sm[i] = k
		i++
	}
	sort.Strings(sm)
	return sm
}

// Add the R packages to the RPM packages
func add(rpms map[string]*PackageInfo, rpacks map[string]*PackageInfo) {
	for k, v := range rpacks {
		rpms[k] = v
	}
}

func FindJarsInRpm(rpmname string) {

}
<|endoftext|>


<commit_before>/*
- Copyright 2014 Yua Shizuki. All rights reserved.
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-*/

package main
import "fmt"
import "github.com/howeyc/fsnotify"
import "io/ioutil"
import "path/filepath"
import "os/exec"
import "os"
import "net"
import "strings"
import "regexp"
import "./txtserve"
import "time"

var controllerListener net.Listener
var controllerConn net.Conn

func runner() {
    err := run()
    if err != nil {
        errExit(err, "")
    }
    done := make(chan bool)
    <-done
}

func run() error {
    err := build(false)
    if err != nil {
        return err
    }
    pwd,_ := os.Getwd()
    exe := filepath.Base(pwd)
    port, err := controller()
    if err != nil {
        return err
    }
    cmd := exec.Command("sudo", "./"+exe, "gokcontroller="+port)
    cmd.Stdin = os.Stdin
    cmd.Stdout = os.Stdout
    cmd.Stderr = os.Stderr
    go cmd.Run()
    return nil
}

/*- http server (child process) controller -*/
func controller() (string, error) {
    var err error
    controllerListener, err = net.Listen("tcp", "127.0.0.1:0")
    if err != nil {
        return "", err
    }
    port := strings.Split(controllerListener.Addr().String(), ":")[1]
    go controllerStart()
    return port, nil
}

func switchOffController() {
    if controllerListener != nil {
        controllerListener.Close()
        controllerListener = nil
    }
    if controllerConn != nil {
        controllerConn.Close()
        controllerConn = nil
    }
}

func controllerStart() {
    var err error
    controllerConn, err = controllerListener.Accept()
    if err != nil {
        return
    }
    go startNotifier(".", make(chan bool))
    ioutil.ReadAll(controllerConn)
    switchOffController()
}
/*- End -*/

func startNotifier(dir string, end chan bool) {
    lastUpdate := time.Now()
    goorgok,_ := regexp.Compile("^([^.]*\\.go|[^.]*\\.gok)$")
    watch, err := fsnotify.NewWatcher()
    if err != nil {
        fmt.Println(err)
        os.Exit(1)
    }
    dirNotifiers := make([](chan bool),0,10)
    files, err := ioutil.ReadDir(dir)
    if err != nil {
        errExit(err, "")
    }
    for _, f := range files {
        if f.IsDir() {
            lenD := len(dirNotifiers)
            dirNotifiers = append(dirNotifiers, make(chan bool))
            startNotifier(dir+"/"+f.Name(), dirNotifiers[lenD])
        }
    }
    watch.Watch(dir)
    for ;; {
        select {
            case event := <-watch.Event:
                if goorgok.Match([]byte(event.Name)) || isDirectory(event.Name) {
                    if time.Since(lastUpdate) < (1000 * time.Millisecond) {
                        continue
                    }
                    lastUpdate = time.Now()
                    fmt.Printf("%v: src code update => ", lastUpdate)
                    txtserve.StopServer()
                    switchOffController()
                    time.Sleep(200 * time.Millisecond)
                    err = run()
                    if err != nil {
                        fmt.Printf("error building server binary\n")
                        txtserve.StartServer(err.Error())
                    } else {
                        fmt.Printf("server binary build successful\n")
                        watch.Close()
                        for _, c := range dirNotifiers {
                            c <- true
                            close(c)
                        }
                        return
                    }
                }
            case shouldEnd := <-end:
                if shouldEnd {
                    watch.Close()
                    for _, c := range dirNotifiers {
                            c <- true
                            close(c)
                    }
                    return
                }
        }
    }
}

<commit_msg>standard<commit_after>/*
- Copyright 2014 Yua Shizuki. All rights reserved.
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-*/

package main
import "fmt"
import "github.com/howeyc/fsnotify"
import "io/ioutil"
import "path/filepath"
import "os/exec"
import "os"
import "net"
import "strings"
import "regexp"
import "github.com/YuaShizuki/gok/txtserve"
import "time"

var controllerListener net.Listener
var controllerConn net.Conn

func runner() {
    err := run()
    if err != nil {
        errExit(err, "")
    }
    done := make(chan bool)
    <-done
}

func run() error {
    err := build(false)
    if err != nil {
        return err
    }
    pwd,_ := os.Getwd()
    exe := filepath.Base(pwd)
    port, err := controller()
    if err != nil {
        return err
    }
    cmd := exec.Command("sudo", "./"+exe, "gokcontroller="+port)
    cmd.Stdin = os.Stdin
    cmd.Stdout = os.Stdout
    cmd.Stderr = os.Stderr
    go cmd.Run()
    return nil
}

/*- http server (child process) controller -*/
func controller() (string, error) {
    var err error
    controllerListener, err = net.Listen("tcp", "127.0.0.1:0")
    if err != nil {
        return "", err
    }
    port := strings.Split(controllerListener.Addr().String(), ":")[1]
    go controllerStart()
    return port, nil
}

func switchOffController() {
    if controllerListener != nil {
        controllerListener.Close()
        controllerListener = nil
    }
    if controllerConn != nil {
        controllerConn.Close()
        controllerConn = nil
    }
}

func controllerStart() {
    var err error
    controllerConn, err = controllerListener.Accept()
    if err != nil {
        return
    }
    go startNotifier(".", make(chan bool))
    ioutil.ReadAll(controllerConn)
    switchOffController()
}
/*- End -*/

func startNotifier(dir string, end chan bool) {
    lastUpdate := time.Now()
    goorgok,_ := regexp.Compile("^([^.]*\\.go|[^.]*\\.gok)$")
    watch, err := fsnotify.NewWatcher()
    if err != nil {
        fmt.Println(err)
        os.Exit(1)
    }
    dirNotifiers := make([](chan bool),0,10)
    files, err := ioutil.ReadDir(dir)
    if err != nil {
        errExit(err, "")
    }
    for _, f := range files {
        if f.IsDir() {
            lenD := len(dirNotifiers)
            dirNotifiers = append(dirNotifiers, make(chan bool))
            startNotifier(dir+"/"+f.Name(), dirNotifiers[lenD])
        }
    }
    watch.Watch(dir)
    for ;; {
        select {
            case event := <-watch.Event:
                if goorgok.Match([]byte(event.Name)) || isDirectory(event.Name) {
                    if time.Since(lastUpdate) < (1000 * time.Millisecond) {
                        continue
                    }
                    lastUpdate = time.Now()
                    fmt.Printf("%v: src code update => ", lastUpdate)
                    txtserve.StopServer()
                    switchOffController()
                    time.Sleep(200 * time.Millisecond)
                    err = run()
                    if err != nil {
                        fmt.Printf("error building server binary\n")
                        txtserve.StartServer(err.Error())
                    } else {
                        fmt.Printf("server binary build successful\n")
                        watch.Close()
                        for _, c := range dirNotifiers {
                            c <- true
                            close(c)
                        }
                        return
                    }
                }
            case shouldEnd := <-end:
                if shouldEnd {
                    watch.Close()
                    for _, c := range dirNotifiers {
                            c <- true
                            close(c)
                    }
                    return
                }
        }
    }
}

<|endoftext|>


<commit_before>//Package atomicTypes provides object locking / unlocking for setting and getting.
package atomicTypes

import (
	"sync"
	"time"
)

//AtomicString provides a string object that is lock safe.
type AtomicString struct {
	valueSync sync.RWMutex
	value     string
}

//Get returns the string value
func (obj *AtomicString) Get() (value string) {
	obj.valueSync.RLock()
	value = obj.value
	obj.valueSync.RUnlock()
	return
}

//Set sets the string value
func (obj *AtomicString) Set(value string) {
	obj.valueSync.Lock()
	obj.value = value
	obj.valueSync.Unlock()
}

//AtomicInt provides an int object that is lock safe.
type AtomicInt struct {
	valueSync sync.RWMutex
	value     int
}

//Get returns the int value
func (obj *AtomicInt) Get() (value int) {
	obj.valueSync.RLock()
	value = obj.value
	obj.valueSync.RUnlock()
	return
}

//Set sets the int value
func (obj *AtomicInt) Set(value int) {
	obj.valueSync.Lock()
	obj.value = value
	obj.valueSync.Unlock()
}

// Increment allows for the value to safely be increased by 1
func (obj *AtomicInt) Increment() int {
	obj.valueSync.Lock()
	defer obj.valueSync.Unlock()

	obj.value++
	return obj.value
}

// Decrement allows for the value to safely be decreased by 1
func (obj *AtomicInt) Decrement() int {
	obj.valueSync.Lock()
	defer obj.valueSync.Unlock()

	obj.value--
	return obj.value
}

//AtomicUInt16 provides an uint16 object that is lock safe.
type AtomicUInt16 struct {
	valueSync sync.RWMutex
	value     uint16
}

//Get returns the int value
func (obj *AtomicUInt16) Get() (value uint16) {
	obj.valueSync.RLock()
	value = obj.value
	obj.valueSync.RUnlock()
	return
}

//Set sets the int value
func (obj *AtomicUInt16) Set(value uint16) {
	obj.valueSync.Lock()
	obj.value = value
	obj.valueSync.Unlock()
}

//AtomicUInt32 provides an uint16 object that is lock safe.
type AtomicUInt32 struct {
	valueSync sync.RWMutex
	value     uint32
}

//Get returns the int value
func (obj *AtomicUInt32) Get() (value uint32) {
	obj.valueSync.RLock()
	value = obj.value
	obj.valueSync.RUnlock()
	return
}

//Set sets the int value
func (obj *AtomicUInt32) Set(value uint32) {
	obj.valueSync.Lock()
	obj.value = value
	obj.valueSync.Unlock()
}

//AtomicBool provides an bool object that is lock safe.
type AtomicBool struct {
	valueSync sync.RWMutex
	value     bool
}

//Get returns the bool value
func (obj *AtomicBool) Get() (value bool) {
	obj.valueSync.RLock()
	value = obj.value
	obj.valueSync.RUnlock()
	return
}

//Set sets the bool value
func (obj *AtomicBool) Set(value bool) {
	obj.valueSync.Lock()
	obj.value = value
	obj.valueSync.Unlock()
}

//ToggleTrue sets the bool value only if the check back true; return if the value was set
func (obj *AtomicBool) ToggleTrue() (changed bool) {
	obj.valueSync.Lock()
	defer obj.valueSync.Unlock()

	changed = false
	if obj.value == false {
		obj.value = true

		changed = true
	}

	return
}

//AtomicTime provides a time.Time object that is lock safe.
type AtomicTime struct {
	valueSync sync.RWMutex
	value     time.Time
}

//Get returns the time.Time value
func (obj *AtomicTime) Get() (value time.Time) {
	obj.valueSync.RLock()
	value = obj.value
	obj.valueSync.RUnlock()
	return
}

//Set sets the time.Time value
func (obj *AtomicTime) Set(value time.Time) {
	obj.valueSync.Lock()
	obj.value = value
	obj.valueSync.Unlock()
}

//AtomicByteArray provides a []byte object that is lock safe.
type AtomicByteArray struct {
	valueSync sync.RWMutex
	value     []byte
}

//Get returns the []byte value
func (obj *AtomicByteArray) Get() (value []byte) {
	obj.valueSync.RLock()
	value = obj.value
	obj.valueSync.RUnlock()
	return
}

//Set sets the []byte value
func (obj *AtomicByteArray) Set(value []byte) {
	obj.valueSync.Lock()
	obj.value = value
	obj.valueSync.Unlock()
}

//AtomicBoolArray provides a []bool object that is lock safe.
type AtomicBoolArray struct {
	valueSync sync.RWMutex
	value     []bool
}

//Get returns the []bool value
func (obj *AtomicBoolArray) Get() (value []bool) {
	obj.valueSync.RLock()
	value = obj.value
	obj.valueSync.RUnlock()
	return
}

//Set sets the []bool value
func (obj *AtomicBoolArray) Set(value []bool) {
	obj.valueSync.Lock()
	obj.value = value
	obj.valueSync.Unlock()
}

//SetByIndex sets a bool by index value
func (obj *AtomicBoolArray) SetByIndex(index int, value bool) {
	obj.valueSync.Lock()
	if len(obj.value) > index {
		obj.value[index] = value
	}
	obj.valueSync.Unlock()
}
<commit_msg>Added Atomic Float.<commit_after>//Package atomicTypes provides object locking / unlocking for setting and getting.
package atomicTypes

import (
	"sync"
	"time"
)

//AtomicString provides a string object that is lock safe.
type AtomicString struct {
	valueSync sync.RWMutex
	value     string
}

//Get returns the string value
func (obj *AtomicString) Get() (value string) {
	obj.valueSync.RLock()
	value = obj.value
	obj.valueSync.RUnlock()
	return
}

//Set sets the string value
func (obj *AtomicString) Set(value string) {
	obj.valueSync.Lock()
	obj.value = value
	obj.valueSync.Unlock()
}

//AtomicInt provides an int object that is lock safe.
type AtomicInt struct {
	valueSync sync.RWMutex
	value     int
}

//Get returns the int value
func (obj *AtomicInt) Get() (value int) {
	obj.valueSync.RLock()
	value = obj.value
	obj.valueSync.RUnlock()
	return
}

//Set sets the int value
func (obj *AtomicInt) Set(value int) {
	obj.valueSync.Lock()
	obj.value = value
	obj.valueSync.Unlock()
}

// Increment allows for the value to safely be increased by 1
func (obj *AtomicInt) Increment() int {
	obj.valueSync.Lock()
	defer obj.valueSync.Unlock()

	obj.value++
	return obj.value
}

// Decrement allows for the value to safely be decreased by 1
func (obj *AtomicInt) Decrement() int {
	obj.valueSync.Lock()
	defer obj.valueSync.Unlock()

	obj.value--
	return obj.value
}

//AtomicUInt16 provides an uint16 object that is lock safe.
type AtomicUInt16 struct {
	valueSync sync.RWMutex
	value     uint16
}

//Get returns the int value
func (obj *AtomicUInt16) Get() (value uint16) {
	obj.valueSync.RLock()
	value = obj.value
	obj.valueSync.RUnlock()
	return
}

//Set sets the int value
func (obj *AtomicUInt16) Set(value uint16) {
	obj.valueSync.Lock()
	obj.value = value
	obj.valueSync.Unlock()
}

//AtomicUInt32 provides an uint16 object that is lock safe.
type AtomicUInt32 struct {
	valueSync sync.RWMutex
	value     uint32
}

//Get returns the int value
func (obj *AtomicUInt32) Get() (value uint32) {
	obj.valueSync.RLock()
	value = obj.value
	obj.valueSync.RUnlock()
	return
}

//Set sets the int value
func (obj *AtomicUInt32) Set(value uint32) {
	obj.valueSync.Lock()
	obj.value = value
	obj.valueSync.Unlock()
}

//AtomicBool provides an bool object that is lock safe.
type AtomicBool struct {
	valueSync sync.RWMutex
	value     bool
}

//Get returns the bool value
func (obj *AtomicBool) Get() (value bool) {
	obj.valueSync.RLock()
	value = obj.value
	obj.valueSync.RUnlock()
	return
}

//Set sets the bool value
func (obj *AtomicBool) Set(value bool) {
	obj.valueSync.Lock()
	obj.value = value
	obj.valueSync.Unlock()
}

//ToggleTrue sets the bool value only if the check back true; return if the value was set
func (obj *AtomicBool) ToggleTrue() (changed bool) {
	obj.valueSync.Lock()
	defer obj.valueSync.Unlock()

	changed = false
	if obj.value == false {
		obj.value = true

		changed = true
	}

	return
}

//AtomicTime provides a time.Time object that is lock safe.
type AtomicTime struct {
	valueSync sync.RWMutex
	value     time.Time
}

//Get returns the time.Time value
func (obj *AtomicTime) Get() (value time.Time) {
	obj.valueSync.RLock()
	value = obj.value
	obj.valueSync.RUnlock()
	return
}

//Set sets the time.Time value
func (obj *AtomicTime) Set(value time.Time) {
	obj.valueSync.Lock()
	obj.value = value
	obj.valueSync.Unlock()
}

//AtomicByteArray provides a []byte object that is lock safe.
type AtomicByteArray struct {
	valueSync sync.RWMutex
	value     []byte
}

//Get returns the []byte value
func (obj *AtomicByteArray) Get() (value []byte) {
	obj.valueSync.RLock()
	value = obj.value
	obj.valueSync.RUnlock()
	return
}

//Set sets the []byte value
func (obj *AtomicByteArray) Set(value []byte) {
	obj.valueSync.Lock()
	obj.value = value
	obj.valueSync.Unlock()
}

//AtomicBoolArray provides a []bool object that is lock safe.
type AtomicBoolArray struct {
	valueSync sync.RWMutex
	value     []bool
}

//Get returns the []bool value
func (obj *AtomicBoolArray) Get() (value []bool) {
	obj.valueSync.RLock()
	value = obj.value
	obj.valueSync.RUnlock()
	return
}

//Set sets the []bool value
func (obj *AtomicBoolArray) Set(value []bool) {
	obj.valueSync.Lock()
	obj.value = value
	obj.valueSync.Unlock()
}

//SetByIndex sets a bool by index value
func (obj *AtomicBoolArray) SetByIndex(index int, value bool) {
	obj.valueSync.Lock()
	if len(obj.value) > index {
		obj.value[index] = value
	}
	obj.valueSync.Unlock()
}

//AtomicFloat64 provides a float64 object that is lock safe.
type AtomicFloat64 struct {
	valueSync sync.RWMutex
	value     float64
}

//Get returns the float64 value
func (obj *AtomicFloat64) Get() (value float64) {
	obj.valueSync.RLock()
	value = obj.value
	obj.valueSync.RUnlock()
	return
}

//Set sets the float64 value
func (obj *AtomicFloat64) Set(value float64) {
	obj.valueSync.Lock()
	obj.value = value
	obj.valueSync.Unlock()
}
<|endoftext|>


<commit_before>package client

import (
	"fmt"
	"io"
	"os"
	"time"

	"strings"

	"github.com/ncodes/cocoon/core/orderer"
	proto "github.com/ncodes/cocoon/core/runtime/golang/proto"
	"github.com/ncodes/cocoon/core/types"
	logging "github.com/op/go-logging"
	cmap "github.com/orcaman/concurrent-map"
	context "golang.org/x/net/context"
	"google.golang.org/grpc"
)

var log = logging.MustGetLogger("connector.client")

// txChannels holds the channels to send transaction responses to
var txRespChannels = cmap.New()

// Client represents a cocoon code GRPC client
// that interacts with a cocoon code.
type Client struct {
	ccodeAddr        string
	stub             proto.StubClient
	conCtx           context.Context
	conCancel        context.CancelFunc
	orderDiscoTicker *time.Ticker
	ordererAddrs     []string
	stream           proto.Stub_TransactClient
	cocoonID         string
}

// NewClient creates a new cocoon code client
func NewClient() *Client {
	return &Client{}
}

// SetCocoonID sets the cocoon id
func (c *Client) SetCocoonID(id string) {
	c.cocoonID = id
}

// SetCocoonCodeAddr sets the cocoon code bind address
func (c *Client) SetCocoonCodeAddr(ccAddr string) {
	c.ccodeAddr = ccAddr
}

// getCCAddr returns the cocoon code bind address.
// For development, if DEV_COCOON_ADDR is set, it connects to it.
func (c *Client) getCCAddr() string {
	if devCCodeAddr := os.Getenv("DEV_COCOON_ADDR"); len(devCCodeAddr) > 0 {
		return devCCodeAddr
	}
	return c.ccodeAddr
}

// Close the stream and cancel connections
func (c *Client) Close() {
	if c.stream != nil {
		c.stream.CloseSend()
	}
	if c.conCancel != nil {
		c.conCancel()
	}
}

// GetStream returns the grpc stream connected to the grpc cocoon code server
func (c *Client) GetStream() proto.Stub_TransactClient {
	return c.stream
}

// Connect connects to a cocoon code server
// running on a known port
func (c *Client) Connect() error {

	log.Info("Starting cocoon code client")

	// start a ticker to continously discover orderer addreses
	go func() {
		c.orderDiscoTicker = time.NewTicker(60 * time.Second)
		for _ = range c.orderDiscoTicker.C {
			var ordererAddrs []string
			ordererAddrs, err := orderer.DiscoverOrderers()
			if err != nil {
				log.Error(err.Error())
				continue
			}
			c.ordererAddrs = ordererAddrs
		}
	}()

	var ordererAddrs []string
	ordererAddrs, err := orderer.DiscoverOrderers()
	if err != nil {
		return err
	}
	c.ordererAddrs = ordererAddrs

	if len(c.ordererAddrs) > 0 {
		log.Infof("Orderer address list updated. Contains %d orderer address(es)", len(c.ordererAddrs))
	} else {
		log.Warning("No orderer address was found. We won't be able to reach the orderer. ")
	}

	conn, err := grpc.Dial(c.getCCAddr(), grpc.WithInsecure())
	if err != nil {
		return fmt.Errorf("failed to connect to cocoon code server. %s", err)
	}
	defer conn.Close()

	log.Debugf("Now connected to cocoon code at port=%s", strings.Split(c.getCCAddr(), ":")[1])

	c.stub = proto.NewStubClient(conn)

	time.Sleep(20 * time.Second)
	if err = c.Do(conn); err != nil {
		return err
	}

	return nil
}

// Do starts a request loop that continously asks the
// server for transactions. When it receives a transaction,
// it processes it and returns the result.
func (c *Client) Do(conn *grpc.ClientConn) error {

	var err error

	// create a context so we have complete controll of the connection
	c.conCtx, c.conCancel = context.WithCancel(context.Background())

	// connect to the cocoon code
	c.stream, err = c.stub.Transact(c.conCtx)
	if err != nil {
		return fmt.Errorf("failed to start transaction stream with cocoon code. %s", err)
	}

	for {

		in, err := c.stream.Recv()
		if err == io.EOF {
			return fmt.Errorf("connection with cocoon code has ended")
		}
		if err != nil {
			if strings.Contains(err.Error(), "context canceled") {
				log.Info("Connection to cocoon code closed")
				return nil
			}
			return fmt.Errorf("failed to read message from cocoon code. %s", err)
		}

		switch in.Invoke {
		case true:
			go func() {
				log.Debugf("New invoke transaction (%s) from cocoon code", in.GetId())
				if err = c.handleInvokeTransaction(in); err != nil {
					log.Error(err.Error())
					c.stream.Send(&proto.Tx{
						Response: true,
						Id:       in.GetId(),
						Status:   500,
						Body:     []byte(err.Error()),
					})
				}
			}()
		case false:
			log.Debugf("New response transaction (%s) from cocoon code", in.GetId())
			go func() {
				if err = c.handleRespTransaction(in); err != nil {
					log.Error(err.Error())
					c.stream.Send(&proto.Tx{
						Response: true,
						Id:       in.GetId(),
						Status:   500,
						Body:     []byte(err.Error()),
					})
				}
			}()
		}
	}
}

// handleInvokeTransaction processes invoke transaction requests
func (c *Client) handleInvokeTransaction(tx *proto.Tx) error {
	switch tx.GetName() {
	case types.TxCreateLedger:
		return c.createLedger(tx)
	case types.TxPut:
		return c.put(tx)
	case types.TxGetLedger:
		return c.getLedger(tx)
	case types.TxGet:
		return c.get(tx, false)
	case types.TxGetByID:
		return c.get(tx, true)
	case types.TxGetBlockByID:
		return c.getBlock(tx)
	case types.TxRangeGet:
		return c.getRange(tx)
	default:
		return c.stream.Send(&proto.Tx{
			Id:       tx.GetId(),
			Response: true,
			Status:   500,
			Body:     []byte(fmt.Sprintf("unsupported transaction name (%s)", tx.GetName())),
		})
	}
}

// handleRespTransaction passes the transaction to a response
// channel with a matching transaction id and deletes the channel afterwards.
func (c *Client) handleRespTransaction(tx *proto.Tx) error {
	if !txRespChannels.Has(tx.GetId()) {
		return fmt.Errorf("response transaction (%s) does not have a corresponding response channel", tx.GetId())
	}

	txRespCh, _ := txRespChannels.Get(tx.GetId())
	txRespCh.(chan *proto.Tx) <- tx
	txRespChannels.Remove(tx.GetId())
	return nil
}

// SendTx sends a transaction to the cocoon code
// and saves the response channel. The response channel will
// be passed a response when it is available in the Transact loop.
func (c *Client) SendTx(tx *proto.Tx, respCh chan *proto.Tx) error {
	txRespChannels.Set(tx.GetId(), respCh)
	if err := c.stream.Send(tx); err != nil {
		txRespChannels.Remove(tx.GetId())
		return err
	}
	log.Debugf("Successfully sent transaction (%s) to cocoon code", tx.GetId())
	return nil
}
<commit_msg>debug<commit_after>package client

import (
	"fmt"
	"io"
	"os"
	"time"

	"strings"

	"github.com/ncodes/cocoon/core/orderer"
	proto "github.com/ncodes/cocoon/core/runtime/golang/proto"
	"github.com/ncodes/cocoon/core/types"
	logging "github.com/op/go-logging"
	cmap "github.com/orcaman/concurrent-map"
	context "golang.org/x/net/context"
	"google.golang.org/grpc"
)

var log = logging.MustGetLogger("connector.client")

// txChannels holds the channels to send transaction responses to
var txRespChannels = cmap.New()

// Client represents a cocoon code GRPC client
// that interacts with a cocoon code.
type Client struct {
	ccodeAddr        string
	stub             proto.StubClient
	conCtx           context.Context
	conCancel        context.CancelFunc
	orderDiscoTicker *time.Ticker
	ordererAddrs     []string
	stream           proto.Stub_TransactClient
	cocoonID         string
}

// NewClient creates a new cocoon code client
func NewClient() *Client {
	return &Client{}
}

// SetCocoonID sets the cocoon id
func (c *Client) SetCocoonID(id string) {
	c.cocoonID = id
}

// SetCocoonCodeAddr sets the cocoon code bind address
func (c *Client) SetCocoonCodeAddr(ccAddr string) {
	c.ccodeAddr = ccAddr
}

// getCCAddr returns the cocoon code bind address.
// For development, if DEV_COCOON_ADDR is set, it connects to it.
func (c *Client) getCCAddr() string {
	if devCCodeAddr := os.Getenv("DEV_COCOON_ADDR"); len(devCCodeAddr) > 0 {
		return devCCodeAddr
	}
	return c.ccodeAddr
}

// Close the stream and cancel connections
func (c *Client) Close() {
	if c.stream != nil {
		c.stream.CloseSend()
	}
	if c.conCancel != nil {
		c.conCancel()
	}
}

// GetStream returns the grpc stream connected to the grpc cocoon code server
func (c *Client) GetStream() proto.Stub_TransactClient {
	return c.stream
}

// Connect connects to a cocoon code server
// running on a known port
func (c *Client) Connect() error {

	log.Info("Starting cocoon code client")

	// start a ticker to continously discover orderer addreses
	go func() {
		c.orderDiscoTicker = time.NewTicker(60 * time.Second)
		for _ = range c.orderDiscoTicker.C {
			var ordererAddrs []string
			ordererAddrs, err := orderer.DiscoverOrderers()
			if err != nil {
				log.Error(err.Error())
				continue
			}
			c.ordererAddrs = ordererAddrs
		}
	}()

	var ordererAddrs []string
	ordererAddrs, err := orderer.DiscoverOrderers()
	if err != nil {
		return err
	}
	c.ordererAddrs = ordererAddrs

	if len(c.ordererAddrs) > 0 {
		log.Infof("Orderer address list updated. Contains %d orderer address(es)", len(c.ordererAddrs))
	} else {
		log.Warning("No orderer address was found. We won't be able to reach the orderer. ")
	}

	conn, err := grpc.Dial(c.getCCAddr(), grpc.WithInsecure())
	if err != nil {
		return fmt.Errorf("failed to connect to cocoon code server. %s", err)
	}
	defer conn.Close()

	log.Debugf("Now connected to cocoon code at port=%s", strings.Split(c.getCCAddr(), ":")[1])

	c.stub = proto.NewStubClient(conn)

	// if err = c.Do(conn); err != nil {
	// 	return err
	// }
	s := make(chan bool)
	<-s

	return nil
}

// Do starts a request loop that continously asks the
// server for transactions. When it receives a transaction,
// it processes it and returns the result.
func (c *Client) Do(conn *grpc.ClientConn) error {

	var err error

	// create a context so we have complete controll of the connection
	c.conCtx, c.conCancel = context.WithCancel(context.Background())

	// connect to the cocoon code
	c.stream, err = c.stub.Transact(c.conCtx)
	if err != nil {
		return fmt.Errorf("failed to start transaction stream with cocoon code. %s", err)
	}

	for {

		in, err := c.stream.Recv()
		if err == io.EOF {
			return fmt.Errorf("connection with cocoon code has ended")
		}
		if err != nil {
			if strings.Contains(err.Error(), "context canceled") {
				log.Info("Connection to cocoon code closed")
				return nil
			}
			return fmt.Errorf("failed to read message from cocoon code. %s", err)
		}

		switch in.Invoke {
		case true:
			go func() {
				log.Debugf("New invoke transaction (%s) from cocoon code", in.GetId())
				if err = c.handleInvokeTransaction(in); err != nil {
					log.Error(err.Error())
					c.stream.Send(&proto.Tx{
						Response: true,
						Id:       in.GetId(),
						Status:   500,
						Body:     []byte(err.Error()),
					})
				}
			}()
		case false:
			log.Debugf("New response transaction (%s) from cocoon code", in.GetId())
			go func() {
				if err = c.handleRespTransaction(in); err != nil {
					log.Error(err.Error())
					c.stream.Send(&proto.Tx{
						Response: true,
						Id:       in.GetId(),
						Status:   500,
						Body:     []byte(err.Error()),
					})
				}
			}()
		}
	}
}

// handleInvokeTransaction processes invoke transaction requests
func (c *Client) handleInvokeTransaction(tx *proto.Tx) error {
	switch tx.GetName() {
	case types.TxCreateLedger:
		return c.createLedger(tx)
	case types.TxPut:
		return c.put(tx)
	case types.TxGetLedger:
		return c.getLedger(tx)
	case types.TxGet:
		return c.get(tx, false)
	case types.TxGetByID:
		return c.get(tx, true)
	case types.TxGetBlockByID:
		return c.getBlock(tx)
	case types.TxRangeGet:
		return c.getRange(tx)
	default:
		return c.stream.Send(&proto.Tx{
			Id:       tx.GetId(),
			Response: true,
			Status:   500,
			Body:     []byte(fmt.Sprintf("unsupported transaction name (%s)", tx.GetName())),
		})
	}
}

// handleRespTransaction passes the transaction to a response
// channel with a matching transaction id and deletes the channel afterwards.
func (c *Client) handleRespTransaction(tx *proto.Tx) error {
	if !txRespChannels.Has(tx.GetId()) {
		return fmt.Errorf("response transaction (%s) does not have a corresponding response channel", tx.GetId())
	}

	txRespCh, _ := txRespChannels.Get(tx.GetId())
	txRespCh.(chan *proto.Tx) <- tx
	txRespChannels.Remove(tx.GetId())
	return nil
}

// SendTx sends a transaction to the cocoon code
// and saves the response channel. The response channel will
// be passed a response when it is available in the Transact loop.
func (c *Client) SendTx(tx *proto.Tx, respCh chan *proto.Tx) error {
	txRespChannels.Set(tx.GetId(), respCh)
	if err := c.stream.Send(tx); err != nil {
		txRespChannels.Remove(tx.GetId())
		return err
	}
	log.Debugf("Successfully sent transaction (%s) to cocoon code", tx.GetId())
	return nil
}
<|endoftext|>


<commit_before>package estafette

import (
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"strings"
	"time"

	"github.com/estafette/estafette-ci-api/docker"

	yaml "gopkg.in/yaml.v2"

	"github.com/ericchiang/k8s"
	"github.com/ericchiang/k8s/api/resource"
	apiv1 "github.com/ericchiang/k8s/api/v1"
	batchv1 "github.com/ericchiang/k8s/apis/batch/v1"
	metav1 "github.com/ericchiang/k8s/apis/meta/v1"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/rs/zerolog/log"
)

// CiBuilderClient is the interface for running kubernetes commands specific to this application
type CiBuilderClient interface {
	CreateCiBuilderJob(CiBuilderParams) (*batchv1.Job, error)
	RemoveCiBuilderJob(string) error
}

type ciBuilderClientImpl struct {
	kubeClient                      *k8s.Client
	dockerHubClient                 docker.DockerHubAPIClient
	EstafetteCiServerBaseURL        string
	EstafetteCiAPIKey               string
	secretDecryptionKey             string
	PrometheusOutboundAPICallTotals *prometheus.CounterVec
}

// NewCiBuilderClient returns a new estafette.CiBuilderClient
func NewCiBuilderClient(estafetteCiServerBaseURL, estafetteCiAPIKey, secretDecryptionKey string, prometheusOutboundAPICallTotals *prometheus.CounterVec) (ciBuilderClient CiBuilderClient, err error) {

	var kubeClient *k8s.Client

	if os.Getenv("KUBERNETES_SERVICE_HOST") != "" && os.Getenv("KUBERNETES_SERVICE_PORT") != "" {

		kubeClient, err = k8s.NewInClusterClient()
		if err != nil {
			log.Error().Err(err).Msg("Creating k8s client failed")
			return
		}

	} else {

		homeDir := os.Getenv("HOME")

		data, err := ioutil.ReadFile(fmt.Sprintf("%v/.kube/config", homeDir))
		if err != nil {
			log.Error().Err(err).Msg("Reading kube config failed")
			return nil, err
		}

		var config k8s.Config
		if err := yaml.Unmarshal(data, &config); err != nil {
			log.Error().Err(err).Msg("Deserializing kube config failed")
			return nil, err
		}

		kubeClient, err = k8s.NewClient(&config)
	}

	dockerHubClient, err := docker.NewDockerHubAPIClient()
	if err != nil {
		return
	}

	ciBuilderClient = &ciBuilderClientImpl{
		kubeClient:                      kubeClient,
		dockerHubClient:                 dockerHubClient,
		EstafetteCiServerBaseURL:        estafetteCiServerBaseURL,
		EstafetteCiAPIKey:               estafetteCiAPIKey,
		secretDecryptionKey:             secretDecryptionKey,
		PrometheusOutboundAPICallTotals: prometheusOutboundAPICallTotals,
	}

	return
}

// CreateCiBuilderJob creates an estafette-ci-builder job in Kubernetes to run the estafette build
func (cbc *ciBuilderClientImpl) CreateCiBuilderJob(ciBuilderParams CiBuilderParams) (job *batchv1.Job, err error) {

	// create job name of max 63 chars
	re := regexp.MustCompile("[^a-zA-Z0-9]+")
	repoName := re.ReplaceAllString(ciBuilderParams.RepoFullName, "-")
	if len(repoName) > 50 {
		repoName = repoName[:50]
	}
	jobName := strings.ToLower(fmt.Sprintf("build-%v-%v", repoName, ciBuilderParams.RepoRevision[:6]))

	// create envvars for job
	estafetteGitSourceName := "ESTAFETTE_GIT_SOURCE"
	estafetteGitSourceValue := ciBuilderParams.RepoSource
	estafetteGitNameName := "ESTAFETTE_GIT_NAME"
	estafetteGitNameValue := ciBuilderParams.RepoFullName
	estafetteGitURLName := "ESTAFETTE_GIT_URL"
	estafetteGitURLValue := ciBuilderParams.RepoURL
	estafetteGitBranchName := "ESTAFETTE_GIT_BRANCH"
	estafetteGitBranchValue := ciBuilderParams.RepoBranch
	estafetteGitRevisionName := "ESTAFETTE_GIT_REVISION"
	estafetteGitRevisionValue := ciBuilderParams.RepoRevision
	estafetteBuildJobNameName := "ESTAFETTE_BUILD_JOB_NAME"
	estafetteBuildJobNameValue := jobName
	estafetteCiServerBaseURLName := "ESTAFETTE_CI_SERVER_BASE_URL"
	estafetteCiServerBaseURLValue := cbc.EstafetteCiServerBaseURL
	estafetteCiServerBuilderEventsURLName := "ESTAFETTE_CI_SERVER_BUILDER_EVENTS_URL"
	estafetteCiServerBuilderEventsURLValue := strings.TrimRight(cbc.EstafetteCiServerBaseURL, "/") + "/events/estafette/ci-builder"
	estafetteCiAPIKeyName := "ESTAFETTE_CI_API_KEY"
	estafetteCiAPIKeyValue := cbc.EstafetteCiAPIKey
	estafetteCiBuilderTrackName := "ESTAFETTE_CI_BUILDER_TRACK"
	estafetteCiBuilderTrackValue := ciBuilderParams.Track

	// temporarily pass build version equal to revision from the outside until estafette supports versioning
	estafetteBuildVersionName := "ESTAFETTE_BUILD_VERSION"
	estafetteBuildVersionValue := ciBuilderParams.VersionNumber
	estafetteBuildVersionPatchName := "ESTAFETTE_BUILD_VERSION_PATCH"
	estafetteBuildVersionPatchValue := fmt.Sprint(ciBuilderParams.AutoIncrement)

	environmentVariables := []*apiv1.EnvVar{
		&apiv1.EnvVar{
			Name:  &estafetteGitSourceName,
			Value: &estafetteGitSourceValue,
		},
		&apiv1.EnvVar{
			Name:  &estafetteGitNameName,
			Value: &estafetteGitNameValue,
		},
		&apiv1.EnvVar{
			Name:  &estafetteGitURLName,
			Value: &estafetteGitURLValue,
		},
		&apiv1.EnvVar{
			Name:  &estafetteGitBranchName,
			Value: &estafetteGitBranchValue,
		},
		&apiv1.EnvVar{
			Name:  &estafetteGitRevisionName,
			Value: &estafetteGitRevisionValue,
		},
		&apiv1.EnvVar{
			Name:  &estafetteBuildVersionName,
			Value: &estafetteBuildVersionValue,
		},
		&apiv1.EnvVar{
			Name:  &estafetteBuildVersionPatchName,
			Value: &estafetteBuildVersionPatchValue,
		},
		&apiv1.EnvVar{
			Name:  &estafetteBuildJobNameName,
			Value: &estafetteBuildJobNameValue,
		},
		&apiv1.EnvVar{
			Name:  &estafetteCiServerBaseURLName,
			Value: &estafetteCiServerBaseURLValue,
		},
		&apiv1.EnvVar{
			Name:  &estafetteCiServerBuilderEventsURLName,
			Value: &estafetteCiServerBuilderEventsURLValue,
		},
		&apiv1.EnvVar{
			Name:  &estafetteCiAPIKeyName,
			Value: &estafetteCiAPIKeyValue,
		},
		&apiv1.EnvVar{
			Name:  &estafetteCiBuilderTrackName,
			Value: &estafetteCiBuilderTrackValue,
		},
	}

	for key, value := range ciBuilderParams.EnvironmentVariables {
		environmentVariables = append(environmentVariables, &apiv1.EnvVar{
			Name:  &key,
			Value: &value,
		})
	}

	// define resource request and limit values to fit reasonably well inside a n1-highmem-4 machine
	cpuRequest := "1.0"
	cpuLimit := "3.0"
	memoryRequest := "2.0Gi"
	memoryLimit := "20.0Gi"

	// other job config
	containerName := "estafette-ci-builder"
	repository := "estafette/estafette-ci-builder"
	tag := ciBuilderParams.Track
	image := fmt.Sprintf("%v:%v", repository, tag)
	imagePullPolicy := "Always"
	digest, err := cbc.dockerHubClient.GetDigestCached(repository, tag)
	if err == nil {
		image = fmt.Sprintf("%v@%v", repository, digest.Digest)
		imagePullPolicy = "IfNotPresent"
	}
	restartPolicy := "Never"
	privileged := true

	job = &batchv1.Job{
		Metadata: &metav1.ObjectMeta{
			Name:      &jobName,
			Namespace: &cbc.kubeClient.Namespace,
			Labels: map[string]string{
				"createdBy": "estafette",
			},
		},
		Spec: &batchv1.JobSpec{
			Template: &apiv1.PodTemplateSpec{
				Metadata: &metav1.ObjectMeta{
					Labels: map[string]string{
						"createdBy": "estafette",
					},
				},
				Spec: &apiv1.PodSpec{
					Containers: []*apiv1.Container{
						&apiv1.Container{
							Name:            &containerName,
							Image:           &image,
							ImagePullPolicy: &imagePullPolicy,
							Args:            []string{fmt.Sprintf("--secret-decryption-key=%v", cbc.secretDecryptionKey)},
							Env:             environmentVariables,
							SecurityContext: &apiv1.SecurityContext{
								Privileged: &privileged,
							},
							Resources: &apiv1.ResourceRequirements{
								Requests: map[string]*resource.Quantity{
									"cpu":    &resource.Quantity{String_: &cpuRequest},
									"memory": &resource.Quantity{String_: &memoryRequest},
								},
								Limits: map[string]*resource.Quantity{
									"cpu":    &resource.Quantity{String_: &cpuLimit},
									"memory": &resource.Quantity{String_: &memoryLimit},
								},
							},
						},
					},
					RestartPolicy: &restartPolicy,
				},
			},
		},
	}

	job, err = cbc.kubeClient.BatchV1().CreateJob(context.Background(), job)
	cbc.PrometheusOutboundAPICallTotals.With(prometheus.Labels{"target": "kubernetes"}).Inc()

	return
}

// RemoveCiBuilderJob waits for a job to finish and then removes it
func (cbc *ciBuilderClientImpl) RemoveCiBuilderJob(jobName string) (err error) {

	// check if job is finished
	job, err := cbc.kubeClient.BatchV1().GetJob(context.Background(), jobName, cbc.kubeClient.Namespace)
	cbc.PrometheusOutboundAPICallTotals.With(prometheus.Labels{"target": "kubernetes"}).Inc()
	if err != nil {
		log.Error().Err(err).
			Str("jobName", jobName).
			Msgf("GetJob call for job %v failed", jobName)
	}

	if err != nil || *job.Status.Succeeded != 1 {
		log.Debug().
			Str("jobName", jobName).
			Msgf("Job is not done yet, watching for job %v to succeed", jobName)

		// watch for job updates
		watcher, err := cbc.kubeClient.BatchV1().WatchJobs(context.Background(), cbc.kubeClient.Namespace, k8s.Timeout(time.Duration(60)*time.Second))
		cbc.PrometheusOutboundAPICallTotals.With(prometheus.Labels{"target": "kubernetes"}).Inc()
		if err != nil {
			log.Error().Err(err).
				Str("jobName", jobName).
				Msgf("WatchJobs call for job %v failed", jobName)
		} else {
			// wait for job to succeed
			for {
				event, job, err := watcher.Next()
				if err != nil {
					log.Error().Err(err)
					break
				}

				if *event.Type == k8s.EventModified && *job.Metadata.Name == jobName && *job.Status.Succeeded == 1 {
					break
				}
			}
		}
	}
	log.Debug().
		Str("jobName", jobName).
		Msgf("Job %v is done, deleting it...", jobName)

	// delete job
	err = cbc.kubeClient.BatchV1().DeleteJob(context.Background(), jobName, cbc.kubeClient.Namespace)
	cbc.PrometheusOutboundAPICallTotals.With(prometheus.Labels{"target": "kubernetes"}).Inc()
	if err != nil {
		log.Error().Err(err).
			Str("jobName", jobName).
			Msgf("Deleting job %v failed", jobName)
		return
	}

	return
}
<commit_msg>prefer to run ci build job on preemptibles<commit_after>package estafette

import (
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"strings"
	"time"

	"github.com/estafette/estafette-ci-api/docker"

	yaml "gopkg.in/yaml.v2"

	"github.com/ericchiang/k8s"
	"github.com/ericchiang/k8s/api/resource"
	apiv1 "github.com/ericchiang/k8s/api/v1"
	batchv1 "github.com/ericchiang/k8s/apis/batch/v1"
	metav1 "github.com/ericchiang/k8s/apis/meta/v1"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/rs/zerolog/log"
)

// CiBuilderClient is the interface for running kubernetes commands specific to this application
type CiBuilderClient interface {
	CreateCiBuilderJob(CiBuilderParams) (*batchv1.Job, error)
	RemoveCiBuilderJob(string) error
}

type ciBuilderClientImpl struct {
	kubeClient                      *k8s.Client
	dockerHubClient                 docker.DockerHubAPIClient
	EstafetteCiServerBaseURL        string
	EstafetteCiAPIKey               string
	secretDecryptionKey             string
	PrometheusOutboundAPICallTotals *prometheus.CounterVec
}

// NewCiBuilderClient returns a new estafette.CiBuilderClient
func NewCiBuilderClient(estafetteCiServerBaseURL, estafetteCiAPIKey, secretDecryptionKey string, prometheusOutboundAPICallTotals *prometheus.CounterVec) (ciBuilderClient CiBuilderClient, err error) {

	var kubeClient *k8s.Client

	if os.Getenv("KUBERNETES_SERVICE_HOST") != "" && os.Getenv("KUBERNETES_SERVICE_PORT") != "" {

		kubeClient, err = k8s.NewInClusterClient()
		if err != nil {
			log.Error().Err(err).Msg("Creating k8s client failed")
			return
		}

	} else {

		homeDir := os.Getenv("HOME")

		data, err := ioutil.ReadFile(fmt.Sprintf("%v/.kube/config", homeDir))
		if err != nil {
			log.Error().Err(err).Msg("Reading kube config failed")
			return nil, err
		}

		var config k8s.Config
		if err := yaml.Unmarshal(data, &config); err != nil {
			log.Error().Err(err).Msg("Deserializing kube config failed")
			return nil, err
		}

		kubeClient, err = k8s.NewClient(&config)
	}

	dockerHubClient, err := docker.NewDockerHubAPIClient()
	if err != nil {
		return
	}

	ciBuilderClient = &ciBuilderClientImpl{
		kubeClient:                      kubeClient,
		dockerHubClient:                 dockerHubClient,
		EstafetteCiServerBaseURL:        estafetteCiServerBaseURL,
		EstafetteCiAPIKey:               estafetteCiAPIKey,
		secretDecryptionKey:             secretDecryptionKey,
		PrometheusOutboundAPICallTotals: prometheusOutboundAPICallTotals,
	}

	return
}

// CreateCiBuilderJob creates an estafette-ci-builder job in Kubernetes to run the estafette build
func (cbc *ciBuilderClientImpl) CreateCiBuilderJob(ciBuilderParams CiBuilderParams) (job *batchv1.Job, err error) {

	// create job name of max 63 chars
	re := regexp.MustCompile("[^a-zA-Z0-9]+")
	repoName := re.ReplaceAllString(ciBuilderParams.RepoFullName, "-")
	if len(repoName) > 50 {
		repoName = repoName[:50]
	}
	jobName := strings.ToLower(fmt.Sprintf("build-%v-%v", repoName, ciBuilderParams.RepoRevision[:6]))

	// create envvars for job
	estafetteGitSourceName := "ESTAFETTE_GIT_SOURCE"
	estafetteGitSourceValue := ciBuilderParams.RepoSource
	estafetteGitNameName := "ESTAFETTE_GIT_NAME"
	estafetteGitNameValue := ciBuilderParams.RepoFullName
	estafetteGitURLName := "ESTAFETTE_GIT_URL"
	estafetteGitURLValue := ciBuilderParams.RepoURL
	estafetteGitBranchName := "ESTAFETTE_GIT_BRANCH"
	estafetteGitBranchValue := ciBuilderParams.RepoBranch
	estafetteGitRevisionName := "ESTAFETTE_GIT_REVISION"
	estafetteGitRevisionValue := ciBuilderParams.RepoRevision
	estafetteBuildJobNameName := "ESTAFETTE_BUILD_JOB_NAME"
	estafetteBuildJobNameValue := jobName
	estafetteCiServerBaseURLName := "ESTAFETTE_CI_SERVER_BASE_URL"
	estafetteCiServerBaseURLValue := cbc.EstafetteCiServerBaseURL
	estafetteCiServerBuilderEventsURLName := "ESTAFETTE_CI_SERVER_BUILDER_EVENTS_URL"
	estafetteCiServerBuilderEventsURLValue := strings.TrimRight(cbc.EstafetteCiServerBaseURL, "/") + "/events/estafette/ci-builder"
	estafetteCiAPIKeyName := "ESTAFETTE_CI_API_KEY"
	estafetteCiAPIKeyValue := cbc.EstafetteCiAPIKey
	estafetteCiBuilderTrackName := "ESTAFETTE_CI_BUILDER_TRACK"
	estafetteCiBuilderTrackValue := ciBuilderParams.Track

	// temporarily pass build version equal to revision from the outside until estafette supports versioning
	estafetteBuildVersionName := "ESTAFETTE_BUILD_VERSION"
	estafetteBuildVersionValue := ciBuilderParams.VersionNumber
	estafetteBuildVersionPatchName := "ESTAFETTE_BUILD_VERSION_PATCH"
	estafetteBuildVersionPatchValue := fmt.Sprint(ciBuilderParams.AutoIncrement)

	environmentVariables := []*apiv1.EnvVar{
		&apiv1.EnvVar{
			Name:  &estafetteGitSourceName,
			Value: &estafetteGitSourceValue,
		},
		&apiv1.EnvVar{
			Name:  &estafetteGitNameName,
			Value: &estafetteGitNameValue,
		},
		&apiv1.EnvVar{
			Name:  &estafetteGitURLName,
			Value: &estafetteGitURLValue,
		},
		&apiv1.EnvVar{
			Name:  &estafetteGitBranchName,
			Value: &estafetteGitBranchValue,
		},
		&apiv1.EnvVar{
			Name:  &estafetteGitRevisionName,
			Value: &estafetteGitRevisionValue,
		},
		&apiv1.EnvVar{
			Name:  &estafetteBuildVersionName,
			Value: &estafetteBuildVersionValue,
		},
		&apiv1.EnvVar{
			Name:  &estafetteBuildVersionPatchName,
			Value: &estafetteBuildVersionPatchValue,
		},
		&apiv1.EnvVar{
			Name:  &estafetteBuildJobNameName,
			Value: &estafetteBuildJobNameValue,
		},
		&apiv1.EnvVar{
			Name:  &estafetteCiServerBaseURLName,
			Value: &estafetteCiServerBaseURLValue,
		},
		&apiv1.EnvVar{
			Name:  &estafetteCiServerBuilderEventsURLName,
			Value: &estafetteCiServerBuilderEventsURLValue,
		},
		&apiv1.EnvVar{
			Name:  &estafetteCiAPIKeyName,
			Value: &estafetteCiAPIKeyValue,
		},
		&apiv1.EnvVar{
			Name:  &estafetteCiBuilderTrackName,
			Value: &estafetteCiBuilderTrackValue,
		},
	}

	for key, value := range ciBuilderParams.EnvironmentVariables {
		environmentVariables = append(environmentVariables, &apiv1.EnvVar{
			Name:  &key,
			Value: &value,
		})
	}

	// define resource request and limit values to fit reasonably well inside a n1-highmem-4 machine
	cpuRequest := "1.0"
	cpuLimit := "3.0"
	memoryRequest := "2.0Gi"
	memoryLimit := "20.0Gi"

	// other job config
	containerName := "estafette-ci-builder"
	repository := "estafette/estafette-ci-builder"
	tag := ciBuilderParams.Track
	image := fmt.Sprintf("%v:%v", repository, tag)
	imagePullPolicy := "Always"
	digest, err := cbc.dockerHubClient.GetDigestCached(repository, tag)
	if err == nil {
		image = fmt.Sprintf("%v@%v", repository, digest.Digest)
		imagePullPolicy = "IfNotPresent"
	}
	restartPolicy := "Never"
	privileged := true

	preemptibleAffinityWeight := int32(10)
	preemptibleAffinityKey := "cloud.google.com/gke-preemptible"
	preemptibleAffinityOperator := "In"

	job = &batchv1.Job{
		Metadata: &metav1.ObjectMeta{
			Name:      &jobName,
			Namespace: &cbc.kubeClient.Namespace,
			Labels: map[string]string{
				"createdBy": "estafette",
			},
		},
		Spec: &batchv1.JobSpec{
			Template: &apiv1.PodTemplateSpec{
				Metadata: &metav1.ObjectMeta{
					Labels: map[string]string{
						"createdBy": "estafette",
					},
				},
				Spec: &apiv1.PodSpec{
					Containers: []*apiv1.Container{
						&apiv1.Container{
							Name:            &containerName,
							Image:           &image,
							ImagePullPolicy: &imagePullPolicy,
							Args:            []string{fmt.Sprintf("--secret-decryption-key=%v", cbc.secretDecryptionKey)},
							Env:             environmentVariables,
							SecurityContext: &apiv1.SecurityContext{
								Privileged: &privileged,
							},
							Resources: &apiv1.ResourceRequirements{
								Requests: map[string]*resource.Quantity{
									"cpu":    &resource.Quantity{String_: &cpuRequest},
									"memory": &resource.Quantity{String_: &memoryRequest},
								},
								Limits: map[string]*resource.Quantity{
									"cpu":    &resource.Quantity{String_: &cpuLimit},
									"memory": &resource.Quantity{String_: &memoryLimit},
								},
							},
						},
					},
					RestartPolicy: &restartPolicy,

					Affinity: &apiv1.Affinity{
						NodeAffinity: &apiv1.NodeAffinity{
							PreferredDuringSchedulingIgnoredDuringExecution: []*apiv1.PreferredSchedulingTerm{
								&apiv1.PreferredSchedulingTerm{
									Weight: &preemptibleAffinityWeight,
									// A node selector term, associated with the corresponding weight.
									Preference: &apiv1.NodeSelectorTerm{
										MatchExpressions: []*apiv1.NodeSelectorRequirement{
											&apiv1.NodeSelectorRequirement{
												Key:      &preemptibleAffinityKey,
												Operator: &preemptibleAffinityOperator,
												Values:   []string{"true"},
											},
										},
									},
								},
							},
						},
					},
				},
			},
		},
	}

	job, err = cbc.kubeClient.BatchV1().CreateJob(context.Background(), job)
	cbc.PrometheusOutboundAPICallTotals.With(prometheus.Labels{"target": "kubernetes"}).Inc()

	return
}

// RemoveCiBuilderJob waits for a job to finish and then removes it
func (cbc *ciBuilderClientImpl) RemoveCiBuilderJob(jobName string) (err error) {

	// check if job is finished
	job, err := cbc.kubeClient.BatchV1().GetJob(context.Background(), jobName, cbc.kubeClient.Namespace)
	cbc.PrometheusOutboundAPICallTotals.With(prometheus.Labels{"target": "kubernetes"}).Inc()
	if err != nil {
		log.Error().Err(err).
			Str("jobName", jobName).
			Msgf("GetJob call for job %v failed", jobName)
	}

	if err != nil || *job.Status.Succeeded != 1 {
		log.Debug().
			Str("jobName", jobName).
			Msgf("Job is not done yet, watching for job %v to succeed", jobName)

		// watch for job updates
		watcher, err := cbc.kubeClient.BatchV1().WatchJobs(context.Background(), cbc.kubeClient.Namespace, k8s.Timeout(time.Duration(60)*time.Second))
		cbc.PrometheusOutboundAPICallTotals.With(prometheus.Labels{"target": "kubernetes"}).Inc()
		if err != nil {
			log.Error().Err(err).
				Str("jobName", jobName).
				Msgf("WatchJobs call for job %v failed", jobName)
		} else {
			// wait for job to succeed
			for {
				event, job, err := watcher.Next()
				if err != nil {
					log.Error().Err(err)
					break
				}

				if *event.Type == k8s.EventModified && *job.Metadata.Name == jobName && *job.Status.Succeeded == 1 {
					break
				}
			}
		}
	}
	log.Debug().
		Str("jobName", jobName).
		Msgf("Job %v is done, deleting it...", jobName)

	// delete job
	err = cbc.kubeClient.BatchV1().DeleteJob(context.Background(), jobName, cbc.kubeClient.Namespace)
	cbc.PrometheusOutboundAPICallTotals.With(prometheus.Labels{"target": "kubernetes"}).Inc()
	if err != nil {
		log.Error().Err(err).
			Str("jobName", jobName).
			Msgf("Deleting job %v failed", jobName)
		return
	}

	return
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"net/http"
	"os"
	"time"

	"github.com/cihangir/gene/example/tinder/workers/account"
	"github.com/cihangir/gene/example/tinder/workers/profile"
	stdprometheus "github.com/prometheus/client_golang/prometheus"
	"golang.org/x/net/context"

	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/metrics"
	kitprometheus "github.com/go-kit/kit/metrics/prometheus"
)

func main() {
	var (
		listen = flag.String("listen", ":8080", "HTTP listen address")
	)
	flag.Parse()

	var logger log.Logger
	logger = log.NewLogfmtLogger(os.Stderr)
	logger = log.NewContext(logger).With("listen", *listen).With("caller", log.DefaultCaller)

	ctx := context.Background()

	fieldKeys := []string{"method", "error"}
	requestCount := kitprometheus.NewCounter(stdprometheus.CounterOpts{
		Namespace: "tinder_api",
		Subsystem: "account_service",
		Name:      "request_count",
		Help:      "Number of requests received.",
	}, fieldKeys)

	requestLatency := metrics.NewTimeHistogram(time.Microsecond, kitprometheus.NewSummary(stdprometheus.SummaryOpts{
		Namespace: "tinder_api",
		Subsystem: "account_service",
		Name:      "request_latency_microseconds",
		Help:      "Total duration of requests in microseconds.",
	}, fieldKeys))

	profileApiEndpoints := []string{
		"profile1.tinder_api.tinder.com",
		"profile2.tinder_api.tinder.com",
	}

	maxAttempt := 5
	maxTime := 10 * time.Second
	qps := 100

	profileService := profile.NewProfileClient(profileApiEndpoints, ctx, maxAttempt, maxTime, qps, logger)

	ctx = context.WithValue(ctx, "profileService", profileService)

	var svc account.AccountService
	svc = account.NewAccount()

	byFacebookIDsHandler := account.NewByFacebookIDsHandler(ctx, svc, account.DefaultMiddlewares("byfacebookids", requestCount, requestLatency, logger))
	byIDsHandler := account.NewByIDsHandler(ctx, svc, account.DefaultMiddlewares("byids", requestCount, requestLatency, logger))
	createHandler := account.NewCreateHandler(ctx, svc, account.DefaultMiddlewares("create", requestCount, requestLatency, logger))
	deleteHandler := account.NewDeleteHandler(ctx, svc, account.DefaultMiddlewares("Delete", requestCount, requestLatency, logger))
	oneHandler := account.NewOneHandler(ctx, svc, account.DefaultMiddlewares("One", requestCount, requestLatency, logger))
	updateHandler := account.NewUpdateHandler(ctx, svc, account.DefaultMiddlewares("Update", requestCount, requestLatency, logger))

	http.Handle("/byfacebookids", byFacebookIDsHandler)
	http.Handle("/byids", byIDsHandler)
	http.Handle("/createhandler", createHandler)
	http.Handle("/delete", deleteHandler)
	http.Handle("/one", oneHandler)
	http.Handle("/update", updateHandler)
	http.Handle("/metrics", stdprometheus.Handler())

	_ = logger.Log("msg", "HTTP", "addr", *listen)
	_ = logger.Log("err", http.ListenAndServe(*listen, nil))
}
<commit_msg>Example: updated main package usage of account worker<commit_after>package main

import (
	"flag"
	"net/http"
	"os"
	"time"

	"github.com/cihangir/gene/example/tinder/workers/account"
	jujuratelimit "github.com/juju/ratelimit"
	stdprometheus "github.com/prometheus/client_golang/prometheus"
	"github.com/sony/gobreaker"
	"golang.org/x/net/context"

	"github.com/go-kit/kit/circuitbreaker"
	"github.com/go-kit/kit/endpoint"
	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/metrics"
	kitprometheus "github.com/go-kit/kit/metrics/prometheus"
	kitratelimit "github.com/go-kit/kit/ratelimit"
)

func main() {
	var (
		listen = flag.String("listen", ":8080", "HTTP listen address")
	)
	flag.Parse()

	var logger log.Logger
	logger = log.NewLogfmtLogger(os.Stderr)
	logger = log.NewContext(logger).With("listen", *listen).With("caller", log.DefaultCaller)

	ctx := context.Background()

	fieldKeys := []string{"method", "error"}
	requestCount := kitprometheus.NewCounter(stdprometheus.CounterOpts{
		Namespace: "tinder_api",
		Subsystem: "account_service",
		Name:      "request_count",
		Help:      "Number of requests received.",
	}, fieldKeys)

	requestLatency := metrics.NewTimeHistogram(time.Microsecond, kitprometheus.NewSummary(stdprometheus.SummaryOpts{
		Namespace: "tinder_api",
		Subsystem: "account_service",
		Name:      "request_latency_microseconds",
		Help:      "Total duration of requests in microseconds.",
	}, fieldKeys))

	profileApiEndpoints := []string{
		"profile1.tinder_api.tinder.com",
		"profile2.tinder_api.tinder.com",
	}

	// maxAttempt := 5
	// maxTime := 10 * time.Second
	qps := 100

	m1 := circuitbreaker.Gobreaker(gobreaker.NewCircuitBreaker(gobreaker.Settings{}))
	m2 := kitratelimit.NewTokenBucketLimiter(jujuratelimit.NewBucketWithRate(float64(qps), int64(qps)))

	profileService := account.NewAccountClient(
		profileApiEndpoints,
		logger,
		nil,
		[]endpoint.Middleware{m1, m2},
	)

	ctx = context.WithValue(ctx, "profileService", profileService)

	var svc account.AccountService
	svc = account.NewAccount()

	byFacebookIDsHandler := account.NewByFacebookIDsHandler(ctx, svc, account.DefaultMiddlewares("byfacebookids", requestCount, requestLatency, logger))
	byIDsHandler := account.NewByIDsHandler(ctx, svc, account.DefaultMiddlewares("byids", requestCount, requestLatency, logger))
	createHandler := account.NewCreateHandler(ctx, svc, account.DefaultMiddlewares("create", requestCount, requestLatency, logger))
	deleteHandler := account.NewDeleteHandler(ctx, svc, account.DefaultMiddlewares("Delete", requestCount, requestLatency, logger))
	oneHandler := account.NewOneHandler(ctx, svc, account.DefaultMiddlewares("One", requestCount, requestLatency, logger))
	updateHandler := account.NewUpdateHandler(ctx, svc, account.DefaultMiddlewares("Update", requestCount, requestLatency, logger))

	http.Handle("/byfacebookids", byFacebookIDsHandler)
	http.Handle("/byids", byIDsHandler)
	http.Handle("/createhandler", createHandler)
	http.Handle("/delete", deleteHandler)
	http.Handle("/one", oneHandler)
	http.Handle("/update", updateHandler)
	http.Handle("/metrics", stdprometheus.Handler())

	_ = logger.Log("msg", "HTTP", "addr", *listen)
	_ = logger.Log("err", http.ListenAndServe(*listen, nil))
}
<|endoftext|>


<commit_before>package syslog

import (
	"fmt"
	"math/rand"
	"net"
	"testing"
	"time"
)

const clienthost = "clienthost"

func panicf(s string, i ...interface{}) { panic(fmt.Sprintf(s, i)) }

type testServer struct {
	ln       net.Listener
	Addr     string
	Close    chan bool
	Messages chan string
}

func newTestServer() *testServer {
	server := testServer{
		Close:    make(chan bool, 1),
		Messages: make(chan string, 20),
	}
	server.listen()
	go server.serve()
	return &server
}

func (s *testServer) listen() {
	var err error
	if s.Addr == "" {
		s.ln, err = net.Listen("tcp", "127.0.0.1:0")
	} else {
		s.ln, err = net.Listen("tcp", s.Addr)
	}
	if err != nil {
		panicf("listen error %v", err)
	}
	if s.Addr == "" {
		s.Addr = s.ln.Addr().String()
	}
}

func (s testServer) handle(conn net.Conn) {
	for {
		buf := make([]byte, 1024)
		n, err := conn.Read(buf)
		if err != nil {
			panicf("Read error")
		} else {
			s.Messages <- string(buf[0:n])
		}
		// todo: make configurable
		if 0 == (rand.Int() % 2) {
			conn.Close()
			return
		}
	}
}

func (s testServer) serve() {
	for {
		select {
		case <-s.Close:
			s.ln.Close()
			return
		default:
			conn, err := s.ln.Accept()
			if err != nil {
				panicf("Accept error: %v", err)
			}
			go s.handle(conn)
		}
	}
}

func generatePackets() []Packet {
	packets := make([]Packet, 10)
	for i, _ := range packets {
		t, _ := time.Parse(time.RFC3339, "2006-01-02T15:04:05Z07:00")
		packets[i] = Packet{
			Severity: SevInfo,
			Facility: LogLocal1,
			Time:     t,
			Hostname: clienthost,
			Tag:      "test",
			Message:  fmt.Sprintf("message %d", i),
		}
	}
	return packets
}

func TestSyslog(t *testing.T) {
	s := newTestServer()

	logger, err := Dial(clienthost, "tcp", s.Addr, nil)
	if err != nil {
		t.Errorf("unexpected dial error %v", err)
	}
	packets := generatePackets()
	for _, p := range packets {
		logger.writePacket(p)
		time.Sleep(100 * time.Millisecond)
	}
	s.Close <- true

	for _, p := range packets {
		expected := p.Generate(0) + "\n"
		select {
		case got := <-s.Messages:
			if got != expected {
				t.Errorf("expected %s, got %s", expected, got)
			}
		default:
			t.Errorf("expected %s, got nothing", expected)
			break
		}
	}
	if l := len(s.Messages); l != 0 {
		t.Errorf("found %d extra messages", l)
	}
}
<commit_msg>Test UDP reconnects<commit_after>package syslog

import (
	"fmt"
	"io"
	"math/rand"
	"net"
	"testing"
	"time"
)

const clienthost = "clienthost"

func panicf(s string, i ...interface{}) { panic(fmt.Sprintf(s, i)) }

type testServer struct {
	Addr     string
	Close    chan bool
	Messages chan string
}

func newTestServer(network string) *testServer {
	server := testServer{
		Close:    make(chan bool, 1),
		Messages: make(chan string, 20),
	}
	switch network {
	case "tcp":
		ln := server.listenTCP()
		go server.serveTCP(ln)
	case "udp":
		conn := server.listenUDP()
		go server.serveUDP(conn)
	}
	return &server
}

func (s *testServer) listenTCP() net.Listener {
	addr := s.Addr
	if addr == "" {
		addr = "127.0.0.1:0"
	}
	ln, err := net.Listen("tcp", addr)
	if err != nil {
		panicf("listen error %v", err)
	}
	if s.Addr == "" {
		s.Addr = ln.Addr().String()
	}
	return ln
}

func (s *testServer) serveTCP(ln net.Listener) {
	for {
		select {
		case <-s.Close:
			ln.Close()
			return
		default:
			conn, err := ln.Accept()
			if err != nil {
				panicf("Accept error: %v", err)
			}
			go handle(conn, s.Messages)
		}
	}
}

func (s *testServer) listenUDP() *net.UDPConn {
	addr := s.Addr
	if addr == "" {
		addr = "127.0.0.1:0"
	}
	udpAddr, err := net.ResolveUDPAddr("udp", addr)
	if err != nil {
		panicf("unexpected error %v", err)
	}

	conn, err := net.ListenUDP("udp", udpAddr)
	if err != nil {
		panicf("listen error %v", err)
	}
	if s.Addr == "" {
		s.Addr = conn.LocalAddr().String()
	}
	return conn
}

func (s *testServer) serveUDP(conn *net.UDPConn) {
	for {
		handle(conn, s.Messages)
		conn = s.listenUDP()
	}
}

func handle(conn io.ReadCloser, messages chan string) {
	for {
		fmt.Println("handle")
		buf := make([]byte, 1024)
		n, err := conn.Read(buf)
		if err != nil {
			panicf("Read error")
		} else {
			fmt.Println("handle", string(buf[0:n]))
			messages <- string(buf[0:n])
		}
		// todo: make configurable
		if 0 == (rand.Int() % 2) {
			fmt.Println("closing")
			conn.Close()
			return
		}
	}
}

func generatePackets() []Packet {
	packets := make([]Packet, 10)
	for i, _ := range packets {
		t, _ := time.Parse(time.RFC3339, "2006-01-02T15:04:05Z07:00")
		packets[i] = Packet{
			Severity: SevInfo,
			Facility: LogLocal1,
			Time:     t,
			Hostname: clienthost,
			Tag:      "test",
			Message:  fmt.Sprintf("message %d", i),
		}
	}
	return packets
}

func TestSyslog(t *testing.T) {
	for _, network := range []string{"tcp", "udp"} {
		s := newTestServer(network)

		logger, err := Dial(clienthost, network, s.Addr, nil)
		if err != nil {
			t.Errorf("unexpected dial error %v", err)
		}
		packets := generatePackets()
		for _, p := range packets {
			logger.writePacket(p)
			time.Sleep(100 * time.Millisecond)
		}
		s.Close <- true

		for _, p := range packets {
			expected := p.Generate(0)
			if network == "tcp" {
				expected = expected + "\n"
			}
			select {
			case got := <-s.Messages:
				if got != expected {
					t.Errorf("expected %s, got %s", expected, got)
				}
			default:
				t.Errorf("expected %s, got nothing", expected)
				break
			}
		}
		if l := len(s.Messages); l != 0 {
			t.Errorf("found %d extra messages", l)
		}
	}
}
<|endoftext|>