<commit_before>package s3api

import (
	"context"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
	"strings"
)

func (s3a *S3ApiServer) mkdir(parentDirectoryPath string, dirName string, fn func(entry *filer_pb.Entry)) error {

	return filer_pb.Mkdir(s3a, parentDirectoryPath, dirName, fn)

}

func (s3a *S3ApiServer) mkFile(parentDirectoryPath string, fileName string, chunks []*filer_pb.FileChunk, fn func(entry *filer_pb.Entry)) error {

	return filer_pb.MkFile(s3a, parentDirectoryPath, fileName, chunks, fn)

}

func (s3a *S3ApiServer) list(parentDirectoryPath, prefix, startFrom string, inclusive bool, limit uint32) (entries []*filer_pb.Entry, isLast bool, err error) {

	err = filer_pb.List(s3a, parentDirectoryPath, prefix, func(entry *filer_pb.Entry, isLastEntry bool) error {
		entries = append(entries, entry)
		if isLastEntry {
			isLast = true
		}
		return nil
	}, startFrom, inclusive, limit)

	if len(entries) == 0 {
		isLast = true
	}

	return

}

func (s3a *S3ApiServer) rm(parentDirectoryPath, entryName string, isDeleteData, isRecursive bool) error {

	return s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {

		err := doDeleteEntry(client, parentDirectoryPath, entryName, isDeleteData, isRecursive)
		if err != nil {
			return err
		}

		return nil
	})

}

func doDeleteEntry(client filer_pb.SeaweedFilerClient, parentDirectoryPath string, entryName string, isDeleteData bool, isRecursive bool) error {
	request := &filer_pb.DeleteEntryRequest{
		Directory:    parentDirectoryPath,
		Name:         entryName,
		IsDeleteData: isDeleteData,
		IsRecursive:  isRecursive,
	}

	glog.V(1).Infof("delete entry %v/%v: %v", parentDirectoryPath, entryName, request)
	if resp, err := client.DeleteEntry(context.Background(), request); err != nil {
		glog.V(0).Infof("delete entry %v: %v", request, err)
		return fmt.Errorf("delete entry %s/%s: %v", parentDirectoryPath, entryName, err)
	} else {
		if resp.Error != "" {
			return fmt.Errorf("delete entry %s/%s: %v", parentDirectoryPath, entryName, resp.Error)
		}
	}
	return nil
}

func (s3a *S3ApiServer) exists(parentDirectoryPath string, entryName string, isDirectory bool) (exists bool, err error) {

	return filer_pb.Exists(s3a, parentDirectoryPath, entryName, isDirectory)

}

func (s3a *S3ApiServer) touch(parentDirectoryPath string, entryName string, entry *filer_pb.Entry) (err error) {

	return filer_pb.Touch(s3a, parentDirectoryPath, entryName, entry)

}

func (s3a *S3ApiServer) getEntry(parentDirectoryPath, entryName string) (entry *filer_pb.Entry, err error) {
	fullPath := util.NewFullPath(parentDirectoryPath, entryName)
	return filer_pb.GetEntry(s3a, fullPath)
}

func objectKey(key *string) *string {
	if strings.HasPrefix(*key, "/") {
		t := (*key)[1:]
		return &t
	}
	return key
}
<commit_msg>better clean up upload directory<commit_after>package s3api

import (
	"context"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
	"strings"
)

func (s3a *S3ApiServer) mkdir(parentDirectoryPath string, dirName string, fn func(entry *filer_pb.Entry)) error {

	return filer_pb.Mkdir(s3a, parentDirectoryPath, dirName, fn)

}

func (s3a *S3ApiServer) mkFile(parentDirectoryPath string, fileName string, chunks []*filer_pb.FileChunk, fn func(entry *filer_pb.Entry)) error {

	return filer_pb.MkFile(s3a, parentDirectoryPath, fileName, chunks, fn)

}

func (s3a *S3ApiServer) list(parentDirectoryPath, prefix, startFrom string, inclusive bool, limit uint32) (entries []*filer_pb.Entry, isLast bool, err error) {

	err = filer_pb.List(s3a, parentDirectoryPath, prefix, func(entry *filer_pb.Entry, isLastEntry bool) error {
		entries = append(entries, entry)
		if isLastEntry {
			isLast = true
		}
		return nil
	}, startFrom, inclusive, limit)

	if len(entries) == 0 {
		isLast = true
	}

	return

}

func (s3a *S3ApiServer) rm(parentDirectoryPath, entryName string, isDeleteData, isRecursive bool) error {

	return s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {

		err := doDeleteEntry(client, parentDirectoryPath, entryName, isDeleteData, isRecursive)
		if err != nil {
			return err
		}

		return nil
	})

}

func doDeleteEntry(client filer_pb.SeaweedFilerClient, parentDirectoryPath string, entryName string, isDeleteData bool, isRecursive bool) error {
	request := &filer_pb.DeleteEntryRequest{
		Directory:            parentDirectoryPath,
		Name:                 entryName,
		IsDeleteData:         isDeleteData,
		IsRecursive:          isRecursive,
		IgnoreRecursiveError: true,
	}

	glog.V(1).Infof("delete entry %v/%v: %v", parentDirectoryPath, entryName, request)
	if resp, err := client.DeleteEntry(context.Background(), request); err != nil {
		glog.V(0).Infof("delete entry %v: %v", request, err)
		return fmt.Errorf("delete entry %s/%s: %v", parentDirectoryPath, entryName, err)
	} else {
		if resp.Error != "" {
			return fmt.Errorf("delete entry %s/%s: %v", parentDirectoryPath, entryName, resp.Error)
		}
	}
	return nil
}

func (s3a *S3ApiServer) exists(parentDirectoryPath string, entryName string, isDirectory bool) (exists bool, err error) {

	return filer_pb.Exists(s3a, parentDirectoryPath, entryName, isDirectory)

}

func (s3a *S3ApiServer) touch(parentDirectoryPath string, entryName string, entry *filer_pb.Entry) (err error) {

	return filer_pb.Touch(s3a, parentDirectoryPath, entryName, entry)

}

func (s3a *S3ApiServer) getEntry(parentDirectoryPath, entryName string) (entry *filer_pb.Entry, err error) {
	fullPath := util.NewFullPath(parentDirectoryPath, entryName)
	return filer_pb.GetEntry(s3a, fullPath)
}

func objectKey(key *string) *string {
	if strings.HasPrefix(*key, "/") {
		t := (*key)[1:]
		return &t
	}
	return key
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cache

import (
	"fmt"
	"os"
	"reflect"
	"strconv"
	"sync"
	"time"

	"k8s.io/klog"

	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/diff"
)

var mutationDetectionEnabled = false

func init() {
	mutationDetectionEnabled, _ = strconv.ParseBool(os.Getenv("KUBE_CACHE_MUTATION_DETECTOR"))
}

type CacheMutationDetector interface {
	AddObject(obj interface{})
	Run(stopCh <-chan struct{})
}

func NewCacheMutationDetector(name string) CacheMutationDetector {
	if !mutationDetectionEnabled {
		return dummyMutationDetector{}
	}
	klog.Warningln("Mutation detector is enabled, this will result in memory leakage.")
	return &defaultCacheMutationDetector{name: name, period: 1 * time.Second}
}

type dummyMutationDetector struct{}

func (dummyMutationDetector) Run(stopCh <-chan struct{}) {
}
func (dummyMutationDetector) AddObject(obj interface{}) {
}

// defaultCacheMutationDetector gives a way to detect if a cached object has been mutated
// It has a list of cached objects and their copies.  I haven't thought of a way
// to see WHO is mutating it, just that it's getting mutated.
type defaultCacheMutationDetector struct {
	name   string
	period time.Duration

	lock       sync.Mutex
	cachedObjs []cacheObj

	// failureFunc is injectable for unit testing.  If you don't have it, the process will panic.
	// This panic is intentional, since turning on this detection indicates you want a strong
	// failure signal.  This failure is effectively a p0 bug and you can't trust process results
	// after a mutation anyway.
	failureFunc func(message string)
}

// cacheObj holds the actual object and a copy
type cacheObj struct {
	cached interface{}
	copied interface{}
}

func (d *defaultCacheMutationDetector) Run(stopCh <-chan struct{}) {
	// we DON'T want protection from panics.  If we're running this code, we want to die
	for {
		d.CompareObjects()

		select {
		case <-stopCh:
			return
		case <-time.After(d.period):
		}
	}
}

// AddObject makes a deep copy of the object for later comparison.  It only works on runtime.Object
// but that covers the vast majority of our cached objects
func (d *defaultCacheMutationDetector) AddObject(obj interface{}) {
	if _, ok := obj.(DeletedFinalStateUnknown); ok {
		return
	}
	if obj, ok := obj.(runtime.Object); ok {
		copiedObj := obj.DeepCopyObject()

		d.lock.Lock()
		defer d.lock.Unlock()
		d.cachedObjs = append(d.cachedObjs, cacheObj{cached: obj, copied: copiedObj})
	}
}

func (d *defaultCacheMutationDetector) CompareObjects() {
	d.lock.Lock()
	defer d.lock.Unlock()

	altered := false
	for i, obj := range d.cachedObjs {
		if !reflect.DeepEqual(obj.cached, obj.copied) {
			fmt.Printf("CACHE %s[%d] ALTERED!\n%v\n", d.name, i, diff.ObjectDiff(obj.cached, obj.copied))
			altered = true
		}
	}

	if altered {
		msg := fmt.Sprintf("cache %s modified", d.name)
		if d.failureFunc != nil {
			d.failureFunc(msg)
			return
		}
		panic(msg)
	}
}
<commit_msg>cache mutation detector: use correct diff function<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cache

import (
	"fmt"
	"os"
	"reflect"
	"strconv"
	"sync"
	"time"

	"k8s.io/klog"

	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/diff"
)

var mutationDetectionEnabled = false

func init() {
	mutationDetectionEnabled, _ = strconv.ParseBool(os.Getenv("KUBE_CACHE_MUTATION_DETECTOR"))
}

type CacheMutationDetector interface {
	AddObject(obj interface{})
	Run(stopCh <-chan struct{})
}

func NewCacheMutationDetector(name string) CacheMutationDetector {
	if !mutationDetectionEnabled {
		return dummyMutationDetector{}
	}
	klog.Warningln("Mutation detector is enabled, this will result in memory leakage.")
	return &defaultCacheMutationDetector{name: name, period: 1 * time.Second}
}

type dummyMutationDetector struct{}

func (dummyMutationDetector) Run(stopCh <-chan struct{}) {
}
func (dummyMutationDetector) AddObject(obj interface{}) {
}

// defaultCacheMutationDetector gives a way to detect if a cached object has been mutated
// It has a list of cached objects and their copies.  I haven't thought of a way
// to see WHO is mutating it, just that it's getting mutated.
type defaultCacheMutationDetector struct {
	name   string
	period time.Duration

	lock       sync.Mutex
	cachedObjs []cacheObj

	// failureFunc is injectable for unit testing.  If you don't have it, the process will panic.
	// This panic is intentional, since turning on this detection indicates you want a strong
	// failure signal.  This failure is effectively a p0 bug and you can't trust process results
	// after a mutation anyway.
	failureFunc func(message string)
}

// cacheObj holds the actual object and a copy
type cacheObj struct {
	cached interface{}
	copied interface{}
}

func (d *defaultCacheMutationDetector) Run(stopCh <-chan struct{}) {
	// we DON'T want protection from panics.  If we're running this code, we want to die
	for {
		d.CompareObjects()

		select {
		case <-stopCh:
			return
		case <-time.After(d.period):
		}
	}
}

// AddObject makes a deep copy of the object for later comparison.  It only works on runtime.Object
// but that covers the vast majority of our cached objects
func (d *defaultCacheMutationDetector) AddObject(obj interface{}) {
	if _, ok := obj.(DeletedFinalStateUnknown); ok {
		return
	}
	if obj, ok := obj.(runtime.Object); ok {
		copiedObj := obj.DeepCopyObject()

		d.lock.Lock()
		defer d.lock.Unlock()
		d.cachedObjs = append(d.cachedObjs, cacheObj{cached: obj, copied: copiedObj})
	}
}

func (d *defaultCacheMutationDetector) CompareObjects() {
	d.lock.Lock()
	defer d.lock.Unlock()

	altered := false
	for i, obj := range d.cachedObjs {
		if !reflect.DeepEqual(obj.cached, obj.copied) {
			fmt.Printf("CACHE %s[%d] ALTERED!\n%v\n", d.name, i, diff.ObjectGoPrintSideBySide(obj.cached, obj.copied))
			altered = true
		}
	}

	if altered {
		msg := fmt.Sprintf("cache %s modified", d.name)
		if d.failureFunc != nil {
			d.failureFunc(msg)
			return
		}
		panic(msg)
	}
}
<|endoftext|>


<commit_before>package morningStar

import (
	"../jsonHttp"
	"bytes"
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"regexp"
	"strconv"
	"sync"
	"time"
)

const PERFORMANCE_URL = `http://www.morningstar.fr/fr/funds/snapshot/snapshot.aspx?tab=1&id=`
const VOLATILITE_URL = `http://www.morningstar.fr/fr/funds/snapshot/snapshot.aspx?tab=2&id=`
const REFRESH_DELAY = 18

var LIST_REQUEST = regexp.MustCompile(`^/list$`)
var PERF_REQUEST = regexp.MustCompile(`^/(.+?)$`)

var ISIN = regexp.MustCompile(`ISIN.:(\S+)`)
var LABEL = regexp.MustCompile(`<h1[^>]*?>((?:.|\n)*?)</h1>`)
var RATING = regexp.MustCompile(`<span\sclass=".*?stars([0-9]).*?">`)
var CATEGORY = regexp.MustCompile(`<span[^>]*?>Catégorie</span>.*?<span[^>]*?>(.*?)</span>`)
var PERF_ONE_MONTH = regexp.MustCompile(`<td[^>]*?>1 mois</td><td[^>]*?>(.*?)</td>`)
var PERF_THREE_MONTH = regexp.MustCompile(`<td[^>]*?>3 mois</td><td[^>]*?>(.*?)</td>`)
var PERF_SIX_MONTH = regexp.MustCompile(`<td[^>]*?>6 mois</td><td[^>]*?>(.*?)</td>`)
var PERF_ONE_YEAR = regexp.MustCompile(`<td[^>]*?>1 an</td><td[^>]*?>(.*?)</td>`)
var VOL_3_YEAR = regexp.MustCompile(`<td[^>]*?>Ecart-type 3 ans.?</td><td[^>]*?>(.*?)</td>`)

type SyncedMap struct {
	sync.RWMutex
	performances map[string]Performance
}

func (m *SyncedMap) get(key string) (Performance, bool) {
	m.RLock()
	defer m.RUnlock()

	performance, ok := m.performances[key]
	return performance, ok
}

func (m *SyncedMap) push(key string, performance Performance) {
	m.Lock()
	defer m.Unlock()
	m.performances[key] = performance
}

var PERFORMANCE_CACHE = SyncedMap{performances: make(map[string]Performance)}

type Performance struct {
	Id            string    `json:"id"`
	Isin          string    `json:"isin"`
	Label         string    `json:"label"`
	Category      string    `json:"category"`
	Rating        string    `json:"rating"`
	OneMonth      float64   `json:"1m"`
	ThreeMonth    float64   `json:"3m"`
	SixMonth      float64   `json:"6m"`
	OneYear       float64   `json:"1y"`
	VolThreeYears float64   `json:"v3y"`
	Score         float64   `json:"score"`
	Update        time.Time `json:"ts"`
}

type Search struct {
	Id    string `json:"i"`
	Label string `json:"n"`
}

type Results struct {
	Results interface{} `json:"results"`
}

func readBody(body io.ReadCloser) ([]byte, error) {
	defer body.Close()
	return ioutil.ReadAll(body)
}

func getBody(url string) ([]byte, error) {
	response, err := http.Get(url)
	if err != nil {
		return nil, errors.New(`Error while retrieving data from ` + url)
	}

	if response.StatusCode >= 400 {
		return nil, errors.New(`Got error ` + strconv.Itoa(response.StatusCode) + ` while getting ` + url)
	}

	body, err := readBody(response.Body)
	if err != nil {
		return nil, errors.New(`Error while reading body of ` + url)
	}

	return body, nil
}

func getLabel(extract *regexp.Regexp, body []byte, defaultValue []byte) []byte {
	match := extract.FindSubmatch(body)
	if match == nil {
		return defaultValue
	}

	return bytes.Replace(match[1], []byte(`&`), []byte(`&`), -1)
}

func getPerformance(extract *regexp.Regexp, body []byte) float64 {
	dotResult := bytes.Replace(getLabel(extract, body, []byte(``)), []byte(`,`), []byte(`.`), -1)
	percentageResult := bytes.Replace(dotResult, []byte(`%`), []byte(``), -1)
	trimResult := bytes.TrimSpace(percentageResult)

	result, err := strconv.ParseFloat(string(trimResult), 64)
	if err != nil {
		return 0.0
	}
	return result
}

func SinglePerformance(morningStarId []byte) (*Performance, error) {
	cleanId := string(bytes.ToLower(morningStarId))

	performance, ok := PERFORMANCE_CACHE.get(cleanId)

	if ok && time.Now().Add(time.Hour*-REFRESH_DELAY).Before(performance.Update) {
		return &performance, nil
	}

	performanceBody, err := getBody(PERFORMANCE_URL + cleanId)
	if err != nil {
		return nil, err
	}

	volatiliteBody, err := getBody(VOLATILITE_URL + cleanId)
	if err != nil {
		return nil, err
	}

	isin := string(getLabel(ISIN, performanceBody, []byte(``)))
	label := string(getLabel(LABEL, performanceBody, []byte(``)))
	rating := string(getLabel(RATING, performanceBody, []byte(`0`)))
	category := string(getLabel(CATEGORY, performanceBody, []byte(``)))
	oneMonth := getPerformance(PERF_ONE_MONTH, performanceBody)
	threeMonths := getPerformance(PERF_THREE_MONTH, performanceBody)
	sixMonths := getPerformance(PERF_SIX_MONTH, performanceBody)
	oneYear := getPerformance(PERF_ONE_YEAR, performanceBody)
	volThreeYears := getPerformance(VOL_3_YEAR, volatiliteBody)

	score := (0.25 * oneMonth) + (0.3 * threeMonths) + (0.25 * sixMonths) + (0.2 * oneYear) - (0.1 * volThreeYears)
	scoreTruncated := float64(int(score*100)) / 100

	performance = Performance{cleanId, isin, label, category, rating, oneMonth, threeMonths, sixMonths, oneYear, volThreeYears, scoreTruncated, time.Now()}

	PERFORMANCE_CACHE.push(cleanId, performance)

	return &performance, nil
}

func singlePerformanceAsync(morningStarId []byte, ch chan<- Performance) {
	performance, err := SinglePerformance(morningStarId)
	if err == nil {
		ch <- *performance
	}
}

func singlePerformanceHandler(w http.ResponseWriter, morningStarId []byte) {
	performance, err := SinglePerformance(morningStarId)

	if err != nil {
		http.Error(w, err.Error(), 500)
	} else {
		jsonHttp.ResponseJson(w, *performance)
	}
}

func listHandler(w http.ResponseWriter, r *http.Request) {
	listBody, err := readBody(r.Body)
	if err != nil {
		http.Error(w, `Error while reading body for list`, 500)
		return
	}

	if len(bytes.TrimSpace(listBody)) == 0 {
		jsonHttp.ResponseJson(w, Results{[0]Performance{}})
		return
	}

	ids := bytes.Split(listBody, []byte(`,`))
	size := len(ids)

	performances := make(chan Performance, size)
	for _, id := range ids {
		go singlePerformanceAsync(id, performances)
	}

	results := make([]Performance, 0, size)
	for performance, ok := range performances {
		if ok {
			results = append(results, performance)
		}
	}

	jsonHttp.ResponseJson(w, Results{results})
}

type Handler struct {
}

func (handler Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	w.Header().Add(`Access-Control-Allow-Origin`, `*`)
	w.Header().Add(`Access-Control-Allow-Headers`, `Content-Type`)
	w.Header().Add(`Access-Control-Allow-Methods`, `GET, POST`)
	w.Header().Add(`X-Content-Type-Options`, `nosniff`)

	urlPath := []byte(r.URL.Path)

	if LIST_REQUEST.Match(urlPath) {
		listHandler(w, r)
	} else if PERF_REQUEST.Match(urlPath) {
		singlePerformanceHandler(w, PERF_REQUEST.FindSubmatch(urlPath)[1])
	}
}
<commit_msg>Update morningStar.go<commit_after>package morningStar

import (
	"../jsonHttp"
	"bytes"
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"regexp"
	"strconv"
	"sync"
	"time"
)

const PERFORMANCE_URL = `http://www.morningstar.fr/fr/funds/snapshot/snapshot.aspx?tab=1&id=`
const VOLATILITE_URL = `http://www.morningstar.fr/fr/funds/snapshot/snapshot.aspx?tab=2&id=`
const REFRESH_DELAY = 18

var LIST_REQUEST = regexp.MustCompile(`^/list$`)
var PERF_REQUEST = regexp.MustCompile(`^/(.+?)$`)

var ISIN = regexp.MustCompile(`ISIN.:(\S+)`)
var LABEL = regexp.MustCompile(`<h1[^>]*?>((?:.|\n)*?)</h1>`)
var RATING = regexp.MustCompile(`<span\sclass=".*?stars([0-9]).*?">`)
var CATEGORY = regexp.MustCompile(`<span[^>]*?>Catégorie</span>.*?<span[^>]*?>(.*?)</span>`)
var PERF_ONE_MONTH = regexp.MustCompile(`<td[^>]*?>1 mois</td><td[^>]*?>(.*?)</td>`)
var PERF_THREE_MONTH = regexp.MustCompile(`<td[^>]*?>3 mois</td><td[^>]*?>(.*?)</td>`)
var PERF_SIX_MONTH = regexp.MustCompile(`<td[^>]*?>6 mois</td><td[^>]*?>(.*?)</td>`)
var PERF_ONE_YEAR = regexp.MustCompile(`<td[^>]*?>1 an</td><td[^>]*?>(.*?)</td>`)
var VOL_3_YEAR = regexp.MustCompile(`<td[^>]*?>Ecart-type 3 ans.?</td><td[^>]*?>(.*?)</td>`)

type SyncedMap struct {
	sync.RWMutex
	performances map[string]Performance
}

func (m *SyncedMap) get(key string) (Performance, bool) {
	m.RLock()
	defer m.RUnlock()

	performance, ok := m.performances[key]
	return performance, ok
}

func (m *SyncedMap) push(key string, performance Performance) {
	m.Lock()
	defer m.Unlock()
	m.performances[key] = performance
}

var PERFORMANCE_CACHE = SyncedMap{performances: make(map[string]Performance)}

type Performance struct {
	Id            string    `json:"id"`
	Isin          string    `json:"isin"`
	Label         string    `json:"label"`
	Category      string    `json:"category"`
	Rating        string    `json:"rating"`
	OneMonth      float64   `json:"1m"`
	ThreeMonth    float64   `json:"3m"`
	SixMonth      float64   `json:"6m"`
	OneYear       float64   `json:"1y"`
	VolThreeYears float64   `json:"v3y"`
	Score         float64   `json:"score"`
	Update        time.Time `json:"ts"`
}

type Search struct {
	Id    string `json:"i"`
	Label string `json:"n"`
}

type Results struct {
	Results interface{} `json:"results"`
}

func readBody(body io.ReadCloser) ([]byte, error) {
	defer body.Close()
	return ioutil.ReadAll(body)
}

func getBody(url string) ([]byte, error) {
	response, err := http.Get(url)
	if err != nil {
		return nil, errors.New(`Error while retrieving data from ` + url)
	}

	if response.StatusCode >= 400 {
		return nil, errors.New(`Got error ` + strconv.Itoa(response.StatusCode) + ` while getting ` + url)
	}

	body, err := readBody(response.Body)
	if err != nil {
		return nil, errors.New(`Error while reading body of ` + url)
	}

	return body, nil
}

func getLabel(extract *regexp.Regexp, body []byte, defaultValue []byte) []byte {
	match := extract.FindSubmatch(body)
	if match == nil {
		return defaultValue
	}

	return bytes.Replace(match[1], []byte(`&`), []byte(`&`), -1)
}

func getPerformance(extract *regexp.Regexp, body []byte) float64 {
	dotResult := bytes.Replace(getLabel(extract, body, []byte(``)), []byte(`,`), []byte(`.`), -1)
	percentageResult := bytes.Replace(dotResult, []byte(`%`), []byte(``), -1)
	trimResult := bytes.TrimSpace(percentageResult)

	result, err := strconv.ParseFloat(string(trimResult), 64)
	if err != nil {
		return 0.0
	}
	return result
}

func SinglePerformance(morningStarId []byte) (*Performance, error) {
	cleanId := string(bytes.ToLower(morningStarId))

	performance, ok := PERFORMANCE_CACHE.get(cleanId)

	if ok && time.Now().Add(time.Hour*-REFRESH_DELAY).Before(performance.Update) {
		return &performance, nil
	}

	performanceBody, err := getBody(PERFORMANCE_URL + cleanId)
	if err != nil {
		return nil, err
	}

	volatiliteBody, err := getBody(VOLATILITE_URL + cleanId)
	if err != nil {
		return nil, err
	}

	isin := string(getLabel(ISIN, performanceBody, []byte(``)))
	label := string(getLabel(LABEL, performanceBody, []byte(``)))
	rating := string(getLabel(RATING, performanceBody, []byte(`0`)))
	category := string(getLabel(CATEGORY, performanceBody, []byte(``)))
	oneMonth := getPerformance(PERF_ONE_MONTH, performanceBody)
	threeMonths := getPerformance(PERF_THREE_MONTH, performanceBody)
	sixMonths := getPerformance(PERF_SIX_MONTH, performanceBody)
	oneYear := getPerformance(PERF_ONE_YEAR, performanceBody)
	volThreeYears := getPerformance(VOL_3_YEAR, volatiliteBody)

	score := (0.25 * oneMonth) + (0.3 * threeMonths) + (0.25 * sixMonths) + (0.2 * oneYear) - (0.1 * volThreeYears)
	scoreTruncated := float64(int(score*100)) / 100

	performance = Performance{cleanId, isin, label, category, rating, oneMonth, threeMonths, sixMonths, oneYear, volThreeYears, scoreTruncated, time.Now()}

	PERFORMANCE_CACHE.push(cleanId, performance)

	return &performance, nil
}

func singlePerformanceAsync(morningStarId []byte, ch chan<- Performance) {
	performance, err := SinglePerformance(morningStarId)
	if err == nil {
		ch <- *performance
	}
}

func singlePerformanceHandler(w http.ResponseWriter, morningStarId []byte) {
	performance, err := SinglePerformance(morningStarId)

	if err != nil {
		http.Error(w, err.Error(), 500)
	} else {
		jsonHttp.ResponseJson(w, *performance)
	}
}

func listHandler(w http.ResponseWriter, r *http.Request) {
	listBody, err := readBody(r.Body)
	if err != nil {
		http.Error(w, `Error while reading body for list`, 500)
		return
	}

	if len(bytes.TrimSpace(listBody)) == 0 {
		jsonHttp.ResponseJson(w, Results{[0]Performance{}})
		return
	}

	ids := bytes.Split(listBody, []byte(`,`))
	size := len(ids)

	performances := make(chan Performance, size)
	for _, id := range ids {
		go singlePerformanceAsync(id, performances)
	}

	results := make([]Performance, 0, size)
	for performance := range performances {
		results = append(results, performance)
	}

	jsonHttp.ResponseJson(w, Results{results})
}

type Handler struct {
}

func (handler Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	w.Header().Add(`Access-Control-Allow-Origin`, `*`)
	w.Header().Add(`Access-Control-Allow-Headers`, `Content-Type`)
	w.Header().Add(`Access-Control-Allow-Methods`, `GET, POST`)
	w.Header().Add(`X-Content-Type-Options`, `nosniff`)

	urlPath := []byte(r.URL.Path)

	if LIST_REQUEST.Match(urlPath) {
		listHandler(w, r)
	} else if PERF_REQUEST.Match(urlPath) {
		singlePerformanceHandler(w, PERF_REQUEST.FindSubmatch(urlPath)[1])
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2018 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package writer

import (
	"github.com/m3db/m3/src/msg/generated/proto/msgpb"
	"github.com/m3db/m3/src/msg/producer"
	"github.com/m3db/m3/src/msg/protocol/proto"

	"go.uber.org/atomic"
)

type message struct {
	*producer.RefCountedMessage

	pb                     msgpb.Message
	meta                   metadata
	initNanos              int64
	retryAtNanos           int64
	expectedProcessAtNanos int64
	retried                int
	// NB(cw) isAcked could be accessed concurrently by the background thread
	// in message writer and acked by consumer service writers.
	// Safe to store value inside struct, as message is never copied by value
	isAcked atomic.Bool
}

func newMessage() *message {
	return &message{
		retryAtNanos: 0,
		retried:      0,
	}
}

// Set sets the message.
func (m *message) Set(meta metadata, rm *producer.RefCountedMessage, initNanos int64) {
	m.initNanos = initNanos
	m.meta = meta
	m.RefCountedMessage = rm
	m.ToProto(&m.pb)
}

// Close resets the states of the message.
func (m *message) Close() {
	m.retryAtNanos = 0
	m.retried = 0
	m.isAcked.Store(false)
	m.ResetProto(&m.pb)
}

// InitNanos returns the nanosecond when the message was initiated.
func (m *message) InitNanos() int64 {
	return m.initNanos
}

// ExpectedProcessAtNanos returns the nanosecond when the message should be processed. Used to calculate processing lag
// in the system.
func (m *message) ExpectedProcessAtNanos() int64 {
	return m.expectedProcessAtNanos
}

// RetryAtNanos returns the timestamp for next retry in nano seconds.
func (m *message) RetryAtNanos() int64 {
	return m.retryAtNanos
}

// SetRetryAtNanos sets the next retry nanos.
func (m *message) SetRetryAtNanos(value int64) {
	if m.retryAtNanos > 0 {
		m.expectedProcessAtNanos = m.retryAtNanos
	} else {
		m.expectedProcessAtNanos = m.initNanos
	}
	m.retryAtNanos = value
}

// WriteTimes returns the times the message has been written.
func (m *message) WriteTimes() int {
	return m.retried
}

// IncWriteTimes increments the times the message has been written.
func (m *message) IncWriteTimes() {
	m.retried++
}

// IsAcked returns true if the message has been acked.
func (m *message) IsAcked() bool {
	return m.isAcked.Load()
}

// Ack acknowledges the message. Duplicated acks on the same message might cause panic.
func (m *message) Ack() {
	m.isAcked.Store(true)
	m.RefCountedMessage.DecRef()
}

func (m *message) ShardID() uint64 {
	return m.meta.shard
}

// Metadata returns the metadata.
func (m *message) Metadata() metadata {
	return m.meta
}

// SetSentAt sets the sentAtNanos on the metadata proto.
func (m *message) SetSentAt(nanos int64) {
	m.pb.Metadata.SentAtNanos = uint64(nanos)
}

// Marshaler returns the marshaler and a bool to indicate whether the marshaler is valid.
func (m *message) Marshaler() (proto.Marshaler, bool) {
	return &m.pb, !m.RefCountedMessage.IsDroppedOrConsumed()
}

func (m *message) ToProto(pb *msgpb.Message) {
	m.meta.ToProto(&pb.Metadata)
	pb.Value = m.RefCountedMessage.Bytes()
}

func (m *message) ResetProto(pb *msgpb.Message) {
	pb.Value = nil
}
<commit_msg>[m3msg] Use atomic value for expected processed at (#3994)<commit_after>// Copyright (c) 2018 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package writer

import (
	"github.com/m3db/m3/src/msg/generated/proto/msgpb"
	"github.com/m3db/m3/src/msg/producer"
	"github.com/m3db/m3/src/msg/protocol/proto"

	"go.uber.org/atomic"
)

type message struct {
	*producer.RefCountedMessage

	pb           msgpb.Message
	meta         metadata
	initNanos    int64
	retryAtNanos int64
	// updated by the writing goroutine and read by the acking goroutine.
	expectedProcessAtNanos atomic.Int64
	retried                int
	// NB(cw) isAcked could be accessed concurrently by the background thread
	// in message writer and acked by consumer service writers.
	// Safe to store value inside struct, as message is never copied by value
	isAcked atomic.Bool
}

func newMessage() *message {
	return &message{
		retryAtNanos: 0,
		retried:      0,
	}
}

// Set sets the message.
func (m *message) Set(meta metadata, rm *producer.RefCountedMessage, initNanos int64) {
	m.initNanos = initNanos
	m.meta = meta
	m.RefCountedMessage = rm
	m.ToProto(&m.pb)
}

// Close resets the states of the message.
func (m *message) Close() {
	m.retryAtNanos = 0
	m.retried = 0
	m.isAcked.Store(false)
	m.ResetProto(&m.pb)
}

// InitNanos returns the nanosecond when the message was initiated.
func (m *message) InitNanos() int64 {
	return m.initNanos
}

// ExpectedProcessAtNanos returns the nanosecond when the message should be processed. Used to calculate processing lag
// in the system.
func (m *message) ExpectedProcessAtNanos() int64 {
	return m.expectedProcessAtNanos.Load()
}

// RetryAtNanos returns the timestamp for next retry in nano seconds.
func (m *message) RetryAtNanos() int64 {
	return m.retryAtNanos
}

// SetRetryAtNanos sets the next retry nanos.
func (m *message) SetRetryAtNanos(value int64) {
	if m.retryAtNanos > 0 {
		m.expectedProcessAtNanos.Store(m.retryAtNanos)
	} else {
		m.expectedProcessAtNanos.Store(m.initNanos)
	}
	m.retryAtNanos = value
}

// WriteTimes returns the times the message has been written.
func (m *message) WriteTimes() int {
	return m.retried
}

// IncWriteTimes increments the times the message has been written.
func (m *message) IncWriteTimes() {
	m.retried++
}

// IsAcked returns true if the message has been acked.
func (m *message) IsAcked() bool {
	return m.isAcked.Load()
}

// Ack acknowledges the message. Duplicated acks on the same message might cause panic.
func (m *message) Ack() {
	m.isAcked.Store(true)
	m.RefCountedMessage.DecRef()
}

func (m *message) ShardID() uint64 {
	return m.meta.shard
}

// Metadata returns the metadata.
func (m *message) Metadata() metadata {
	return m.meta
}

// SetSentAt sets the sentAtNanos on the metadata proto.
func (m *message) SetSentAt(nanos int64) {
	m.pb.Metadata.SentAtNanos = uint64(nanos)
}

// Marshaler returns the marshaler and a bool to indicate whether the marshaler is valid.
func (m *message) Marshaler() (proto.Marshaler, bool) {
	return &m.pb, !m.RefCountedMessage.IsDroppedOrConsumed()
}

func (m *message) ToProto(pb *msgpb.Message) {
	m.meta.ToProto(&pb.Metadata)
	pb.Value = m.RefCountedMessage.Bytes()
}

func (m *message) ResetProto(pb *msgpb.Message) {
	pb.Value = nil
}
<|endoftext|>


<commit_before>package procspy

import (
	"bytes"
	"net"
)

// ProcNet is an iterator to parse /proc/net/tcp{,6} files.
type ProcNet struct {
	b                       []byte
	c                       Connection
	bytesLocal, bytesRemote [16]byte
	seen                    map[uint64]struct{}
}

// NewProcNet gives a new ProcNet parser.
func NewProcNet(b []byte) *ProcNet {
	return &ProcNet{
		b:           b,
		c:           Connection{},
		seen:        map[uint64]struct{}{},
	}
}

// Next returns the next connection. All buffers are re-used, so if you want
// to keep the IPs you have to copy them.
func (p *ProcNet) Next() *Connection {
again:
	if len(p.b) == 0 {
		return nil
	}
	b := p.b

	if p.b[2] == 's' {
		// Skip header
		p.b = nextLine(b)
		goto again
	}

	var (
		local, remote, inode []byte
	)
	_, b = nextField(b) // 'sl' column
	local, b = nextField(b)
	remote, b = nextField(b)
	_, b = nextField(b) // 'st' column
	_, b = nextField(b) // 'tx_queue' column
	_, b = nextField(b) // 'rx_queue' column
	_, b = nextField(b) // 'tr' column
	_, b = nextField(b) // 'uid' column
	_, b = nextField(b) // 'timeout' column
	inode, b = nextField(b)

	p.c.LocalAddress, p.c.LocalPort = scanAddressNA(local, &p.bytesLocal)
	p.c.RemoteAddress, p.c.RemotePort = scanAddressNA(remote, &p.bytesRemote)
	p.c.inode = parseDec(inode)
	p.b = nextLine(b)
	if _, alreadySeen := p.seen[p.c.inode]; alreadySeen {
		goto again
	}
	p.seen[p.c.inode] = struct{}{}
	return &p.c
}

// scanAddressNA parses 'A12CF62E:00AA' to the address/port. Handles IPv4 and
// IPv6 addresses. The address is a big endian 32 bit ints, hex encoded. We
// just decode the hex and flip the bytes in every group of 4.
func scanAddressNA(in []byte, buf *[16]byte) (net.IP, uint16) {
	col := bytes.IndexByte(in, ':')
	if col == -1 {
		return nil, 0
	}

	// Network address is big endian. Can be either ipv4 or ipv6.
	address := hexDecode32bigNA(in[:col], buf)
	return net.IP(address), uint16(parseHex(in[col+1:]))
}

// hexDecode32big decodes sequences of 32bit big endian bytes.
func hexDecode32bigNA(src []byte, buf *[16]byte) []byte {
	blocks := len(src) / 8
	for block := 0; block < blocks; block++ {
		for i := 0; i < 4; i++ {
			a := fromHexChar(src[block*8+i*2])
			b := fromHexChar(src[block*8+i*2+1])
			buf[block*4+3-i] = (a << 4) | b
		}
	}
	return buf[:blocks*4]
}

func nextField(s []byte) ([]byte, []byte) {
	// Skip whitespace.
	for i, b := range s {
		if b != ' ' {
			s = s[i:]
			break
		}
	}

	// Up until the next whitespace field.
	for i, b := range s {
		if b == ' ' {
			return s[:i], s[i:]
		}
	}

	return nil, nil
}

func nextLine(s []byte) []byte {
	i := bytes.IndexByte(s, '\n')
	if i == -1 {
		return nil
	}
	return s[i+1:]
}

// Simplified copy of strconv.ParseUint(16).
func parseHex(s []byte) uint {
	n := uint(0)
	for i := 0; i < len(s); i++ {
		n *= 16
		n += uint(fromHexChar(s[i]))
	}
	return n
}

// Simplified copy of strconv.ParseUint(10).
func parseDec(s []byte) uint64 {
	n := uint64(0)
	for _, c := range s {
		n *= 10
		n += uint64(c - '0')
	}
	return n
}

// hexDecode32big decodes sequences of 32bit big endian bytes.
func hexDecode32big(src []byte) []byte {
	dst := make([]byte, len(src)/2)
	blocks := len(src) / 8
	for block := 0; block < blocks; block++ {
		for i := 0; i < 4; i++ {
			a := fromHexChar(src[block*8+i*2])
			b := fromHexChar(src[block*8+i*2+1])
			dst[block*4+3-i] = (a << 4) | b
		}
	}
	return dst
}

// fromHexChar converts a hex character into its value.
func fromHexChar(c byte) uint8 {
	switch {
	case '0' <= c && c <= '9':
		return c - '0'
	case 'a' <= c && c <= 'f':
		return c - 'a' + 10
	case 'A' <= c && c <= 'F':
		return c - 'A' + 10
	}
	return 0
}
<commit_msg>Make linter happy<commit_after>package procspy

import (
	"bytes"
	"net"
)

// ProcNet is an iterator to parse /proc/net/tcp{,6} files.
type ProcNet struct {
	b                       []byte
	c                       Connection
	bytesLocal, bytesRemote [16]byte
	seen                    map[uint64]struct{}
}

// NewProcNet gives a new ProcNet parser.
func NewProcNet(b []byte) *ProcNet {
	return &ProcNet{
		b:    b,
		c:    Connection{},
		seen: map[uint64]struct{}{},
	}
}

// Next returns the next connection. All buffers are re-used, so if you want
// to keep the IPs you have to copy them.
func (p *ProcNet) Next() *Connection {
again:
	if len(p.b) == 0 {
		return nil
	}
	b := p.b

	if p.b[2] == 's' {
		// Skip header
		p.b = nextLine(b)
		goto again
	}

	var (
		local, remote, inode []byte
	)
	_, b = nextField(b) // 'sl' column
	local, b = nextField(b)
	remote, b = nextField(b)
	_, b = nextField(b) // 'st' column
	_, b = nextField(b) // 'tx_queue' column
	_, b = nextField(b) // 'rx_queue' column
	_, b = nextField(b) // 'tr' column
	_, b = nextField(b) // 'uid' column
	_, b = nextField(b) // 'timeout' column
	inode, b = nextField(b)

	p.c.LocalAddress, p.c.LocalPort = scanAddressNA(local, &p.bytesLocal)
	p.c.RemoteAddress, p.c.RemotePort = scanAddressNA(remote, &p.bytesRemote)
	p.c.inode = parseDec(inode)
	p.b = nextLine(b)
	if _, alreadySeen := p.seen[p.c.inode]; alreadySeen {
		goto again
	}
	p.seen[p.c.inode] = struct{}{}
	return &p.c
}

// scanAddressNA parses 'A12CF62E:00AA' to the address/port. Handles IPv4 and
// IPv6 addresses. The address is a big endian 32 bit ints, hex encoded. We
// just decode the hex and flip the bytes in every group of 4.
func scanAddressNA(in []byte, buf *[16]byte) (net.IP, uint16) {
	col := bytes.IndexByte(in, ':')
	if col == -1 {
		return nil, 0
	}

	// Network address is big endian. Can be either ipv4 or ipv6.
	address := hexDecode32bigNA(in[:col], buf)
	return net.IP(address), uint16(parseHex(in[col+1:]))
}

// hexDecode32big decodes sequences of 32bit big endian bytes.
func hexDecode32bigNA(src []byte, buf *[16]byte) []byte {
	blocks := len(src) / 8
	for block := 0; block < blocks; block++ {
		for i := 0; i < 4; i++ {
			a := fromHexChar(src[block*8+i*2])
			b := fromHexChar(src[block*8+i*2+1])
			buf[block*4+3-i] = (a << 4) | b
		}
	}
	return buf[:blocks*4]
}

func nextField(s []byte) ([]byte, []byte) {
	// Skip whitespace.
	for i, b := range s {
		if b != ' ' {
			s = s[i:]
			break
		}
	}

	// Up until the next whitespace field.
	for i, b := range s {
		if b == ' ' {
			return s[:i], s[i:]
		}
	}

	return nil, nil
}

func nextLine(s []byte) []byte {
	i := bytes.IndexByte(s, '\n')
	if i == -1 {
		return nil
	}
	return s[i+1:]
}

// Simplified copy of strconv.ParseUint(16).
func parseHex(s []byte) uint {
	n := uint(0)
	for i := 0; i < len(s); i++ {
		n *= 16
		n += uint(fromHexChar(s[i]))
	}
	return n
}

// Simplified copy of strconv.ParseUint(10).
func parseDec(s []byte) uint64 {
	n := uint64(0)
	for _, c := range s {
		n *= 10
		n += uint64(c - '0')
	}
	return n
}

// hexDecode32big decodes sequences of 32bit big endian bytes.
func hexDecode32big(src []byte) []byte {
	dst := make([]byte, len(src)/2)
	blocks := len(src) / 8
	for block := 0; block < blocks; block++ {
		for i := 0; i < 4; i++ {
			a := fromHexChar(src[block*8+i*2])
			b := fromHexChar(src[block*8+i*2+1])
			dst[block*4+3-i] = (a << 4) | b
		}
	}
	return dst
}

// fromHexChar converts a hex character into its value.
func fromHexChar(c byte) uint8 {
	switch {
	case '0' <= c && c <= '9':
		return c - '0'
	case 'a' <= c && c <= 'f':
		return c - 'a' + 10
	case 'A' <= c && c <= 'F':
		return c - 'A' + 10
	}
	return 0
}
<|endoftext|>


<commit_before>package main

import "fmt"
import "io/ioutil"
import "os"

func main() {
	fmt.Println("Advent of code: Golang\nTask 8")

	PrintCurrentDir()

	file, err := ioutil.ReadFile("../input.txt")
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
        defer file.Close()

	content := string(file)
	fmt.Println(content)
}

func PrintCurrentDir() {
	pwd, err := os.Getwd()
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	fmt.Println("current dir: " + pwd)
}

<commit_msg>defer before error check - always call<commit_after>package main

import "fmt"
import "io/ioutil"
import "os"

func main() {
	fmt.Println("Advent of code: Golang\nTask 8")

	PrintCurrentDir()

	file, err := ioutil.ReadFile("../input.txt")
        defer file.Close()
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	content := string(file)
	fmt.Println(content)
}

func PrintCurrentDir() {
	pwd, err := os.Getwd()
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	fmt.Println("current dir: " + pwd)
}

<|endoftext|>


<commit_before>package simple

import (
	"testing"

	baloo "gopkg.in/h2non/baloo.v3"
)

// test stores the HTTP testing client preconfigured
var test = baloo.New("http://httpbin.org")

func TestBalooHeaders(t *testing.T) {
	test.Get("/get").
		SetHeader("Foo", "Bar").
		Expect(t).
		Status(200).
		Header("Server", "meinheld").
		Header("Content-Type", "json").
		Type("json").
		Done()
}
<commit_msg>fix(header):<commit_after>package simple

import (
	"testing"

	baloo "gopkg.in/h2non/baloo.v3"
)

// test stores the HTTP testing client preconfigured
var test = baloo.New("http://httpbin.org")

func TestBalooHeaders(t *testing.T) {
	test.Get("/get").
		SetHeader("Foo", "Bar").
		Expect(t).
		Status(200).
		Header("Server", "gunicorn").
		Header("Content-Type", "json").
		Type("json").
		Done()
}
<|endoftext|>


<commit_before>package tunnelproxymanager

import (
	"errors"
	"fmt"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/route53"
	"github.com/koding/logging"
)

const (
	// callerReferance is used as unique id for idempotency on aws side
	callerReferance         = "tunnelproxy_dev_2"
	hostedZoneName          = "tunnelproxy.koding.com"
	hostedZoneComment       = "Hosted zone for tunnel proxies"
	validStateForHostedZone = "INSYNC" // taken from aws response

	maxIterationCount = 100
)

var (
	errHostedZoneNotFound           = errors.New("hosted zone not found")
	errDeadlineReachedForChangeInfo = errors.New("deadline for change info")
)

// RecordManager manages Route53 records
type RecordManager struct {
	// aws services
	route53 *route53.Route53

	// application wide parameters
	hostedZone *route53.HostedZone
	region     string

	// general usage
	log logging.Logger
}

// NewRecordManager creates a RecordManager
func NewRecordManager(config *aws.Config, log logging.Logger, region string) *RecordManager {
	return &RecordManager{
		route53: route53.New(config),
		log:     log.New("recordmanager"),
		region:  region,
	}
}

// Init initializes record manager's prerequisites
func (r *RecordManager) Init() error {
	r.log.Debug("Configuring...")

	if err := r.ensureHostedZone(); err != nil {
		return err
	}

	r.log.Info("init is done")
	return nil
}

func (r *RecordManager) ensureHostedZone() error {
	hostedZoneLogger := r.log.New("HostedZone").New("name", hostedZoneName)
	hostedZoneLogger.Debug("Trying to get existing Hosted Zone")

	err := r.getHostedZone(hostedZoneLogger)
	if err == nil {
		hostedZoneLogger.Debug("Hosted Zone exists..")
		return nil
	}

	if err != errHostedZoneNotFound {
		return err
	}

	hostedZoneLogger.Debug("Not found, creating...")
	return r.createHostedZone(hostedZoneLogger)
}

func (r *RecordManager) getHostedZone(hostedZoneLogger logging.Logger) error {
	iteration := 0
	// try to get our hosted zone
	for {
		// just be paranoid about remove api calls, dont harden too much
		if iteration == maxIterationCount {
			return errors.New("iteration terminated")
		}

		log := hostedZoneLogger.New("iteration", iteration)

		iteration++

		// for pagination
		var nextMarker *string

		log.Debug("Fetching hosted zone")
		listHostedZonesResp, err := r.route53.ListHostedZones(
			&route53.ListHostedZonesInput{
				Marker: nextMarker,
			}, // we dont have anything to filter
		)
		if err != nil {
			return err
		}

		if listHostedZonesResp == nil || listHostedZonesResp.HostedZones == nil {
			return errors.New("malformed response")
		}

		for _, hostedZone := range listHostedZonesResp.HostedZones {
			if *hostedZone.CallerReference == callerReferance {
				r.hostedZone = hostedZone
				return nil
			}
		}

		// if our result set is truncated we can try to fecth again, but if we
		// reach to end, nothing to do left
		if !*listHostedZonesResp.IsTruncated {
			return errHostedZoneNotFound
		}

		// assign next marker
		nextMarker = listHostedZonesResp.NextMarker
	}
}

// createHostedZone creates hosted zone and makes sure that it is in to be used
// state
func (r *RecordManager) createHostedZone(hostedZoneLogger logging.Logger) error {
	hostedZoneLogger.Debug("create hosted zone started")

	// CreateHostedZone is not idempotent, multiple calls to this function
	// result in duplicate records, fyi
	resp, err := r.route53.CreateHostedZone(&route53.CreateHostedZoneInput{
		CallerReference: aws.String(callerReferance),
		Name:            aws.String(hostedZoneName),
		HostedZoneConfig: &route53.HostedZoneConfig{
			Comment: aws.String(hostedZoneComment),
		},
	})
	if err != nil {
		return err
	}

	if resp == nil || resp.ChangeInfo == nil {
		return errors.New("malformed response, resp is nil")
	}

	changeInfo := resp.ChangeInfo
	deadline := time.After(time.Minute * 5) // at most i've seen ~3min

	// make sure it propagated
	for {
		// if our change propagated, we can return
		if *changeInfo.Status == validStateForHostedZone {
			hostedZoneLogger.Debug("hosted zone status is valid")
			break
		}

		select {
		case <-deadline:
			return errDeadlineReachedForChangeInfo
		default:
			time.Sleep(time.Second * 3) // poor man's throttling
			hostedZoneLogger.New("changeInfoID", *changeInfo.ID).Debug("fetching latest status")
			getChangeResp, err := r.route53.GetChange(&route53.GetChangeInput{
				ID: changeInfo.ID,
			})
			if err != nil {
				return err
			}

			if getChangeResp == nil || getChangeResp.ChangeInfo == nil {
				return errors.New("malformed response, getChangeResp is nil")
			}

			changeInfo = getChangeResp.ChangeInfo
		}
	}

	r.hostedZone = resp.HostedZone

	hostedZoneLogger.Debug("create hosted finished successfully")

	return nil
}

// UpsertRecordSet updates record set for current ResourceRecordSet
func (r *RecordManager) UpsertRecordSet(instances []*string) error {
	if r.hostedZone == nil {
		return errors.New("hosted zone is not set")
	}

	resourceRecords := make([]*route53.ResourceRecord, 0)
	for _, instance := range instances {
		resourceRecords = append(resourceRecords, &route53.ResourceRecord{
			Value: instance,
		})
	}
	params := &route53.ChangeResourceRecordSetsInput{
		ChangeBatch: &route53.ChangeBatch{
			Changes: []*route53.Change{
				&route53.Change{
					Action: aws.String("UPSERT"),
					ResourceRecordSet: &route53.ResourceRecordSet{
						Name:            aws.String(hostedZoneName),
						Type:            aws.String("A"),
						Region:          aws.String(r.region),
						ResourceRecords: resourceRecords,
						// use region name as identifer
						SetIdentifier: aws.String(r.region),
						TTL:           aws.Long(1),
					},
				},
			},
			Comment: aws.String(
				fmt.Sprintf(
					"Record set for zone: %s region: %s",
					hostedZoneName,
					r.region,
				),
			),
		},
		HostedZoneID: r.hostedZone.ID,
	}

	_, err := r.route53.ChangeResourceRecordSets(params)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>Tunnel: write doc for ResourceRecordSet<commit_after>package tunnelproxymanager

import (
	"errors"
	"fmt"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/route53"
	"github.com/koding/logging"
)

const (
	// callerReferance is used as unique id for idempotency on aws side
	callerReferance         = "tunnelproxy_dev_2"
	hostedZoneName          = "tunnelproxy.koding.com"
	hostedZoneComment       = "Hosted zone for tunnel proxies"
	validStateForHostedZone = "INSYNC" // taken from aws response

	maxIterationCount = 100
)

var (
	errHostedZoneNotFound           = errors.New("hosted zone not found")
	errDeadlineReachedForChangeInfo = errors.New("deadline for change info")
)

// RecordManager manages Route53 records
type RecordManager struct {
	// aws services
	route53 *route53.Route53

	// application wide parameters
	hostedZone *route53.HostedZone
	region     string

	// general usage
	log logging.Logger
}

// NewRecordManager creates a RecordManager
func NewRecordManager(config *aws.Config, log logging.Logger, region string) *RecordManager {
	return &RecordManager{
		route53: route53.New(config),
		log:     log.New("recordmanager"),
		region:  region,
	}
}

// Init initializes record manager's prerequisites
func (r *RecordManager) Init() error {
	r.log.Debug("Configuring...")

	if err := r.ensureHostedZone(); err != nil {
		return err
	}

	r.log.Info("init is done")
	return nil
}

func (r *RecordManager) ensureHostedZone() error {
	hostedZoneLogger := r.log.New("HostedZone").New("name", hostedZoneName)
	hostedZoneLogger.Debug("Trying to get existing Hosted Zone")

	err := r.getHostedZone(hostedZoneLogger)
	if err == nil {
		hostedZoneLogger.Debug("Hosted Zone exists..")
		return nil
	}

	if err != errHostedZoneNotFound {
		return err
	}

	hostedZoneLogger.Debug("Not found, creating...")
	return r.createHostedZone(hostedZoneLogger)
}

func (r *RecordManager) getHostedZone(hostedZoneLogger logging.Logger) error {
	iteration := 0
	// try to get our hosted zone
	for {
		// just be paranoid about remove api calls, dont harden too much
		if iteration == maxIterationCount {
			return errors.New("iteration terminated")
		}

		log := hostedZoneLogger.New("iteration", iteration)

		iteration++

		// for pagination
		var nextMarker *string

		log.Debug("Fetching hosted zone")
		listHostedZonesResp, err := r.route53.ListHostedZones(
			&route53.ListHostedZonesInput{
				Marker: nextMarker,
			}, // we dont have anything to filter
		)
		if err != nil {
			return err
		}

		if listHostedZonesResp == nil || listHostedZonesResp.HostedZones == nil {
			return errors.New("malformed response")
		}

		for _, hostedZone := range listHostedZonesResp.HostedZones {
			if *hostedZone.CallerReference == callerReferance {
				r.hostedZone = hostedZone
				return nil
			}
		}

		// if our result set is truncated we can try to fecth again, but if we
		// reach to end, nothing to do left
		if !*listHostedZonesResp.IsTruncated {
			return errHostedZoneNotFound
		}

		// assign next marker
		nextMarker = listHostedZonesResp.NextMarker
	}
}

// createHostedZone creates hosted zone and makes sure that it is in to be used
// state
func (r *RecordManager) createHostedZone(hostedZoneLogger logging.Logger) error {
	hostedZoneLogger.Debug("create hosted zone started")

	// CreateHostedZone is not idempotent, multiple calls to this function
	// result in duplicate records, fyi
	resp, err := r.route53.CreateHostedZone(&route53.CreateHostedZoneInput{
		CallerReference: aws.String(callerReferance),
		Name:            aws.String(hostedZoneName),
		HostedZoneConfig: &route53.HostedZoneConfig{
			Comment: aws.String(hostedZoneComment),
		},
	})
	if err != nil {
		return err
	}

	if resp == nil || resp.ChangeInfo == nil {
		return errors.New("malformed response, resp is nil")
	}

	changeInfo := resp.ChangeInfo
	deadline := time.After(time.Minute * 5) // at most i've seen ~3min

	// make sure it propagated
	for {
		// if our change propagated, we can return
		if *changeInfo.Status == validStateForHostedZone {
			hostedZoneLogger.Debug("hosted zone status is valid")
			break
		}

		select {
		case <-deadline:
			return errDeadlineReachedForChangeInfo
		default:
			time.Sleep(time.Second * 3) // poor man's throttling
			hostedZoneLogger.New("changeInfoID", *changeInfo.ID).Debug("fetching latest status")
			getChangeResp, err := r.route53.GetChange(&route53.GetChangeInput{
				ID: changeInfo.ID,
			})
			if err != nil {
				return err
			}

			if getChangeResp == nil || getChangeResp.ChangeInfo == nil {
				return errors.New("malformed response, getChangeResp is nil")
			}

			changeInfo = getChangeResp.ChangeInfo
		}
	}

	r.hostedZone = resp.HostedZone

	hostedZoneLogger.Debug("create hosted finished successfully")

	return nil
}

// UpsertRecordSet updates record set for current ResourceRecordSet
func (r *RecordManager) UpsertRecordSet(instances []*string) error {
	if r.hostedZone == nil {
		return errors.New("hosted zone is not set")
	}

	resourceRecords := make([]*route53.ResourceRecord, 0)
	for _, instance := range instances {
		resourceRecords = append(resourceRecords, &route53.ResourceRecord{
			Value: instance,
		})
	}
	params := &route53.ChangeResourceRecordSetsInput{
		ChangeBatch: &route53.ChangeBatch{
			// contains one Change element for each resource record set that you
			// want to create or delete.
			Changes: []*route53.Change{
				&route53.Change{
					Action: aws.String("UPSERT"),
					ResourceRecordSet: &route53.ResourceRecordSet{
						// The domain name of the current resource record set.
						Name: aws.String(hostedZoneName),
						// The type of the current resource record set.
						Type: aws.String("A"),
						// Latency-based resource record sets only: Among
						// resource record sets that have the same combination
						// of DNS name and type, a value that specifies the AWS
						// region for the current resource record set.
						Region: aws.String(r.region),
						// that contains the resource records for the current
						// resource record set.
						ResourceRecords: resourceRecords,
						// Weighted, Latency, Geo, and Failover resource record sets only
						// use region name as identifer
						SetIdentifier: aws.String(r.region),
						// The cache time to live for the current resource record set.
						TTL: aws.Long(1),
					},
				},
			},
			Comment: aws.String(
				fmt.Sprintf(
					"Record set for zone: %s region: %s",
					hostedZoneName,
					r.region,
				),
			),
		},
		HostedZoneID: r.hostedZone.ID,
	}

	_, err := r.route53.ChangeResourceRecordSets(params)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package git

import (
	"fmt"
	"io/ioutil"
	"net/url"
	"os"
	"os/exec"
	"path"
	"strings"

	log "github.com/sirupsen/logrus"
)

// Client is a generic git client interface
type Client interface {
	Root() string
	Init() error
	Fetch() error
	Checkout(revision string) error
	LsRemote(revision string) (string, error)
	CommitSHA() (string, error)
	Reset() error
}

// ClientFactory is a factory of Git Clients
// Primarily used to support creation of mock git clients during unit testing
type ClientFactory interface {
	NewClient(repoURL, path, username, password, sshPrivateKey string) Client
}

// nativeGitClient implements Client interface using git CLI
type nativeGitClient struct {
	repoURL       string
	root          string
	username      string
	password      string
	sshPrivateKey string
}

type factory struct{}

func NewFactory() ClientFactory {
	return &factory{}
}

func (f *factory) NewClient(repoURL, path, username, password, sshPrivateKey string) Client {
	return &nativeGitClient{
		repoURL:       repoURL,
		root:          path,
		username:      username,
		password:      password,
		sshPrivateKey: sshPrivateKey,
	}
}

func (m *nativeGitClient) Root() string {
	return m.root
}

// Init initializes a local git repository and sets the remote origin
func (m *nativeGitClient) Init() error {
	var needInit bool
	if _, err := os.Stat(m.root); os.IsNotExist(err) {
		needInit = true
	} else {
		_, err = m.runCmd("git", "status")
		needInit = err != nil
	}
	if needInit {
		log.Infof("Initializing %s to %s", m.repoURL, m.root)
		_, err := exec.Command("rm", "-rf", m.root).Output()
		if err != nil {
			return fmt.Errorf("unable to clean repo at %s: %v", m.root, err)
		}
		err = os.MkdirAll(m.root, 0755)
		if err != nil {
			return err
		}
		if _, err := m.runCmd("git", "init"); err != nil {
			return err
		}
		if _, err := m.runCmd("git", "remote", "add", "origin", m.repoURL); err != nil {
			return err
		}
	}
	// always set credentials since it can change
	err := m.setCredentials()
	if err != nil {
		return err
	}
	return nil
}

// setCredentials sets a local credentials file to connect to a remote git repository
func (m *nativeGitClient) setCredentials() error {
	if m.password != "" {
		log.Debug("Setting password credentials")
		gitCredentialsFile := path.Join(m.root, ".git", "credentials")
		urlObj, err := url.ParseRequestURI(m.repoURL)
		if err != nil {
			return err
		}
		urlObj.User = url.UserPassword(m.username, m.password)
		cmdURL := urlObj.String()
		err = ioutil.WriteFile(gitCredentialsFile, []byte(cmdURL), 0600)
		if err != nil {
			return fmt.Errorf("failed to set git credentials: %v", err)
		}
		_, err = m.runCmd("git", "config", "--local", "credential.helper", fmt.Sprintf("store --file=%s", gitCredentialsFile))
		if err != nil {
			return err
		}
	}
	if IsSSHURL(m.repoURL) {
		sshCmd := gitSSHCommand
		if m.sshPrivateKey != "" {
			log.Debug("Setting SSH credentials")
			sshPrivateKeyFile := path.Join(".git", "ssh-private-key")
			err := ioutil.WriteFile(sshPrivateKeyFile, []byte(m.sshPrivateKey), 0600)
			if err != nil {
				return fmt.Errorf("failed to set git credentials: %v", err)
			}
			sshCmd += sshCmd + " -i " + sshPrivateKeyFile
		}
		_, err := m.runCmd("git", "config", "--local", "core.sshCommand", sshCmd)
		if err != nil {
			return err
		}
	}
	return nil
}

// Fetch fetches latest updates from origin
func (m *nativeGitClient) Fetch() error {
	var err error
	log.Debugf("Fetching repo %s at %s", m.repoURL, m.root)
	if _, err = m.runCmd("git", "fetch", "origin"); err != nil {
		return err
	}
	// git fetch does not update the HEAD reference. The following command will update the local
	// knowledge of what remote considers the “default branch”
	// See: https://stackoverflow.com/questions/8839958/how-does-origin-head-get-set
	if _, err := m.runCmd("git", "remote", "set-head", "origin", "-a"); err != nil {
		return err
	}
	// Delete all local branches (we must first detach so we are not checked out a branch we are about to delete)
	if _, err = m.runCmd("git", "checkout", "--detach", "origin/HEAD"); err != nil {
		return err
	}
	branchesOut, err := m.runCmd("git", "for-each-ref", "--format=%(refname:short)", "refs/heads/")
	if err != nil {
		return err
	}
	branchesOut = strings.TrimSpace(branchesOut)
	if branchesOut != "" {
		branches := strings.Split(branchesOut, "\n")
		args := []string{"branch", "-D"}
		args = append(args, branches...)
		if _, err = m.runCmd("git", args...); err != nil {
			return err
		}
	}
	return nil
}

// Reset resets local changes in a repository
func (m *nativeGitClient) Reset() error {
	if _, err := m.runCmd("git", "reset", "--hard", "origin/HEAD"); err != nil {
		return err
	}
	if _, err := m.runCmd("git", "clean", "-f"); err != nil {
		return err
	}
	return nil
}

// Checkout checkout specified git sha
func (m *nativeGitClient) Checkout(revision string) error {
	if revision == "" || revision == "HEAD" {
		revision = "origin/HEAD"
	}
	if _, err := m.runCmd("git", "checkout", revision); err != nil {
		return err
	}
	return nil
}

// LsRemote returns the commit SHA of a specific branch, tag, or HEAD
func (m *nativeGitClient) LsRemote(revision string) (string, error) {
	var args []string
	if revision == "" || revision == "HEAD" {
		args = []string{"ls-remote", "origin", "HEAD"}
	} else {
		args = []string{"ls-remote", "--head", "--tags", "origin", revision}

	}
	out, err := m.runCmd("git", args...)
	if err != nil {
		return "", err
	}
	if out == "" {
		// if doesn't exist in remote, assume revision is a commit sha and return it
		return revision, nil
	}
	// 3f4ec0ab2263038ba91d3b594b2188fc108fc8d7	refs/heads/master
	return strings.Fields(out)[0], nil
}

// CommitSHA returns current commit sha from `git rev-parse HEAD`
func (m *nativeGitClient) CommitSHA() (string, error) {
	out, err := m.runCmd("git", "rev-parse", "HEAD")
	if err != nil {
		return "", err
	}
	return strings.TrimSpace(out), nil
}

// runCmd is a convenience function to run a command in a given directory and return its output
func (m *nativeGitClient) runCmd(command string, args ...string) (string, error) {
	cmd := exec.Command(command, args...)
	log.Debug(strings.Join(cmd.Args, " "))
	cmd.Dir = m.root
	env := os.Environ()
	env = append(env, "GIT_ASKPASS=")
	cmd.Env = env
	out, err := cmd.Output()
	if len(out) > 0 {
		log.Debug(string(out))
	}
	if err != nil {
		exErr, ok := err.(*exec.ExitError)
		if ok {
			errOutput := strings.Split(string(exErr.Stderr), "\n")[0]
			log.Debug(errOutput)
			return string(out), fmt.Errorf("'%s' failed: %v", strings.Join(cmd.Args, " "), errOutput)
		}
		return string(out), fmt.Errorf("'%s' failed: %v", strings.Join(cmd.Args, " "), err)
	}
	return string(out), nil
}
<commit_msg>Move local branch deletion as part of git Reset() (resolves #185) (#222)<commit_after>package git

import (
	"fmt"
	"io/ioutil"
	"net/url"
	"os"
	"os/exec"
	"path"
	"strings"

	log "github.com/sirupsen/logrus"
)

// Client is a generic git client interface
type Client interface {
	Root() string
	Init() error
	Fetch() error
	Checkout(revision string) error
	LsRemote(revision string) (string, error)
	CommitSHA() (string, error)
	Reset() error
}

// ClientFactory is a factory of Git Clients
// Primarily used to support creation of mock git clients during unit testing
type ClientFactory interface {
	NewClient(repoURL, path, username, password, sshPrivateKey string) Client
}

// nativeGitClient implements Client interface using git CLI
type nativeGitClient struct {
	repoURL       string
	root          string
	username      string
	password      string
	sshPrivateKey string
}

type factory struct{}

func NewFactory() ClientFactory {
	return &factory{}
}

func (f *factory) NewClient(repoURL, path, username, password, sshPrivateKey string) Client {
	return &nativeGitClient{
		repoURL:       repoURL,
		root:          path,
		username:      username,
		password:      password,
		sshPrivateKey: sshPrivateKey,
	}
}

func (m *nativeGitClient) Root() string {
	return m.root
}

// Init initializes a local git repository and sets the remote origin
func (m *nativeGitClient) Init() error {
	var needInit bool
	if _, err := os.Stat(m.root); os.IsNotExist(err) {
		needInit = true
	} else {
		_, err = m.runCmd("git", "status")
		needInit = err != nil
	}
	if needInit {
		log.Infof("Initializing %s to %s", m.repoURL, m.root)
		_, err := exec.Command("rm", "-rf", m.root).Output()
		if err != nil {
			return fmt.Errorf("unable to clean repo at %s: %v", m.root, err)
		}
		err = os.MkdirAll(m.root, 0755)
		if err != nil {
			return err
		}
		if _, err := m.runCmd("git", "init"); err != nil {
			return err
		}
		if _, err := m.runCmd("git", "remote", "add", "origin", m.repoURL); err != nil {
			return err
		}
	}
	// always set credentials since it can change
	err := m.setCredentials()
	if err != nil {
		return err
	}
	return nil
}

// setCredentials sets a local credentials file to connect to a remote git repository
func (m *nativeGitClient) setCredentials() error {
	if m.password != "" {
		log.Debug("Setting password credentials")
		gitCredentialsFile := path.Join(m.root, ".git", "credentials")
		urlObj, err := url.ParseRequestURI(m.repoURL)
		if err != nil {
			return err
		}
		urlObj.User = url.UserPassword(m.username, m.password)
		cmdURL := urlObj.String()
		err = ioutil.WriteFile(gitCredentialsFile, []byte(cmdURL), 0600)
		if err != nil {
			return fmt.Errorf("failed to set git credentials: %v", err)
		}
		_, err = m.runCmd("git", "config", "--local", "credential.helper", fmt.Sprintf("store --file=%s", gitCredentialsFile))
		if err != nil {
			return err
		}
	}
	if IsSSHURL(m.repoURL) {
		sshCmd := gitSSHCommand
		if m.sshPrivateKey != "" {
			log.Debug("Setting SSH credentials")
			sshPrivateKeyFile := path.Join(".git", "ssh-private-key")
			err := ioutil.WriteFile(sshPrivateKeyFile, []byte(m.sshPrivateKey), 0600)
			if err != nil {
				return fmt.Errorf("failed to set git credentials: %v", err)
			}
			sshCmd += sshCmd + " -i " + sshPrivateKeyFile
		}
		_, err := m.runCmd("git", "config", "--local", "core.sshCommand", sshCmd)
		if err != nil {
			return err
		}
	}
	return nil
}

// Fetch fetches latest updates from origin
func (m *nativeGitClient) Fetch() error {
	var err error
	log.Debugf("Fetching repo %s at %s", m.repoURL, m.root)
	if _, err = m.runCmd("git", "fetch", "origin"); err != nil {
		return err
	}
	// git fetch does not update the HEAD reference. The following command will update the local
	// knowledge of what remote considers the “default branch”
	// See: https://stackoverflow.com/questions/8839958/how-does-origin-head-get-set
	if _, err := m.runCmd("git", "remote", "set-head", "origin", "-a"); err != nil {
		return err
	}
	return nil
}

// Reset resets local changes in a repository
func (m *nativeGitClient) Reset() error {
	if _, err := m.runCmd("git", "reset", "--hard", "origin/HEAD"); err != nil {
		return err
	}
	// Delete all local branches (we must first detach so we are not checked out a branch we are about to delete)
	if _, err := m.runCmd("git", "checkout", "--detach", "origin/HEAD"); err != nil {
		return err
	}
	branchesOut, err := m.runCmd("git", "for-each-ref", "--format=%(refname:short)", "refs/heads/")
	if err != nil {
		return err
	}
	branchesOut = strings.TrimSpace(branchesOut)
	if branchesOut != "" {
		branches := strings.Split(branchesOut, "\n")
		args := []string{"branch", "-D"}
		args = append(args, branches...)
		if _, err = m.runCmd("git", args...); err != nil {
			return err
		}
	}
	if _, err := m.runCmd("git", "clean", "-f"); err != nil {
		return err
	}
	return nil
}

// Checkout checkout specified git sha
func (m *nativeGitClient) Checkout(revision string) error {
	if revision == "" || revision == "HEAD" {
		revision = "origin/HEAD"
	}
	if _, err := m.runCmd("git", "checkout", revision); err != nil {
		return err
	}
	return nil
}

// LsRemote returns the commit SHA of a specific branch, tag, or HEAD
func (m *nativeGitClient) LsRemote(revision string) (string, error) {
	var args []string
	if revision == "" || revision == "HEAD" {
		args = []string{"ls-remote", "origin", "HEAD"}
	} else {
		args = []string{"ls-remote", "--head", "--tags", "origin", revision}

	}
	out, err := m.runCmd("git", args...)
	if err != nil {
		return "", err
	}
	if out == "" {
		// if doesn't exist in remote, assume revision is a commit sha and return it
		return revision, nil
	}
	// 3f4ec0ab2263038ba91d3b594b2188fc108fc8d7	refs/heads/master
	return strings.Fields(out)[0], nil
}

// CommitSHA returns current commit sha from `git rev-parse HEAD`
func (m *nativeGitClient) CommitSHA() (string, error) {
	out, err := m.runCmd("git", "rev-parse", "HEAD")
	if err != nil {
		return "", err
	}
	return strings.TrimSpace(out), nil
}

// runCmd is a convenience function to run a command in a given directory and return its output
func (m *nativeGitClient) runCmd(command string, args ...string) (string, error) {
	cmd := exec.Command(command, args...)
	log.Debug(strings.Join(cmd.Args, " "))
	cmd.Dir = m.root
	env := os.Environ()
	env = append(env, "GIT_ASKPASS=")
	cmd.Env = env
	out, err := cmd.Output()
	if len(out) > 0 {
		log.Debug(string(out))
	}
	if err != nil {
		exErr, ok := err.(*exec.ExitError)
		if ok {
			errOutput := strings.Split(string(exErr.Stderr), "\n")[0]
			log.Debug(errOutput)
			return string(out), fmt.Errorf("'%s' failed: %v", strings.Join(cmd.Args, " "), errOutput)
		}
		return string(out), fmt.Errorf("'%s' failed: %v", strings.Join(cmd.Args, " "), err)
	}
	return string(out), nil
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"fmt"

	"github.com/urfave/cli"
)

// Called when thaum is actually run.
func onRun(c *cli.Context) error {
	template := c.Args().Get(0)
	name := c.Args().Get(1)

	if len(c.Args()) == 0 {
		cli.ShowAppHelp(c)
		return nil
	}

	if name == "" {
		ErrorLog(fmt.Sprintf("Thaum requires a name for your %q template.", template))
		return nil
	}

	compile(template, name)

	return nil
}

func main() {
	cli.AppHelpTemplate = HELP_TEMPLATE

	app := cli.NewApp()
	app.Name = "thaum"
	app.Usage = "Generate micro-boilerplates"
	app.Action = onRun
	app.Version = "0.1.0"

	app.Run(os.Args)
}
<commit_msg>Edit "name for template" error message to include example<commit_after>package main

import (
	"os"
	"fmt"

	"github.com/urfave/cli"
)

// Called when thaum is actually run.
func onRun(c *cli.Context) error {
	template := c.Args().Get(0)
	name := c.Args().Get(1)

	if len(c.Args()) == 0 {
		cli.ShowAppHelp(c)
		return nil
	}

	if name == "" {
		ErrorLog(fmt.Sprintf("Thaum requires a name for your %q template. Example: \n\n $ thaum <template> <name> ", template))
		return nil
	}

	compile(template, name)

	return nil
}

func main() {
	cli.AppHelpTemplate = HELP_TEMPLATE

	app := cli.NewApp()
	app.Name = "thaum"
	app.Usage = "Generate micro-boilerplates"
	app.Action = onRun
	app.Version = "0.1.0"

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package times

import "time"

type Times struct {
	time.Time
	timezone string
	location time.Location
}

// This will return an instance of Times the current time localized to UTC
func Now() Times {
	timezone = "UTC"
	location, err := LoadLocation(timezone)
	now := time.Now().UTC()
	return Times{now, timezone, location}
}

// This will localixe
func (t Times) Localize() {

}
<commit_msg>moar fixes<commit_after>package times

import "time"

type Times struct {
	time.Time
	timezone string
	location *time.Location
}

// This will return an instance of Times the current time localized to UTC
func Now() Times {
	timezone := "UTC"
	location, _ := time.LoadLocation(timezone)
	now := time.Now().UTC()
	return Times{now, timezone, location}
}

// This will localixe
func (t Times) Localize() {

}
<|endoftext|>


<commit_before>/*

memory is a simple example game based on memory--where players take turn
flipping over two cards, and keeping them if they match.

*/
package memory

import (
	"github.com/jkomoros/boardgame"
	"strconv"
	"strings"
)

type gameDelegate struct {
	boardgame.DefaultGameDelegate
}

func (g *gameDelegate) Name() string {
	return "memory"
}

func (g *gameDelegate) DisplayName() string {
	return "Memory"
}

func (g *gameDelegate) DefaultNumPlayeres() int {
	return 2
}

func (g *gameDelegate) LegalNumPlayers(numPlayers int) bool {
	return numPlayers < 4 && numPlayers > 1
}

func (g *gameDelegate) EmptyGameState() boardgame.MutableGameState {

	cards := g.Manager().Chest().Deck(cardsDeckName)

	if cards == nil {
		return nil
	}

	return &gameState{
		CurrentPlayer: 0,
		HiddenCards:   boardgame.NewSizedStack(cards, len(cards.Components())),
		RevealedCards: boardgame.NewSizedStack(cards, len(cards.Components())),
	}
}

func (g *gameDelegate) EmptyPlayerState(playerIndex int) boardgame.MutablePlayerState {

	cards := g.Manager().Chest().Deck(cardsDeckName)

	if cards == nil {
		return nil
	}

	return &playerState{
		playerIndex: playerIndex,
		WonCards:    boardgame.NewGrowableStack(cards, 0),
	}
}

func (g *gameDelegate) DistributeComponentToStarterStack(state boardgame.State, c *boardgame.Component) (boardgame.Stack, error) {
	game, _ := concreteStates(state)

	return game.HiddenCards, nil

}

func (g *gameDelegate) FinishSetUp(state boardgame.MutableState) {
	game, players := concreteStates(state)

	game.HiddenCards.Shuffle()

	players[0].CardsLeftToReveal = 2
}

func (g *gameDelegate) Diagram(state boardgame.State) string {
	game, players := concreteStates(state)

	var result []string

	result = append(result, "Board")

	for i, c := range game.HiddenCards.Components() {

		//If there's no hidden card in this slot, see if there is a revealed one.
		if c == nil {
			c = game.RevealedCards.ComponentAt(i)
		}

		value := "<empty>"

		if c != nil {
			value = c.Values.(*cardValue).Type
		}

		result = append(result, "\t"+value)

	}

	result = append(result, "*****")

	for i, player := range players {
		playerName := "Player" + strconv.Itoa(i)
		if i == game.CurrentPlayer {
			playerName += " *CURRENT* " + strconv.Itoa(player.CardsLeftToReveal)
		}
		result = append(result, playerName)
		result = append(result, strconv.Itoa(player.WonCards.NumComponents()))
	}

	return strings.Join(result, "\n")
}

var policy *boardgame.StatePolicy

func (g *gameDelegate) StateSanitizationPolicy() *boardgame.StatePolicy {

	if policy == nil {
		policy = &boardgame.StatePolicy{
			Game: map[string]boardgame.GroupPolicy{
				"HiddenCards": boardgame.GroupPolicy{
					boardgame.GroupAll: boardgame.PolicyLen,
				},
			},
		}
	}

	return policy

}

func NewManager(storage boardgame.StorageManager) *boardgame.GameManager {
	chest := boardgame.NewComponentChest()

	cards := boardgame.NewDeck()

	for _, val := range cardNames {
		cards.AddComponentMulti(&cardValue{
			Type: val,
		}, 2)
	}

	cards.SetShadowValues(&cardValue{
		Type: "<hidden>",
	})

	chest.AddDeck(cardsDeckName, cards)

	manager := boardgame.NewGameManager(&gameDelegate{}, chest, storage)

	if manager == nil {
		panic("No manager returned")
	}

	manager.AddFixUpMove(&MoveAdvanceNextPlayer{})
	manager.AddFixUpMove(&MoveCaptureCards{})
	manager.AddPlayerMove(&MoveRevealCard{})
	manager.AddPlayerMove(&MoveHideCards{})

	manager.SetUp()

	return manager
}
<commit_msg>Implement CheckGameFinished logic for memory. Not tested. Part of #174.<commit_after>/*

memory is a simple example game based on memory--where players take turn
flipping over two cards, and keeping them if they match.

*/
package memory

import (
	"github.com/jkomoros/boardgame"
	"strconv"
	"strings"
)

type gameDelegate struct {
	boardgame.DefaultGameDelegate
}

func (g *gameDelegate) Name() string {
	return "memory"
}

func (g *gameDelegate) DisplayName() string {
	return "Memory"
}

func (g *gameDelegate) DefaultNumPlayeres() int {
	return 2
}

func (g *gameDelegate) LegalNumPlayers(numPlayers int) bool {
	return numPlayers < 4 && numPlayers > 1
}

func (g *gameDelegate) EmptyGameState() boardgame.MutableGameState {

	cards := g.Manager().Chest().Deck(cardsDeckName)

	if cards == nil {
		return nil
	}

	return &gameState{
		CurrentPlayer: 0,
		HiddenCards:   boardgame.NewSizedStack(cards, len(cards.Components())),
		RevealedCards: boardgame.NewSizedStack(cards, len(cards.Components())),
	}
}

func (g *gameDelegate) EmptyPlayerState(playerIndex int) boardgame.MutablePlayerState {

	cards := g.Manager().Chest().Deck(cardsDeckName)

	if cards == nil {
		return nil
	}

	return &playerState{
		playerIndex: playerIndex,
		WonCards:    boardgame.NewGrowableStack(cards, 0),
	}
}

func (g *gameDelegate) DistributeComponentToStarterStack(state boardgame.State, c *boardgame.Component) (boardgame.Stack, error) {
	game, _ := concreteStates(state)

	return game.HiddenCards, nil

}

func (g *gameDelegate) FinishSetUp(state boardgame.MutableState) {
	game, players := concreteStates(state)

	game.HiddenCards.Shuffle()

	players[0].CardsLeftToReveal = 2
}

func (g *gameDelegate) Diagram(state boardgame.State) string {
	game, players := concreteStates(state)

	var result []string

	result = append(result, "Board")

	for i, c := range game.HiddenCards.Components() {

		//If there's no hidden card in this slot, see if there is a revealed one.
		if c == nil {
			c = game.RevealedCards.ComponentAt(i)
		}

		value := "<empty>"

		if c != nil {
			value = c.Values.(*cardValue).Type
		}

		result = append(result, "\t"+value)

	}

	result = append(result, "*****")

	for i, player := range players {
		playerName := "Player" + strconv.Itoa(i)
		if i == game.CurrentPlayer {
			playerName += " *CURRENT* " + strconv.Itoa(player.CardsLeftToReveal)
		}
		result = append(result, playerName)
		result = append(result, strconv.Itoa(player.WonCards.NumComponents()))
	}

	return strings.Join(result, "\n")
}

var policy *boardgame.StatePolicy

func (g *gameDelegate) StateSanitizationPolicy() *boardgame.StatePolicy {

	if policy == nil {
		policy = &boardgame.StatePolicy{
			Game: map[string]boardgame.GroupPolicy{
				"HiddenCards": boardgame.GroupPolicy{
					boardgame.GroupAll: boardgame.PolicyLen,
				},
			},
		}
	}

	return policy

}

func (g *gameDelegate) CheckGameFinished(state boardgame.State) (finished bool, winners []int) {
	game, players := concreteStates(state)

	if game.HiddenCards.NumComponents() != 0 || game.RevealedCards.NumComponents() != 0 {
		return false, nil
	}

	//If we get to here, the game is over. Who won?
	maxScore := 0

	for _, player := range players {
		score := player.WonCards.NumComponents()
		if score > maxScore {
			maxScore = score
		}
	}

	for i, player := range players {
		score := player.WonCards.NumComponents()

		if score >= maxScore {
			winners = append(winners, i)
		}
	}

	return true, winners

}

func NewManager(storage boardgame.StorageManager) *boardgame.GameManager {
	chest := boardgame.NewComponentChest()

	cards := boardgame.NewDeck()

	for _, val := range cardNames {
		cards.AddComponentMulti(&cardValue{
			Type: val,
		}, 2)
	}

	cards.SetShadowValues(&cardValue{
		Type: "<hidden>",
	})

	chest.AddDeck(cardsDeckName, cards)

	manager := boardgame.NewGameManager(&gameDelegate{}, chest, storage)

	if manager == nil {
		panic("No manager returned")
	}

	manager.AddFixUpMove(&MoveAdvanceNextPlayer{})
	manager.AddFixUpMove(&MoveCaptureCards{})
	manager.AddPlayerMove(&MoveRevealCard{})
	manager.AddPlayerMove(&MoveHideCards{})

	manager.SetUp()

	return manager
}
<|endoftext|>


<commit_before>// Copyright 2016 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/google/syzkaller/pkg/report"
)

var (
	flagLinux = flag.String("linux", "", "path to linux")
)

func main() {
	flag.Parse()
	if len(flag.Args()) != 1 {
		fmt.Fprintf(os.Stderr, "usage: syz-symbolize [flags] kernel_log_file\n")
		flag.PrintDefaults()
		os.Exit(1)
	}
	text, err := ioutil.ReadFile(flag.Args()[0])
	if err != nil {
		fmt.Fprintf(os.Stderr, "failed to open input file: %v\n", err)
		os.Exit(1)
	}
	if console := report.ExtractConsoleOutput(text); len(console) != 0 {
		text = console
	}
	text, err = report.Symbolize(filepath.Join(*flagLinux, "vmlinux"), text, nil)
	if err != nil {
		fmt.Fprintf(os.Stderr, "failed to symbolize: %v\n", err)
		os.Exit(1)
	}
	os.Stdout.Write(text)
}
<commit_msg>tools/syz-symbolize: add report mode<commit_after>// Copyright 2016 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/google/syzkaller/pkg/report"
)

var (
	flagKernelSrc = flag.String("kernel_src", "", "path to kernel sources")
	flagKernelObj = flag.String("kernel_obj", "", "path to kernel build dir")
	flagReport    = flag.Bool("report", false, "extract report from the log")
)

func main() {
	flag.Parse()
	if len(flag.Args()) != 1 {
		fmt.Fprintf(os.Stderr, "usage: syz-symbolize [flags] kernel_log_file\n")
		flag.PrintDefaults()
		os.Exit(1)
	}
	if *flagKernelSrc == "" {
		*flagKernelSrc = *flagKernelObj
	}
	text, err := ioutil.ReadFile(flag.Args()[0])
	if err != nil {
		fmt.Fprintf(os.Stderr, "failed to open input file: %v\n", err)
		os.Exit(1)
	}
	if *flagReport {
		desc, text, _, _ := report.Parse(text, nil)
		text, err = report.Symbolize(filepath.Join(*flagKernelObj, "vmlinux"), text, nil)
		if err != nil {
			fmt.Fprintf(os.Stderr, "failed to symbolize: %v\n", err)
			os.Exit(1)
		}
		guiltyFile := report.ExtractGuiltyFile(text)
		fmt.Printf("%v\n\n", desc)
		os.Stdout.Write(text)
		fmt.Printf("\n")
		fmt.Printf("guilty file: %v\n", guiltyFile)
		if guiltyFile != "" {
			maintainers, err := report.GetMaintainers(*flagKernelSrc, guiltyFile)
			if err != nil {
				fmt.Fprintf(os.Stderr, "failed to get maintainers: %v\n", err)
				os.Exit(1)
			}
			fmt.Printf("maintainers: %v\n", maintainers)
		}
	} else {
		if console := report.ExtractConsoleOutput(text); len(console) != 0 {
			text = console
		}
		text, err = report.Symbolize(filepath.Join(*flagKernelObj, "vmlinux"), text, nil)
		if err != nil {
			fmt.Fprintf(os.Stderr, "failed to symbolize: %v\n", err)
			os.Exit(1)
		}
		os.Stdout.Write(text)
	}
}
<|endoftext|>