<commit_before>//  Copyright (c) 2017 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 		http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package mem

import (
	"math"
	"sort"

	"github.com/RoaringBitmap/roaring"
	"github.com/blevesearch/bleve/analysis"
	"github.com/blevesearch/bleve/document"
	"github.com/blevesearch/bleve/index"
)

// NewFromAnalyzedDocs places the analyzed document mutations into a new segment
func NewFromAnalyzedDocs(results []*index.AnalysisResult) *Segment {
	s := New()

	// ensure that _id field get fieldID 0
	s.getOrDefineField("_id")

	// fill Dicts/DictKeys and preallocate memory
	s.initializeDict(results)

	// walk each doc
	for _, result := range results {
		s.processDocument(result)
	}

	// go back and sort the dictKeys
	for _, dict := range s.DictKeys {
		sort.Strings(dict)
	}

	// compute memory usage of segment
	s.updateSizeInBytes()

	// professional debugging
	//
	// log.Printf("fields: %v\n", s.FieldsMap)
	// log.Printf("fieldsInv: %v\n", s.FieldsInv)
	// log.Printf("fieldsLoc: %v\n", s.FieldsLoc)
	// log.Printf("dicts: %v\n", s.Dicts)
	// log.Printf("dict keys: %v\n", s.DictKeys)
	// for i, posting := range s.Postings {
	// 	log.Printf("posting %d: %v\n", i, posting)
	// }
	// for i, freq := range s.Freqs {
	// 	log.Printf("freq %d: %v\n", i, freq)
	// }
	// for i, norm := range s.Norms {
	// 	log.Printf("norm %d: %v\n", i, norm)
	// }
	// for i, field := range s.Locfields {
	// 	log.Printf("field %d: %v\n", i, field)
	// }
	// for i, start := range s.Locstarts {
	// 	log.Printf("start %d: %v\n", i, start)
	// }
	// for i, end := range s.Locends {
	// 	log.Printf("end %d: %v\n", i, end)
	// }
	// for i, pos := range s.Locpos {
	// 	log.Printf("pos %d: %v\n", i, pos)
	// }
	// for i, apos := range s.Locarraypos {
	// 	log.Printf("apos %d: %v\n", i, apos)
	// }
	// log.Printf("stored: %v\n", s.Stored)
	// log.Printf("stored types: %v\n", s.StoredTypes)
	// log.Printf("stored pos: %v\n", s.StoredPos)

	return s
}

// fill Dicts/DictKeys and preallocate memory for postings
func (s *Segment) initializeDict(results []*index.AnalysisResult) {
	var numPostingsLists int

	numTermsPerPostingsList := make([]int, 0, 64)

	var numTokenFrequencies int

	processField := func(fieldID uint16, tf analysis.TokenFrequencies) {
		for term, _ := range tf {
			pidPlus1, exists := s.Dicts[fieldID][term]
			if !exists {
				numPostingsLists++
				pidPlus1 = uint64(numPostingsLists)
				s.Dicts[fieldID][term] = pidPlus1
				s.DictKeys[fieldID] = append(s.DictKeys[fieldID], term)
				numTermsPerPostingsList = append(numTermsPerPostingsList, 0)
			}
			pid := pidPlus1 - 1
			numTermsPerPostingsList[pid]++
		}
		numTokenFrequencies += len(tf)
	}

	for _, result := range results {
		// walk each composite field
		for _, field := range result.Document.CompositeFields {
			fieldID := uint16(s.getOrDefineField(field.Name()))
			_, tf := field.Analyze()
			processField(fieldID, tf)
		}

		// walk each field
		for i, field := range result.Document.Fields {
			fieldID := uint16(s.getOrDefineField(field.Name()))
			tf := result.Analyzed[i]
			processField(fieldID, tf)
		}
	}

	s.Postings = make([]*roaring.Bitmap, numPostingsLists)
	for i := 0; i < numPostingsLists; i++ {
		s.Postings[i] = roaring.New()
	}
	s.PostingsLocs = make([]*roaring.Bitmap, numPostingsLists)
	for i := 0; i < numPostingsLists; i++ {
		s.PostingsLocs[i] = roaring.New()
	}

	s.Freqs = make([][]uint64, numPostingsLists)
	s.Norms = make([][]float32, numPostingsLists)
	s.Locfields = make([][]uint16, numPostingsLists)
	s.Locstarts = make([][]uint64, numPostingsLists)
	s.Locends = make([][]uint64, numPostingsLists)
	s.Locpos = make([][]uint64, numPostingsLists)
	s.Locarraypos = make([][][]uint64, numPostingsLists)

	uint64Backing := make([]uint64, numTokenFrequencies)
	float32Backing := make([]float32, numTokenFrequencies)

	for i, numTerms := range numTermsPerPostingsList {
		s.Freqs[i] = uint64Backing[0:0]
		uint64Backing = uint64Backing[numTerms:]

		s.Norms[i] = float32Backing[0:0]
		float32Backing = float32Backing[numTerms:]
	}
}

func (s *Segment) processDocument(result *index.AnalysisResult) {
	// used to collate information across fields
	docMap := make(map[uint16]analysis.TokenFrequencies, len(s.FieldsMap))
	fieldLens := make(map[uint16]int, len(s.FieldsMap))

	docNum := uint64(s.addDocument())

	processField := func(field uint16, name string, l int, tf analysis.TokenFrequencies) {
		fieldLens[field] += l
		if existingFreqs, ok := docMap[field]; ok {
			existingFreqs.MergeAll(name, tf)
		} else {
			docMap[field] = tf
		}
	}

	storeField := func(docNum uint64, field uint16, typ byte, val []byte, pos []uint64) {
		s.Stored[docNum][field] = append(s.Stored[docNum][field], val)
		s.StoredTypes[docNum][field] = append(s.StoredTypes[docNum][field], typ)
		s.StoredPos[docNum][field] = append(s.StoredPos[docNum][field], pos)
	}

	// walk each composite field
	for _, field := range result.Document.CompositeFields {
		fieldID := uint16(s.getOrDefineField(field.Name()))
		l, tf := field.Analyze()
		processField(fieldID, field.Name(), l, tf)
	}

	// walk each field
	for i, field := range result.Document.Fields {
		fieldID := uint16(s.getOrDefineField(field.Name()))
		l := result.Length[i]
		tf := result.Analyzed[i]
		processField(fieldID, field.Name(), l, tf)
		if field.Options().IsStored() {
			storeField(docNum, fieldID, encodeFieldType(field), field.Value(), field.ArrayPositions())
		}

		if field.Options().IncludeDocValues() {
			s.DocValueFields[fieldID] = true
		}
	}

	// now that its been rolled up into docMap, walk that
	for fieldID, tokenFrequencies := range docMap {
		for term, tokenFreq := range tokenFrequencies {
			pid := s.Dicts[fieldID][term]-1
			bs := s.Postings[pid]
			bs.AddInt(int(docNum))
			s.Freqs[pid] = append(s.Freqs[pid], uint64(tokenFreq.Frequency()))
			s.Norms[pid] = append(s.Norms[pid], float32(1.0/math.Sqrt(float64(fieldLens[fieldID]))))
			locationBS := s.PostingsLocs[pid]
			if len(tokenFreq.Locations) > 0 {
				locationBS.AddInt(int(docNum))
				for _, loc := range tokenFreq.Locations {
					var locf = fieldID
					if loc.Field != "" {
						locf = uint16(s.getOrDefineField(loc.Field))
					}
					s.Locfields[pid] = append(s.Locfields[pid], locf)
					s.Locstarts[pid] = append(s.Locstarts[pid], uint64(loc.Start))
					s.Locends[pid] = append(s.Locends[pid], uint64(loc.End))
					s.Locpos[pid] = append(s.Locpos[pid], uint64(loc.Position))
					if len(loc.ArrayPositions) > 0 {
						s.Locarraypos[pid] = append(s.Locarraypos[pid], loc.ArrayPositions)
					} else {
						s.Locarraypos[pid] = append(s.Locarraypos[pid], nil)
					}
				}
			}
		}
	}
}

func (s *Segment) getOrDefineField(name string) int {
	fieldID, ok := s.FieldsMap[name]
	if !ok {
		fieldID = uint16(len(s.FieldsInv) + 1)
		s.FieldsMap[name] = fieldID
		s.FieldsInv = append(s.FieldsInv, name)
		s.Dicts = append(s.Dicts, make(map[string]uint64))
		s.DictKeys = append(s.DictKeys, make([]string, 0))
	}
	return int(fieldID - 1)
}

func (s *Segment) addDocument() int {
	docNum := len(s.Stored)
	s.Stored = append(s.Stored, map[uint16][][]byte{})
	s.StoredTypes = append(s.StoredTypes, map[uint16][]byte{})
	s.StoredPos = append(s.StoredPos, map[uint16][][]uint64{})
	return docNum
}

func encodeFieldType(f document.Field) byte {
	fieldType := byte('x')
	switch f.(type) {
	case *document.TextField:
		fieldType = 't'
	case *document.NumericField:
		fieldType = 'n'
	case *document.DateTimeField:
		fieldType = 'd'
	case *document.BooleanField:
		fieldType = 'b'
	case *document.GeoPointField:
		fieldType = 'g'
	case *document.CompositeField:
		fieldType = 'c'
	}
	return fieldType
}
<commit_msg>scorch mem segment prealloc's Locfields/starts/ends/pos/arraypos<commit_after>//  Copyright (c) 2017 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 		http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package mem

import (
	"math"
	"sort"

	"github.com/RoaringBitmap/roaring"
	"github.com/blevesearch/bleve/analysis"
	"github.com/blevesearch/bleve/document"
	"github.com/blevesearch/bleve/index"
)

// NewFromAnalyzedDocs places the analyzed document mutations into a new segment
func NewFromAnalyzedDocs(results []*index.AnalysisResult) *Segment {
	s := New()

	// ensure that _id field get fieldID 0
	s.getOrDefineField("_id")

	// fill Dicts/DictKeys and preallocate memory
	s.initializeDict(results)

	// walk each doc
	for _, result := range results {
		s.processDocument(result)
	}

	// go back and sort the dictKeys
	for _, dict := range s.DictKeys {
		sort.Strings(dict)
	}

	// compute memory usage of segment
	s.updateSizeInBytes()

	// professional debugging
	//
	// log.Printf("fields: %v\n", s.FieldsMap)
	// log.Printf("fieldsInv: %v\n", s.FieldsInv)
	// log.Printf("fieldsLoc: %v\n", s.FieldsLoc)
	// log.Printf("dicts: %v\n", s.Dicts)
	// log.Printf("dict keys: %v\n", s.DictKeys)
	// for i, posting := range s.Postings {
	// 	log.Printf("posting %d: %v\n", i, posting)
	// }
	// for i, freq := range s.Freqs {
	// 	log.Printf("freq %d: %v\n", i, freq)
	// }
	// for i, norm := range s.Norms {
	// 	log.Printf("norm %d: %v\n", i, norm)
	// }
	// for i, field := range s.Locfields {
	// 	log.Printf("field %d: %v\n", i, field)
	// }
	// for i, start := range s.Locstarts {
	// 	log.Printf("start %d: %v\n", i, start)
	// }
	// for i, end := range s.Locends {
	// 	log.Printf("end %d: %v\n", i, end)
	// }
	// for i, pos := range s.Locpos {
	// 	log.Printf("pos %d: %v\n", i, pos)
	// }
	// for i, apos := range s.Locarraypos {
	// 	log.Printf("apos %d: %v\n", i, apos)
	// }
	// log.Printf("stored: %v\n", s.Stored)
	// log.Printf("stored types: %v\n", s.StoredTypes)
	// log.Printf("stored pos: %v\n", s.StoredPos)

	return s
}

// fill Dicts/DictKeys and preallocate memory for postings
func (s *Segment) initializeDict(results []*index.AnalysisResult) {
	var numPostingsLists int

	numTermsPerPostingsList := make([]int, 0, 64) // Keyed by postings list id.
	numLocsPerPostingsList := make([]int, 0, 64)  // Keyed by postings list id.

	var numTokenFrequencies int
	var numLocs int

	processField := func(fieldID uint16, tfs analysis.TokenFrequencies) {
		for term, tf := range tfs {
			pidPlus1, exists := s.Dicts[fieldID][term]
			if !exists {
				numPostingsLists++
				pidPlus1 = uint64(numPostingsLists)
				s.Dicts[fieldID][term] = pidPlus1
				s.DictKeys[fieldID] = append(s.DictKeys[fieldID], term)
				numTermsPerPostingsList = append(numTermsPerPostingsList, 0)
				numLocsPerPostingsList = append(numLocsPerPostingsList, 0)
			}
			pid := pidPlus1 - 1
			numTermsPerPostingsList[pid] += 1
			numLocsPerPostingsList[pid] += len(tf.Locations)
			numLocs += len(tf.Locations)
		}
		numTokenFrequencies += len(tfs)
	}

	for _, result := range results {
		// walk each composite field
		for _, field := range result.Document.CompositeFields {
			fieldID := uint16(s.getOrDefineField(field.Name()))
			_, tf := field.Analyze()
			processField(fieldID, tf)
		}

		// walk each field
		for i, field := range result.Document.Fields {
			fieldID := uint16(s.getOrDefineField(field.Name()))
			tf := result.Analyzed[i]
			processField(fieldID, tf)
		}
	}

	s.Postings = make([]*roaring.Bitmap, numPostingsLists)
	for i := 0; i < numPostingsLists; i++ {
		s.Postings[i] = roaring.New()
	}
	s.PostingsLocs = make([]*roaring.Bitmap, numPostingsLists)
	for i := 0; i < numPostingsLists; i++ {
		s.PostingsLocs[i] = roaring.New()
	}

	s.Freqs = make([][]uint64, numPostingsLists)
	s.Norms = make([][]float32, numPostingsLists)

	uint64Backing := make([]uint64, numTokenFrequencies)
	float32Backing := make([]float32, numTokenFrequencies)

	for pid, numTerms := range numTermsPerPostingsList {
		s.Freqs[pid] = uint64Backing[0:0]
		uint64Backing = uint64Backing[numTerms:]

		s.Norms[pid] = float32Backing[0:0]
		float32Backing = float32Backing[numTerms:]
	}

	s.Locfields = make([][]uint16, numPostingsLists)
	s.Locstarts = make([][]uint64, numPostingsLists)
	s.Locends = make([][]uint64, numPostingsLists)
	s.Locpos = make([][]uint64, numPostingsLists)
	s.Locarraypos = make([][][]uint64, numPostingsLists)

	uint16Backing := make([]uint16, numLocs)    // For Locfields.
	uint64Backing = make([]uint64, numLocs*3)   // For Locstarts, Locends, Locpos.
	auint64Backing := make([][]uint64, numLocs) // For Locarraypos.

	for pid, numLocs := range numLocsPerPostingsList {
		s.Locfields[pid] = uint16Backing[0:0]
		uint16Backing = uint16Backing[numLocs:]

		s.Locstarts[pid] = uint64Backing[0:0]
		uint64Backing = uint64Backing[numLocs:]

		s.Locends[pid] = uint64Backing[0:0]
		uint64Backing = uint64Backing[numLocs:]

		s.Locpos[pid] = uint64Backing[0:0]
		uint64Backing = uint64Backing[numLocs:]

		s.Locarraypos[pid] = auint64Backing[0:0]
		auint64Backing = auint64Backing[numLocs:]
	}
}

func (s *Segment) processDocument(result *index.AnalysisResult) {
	// used to collate information across fields
	docMap := make(map[uint16]analysis.TokenFrequencies, len(s.FieldsMap))
	fieldLens := make(map[uint16]int, len(s.FieldsMap))

	docNum := uint64(s.addDocument())

	processField := func(field uint16, name string, l int, tf analysis.TokenFrequencies) {
		fieldLens[field] += l
		if existingFreqs, ok := docMap[field]; ok {
			existingFreqs.MergeAll(name, tf)
		} else {
			docMap[field] = tf
		}
	}

	storeField := func(docNum uint64, field uint16, typ byte, val []byte, pos []uint64) {
		s.Stored[docNum][field] = append(s.Stored[docNum][field], val)
		s.StoredTypes[docNum][field] = append(s.StoredTypes[docNum][field], typ)
		s.StoredPos[docNum][field] = append(s.StoredPos[docNum][field], pos)
	}

	// walk each composite field
	for _, field := range result.Document.CompositeFields {
		fieldID := uint16(s.getOrDefineField(field.Name()))
		l, tf := field.Analyze()
		processField(fieldID, field.Name(), l, tf)
	}

	// walk each field
	for i, field := range result.Document.Fields {
		fieldID := uint16(s.getOrDefineField(field.Name()))
		l := result.Length[i]
		tf := result.Analyzed[i]
		processField(fieldID, field.Name(), l, tf)
		if field.Options().IsStored() {
			storeField(docNum, fieldID, encodeFieldType(field), field.Value(), field.ArrayPositions())
		}

		if field.Options().IncludeDocValues() {
			s.DocValueFields[fieldID] = true
		}
	}

	// now that its been rolled up into docMap, walk that
	for fieldID, tokenFrequencies := range docMap {
		for term, tokenFreq := range tokenFrequencies {
			pid := s.Dicts[fieldID][term] - 1
			bs := s.Postings[pid]
			bs.AddInt(int(docNum))
			s.Freqs[pid] = append(s.Freqs[pid], uint64(tokenFreq.Frequency()))
			s.Norms[pid] = append(s.Norms[pid], float32(1.0/math.Sqrt(float64(fieldLens[fieldID]))))
			locationBS := s.PostingsLocs[pid]
			if len(tokenFreq.Locations) > 0 {
				locationBS.AddInt(int(docNum))
				for _, loc := range tokenFreq.Locations {
					var locf = fieldID
					if loc.Field != "" {
						locf = uint16(s.getOrDefineField(loc.Field))
					}
					s.Locfields[pid] = append(s.Locfields[pid], locf)
					s.Locstarts[pid] = append(s.Locstarts[pid], uint64(loc.Start))
					s.Locends[pid] = append(s.Locends[pid], uint64(loc.End))
					s.Locpos[pid] = append(s.Locpos[pid], uint64(loc.Position))
					if len(loc.ArrayPositions) > 0 {
						s.Locarraypos[pid] = append(s.Locarraypos[pid], loc.ArrayPositions)
					} else {
						s.Locarraypos[pid] = append(s.Locarraypos[pid], nil)
					}
				}
			}
		}
	}
}

func (s *Segment) getOrDefineField(name string) int {
	fieldID, ok := s.FieldsMap[name]
	if !ok {
		fieldID = uint16(len(s.FieldsInv) + 1)
		s.FieldsMap[name] = fieldID
		s.FieldsInv = append(s.FieldsInv, name)
		s.Dicts = append(s.Dicts, make(map[string]uint64))
		s.DictKeys = append(s.DictKeys, make([]string, 0))
	}
	return int(fieldID - 1)
}

func (s *Segment) addDocument() int {
	docNum := len(s.Stored)
	s.Stored = append(s.Stored, map[uint16][][]byte{})
	s.StoredTypes = append(s.StoredTypes, map[uint16][]byte{})
	s.StoredPos = append(s.StoredPos, map[uint16][][]uint64{})
	return docNum
}

func encodeFieldType(f document.Field) byte {
	fieldType := byte('x')
	switch f.(type) {
	case *document.TextField:
		fieldType = 't'
	case *document.NumericField:
		fieldType = 'n'
	case *document.DateTimeField:
		fieldType = 'd'
	case *document.BooleanField:
		fieldType = 'b'
	case *document.GeoPointField:
		fieldType = 'g'
	case *document.CompositeField:
		fieldType = 'c'
	}
	return fieldType
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"compress/lzw"
	"crypto/md5"
	"encoding/gob"
	"fmt"
	"log"
	"path"
	"sync"
	"time"

	dep "github.com/hashicorp/consul-template/dependency"
	consulapi "github.com/hashicorp/consul/api"
)

const (
	// sessionCreateRetry is the amount of time we wait
	// to recreate a session when lost.
	sessionCreateRetry = 15 * time.Second

	// lockRetry is the interval on which we try to re-acquire locks
	lockRetry = 10 * time.Second

	// listRetry is the interval on which we retry listing a data path
	listRetry = 10 * time.Second

	// templateDataFlag is added as a flag to the shared data values
	// so that we can use it as a sanity check
	templateDataFlag = 0x22b9a127a2c03520
)

// templateData is GOB encoded share the depdency values
type templateData struct {
	Data map[string]interface{}
}

// DedupManager is used to de-duplicate which instance of Consul-Template
// is handling each template. For each template, a lock path is determined
// using the MD5 of the template. This path is used to elect a "leader"
// instance.
//
// The leader instance operations like usual, but any time a template is
// rendered, any of the data required for rendering is stored in the
// Consul KV store under the lock path.
//
// The follower instances depend on the leader to do the primary watching
// and rendering, and instead only watch the aggregated data in the KV.
// Followers wait for updates and re-render the template.
//
// If a template depends on 50 views, and is running on 50 machines, that
// would normally require 2500 blocking queries. Using deduplication, one
// instance has 50 view queries, plus 50 additional queries on the lock
// path for a total of 100.
//
type DedupManager struct {
	// config is the consul-template configuration
	config *Config

	// clients is used to access the underlying clinets
	clients *dep.ClientSet

	// Brain is where we inject udpates
	brain *Brain

	// templates is the set of templates we are trying to dedup
	templates []*Template

	// leader tracks if we are currently the leader
	leader     map[*Template]<-chan struct{}
	leaderLock sync.RWMutex

	// lastWrite tracks the hash of the data paths
	lastWrite     map[*Template][]byte
	lastWriteLock sync.RWMutex

	// updateCh is used to indicate an update watched data
	updateCh chan struct{}

	// wg is used to wait for a clean shutdown
	wg sync.WaitGroup

	stop     bool
	stopCh   chan struct{}
	stopLock sync.Mutex
}

// NewDedupManager creates a new Dedup manager
func NewDedupManager(config *Config, clients *dep.ClientSet, brain *Brain, templates []*Template) (*DedupManager, error) {
	d := &DedupManager{
		config:    config,
		clients:   clients,
		brain:     brain,
		templates: templates,
		leader:    make(map[*Template]<-chan struct{}),
		lastWrite: make(map[*Template][]byte),
		updateCh:  make(chan struct{}, 1),
		stopCh:    make(chan struct{}),
	}
	return d, nil
}

// Start is used to start the de-duplication manager
func (d *DedupManager) Start() error {
	log.Printf("[INFO] (dedup) starting de-duplication manager")

	client, err := d.clients.Consul()
	if err != nil {
		return err
	}
	go d.createSession(client)

	// Start to watch each template
	for _, t := range d.templates {
		go d.watchTemplate(client, t)
	}
	return nil
}

// Stop is used to stop the de-duplication manager
func (d *DedupManager) Stop() error {
	d.stopLock.Lock()
	defer d.stopLock.Unlock()
	if d.stop {
		return nil
	}

	log.Printf("[INFO] (dedup) stopping de-duplication manager")
	d.stop = true
	close(d.stopCh)
	d.wg.Wait()
	return nil
}

// createSession is used to create and maintain a session to Consul
func (d *DedupManager) createSession(client *consulapi.Client) {
START:
	log.Printf("[INFO] (dedup) attempting to create session")
	session := client.Session()
	sessionCh := make(chan struct{})
	ttl := fmt.Sprintf("%ds", d.config.Deduplicate.TTL/time.Second)
	se := &consulapi.SessionEntry{
		Name:     "Consul-Template de-duplication",
		Behavior: "delete",
		TTL:      ttl,
	}
	id, _, err := session.Create(se, nil)
	if err != nil {
		log.Printf("[ERR] (dedup) failed to create session: %v", err)
		goto WAIT
	}
	log.Printf("[INFO] (dedup) created session %s", id)

	// Attempt to lock each template
	for _, t := range d.templates {
		d.wg.Add(1)
		go d.attemptLock(client, id, sessionCh, t)
	}

	// Renew our session periodically
	if err := session.RenewPeriodic("15s", id, nil, d.stopCh); err != nil {
		log.Printf("[ERR] (dedup) failed to renew session: %v", err)
	}
	close(sessionCh)

WAIT:
	select {
	case <-time.After(sessionCreateRetry):
		goto START
	case <-d.stopCh:
		return
	}
}

// IsLeader checks if we are currently the leader instance
func (d *DedupManager) IsLeader(tmpl *Template) bool {
	d.leaderLock.RLock()
	defer d.leaderLock.RUnlock()

	lockCh, ok := d.leader[tmpl]
	if !ok {
		return false
	}
	select {
	case <-lockCh:
		return false
	default:
		return true
	}
}

// UpdateDeps is used to update the values of the dependencies for a template
func (d *DedupManager) UpdateDeps(t *Template, deps []dep.Dependency) error {
	// Calculate the path to write updates to
	dataPath := path.Join(d.config.Deduplicate.Prefix, t.hexMD5, "data")

	// Package up the dependency data
	td := templateData{
		Data: make(map[string]interface{}),
	}
	for _, dp := range deps {
		// Skip any dependencies that can't be shared
		if !dep.CanShare(dp) {
			continue
		}

		// Pull the current value from the brain
		val, ok := d.brain.Recall(dp)
		if ok {
			td.Data[dp.HashCode()] = val
		}
	}

	// Encode via GOB and LZW compress
	var buf bytes.Buffer
	compress := lzw.NewWriter(&buf, lzw.LSB, 8)
	enc := gob.NewEncoder(compress)
	if err := enc.Encode(&td); err != nil {
		return fmt.Errorf("encode failed: %v", err)
	}
	compress.Close()

	// Compute MD5 of the buffer
	hash := md5.Sum(buf.Bytes())
	d.lastWriteLock.RLock()
	existing, ok := d.lastWrite[t]
	d.lastWriteLock.RUnlock()
	if ok && bytes.Equal(existing, hash[:]) {
		log.Printf("[INFO] (dedup) de-duplicate data '%s' already current",
			dataPath)
		return nil
	}

	// Write the KV update
	kvPair := consulapi.KVPair{
		Key:   dataPath,
		Value: buf.Bytes(),
		Flags: templateDataFlag,
	}
	client, err := d.clients.Consul()
	if err != nil {
		return fmt.Errorf("failed to get consul client: %v", err)
	}
	if _, err := client.KV().Put(&kvPair, nil); err != nil {
		return fmt.Errorf("failed to write '%s': %v", dataPath, err)
	}
	log.Printf("[INFO] (dedup) updated de-duplicate data '%s'", dataPath)
	d.lastWriteLock.Lock()
	d.lastWrite[t] = hash[:]
	d.lastWriteLock.Unlock()
	return nil
}

// UpdateCh returns a channel to watch for depedency updates
func (d *DedupManager) UpdateCh() <-chan struct{} {
	return d.updateCh
}

// setLeader sets if we are currently the leader instance
func (d *DedupManager) setLeader(tmpl *Template, lockCh <-chan struct{}) {
	// Update the lock state
	d.leaderLock.Lock()
	if lockCh != nil {
		d.leader[tmpl] = lockCh
	} else {
		delete(d.leader, tmpl)
	}
	d.leaderLock.Unlock()

	// Clear the lastWrite hash if we've lost leadership
	if lockCh == nil {
		d.lastWriteLock.Lock()
		delete(d.lastWrite, tmpl)
		d.lastWriteLock.Unlock()
	}

	// Do an async notify of an update
	select {
	case d.updateCh <- struct{}{}:
	default:
	}
}

func (d *DedupManager) watchTemplate(client *consulapi.Client, t *Template) {
	log.Printf("[INFO] (dedup) starting watch for template hash %s", t.hexMD5)
	path := path.Join(d.config.Deduplicate.Prefix, t.hexMD5, "data")

	// Determine if stale queries are allowed
	var allowStale bool
	if d.config.MaxStale != 0 {
		allowStale = true
	}

	// Setup our query options
	opts := &consulapi.QueryOptions{
		AllowStale: allowStale,
		WaitTime:   60 * time.Second,
	}

START:
	// Stop listening if we're stopped
	select {
	case <-d.stopCh:
		return
	default:
	}

	// If we are current the leader, wait for leadership lost
	d.leaderLock.RLock()
	lockCh, ok := d.leader[t]
	d.leaderLock.RUnlock()
	if ok {
		select {
		case <-lockCh:
			goto START
		case <-d.stopCh:
			return
		}
	}

	// Block for updates on the data key
	log.Printf("[INFO] (dedup) listing data for template hash %s", t.hexMD5)
	pair, meta, err := client.KV().Get(path, opts)
	if err != nil {
		log.Printf("[ERR] (dedup) failed to get '%s': %v", path, err)
		select {
		case <-time.After(listRetry):
			goto START
		case <-d.stopCh:
			return
		}
	}
	opts.WaitIndex = meta.LastIndex

	// If we've exceeded the maximum staleness, retry without stale
	if allowStale && meta.LastContact > d.config.MaxStale {
		allowStale = false
		log.Printf("[DEBUG] (dedup) %s stale data (last contact exceeded max_stale)", path)
		goto START
	}

	// Re-enable stale queries if allowed
	if d.config.MaxStale != 0 {
		allowStale = true
	}

	// Stop listening if we're stopped
	select {
	case <-d.stopCh:
		return
	default:
	}

	// If we are current the leader, wait for leadership lost
	d.leaderLock.RLock()
	lockCh, ok = d.leader[t]
	d.leaderLock.RUnlock()
	if ok {
		select {
		case <-lockCh:
			goto START
		case <-d.stopCh:
			return
		}
	}

	// Parse the data file
	if pair != nil && pair.Flags == templateDataFlag {
		d.parseData(pair.Key, pair.Value)
	}
	goto START
}

// parseData is used to update brain from a KV data pair
func (d *DedupManager) parseData(path string, raw []byte) {
	// Setup the decompression and decoders
	r := bytes.NewReader(raw)
	decompress := lzw.NewReader(r, lzw.LSB, 8)
	defer decompress.Close()
	dec := gob.NewDecoder(decompress)

	// Decode the data
	var td templateData
	if err := dec.Decode(&td); err != nil {
		log.Printf("[ERR] (dedup) failed to decode '%s': %v",
			path, err)
		return
	}
	log.Printf("[INFO] (dedup) loading %d dependencies from '%s'",
		len(td.Data), path)

	// Update the data in the brain
	for hashCode, value := range td.Data {
		d.brain.ForceSet(hashCode, value)
	}

	// Trigger the updateCh
	select {
	case d.updateCh <- struct{}{}:
	default:
	}
}

func (d *DedupManager) attemptLock(client *consulapi.Client, session string, sessionCh chan struct{}, t *Template) {
	defer d.wg.Done()
START:
	log.Printf("[INFO] (dedup) attempting lock for template hash %s", t.hexMD5)
	basePath := path.Join(d.config.Deduplicate.Prefix, t.hexMD5)
	lopts := &consulapi.LockOptions{
		Key:     path.Join(basePath, "lock"),
		Session: session,
	}
	lock, err := client.LockOpts(lopts)
	if err != nil {
		log.Printf("[ERR] (dedup) failed to create lock '%s': %v",
			lopts.Key, err)
		return
	}

	var retryCh <-chan time.Time
	leaderCh, err := lock.Lock(sessionCh)
	if err != nil {
		log.Printf("[ERR] (dedup) failed to acquire lock '%s': %v",
			lopts.Key, err)
		retryCh = time.After(lockRetry)
	} else {
		log.Printf("[INFO] (dedup) acquired lock '%s'", lopts.Key)
		d.setLeader(t, leaderCh)
	}

	select {
	case <-retryCh:
		retryCh = nil
		goto START
	case <-leaderCh:
		log.Printf("[WARN] (dedup) lost lock ownership '%s'", lopts.Key)
		d.setLeader(t, nil)
		goto START
	case <-sessionCh:
		log.Printf("[INFO] (dedup) releasing lock '%s'", lopts.Key)
		d.setLeader(t, nil)
		lock.Unlock()
	case <-d.stopCh:
		log.Printf("[INFO] (dedup) releasing lock '%s'", lopts.Key)
		lock.Unlock()
	}
}
<commit_msg>Use new MonitorRetries for robustness<commit_after>package main

import (
	"bytes"
	"compress/lzw"
	"crypto/md5"
	"encoding/gob"
	"fmt"
	"log"
	"path"
	"sync"
	"time"

	dep "github.com/hashicorp/consul-template/dependency"
	consulapi "github.com/hashicorp/consul/api"
)

const (
	// sessionCreateRetry is the amount of time we wait
	// to recreate a session when lost.
	sessionCreateRetry = 15 * time.Second

	// lockRetry is the interval on which we try to re-acquire locks
	lockRetry = 10 * time.Second

	// listRetry is the interval on which we retry listing a data path
	listRetry = 10 * time.Second

	// templateDataFlag is added as a flag to the shared data values
	// so that we can use it as a sanity check
	templateDataFlag = 0x22b9a127a2c03520
)

// templateData is GOB encoded share the depdency values
type templateData struct {
	Data map[string]interface{}
}

// DedupManager is used to de-duplicate which instance of Consul-Template
// is handling each template. For each template, a lock path is determined
// using the MD5 of the template. This path is used to elect a "leader"
// instance.
//
// The leader instance operations like usual, but any time a template is
// rendered, any of the data required for rendering is stored in the
// Consul KV store under the lock path.
//
// The follower instances depend on the leader to do the primary watching
// and rendering, and instead only watch the aggregated data in the KV.
// Followers wait for updates and re-render the template.
//
// If a template depends on 50 views, and is running on 50 machines, that
// would normally require 2500 blocking queries. Using deduplication, one
// instance has 50 view queries, plus 50 additional queries on the lock
// path for a total of 100.
//
type DedupManager struct {
	// config is the consul-template configuration
	config *Config

	// clients is used to access the underlying clinets
	clients *dep.ClientSet

	// Brain is where we inject udpates
	brain *Brain

	// templates is the set of templates we are trying to dedup
	templates []*Template

	// leader tracks if we are currently the leader
	leader     map[*Template]<-chan struct{}
	leaderLock sync.RWMutex

	// lastWrite tracks the hash of the data paths
	lastWrite     map[*Template][]byte
	lastWriteLock sync.RWMutex

	// updateCh is used to indicate an update watched data
	updateCh chan struct{}

	// wg is used to wait for a clean shutdown
	wg sync.WaitGroup

	stop     bool
	stopCh   chan struct{}
	stopLock sync.Mutex
}

// NewDedupManager creates a new Dedup manager
func NewDedupManager(config *Config, clients *dep.ClientSet, brain *Brain, templates []*Template) (*DedupManager, error) {
	d := &DedupManager{
		config:    config,
		clients:   clients,
		brain:     brain,
		templates: templates,
		leader:    make(map[*Template]<-chan struct{}),
		lastWrite: make(map[*Template][]byte),
		updateCh:  make(chan struct{}, 1),
		stopCh:    make(chan struct{}),
	}
	return d, nil
}

// Start is used to start the de-duplication manager
func (d *DedupManager) Start() error {
	log.Printf("[INFO] (dedup) starting de-duplication manager")

	client, err := d.clients.Consul()
	if err != nil {
		return err
	}
	go d.createSession(client)

	// Start to watch each template
	for _, t := range d.templates {
		go d.watchTemplate(client, t)
	}
	return nil
}

// Stop is used to stop the de-duplication manager
func (d *DedupManager) Stop() error {
	d.stopLock.Lock()
	defer d.stopLock.Unlock()
	if d.stop {
		return nil
	}

	log.Printf("[INFO] (dedup) stopping de-duplication manager")
	d.stop = true
	close(d.stopCh)
	d.wg.Wait()
	return nil
}

// createSession is used to create and maintain a session to Consul
func (d *DedupManager) createSession(client *consulapi.Client) {
START:
	log.Printf("[INFO] (dedup) attempting to create session")
	session := client.Session()
	sessionCh := make(chan struct{})
	ttl := fmt.Sprintf("%ds", d.config.Deduplicate.TTL/time.Second)
	se := &consulapi.SessionEntry{
		Name:     "Consul-Template de-duplication",
		Behavior: "delete",
		TTL:      ttl,
	}
	id, _, err := session.Create(se, nil)
	if err != nil {
		log.Printf("[ERR] (dedup) failed to create session: %v", err)
		goto WAIT
	}
	log.Printf("[INFO] (dedup) created session %s", id)

	// Attempt to lock each template
	for _, t := range d.templates {
		d.wg.Add(1)
		go d.attemptLock(client, id, sessionCh, t)
	}

	// Renew our session periodically
	if err := session.RenewPeriodic("15s", id, nil, d.stopCh); err != nil {
		log.Printf("[ERR] (dedup) failed to renew session: %v", err)
	}
	close(sessionCh)

WAIT:
	select {
	case <-time.After(sessionCreateRetry):
		goto START
	case <-d.stopCh:
		return
	}
}

// IsLeader checks if we are currently the leader instance
func (d *DedupManager) IsLeader(tmpl *Template) bool {
	d.leaderLock.RLock()
	defer d.leaderLock.RUnlock()

	lockCh, ok := d.leader[tmpl]
	if !ok {
		return false
	}
	select {
	case <-lockCh:
		return false
	default:
		return true
	}
}

// UpdateDeps is used to update the values of the dependencies for a template
func (d *DedupManager) UpdateDeps(t *Template, deps []dep.Dependency) error {
	// Calculate the path to write updates to
	dataPath := path.Join(d.config.Deduplicate.Prefix, t.hexMD5, "data")

	// Package up the dependency data
	td := templateData{
		Data: make(map[string]interface{}),
	}
	for _, dp := range deps {
		// Skip any dependencies that can't be shared
		if !dep.CanShare(dp) {
			continue
		}

		// Pull the current value from the brain
		val, ok := d.brain.Recall(dp)
		if ok {
			td.Data[dp.HashCode()] = val
		}
	}

	// Encode via GOB and LZW compress
	var buf bytes.Buffer
	compress := lzw.NewWriter(&buf, lzw.LSB, 8)
	enc := gob.NewEncoder(compress)
	if err := enc.Encode(&td); err != nil {
		return fmt.Errorf("encode failed: %v", err)
	}
	compress.Close()

	// Compute MD5 of the buffer
	hash := md5.Sum(buf.Bytes())
	d.lastWriteLock.RLock()
	existing, ok := d.lastWrite[t]
	d.lastWriteLock.RUnlock()
	if ok && bytes.Equal(existing, hash[:]) {
		log.Printf("[INFO] (dedup) de-duplicate data '%s' already current",
			dataPath)
		return nil
	}

	// Write the KV update
	kvPair := consulapi.KVPair{
		Key:   dataPath,
		Value: buf.Bytes(),
		Flags: templateDataFlag,
	}
	client, err := d.clients.Consul()
	if err != nil {
		return fmt.Errorf("failed to get consul client: %v", err)
	}
	if _, err := client.KV().Put(&kvPair, nil); err != nil {
		return fmt.Errorf("failed to write '%s': %v", dataPath, err)
	}
	log.Printf("[INFO] (dedup) updated de-duplicate data '%s'", dataPath)
	d.lastWriteLock.Lock()
	d.lastWrite[t] = hash[:]
	d.lastWriteLock.Unlock()
	return nil
}

// UpdateCh returns a channel to watch for depedency updates
func (d *DedupManager) UpdateCh() <-chan struct{} {
	return d.updateCh
}

// setLeader sets if we are currently the leader instance
func (d *DedupManager) setLeader(tmpl *Template, lockCh <-chan struct{}) {
	// Update the lock state
	d.leaderLock.Lock()
	if lockCh != nil {
		d.leader[tmpl] = lockCh
	} else {
		delete(d.leader, tmpl)
	}
	d.leaderLock.Unlock()

	// Clear the lastWrite hash if we've lost leadership
	if lockCh == nil {
		d.lastWriteLock.Lock()
		delete(d.lastWrite, tmpl)
		d.lastWriteLock.Unlock()
	}

	// Do an async notify of an update
	select {
	case d.updateCh <- struct{}{}:
	default:
	}
}

func (d *DedupManager) watchTemplate(client *consulapi.Client, t *Template) {
	log.Printf("[INFO] (dedup) starting watch for template hash %s", t.hexMD5)
	path := path.Join(d.config.Deduplicate.Prefix, t.hexMD5, "data")

	// Determine if stale queries are allowed
	var allowStale bool
	if d.config.MaxStale != 0 {
		allowStale = true
	}

	// Setup our query options
	opts := &consulapi.QueryOptions{
		AllowStale: allowStale,
		WaitTime:   60 * time.Second,
	}

START:
	// Stop listening if we're stopped
	select {
	case <-d.stopCh:
		return
	default:
	}

	// If we are current the leader, wait for leadership lost
	d.leaderLock.RLock()
	lockCh, ok := d.leader[t]
	d.leaderLock.RUnlock()
	if ok {
		select {
		case <-lockCh:
			goto START
		case <-d.stopCh:
			return
		}
	}

	// Block for updates on the data key
	log.Printf("[INFO] (dedup) listing data for template hash %s", t.hexMD5)
	pair, meta, err := client.KV().Get(path, opts)
	if err != nil {
		log.Printf("[ERR] (dedup) failed to get '%s': %v", path, err)
		select {
		case <-time.After(listRetry):
			goto START
		case <-d.stopCh:
			return
		}
	}
	opts.WaitIndex = meta.LastIndex

	// If we've exceeded the maximum staleness, retry without stale
	if allowStale && meta.LastContact > d.config.MaxStale {
		allowStale = false
		log.Printf("[DEBUG] (dedup) %s stale data (last contact exceeded max_stale)", path)
		goto START
	}

	// Re-enable stale queries if allowed
	if d.config.MaxStale != 0 {
		allowStale = true
	}

	// Stop listening if we're stopped
	select {
	case <-d.stopCh:
		return
	default:
	}

	// If we are current the leader, wait for leadership lost
	d.leaderLock.RLock()
	lockCh, ok = d.leader[t]
	d.leaderLock.RUnlock()
	if ok {
		select {
		case <-lockCh:
			goto START
		case <-d.stopCh:
			return
		}
	}

	// Parse the data file
	if pair != nil && pair.Flags == templateDataFlag {
		d.parseData(pair.Key, pair.Value)
	}
	goto START
}

// parseData is used to update brain from a KV data pair
func (d *DedupManager) parseData(path string, raw []byte) {
	// Setup the decompression and decoders
	r := bytes.NewReader(raw)
	decompress := lzw.NewReader(r, lzw.LSB, 8)
	defer decompress.Close()
	dec := gob.NewDecoder(decompress)

	// Decode the data
	var td templateData
	if err := dec.Decode(&td); err != nil {
		log.Printf("[ERR] (dedup) failed to decode '%s': %v",
			path, err)
		return
	}
	log.Printf("[INFO] (dedup) loading %d dependencies from '%s'",
		len(td.Data), path)

	// Update the data in the brain
	for hashCode, value := range td.Data {
		d.brain.ForceSet(hashCode, value)
	}

	// Trigger the updateCh
	select {
	case d.updateCh <- struct{}{}:
	default:
	}
}

func (d *DedupManager) attemptLock(client *consulapi.Client, session string, sessionCh chan struct{}, t *Template) {
	defer d.wg.Done()
START:
	log.Printf("[INFO] (dedup) attempting lock for template hash %s", t.hexMD5)
	basePath := path.Join(d.config.Deduplicate.Prefix, t.hexMD5)
	lopts := &consulapi.LockOptions{
		Key:              path.Join(basePath, "lock"),
		Session:          session,
		MonitorRetries:   3,
		MonitorRetryTime: 3 * time.Second,
	}
	lock, err := client.LockOpts(lopts)
	if err != nil {
		log.Printf("[ERR] (dedup) failed to create lock '%s': %v",
			lopts.Key, err)
		return
	}

	var retryCh <-chan time.Time
	leaderCh, err := lock.Lock(sessionCh)
	if err != nil {
		log.Printf("[ERR] (dedup) failed to acquire lock '%s': %v",
			lopts.Key, err)
		retryCh = time.After(lockRetry)
	} else {
		log.Printf("[INFO] (dedup) acquired lock '%s'", lopts.Key)
		d.setLeader(t, leaderCh)
	}

	select {
	case <-retryCh:
		retryCh = nil
		goto START
	case <-leaderCh:
		log.Printf("[WARN] (dedup) lost lock ownership '%s'", lopts.Key)
		d.setLeader(t, nil)
		goto START
	case <-sessionCh:
		log.Printf("[INFO] (dedup) releasing lock '%s'", lopts.Key)
		d.setLeader(t, nil)
		lock.Unlock()
	case <-d.stopCh:
		log.Printf("[INFO] (dedup) releasing lock '%s'", lopts.Key)
		lock.Unlock()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"image"
	"math/rand"
	"sync/atomic"
)

func traceImage(pxWide, pxHigh int, accel accelerationStructure, cam camera, quality int, completed *uint64) image.Image {

	accum := newAccumulator(pxWide, pxHigh)

	// Trace the image.
	pxPitch := 2.0 / float64(pxWide)
	for pxX := 0; pxX < pxWide; pxX++ {
		for pxY := 0; pxY < pxHigh; pxY++ {
			for i := 0; i < quality; i++ {
				x := (float64(pxX-pxWide/2) + rand.Float64()) * pxPitch
				y := (float64(pxY-pxHigh/2) + rand.Float64()) * pxPitch * -1.0
				r := cam.makeRay(x, y)
				r.dir = r.dir.unit()
				accum.add(pxX, pxY, tracePath(accel, r))
				atomic.AddUint64(completed, 1)

			}
		}
	}

	return accum.toImage(1.0)
}

func tracePath(accel accelerationStructure, r ray) colour {

	intersection, hit := accel.closestHit(r)
	if !hit {
		return colour{0, 0, 0}
	}

	// Calculate probability of emitting.
	pEmit := 0.1
	if intersection.emittance != 0 {
		pEmit = 1.0
	}

	// Handle emit case.
	if rand.Float64() < pEmit {
		return intersection.colour.
			scale(intersection.emittance / pEmit)
	}

	// Find where the ray hit. Reduce the intersection distance by a small
	// amount so that reflected rays don't intersect with it immediately.
	hitLoc := r.at(addULPs(intersection.distance, -50))

	// Orient the unit normal towards the ray origin.
	if intersection.unitNormal.dot(r.dir) > 0 {
		intersection.unitNormal = intersection.unitNormal.scale(-1.0)
	}

	// Create a random vector on the hemisphere towards the normal.
	rnd := vector{rand.NormFloat64(), rand.NormFloat64(), rand.NormFloat64()}
	rnd = rnd.unit()
	if rnd.dot(intersection.unitNormal) < 0 {
		rnd = rnd.scale(-1.0)
	}

	// Apply the BRDF (bidirectional reflection distribution function).
	brdf := rnd.dot(intersection.unitNormal)

	return tracePath(accel, ray{start: hitLoc, dir: rnd}).
		scale(brdf / (1 - pEmit)).
		mul(intersection.colour)
}
<commit_msg>Change loop nesting ordering so that quality is in the outer loop<commit_after>package main

import (
	"image"
	"math/rand"
	"sync/atomic"
)

func traceImage(pxWide, pxHigh int, accel accelerationStructure, cam camera, quality int, completed *uint64) image.Image {

	accum := newAccumulator(pxWide, pxHigh)

	// Trace the image.
	pxPitch := 2.0 / float64(pxWide)
	for i := 0; i < quality; i++ {
		for pxX := 0; pxX < pxWide; pxX++ {
			for pxY := 0; pxY < pxHigh; pxY++ {
				x := (float64(pxX-pxWide/2) + rand.Float64()) * pxPitch
				y := (float64(pxY-pxHigh/2) + rand.Float64()) * pxPitch * -1.0
				r := cam.makeRay(x, y)
				r.dir = r.dir.unit()
				accum.add(pxX, pxY, tracePath(accel, r))
				atomic.AddUint64(completed, 1)

			}
		}
	}

	return accum.toImage(1.0)
}

func tracePath(accel accelerationStructure, r ray) colour {

	intersection, hit := accel.closestHit(r)
	if !hit {
		return colour{0, 0, 0}
	}

	// Calculate probability of emitting.
	pEmit := 0.1
	if intersection.emittance != 0 {
		pEmit = 1.0
	}

	// Handle emit case.
	if rand.Float64() < pEmit {
		return intersection.colour.
			scale(intersection.emittance / pEmit)
	}

	// Find where the ray hit. Reduce the intersection distance by a small
	// amount so that reflected rays don't intersect with it immediately.
	hitLoc := r.at(addULPs(intersection.distance, -50))

	// Orient the unit normal towards the ray origin.
	if intersection.unitNormal.dot(r.dir) > 0 {
		intersection.unitNormal = intersection.unitNormal.scale(-1.0)
	}

	// Create a random vector on the hemisphere towards the normal.
	rnd := vector{rand.NormFloat64(), rand.NormFloat64(), rand.NormFloat64()}
	rnd = rnd.unit()
	if rnd.dot(intersection.unitNormal) < 0 {
		rnd = rnd.scale(-1.0)
	}

	// Apply the BRDF (bidirectional reflection distribution function).
	brdf := rnd.dot(intersection.unitNormal)

	return tracePath(accel, ray{start: hitLoc, dir: rnd}).
		scale(brdf / (1 - pEmit)).
		mul(intersection.colour)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
)

func usage() {
	fmt.Println(`Usage: get [<path>] [-v] [-h] [-d]

    -v, --version                   Prints the version and exits.
    -h, --help                      Prints the usage information.
    -d, --debug                     Logs debugging information to STDOUT.

Arguments:

    path                            The path to place or update the
                                    repostories. Defaults to the path
                                    in ~/.get.

To learn more or to contribute, please see github.com/pearkes/get`)
	os.Exit(1)
}
<commit_msg>Correct file to ~/.getconfig in usage instructions<commit_after>package main

import (
	"fmt"
	"os"
)

func usage() {
	fmt.Println(`Usage: get [<path>] [-v] [-h] [-d]

    -v, --version                   Prints the version and exits.
    -h, --help                      Prints the usage information.
    -d, --debug                     Logs debugging information to STDOUT.

Arguments:

    path                            The path to place or update the
                                    repostories. Defaults to the path
                                    in ~/.getconfig.

To learn more or to contribute, please see github.com/pearkes/get`)
	os.Exit(1)
}
<|endoftext|>


<commit_before>package test

import (
	"net/http"
	"net/http/httptest"
	"path/filepath"
	"runtime"
	"testing"

	_ "github.com/mihailo-misic/company-resource-api/routers"
	. "github.com/smartystreets/goconvey/convey"

	"github.com/astaxie/beego"
)

func init() {
	_, file, _, _ := runtime.Caller(1)
	appPath, _ := filepath.Abs(filepath.Dir(filepath.Join(file, "..")))
	appPath = filepath.Join(appPath, "..")
	beego.TestBeegoInit(appPath)
}

// TestGet is a sample to run an endpoint test
func TestGet(t *testing.T) {
	r, _ := http.NewRequest("GET", "/v1/object", nil)
	w := httptest.NewRecorder()
	beego.BeeApp.Handlers.ServeHTTP(w, r)

	beego.Trace("testing", "TestGet", "Code[%d]\n%s", w.Code, w.Body.String())

	Convey("Subject: Test Station Endpoint\n", t, func() {
		Convey("Status Code Should Be 200", func() {
			So(w.Code, ShouldEqual, 200)
		})

		Convey("The Result Should Not Be Empty", func() {
			So(w.Body.Len(), ShouldBeGreaterThan, 0)
		})
	})
}
<commit_msg>Fixing tests...<commit_after>package test

import (
	"net/http"
	"net/http/httptest"
	"path/filepath"
	"runtime"
	"testing"

	_ "github.com/mihailo-misic/company-resource-api/routers"
	. "github.com/smartystreets/goconvey/convey"

	"fmt"
	"github.com/astaxie/beego"
)

func init() {
	_, file, _, _ := runtime.Caller(1)
	appPath, _ := filepath.Abs(filepath.Dir(filepath.Join(file, "..")))
	appPath = filepath.Join(appPath, "..")
	beego.TestBeegoInit(appPath)
}

// TestGet is a sample to run an endpoint test
func TestGet(t *testing.T) {
	r, _ := http.NewRequest("GET", "/v1/object", nil)
	w := httptest.NewRecorder()
	beego.BeeApp.Handlers.ServeHTTP(w, r)

	beego.Trace("testing", "TestGet", fmt.Sprintf("Code[%d]\n%s", w.Code, w.Body.String()))

	fmt.Printf("\n%+v\n", w.HeaderMap)

	Convey("Subject: Test Station Endpoint\n", t, func() {
		Convey("Status Code Should Be 200", func() {
			So(w.Code, ShouldEqual, 200)
		})

		Convey("The Result Should Not Be Empty", func() {
			So(w.Body.Len(), ShouldBeGreaterThan, 0)
		})
	})
}
<|endoftext|>


<commit_before>package test

import (
	"runtime"
	"path/filepath"
	_ "github.com/scmo/foodchain-backend/routers"
	"github.com/astaxie/beego"
	"testing"

	"github.com/scmo/foodchain-backend/services"
	"github.com/scmo/foodchain-backend/tests/db_test"
	"github.com/scmo/foodchain-backend/models"
)

func init() {
	_, file, _, _ := runtime.Caller(1)
	apppath, _ := filepath.Abs(filepath.Dir(filepath.Join(file, ".." + string(filepath.Separator))))
	beego.TestBeegoInit(apppath)

	db_test.Setup()
}

func TestAddLackToContributions(t *testing.T) {
	contributions := make([] *models.Contribution, 0)
	iLack := models.InspectionLack{ContributionCode:5416, ControlCategoryId:"12.01_2017", PointGroupCode:1110, ControlPointId:"01", LackId:1}
	contribution := services.GetContributionByInspectionLack(&iLack)
	contributions = services.AddContributionToContributions(contributions, contribution)

	iLack = models.InspectionLack{ContributionCode:5416, ControlCategoryId:"12.01_2017", PointGroupCode:1150, ControlPointId:"01", LackId:7}
	contribution = services.GetContributionByInspectionLack(&iLack)
	contributions2 := services.AddContributionToContributions(contributions, contribution)

	iLack = models.InspectionLack{ContributionCode:5416, ControlCategoryId:"12.01_2017", PointGroupCode:1110, ControlPointId:"02", LackId:4}
	contribution = services.GetContributionByInspectionLack(&iLack)
	contributions3 := services.AddContributionToContributions(contributions, contribution)

	iLack = models.InspectionLack{ContributionCode:5416, ControlCategoryId:"12.01_2017", PointGroupCode:1110, ControlPointId:"01", LackId:2}
	contribution = services.GetContributionByInspectionLack(&iLack)
	contributions4 := services.AddContributionToContributions(contributions, contribution)

	iLack = models.InspectionLack{ContributionCode:5417, ControlCategoryId:"12.01_2017", PointGroupCode:1110, ControlPointId:"01", LackId:28}
	contribution = services.GetContributionByInspectionLack(&iLack)
	contributions5 := services.AddContributionToContributions(contributions, contribution)

	Convey("Subject: Test Generating Contributions with Lacks \n", t, func() {
		Convey("Length should 1", func() {
			So(len(contributions), ShouldEqual, 1)
		})
		Convey("Length should still be 1", func() {
			So(len(contributions2), ShouldEqual, 1)
		})
		Convey("ControlCategory should have two PointGroups", func() {
			So(len(contributions2[0].ControlCategories[0].PointGroups), ShouldEqual, 2)
		})
		Convey("PointGroup[0] should have two ControlPoints", func() {
			So(len(contributions3[0].ControlCategories[0].PointGroups[0].ControlPoints), ShouldEqual, 2)
		})
		Convey("ControlPoint[0] should have two Lacks", func() {
			So(len(contributions4[0].ControlCategories[0].PointGroups[0].ControlPoints[0].Lacks), ShouldEqual, 2)
		})
		Convey("Length should be 2", func() {
			So(len(contributions5), ShouldEqual, 2)
		})
	})
}

<commit_msg>Import convey<commit_after>package test

import (
	"runtime"
	"path/filepath"
	_ "github.com/scmo/foodchain-backend/routers"
	"github.com/astaxie/beego"
	"testing"

	"github.com/scmo/foodchain-backend/services"
	"github.com/scmo/foodchain-backend/tests/db_test"
	"github.com/scmo/foodchain-backend/models"
	. "github.com/smartystreets/goconvey/convey"
)

func init() {
	_, file, _, _ := runtime.Caller(1)
	apppath, _ := filepath.Abs(filepath.Dir(filepath.Join(file, ".." + string(filepath.Separator))))
	beego.TestBeegoInit(apppath)

	db_test.Setup()
}

func TestAddLackToContributions(t *testing.T) {
	contributions := make([] *models.Contribution, 0)
	iLack := models.InspectionLack{ContributionCode:5416, ControlCategoryId:"12.01_2017", PointGroupCode:1110, ControlPointId:"01", LackId:1}
	contribution := services.GetContributionByInspectionLack(&iLack)
	contributions = services.AddContributionToContributions(contributions, contribution)

	iLack = models.InspectionLack{ContributionCode:5416, ControlCategoryId:"12.01_2017", PointGroupCode:1150, ControlPointId:"01", LackId:7}
	contribution = services.GetContributionByInspectionLack(&iLack)
	contributions2 := services.AddContributionToContributions(contributions, contribution)

	iLack = models.InspectionLack{ContributionCode:5416, ControlCategoryId:"12.01_2017", PointGroupCode:1110, ControlPointId:"02", LackId:4}
	contribution = services.GetContributionByInspectionLack(&iLack)
	contributions3 := services.AddContributionToContributions(contributions, contribution)

	iLack = models.InspectionLack{ContributionCode:5416, ControlCategoryId:"12.01_2017", PointGroupCode:1110, ControlPointId:"01", LackId:2}
	contribution = services.GetContributionByInspectionLack(&iLack)
	contributions4 := services.AddContributionToContributions(contributions, contribution)

	iLack = models.InspectionLack{ContributionCode:5417, ControlCategoryId:"12.01_2017", PointGroupCode:1110, ControlPointId:"01", LackId:28}
	contribution = services.GetContributionByInspectionLack(&iLack)
	contributions5 := services.AddContributionToContributions(contributions, contribution)

	Convey("Subject: Test Generating Contributions with Lacks \n", t, func() {
		Convey("Length should 1", func() {
			So(len(contributions), ShouldEqual, 1)
		})
		Convey("Length should still be 1", func() {
			So(len(contributions2), ShouldEqual, 1)
		})
		Convey("ControlCategory should have two PointGroups", func() {
			So(len(contributions2[0].ControlCategories[0].PointGroups), ShouldEqual, 2)
		})
		Convey("PointGroup[0] should have two ControlPoints", func() {
			So(len(contributions3[0].ControlCategories[0].PointGroups[0].ControlPoints), ShouldEqual, 2)
		})
		Convey("ControlPoint[0] should have two Lacks", func() {
			So(len(contributions4[0].ControlCategories[0].PointGroups[0].ControlPoints[0].Lacks), ShouldEqual, 2)
		})
		Convey("Length should be 2", func() {
			So(len(contributions5), ShouldEqual, 2)
		})
	})
}

<|endoftext|>


<commit_before>package stackongo

import (
	"os"
	"strings"
	"fmt"
)

// AllUsers returns all users in site
func (session Session) AllUsers(params map[string]string) (output *Users, error os.Error) {
	output = new(Users)
	error = session.get("users", params, output)
	return
}

// Users returns the users with the given ids
func (session Session) GetUsers(ids []int, params map[string]string) (output *Users, error os.Error) {
	string_ids := []string{}
	for _, v := range ids {
		string_ids = append(string_ids, fmt.Sprintf("%v", v))
	}
	request_path := strings.Join([]string{"users", strings.Join(string_ids, ";")}, "/")

	output = new(Users)
	error = session.get(request_path, params, output)
	return
}

// AuthenticatedUser returns the user associated with the passed auth_token.
func (session Session) AuthenticatedUser(params map[string]string, auth map[string]string) (output User, error os.Error) {
	//add auth params
	for key, value := range auth {
		params[key] = value
	}

	collection := new(Users)
	error = session.get("me", params, collection)

	if len(collection.Items) > 0 {
		output = collection.Items[0]
	} else {
		error = os.NewError("User not found")
	}

	return

}

// Moderators returns those users on a site who can exercise moderation powers.
func (session Session) Moderators(params map[string]string) (output *Users, error os.Error) {
	output = new(Users)
	error = session.get("users/moderators", params, output)
	return
}

// ElectedModerators returns those users on a site who both have moderator powers, and were actually elected.
func (session Session) ElectedModerators(params map[string]string) (output *Users, error os.Error) {
	output = new(Users)
	error = session.get("users/moderators/elected", params, output)
	return
}
<commit_msg>Better error handling for authenticated user request<commit_after>package stackongo

import (
	"os"
	"strings"
	"fmt"
)

// AllUsers returns all users in site
func (session Session) AllUsers(params map[string]string) (output *Users, error os.Error) {
	output = new(Users)
	error = session.get("users", params, output)
	return
}

// Users returns the users with the given ids
func (session Session) GetUsers(ids []int, params map[string]string) (output *Users, error os.Error) {
	string_ids := []string{}
	for _, v := range ids {
		string_ids = append(string_ids, fmt.Sprintf("%v", v))
	}
	request_path := strings.Join([]string{"users", strings.Join(string_ids, ";")}, "/")

	output = new(Users)
	error = session.get(request_path, params, output)
	return
}

// AuthenticatedUser returns the user associated with the passed auth_token.
func (session Session) AuthenticatedUser(params map[string]string, auth map[string]string) (output User, error os.Error) {
	//add auth params
	for key, value := range auth {
		params[key] = value
	}

	collection := new(Users)
	error = session.get("me", params, collection)

	if error != nil {
    error = os.NewError(collection.Error_name + ": " + collection.Error_message)
		return
	}

	if len(collection.Items) > 0 {
		output = collection.Items[0]
	} else {
		error = os.NewError("User not found")
	}

	return

}

// Moderators returns those users on a site who can exercise moderation powers.
func (session Session) Moderators(params map[string]string) (output *Users, error os.Error) {
	output = new(Users)
	error = session.get("users/moderators", params, output)
	return
}

// ElectedModerators returns those users on a site who both have moderator powers, and were actually elected.
func (session Session) ElectedModerators(params map[string]string) (output *Users, error os.Error) {
	output = new(Users)
	error = session.get("users/moderators/elected", params, output)
	return
}
<|endoftext|>


<commit_before>package scipipe

import (
	// "github.com/go-errors/errors"
	//"os"
	"os/exec"
	re "regexp"
)

func ExecCmd(cmd string) {
	Info.Println("Executing command: ", cmd)
	combOutput, err := exec.Command("bash", "-lc", cmd).CombinedOutput()
	if err != nil {
		Error.Println("Could not execute command `" + cmd + "`: " + string(combOutput))
	}
}

func Check(err error) {
	if err != nil {
		panic(err)
	}
}

func copyMapStrStr(m map[string]string) (nm map[string]string) {
	nm = make(map[string]string)
	for k, v := range m {
		nm[k] = v
	}
	return nm
}

// Return the regular expression used to parse the place-holder syntax for in-, out- and
// parameter ports, that can be used to instantiate a SciProcess.
func getShellCommandPlaceHolderRegex() *re.Regexp {
	r, err := re.Compile("{(o|os|i|is|p):([^{}:]+)}")
	Check(err)
	return r
}
<commit_msg>Return output from ExecCmd function<commit_after>package scipipe

import (
	// "github.com/go-errors/errors"
	//"os"
	"os"
	"os/exec"
	re "regexp"
)

func ExecCmd(cmd string) string {
	Info.Println("Executing command: ", cmd)
	combOutput, err := exec.Command("bash", "-lc", cmd).CombinedOutput()
	if err != nil {
		Error.Println("Could not execute command `" + cmd + "`: " + string(combOutput))
		os.Exit(128)
	}
	return string(combOutput)
}

func Check(err error) {
	if err != nil {
		panic(err)
	}
}

func copyMapStrStr(m map[string]string) (nm map[string]string) {
	nm = make(map[string]string)
	for k, v := range m {
		nm[k] = v
	}
	return nm
}

// Return the regular expression used to parse the place-holder syntax for in-, out- and
// parameter ports, that can be used to instantiate a SciProcess.
func getShellCommandPlaceHolderRegex() *re.Regexp {
	r, err := re.Compile("{(o|os|i|is|p):([^{}:]+)}")
	Check(err)
	return r
}
<|endoftext|>


<commit_before>package docker

/*
#include <sys/ioctl.h>
#include <linux/fs.h>
#include <errno.h>

// See linux.git/fs/btrfs/ioctl.h
#define BTRFS_IOCTL_MAGIC 0x94
#define BTRFS_IOC_CLONE _IOW(BTRFS_IOCTL_MAGIC, 9, int)

int
btrfs_reflink(int fd_out, int fd_in)
{
  int res;
  res = ioctl(fd_out, BTRFS_IOC_CLONE, fd_in);
  if (res < 0)
    return errno;
  return 0;
}

*/
import "C"
import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"strings"
	"syscall"
)

// Compare two Config struct. Do not compare the "Image" nor "Hostname" fields
// If OpenStdin is set, then it differs
func CompareConfig(a, b *Config) bool {
	if a == nil || b == nil ||
		a.OpenStdin || b.OpenStdin {
		return false
	}
	if a.AttachStdout != b.AttachStdout ||
		a.AttachStderr != b.AttachStderr ||
		a.User != b.User ||
		a.Memory != b.Memory ||
		a.MemorySwap != b.MemorySwap ||
		a.CpuShares != b.CpuShares ||
		a.OpenStdin != b.OpenStdin ||
		a.Tty != b.Tty ||
		a.VolumesFrom != b.VolumesFrom {
		return false
	}
	if len(a.Cmd) != len(b.Cmd) ||
		len(a.Dns) != len(b.Dns) ||
		len(a.Env) != len(b.Env) ||
		len(a.PortSpecs) != len(b.PortSpecs) ||
		len(a.Entrypoint) != len(b.Entrypoint) ||
		len(a.Volumes) != len(b.Volumes) {
		return false
	}

	for i := 0; i < len(a.Cmd); i++ {
		if a.Cmd[i] != b.Cmd[i] {
			return false
		}
	}
	for i := 0; i < len(a.Dns); i++ {
		if a.Dns[i] != b.Dns[i] {
			return false
		}
	}
	for i := 0; i < len(a.Env); i++ {
		if a.Env[i] != b.Env[i] {
			return false
		}
	}
	for i := 0; i < len(a.PortSpecs); i++ {
		if a.PortSpecs[i] != b.PortSpecs[i] {
			return false
		}
	}
	for i := 0; i < len(a.Entrypoint); i++ {
		if a.Entrypoint[i] != b.Entrypoint[i] {
			return false
		}
	}
	for key := range a.Volumes {
		if _, exists := b.Volumes[key]; !exists {
			return false
		}
	}
	return true
}

func MergeConfig(userConf, imageConf *Config) {
	if userConf.User == "" {
		userConf.User = imageConf.User
	}
	if userConf.Memory == 0 {
		userConf.Memory = imageConf.Memory
	}
	if userConf.MemorySwap == 0 {
		userConf.MemorySwap = imageConf.MemorySwap
	}
	if userConf.CpuShares == 0 {
		userConf.CpuShares = imageConf.CpuShares
	}
	if userConf.PortSpecs == nil || len(userConf.PortSpecs) == 0 {
		userConf.PortSpecs = imageConf.PortSpecs
	} else {
		for _, imagePortSpec := range imageConf.PortSpecs {
			found := false
			imageNat, _ := parseNat(imagePortSpec)
			for _, userPortSpec := range userConf.PortSpecs {
				userNat, _ := parseNat(userPortSpec)
				if imageNat.Proto == userNat.Proto && imageNat.Backend == userNat.Backend {
					found = true
				}
			}
			if !found {
				userConf.PortSpecs = append(userConf.PortSpecs, imagePortSpec)
			}
		}
	}
	if !userConf.Tty {
		userConf.Tty = imageConf.Tty
	}
	if !userConf.OpenStdin {
		userConf.OpenStdin = imageConf.OpenStdin
	}
	if !userConf.StdinOnce {
		userConf.StdinOnce = imageConf.StdinOnce
	}
	if userConf.Env == nil || len(userConf.Env) == 0 {
		userConf.Env = imageConf.Env
	} else {
		for _, imageEnv := range imageConf.Env {
			found := false
			imageEnvKey := strings.Split(imageEnv, "=")[0]
			for _, userEnv := range userConf.Env {
				userEnvKey := strings.Split(userEnv, "=")[0]
				if imageEnvKey == userEnvKey {
					found = true
				}
			}
			if !found {
				userConf.Env = append(userConf.Env, imageEnv)
			}
		}
	}
	if userConf.Cmd == nil || len(userConf.Cmd) == 0 {
		userConf.Cmd = imageConf.Cmd
	}
	if userConf.Dns == nil || len(userConf.Dns) == 0 {
		userConf.Dns = imageConf.Dns
	} else {
		//duplicates aren't an issue here
		userConf.Dns = append(userConf.Dns, imageConf.Dns...)
	}
	if userConf.Entrypoint == nil || len(userConf.Entrypoint) == 0 {
		userConf.Entrypoint = imageConf.Entrypoint
	}
	if userConf.WorkingDir == "" {
		userConf.WorkingDir = imageConf.WorkingDir
	}
	if userConf.VolumesFrom == "" {
		userConf.VolumesFrom = imageConf.VolumesFrom
	}
	if userConf.Volumes == nil || len(userConf.Volumes) == 0 {
		userConf.Volumes = imageConf.Volumes
	} else {
		for k, v := range imageConf.Volumes {
			userConf.Volumes[k] = v
		}
	}
}

func parseLxcConfOpts(opts ListOpts) ([]KeyValuePair, error) {
	out := make([]KeyValuePair, len(opts))
	for i, o := range opts {
		k, v, err := parseLxcOpt(o)
		if err != nil {
			return nil, err
		}
		out[i] = KeyValuePair{Key: k, Value: v}
	}
	return out, nil
}

func parseLxcOpt(opt string) (string, string, error) {
	parts := strings.SplitN(opt, "=", 2)
	if len(parts) != 2 {
		return "", "", fmt.Errorf("Unable to parse lxc conf option: %s", opt)
	}
	return strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1]), nil
}

func RootIsShared() bool {
	if data, err := ioutil.ReadFile("/proc/self/mountinfo"); err == nil {
		for _, line := range strings.Split(string(data), "\n") {
			cols := strings.Split(line, " ")
			if len(cols) >= 6 && cols[3] == "/" && cols[4] == "/" {
				return strings.HasPrefix(cols[6], "shared")
			}
		}
	}

	// No idea, probably safe to assume so
	return true
}

func BtrfsReflink(fd_out, fd_in uintptr) error {
	res := C.btrfs_reflink(C.int(fd_out), C.int(fd_in))
	if res != 0 {
		return syscall.Errno(res)
	}
	return nil
}

func CopyFile(dstFile, srcFile *os.File) error {
	err := BtrfsReflink(dstFile.Fd(), srcFile.Fd())
	if err == nil {
		return nil
	}

	// Fall back to normal copy
	_, err = io.Copy(dstFile, srcFile)
	return err
}
<commit_msg>RootIsShared: Fix root detection<commit_after>package docker

/*
#include <sys/ioctl.h>
#include <linux/fs.h>
#include <errno.h>

// See linux.git/fs/btrfs/ioctl.h
#define BTRFS_IOCTL_MAGIC 0x94
#define BTRFS_IOC_CLONE _IOW(BTRFS_IOCTL_MAGIC, 9, int)

int
btrfs_reflink(int fd_out, int fd_in)
{
  int res;
  res = ioctl(fd_out, BTRFS_IOC_CLONE, fd_in);
  if (res < 0)
    return errno;
  return 0;
}

*/
import "C"
import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"strings"
	"syscall"
)

// Compare two Config struct. Do not compare the "Image" nor "Hostname" fields
// If OpenStdin is set, then it differs
func CompareConfig(a, b *Config) bool {
	if a == nil || b == nil ||
		a.OpenStdin || b.OpenStdin {
		return false
	}
	if a.AttachStdout != b.AttachStdout ||
		a.AttachStderr != b.AttachStderr ||
		a.User != b.User ||
		a.Memory != b.Memory ||
		a.MemorySwap != b.MemorySwap ||
		a.CpuShares != b.CpuShares ||
		a.OpenStdin != b.OpenStdin ||
		a.Tty != b.Tty ||
		a.VolumesFrom != b.VolumesFrom {
		return false
	}
	if len(a.Cmd) != len(b.Cmd) ||
		len(a.Dns) != len(b.Dns) ||
		len(a.Env) != len(b.Env) ||
		len(a.PortSpecs) != len(b.PortSpecs) ||
		len(a.Entrypoint) != len(b.Entrypoint) ||
		len(a.Volumes) != len(b.Volumes) {
		return false
	}

	for i := 0; i < len(a.Cmd); i++ {
		if a.Cmd[i] != b.Cmd[i] {
			return false
		}
	}
	for i := 0; i < len(a.Dns); i++ {
		if a.Dns[i] != b.Dns[i] {
			return false
		}
	}
	for i := 0; i < len(a.Env); i++ {
		if a.Env[i] != b.Env[i] {
			return false
		}
	}
	for i := 0; i < len(a.PortSpecs); i++ {
		if a.PortSpecs[i] != b.PortSpecs[i] {
			return false
		}
	}
	for i := 0; i < len(a.Entrypoint); i++ {
		if a.Entrypoint[i] != b.Entrypoint[i] {
			return false
		}
	}
	for key := range a.Volumes {
		if _, exists := b.Volumes[key]; !exists {
			return false
		}
	}
	return true
}

func MergeConfig(userConf, imageConf *Config) {
	if userConf.User == "" {
		userConf.User = imageConf.User
	}
	if userConf.Memory == 0 {
		userConf.Memory = imageConf.Memory
	}
	if userConf.MemorySwap == 0 {
		userConf.MemorySwap = imageConf.MemorySwap
	}
	if userConf.CpuShares == 0 {
		userConf.CpuShares = imageConf.CpuShares
	}
	if userConf.PortSpecs == nil || len(userConf.PortSpecs) == 0 {
		userConf.PortSpecs = imageConf.PortSpecs
	} else {
		for _, imagePortSpec := range imageConf.PortSpecs {
			found := false
			imageNat, _ := parseNat(imagePortSpec)
			for _, userPortSpec := range userConf.PortSpecs {
				userNat, _ := parseNat(userPortSpec)
				if imageNat.Proto == userNat.Proto && imageNat.Backend == userNat.Backend {
					found = true
				}
			}
			if !found {
				userConf.PortSpecs = append(userConf.PortSpecs, imagePortSpec)
			}
		}
	}
	if !userConf.Tty {
		userConf.Tty = imageConf.Tty
	}
	if !userConf.OpenStdin {
		userConf.OpenStdin = imageConf.OpenStdin
	}
	if !userConf.StdinOnce {
		userConf.StdinOnce = imageConf.StdinOnce
	}
	if userConf.Env == nil || len(userConf.Env) == 0 {
		userConf.Env = imageConf.Env
	} else {
		for _, imageEnv := range imageConf.Env {
			found := false
			imageEnvKey := strings.Split(imageEnv, "=")[0]
			for _, userEnv := range userConf.Env {
				userEnvKey := strings.Split(userEnv, "=")[0]
				if imageEnvKey == userEnvKey {
					found = true
				}
			}
			if !found {
				userConf.Env = append(userConf.Env, imageEnv)
			}
		}
	}
	if userConf.Cmd == nil || len(userConf.Cmd) == 0 {
		userConf.Cmd = imageConf.Cmd
	}
	if userConf.Dns == nil || len(userConf.Dns) == 0 {
		userConf.Dns = imageConf.Dns
	} else {
		//duplicates aren't an issue here
		userConf.Dns = append(userConf.Dns, imageConf.Dns...)
	}
	if userConf.Entrypoint == nil || len(userConf.Entrypoint) == 0 {
		userConf.Entrypoint = imageConf.Entrypoint
	}
	if userConf.WorkingDir == "" {
		userConf.WorkingDir = imageConf.WorkingDir
	}
	if userConf.VolumesFrom == "" {
		userConf.VolumesFrom = imageConf.VolumesFrom
	}
	if userConf.Volumes == nil || len(userConf.Volumes) == 0 {
		userConf.Volumes = imageConf.Volumes
	} else {
		for k, v := range imageConf.Volumes {
			userConf.Volumes[k] = v
		}
	}
}

func parseLxcConfOpts(opts ListOpts) ([]KeyValuePair, error) {
	out := make([]KeyValuePair, len(opts))
	for i, o := range opts {
		k, v, err := parseLxcOpt(o)
		if err != nil {
			return nil, err
		}
		out[i] = KeyValuePair{Key: k, Value: v}
	}
	return out, nil
}

func parseLxcOpt(opt string) (string, string, error) {
	parts := strings.SplitN(opt, "=", 2)
	if len(parts) != 2 {
		return "", "", fmt.Errorf("Unable to parse lxc conf option: %s", opt)
	}
	return strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1]), nil
}

func RootIsShared() bool {
	if data, err := ioutil.ReadFile("/proc/self/mountinfo"); err == nil {
		for _, line := range strings.Split(string(data), "\n") {
			cols := strings.Split(line, " ")
			if len(cols) >= 6 && cols[4] == "/" {
				return strings.HasPrefix(cols[6], "shared")
			}
		}
	}

	// No idea, probably safe to assume so
	return true
}

func BtrfsReflink(fd_out, fd_in uintptr) error {
	res := C.btrfs_reflink(C.int(fd_out), C.int(fd_in))
	if res != 0 {
		return syscall.Errno(res)
	}
	return nil
}

func CopyFile(dstFile, srcFile *os.File) error {
	err := BtrfsReflink(dstFile.Fd(), srcFile.Fd())
	if err == nil {
		return nil
	}

	// Fall back to normal copy
	_, err = io.Copy(dstFile, srcFile)
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/samuel/go-gettext/gettext"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"path"
	"regexp"
	"strconv"
	"strings"
)

func IsParamSet(r *http.Request, param string) bool {
	return len(r.URL.Query().Get(param)) > 0
}

func Lang(r *http.Request) string {
	lang := r.URL.Query().Get("lang")
	if len(lang) == 0 {
		lang = "en_US"
	}
	return lang
}

func GetQS(q url.Values, param string, deflt int) (num int, str string) {
	str = q.Get(param)
	num, err := strconv.Atoi(str)
	if err != nil {
		num = deflt
		str = ""
	} else {
		str = fmt.Sprintf("&%s=%s", param, str)
	}
	return
}

func GetHost(r *http.Request) (host string, err error) {
	// get remote ip
	host = r.Header.Get("X-Forwarded-For")
	if len(host) > 0 {
		parts := strings.Split(host, ",")
		// apache will append the remote address
		host = strings.TrimSpace(parts[len(parts)-1])
	} else {
		host, _, err = net.SplitHostPort(r.RemoteAddr)
	}
	return
}

var TBBUserAgents = regexp.MustCompile(`^Mozilla/5\.0 \([^)]*\) Gecko/20100101 Firefox/[\d]+\.0$`)
var OrfoxUserAgents = regexp.MustCompile(`^Mozilla/5\.0 \(Android; Mobile; rv:38.0\) Gecko/38.0 Firefox/38.0$`)

func LikelyTBB(ua string) bool {
	return TBBUserAgents.MatchString(ua) || OrfoxUserAgents.MatchString(ua)
}

func FuncMap(domain *gettext.Domain) template.FuncMap {
	return template.FuncMap{
		"UnEscaped": func(x string) interface{} {
			return template.HTML(x)
		},
		"UnEscapedURL": func(x string) interface{} {
			return template.URL(x)
		},
		"GetText": func(lang string, text string) string {
			return domain.GetText(lang, text)
		},
		"Equal": func(one string, two string) bool {
			return one == two
		},
		"Not": func(b bool) bool {
			return !b
		},
		"And": func(a bool, b bool) bool {
			return a && b
		},
	}
}

var Layout *template.Template

func CompileTemplate(base string, domain *gettext.Domain, templateName string) *template.Template {
	if Layout == nil {
		Layout = template.New("")
		Layout = Layout.Funcs(FuncMap(domain))
		Layout = template.Must(Layout.ParseFiles(
			path.Join(base, "public/base.html"),
			path.Join(base, "public/torbutton.html"),
		))
	}
	l, err := Layout.Clone()
	if err != nil {
		log.Fatal(err)
	}
	return template.Must(l.ParseFiles(path.Join(base, "public/", templateName)))
}

type locale struct {
	Code string
	Name string
}

func GetLocaleList(base string) map[string]string {
	// populated from https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes
	// and https://en.wikipedia.org/w/api.php?action=sitematrix&format=json
	haveTranslatedNames := map[string]string{
		"af":          "Afrikaans",
		"ar":          "العربية",
		"az":          "Azərbaycanca",
		"bg":          "български",
		"bs":          "Bosanski jezik",
		"ca":          "Català",
		"cs":          "Čeština",
		"cy":          "Cymraeg",
		"da":          "Dansk",
		"de":          "Deutsch",
		"el":          "ελληνικά",
		"en_GB":       "English (United Kingdom)",
		"eo":          "Esperanto",
		"es":          "Español",
		"es_AR":       "Español (Argentina)",
		"es_CO":       "Español (Colombia)",
		"es_MX":       "Español (Mexico)",
		"et":          "Eesti",
		"eu":          "Euskara",
		"fa":          "فارسی",
		"fi":          "Suomi",
		"fr":          "Français",
		"fr_CA":       "Français (Canadien)",
		"gl":          "Galego",
		"he":          "עברית",
		"hi":          "हिन्दी",
		"hr":          "Hrvatski jezik",
		"hr_HR":       "Hrvatski jezik (Croatia)",
		"hu":          "Magyar",
		"id":          "Bahasa Indonesia",
		"is":          "íslenska",
		"it":          "Italiano",
		"ja":          "日本語",
		"km":          "មែរ",
		"kn":          "ಕನ್ನಡ",
		"ko":          "한국어",
		"ko_KR":       "한국어 (South Korea)",
		"lb":          "Lëtzebuergesch",
		"lo":          "ລາວ",
		"lv":          "Latviešu valoda",
		"mk":          "македонски јазик",
		"ms_MY":       "Bahasa Melayu",
		"my":          "ဗမာစာ",
		"nb":          "Norsk bokmål",
		"nl":          "Nederlands",
		"nl_BE":       "Vlaams",
		"nn":          "Norsk nynorsk",
		"pa":          "ਪੰਜਾਬੀ",
		"pl":          "Język polski",
		"pl_PL":       "Język polski (Poland)",
		"pt":          "Português",
		"pt_BR":       "Português do Brasil",
		"ro":          "română",
		"ru":          "русский язык",
		"ru@petr1708": "Russian Petrine orthography",
		"si_LK":       "සිංහල",
		"sk":          "Slovenčina",
		"sk_SK":       "Slovenčina (Slovakia)",
		"sl":          "Slovenski jezik",
		"sl_SI":       "Slovenski jezik (Slovenia)",
		"sq":          "shqip",
		"sr":          "српски језик",
		"sv":          "Svenska",
		"ta":          "தமிழ்",
		"te_IN":       "తెలుగు",
		"th":          "ไทย",
		"tr":          "Türkçe",
		"uk":          "українська мова",
		"zh_CN":       "中文简体",
		"zh_HK":       "中文繁體",
		"zh_TW":       "中文繁體",
	}

	// for all folders in locale which match a locale from https://www.transifex.com/api/2/languages/
	// use the language name unless we have an override
	webLocales, err := FetchTranslationLocales(base)
	if err != nil {
		log.Printf("Failed to get up to date language list, using fallback.")
		return haveTranslatedNames
	}

	return GetInstalledLocales(base, webLocales, haveTranslatedNames)
}

func FetchTranslationLocales(base string) (map[string]locale, error) {
	file, err := os.Open(path.Join(base, "data/langs"))
	if err != nil {
		return nil, err
	}
	defer file.Close()

	webLocales := make(map[string]locale)
	// Parse the api response into a list of possible locales
	dec := json.NewDecoder(file)
	for {
		var webList []locale
		if err = dec.Decode(&webList); err == io.EOF {
			break
		} else if err != nil {
			log.Fatal(err)
		}

		// The api returns an array, so we need to map it
		for _, l := range webList {
			webLocales[l.Code] = l
		}
	}

	return webLocales, nil
}

// Get a list of all languages installed in our locale folder with translations if available
func GetInstalledLocales(base string, webLocales map[string]locale, nameTranslations map[string]string) map[string]string {
	localFiles, err := ioutil.ReadDir(path.Join(base, "locale"))

	if err != nil {
		log.Print("No locales found in 'locale'. Try running 'make i18n'.")
		log.Fatal(err)
	}

	locales := make(map[string]string, len(localFiles))
	locales["en_US"] = "English"

	for _, f := range localFiles {
		// TODO: Ensure a language has 100% of the template file
		// Currently this is what should be on the torcheck_completed
		// branch on the translations git should be, so we don't really
		// have to check it in theory...
		code := f.Name()

		// Only accept folders which have corresponding locale
		if !f.IsDir() || webLocales[code] == (locale{}) {
			continue
		}

		// If we have a translated name for a given locale, use it
		if transName := nameTranslations[code]; transName != "" {
			locales[code] = transName
		} else {
			log.Print("No translated name for code: " + code)
			locales[code] = webLocales[code].Name
		}
	}

	return locales
}
<commit_msg>Update `haveTranslatedNames` for the latest "torcheck_completed"<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/samuel/go-gettext/gettext"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"path"
	"regexp"
	"strconv"
	"strings"
)

func IsParamSet(r *http.Request, param string) bool {
	return len(r.URL.Query().Get(param)) > 0
}

func Lang(r *http.Request) string {
	lang := r.URL.Query().Get("lang")
	if len(lang) == 0 {
		lang = "en_US"
	}
	return lang
}

func GetQS(q url.Values, param string, deflt int) (num int, str string) {
	str = q.Get(param)
	num, err := strconv.Atoi(str)
	if err != nil {
		num = deflt
		str = ""
	} else {
		str = fmt.Sprintf("&%s=%s", param, str)
	}
	return
}

func GetHost(r *http.Request) (host string, err error) {
	// get remote ip
	host = r.Header.Get("X-Forwarded-For")
	if len(host) > 0 {
		parts := strings.Split(host, ",")
		// apache will append the remote address
		host = strings.TrimSpace(parts[len(parts)-1])
	} else {
		host, _, err = net.SplitHostPort(r.RemoteAddr)
	}
	return
}

var TBBUserAgents = regexp.MustCompile(`^Mozilla/5\.0 \([^)]*\) Gecko/20100101 Firefox/[\d]+\.0$`)
var OrfoxUserAgents = regexp.MustCompile(`^Mozilla/5\.0 \(Android; Mobile; rv:38.0\) Gecko/38.0 Firefox/38.0$`)

func LikelyTBB(ua string) bool {
	return TBBUserAgents.MatchString(ua) || OrfoxUserAgents.MatchString(ua)
}

func FuncMap(domain *gettext.Domain) template.FuncMap {
	return template.FuncMap{
		"UnEscaped": func(x string) interface{} {
			return template.HTML(x)
		},
		"UnEscapedURL": func(x string) interface{} {
			return template.URL(x)
		},
		"GetText": func(lang string, text string) string {
			return domain.GetText(lang, text)
		},
		"Equal": func(one string, two string) bool {
			return one == two
		},
		"Not": func(b bool) bool {
			return !b
		},
		"And": func(a bool, b bool) bool {
			return a && b
		},
	}
}

var Layout *template.Template

func CompileTemplate(base string, domain *gettext.Domain, templateName string) *template.Template {
	if Layout == nil {
		Layout = template.New("")
		Layout = Layout.Funcs(FuncMap(domain))
		Layout = template.Must(Layout.ParseFiles(
			path.Join(base, "public/base.html"),
			path.Join(base, "public/torbutton.html"),
		))
	}
	l, err := Layout.Clone()
	if err != nil {
		log.Fatal(err)
	}
	return template.Must(l.ParseFiles(path.Join(base, "public/", templateName)))
}

type locale struct {
	Code string
	Name string
}

func GetLocaleList(base string) map[string]string {
	// populated from https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes
	// and https://en.wikipedia.org/w/api.php?action=sitematrix&format=json
	haveTranslatedNames := map[string]string{
		"ar":          "العربية",
		"bg":          "български",
		"bn":          "বাংলা",
		"bs":          "Bosanski jezik",
		"ca":          "Català",
		"cs":          "Čeština",
		"da":          "Dansk",
		"de":          "Deutsch",
		"el":          "ελληνικά",
		"en_GB":       "English (United Kingdom)",
		"eo":          "Esperanto",
		"es":          "Español",
		"es_AR":       "Español (Argentina)",
		"es_MX":       "Español (Mexico)",
		"et":          "Eesti",
		"eu":          "Euskara",
		"fa":          "فارسی",
		"fi":          "Suomi",
		"fr":          "Français",
		"ga":          "Gaeilge",
		"he":          "עברית",
		"hi":          "हिन्दी",
		"hr":          "Hrvatski jezik",
		"hr_HR":       "Hrvatski jezik (Croatia)",
		"hu":          "Magyar",
		"id":          "Bahasa Indonesia",
		"is":          "íslenska",
		"it":          "Italiano",
		"ja":          "日本語",
		"ka":          "ქართული",
		"ko":          "한국어",
		"lt":          "lietuvių kalba",
		"lv":          "Latviešu valoda",
		"mk":          "македонски јазик",
		"ms_MY":       "Bahasa Melayu",
		"nb":          "Norsk bokmål",
		"nl":          "Nederlands",
		"nl_BE":       "Vlaams",
		"nn":          "Norsk nynorsk",
		"pa":          "ਪੰਜਾਬੀ",
		"pl":          "Język polski",
		"pt":          "Português",
		"pt_BR":       "Português brasileiro",
		"pt_PT":       "Português europeu",
		"ro":          "română",
		"ru":          "русский язык",
		"sk":          "Slovenčina",
		"sq":          "shqip",
		"sr":          "српски језик",
		"sv":          "Svenska",
		"ta":          "தமிழ்",
		"th":          "ไทย",
		"tr":          "Türkçe",
		"uk":          "українська мова",
		"vi":          "Tiếng Việt",
		"zh_CN":       "中文简体",
		"zh_HK":       "中文繁體",
		"zh_TW":       "中文繁體",
	}

	// for all folders in locale which match a locale from https://www.transifex.com/api/2/languages/
	// use the language name unless we have an override
	webLocales, err := FetchTranslationLocales(base)
	if err != nil {
		log.Printf("Failed to get up to date language list, using fallback.")
		return haveTranslatedNames
	}

	return GetInstalledLocales(base, webLocales, haveTranslatedNames)
}

func FetchTranslationLocales(base string) (map[string]locale, error) {
	file, err := os.Open(path.Join(base, "data/langs"))
	if err != nil {
		return nil, err
	}
	defer file.Close()

	webLocales := make(map[string]locale)
	// Parse the api response into a list of possible locales
	dec := json.NewDecoder(file)
	for {
		var webList []locale
		if err = dec.Decode(&webList); err == io.EOF {
			break
		} else if err != nil {
			log.Fatal(err)
		}

		// The api returns an array, so we need to map it
		for _, l := range webList {
			webLocales[l.Code] = l
		}
	}

	return webLocales, nil
}

// Get a list of all languages installed in our locale folder with translations if available
func GetInstalledLocales(base string, webLocales map[string]locale, nameTranslations map[string]string) map[string]string {
	localFiles, err := ioutil.ReadDir(path.Join(base, "locale"))

	if err != nil {
		log.Print("No locales found in 'locale'. Try running 'make i18n'.")
		log.Fatal(err)
	}

	locales := make(map[string]string, len(localFiles))
	locales["en_US"] = "English"

	for _, f := range localFiles {
		// TODO: Ensure a language has 100% of the template file
		// Currently this is what should be on the torcheck_completed
		// branch on the translations git should be, so we don't really
		// have to check it in theory...
		code := f.Name()

		// Only accept folders which have corresponding locale
		if !f.IsDir() || webLocales[code] == (locale{}) {
			continue
		}

		// If we have a translated name for a given locale, use it
		if transName := nameTranslations[code]; transName != "" {
			locales[code] = transName
		} else {
			log.Print("No translated name for code: " + code)
			locales[code] = webLocales[code].Name
		}
	}

	return locales
}
<|endoftext|>


<commit_before>package main

import (
	"code.google.com/p/go-uuid/uuid"
	"github.com/clawio/service.auth/lib"
	"io"
	"net/http"
	"os"
	"path"
	"strings"
)

func getPathFromReq(r *http.Request) string {

	if len(r.URL.Path) > len(endPoint) {
		return path.Clean(r.URL.Path[len(endPoint):])
	}
	return ""
}

// getHome returns the user home directory.
// the logical home has this layout.
// local/users/<letter>/<pid>
// Example: /local/users/o/ourense
// idt.Pid must be always non-empty
func getHome(idt *lib.Identity) string {

	pid := path.Clean(idt.Pid)

	if pid == "" {
		panic("idt.Pid must not be empty")
	}

	return path.Join("/local", "users", string(pid[0]), pid)
}

func isUnderHome(p string, idt *lib.Identity) bool {

	p = path.Clean(p)

	if strings.HasPrefix(p, getHome(idt)) {
		return true
	}

	return false
}

func copyFile(src, dst string, size int64) (err error) {
	reader, err := os.Open(src)
	if err != nil {
		return err
	}
	defer reader.Close()

	writer, err := os.Create(dst)
	if err != nil {
		return err
	}
	defer writer.Close()

	_, err = io.CopyN(writer, reader, size)
	if err != nil {
		return err
	}
	return nil
}

func copyDir(src, dst string) (err error) {
	err = os.Mkdir(dst, dirPerm)
	if err != nil {
		return err
	}

	directory, err := os.Open(src)
	if err != nil {
		return err
	}
	defer directory.Close()

	objects, err := directory.Readdir(-1)

	for _, obj := range objects {

		_src := path.Join(src, obj.Name())
		_dst := path.Join(dst, obj.Name())

		if obj.IsDir() {
			// create sub-directories - recursively
			err = copyDir(_src, _dst)
			if err != nil {
				return err
			}
		} else {
			// perform copy
			err = copyFile(_src, _dst, obj.Size())
			if err != nil {
				return err
			}
		}
	}
	return
}

// getTraceID returns the traceID that comes in the request
// or generate a new one
func getTraceID(r *http.Request) string {
	traceID := r.Header.Get("CIO-TraceID")
	if traceID == "" {
		return uuid.New()
	}
	return traceID
}
<commit_msg>Fixed logical path retrieval<commit_after>package main

import (
	"code.google.com/p/go-uuid/uuid"
	"github.com/clawio/service.auth/lib"
	"io"
	"net/http"
	"os"
	"path"
	"strings"
)

func getPathFromReq(r *http.Request) string {
	return path.Clean(r.URL.Path)
}

// getHome returns the user home directory.
// the logical home has this layout.
// local/users/<letter>/<pid>
// Example: /local/users/o/ourense
// idt.Pid must be always non-empty
func getHome(idt *lib.Identity) string {

	pid := path.Clean(idt.Pid)

	if pid == "" {
		panic("idt.Pid must not be empty")
	}

	return path.Join("/local", "users", string(pid[0]), pid)
}

func isUnderHome(p string, idt *lib.Identity) bool {

	p = path.Clean(p)

	if strings.HasPrefix(p, getHome(idt)) {
		return true
	}

	return false
}

func copyFile(src, dst string, size int64) (err error) {
	reader, err := os.Open(src)
	if err != nil {
		return err
	}
	defer reader.Close()

	writer, err := os.Create(dst)
	if err != nil {
		return err
	}
	defer writer.Close()

	_, err = io.CopyN(writer, reader, size)
	if err != nil {
		return err
	}
	return nil
}

func copyDir(src, dst string) (err error) {
	err = os.Mkdir(dst, dirPerm)
	if err != nil {
		return err
	}

	directory, err := os.Open(src)
	if err != nil {
		return err
	}
	defer directory.Close()

	objects, err := directory.Readdir(-1)

	for _, obj := range objects {

		_src := path.Join(src, obj.Name())
		_dst := path.Join(dst, obj.Name())

		if obj.IsDir() {
			// create sub-directories - recursively
			err = copyDir(_src, _dst)
			if err != nil {
				return err
			}
		} else {
			// perform copy
			err = copyFile(_src, _dst, obj.Size())
			if err != nil {
				return err
			}
		}
	}
	return
}

// getTraceID returns the traceID that comes in the request
// or generate a new one
func getTraceID(r *http.Request) string {
	traceID := r.Header.Get("CIO-TraceID")
	if traceID == "" {
		return uuid.New()
	}
	return traceID
}
<|endoftext|>


<commit_before>package bip32

import (
	"bytes"
	"code.google.com/p/go.crypto/ripemd160"
	"crypto/sha256"
	"encoding/binary"
	"errors"
	"github.com/FactomProject/basen"
	"github.com/FactomProject/btcutilecc"
	"io"
	"math/big"
)

var (
	curve                 = btcutil.Secp256k1()
	curveParams           = curve.Params()
	BitcoinBase58Encoding = basen.NewEncoding("123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz")
)

//
// Hashes
//

func hashSha256(data []byte) []byte {
	hasher := sha256.New()
	hasher.Write(data)
	return hasher.Sum(nil)
}

func hashDoubleSha256(data []byte) []byte {
	return hashSha256(hashSha256(data))
}

func hashRipeMD160(data []byte) []byte {
	hasher := ripemd160.New()
	io.WriteString(hasher, string(data))
	return hasher.Sum(nil)
}

func hash160(data []byte) []byte {
	return hashRipeMD160(hashSha256(data))
}

//
// Encoding
//

func checksum(data []byte) []byte {
	return hashDoubleSha256(data)[:4]
}

func addChecksumToBytes(data []byte) []byte {
	checksum := checksum(data)
	return append(data, checksum...)
}

func base58Encode(data []byte) []byte {
	return []byte(BitcoinBase58Encoding.EncodeToString(data))
}

// Keys
func publicKeyForPrivateKey(key []byte) []byte {
	return compressPublicKey(curve.ScalarBaseMult([]byte(key)))
}

func addPublicKeys(key1 []byte, key2 []byte) []byte {
	x1, y1 := expandPublicKey(key1)
	x2, y2 := expandPublicKey(key2)
	return compressPublicKey(curve.Add(x1, y1, x2, y2))
}

func addPrivateKeys(key1 []byte, key2 []byte) []byte {
	var key1Int big.Int
	var key2Int big.Int
	key1Int.SetBytes(key1)
	key2Int.SetBytes(key2)

	key1Int.Add(&key1Int, &key2Int)
	key1Int.Mod(&key1Int, curve.Params().N)

	return key1Int.Bytes()
}

func compressPublicKey(x *big.Int, y *big.Int) []byte {
	var key bytes.Buffer

	// Write header; 0x2 for even y value; 0x3 for odd
	key.WriteByte(byte(0x2) + byte(y.Bit(0)))

	// Write X coord; Pad the key so x is aligned with the LSB. Pad size is key length - header size (1) - xBytes size
	xBytes := x.Bytes()
	for i := 0; i < (PublicKeyCompressedLength - 1 - len(xBytes)); i++ {
		key.WriteByte(0x0)
	}
	key.Write(xBytes)

	return key.Bytes()
}

// As described at https://bitcointa.lk/threads/compressed-keys-y-from-x.95735/
func expandPublicKey(key []byte) (*big.Int, *big.Int) {
	Y := big.NewInt(0)
	X := big.NewInt(0)
	qPlus1Div4 := big.NewInt(0)
	X.SetBytes(key[1:])

	// y^2 = x^3 + ax^2 + b
	// a = 0
	// => y^2 = x^3 + b
	ySquared := X.Exp(X, big.NewInt(3), nil)
	ySquared.Add(ySquared, curveParams.B)

	qPlus1Div4.Add(curveParams.P, big.NewInt(1))
	qPlus1Div4.Div(qPlus1Div4, big.NewInt(4))

	// sqrt(n) = n^((q+1)/4) if q = 3 mod 4
	Y.Exp(ySquared, qPlus1Div4, curveParams.P)

	if uint32(key[0])%2 == 0 {
		Y.Sub(curveParams.P, Y)
	}

	return X, Y
}

func validatePrivateKey(key []byte) error {
	keyInt, _ := binary.ReadVarint(bytes.NewBuffer(key))
	if keyInt == 0 || bytes.Compare(key, curveParams.N.Bytes()) >= 0 {
		return errors.New("Invalid seed")
	}

	return nil
}

func validateChildPublicKey(key []byte) error {
	x, y := expandPublicKey(key)

	if x.Sign() == 0 || y.Sign() == 0 {
		return errors.New("Invalid public key")
	}

	return nil
}

//
// Numerical
//
func uint32Bytes(i uint32) []byte {
	bytes := make([]byte, 4)
	binary.BigEndian.PutUint32(bytes, i)
	return bytes
}
<commit_msg>replaced deprecated reference to code.google.com<commit_after>package bip32

import (
	"bytes"
	"golang.org/x/crypto/ripemd160"
	"crypto/sha256"
	"encoding/binary"
	"errors"
	"github.com/FactomProject/basen"
	"github.com/FactomProject/btcutilecc"
	"io"
	"math/big"
)

var (
	curve                 = btcutil.Secp256k1()
	curveParams           = curve.Params()
	BitcoinBase58Encoding = basen.NewEncoding("123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz")
)

//
// Hashes
//

func hashSha256(data []byte) []byte {
	hasher := sha256.New()
	hasher.Write(data)
	return hasher.Sum(nil)
}

func hashDoubleSha256(data []byte) []byte {
	return hashSha256(hashSha256(data))
}

func hashRipeMD160(data []byte) []byte {
	hasher := ripemd160.New()
	io.WriteString(hasher, string(data))
	return hasher.Sum(nil)
}

func hash160(data []byte) []byte {
	return hashRipeMD160(hashSha256(data))
}

//
// Encoding
//

func checksum(data []byte) []byte {
	return hashDoubleSha256(data)[:4]
}

func addChecksumToBytes(data []byte) []byte {
	checksum := checksum(data)
	return append(data, checksum...)
}

func base58Encode(data []byte) []byte {
	return []byte(BitcoinBase58Encoding.EncodeToString(data))
}

// Keys
func publicKeyForPrivateKey(key []byte) []byte {
	return compressPublicKey(curve.ScalarBaseMult([]byte(key)))
}

func addPublicKeys(key1 []byte, key2 []byte) []byte {
	x1, y1 := expandPublicKey(key1)
	x2, y2 := expandPublicKey(key2)
	return compressPublicKey(curve.Add(x1, y1, x2, y2))
}

func addPrivateKeys(key1 []byte, key2 []byte) []byte {
	var key1Int big.Int
	var key2Int big.Int
	key1Int.SetBytes(key1)
	key2Int.SetBytes(key2)

	key1Int.Add(&key1Int, &key2Int)
	key1Int.Mod(&key1Int, curve.Params().N)

	return key1Int.Bytes()
}

func compressPublicKey(x *big.Int, y *big.Int) []byte {
	var key bytes.Buffer

	// Write header; 0x2 for even y value; 0x3 for odd
	key.WriteByte(byte(0x2) + byte(y.Bit(0)))

	// Write X coord; Pad the key so x is aligned with the LSB. Pad size is key length - header size (1) - xBytes size
	xBytes := x.Bytes()
	for i := 0; i < (PublicKeyCompressedLength - 1 - len(xBytes)); i++ {
		key.WriteByte(0x0)
	}
	key.Write(xBytes)

	return key.Bytes()
}

// As described at https://bitcointa.lk/threads/compressed-keys-y-from-x.95735/
func expandPublicKey(key []byte) (*big.Int, *big.Int) {
	Y := big.NewInt(0)
	X := big.NewInt(0)
	qPlus1Div4 := big.NewInt(0)
	X.SetBytes(key[1:])

	// y^2 = x^3 + ax^2 + b
	// a = 0
	// => y^2 = x^3 + b
	ySquared := X.Exp(X, big.NewInt(3), nil)
	ySquared.Add(ySquared, curveParams.B)

	qPlus1Div4.Add(curveParams.P, big.NewInt(1))
	qPlus1Div4.Div(qPlus1Div4, big.NewInt(4))

	// sqrt(n) = n^((q+1)/4) if q = 3 mod 4
	Y.Exp(ySquared, qPlus1Div4, curveParams.P)

	if uint32(key[0])%2 == 0 {
		Y.Sub(curveParams.P, Y)
	}

	return X, Y
}

func validatePrivateKey(key []byte) error {
	keyInt, _ := binary.ReadVarint(bytes.NewBuffer(key))
	if keyInt == 0 || bytes.Compare(key, curveParams.N.Bytes()) >= 0 {
		return errors.New("Invalid seed")
	}

	return nil
}

func validateChildPublicKey(key []byte) error {
	x, y := expandPublicKey(key)

	if x.Sign() == 0 || y.Sign() == 0 {
		return errors.New("Invalid public key")
	}

	return nil
}

//
// Numerical
//
func uint32Bytes(i uint32) []byte {
	bytes := make([]byte, 4)
	binary.BigEndian.PutUint32(bytes, i)
	return bytes
}
<|endoftext|>