<commit_before>//  Copyright (c) 2017 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 		http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package mem

import (
	"math"
	"sort"

	"github.com/RoaringBitmap/roaring"
	"github.com/blevesearch/bleve/analysis"
	"github.com/blevesearch/bleve/document"
	"github.com/blevesearch/bleve/index"
)

// NewFromAnalyzedDocs places the analyzed document mutations into a new segment
func NewFromAnalyzedDocs(results []*index.AnalysisResult) *Segment {
	s := New()

	// ensure that _id field get fieldID 0
	s.getOrDefineField("_id")

	// fill Dicts/DictKeys and preallocate memory
	s.initializeDict(results)

	// walk each doc
	for _, result := range results {
		s.processDocument(result)
	}

	// go back and sort the dictKeys
	for _, dict := range s.DictKeys {
		sort.Strings(dict)
	}

	// compute memory usage of segment
	s.updateSize()

	// professional debugging
	//
	// log.Printf("fields: %v\n", s.FieldsMap)
	// log.Printf("fieldsInv: %v\n", s.FieldsInv)
	// log.Printf("fieldsLoc: %v\n", s.FieldsLoc)
	// log.Printf("dicts: %v\n", s.Dicts)
	// log.Printf("dict keys: %v\n", s.DictKeys)
	// for i, posting := range s.Postings {
	// 	log.Printf("posting %d: %v\n", i, posting)
	// }
	// for i, freq := range s.Freqs {
	// 	log.Printf("freq %d: %v\n", i, freq)
	// }
	// for i, norm := range s.Norms {
	// 	log.Printf("norm %d: %v\n", i, norm)
	// }
	// for i, field := range s.Locfields {
	// 	log.Printf("field %d: %v\n", i, field)
	// }
	// for i, start := range s.Locstarts {
	// 	log.Printf("start %d: %v\n", i, start)
	// }
	// for i, end := range s.Locends {
	// 	log.Printf("end %d: %v\n", i, end)
	// }
	// for i, pos := range s.Locpos {
	// 	log.Printf("pos %d: %v\n", i, pos)
	// }
	// for i, apos := range s.Locarraypos {
	// 	log.Printf("apos %d: %v\n", i, apos)
	// }
	// log.Printf("stored: %v\n", s.Stored)
	// log.Printf("stored types: %v\n", s.StoredTypes)
	// log.Printf("stored pos: %v\n", s.StoredPos)

	return s
}

// fill Dicts/DictKeys and preallocate memory for postings
func (s *Segment) initializeDict(results []*index.AnalysisResult) {
	var numPostingsLists int

	numTermsPerPostingsList := make([]int, 0, 64) // Keyed by postings list id.
	numLocsPerPostingsList := make([]int, 0, 64)  // Keyed by postings list id.

	var numTokenFrequencies int
	var totLocs int

	// initial scan for all fieldID's to sort them
	for _, result := range results {
		for _, field := range result.Document.CompositeFields {
			s.getOrDefineField(field.Name())
		}
		for _, field := range result.Document.Fields {
			s.getOrDefineField(field.Name())
		}
	}
	sort.Strings(s.FieldsInv[1:]) // keep _id as first field
	s.FieldsMap = make(map[string]uint16, len(s.FieldsInv))
	for fieldID, fieldName := range s.FieldsInv {
		s.FieldsMap[fieldName] = uint16(fieldID + 1)
	}

	processField := func(fieldID uint16, tfs analysis.TokenFrequencies) {
		dict := s.Dicts[fieldID]
		dictKeys := s.DictKeys[fieldID]
		for term, tf := range tfs {
			pidPlus1, exists := dict[term]
			if !exists {
				numPostingsLists++
				pidPlus1 = uint64(numPostingsLists)
				dict[term] = pidPlus1
				dictKeys = append(dictKeys, term)
				numTermsPerPostingsList = append(numTermsPerPostingsList, 0)
				numLocsPerPostingsList = append(numLocsPerPostingsList, 0)
			}
			pid := pidPlus1 - 1
			numTermsPerPostingsList[pid] += 1
			numLocsPerPostingsList[pid] += len(tf.Locations)
			totLocs += len(tf.Locations)
		}
		numTokenFrequencies += len(tfs)
		s.DictKeys[fieldID] = dictKeys
	}

	for _, result := range results {
		// walk each composite field
		for _, field := range result.Document.CompositeFields {
			fieldID := uint16(s.getOrDefineField(field.Name()))
			_, tf := field.Analyze()
			processField(fieldID, tf)
		}

		// walk each field
		for i, field := range result.Document.Fields {
			fieldID := uint16(s.getOrDefineField(field.Name()))
			tf := result.Analyzed[i]
			processField(fieldID, tf)
		}
	}

	s.Postings = make([]*roaring.Bitmap, numPostingsLists)
	for i := 0; i < numPostingsLists; i++ {
		s.Postings[i] = roaring.New()
	}
	s.PostingsLocs = make([]*roaring.Bitmap, numPostingsLists)
	for i := 0; i < numPostingsLists; i++ {
		s.PostingsLocs[i] = roaring.New()
	}

	// Preallocate big, contiguous backing arrays.
	auint64Backing := make([][]uint64, numPostingsLists*4+totLocs) // For Freqs, Locstarts, Locends, Locpos, sub-Locarraypos.
	uint64Backing := make([]uint64, numTokenFrequencies+totLocs*3) // For sub-Freqs, sub-Locstarts, sub-Locends, sub-Locpos.
	float32Backing := make([]float32, numTokenFrequencies)         // For sub-Norms.
	uint16Backing := make([]uint16, totLocs)                       // For sub-Locfields.

	// Point top-level slices to the backing arrays.
	s.Freqs = auint64Backing[0:numPostingsLists]
	auint64Backing = auint64Backing[numPostingsLists:]

	s.Norms = make([][]float32, numPostingsLists)

	s.Locfields = make([][]uint16, numPostingsLists)

	s.Locstarts = auint64Backing[0:numPostingsLists]
	auint64Backing = auint64Backing[numPostingsLists:]

	s.Locends = auint64Backing[0:numPostingsLists]
	auint64Backing = auint64Backing[numPostingsLists:]

	s.Locpos = auint64Backing[0:numPostingsLists]
	auint64Backing = auint64Backing[numPostingsLists:]

	s.Locarraypos = make([][][]uint64, numPostingsLists)

	// Point sub-slices to the backing arrays.
	for pid, numTerms := range numTermsPerPostingsList {
		s.Freqs[pid] = uint64Backing[0:0]
		uint64Backing = uint64Backing[numTerms:]

		s.Norms[pid] = float32Backing[0:0]
		float32Backing = float32Backing[numTerms:]
	}

	for pid, numLocs := range numLocsPerPostingsList {
		s.Locfields[pid] = uint16Backing[0:0]
		uint16Backing = uint16Backing[numLocs:]

		s.Locstarts[pid] = uint64Backing[0:0]
		uint64Backing = uint64Backing[numLocs:]

		s.Locends[pid] = uint64Backing[0:0]
		uint64Backing = uint64Backing[numLocs:]

		s.Locpos[pid] = uint64Backing[0:0]
		uint64Backing = uint64Backing[numLocs:]

		s.Locarraypos[pid] = auint64Backing[0:0]
		auint64Backing = auint64Backing[numLocs:]
	}
}

func (s *Segment) processDocument(result *index.AnalysisResult) {
	// used to collate information across fields
	docMap := make(map[uint16]analysis.TokenFrequencies, len(s.FieldsMap))
	fieldLens := make(map[uint16]int, len(s.FieldsMap))

	docNum := uint64(s.addDocument())

	processField := func(field uint16, name string, l int, tf analysis.TokenFrequencies) {
		fieldLens[field] += l
		if existingFreqs, ok := docMap[field]; ok {
			existingFreqs.MergeAll(name, tf)
		} else {
			docMap[field] = tf
		}
	}

	// walk each composite field
	for _, field := range result.Document.CompositeFields {
		fieldID := uint16(s.getOrDefineField(field.Name()))
		l, tf := field.Analyze()
		processField(fieldID, field.Name(), l, tf)
	}

	docStored := s.Stored[docNum]
	docStoredTypes := s.StoredTypes[docNum]
	docStoredPos := s.StoredPos[docNum]

	// walk each field
	for i, field := range result.Document.Fields {
		fieldID := uint16(s.getOrDefineField(field.Name()))
		l := result.Length[i]
		tf := result.Analyzed[i]
		processField(fieldID, field.Name(), l, tf)
		if field.Options().IsStored() {
			docStored[fieldID] = append(docStored[fieldID], field.Value())
			docStoredTypes[fieldID] = append(docStoredTypes[fieldID], encodeFieldType(field))
			docStoredPos[fieldID] = append(docStoredPos[fieldID], field.ArrayPositions())
		}

		if field.Options().IncludeDocValues() {
			s.DocValueFields[fieldID] = true
		}
	}

	// now that its been rolled up into docMap, walk that
	for fieldID, tokenFrequencies := range docMap {
		dict := s.Dicts[fieldID]
		norm := float32(1.0 / math.Sqrt(float64(fieldLens[fieldID])))
		for term, tokenFreq := range tokenFrequencies {
			pid := dict[term] - 1
			bs := s.Postings[pid]
			bs.AddInt(int(docNum))
			s.Freqs[pid] = append(s.Freqs[pid], uint64(tokenFreq.Frequency()))
			s.Norms[pid] = append(s.Norms[pid], norm)
			locationBS := s.PostingsLocs[pid]
			if len(tokenFreq.Locations) > 0 {
				locationBS.AddInt(int(docNum))

				locfields := s.Locfields[pid]
				locstarts := s.Locstarts[pid]
				locends := s.Locends[pid]
				locpos := s.Locpos[pid]
				locarraypos := s.Locarraypos[pid]

				for _, loc := range tokenFreq.Locations {
					var locf = fieldID
					if loc.Field != "" {
						locf = uint16(s.getOrDefineField(loc.Field))
					}
					locfields = append(locfields, locf)
					locstarts = append(locstarts, uint64(loc.Start))
					locends = append(locends, uint64(loc.End))
					locpos = append(locpos, uint64(loc.Position))
					if len(loc.ArrayPositions) > 0 {
						locarraypos = append(locarraypos, loc.ArrayPositions)
					} else {
						locarraypos = append(locarraypos, nil)
					}
				}

				s.Locfields[pid] = locfields
				s.Locstarts[pid] = locstarts
				s.Locends[pid] = locends
				s.Locpos[pid] = locpos
				s.Locarraypos[pid] = locarraypos
			}
		}
	}
}

func (s *Segment) getOrDefineField(name string) int {
	fieldIDPlus1, ok := s.FieldsMap[name]
	if !ok {
		fieldIDPlus1 = uint16(len(s.FieldsInv) + 1)
		s.FieldsMap[name] = fieldIDPlus1
		s.FieldsInv = append(s.FieldsInv, name)
		s.Dicts = append(s.Dicts, make(map[string]uint64))
		s.DictKeys = append(s.DictKeys, make([]string, 0))
	}
	return int(fieldIDPlus1 - 1)
}

func (s *Segment) addDocument() int {
	docNum := len(s.Stored)
	s.Stored = append(s.Stored, map[uint16][][]byte{})
	s.StoredTypes = append(s.StoredTypes, map[uint16][]byte{})
	s.StoredPos = append(s.StoredPos, map[uint16][][]uint64{})
	return docNum
}

func encodeFieldType(f document.Field) byte {
	fieldType := byte('x')
	switch f.(type) {
	case *document.TextField:
		fieldType = 't'
	case *document.NumericField:
		fieldType = 'n'
	case *document.DateTimeField:
		fieldType = 'd'
	case *document.BooleanField:
		fieldType = 'b'
	case *document.GeoPointField:
		fieldType = 'g'
	case *document.CompositeField:
		fieldType = 'c'
	}
	return fieldType
}
<commit_msg>scorch mem processDocument reuses fieldLens/docMap arrays<commit_after>//  Copyright (c) 2017 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 		http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package mem

import (
	"math"
	"sort"

	"github.com/RoaringBitmap/roaring"
	"github.com/blevesearch/bleve/analysis"
	"github.com/blevesearch/bleve/document"
	"github.com/blevesearch/bleve/index"
)

// NewFromAnalyzedDocs places the analyzed document mutations into a new segment
func NewFromAnalyzedDocs(results []*index.AnalysisResult) *Segment {
	s := New()

	// ensure that _id field get fieldID 0
	s.getOrDefineField("_id")

	// fill Dicts/DictKeys and preallocate memory
	s.initializeDict(results)

	// walk each doc
	fieldLensReuse := make([]int, len(s.FieldsMap))
	docMapReuse := make([]analysis.TokenFrequencies, len(s.FieldsMap))
	for _, result := range results {
		s.processDocument(result, fieldLensReuse, docMapReuse)
	}

	// go back and sort the dictKeys
	for _, dict := range s.DictKeys {
		sort.Strings(dict)
	}

	// compute memory usage of segment
	s.updateSize()

	// professional debugging
	//
	// log.Printf("fields: %v\n", s.FieldsMap)
	// log.Printf("fieldsInv: %v\n", s.FieldsInv)
	// log.Printf("fieldsLoc: %v\n", s.FieldsLoc)
	// log.Printf("dicts: %v\n", s.Dicts)
	// log.Printf("dict keys: %v\n", s.DictKeys)
	// for i, posting := range s.Postings {
	// 	log.Printf("posting %d: %v\n", i, posting)
	// }
	// for i, freq := range s.Freqs {
	// 	log.Printf("freq %d: %v\n", i, freq)
	// }
	// for i, norm := range s.Norms {
	// 	log.Printf("norm %d: %v\n", i, norm)
	// }
	// for i, field := range s.Locfields {
	// 	log.Printf("field %d: %v\n", i, field)
	// }
	// for i, start := range s.Locstarts {
	// 	log.Printf("start %d: %v\n", i, start)
	// }
	// for i, end := range s.Locends {
	// 	log.Printf("end %d: %v\n", i, end)
	// }
	// for i, pos := range s.Locpos {
	// 	log.Printf("pos %d: %v\n", i, pos)
	// }
	// for i, apos := range s.Locarraypos {
	// 	log.Printf("apos %d: %v\n", i, apos)
	// }
	// log.Printf("stored: %v\n", s.Stored)
	// log.Printf("stored types: %v\n", s.StoredTypes)
	// log.Printf("stored pos: %v\n", s.StoredPos)

	return s
}

// fill Dicts/DictKeys and preallocate memory for postings
func (s *Segment) initializeDict(results []*index.AnalysisResult) {
	var numPostingsLists int

	numTermsPerPostingsList := make([]int, 0, 64) // Keyed by postings list id.
	numLocsPerPostingsList := make([]int, 0, 64)  // Keyed by postings list id.

	var numTokenFrequencies int
	var totLocs int

	// initial scan for all fieldID's to sort them
	for _, result := range results {
		for _, field := range result.Document.CompositeFields {
			s.getOrDefineField(field.Name())
		}
		for _, field := range result.Document.Fields {
			s.getOrDefineField(field.Name())
		}
	}
	sort.Strings(s.FieldsInv[1:]) // keep _id as first field
	s.FieldsMap = make(map[string]uint16, len(s.FieldsInv))
	for fieldID, fieldName := range s.FieldsInv {
		s.FieldsMap[fieldName] = uint16(fieldID + 1)
	}

	processField := func(fieldID uint16, tfs analysis.TokenFrequencies) {
		dict := s.Dicts[fieldID]
		dictKeys := s.DictKeys[fieldID]
		for term, tf := range tfs {
			pidPlus1, exists := dict[term]
			if !exists {
				numPostingsLists++
				pidPlus1 = uint64(numPostingsLists)
				dict[term] = pidPlus1
				dictKeys = append(dictKeys, term)
				numTermsPerPostingsList = append(numTermsPerPostingsList, 0)
				numLocsPerPostingsList = append(numLocsPerPostingsList, 0)
			}
			pid := pidPlus1 - 1
			numTermsPerPostingsList[pid] += 1
			numLocsPerPostingsList[pid] += len(tf.Locations)
			totLocs += len(tf.Locations)
		}
		numTokenFrequencies += len(tfs)
		s.DictKeys[fieldID] = dictKeys
	}

	for _, result := range results {
		// walk each composite field
		for _, field := range result.Document.CompositeFields {
			fieldID := uint16(s.getOrDefineField(field.Name()))
			_, tf := field.Analyze()
			processField(fieldID, tf)
		}

		// walk each field
		for i, field := range result.Document.Fields {
			fieldID := uint16(s.getOrDefineField(field.Name()))
			tf := result.Analyzed[i]
			processField(fieldID, tf)
		}
	}

	s.Postings = make([]*roaring.Bitmap, numPostingsLists)
	for i := 0; i < numPostingsLists; i++ {
		s.Postings[i] = roaring.New()
	}
	s.PostingsLocs = make([]*roaring.Bitmap, numPostingsLists)
	for i := 0; i < numPostingsLists; i++ {
		s.PostingsLocs[i] = roaring.New()
	}

	// Preallocate big, contiguous backing arrays.
	auint64Backing := make([][]uint64, numPostingsLists*4+totLocs) // For Freqs, Locstarts, Locends, Locpos, sub-Locarraypos.
	uint64Backing := make([]uint64, numTokenFrequencies+totLocs*3) // For sub-Freqs, sub-Locstarts, sub-Locends, sub-Locpos.
	float32Backing := make([]float32, numTokenFrequencies)         // For sub-Norms.
	uint16Backing := make([]uint16, totLocs)                       // For sub-Locfields.

	// Point top-level slices to the backing arrays.
	s.Freqs = auint64Backing[0:numPostingsLists]
	auint64Backing = auint64Backing[numPostingsLists:]

	s.Norms = make([][]float32, numPostingsLists)

	s.Locfields = make([][]uint16, numPostingsLists)

	s.Locstarts = auint64Backing[0:numPostingsLists]
	auint64Backing = auint64Backing[numPostingsLists:]

	s.Locends = auint64Backing[0:numPostingsLists]
	auint64Backing = auint64Backing[numPostingsLists:]

	s.Locpos = auint64Backing[0:numPostingsLists]
	auint64Backing = auint64Backing[numPostingsLists:]

	s.Locarraypos = make([][][]uint64, numPostingsLists)

	// Point sub-slices to the backing arrays.
	for pid, numTerms := range numTermsPerPostingsList {
		s.Freqs[pid] = uint64Backing[0:0]
		uint64Backing = uint64Backing[numTerms:]

		s.Norms[pid] = float32Backing[0:0]
		float32Backing = float32Backing[numTerms:]
	}

	for pid, numLocs := range numLocsPerPostingsList {
		s.Locfields[pid] = uint16Backing[0:0]
		uint16Backing = uint16Backing[numLocs:]

		s.Locstarts[pid] = uint64Backing[0:0]
		uint64Backing = uint64Backing[numLocs:]

		s.Locends[pid] = uint64Backing[0:0]
		uint64Backing = uint64Backing[numLocs:]

		s.Locpos[pid] = uint64Backing[0:0]
		uint64Backing = uint64Backing[numLocs:]

		s.Locarraypos[pid] = auint64Backing[0:0]
		auint64Backing = auint64Backing[numLocs:]
	}
}

func (s *Segment) processDocument(result *index.AnalysisResult,
	fieldLens []int, docMap []analysis.TokenFrequencies) {
	// clear the fieldLens and docMap for reuse
	n := len(s.FieldsMap)
	for i := 0; i < n; i++ {
		fieldLens[i] = 0
		docMap[i] = nil
	}

	docNum := uint64(s.addDocument())

	processField := func(fieldID uint16, name string, l int, tf analysis.TokenFrequencies) {
		fieldLens[fieldID] += l

		existingFreqs := docMap[fieldID]
		if existingFreqs != nil {
			existingFreqs.MergeAll(name, tf)
		} else {
			docMap[fieldID] = tf
		}
	}

	// walk each composite field
	for _, field := range result.Document.CompositeFields {
		fieldID := uint16(s.getOrDefineField(field.Name()))
		l, tf := field.Analyze()
		processField(fieldID, field.Name(), l, tf)
	}

	docStored := s.Stored[docNum]
	docStoredTypes := s.StoredTypes[docNum]
	docStoredPos := s.StoredPos[docNum]

	// walk each field
	for i, field := range result.Document.Fields {
		fieldID := uint16(s.getOrDefineField(field.Name()))
		l := result.Length[i]
		tf := result.Analyzed[i]
		processField(fieldID, field.Name(), l, tf)
		if field.Options().IsStored() {
			docStored[fieldID] = append(docStored[fieldID], field.Value())
			docStoredTypes[fieldID] = append(docStoredTypes[fieldID], encodeFieldType(field))
			docStoredPos[fieldID] = append(docStoredPos[fieldID], field.ArrayPositions())
		}

		if field.Options().IncludeDocValues() {
			s.DocValueFields[fieldID] = true
		}
	}

	// now that its been rolled up into docMap, walk that
	for fieldID, tokenFrequencies := range docMap {
		dict := s.Dicts[fieldID]
		norm := float32(1.0 / math.Sqrt(float64(fieldLens[fieldID])))
		for term, tokenFreq := range tokenFrequencies {
			pid := dict[term] - 1
			bs := s.Postings[pid]
			bs.AddInt(int(docNum))
			s.Freqs[pid] = append(s.Freqs[pid], uint64(tokenFreq.Frequency()))
			s.Norms[pid] = append(s.Norms[pid], norm)
			locationBS := s.PostingsLocs[pid]
			if len(tokenFreq.Locations) > 0 {
				locationBS.AddInt(int(docNum))

				locfields := s.Locfields[pid]
				locstarts := s.Locstarts[pid]
				locends := s.Locends[pid]
				locpos := s.Locpos[pid]
				locarraypos := s.Locarraypos[pid]

				for _, loc := range tokenFreq.Locations {
					var locf = uint16(fieldID)
					if loc.Field != "" {
						locf = uint16(s.getOrDefineField(loc.Field))
					}
					locfields = append(locfields, locf)
					locstarts = append(locstarts, uint64(loc.Start))
					locends = append(locends, uint64(loc.End))
					locpos = append(locpos, uint64(loc.Position))
					if len(loc.ArrayPositions) > 0 {
						locarraypos = append(locarraypos, loc.ArrayPositions)
					} else {
						locarraypos = append(locarraypos, nil)
					}
				}

				s.Locfields[pid] = locfields
				s.Locstarts[pid] = locstarts
				s.Locends[pid] = locends
				s.Locpos[pid] = locpos
				s.Locarraypos[pid] = locarraypos
			}
		}
	}
}

func (s *Segment) getOrDefineField(name string) int {
	fieldIDPlus1, ok := s.FieldsMap[name]
	if !ok {
		fieldIDPlus1 = uint16(len(s.FieldsInv) + 1)
		s.FieldsMap[name] = fieldIDPlus1
		s.FieldsInv = append(s.FieldsInv, name)
		s.Dicts = append(s.Dicts, make(map[string]uint64))
		s.DictKeys = append(s.DictKeys, make([]string, 0))
	}
	return int(fieldIDPlus1 - 1)
}

func (s *Segment) addDocument() int {
	docNum := len(s.Stored)
	s.Stored = append(s.Stored, map[uint16][][]byte{})
	s.StoredTypes = append(s.StoredTypes, map[uint16][]byte{})
	s.StoredPos = append(s.StoredPos, map[uint16][][]uint64{})
	return docNum
}

func encodeFieldType(f document.Field) byte {
	fieldType := byte('x')
	switch f.(type) {
	case *document.TextField:
		fieldType = 't'
	case *document.NumericField:
		fieldType = 'n'
	case *document.DateTimeField:
		fieldType = 'd'
	case *document.BooleanField:
		fieldType = 'b'
	case *document.GeoPointField:
		fieldType = 'g'
	case *document.CompositeField:
		fieldType = 'c'
	}
	return fieldType
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"github.com/golang/glog"
	"github.com/nelhage/livegrep/server"
	"github.com/nelhage/livegrep/server/config"
	"github.com/nelhage/livegrep/server/middleware"
	"io/ioutil"
	"net/http"
	"os"
)

var (
	serveAddr *string = flag.String("listen", "127.0.0.1:8910", "The address to listen on")
	docRoot   *string = flag.String("docroot", "./web", "The livegrep document root (web/ directory)")
)

// var backendAddr *string = flag.String("connect", "localhost:9999", "The address to connect to")

func main() {
	flag.Parse()

	if len(flag.Args()) == 0 {
		glog.Fatalf("Usage: %s CONFIG.json", os.Args[0])
	}

	data, err := ioutil.ReadFile(flag.Arg(0))
	if err != nil {
		glog.Fatalf(err.Error())
	}

	cfg := &config.Config{
		DocRoot: *docRoot,
	}
	if err = json.Unmarshal(data, &cfg); err != nil {
		glog.Fatalf("reading %s: %s", flag.Arg(0), err.Error())
	}

	handler, err := server.New(cfg)
	if err != nil {
		panic(err.Error())
	}

	if cfg.ReverseProxy {
		handler = middleware.UnwrapProxyHeaders(handler)
	}

	glog.Fatal(http.ListenAndServe(*serveAddr, handler))
}
<commit_msg>Add some debug endpoints to livegrep.<commit_after>package main

import (
	"encoding/json"
	_ "expvar"
	"flag"
	"github.com/golang/glog"
	"github.com/nelhage/livegrep/server"
	"github.com/nelhage/livegrep/server/config"
	"github.com/nelhage/livegrep/server/middleware"
	"io/ioutil"
	"net/http"
	_ "net/http/pprof"
	"os"
)

var (
	serveAddr *string = flag.String("listen", "127.0.0.1:8910", "The address to listen on")
	docRoot   *string = flag.String("docroot", "./web", "The livegrep document root (web/ directory)")
)

// var backendAddr *string = flag.String("connect", "localhost:9999", "The address to connect to")

func main() {
	flag.Parse()

	if len(flag.Args()) == 0 {
		glog.Fatalf("Usage: %s CONFIG.json", os.Args[0])
	}

	data, err := ioutil.ReadFile(flag.Arg(0))
	if err != nil {
		glog.Fatalf(err.Error())
	}

	cfg := &config.Config{
		DocRoot: *docRoot,
	}
	if err = json.Unmarshal(data, &cfg); err != nil {
		glog.Fatalf("reading %s: %s", flag.Arg(0), err.Error())
	}

	handler, err := server.New(cfg)
	if err != nil {
		panic(err.Error())
	}

	if cfg.ReverseProxy {
		handler = middleware.UnwrapProxyHeaders(handler)
	}

	http.DefaultServeMux.Handle("/", handler)

	glog.Fatal(http.ListenAndServe(*serveAddr, nil))
}
<|endoftext|>


<commit_before>package gcache

import (
	"time"

	"google.golang.org/api/googleapi"
)

const (
	// MinimumField is minimim search fields on Google Drive API.
	MinimumField googleapi.Field = "files/id"

	defaultField googleapi.Field = "files(" +
		"name," +
		"createdTime," +
		"appProperties," +
		"modifiedTime," +
		"md5Checksum," +
		"size," +
		"id)"

	msec100 = time.Duration(100) * time.Millisecond
)
<commit_msg>refactor<commit_after>package gcache

import (
	"time"

	"google.golang.org/api/googleapi"
)

const (
	// MinimumField is minimim search fields on Google Drive API.
	MinimumField googleapi.Field = "files/id"

	defaultField googleapi.Field = "files(" +
		"createdTime," +
		"appProperties," +
		"modifiedTime," +
		"md5Checksum," +
		"size," +
		"name," +
		"id)"

	msec100 = time.Duration(100) * time.Millisecond
)
<|endoftext|>


<commit_before>package main

const NGINX_BUILD_VERSION = "0.7.2"

// nginx
const (
	NGINX_VERSION             = "1.9.12"
	NGINX_DOWNLOAD_URL_PREFIX = "http://nginx.org/download"
)

// pcre
const (
	PCRE_VERSION             = "8.38"
	PCRE_DOWNLOAD_URL_PREFIX = "http://ftp.csx.cam.ac.uk/pub/software/programming/pcre"
)

// openssl
const (
	OPENSSL_VERSION             = "1.0.2g"
	OPENSSL_DOWNLOAD_URL_PREFIX = "http://www.openssl.org/source"
)

// zlib
const (
	ZLIB_VERSION             = "1.2.8"
	ZLIB_DOWNLOAD_URL_PREFIX = "http://zlib.net"
)

// openResty
const (
	OPENRESTY_VERSION             = "1.9.7.3"
	OPENRESTY_DOWNLOAD_URL_PREFIX = "https://openresty.org/download"
)

// tengine
const (
	TENGINE_VERSION             = "2.1.2"
	TENGINE_DOWNLOAD_URL_PREFIX = "http://tengine.taobao.org/download"
)

// component enumerations
const (
	COMPONENT_NGINX = iota
	COMPONENT_OPENRESTY
	COMPONENT_TENGINE
	COMPONENT_PCRE
	COMPONENT_OPENSSL
	COMPONENT_ZLIB
	COMPONENT_MAX
)
<commit_msg>bumped openresty version to 1.9.7.4.<commit_after>package main

const NGINX_BUILD_VERSION = "0.7.2"

// nginx
const (
	NGINX_VERSION             = "1.9.12"
	NGINX_DOWNLOAD_URL_PREFIX = "http://nginx.org/download"
)

// pcre
const (
	PCRE_VERSION             = "8.38"
	PCRE_DOWNLOAD_URL_PREFIX = "http://ftp.csx.cam.ac.uk/pub/software/programming/pcre"
)

// openssl
const (
	OPENSSL_VERSION             = "1.0.2g"
	OPENSSL_DOWNLOAD_URL_PREFIX = "http://www.openssl.org/source"
)

// zlib
const (
	ZLIB_VERSION             = "1.2.8"
	ZLIB_DOWNLOAD_URL_PREFIX = "http://zlib.net"
)

// openResty
const (
	OPENRESTY_VERSION             = "1.9.7.4"
	OPENRESTY_DOWNLOAD_URL_PREFIX = "https://openresty.org/download"
)

// tengine
const (
	TENGINE_VERSION             = "2.1.2"
	TENGINE_DOWNLOAD_URL_PREFIX = "http://tengine.taobao.org/download"
)

// component enumerations
const (
	COMPONENT_NGINX = iota
	COMPONENT_OPENRESTY
	COMPONENT_TENGINE
	COMPONENT_PCRE
	COMPONENT_OPENSSL
	COMPONENT_ZLIB
	COMPONENT_MAX
)
<|endoftext|>


<commit_before>// ratestat project ratestat.go
package ratestat

import (
	"sync/atomic"
	"time"
)

type RateStat struct {
	Buckets   []uint64
	curBucket int32
	interval  time.Duration
	length    int32
}

func (rs *RateStat) Log(count uint64) {
	bucket := atomic.LoadInt32(&rs.curBucket)
	atomic.AddUint64(&rs.Buckets[bucket], count)
}

func (rs *RateStat) Value() uint64 {
	val := uint64(0)
	for k := range rs.Buckets {
		val += atomic.LoadUint64(&rs.Buckets[k])
	}
	return val
}

func (rs *RateStat) manage() {
	for range time.Tick(rs.interval) {
		bucket := atomic.LoadInt32(&rs.curBucket)
		bucket++
		if bucket >= rs.length {
			bucket = 0
		}
		atomic.StoreUint64(&rs.Buckets[bucket], 0)
		atomic.StoreInt32(&rs.curBucket, bucket)
	}
}

func New(buckets int32, interval time.Duration) *RateStat {
	rs := &RateStat{
		Buckets:  make([]uint64, buckets),
		interval: interval,
		length:   buckets,
	}
	go rs.manage()
	return rs
}
<commit_msg>Optimization thanks to Freman<commit_after>// ratestat project ratestat.go
package ratestat

import (
	"sync/atomic"
	"time"
)

type RateStat struct {
	Buckets   []uint64
	curValue  uint64
	curBucket int32
	interval  time.Duration
	length    int32
}

func (rs *RateStat) Log(count uint64) {
	bucket := atomic.LoadInt32(&rs.curBucket)
	atomic.AddUint64(&rs.Buckets[bucket], count)
	atomic.AddUint64(&rs.curValue, count)
}

func (rs *RateStat) Value() uint64 {
	return atomic.LoadUint64(&rs.curValue)
}

func (rs *RateStat) manage() {
	for range time.Tick(rs.interval) {
		bucket := atomic.LoadInt32(&rs.curBucket)
		bucket++
		if bucket >= rs.length {
			bucket = 0
		}
		atomic.StoreInt32(&rs.curBucket, bucket)
		old := atomic.SwapUint64(&rs.Buckets[bucket], 0)
		atomic.AddUint64(&rs.curValue, -old)
	}
}

func New(buckets int32, interval time.Duration) *RateStat {
	rs := &RateStat{
		Buckets:  make([]uint64, buckets),
		interval: interval,
		length:   buckets,
	}
	go rs.manage()
	return rs
}
<|endoftext|>


<commit_before>package gxkafka

import (
	"log"
	"os"
	"strconv"
	"testing"
	"time"
)

import (
	"github.com/Shopify/sarama"
)

func init() {
	sarama.Logger = log.New(os.Stdout, "[Sarama] ", log.LstdFlags)
}

type Message struct {
	Content string "json:`content`"
}

func TestKafkaProducer(t *testing.T) {
	var (
		id        = "producer-client-id"
		zk        = "127.0.0.1:2181/kafka"
		topic     = "test1"
		err       error
		partition int32
		offset    int64
		producer  Producer
		message   Message
	)

	if producer, err = NewProducerWithZk(id, zk, HASH, true); err != nil {
		t.Errorf("NewProducerWithZk(id:%s, zk:%s) = err{%v}", id, zk, err)
	}
	defer producer.Stop()

	for i := 0; i < 10; i++ {
		message.Content = "hello:" + strconv.Itoa(i)
		partition, offset, err = producer.SendMessage(topic, message.Content, message)
		if err != nil {
			t.Fatalf("FAILED to produce message:%v, err:%v", message, err)
		}
		t.Logf("send telemetry message:%#v, kafka partition:%d, kafka offset:%d",
			message, partition, offset)
		time.Sleep(1e9)
	}
}
<commit_msg>modify message in unit test case<commit_after>package gxkafka

import (
	"log"
	"os"
	"strconv"
	"testing"
	"time"
)

import (
	"github.com/Shopify/sarama"
)

func init() {
	sarama.Logger = log.New(os.Stdout, "[Sarama] ", log.LstdFlags)
}

type Message struct {
	EventName string `json:"event_name,omitempty"`
	Timestamp int64  `json:"timestamp,omitempty"`
	IP        string `json:"ip,omitempty"`
	PlanID    int64  `json:"plan_id,omitempty"`
	UUID      string `json:"uuid,omitempty"`
	OS        int    `json:"os,omitempty"`
	AppID     int    `json:"app_id,omitempty"`
	Token     string `json:"token,omitempty"`
	Status    string `json:"status,omitempty"`
}

func TestKafkaProducer(t *testing.T) {
	var (
		id        = "producer-client-id"
		zk        = "127.0.0.1:2181/kafka"
		topic     = "test1"
		err       error
		partition int32
		offset    int64
		producer  Producer
		message   Message
	)

	if producer, err = NewProducerWithZk(id, zk, HASH, true); err != nil {
		t.Errorf("NewProducerWithZk(id:%s, zk:%s) = err{%v}", id, zk, err)
	}
	defer producer.Stop()

	message.EventName = "event"
	message.Timestamp = time.Now().Unix()
	message.IP = "127.0.0.1"
	message.PlanID = 123
	message.UUID = "a-b-c-d-e"
	message.OS = 2
	message.AppID = 1
	message.Token = "1234"
	message.Status = "good"

	for i := 0; i < 10; i++ {
		message.UUID = "hello:" + strconv.Itoa(i)
		partition, offset, err = producer.SendMessage(topic, message.UUID, message)
		if err != nil {
			t.Fatalf("FAILED to produce message:%v, err:%v", message, err)
		}
		t.Logf("send telemetry message:%#v, kafka partition:%d, kafka offset:%d",
			message, partition, offset)
		time.Sleep(1e9)
	}
}
<|endoftext|>


<commit_before>package remote

import (
	"bytes"
	"fmt"
	"os"
	"strings"
	"testing"
	. "github.com/smartystreets/goconvey/convey"
)

func TestRemote(t *testing.T) {

	Convey("Login", t, func() {
		pem_file := "id_dsa"
		if _, err := os.Stat(pem_file); err == nil {
			ssh, err := New("50.16.204.123:22", "todd", "id_dsa")
			So(err, ShouldEqual, nil)
			err = ssh.Launch("sleep 10 && date", "background")
			So(err, ShouldEqual, nil)
			content, _ := ssh.Run("date", false)
			ssh.CopyTo(strings.NewReader(content), "uploadfile")
			var out bytes.Buffer
			ssh.CopyFrom("uploadfile", &out)

			ok := bytes.Equal([]byte(content), out.Bytes())
			So(ok, ShouldEqual, true)
		} else {
			fmt.Println("No credentials so SSH Test ignored")
			So(true, ShouldEqual, true)
		}
	})
	Convey("BigFile", t, func() {
		ssh, _ := New("50.16.204.123:22", "todd", "id_dsa")

		ssh.SimpleFileCopyTo("outbin", "tb")
		ssh.SimpleFileCopyFrom("tb", "outbin2")

	})
}
<commit_msg>make sure remote_test.go does not run if pem file is not available<commit_after>package remote

import (
	"bytes"
	"fmt"
	"os"
	"strings"
	"testing"
	. "github.com/smartystreets/goconvey/convey"
)

func TestRemote(t *testing.T) {

	Convey("Login", t, func() {
		pem_file := "id_dsa"
		if _, err := os.Stat(pem_file); err == nil {
			ssh, err := New("50.16.204.123:22", "todd", pem_file)
			So(err, ShouldEqual, nil)
			err = ssh.Launch("sleep 10 && date", "background")
			So(err, ShouldEqual, nil)
			content, _ := ssh.Run("date", false)
			ssh.CopyTo(strings.NewReader(content), "uploadfile")
			var out bytes.Buffer
			ssh.CopyFrom("uploadfile", &out)

			ok := bytes.Equal([]byte(content), out.Bytes())
			So(ok, ShouldEqual, true)
		} else {
			fmt.Println("No credentials to SSH. Test ignored.")
			So(true, ShouldEqual, true)
		}
	})
	Convey("BigFile", t, func() {
		pem_file := "id_dsa"
		if _, err := os.Stat(pem_file); err == nil {
			ssh, _ := New("50.16.204.123:22", "todd", pem_file)

			ssh.SimpleFileCopyTo("outbin", "tb")
			ssh.SimpleFileCopyFrom("tb", "outbin2")
		} else {
			fmt.Println("No credentials to SSH. Test ignored.")
			So(true, ShouldEqual, true)
		}

	})
}
<|endoftext|>


<commit_before>package db

import (
	"bytes"
	"encoding/hex"
	"strconv"
	"sync"
	"testing"

	"github.com/OpenBazaar/openbazaar-go/repo"
	"github.com/OpenBazaar/openbazaar-go/schema"
	"github.com/OpenBazaar/wallet-interface"
	"github.com/btcsuite/btcd/chaincfg/chainhash"
	"github.com/btcsuite/btcd/wire"
)

func mustNewUtxo() wallet.Utxo {
	sh1, _ := chainhash.NewHashFromStr("e941e1c32b3dd1a68edc3af9f7fe711f35aaca60f758c2dd49561e45ca2c41c0")
	outpoint := wire.NewOutPoint(sh1, 0)
	return wallet.Utxo{
		Op:           *outpoint,
		AtHeight:     300000,
		Value:        100000000,
		ScriptPubkey: []byte("scriptpubkey"),
		WatchOnly:    false,
	}
}

func buildNewUnspentTransactionOutputStore() (repo.UnspentTransactionOutputStore, func(), error) {
	appSchema := schema.MustNewCustomSchemaManager(schema.SchemaContext{
		DataPath:        schema.GenerateTempPath(),
		TestModeEnabled: true,
	})
	if err := appSchema.BuildSchemaDirectories(); err != nil {
		return nil, nil, err
	}
	if err := appSchema.InitializeDatabase(); err != nil {
		return nil, nil, err
	}
	database, err := appSchema.OpenDatabase()
	if err != nil {
		return nil, nil, err
	}
	return NewUnspentTransactionStore(database, new(sync.Mutex), wallet.Bitcoin), appSchema.DestroySchemaDirectories, nil
}

func TestUtxoPut(t *testing.T) {
	var uxdb, teardown, err = buildNewUnspentTransactionOutputStore()
	if err != nil {
		t.Fatal(err)
	}
	defer teardown()

	utxo := mustNewUtxo()
	err = uxdb.Put(utxo)
	if err != nil {
		t.Error(err)
	}
	stmt, _ := uxdb.PrepareQuery("select outpoint, value, height, scriptPubKey from utxos where outpoint=?")
	defer stmt.Close()

	var outpoint string
	var value int
	var height int
	var scriptPubkey string
	o := utxo.Op.Hash.String() + ":" + strconv.Itoa(int(utxo.Op.Index))
	err = stmt.QueryRow(o).Scan(&outpoint, &value, &height, &scriptPubkey)
	if err != nil {
		t.Error(err)
	}
	if outpoint != o {
		t.Error("Utxo db returned wrong outpoint")
	}
	if value != int(utxo.Value) {
		t.Error("Utxo db returned wrong value")
	}
	if height != int(utxo.AtHeight) {
		t.Error("Utxo db returned wrong height")
	}
	if scriptPubkey != hex.EncodeToString(utxo.ScriptPubkey) {
		t.Error("Utxo db returned wrong scriptPubKey")
	}
}

func TestUtxoGetAll(t *testing.T) {
	var uxdb, teardown, err = buildNewUnspentTransactionOutputStore()
	if err != nil {
		t.Fatal(err)
	}
	defer teardown()

	utxo := mustNewUtxo()
	err = uxdb.Put(utxo)
	if err != nil {
		t.Error(err)
	}
	utxos, err := uxdb.GetAll()
	if err != nil {
		t.Error(err)
	}
	if utxos[0].Op.Hash.String() != utxo.Op.Hash.String() {
		t.Error("Utxo db returned wrong outpoint hash")
	}
	if utxos[0].Op.Index != utxo.Op.Index {
		t.Error("Utxo db returned wrong outpoint index")
	}
	if utxos[0].Value != utxo.Value {
		t.Error("Utxo db returned wrong value")
	}
	if utxos[0].AtHeight != utxo.AtHeight {
		t.Error("Utxo db returned wrong height")
	}
	if !bytes.Equal(utxos[0].ScriptPubkey, utxo.ScriptPubkey) {
		t.Error("Utxo db returned wrong scriptPubKey")
	}
}

func TestSetWatchOnlyUtxo(t *testing.T) {
	var uxdb, teardown, err = buildNewUnspentTransactionOutputStore()
	if err != nil {
		t.Fatal(err)
	}
	defer teardown()

	utxo := mustNewUtxo()
	err = uxdb.Put(utxo)
	if err != nil {
		t.Error(err)
	}
	err = uxdb.SetWatchOnly(utxo)
	if err != nil {
		t.Error(err)
	}
	stmt, _ := uxdb.PrepareQuery("select watchOnly from utxos where outpoint=?")
	defer stmt.Close()

	var watchOnlyInt int
	o := utxo.Op.Hash.String() + ":" + strconv.Itoa(int(utxo.Op.Index))
	err = stmt.QueryRow(o).Scan(&watchOnlyInt)
	if err != nil {
		t.Error(err)
	}
	if watchOnlyInt != 1 {
		t.Error("Utxo freeze failed")
	}

}

func TestDeleteUtxo(t *testing.T) {
	var uxdb, teardown, err = buildNewUnspentTransactionOutputStore()
	if err != nil {
		t.Fatal(err)
	}
	defer teardown()

	utxo := mustNewUtxo()
	err = uxdb.Put(utxo)
	if err != nil {
		t.Error(err)
	}
	err = uxdb.Delete(utxo)
	if err != nil {
		t.Error(err)
	}
	utxos, err := uxdb.GetAll()
	if err != nil {
		t.Error(err)
	}
	if len(utxos) != 0 {
		t.Error("Utxo db delete failed")
	}
}
<commit_msg>refactor setup of uxdb with Utxo into a function<commit_after>package db

import (
	"bytes"
	"encoding/hex"
	"strconv"
	"sync"
	"testing"

	"github.com/OpenBazaar/openbazaar-go/repo"
	"github.com/OpenBazaar/openbazaar-go/schema"
	"github.com/OpenBazaar/wallet-interface"
	"github.com/btcsuite/btcd/chaincfg/chainhash"
	"github.com/btcsuite/btcd/wire"
)

func mustNewUtxo() wallet.Utxo {
	sh1, _ := chainhash.NewHashFromStr("e941e1c32b3dd1a68edc3af9f7fe711f35aaca60f758c2dd49561e45ca2c41c0")
	outpoint := wire.NewOutPoint(sh1, 0)
	return wallet.Utxo{
		Op:           *outpoint,
		AtHeight:     300000,
		Value:        100000000,
		ScriptPubkey: []byte("scriptpubkey"),
		WatchOnly:    false,
	}
}

func buildNewUnspentTransactionOutputStore() (repo.UnspentTransactionOutputStore, func(), error) {
	appSchema := schema.MustNewCustomSchemaManager(schema.SchemaContext{
		DataPath:        schema.GenerateTempPath(),
		TestModeEnabled: true,
	})
	if err := appSchema.BuildSchemaDirectories(); err != nil {
		return nil, nil, err
	}
	if err := appSchema.InitializeDatabase(); err != nil {
		return nil, nil, err
	}
	database, err := appSchema.OpenDatabase()
	if err != nil {
		return nil, nil, err
	}
	return NewUnspentTransactionStore(database, new(sync.Mutex), wallet.Bitcoin), appSchema.DestroySchemaDirectories, nil
}

func mustNewUxdbWithUtxo() (repo.UnspentTransactionOutputStore, wallet.Utxo, error) {
	var uxdb, teardown, err = buildNewUnspentTransactionOutputStore()
	utxo := mustNewUtxo()
	if err != nil {
		return nil, utxo, err
	}
	defer teardown()
	return uxdb, utxo, uxdb.Put(utxo)
}

func TestUtxoPut(t *testing.T) {
	var uxdb, utxo, err = mustNewUxdbWithUtxo()
	if err != nil {
		t.Error(err)
	}
	stmt, _ := uxdb.PrepareQuery("select outpoint, value, height, scriptPubKey from utxos where outpoint=?")
	defer stmt.Close()

	var outpoint string
	var value int
	var height int
	var scriptPubkey string
	o := utxo.Op.Hash.String() + ":" + strconv.Itoa(int(utxo.Op.Index))
	err = stmt.QueryRow(o).Scan(&outpoint, &value, &height, &scriptPubkey)
	if err != nil {
		t.Error(err)
	}
	if outpoint != o {
		t.Error("Utxo db returned wrong outpoint")
	}
	if value != int(utxo.Value) {
		t.Error("Utxo db returned wrong value")
	}
	if height != int(utxo.AtHeight) {
		t.Error("Utxo db returned wrong height")
	}
	if scriptPubkey != hex.EncodeToString(utxo.ScriptPubkey) {
		t.Error("Utxo db returned wrong scriptPubKey")
	}
}

func TestUtxoGetAll(t *testing.T) {
	var uxdb, utxo, err = mustNewUxdbWithUtxo()
	if err != nil {
		t.Error(err)
	}
	utxos, err := uxdb.GetAll()
	if err != nil {
		t.Error(err)
	}
	if utxos[0].Op.Hash.String() != utxo.Op.Hash.String() {
		t.Error("Utxo db returned wrong outpoint hash")
	}
	if utxos[0].Op.Index != utxo.Op.Index {
		t.Error("Utxo db returned wrong outpoint index")
	}
	if utxos[0].Value != utxo.Value {
		t.Error("Utxo db returned wrong value")
	}
	if utxos[0].AtHeight != utxo.AtHeight {
		t.Error("Utxo db returned wrong height")
	}
	if !bytes.Equal(utxos[0].ScriptPubkey, utxo.ScriptPubkey) {
		t.Error("Utxo db returned wrong scriptPubKey")
	}
}

func TestSetWatchOnlyUtxo(t *testing.T) {
	var uxdb, utxo, err = mustNewUxdbWithUtxo()
	if err != nil {
		t.Error(err)
	}
	err = uxdb.SetWatchOnly(utxo)
	if err != nil {
		t.Error(err)
	}
	stmt, _ := uxdb.PrepareQuery("select watchOnly from utxos where outpoint=?")
	defer stmt.Close()

	var watchOnlyInt int
	o := utxo.Op.Hash.String() + ":" + strconv.Itoa(int(utxo.Op.Index))
	err = stmt.QueryRow(o).Scan(&watchOnlyInt)
	if err != nil {
		t.Error(err)
	}
	if watchOnlyInt != 1 {
		t.Error("Utxo freeze failed")
	}

}

func TestDeleteUtxo(t *testing.T) {
	var uxdb, utxo, err = mustNewUxdbWithUtxo()
	if err != nil {
		t.Error(err)
	}
	err = uxdb.Delete(utxo)
	if err != nil {
		t.Error(err)
	}
	utxos, err := uxdb.GetAll()
	if err != nil {
		t.Error(err)
	}
	if len(utxos) != 0 {
		t.Error("Utxo db delete failed")
	}
}
<|endoftext|>


<commit_before>package fsrepo

import (
	"fmt"
	"io"
	"os"
	"path"
	"strconv"
	"sync"

	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
	levelds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/leveldb"
	ldbopts "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
	repo "github.com/jbenet/go-ipfs/repo"
	"github.com/jbenet/go-ipfs/repo/common"
	config "github.com/jbenet/go-ipfs/repo/config"
	lockfile "github.com/jbenet/go-ipfs/repo/fsrepo/lock"
	serialize "github.com/jbenet/go-ipfs/repo/fsrepo/serialize"
	dir "github.com/jbenet/go-ipfs/thirdparty/dir"
	"github.com/jbenet/go-ipfs/thirdparty/eventlog"
	u "github.com/jbenet/go-ipfs/util"
	util "github.com/jbenet/go-ipfs/util"
	ds2 "github.com/jbenet/go-ipfs/util/datastore2"
	debugerror "github.com/jbenet/go-ipfs/util/debugerror"
)

const (
	defaultDataStoreDirectory = "datastore"
)

var (

	// packageLock must be held to while performing any operation that modifies an
	// FSRepo's state field. This includes Init, Open, Close, and Remove.
	packageLock sync.Mutex

	// onlyOne keeps track of open FSRepo instances.
	//
	// TODO: once command Context / Repo integration is cleaned up,
	// this can be removed. Right now, this makes ConfigCmd.Run
	// function try to open the repo twice:
	//
	//     $ ipfs daemon &
	//     $ ipfs config foo
	//
	// The reason for the above is that in standalone mode without the
	// daemon, `ipfs config` tries to save work by not building the
	// full IpfsNode, but accessing the Repo directly.
	onlyOne repo.OnlyOne
)

// FSRepo represents an IPFS FileSystem Repo. It is safe for use by multiple
// callers.
type FSRepo struct {
	// state is the FSRepo's state (unopened, opened, closed)
	state state
	// path is the file-system path
	path string
	// lockfile is the file system lock to prevent others from opening
	// the same fsrepo path concurrently
	lockfile io.Closer
	// config is set on Open, guarded by packageLock
	config *config.Config
	// ds is set on Open
	ds ds2.ThreadSafeDatastoreCloser
}

var _ repo.Repo = (*FSRepo)(nil)

// Open the FSRepo at path. Returns an error if the repo is not
// initialized.
func Open(repoPath string) (repo.Repo, error) {
	fn := func() (repo.Repo, error) {
		return open(repoPath)
	}
	return onlyOne.Open(repoPath, fn)
}

func open(repoPath string) (repo.Repo, error) {
	packageLock.Lock()
	defer packageLock.Unlock()

	r := &FSRepo{
		path:  path.Clean(repoPath),
		state: unopened, // explicitly set for clarity
	}

	expPath, err := u.TildeExpansion(r.path)
	if err != nil {
		return nil, err
	}
	r.path = expPath

	if r.state != unopened {
		return nil, debugerror.Errorf("repo is %s", r.state)
	}
	if !isInitializedUnsynced(r.path) {
		return nil, debugerror.New("ipfs not initialized, please run 'ipfs init'")
	}
	// check repo path, then check all constituent parts.
	// TODO acquire repo lock
	// TODO if err := initCheckDir(logpath); err != nil { // }
	if err := dir.Writable(r.path); err != nil {
		return nil, err
	}

	if err := r.openConfig(); err != nil {
		return nil, err
	}

	if err := r.openDatastore(); err != nil {
		return nil, err
	}

	// log.Debugf("writing eventlogs to ...", c.path)
	configureEventLoggerAtRepoPath(r.config, r.path)

	if err := r.transitionToOpened(); err != nil {
		return nil, err
	}
	return r, nil
}

// ConfigAt returns an error if the FSRepo at the given path is not
// initialized. This function allows callers to read the config file even when
// another process is running and holding the lock.
func ConfigAt(repoPath string) (*config.Config, error) {

	// packageLock must be held to ensure that the Read is atomic.
	packageLock.Lock()
	defer packageLock.Unlock()

	configFilename, err := config.Filename(repoPath)
	if err != nil {
		return nil, err
	}
	return serialize.Load(configFilename)
}

// configIsInitialized returns true if the repo is initialized at
// provided |path|.
func configIsInitialized(path string) bool {
	configFilename, err := config.Filename(path)
	if err != nil {
		return false
	}
	if !util.FileExists(configFilename) {
		return false
	}
	return true
}

func initConfig(path string, conf *config.Config) error {
	if configIsInitialized(path) {
		return nil
	}
	configFilename, err := config.Filename(path)
	if err != nil {
		return err
	}
	// initialization is the one time when it's okay to write to the config
	// without reading the config from disk and merging any user-provided keys
	// that may exist.
	if err := serialize.WriteConfigFile(configFilename, conf); err != nil {
		return err
	}
	return nil
}

// Init initializes a new FSRepo at the given path with the provided config.
// TODO add support for custom datastores.
func Init(repoPath string, conf *config.Config) error {

	// packageLock must be held to ensure that the repo is not initialized more
	// than once.
	packageLock.Lock()
	defer packageLock.Unlock()

	if isInitializedUnsynced(repoPath) {
		return nil
	}

	if err := initConfig(repoPath, conf); err != nil {
		return err
	}

	// The actual datastore contents are initialized lazily when Opened.
	// During Init, we merely check that the directory is writeable.
	p := path.Join(repoPath, defaultDataStoreDirectory)
	if err := dir.Writable(p); err != nil {
		return debugerror.Errorf("datastore: %s", err)
	}

	if err := dir.Writable(path.Join(repoPath, "logs")); err != nil {
		return err
	}

	return nil
}

// Remove recursively removes the FSRepo at |path|.
func Remove(repoPath string) error {
	repoPath = path.Clean(repoPath)
	return os.RemoveAll(repoPath)
}

// LockedByOtherProcess returns true if the FSRepo is locked by another
// process. If true, then the repo cannot be opened by this process.
func LockedByOtherProcess(repoPath string) bool {
	repoPath = path.Clean(repoPath)

	// TODO replace this with the "api" file
	// https://github.com/ipfs/specs/tree/master/repo/fs-repo

	// NB: the lock is only held when repos are Open
	return lockfile.Locked(repoPath)
}

// openConfig returns an error if the config file is not present.
func (r *FSRepo) openConfig() error {
	configFilename, err := config.Filename(r.path)
	if err != nil {
		return err
	}
	conf, err := serialize.Load(configFilename)
	if err != nil {
		return err
	}
	r.config = conf
	return nil
}

// openDatastore returns an error if the config file is not present.
func (r *FSRepo) openDatastore() error {
	dsPath := path.Join(r.path, defaultDataStoreDirectory)
	ds, err := levelds.NewDatastore(dsPath, &levelds.Options{
		Compression: ldbopts.NoCompression,
	})
	if err != nil {
		return debugerror.New("unable to open leveldb datastore")
	}
	r.ds = ds
	return nil
}

func configureEventLoggerAtRepoPath(c *config.Config, repoPath string) {
	eventlog.Configure(eventlog.LevelInfo)
	eventlog.Configure(eventlog.LdJSONFormatter)
	rotateConf := eventlog.LogRotatorConfig{
		Filename:   path.Join(repoPath, "logs", "events.log"),
		MaxSizeMB:  c.Log.MaxSizeMB,
		MaxBackups: c.Log.MaxBackups,
		MaxAgeDays: c.Log.MaxAgeDays,
	}
	eventlog.Configure(eventlog.OutputRotatingLogFile(rotateConf))
}

// Close closes the FSRepo, releasing held resources.
func (r *FSRepo) Close() error {
	packageLock.Lock()
	defer packageLock.Unlock()

	if r.state != opened {
		return debugerror.Errorf("repo is %s", r.state)
	}

	if err := r.ds.Close(); err != nil {
		return err
	}

	// This code existed in the previous versions, but
	// EventlogComponent.Close was never called. Preserving here
	// pending further discussion.
	//
	// TODO It isn't part of the current contract, but callers may like for us
	// to disable logging once the component is closed.
	// eventlog.Configure(eventlog.Output(os.Stderr))

	return r.transitionToClosed()
}

// Config returns the FSRepo's config. This method must not be called if the
// repo is not open.
//
// Result when not Open is undefined. The method may panic if it pleases.
func (r *FSRepo) Config() *config.Config {

	// It is not necessary to hold the package lock since the repo is in an
	// opened state. The package lock is _not_ meant to ensure that the repo is
	// thread-safe. The package lock is only meant to guard againt removal and
	// coordinate the lockfile. However, we provide thread-safety to keep
	// things simple.
	packageLock.Lock()
	defer packageLock.Unlock()

	if r.state != opened {
		panic(fmt.Sprintln("repo is", r.state))
	}
	return r.config
}

// setConfigUnsynced is for private use.
func (r *FSRepo) setConfigUnsynced(updated *config.Config) error {
	configFilename, err := config.Filename(r.path)
	if err != nil {
		return err
	}
	// to avoid clobbering user-provided keys, must read the config from disk
	// as a map, write the updated struct values to the map and write the map
	// to disk.
	var mapconf map[string]interface{}
	if err := serialize.ReadConfigFile(configFilename, &mapconf); err != nil {
		return err
	}
	m, err := config.ToMap(updated)
	if err != nil {
		return err
	}
	for k, v := range m {
		mapconf[k] = v
	}
	if err := serialize.WriteConfigFile(configFilename, mapconf); err != nil {
		return err
	}
	*r.config = *updated // copy so caller cannot modify this private config
	return nil
}

// SetConfig updates the FSRepo's config.
func (r *FSRepo) SetConfig(updated *config.Config) error {

	// packageLock is held to provide thread-safety.
	packageLock.Lock()
	defer packageLock.Unlock()

	return r.setConfigUnsynced(updated)
}

// GetConfigKey retrieves only the value of a particular key.
func (r *FSRepo) GetConfigKey(key string) (interface{}, error) {
	packageLock.Lock()
	defer packageLock.Unlock()

	if r.state != opened {
		return nil, debugerror.Errorf("repo is %s", r.state)
	}

	filename, err := config.Filename(r.path)
	if err != nil {
		return nil, err
	}
	var cfg map[string]interface{}
	if err := serialize.ReadConfigFile(filename, &cfg); err != nil {
		return nil, err
	}
	return common.MapGetKV(cfg, key)
}

// SetConfigKey writes the value of a particular key.
func (r *FSRepo) SetConfigKey(key string, value interface{}) error {
	packageLock.Lock()
	defer packageLock.Unlock()

	if r.state != opened {
		return debugerror.Errorf("repo is %s", r.state)
	}

	filename, err := config.Filename(r.path)
	if err != nil {
		return err
	}
	switch v := value.(type) {
	case string:
		if i, err := strconv.Atoi(v); err == nil {
			value = i
		}
	}
	var mapconf map[string]interface{}
	if err := serialize.ReadConfigFile(filename, &mapconf); err != nil {
		return err
	}
	if err := common.MapSetKV(mapconf, key, value); err != nil {
		return err
	}
	conf, err := config.FromMap(mapconf)
	if err != nil {
		return err
	}
	if err := serialize.WriteConfigFile(filename, mapconf); err != nil {
		return err
	}
	return r.setConfigUnsynced(conf) // TODO roll this into this method
}

// Datastore returns a repo-owned datastore. If FSRepo is Closed, return value
// is undefined.
func (r *FSRepo) Datastore() ds.ThreadSafeDatastore {
	packageLock.Lock()
	d := r.ds
	packageLock.Unlock()
	return d
}

var _ io.Closer = &FSRepo{}
var _ repo.Repo = &FSRepo{}

// IsInitialized returns true if the repo is initialized at provided |path|.
func IsInitialized(path string) bool {
	// packageLock is held to ensure that another caller doesn't attempt to
	// Init or Remove the repo while this call is in progress.
	packageLock.Lock()
	defer packageLock.Unlock()

	return isInitializedUnsynced(path)
}

// private methods below this point. NB: packageLock must held by caller.

// isInitializedUnsynced reports whether the repo is initialized. Caller must
// hold the packageLock.
func isInitializedUnsynced(repoPath string) bool {
	if !configIsInitialized(repoPath) {
		return false
	}
	if !util.FileExists(path.Join(repoPath, defaultDataStoreDirectory)) {
		return false
	}
	return true
}

// transitionToOpened manages the state transition to |opened|. Caller must hold
// the package mutex.
func (r *FSRepo) transitionToOpened() error {
	r.state = opened
	closer, err := lockfile.Lock(r.path)
	if err != nil {
		return err
	}
	r.lockfile = closer
	return nil
}

// transitionToClosed manages the state transition to |closed|. Caller must
// hold the package mutex.
func (r *FSRepo) transitionToClosed() error {
	r.state = closed
	if err := r.lockfile.Close(); err != nil {
		return err
	}
	return nil
}
<commit_msg>Clean up fsrepo path handling<commit_after>package fsrepo

import (
	"fmt"
	"io"
	"os"
	"path"
	"strconv"
	"sync"

	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
	levelds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/leveldb"
	ldbopts "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
	repo "github.com/jbenet/go-ipfs/repo"
	"github.com/jbenet/go-ipfs/repo/common"
	config "github.com/jbenet/go-ipfs/repo/config"
	lockfile "github.com/jbenet/go-ipfs/repo/fsrepo/lock"
	serialize "github.com/jbenet/go-ipfs/repo/fsrepo/serialize"
	dir "github.com/jbenet/go-ipfs/thirdparty/dir"
	"github.com/jbenet/go-ipfs/thirdparty/eventlog"
	u "github.com/jbenet/go-ipfs/util"
	util "github.com/jbenet/go-ipfs/util"
	ds2 "github.com/jbenet/go-ipfs/util/datastore2"
	debugerror "github.com/jbenet/go-ipfs/util/debugerror"
)

const (
	defaultDataStoreDirectory = "datastore"
)

var (

	// packageLock must be held to while performing any operation that modifies an
	// FSRepo's state field. This includes Init, Open, Close, and Remove.
	packageLock sync.Mutex

	// onlyOne keeps track of open FSRepo instances.
	//
	// TODO: once command Context / Repo integration is cleaned up,
	// this can be removed. Right now, this makes ConfigCmd.Run
	// function try to open the repo twice:
	//
	//     $ ipfs daemon &
	//     $ ipfs config foo
	//
	// The reason for the above is that in standalone mode without the
	// daemon, `ipfs config` tries to save work by not building the
	// full IpfsNode, but accessing the Repo directly.
	onlyOne repo.OnlyOne
)

// FSRepo represents an IPFS FileSystem Repo. It is safe for use by multiple
// callers.
type FSRepo struct {
	// state is the FSRepo's state (unopened, opened, closed)
	state state
	// path is the file-system path
	path string
	// lockfile is the file system lock to prevent others from opening
	// the same fsrepo path concurrently
	lockfile io.Closer
	// config is set on Open, guarded by packageLock
	config *config.Config
	// ds is set on Open
	ds ds2.ThreadSafeDatastoreCloser
}

var _ repo.Repo = (*FSRepo)(nil)

// Open the FSRepo at path. Returns an error if the repo is not
// initialized.
func Open(repoPath string) (repo.Repo, error) {
	fn := func() (repo.Repo, error) {
		return open(repoPath)
	}
	return onlyOne.Open(repoPath, fn)
}

func open(repoPath string) (repo.Repo, error) {
	packageLock.Lock()
	defer packageLock.Unlock()

	expPath, err := u.TildeExpansion(path.Clean(repoPath))
	if err != nil {
		return nil, err
	}

	r := &FSRepo{
		path: expPath,
	}

	if !isInitializedUnsynced(r.path) {
		return nil, debugerror.New("ipfs not initialized, please run 'ipfs init'")
	}
	// check repo path, then check all constituent parts.
	// TODO acquire repo lock
	// TODO if err := initCheckDir(logpath); err != nil { // }
	if err := dir.Writable(r.path); err != nil {
		return nil, err
	}

	if err := r.openConfig(); err != nil {
		return nil, err
	}

	if err := r.openDatastore(); err != nil {
		return nil, err
	}

	// log.Debugf("writing eventlogs to ...", c.path)
	configureEventLoggerAtRepoPath(r.config, r.path)

	if err := r.transitionToOpened(); err != nil {
		return nil, err
	}
	return r, nil
}

// ConfigAt returns an error if the FSRepo at the given path is not
// initialized. This function allows callers to read the config file even when
// another process is running and holding the lock.
func ConfigAt(repoPath string) (*config.Config, error) {

	// packageLock must be held to ensure that the Read is atomic.
	packageLock.Lock()
	defer packageLock.Unlock()

	configFilename, err := config.Filename(repoPath)
	if err != nil {
		return nil, err
	}
	return serialize.Load(configFilename)
}

// configIsInitialized returns true if the repo is initialized at
// provided |path|.
func configIsInitialized(path string) bool {
	configFilename, err := config.Filename(path)
	if err != nil {
		return false
	}
	if !util.FileExists(configFilename) {
		return false
	}
	return true
}

func initConfig(path string, conf *config.Config) error {
	if configIsInitialized(path) {
		return nil
	}
	configFilename, err := config.Filename(path)
	if err != nil {
		return err
	}
	// initialization is the one time when it's okay to write to the config
	// without reading the config from disk and merging any user-provided keys
	// that may exist.
	if err := serialize.WriteConfigFile(configFilename, conf); err != nil {
		return err
	}
	return nil
}

// Init initializes a new FSRepo at the given path with the provided config.
// TODO add support for custom datastores.
func Init(repoPath string, conf *config.Config) error {

	// packageLock must be held to ensure that the repo is not initialized more
	// than once.
	packageLock.Lock()
	defer packageLock.Unlock()

	if isInitializedUnsynced(repoPath) {
		return nil
	}

	if err := initConfig(repoPath, conf); err != nil {
		return err
	}

	// The actual datastore contents are initialized lazily when Opened.
	// During Init, we merely check that the directory is writeable.
	p := path.Join(repoPath, defaultDataStoreDirectory)
	if err := dir.Writable(p); err != nil {
		return debugerror.Errorf("datastore: %s", err)
	}

	if err := dir.Writable(path.Join(repoPath, "logs")); err != nil {
		return err
	}

	return nil
}

// Remove recursively removes the FSRepo at |path|.
func Remove(repoPath string) error {
	repoPath = path.Clean(repoPath)
	return os.RemoveAll(repoPath)
}

// LockedByOtherProcess returns true if the FSRepo is locked by another
// process. If true, then the repo cannot be opened by this process.
func LockedByOtherProcess(repoPath string) bool {
	repoPath = path.Clean(repoPath)

	// TODO replace this with the "api" file
	// https://github.com/ipfs/specs/tree/master/repo/fs-repo

	// NB: the lock is only held when repos are Open
	return lockfile.Locked(repoPath)
}

// openConfig returns an error if the config file is not present.
func (r *FSRepo) openConfig() error {
	configFilename, err := config.Filename(r.path)
	if err != nil {
		return err
	}
	conf, err := serialize.Load(configFilename)
	if err != nil {
		return err
	}
	r.config = conf
	return nil
}

// openDatastore returns an error if the config file is not present.
func (r *FSRepo) openDatastore() error {
	dsPath := path.Join(r.path, defaultDataStoreDirectory)
	ds, err := levelds.NewDatastore(dsPath, &levelds.Options{
		Compression: ldbopts.NoCompression,
	})
	if err != nil {
		return debugerror.New("unable to open leveldb datastore")
	}
	r.ds = ds
	return nil
}

func configureEventLoggerAtRepoPath(c *config.Config, repoPath string) {
	eventlog.Configure(eventlog.LevelInfo)
	eventlog.Configure(eventlog.LdJSONFormatter)
	rotateConf := eventlog.LogRotatorConfig{
		Filename:   path.Join(repoPath, "logs", "events.log"),
		MaxSizeMB:  c.Log.MaxSizeMB,
		MaxBackups: c.Log.MaxBackups,
		MaxAgeDays: c.Log.MaxAgeDays,
	}
	eventlog.Configure(eventlog.OutputRotatingLogFile(rotateConf))
}

// Close closes the FSRepo, releasing held resources.
func (r *FSRepo) Close() error {
	packageLock.Lock()
	defer packageLock.Unlock()

	if r.state != opened {
		return debugerror.Errorf("repo is %s", r.state)
	}

	if err := r.ds.Close(); err != nil {
		return err
	}

	// This code existed in the previous versions, but
	// EventlogComponent.Close was never called. Preserving here
	// pending further discussion.
	//
	// TODO It isn't part of the current contract, but callers may like for us
	// to disable logging once the component is closed.
	// eventlog.Configure(eventlog.Output(os.Stderr))

	return r.transitionToClosed()
}

// Config returns the FSRepo's config. This method must not be called if the
// repo is not open.
//
// Result when not Open is undefined. The method may panic if it pleases.
func (r *FSRepo) Config() *config.Config {

	// It is not necessary to hold the package lock since the repo is in an
	// opened state. The package lock is _not_ meant to ensure that the repo is
	// thread-safe. The package lock is only meant to guard againt removal and
	// coordinate the lockfile. However, we provide thread-safety to keep
	// things simple.
	packageLock.Lock()
	defer packageLock.Unlock()

	if r.state != opened {
		panic(fmt.Sprintln("repo is", r.state))
	}
	return r.config
}

// setConfigUnsynced is for private use.
func (r *FSRepo) setConfigUnsynced(updated *config.Config) error {
	configFilename, err := config.Filename(r.path)
	if err != nil {
		return err
	}
	// to avoid clobbering user-provided keys, must read the config from disk
	// as a map, write the updated struct values to the map and write the map
	// to disk.
	var mapconf map[string]interface{}
	if err := serialize.ReadConfigFile(configFilename, &mapconf); err != nil {
		return err
	}
	m, err := config.ToMap(updated)
	if err != nil {
		return err
	}
	for k, v := range m {
		mapconf[k] = v
	}
	if err := serialize.WriteConfigFile(configFilename, mapconf); err != nil {
		return err
	}
	*r.config = *updated // copy so caller cannot modify this private config
	return nil
}

// SetConfig updates the FSRepo's config.
func (r *FSRepo) SetConfig(updated *config.Config) error {

	// packageLock is held to provide thread-safety.
	packageLock.Lock()
	defer packageLock.Unlock()

	return r.setConfigUnsynced(updated)
}

// GetConfigKey retrieves only the value of a particular key.
func (r *FSRepo) GetConfigKey(key string) (interface{}, error) {
	packageLock.Lock()
	defer packageLock.Unlock()

	if r.state != opened {
		return nil, debugerror.Errorf("repo is %s", r.state)
	}

	filename, err := config.Filename(r.path)
	if err != nil {
		return nil, err
	}
	var cfg map[string]interface{}
	if err := serialize.ReadConfigFile(filename, &cfg); err != nil {
		return nil, err
	}
	return common.MapGetKV(cfg, key)
}

// SetConfigKey writes the value of a particular key.
func (r *FSRepo) SetConfigKey(key string, value interface{}) error {
	packageLock.Lock()
	defer packageLock.Unlock()

	if r.state != opened {
		return debugerror.Errorf("repo is %s", r.state)
	}

	filename, err := config.Filename(r.path)
	if err != nil {
		return err
	}
	switch v := value.(type) {
	case string:
		if i, err := strconv.Atoi(v); err == nil {
			value = i
		}
	}
	var mapconf map[string]interface{}
	if err := serialize.ReadConfigFile(filename, &mapconf); err != nil {
		return err
	}
	if err := common.MapSetKV(mapconf, key, value); err != nil {
		return err
	}
	conf, err := config.FromMap(mapconf)
	if err != nil {
		return err
	}
	if err := serialize.WriteConfigFile(filename, mapconf); err != nil {
		return err
	}
	return r.setConfigUnsynced(conf) // TODO roll this into this method
}

// Datastore returns a repo-owned datastore. If FSRepo is Closed, return value
// is undefined.
func (r *FSRepo) Datastore() ds.ThreadSafeDatastore {
	packageLock.Lock()
	d := r.ds
	packageLock.Unlock()
	return d
}

var _ io.Closer = &FSRepo{}
var _ repo.Repo = &FSRepo{}

// IsInitialized returns true if the repo is initialized at provided |path|.
func IsInitialized(path string) bool {
	// packageLock is held to ensure that another caller doesn't attempt to
	// Init or Remove the repo while this call is in progress.
	packageLock.Lock()
	defer packageLock.Unlock()

	return isInitializedUnsynced(path)
}

// private methods below this point. NB: packageLock must held by caller.

// isInitializedUnsynced reports whether the repo is initialized. Caller must
// hold the packageLock.
func isInitializedUnsynced(repoPath string) bool {
	if !configIsInitialized(repoPath) {
		return false
	}
	if !util.FileExists(path.Join(repoPath, defaultDataStoreDirectory)) {
		return false
	}
	return true
}

// transitionToOpened manages the state transition to |opened|. Caller must hold
// the package mutex.
func (r *FSRepo) transitionToOpened() error {
	r.state = opened
	closer, err := lockfile.Lock(r.path)
	if err != nil {
		return err
	}
	r.lockfile = closer
	return nil
}

// transitionToClosed manages the state transition to |closed|. Caller must
// hold the package mutex.
func (r *FSRepo) transitionToClosed() error {
	r.state = closed
	if err := r.lockfile.Close(); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package repo

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"gx/ipfs/QmXYjuNuxVzXKJCfWasQk1RqkhVLDM9jtUKhqc2WPQmFSB/go-libp2p-peer"
	"io/ioutil"
	"os"
	"path"
	"time"

	"github.com/OpenBazaar/openbazaar-go/ipfs"
	"github.com/ipfs/go-ipfs/commands"
	"github.com/ipfs/go-ipfs/core"
	"github.com/ipfs/go-ipfs/namesys"
	ipfsPath "github.com/ipfs/go-ipfs/path"
)

var getObjectFromIPFSCache = map[string]getObjectFromIPFSCacheEntry{}

type getObjectFromIPFSCacheEntry struct {
	bytes   []byte
	created time.Time
}

// PublishObjectToIPFS writes the given data to IPFS labeled as the given name
func PublishObjectToIPFS(ctx commands.Context, ipfsNode *core.IpfsNode, tempDir string, name string, data interface{}) (string, error) {
	serializedData, err := json.MarshalIndent(data, "", "    ")
	if err != nil {
		return "", err
	}
	h := sha256.Sum256(serializedData)

	tmpPath := path.Join(tempDir, hex.EncodeToString(h[:])+".json")
	err = ioutil.WriteFile(tmpPath, serializedData, os.ModePerm)
	if err != nil {
		return "", err
	}
	hash, err := ipfs.AddFile(ctx, tmpPath)
	if err != nil {
		return "", err
	}
	err = os.Remove(tmpPath)
	if err != nil {
		return "", err
	}

	return hash, ipfs.PublishAltRoot(ctx, ipfsNode, name, ipfsPath.FromString("/ipfs/"+hash), time.Now().Add(namesys.DefaultPublishLifetime))
}

// GetObjectFromIPFS gets the requested name from ipfs or the local cache
func GetObjectFromIPFS(ctx commands.Context, p peer.ID, name string, maxCacheLen time.Duration) ([]byte, error) {
	entry, ok := getObjectFromIPFSCache[getIPFSCacheKey(p, name)]
	if !ok {
		return fetchObjectFromIPFS(ctx, p, name)
	}

	if entry.created.Add(maxCacheLen).Before(time.Now()) {
		return fetchObjectFromIPFS(ctx, p, name)
	}

	return entry.bytes, nil
}

// fetchObjectFromIPFS gets the requested object from ipfs
func fetchObjectFromIPFS(ctx commands.Context, p peer.ID, name string) ([]byte, error) {
	root, err := ipfs.ResolveAltRoot(ctx, p, name, time.Minute)
	if err != nil {
		return nil, err
	}
	bytes, err := ipfs.Cat(ctx, root, time.Minute)
	if err != nil {
		return nil, err
	}
	getObjectFromIPFSCache[getIPFSCacheKey(p, name)] = getObjectFromIPFSCacheEntry{
		bytes:   bytes,
		created: time.Now(),
	}

	return bytes, nil
}

func getIPFSCacheKey(p peer.ID, name string) string {
	return p.Pretty() + "|" + name
}
<commit_msg>REFACTOR: Update ipfs inventory cache to lock and update cache in background after cache hits.<commit_after>package repo

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"gx/ipfs/QmXYjuNuxVzXKJCfWasQk1RqkhVLDM9jtUKhqc2WPQmFSB/go-libp2p-peer"
	"io/ioutil"
	"os"
	"path"
	"sync"
	"time"

	"github.com/OpenBazaar/openbazaar-go/ipfs"
	"github.com/ipfs/go-ipfs/commands"
	"github.com/ipfs/go-ipfs/core"
	"github.com/ipfs/go-ipfs/namesys"
	ipfsPath "github.com/ipfs/go-ipfs/path"
)

var (
	getObjectFromIPFSCache   = map[string]getObjectFromIPFSCacheEntry{}
	getObjectFromIPFSCacheMu = sync.Mutex{}
)

type getObjectFromIPFSCacheEntry struct {
	bytes   []byte
	created time.Time
}

// PublishObjectToIPFS writes the given data to IPFS labeled as the given name
func PublishObjectToIPFS(ctx commands.Context, ipfsNode *core.IpfsNode, tempDir string, name string, data interface{}) (string, error) {
	serializedData, err := json.MarshalIndent(data, "", "    ")
	if err != nil {
		return "", err
	}
	h := sha256.Sum256(serializedData)

	tmpPath := path.Join(tempDir, hex.EncodeToString(h[:])+".json")
	err = ioutil.WriteFile(tmpPath, serializedData, os.ModePerm)
	if err != nil {
		return "", err
	}
	hash, err := ipfs.AddFile(ctx, tmpPath)
	if err != nil {
		return "", err
	}
	err = os.Remove(tmpPath)
	if err != nil {
		return "", err
	}

	return hash, ipfs.PublishAltRoot(ctx, ipfsNode, name, ipfsPath.FromString("/ipfs/"+hash), time.Now().Add(namesys.DefaultPublishLifetime))
}

// GetObjectFromIPFS gets the requested name from ipfs or the local cache
func GetObjectFromIPFS(ctx commands.Context, p peer.ID, name string, maxCacheLen time.Duration) ([]byte, error) {
	getObjectFromIPFSCacheMu.Lock()
	defer getObjectFromIPFSCacheMu.Unlock()

	entry, ok := getObjectFromIPFSCache[getIPFSCacheKey(p, name)]
	if !ok {
		return fetchObjectFromIPFS(ctx, p, name)
	}

	if entry.created.Add(maxCacheLen).Before(time.Now()) {
		objBytes, err := fetchObjectFromIPFS(ctx, p, name)
		if err != nil {
			return nil, err
		}

		getObjectFromIPFSCache[getIPFSCacheKey(p, name)] = getObjectFromIPFSCacheEntry{
			bytes:   objBytes,
			created: time.Now(),
		}

		return objBytes, nil
	}

	// Update cache in background after a successful read
	func() {
		getObjectFromIPFSCacheMu.Lock()
		defer getObjectFromIPFSCacheMu.Unlock()
		objBytes, err := fetchObjectFromIPFS(ctx, p, name)
		if err != nil {
			log.Error("error update inventory cache:", err)
		}

		getObjectFromIPFSCache[getIPFSCacheKey(p, name)] = getObjectFromIPFSCacheEntry{
			bytes:   objBytes,
			created: time.Now(),
		}
	}()

	return entry.bytes, nil
}

// fetchObjectFromIPFS gets the requested object from ipfs
func fetchObjectFromIPFS(ctx commands.Context, p peer.ID, name string) ([]byte, error) {
	root, err := ipfs.ResolveAltRoot(ctx, p, name, time.Minute)
	if err != nil {
		return nil, err
	}
	bytes, err := ipfs.Cat(ctx, root, time.Minute)
	if err != nil {
		return nil, err
	}

	return bytes, nil
}

func getIPFSCacheKey(p peer.ID, name string) string {
	return p.Pretty() + "|" + name
}
<|endoftext|>


<commit_before>package controller

import (
	"fmt"
	_ "os"

	"github.com/orangesys/orangeapi/common"
	"github.com/orangesys/orangeapi/kong"
	"github.com/orangesys/orangeapi/firebase"
	"github.com/orangesys/orangeapi/config"
)

func create_kong_api_plugin(config *config.KongConfiguration, name, writepassword string) error {
	client := kong.NewClient(nil, config)
	influxdbAPI := &kong.API{
	    Name: name + "-influxdb",
	    UpstreamURL: "http://" + name + "-influxdb.default",
	    RequestHost: name + ".i.orangesys.io",
	}
	_, iresp, err := client.APIService.Create(influxdbAPI)
	if iresp.StatusCode != 201 {
		return fmt.Errorf("%s", "can not create influxdb api")
	}
	if err != nil {
		return err
	}

	grafanaAPI := &kong.API{
	    Name: name + "-grafana",
	    UpstreamURL: "http://" + name + "-grafana.default",
	    RequestHost: name + ".g.orangesys.io",
	}
	_, gresp, err := client.APIService.Create(grafanaAPI)
	if gresp.StatusCode != 201 {
		return fmt.Errorf("%s", "can not create grafana api")
	}
	if err != nil {
		return err
	}

	apiName := name + "-influxdb"
	JWTPlugin := &kong.Plugin{
	    Name: "jwt",
	}
	_, jwtresp, err := client.PluginService.Create(JWTPlugin, apiName)
	if jwtresp.StatusCode != 201 {
		return fmt.Errorf("%s %s", "can not create jwt plugin with api", apiName)
	}
	if err != nil {
		return err
	}

	correlationIDPlugin := &kong.Plugin{
	    Name: "correlation-id",
	    Config: kong.CorrelationIDPluginConfig{
			HeaderName: "Orangesys-Request-ID",
			Generator: "tracker",
	    },
	}
	_, cidresp, err := client.PluginService.Create(correlationIDPlugin, apiName)
	if cidresp.StatusCode != 201 {
		return fmt.Errorf("%s %s", "can not create correlation-id plugin with api", apiName)
	}
	if err != nil {
		return err
	}

	querystring := "u:_write,p:" + writepassword
	requesttransformerPlugin := &kong.Plugin{
	    Name: "request-transformer",
	    Config: kong.RequestTransformerPluginConfig{
			RemoveQueryString: "jwt",
			AddQueryString: querystring,
            },
        }
	_, rfresp, err := client.PluginService.Create(requesttransformerPlugin, apiName)
	if rfresp.StatusCode != 201 {
		return fmt.Errorf("%s %s", "can not create request-transformer plugin with api", apiName)
	}
	if err != nil {
		return err
	}
	return nil
}

func create_kong_consumer_with_jwt(config *config.KongConfiguration, name string) (string, string, error) {
	client := kong.NewClient(nil, config)
	generateConsumer := &kong.Consumer{
	    Username: name,
	}
	_, cresp, err := client.ConsumerService.Create(generateConsumer)
	if cresp.StatusCode != 201 {
		return "", "", fmt.Errorf("%s %s", "can not create consumer", name)
	}
	if err != nil {
		return "", "", err
	}

	_k, _ := common.UUID()
	_s, _ := common.UUID()
	generateConfig := &kong.JWTCredential{
		Key: _k,
		Secret: _s,
	}
	_, jwtresp, err := client.JWTService.Create(name, generateConfig)
	if jwtresp.StatusCode != 201 {
		return "", "", fmt.Errorf("%s %s", "can not create jwt plugin with api", name)
	}
	if err != nil {
		return "", "", err
	}
	return _k, _s, nil
}

func CreateConsumer(name, wp, uuid string) error {
//	name := "rlxebz"
//	wp := "mypassword"
//	uuid := "iGzNX6QzfudVlwKtR8CQCj0itIU2"

	kongconfig, err := config.LoadKongConfig()
	if err != nil {
		return  err
	}

	if err = create_kong_api_plugin(kongconfig, name, wp); err != nil {
		return err
	}

	key, secret, cerr := create_kong_consumer_with_jwt(kongconfig, name)
	if cerr != nil {
		return cerr
	}
	consumer := common.Consumer{
		Iss: key,
		Secret: secret,
	}
	consumer_jwt_token, jerr := consumer.CreateToken()
	if jerr != nil {
		return jerr
	}

        firebaseconfig, err := config.LoadFirebaseConfig()

        if err != nil {
            return err
        }
        user := firebase.FirebaseConfiguration{
	    Config: firebaseconfig,
	    UUID: uuid,
	    ConsumerID: name,
	    Token: consumer_jwt_token,
        }
        if err = user.SaveToken(); err !=nil {
            return err
        }
	return nil
}
<commit_msg>fix CreateConsumer<commit_after>package controller

import (
	"fmt"
	"errors"
	_ "os"

	"github.com/orangesys/orangeapi/common"
	"github.com/orangesys/orangeapi/kong"
	"github.com/orangesys/orangeapi/helm"
	"github.com/orangesys/orangeapi/k8s"
	"github.com/orangesys/orangeapi/firebase"
	"github.com/orangesys/orangeapi/config"
)
var ErrNotExist = errors.New("object does not exist")

func create_kong_api_plugin(config *config.KongConfiguration, name, writepassword string) error {
	client := kong.NewClient(nil, config)
	influxdbAPI := &kong.API{
	    Name: name + "-influxdb",
	    UpstreamURL: "http://" + name + "-influxdb.default",
	    RequestHost: name + ".i.orangesys.io",
	}
	_, iresp, err := client.APIService.Create(influxdbAPI)
	if iresp.StatusCode != 201 {
		return fmt.Errorf("%s", "can not create influxdb api")
	}
	if err != nil {
		return err
	}

	grafanaAPI := &kong.API{
	    Name: name + "-grafana",
	    UpstreamURL: "http://" + name + "-grafana.default",
	    RequestHost: name + ".g.orangesys.io",
	}
	_, gresp, err := client.APIService.Create(grafanaAPI)
	if gresp.StatusCode != 201 {
		return fmt.Errorf("%s", "can not create grafana api")
	}
	if err != nil {
		return err
	}

	apiName := name + "-influxdb"
	JWTPlugin := &kong.Plugin{
	    Name: "jwt",
	}
	_, jwtresp, err := client.PluginService.Create(JWTPlugin, apiName)
	if jwtresp.StatusCode != 201 {
		return fmt.Errorf("%s %s", "can not create jwt plugin with api", apiName)
	}
	if err != nil {
		return err
	}

	correlationIDPlugin := &kong.Plugin{
	    Name: "correlation-id",
	    Config: kong.CorrelationIDPluginConfig{
			HeaderName: "Orangesys-Request-ID",
			Generator: "tracker",
	    },
	}
	_, cidresp, err := client.PluginService.Create(correlationIDPlugin, apiName)
	if cidresp.StatusCode != 201 {
		return fmt.Errorf("%s %s", "can not create correlation-id plugin with api", apiName)
	}
	if err != nil {
		return err
	}

	querystring := "u:_write,p:" + writepassword
	requesttransformerPlugin := &kong.Plugin{
	    Name: "request-transformer",
	    Config: kong.RequestTransformerPluginConfig{
			RemoveQueryString: "jwt",
			AddQueryString: querystring,
            },
        }
	_, rfresp, err := client.PluginService.Create(requesttransformerPlugin, apiName)
	if rfresp.StatusCode != 201 {
		return fmt.Errorf("%s %s", "can not create request-transformer plugin with api", apiName)
	}
	if err != nil {
		return err
	}
	return nil
}

func create_kong_consumer_with_jwt(config *config.KongConfiguration, name string) (string, string, error) {
	client := kong.NewClient(nil, config)
	generateConsumer := &kong.Consumer{
	    Username: name,
	}
	_, cresp, err := client.ConsumerService.Create(generateConsumer)
	if cresp.StatusCode != 201 {
		return "", "", fmt.Errorf("%s %s", "can not create consumer", name)
	}
	if err != nil {
		return "", "", err
	}

	_k, _ := common.UUID()
	_s, _ := common.UUID()
	generateConfig := &kong.JWTCredential{
		Key: _k,
		Secret: _s,
	}
	_, jwtresp, err := client.JWTService.Create(name, generateConfig)
	if jwtresp.StatusCode != 201 {
		return "", "", fmt.Errorf("%s %s", "can not create jwt plugin with api", name)
	}
	if err != nil {
		return "", "", err
	}
	return _k, _s, nil
}

func deploy_influxdb_grafana(name, retention, pvcsize string) (string, error) {
		namespace := "default"
		data := "write-password"
		key := name + "-i-influxdb"

	  influxdb := helm.InfluxdbCommander{
		    Name: name,
		    Retention: retention,
		    Pvcsize: pvcsize,
	  }
		if err := influxdb.InstallInfluxdb(); err != nil {
		    return "", fmt.Errorf("%s %s", "can not deploy influxdb with", name)
		}

		grafana := helm.GrafanaCommander{
				Name: name,
		}
		if err := grafana.InstallGrafana(); err != nil {
		    return "", fmt.Errorf("%s %s", "can not deploy grafana with", name)
		}
		writepassword, err := k8s.GetSecret(namespace,key, data)
		if err != nil {
			return "", err
		}
		return writepassword, nil
}

func CreateConsumer(name, retention, pvcsize, uuid string) error {
//	name := "rlxebz"
//	wp := "mypassword"
//	uuid := "iGzNX6QzfudVlwKtR8CQCj0itIU2"

	wp, err := deploy_influxdb_grafana(name, retention, pvcsize)
	if err != nil {
		return err
	}

	kongconfig, kerr := config.LoadKongConfig()
	if err != nil {
		return  kerr
	}

	if err := create_kong_api_plugin(kongconfig, name, wp); err != nil {
		return err
	}

	key, secret, err := create_kong_consumer_with_jwt(kongconfig, name)
	if err != nil {
		return err
	}
	consumer := common.Consumer{
		Iss: key,
		Secret: secret,
	}
	consumer_jwt_token, err := consumer.CreateToken()
	if err != nil {
		return err
	}

  firebaseconfig, err := config.LoadFirebaseConfig()

  if err != nil {
      return err
  }
  user := firebase.FirebaseConfiguration{
			Config: firebaseconfig,
			UUID: uuid,
			ConsumerID: name,
			Token: consumer_jwt_token,
  }
  if err := user.SaveToken(); err != nil {
      return err
  }
	return nil
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package builds

import (
	"net/http"

	h "github.com/ernestio/api-gateway/helpers"
	"github.com/ernestio/api-gateway/models"
)

// Delete : Deletes an environment by name, generating a delete build
func Delete(au models.User, name string) (int, []byte) {
	var e models.Env
	var m models.Mapping

	err := e.FindByName(name)
	if err != nil {
		h.L.Error(err.Error())
		return 404, []byte("Environment not found")
	}

	if st, res := h.IsAuthorizedToResource(&au, h.GetEnv, e.GetType(), e.Name); st != 200 {
		return st, res
	}

	err = m.Delete(name)
	if err != nil {
		h.L.Error(err.Error())
		return 500, []byte(`"Couldn't map the environment"`)
	}

	b := models.Build{
		EnvironmentID: e.ID,
		UserID:        au.ID,
		Username:      au.Username,
		Type:          "destroy",
		Mapping:       m,
	}

	err = b.Save()
	if err != nil {
		h.L.Error(err.Error())
		return 500, []byte(`"Couldn't create the deletion build"`)
	}

	if err := b.RequestDeletion(&m); err != nil {
		h.L.Error(err.Error())
		return 500, []byte(`"Couldn't call build.delete"`)
	}

	return http.StatusOK, []byte(`{"id":"` + b.ID + `"}`)
}
<commit_msg>fixed missing id on deletion<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package builds

import (
	"net/http"

	h "github.com/ernestio/api-gateway/helpers"
	"github.com/ernestio/api-gateway/models"
)

// Delete : Deletes an environment by name, generating a delete build
func Delete(au models.User, name string) (int, []byte) {
	var e models.Env
	var m models.Mapping

	err := e.FindByName(name)
	if err != nil {
		h.L.Error(err.Error())
		return 404, []byte("Environment not found")
	}

	if st, res := h.IsAuthorizedToResource(&au, h.GetEnv, e.GetType(), e.Name); st != 200 {
		return st, res
	}

	err = m.Delete(name)
	if err != nil {
		h.L.Error(err.Error())
		return 500, []byte(`"Couldn't map the environment"`)
	}

	b := models.Build{
		ID:            m["id"].(string),
		EnvironmentID: e.ID,
		UserID:        au.ID,
		Username:      au.Username,
		Type:          "destroy",
		Mapping:       m,
	}

	err = b.Save()
	if err != nil {
		h.L.Error(err.Error())
		return 500, []byte(`"Couldn't create the deletion build"`)
	}

	if err := b.RequestDeletion(&m); err != nil {
		h.L.Error(err.Error())
		return 500, []byte(`"Couldn't call build.delete"`)
	}

	return http.StatusOK, []byte(`{"id":"` + b.ID + `"}`)
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/mitchellh/cli"
	"github.com/wantedly/developers-account-mapper/command"
)

func Commands(meta *command.Meta) map[string]cli.CommandFactory {
	return map[string]cli.CommandFactory{
		"register": func() (cli.Command, error) {
			return &command.RegisterCommand{
				Meta: *meta,
			}, nil
		},
		"list": func() (cli.Command, error) {
			return &command.ListCommand{
				Meta: *meta,
			}, nil
		},
		"setenv": func() (cli.Command, error) {
			return &command.ExecCommand{
				Meta: *meta,
			}, nil
		},
		"to-github-name": func() (cli.Command, error) {
			return &command.ToGithubNameCommand{
				Meta: *meta,
			}, nil
		},

		"version": func() (cli.Command, error) {
			return &command.VersionCommand{
				Meta:     *meta,
				Version:  Version,
				Revision: GitCommit,
				Name:     Name,
			}, nil
		},
	}
}
<commit_msg>Rename to exec<commit_after>package main

import (
	"github.com/mitchellh/cli"
	"github.com/wantedly/developers-account-mapper/command"
)

func Commands(meta *command.Meta) map[string]cli.CommandFactory {
	return map[string]cli.CommandFactory{
		"register": func() (cli.Command, error) {
			return &command.RegisterCommand{
				Meta: *meta,
			}, nil
		},
		"list": func() (cli.Command, error) {
			return &command.ListCommand{
				Meta: *meta,
			}, nil
		},
		"exec": func() (cli.Command, error) {
			return &command.ExecCommand{
				Meta: *meta,
			}, nil
		},
		"to-github-name": func() (cli.Command, error) {
			return &command.ToGithubNameCommand{
				Meta: *meta,
			}, nil
		},

		"version": func() (cli.Command, error) {
			return &command.VersionCommand{
				Meta:     *meta,
				Version:  Version,
				Revision: GitCommit,
				Name:     Name,
			}, nil
		},
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"strconv"
	"strings"
)

type readLiner interface {
	ReadLine() (string, error)
}

type commandContext struct {
	args           []string
	stdin          readLiner
	stdout, stderr io.Writer
	pty            bool
}

type command interface {
	execute(context commandContext) (uint32, error)
}

var commands = map[string]command{
	"sh":    cmdShell{},
	"true":  cmdTrue{},
	"false": cmdFalse{},
	"echo":  cmdEcho{},
	"cat":   cmdCat{},
}

var shellProgram = []string{"sh"}

func executeProgram(context commandContext) (uint32, error) {
	if len(context.args) == 0 {
		return 0, nil
	}
	command := commands[context.args[0]]
	if command == nil {
		_, err := fmt.Fprintf(context.stderr, "%v: command not found\n", context.args[0])
		return 127, err
	}
	return command.execute(context)
}

type cmdShell struct{}

func (cmdShell) execute(context commandContext) (uint32, error) {
	var prompt string
	if context.pty {
		prompt = "$ "
	}
	var lastStatus uint32
	var line string
	var err error
	for {
		_, err = fmt.Fprint(context.stdout, prompt)
		if err != nil {
			return 0, err
		}
		line, err = context.stdin.ReadLine()
		if err != nil {
			return 0, err
		}
		args := strings.Fields(line)
		if len(args) == 0 {
			continue
		}
		if args[0] == "exit" {
			var err error
			var status uint64 = uint64(lastStatus)
			if len(args) > 1 {
				status, err = strconv.ParseUint(args[1], 10, 32)
				if err != nil {
					status = 255
				}
			}
			return uint32(status), nil
		}
		newContext := context
		newContext.args = args
		if lastStatus, err = executeProgram(newContext); err != nil {
			return 0, err
		}
	}
}

type cmdTrue struct{}

func (cmdTrue) execute(context commandContext) (uint32, error) {
	return 0, nil
}

type cmdFalse struct{}

func (cmdFalse) execute(context commandContext) (uint32, error) {
	return 1, nil
}

type cmdEcho struct{}

func (cmdEcho) execute(context commandContext) (uint32, error) {
	_, err := fmt.Fprintln(context.stdout, strings.Join(context.args[1:], " "))
	return 0, err
}

type cmdCat struct{}

func (cmdCat) execute(context commandContext) (uint32, error) {
	if len(context.args) > 1 {
		for _, file := range context.args[1:] {
			if _, err := fmt.Fprintf(context.stderr, "%v: %v: No such file or directory\n", context.args[0], file); err != nil {
				return 0, err
			}
		}
		return 1, nil
	}
	var line string
	var err error
	for err == nil {
		line, err = context.stdin.ReadLine()
		if err == nil {
			_, err = fmt.Fprintln(context.stdout, line)
		}
	}
	return 0, err
}
<commit_msg>shell: return status codes even on errors<commit_after>package main

import (
	"fmt"
	"io"
	"strconv"
	"strings"
)

type readLiner interface {
	ReadLine() (string, error)
}

type commandContext struct {
	args           []string
	stdin          readLiner
	stdout, stderr io.Writer
	pty            bool
}

type command interface {
	execute(context commandContext) (uint32, error)
}

var commands = map[string]command{
	"sh":    cmdShell{},
	"true":  cmdTrue{},
	"false": cmdFalse{},
	"echo":  cmdEcho{},
	"cat":   cmdCat{},
}

var shellProgram = []string{"sh"}

func executeProgram(context commandContext) (uint32, error) {
	if len(context.args) == 0 {
		return 0, nil
	}
	command := commands[context.args[0]]
	if command == nil {
		_, err := fmt.Fprintf(context.stderr, "%v: command not found\n", context.args[0])
		return 127, err
	}
	return command.execute(context)
}

type cmdShell struct{}

func (cmdShell) execute(context commandContext) (uint32, error) {
	var prompt string
	if context.pty {
		prompt = "$ "
	}
	var lastStatus uint32
	var line string
	var err error
	for {
		_, err = fmt.Fprint(context.stdout, prompt)
		if err != nil {
			return lastStatus, err
		}
		line, err = context.stdin.ReadLine()
		if err != nil {
			return lastStatus, err
		}
		args := strings.Fields(line)
		if len(args) == 0 {
			continue
		}
		if args[0] == "exit" {
			var err error
			var status uint64 = uint64(lastStatus)
			if len(args) > 1 {
				status, err = strconv.ParseUint(args[1], 10, 32)
				if err != nil {
					status = 255
				}
			}
			return uint32(status), nil
		}
		newContext := context
		newContext.args = args
		if lastStatus, err = executeProgram(newContext); err != nil {
			return lastStatus, err
		}
	}
}

type cmdTrue struct{}

func (cmdTrue) execute(context commandContext) (uint32, error) {
	return 0, nil
}

type cmdFalse struct{}

func (cmdFalse) execute(context commandContext) (uint32, error) {
	return 1, nil
}

type cmdEcho struct{}

func (cmdEcho) execute(context commandContext) (uint32, error) {
	_, err := fmt.Fprintln(context.stdout, strings.Join(context.args[1:], " "))
	return 0, err
}

type cmdCat struct{}

func (cmdCat) execute(context commandContext) (uint32, error) {
	if len(context.args) > 1 {
		for _, file := range context.args[1:] {
			if _, err := fmt.Fprintf(context.stderr, "%v: %v: No such file or directory\n", context.args[0], file); err != nil {
				return 0, err
			}
		}
		return 1, nil
	}
	var line string
	var err error
	for err == nil {
		line, err = context.stdin.ReadLine()
		if err == nil {
			_, err = fmt.Fprintln(context.stdout, line)
		}
	}
	return 0, err
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"os/signal"

	"github.com/hashicorp/terraform/command"
	"github.com/mitchellh/cli"
)

// Commands is the mapping of all the available Terraform commands.
var Commands map[string]cli.CommandFactory
var PlumbingCommands map[string]struct{}

// Ui is the cli.Ui used for communicating to the outside world.
var Ui cli.Ui

const (
	ErrorPrefix  = "e:"
	OutputPrefix = "o:"
)

func init() {
	Ui = &cli.PrefixedUi{
		AskPrefix:    OutputPrefix,
		OutputPrefix: OutputPrefix,
		InfoPrefix:   OutputPrefix,
		ErrorPrefix:  ErrorPrefix,
		Ui:           &cli.BasicUi{Writer: os.Stdout, ErrorWriter: os.Stderr},
	}

	meta := command.Meta{
		Color:       true,
		ContextOpts: &ContextOpts,
		Ui:          Ui,
	}

	PlumbingCommands = map[string]struct{}{
		"state": struct{}{}, // includes all subcommands
	}

	Commands = map[string]cli.CommandFactory{
		"apply": func() (cli.Command, error) {
			return &command.ApplyCommand{
				Meta:       meta,
				ShutdownCh: makeShutdownCh(),
			}, nil
		},

		"destroy": func() (cli.Command, error) {
			return &command.ApplyCommand{
				Meta:       meta,
				Destroy:    true,
				ShutdownCh: makeShutdownCh(),
			}, nil
		},

		"fmt": func() (cli.Command, error) {
			return &command.FmtCommand{
				Meta: meta,
			}, nil
		},

		"get": func() (cli.Command, error) {
			return &command.GetCommand{
				Meta: meta,
			}, nil
		},

		"graph": func() (cli.Command, error) {
			return &command.GraphCommand{
				Meta: meta,
			}, nil
		},

		"import": func() (cli.Command, error) {
			return &command.ImportCommand{
				Meta: meta,
			}, nil
		},

		"init": func() (cli.Command, error) {
			return &command.InitCommand{
				Meta: meta,
			}, nil
		},

		"internal-plugin": func() (cli.Command, error) {
			return &command.InternalPluginCommand{
				Meta: meta,
			}, nil
		},

		"output": func() (cli.Command, error) {
			return &command.OutputCommand{
				Meta: meta,
			}, nil
		},

		"plan": func() (cli.Command, error) {
			return &command.PlanCommand{
				Meta: meta,
			}, nil
		},

		"push": func() (cli.Command, error) {
			return &command.PushCommand{
				Meta: meta,
			}, nil
		},

		"refresh": func() (cli.Command, error) {
			return &command.RefreshCommand{
				Meta: meta,
			}, nil
		},

		"remote": func() (cli.Command, error) {
			return &command.RemoteCommand{
				Meta: meta,
			}, nil
		},

		"show": func() (cli.Command, error) {
			return &command.ShowCommand{
				Meta: meta,
			}, nil
		},

		"taint": func() (cli.Command, error) {
			return &command.TaintCommand{
				Meta: meta,
			}, nil
		},

		"validate": func() (cli.Command, error) {
			return &command.ValidateCommand{
				Meta: meta,
			}, nil
		},

		"version": func() (cli.Command, error) {
			return &command.VersionCommand{
				Meta:              meta,
				Revision:          GitCommit,
				Version:           Version,
				VersionPrerelease: VersionPrerelease,
				CheckFunc:         commandVersionCheck,
			}, nil
		},

		"untaint": func() (cli.Command, error) {
			return &command.UntaintCommand{
				Meta: meta,
			}, nil
		},

		//-----------------------------------------------------------
		// Plumbing
		//-----------------------------------------------------------

		"state": func() (cli.Command, error) {
			return &command.StateCommand{
				Meta: meta,
			}, nil
		},

		"state list": func() (cli.Command, error) {
			return &command.StateListCommand{
				Meta: meta,
			}, nil
		},

		"state show": func() (cli.Command, error) {
			return &command.StateShowCommand{
				Meta: meta,
			}, nil
		},
	}
}

// makeShutdownCh creates an interrupt listener and returns a channel.
// A message will be sent on the channel for every interrupt received.
func makeShutdownCh() <-chan struct{} {
	resultCh := make(chan struct{})

	signalCh := make(chan os.Signal, 4)
	signal.Notify(signalCh, os.Interrupt)
	go func() {
		for {
			<-signalCh
			resultCh <- struct{}{}
		}
	}()

	return resultCh
}
<commit_msg>Revert "core: Keep errors/warning out of stdout for ease of automation"<commit_after>package main

import (
	"os"
	"os/signal"

	"github.com/hashicorp/terraform/command"
	"github.com/mitchellh/cli"
)

// Commands is the mapping of all the available Terraform commands.
var Commands map[string]cli.CommandFactory
var PlumbingCommands map[string]struct{}

// Ui is the cli.Ui used for communicating to the outside world.
var Ui cli.Ui

const (
	ErrorPrefix  = "e:"
	OutputPrefix = "o:"
)

func init() {
	Ui = &cli.PrefixedUi{
		AskPrefix:    OutputPrefix,
		OutputPrefix: OutputPrefix,
		InfoPrefix:   OutputPrefix,
		ErrorPrefix:  ErrorPrefix,
		Ui:           &cli.BasicUi{Writer: os.Stdout},
	}

	meta := command.Meta{
		Color:       true,
		ContextOpts: &ContextOpts,
		Ui:          Ui,
	}

	PlumbingCommands = map[string]struct{}{
		"state": struct{}{}, // includes all subcommands
	}

	Commands = map[string]cli.CommandFactory{
		"apply": func() (cli.Command, error) {
			return &command.ApplyCommand{
				Meta:       meta,
				ShutdownCh: makeShutdownCh(),
			}, nil
		},

		"destroy": func() (cli.Command, error) {
			return &command.ApplyCommand{
				Meta:       meta,
				Destroy:    true,
				ShutdownCh: makeShutdownCh(),
			}, nil
		},

		"fmt": func() (cli.Command, error) {
			return &command.FmtCommand{
				Meta: meta,
			}, nil
		},

		"get": func() (cli.Command, error) {
			return &command.GetCommand{
				Meta: meta,
			}, nil
		},

		"graph": func() (cli.Command, error) {
			return &command.GraphCommand{
				Meta: meta,
			}, nil
		},

		"import": func() (cli.Command, error) {
			return &command.ImportCommand{
				Meta: meta,
			}, nil
		},

		"init": func() (cli.Command, error) {
			return &command.InitCommand{
				Meta: meta,
			}, nil
		},

		"internal-plugin": func() (cli.Command, error) {
			return &command.InternalPluginCommand{
				Meta: meta,
			}, nil
		},

		"output": func() (cli.Command, error) {
			return &command.OutputCommand{
				Meta: meta,
			}, nil
		},

		"plan": func() (cli.Command, error) {
			return &command.PlanCommand{
				Meta: meta,
			}, nil
		},

		"push": func() (cli.Command, error) {
			return &command.PushCommand{
				Meta: meta,
			}, nil
		},

		"refresh": func() (cli.Command, error) {
			return &command.RefreshCommand{
				Meta: meta,
			}, nil
		},

		"remote": func() (cli.Command, error) {
			return &command.RemoteCommand{
				Meta: meta,
			}, nil
		},

		"show": func() (cli.Command, error) {
			return &command.ShowCommand{
				Meta: meta,
			}, nil
		},

		"taint": func() (cli.Command, error) {
			return &command.TaintCommand{
				Meta: meta,
			}, nil
		},

		"validate": func() (cli.Command, error) {
			return &command.ValidateCommand{
				Meta: meta,
			}, nil
		},

		"version": func() (cli.Command, error) {
			return &command.VersionCommand{
				Meta:              meta,
				Revision:          GitCommit,
				Version:           Version,
				VersionPrerelease: VersionPrerelease,
				CheckFunc:         commandVersionCheck,
			}, nil
		},

		"untaint": func() (cli.Command, error) {
			return &command.UntaintCommand{
				Meta: meta,
			}, nil
		},

		//-----------------------------------------------------------
		// Plumbing
		//-----------------------------------------------------------

		"state": func() (cli.Command, error) {
			return &command.StateCommand{
				Meta: meta,
			}, nil
		},

		"state list": func() (cli.Command, error) {
			return &command.StateListCommand{
				Meta: meta,
			}, nil
		},

		"state show": func() (cli.Command, error) {
			return &command.StateShowCommand{
				Meta: meta,
			}, nil
		},
	}
}

// makeShutdownCh creates an interrupt listener and returns a channel.
// A message will be sent on the channel for every interrupt received.
func makeShutdownCh() <-chan struct{} {
	resultCh := make(chan struct{})

	signalCh := make(chan os.Signal, 4)
	signal.Notify(signalCh, os.Interrupt)
	go func() {
		for {
			<-signalCh
			resultCh <- struct{}{}
		}
	}()

	return resultCh
}
<|endoftext|>