<commit_before>package coreunix

import (
	"errors"
	"io"
	"os"
	gopath "path"

	"github.com/jbenet/go-ipfs/commands/files"
	core "github.com/jbenet/go-ipfs/core"
	importer "github.com/jbenet/go-ipfs/importer"
	chunk "github.com/jbenet/go-ipfs/importer/chunk"
	merkledag "github.com/jbenet/go-ipfs/merkledag"
	"github.com/jbenet/go-ipfs/pin"
	"github.com/jbenet/go-ipfs/thirdparty/eventlog"
	unixfs "github.com/jbenet/go-ipfs/unixfs"
)

var log = eventlog.Logger("coreunix")

// Add builds a merkledag from the a reader, pinning all objects to the local
// datastore. Returns a key representing the root node.
func Add(n *core.IpfsNode, r io.Reader) (string, error) {
	// TODO more attractive function signature importer.BuildDagFromReader
	dagNode, err := importer.BuildDagFromReader(
		r,
		n.DAG,
		n.Pinning.GetManual(), // Fix this interface
		chunk.DefaultSplitter,
	)
	if err != nil {
		return "", err
	}
	if err := n.Pinning.Flush(); err != nil {
		return "", err
	}
	k, err := dagNode.Key()
	if err != nil {
		return "", err
	}

	return k.String(), nil
}

// AddR recursively adds files in |path|.
func AddR(n *core.IpfsNode, root string) (key string, err error) {
	f, err := os.Open(root)
	if err != nil {
		return "", err
	}
	defer f.Close()
	ff, err := files.NewSerialFile(root, f)
	if err != nil {
		return "", err
	}
	dagnode, err := addFile(n, ff)
	if err != nil {
		return "", err
	}
	k, err := dagnode.Key()
	if err != nil {
		return "", err
	}
	return k.String(), nil
}

func add(n *core.IpfsNode, readers []io.Reader) ([]*merkledag.Node, error) {
	mp, ok := n.Pinning.(pin.ManualPinner)
	if !ok {
		return nil, errors.New("invalid pinner type! expected manual pinner")
	}
	dagnodes := make([]*merkledag.Node, 0)
	for _, reader := range readers {
		node, err := importer.BuildDagFromReader(reader, n.DAG, mp, chunk.DefaultSplitter)
		if err != nil {
			return nil, err
		}
		dagnodes = append(dagnodes, node)
	}
	err := n.Pinning.Flush()
	if err != nil {
		return nil, err
	}
	return dagnodes, nil
}

func addNode(n *core.IpfsNode, node *merkledag.Node) error {
	err := n.DAG.AddRecursive(node) // add the file to the graph + local storage
	if err != nil {
		return err
	}
	err = n.Pinning.Pin(node, true) // ensure we keep it
	if err != nil {
		return err
	}
	return nil
}

func addFile(n *core.IpfsNode, file files.File) (*merkledag.Node, error) {
	if file.IsDirectory() {
		return addDir(n, file)
	}

	dns, err := add(n, []io.Reader{file})
	if err != nil {
		return nil, err
	}

	return dns[len(dns)-1], nil // last dag node is the file.
}

func addDir(n *core.IpfsNode, dir files.File) (*merkledag.Node, error) {

	tree := &merkledag.Node{Data: unixfs.FolderPBData()}

Loop:
	for {
		file, err := dir.NextFile()
		switch {
		case err != nil && err != io.EOF:
			return nil, err
		case err == io.EOF:
			break Loop
		}

		node, err := addFile(n, file)
		if err != nil {
			return nil, err
		}

		_, name := gopath.Split(file.FileName())

		err = tree.AddNodeLink(name, node)
		if err != nil {
			return nil, err
		}
	}

	err := addNode(n, tree)
	if err != nil {
		return nil, err
	}
	return tree, nil
}
<commit_msg>core/coreunix: Created 'AddWrapped' function to add files with a wrapped directory (to preserve filename in path)<commit_after>package coreunix

import (
	"errors"
	"io"
	"io/ioutil"
	"os"
	gopath "path"

	"github.com/jbenet/go-ipfs/commands/files"
	core "github.com/jbenet/go-ipfs/core"
	importer "github.com/jbenet/go-ipfs/importer"
	chunk "github.com/jbenet/go-ipfs/importer/chunk"
	merkledag "github.com/jbenet/go-ipfs/merkledag"
	"github.com/jbenet/go-ipfs/pin"
	"github.com/jbenet/go-ipfs/thirdparty/eventlog"
	unixfs "github.com/jbenet/go-ipfs/unixfs"
)

var log = eventlog.Logger("coreunix")

// Add builds a merkledag from the a reader, pinning all objects to the local
// datastore. Returns a key representing the root node.
func Add(n *core.IpfsNode, r io.Reader) (string, error) {
	// TODO more attractive function signature importer.BuildDagFromReader
	dagNode, err := importer.BuildDagFromReader(
		r,
		n.DAG,
		n.Pinning.GetManual(), // Fix this interface
		chunk.DefaultSplitter,
	)
	if err != nil {
		return "", err
	}
	if err := n.Pinning.Flush(); err != nil {
		return "", err
	}
	k, err := dagNode.Key()
	if err != nil {
		return "", err
	}

	return k.String(), nil
}

// AddR recursively adds files in |path|.
func AddR(n *core.IpfsNode, root string) (key string, err error) {
	f, err := os.Open(root)
	if err != nil {
		return "", err
	}
	defer f.Close()
	ff, err := files.NewSerialFile(root, f)
	if err != nil {
		return "", err
	}
	dagnode, err := addFile(n, ff)
	if err != nil {
		return "", err
	}
	k, err := dagnode.Key()
	if err != nil {
		return "", err
	}
	return k.String(), nil
}

// AddWrapped adds data from a reader, and wraps it with a directory object
// to preserve the filename.
func AddWrapped(n *core.IpfsNode, r io.Reader, filename string) (string, *merkledag.Node, error) {
	file := files.NewReaderFile(filename, ioutil.NopCloser(r), nil)
	dir := files.NewSliceFile("", []files.File{file})
	dagnode, err := addDir(n, dir)
	if err != nil {
		return "", nil, err
	}
	k, err := dagnode.Key()
	if err != nil {
		return "", nil, err
	}
	return gopath.Join(k.String(), filename), dagnode, nil
}

func add(n *core.IpfsNode, readers []io.Reader) ([]*merkledag.Node, error) {
	mp, ok := n.Pinning.(pin.ManualPinner)
	if !ok {
		return nil, errors.New("invalid pinner type! expected manual pinner")
	}
	dagnodes := make([]*merkledag.Node, 0)
	for _, reader := range readers {
		node, err := importer.BuildDagFromReader(reader, n.DAG, mp, chunk.DefaultSplitter)
		if err != nil {
			return nil, err
		}
		dagnodes = append(dagnodes, node)
	}
	err := n.Pinning.Flush()
	if err != nil {
		return nil, err
	}
	return dagnodes, nil
}

func addNode(n *core.IpfsNode, node *merkledag.Node) error {
	err := n.DAG.AddRecursive(node) // add the file to the graph + local storage
	if err != nil {
		return err
	}
	err = n.Pinning.Pin(node, true) // ensure we keep it
	if err != nil {
		return err
	}
	return nil
}

func addFile(n *core.IpfsNode, file files.File) (*merkledag.Node, error) {
	if file.IsDirectory() {
		return addDir(n, file)
	}

	dns, err := add(n, []io.Reader{file})
	if err != nil {
		return nil, err
	}

	return dns[len(dns)-1], nil // last dag node is the file.
}

func addDir(n *core.IpfsNode, dir files.File) (*merkledag.Node, error) {

	tree := &merkledag.Node{Data: unixfs.FolderPBData()}

Loop:
	for {
		file, err := dir.NextFile()
		switch {
		case err != nil && err != io.EOF:
			return nil, err
		case err == io.EOF:
			break Loop
		}

		node, err := addFile(n, file)
		if err != nil {
			return nil, err
		}

		_, name := gopath.Split(file.FileName())

		err = tree.AddNodeLink(name, node)
		if err != nil {
			return nil, err
		}
	}

	err := addNode(n, tree)
	if err != nil {
		return nil, err
	}
	return tree, nil
}
<|endoftext|>


<commit_before>package index

import (
	"fmt"
	"log"
	"sort"
)

type Fields interface {
	// Iterator of string
	Terms(field string) Terms
}

type MultiFields struct {
	subs      []Fields
	subSlices []ReaderSlice
	termsMap  map[string]Terms // synchronized
}

func NewMultiFields(subs []Fields, subSlices []ReaderSlice) MultiFields {
	return MultiFields{subs, subSlices, make(map[string]Terms)}
}

func (mf MultiFields) Terms(field string) Terms {
	if ans, ok := mf.termsMap[field]; ok {
		return ans
	}

	// Lazy init: first time this field is requested, we
	// create & add to terms:
	subs2 := make([]Terms, 0)
	slices2 := make([]ReaderSlice, 0)

	// Gather all sub-readers that share this field
	for i, v := range mf.subs {
		terms := v.Terms(field)
		if terms.Iterator != nil {
			subs2 = append(subs2, terms)
			slices2 = append(slices2, mf.subSlices[i])
		}
	}
	if len(subs2) == 0 {
		return nil
		// don't cache this case with an unbounded cache, since the number of fields that don't exist
		// is unbounded.
	}
	ans := NewMultiTerms(subs2, slices2)
	mf.termsMap[field] = ans
	return ans
}

func GetMultiFields(r IndexReader) Fields {
	log.Print("Obtaining MultiFields from ", r)
	leaves := r.Leaves()
	switch len(leaves) {
	case 0:
		log.Print("No fields are found.")
		// no fields
		return nil
	case 1:
		// already an atomic reader / reader with one leave
		return leaves[0].Reader().(AtomicReader).Fields()
	default:
		fields := make([]Fields, 0)
		slices := make([]ReaderSlice, 0)
		for _, ctx := range leaves {
			f := ctx.Reader().(AtomicReader).Fields()
			if f == nil {
				continue
			}
			fields = append(fields, f)
			slices = append(slices, ReaderSlice{ctx.DocBase, r.MaxDoc(), len(fields)})
		}
		log.Printf("Found %v fields in %v slices.", len(fields), len(slices))
		switch len(fields) {
		case 0:
			return nil
		case 1:
			return fields[0]
		default:
			return NewMultiFields(fields, slices)
		}
	}
}

func GetMultiTerms(r IndexReader, field string) Terms {
	log.Printf("Loading field '%v' from %v", field, r)
	fields := GetMultiFields(r)
	if fields.Terms == nil {
		return nil
	}
	return fields.Terms(field)
}

type FieldInfo struct {
	// Field's name
	name string
	// Internal field number
	number int32

	indexed      bool
	docValueType DocValuesType

	// True if any document indexed term vectors
	storeTermVector bool

	normType      DocValuesType
	omitNorms     bool
	indexOptions  IndexOptions
	storePayloads bool

	attributes map[string]string
}

func NewFieldInfo(name string, indexed bool, number int32, storeTermVector, omitNorms, storePayloads bool,
	indexOptions IndexOptions, docValues, normsType DocValuesType, attributes map[string]string) FieldInfo {
	fi := FieldInfo{name: name, indexed: indexed, number: number, docValueType: docValues, attributes: attributes}
	if indexed {
		fi.storeTermVector = storeTermVector
		fi.storePayloads = storePayloads
		fi.omitNorms = omitNorms
		fi.indexOptions = indexOptions
		if !omitNorms {
			fi.normType = normsType
		}
	} // for non-indexed fields, leave defaults
	// assert checkConsistency()
	return fi
}

func (fi FieldInfo) String() string {
	return fmt.Sprintf("%v-%v, isIndexed=%v, docValueType=%v, hasVectors=%v, normType=%v, omitNorms=%v, indexOptions=%v, hasPayloads=%v, attributes=%v",
		fi.number, fi.name, fi.indexed, fi.docValueType, fi.storeTermVector, fi.normType, fi.omitNorms, fi.indexOptions, fi.storePayloads, fi.attributes)
}

type Int32Slice []int32

func (p Int32Slice) Len() int           { return len(p) }
func (p Int32Slice) Less(i, j int) bool { return p[i] < p[j] }
func (p Int32Slice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

// index/FieldInfos.java

// Collection of FieldInfo(s) (accessible by number of by name)
type FieldInfos struct {
	hasFreq      bool
	hasProx      bool
	hasPayloads  bool
	hasOffsets   bool
	hasVectors   bool
	hasNorms     bool
	hasDocValues bool

	byNumber map[int32]FieldInfo
	byName   map[string]FieldInfo
	values   []FieldInfo // sorted by ID
}

func NewFieldInfos(infos []FieldInfo) FieldInfos {
	self := FieldInfos{byNumber: make(map[int32]FieldInfo), byName: make(map[string]FieldInfo)}

	numbers := make([]int32, 0)
	for _, info := range infos {
		if prev, ok := self.byNumber[info.number]; ok {
			panic(fmt.Sprintf("duplicate field numbers: %v and %v have: %v", prev.name, info.name, info.number))
		}
		self.byNumber[info.number] = info
		numbers = append(numbers, info.number)
		if prev, ok := self.byName[info.name]; ok {
			panic(fmt.Sprintf("duplicate field names: %v and %v have: %v", prev.number, info.number, info.name))
		}
		self.byName[info.name] = info

		self.hasVectors = self.hasVectors || info.storeTermVector
		self.hasProx = self.hasProx || info.indexed && info.indexOptions >= INDEX_OPT_DOCS_AND_FREQS_AND_POSITIONS
		self.hasFreq = self.hasFreq || info.indexed && info.indexOptions != INDEX_OPT_DOCS_ONLY
		self.hasOffsets = self.hasOffsets || info.indexed && info.indexOptions >= INDEX_OPT_DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS
		self.hasNorms = self.hasNorms || info.normType != 0
		self.hasDocValues = self.hasDocValues || info.docValueType != 0
		self.hasPayloads = self.hasPayloads || info.storePayloads
	}

	sort.Sort(Int32Slice(numbers))
	self.values = make([]FieldInfo, len(infos))
	for i, v := range numbers {
		self.values[int32(i)] = self.byNumber[v]
	}

	return self
}

func (fis FieldInfos) String() string {
	return fmt.Sprintf(`
hasFreq = %v
hasProx = %v
hasPayloads = %v
hasOffsets = %v
hasVectors = %v
hasNorms = %v
hasDocValues = %v
%v`, fis.hasFreq, fis.hasProx, fis.hasPayloads, fis.hasOffsets, fis.hasVectors, fis.hasNorms, fis.hasDocValues, fis.values)
}

type FieldNumbers struct {
	numberToName map[int]string
	nameToNumber map[string]int
	// We use this to enforce that a given field never changes DV type,
	// even across segments / IndexWriter sessions:
	docValuesType map[string]DocValuesType
	// TODO: we should similarly catch an attempt to turn norms back on
	// after they were already ommitted; today we silently discard the
	// norm but this is badly trappy
	lowestUnassignedFieldNumber int
}

func newFieldNumbers() *FieldNumbers {
	return &FieldNumbers{
		nameToNumber:  make(map[string]int),
		numberToName:  make(map[int]string),
		docValuesType: make(map[string]DocValuesType),
	}
}

/*
Returns the global field number for the given field name. If the name
does not exist yet it tries to add it with the given preferred field
number assigned if possible otherwise the first unassigned field
number is used as the field number.
*/
func (fn *FieldNumbers) addOrGet(name string, preferredNumber int, dv DocValuesType) int {
	if dv != 0 {
		currentDv, ok := fn.docValuesType[name]
		if !ok || currentDv == 0 {
			fn.docValuesType[name] = dv
		} else if currentDv != dv {
			log.Panicf("cannot change DocValues type from %v to %v for field '%v'", currentDv, dv, name)
		}
	}
	number, ok := fn.nameToNumber[name]
	if !ok {
		_, ok = fn.numberToName[preferredNumber]
		if preferredNumber != -1 && !ok {
			// cool - we can use this number globally
			number = preferredNumber
		} else {
			// find a new FieldNumber
			for _, ok = fn.numberToName[fn.lowestUnassignedFieldNumber]; ok; {
				// might not be up to date - lets do the work once needed
				fn.lowestUnassignedFieldNumber++
			}
			number = fn.lowestUnassignedFieldNumber
		}

		fn.numberToName[number] = name
		fn.nameToNumber[name] = number
	}
	return number
}

type FieldInfosBuilder struct {
}

func newFieldInfosBuilder(globalFieldNumbers *FieldNumbers) *FieldInfosBuilder {
	panic("not implemented yet")
}

// index/FieldInfo.java

type IndexOptions int

const (
	INDEX_OPT_DOCS_ONLY                                = IndexOptions(1)
	INDEX_OPT_DOCS_AND_FREQS                           = IndexOptions(2)
	INDEX_OPT_DOCS_AND_FREQS_AND_POSITIONS             = IndexOptions(3)
	INDEX_OPT_DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS = IndexOptions(4)
)

type DocValuesType int

const (
	DOC_VALUES_TYPE_NUMERIC    = DocValuesType(1)
	DOC_VALUES_TYPE_BINARY     = DocValuesType(2)
	DOC_VALUES_TYPE_SORTED     = DocValuesType(3)
	DOC_VALUES_TYPE_SORTED_SET = DocValuesType(4)
)
<commit_msg>implement FieldInfosBuilder constructor<commit_after>package index

import (
	"fmt"
	"log"
	"sort"
)

type Fields interface {
	// Iterator of string
	Terms(field string) Terms
}

type MultiFields struct {
	subs      []Fields
	subSlices []ReaderSlice
	termsMap  map[string]Terms // synchronized
}

func NewMultiFields(subs []Fields, subSlices []ReaderSlice) MultiFields {
	return MultiFields{subs, subSlices, make(map[string]Terms)}
}

func (mf MultiFields) Terms(field string) Terms {
	if ans, ok := mf.termsMap[field]; ok {
		return ans
	}

	// Lazy init: first time this field is requested, we
	// create & add to terms:
	subs2 := make([]Terms, 0)
	slices2 := make([]ReaderSlice, 0)

	// Gather all sub-readers that share this field
	for i, v := range mf.subs {
		terms := v.Terms(field)
		if terms.Iterator != nil {
			subs2 = append(subs2, terms)
			slices2 = append(slices2, mf.subSlices[i])
		}
	}
	if len(subs2) == 0 {
		return nil
		// don't cache this case with an unbounded cache, since the number of fields that don't exist
		// is unbounded.
	}
	ans := NewMultiTerms(subs2, slices2)
	mf.termsMap[field] = ans
	return ans
}

func GetMultiFields(r IndexReader) Fields {
	log.Print("Obtaining MultiFields from ", r)
	leaves := r.Leaves()
	switch len(leaves) {
	case 0:
		log.Print("No fields are found.")
		// no fields
		return nil
	case 1:
		// already an atomic reader / reader with one leave
		return leaves[0].Reader().(AtomicReader).Fields()
	default:
		fields := make([]Fields, 0)
		slices := make([]ReaderSlice, 0)
		for _, ctx := range leaves {
			f := ctx.Reader().(AtomicReader).Fields()
			if f == nil {
				continue
			}
			fields = append(fields, f)
			slices = append(slices, ReaderSlice{ctx.DocBase, r.MaxDoc(), len(fields)})
		}
		log.Printf("Found %v fields in %v slices.", len(fields), len(slices))
		switch len(fields) {
		case 0:
			return nil
		case 1:
			return fields[0]
		default:
			return NewMultiFields(fields, slices)
		}
	}
}

func GetMultiTerms(r IndexReader, field string) Terms {
	log.Printf("Loading field '%v' from %v", field, r)
	fields := GetMultiFields(r)
	if fields.Terms == nil {
		return nil
	}
	return fields.Terms(field)
}

type FieldInfo struct {
	// Field's name
	name string
	// Internal field number
	number int32

	indexed      bool
	docValueType DocValuesType

	// True if any document indexed term vectors
	storeTermVector bool

	normType      DocValuesType
	omitNorms     bool
	indexOptions  IndexOptions
	storePayloads bool

	attributes map[string]string
}

func NewFieldInfo(name string, indexed bool, number int32, storeTermVector, omitNorms, storePayloads bool,
	indexOptions IndexOptions, docValues, normsType DocValuesType, attributes map[string]string) FieldInfo {
	fi := FieldInfo{name: name, indexed: indexed, number: number, docValueType: docValues, attributes: attributes}
	if indexed {
		fi.storeTermVector = storeTermVector
		fi.storePayloads = storePayloads
		fi.omitNorms = omitNorms
		fi.indexOptions = indexOptions
		if !omitNorms {
			fi.normType = normsType
		}
	} // for non-indexed fields, leave defaults
	// assert checkConsistency()
	return fi
}

func (fi FieldInfo) String() string {
	return fmt.Sprintf("%v-%v, isIndexed=%v, docValueType=%v, hasVectors=%v, normType=%v, omitNorms=%v, indexOptions=%v, hasPayloads=%v, attributes=%v",
		fi.number, fi.name, fi.indexed, fi.docValueType, fi.storeTermVector, fi.normType, fi.omitNorms, fi.indexOptions, fi.storePayloads, fi.attributes)
}

type Int32Slice []int32

func (p Int32Slice) Len() int           { return len(p) }
func (p Int32Slice) Less(i, j int) bool { return p[i] < p[j] }
func (p Int32Slice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

// index/FieldInfos.java

// Collection of FieldInfo(s) (accessible by number of by name)
type FieldInfos struct {
	hasFreq      bool
	hasProx      bool
	hasPayloads  bool
	hasOffsets   bool
	hasVectors   bool
	hasNorms     bool
	hasDocValues bool

	byNumber map[int32]FieldInfo
	byName   map[string]FieldInfo
	values   []FieldInfo // sorted by ID
}

func NewFieldInfos(infos []FieldInfo) FieldInfos {
	self := FieldInfos{byNumber: make(map[int32]FieldInfo), byName: make(map[string]FieldInfo)}

	numbers := make([]int32, 0)
	for _, info := range infos {
		if prev, ok := self.byNumber[info.number]; ok {
			panic(fmt.Sprintf("duplicate field numbers: %v and %v have: %v", prev.name, info.name, info.number))
		}
		self.byNumber[info.number] = info
		numbers = append(numbers, info.number)
		if prev, ok := self.byName[info.name]; ok {
			panic(fmt.Sprintf("duplicate field names: %v and %v have: %v", prev.number, info.number, info.name))
		}
		self.byName[info.name] = info

		self.hasVectors = self.hasVectors || info.storeTermVector
		self.hasProx = self.hasProx || info.indexed && info.indexOptions >= INDEX_OPT_DOCS_AND_FREQS_AND_POSITIONS
		self.hasFreq = self.hasFreq || info.indexed && info.indexOptions != INDEX_OPT_DOCS_ONLY
		self.hasOffsets = self.hasOffsets || info.indexed && info.indexOptions >= INDEX_OPT_DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS
		self.hasNorms = self.hasNorms || info.normType != 0
		self.hasDocValues = self.hasDocValues || info.docValueType != 0
		self.hasPayloads = self.hasPayloads || info.storePayloads
	}

	sort.Sort(Int32Slice(numbers))
	self.values = make([]FieldInfo, len(infos))
	for i, v := range numbers {
		self.values[int32(i)] = self.byNumber[v]
	}

	return self
}

func (fis FieldInfos) String() string {
	return fmt.Sprintf(`
hasFreq = %v
hasProx = %v
hasPayloads = %v
hasOffsets = %v
hasVectors = %v
hasNorms = %v
hasDocValues = %v
%v`, fis.hasFreq, fis.hasProx, fis.hasPayloads, fis.hasOffsets, fis.hasVectors, fis.hasNorms, fis.hasDocValues, fis.values)
}

type FieldNumbers struct {
	numberToName map[int]string
	nameToNumber map[string]int
	// We use this to enforce that a given field never changes DV type,
	// even across segments / IndexWriter sessions:
	docValuesType map[string]DocValuesType
	// TODO: we should similarly catch an attempt to turn norms back on
	// after they were already ommitted; today we silently discard the
	// norm but this is badly trappy
	lowestUnassignedFieldNumber int
}

func newFieldNumbers() *FieldNumbers {
	return &FieldNumbers{
		nameToNumber:  make(map[string]int),
		numberToName:  make(map[int]string),
		docValuesType: make(map[string]DocValuesType),
	}
}

/*
Returns the global field number for the given field name. If the name
does not exist yet it tries to add it with the given preferred field
number assigned if possible otherwise the first unassigned field
number is used as the field number.
*/
func (fn *FieldNumbers) addOrGet(name string, preferredNumber int, dv DocValuesType) int {
	if dv != 0 {
		currentDv, ok := fn.docValuesType[name]
		if !ok || currentDv == 0 {
			fn.docValuesType[name] = dv
		} else if currentDv != dv {
			log.Panicf("cannot change DocValues type from %v to %v for field '%v'", currentDv, dv, name)
		}
	}
	number, ok := fn.nameToNumber[name]
	if !ok {
		_, ok = fn.numberToName[preferredNumber]
		if preferredNumber != -1 && !ok {
			// cool - we can use this number globally
			number = preferredNumber
		} else {
			// find a new FieldNumber
			for _, ok = fn.numberToName[fn.lowestUnassignedFieldNumber]; ok; {
				// might not be up to date - lets do the work once needed
				fn.lowestUnassignedFieldNumber++
			}
			number = fn.lowestUnassignedFieldNumber
		}

		fn.numberToName[number] = name
		fn.nameToNumber[name] = number
	}
	return number
}

type FieldInfosBuilder struct {
	byName             map[string]*FieldInfo
	globalFieldNumbers *FieldNumbers
}

func newFieldInfosBuilder(globalFieldNumbers *FieldNumbers) *FieldInfosBuilder {
	assert(globalFieldNumbers != nil)
	return &FieldInfosBuilder{
		byName:             make(map[string]*FieldInfo),
		globalFieldNumbers: globalFieldNumbers,
	}
}

// index/FieldInfo.java

type IndexOptions int

const (
	INDEX_OPT_DOCS_ONLY                                = IndexOptions(1)
	INDEX_OPT_DOCS_AND_FREQS                           = IndexOptions(2)
	INDEX_OPT_DOCS_AND_FREQS_AND_POSITIONS             = IndexOptions(3)
	INDEX_OPT_DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS = IndexOptions(4)
)

type DocValuesType int

const (
	DOC_VALUES_TYPE_NUMERIC    = DocValuesType(1)
	DOC_VALUES_TYPE_BINARY     = DocValuesType(2)
	DOC_VALUES_TYPE_SORTED     = DocValuesType(3)
	DOC_VALUES_TYPE_SORTED_SET = DocValuesType(4)
)
<|endoftext|>


<commit_before>package php

import (
	"stephensearles.com/php/ast"
	"stephensearles.com/php/token"
)

func (p *parser) parseIf() *ast.IfStmt {
	p.expect(token.OpenParen)
	n := &ast.IfStmt{}
	p.next()
	n.Condition = p.parseExpression()
	p.expect(token.CloseParen)

	if p.peek().typ == token.TernaryOperator2 {
		p.expect(token.TernaryOperator2)
		n.TrueBranch = p.parseStatementsUntil(token.EndIf, token.ElseIf, token.Else)
	} else {
		p.next()
		n.TrueBranch = p.parseStmt()
		p.next()
	}

	switch p.current.typ {
	case token.ElseIf:
		n.FalseBranch = p.parseIf()
	case token.Else:
		p.next()
		if p.current.typ == token.TernaryOperator2 {
			n.FalseBranch = p.parseStatementsUntil(token.EndIf)
		} else {
			n.FalseBranch = p.parseStmt()
		}
	default:
		n.FalseBranch = ast.Block{}
	}
	return n
}

func (p *parser) parseWhile() ast.Statement {
	p.expect(token.OpenParen)
	term := p.parseNextExpression()
	p.expect(token.CloseParen)
	p.next()
	block := p.parseStmt()
	return &ast.WhileStmt{
		Termination: term,
		LoopBlock:   block,
	}
}

func (p *parser) parseForeach() ast.Statement {
	stmt := &ast.ForeachStmt{}
	p.expect(token.OpenParen)
	stmt.Source = p.parseNextExpression()
	p.expect(token.AsOperator)
	if p.peek().typ == token.AmpersandOperator {
		p.expect(token.AmpersandOperator)
	}
	p.expect(token.VariableOperator)
	p.next()
	first := ast.NewVariable(p.current.val)
	if p.peek().typ == token.ArrayKeyOperator {
		stmt.Key = first
		p.expect(token.ArrayKeyOperator)
		if p.peek().typ == token.AmpersandOperator {
			p.expect(token.AmpersandOperator)
		}
		p.expect(token.VariableOperator)
		p.next()
		stmt.Value = ast.NewVariable(p.current.val)
	} else {
		stmt.Value = first
	}
	p.expect(token.CloseParen)
	p.next()
	stmt.LoopBlock = p.parseStmt()
	return stmt
}

func (p *parser) parseFor() ast.Statement {
	stmt := &ast.ForStmt{}
	p.expect(token.OpenParen)
	stmt.Initialization = p.parseNextExpression()
	p.expect(token.StatementEnd)
	stmt.Termination = p.parseNextExpression()
	p.expect(token.StatementEnd)
	stmt.Iteration = p.parseNextExpression()
	p.expect(token.CloseParen)
	p.next()
	stmt.LoopBlock = p.parseStmt()
	return stmt
}

func (p *parser) parseDo() ast.Statement {
	block := p.parseBlock()
	p.expect(token.While)
	p.expect(token.OpenParen)
	term := p.parseNextExpression()
	p.expect(token.CloseParen)
	p.expectStmtEnd()
	return &ast.DoWhileStmt{
		Termination: term,
		LoopBlock:   block,
	}
}

func (p *parser) parseSwitch() ast.Statement {
	stmt := ast.SwitchStmt{}
	p.expect(token.OpenParen)
	stmt.Expression = p.parseExpression()
	p.expectCurrent(token.CloseParen)
	p.expect(token.BlockBegin)
	p.next()
	for {
		switch p.current.typ {
		case token.Case:
			expr := p.parseNextExpression()
			p.expect(token.TernaryOperator2)
			p.next()
			stmt.Cases = append(stmt.Cases, &ast.SwitchCase{
				Expression: expr,
				Block:      *(p.parseSwitchBlock()),
			})
		case token.Default:
			p.expect(token.TernaryOperator2)
			p.next()
			stmt.DefaultCase = p.parseSwitchBlock()
		case token.BlockEnd:
			return stmt
		default:
			p.errorf("Unexpected token. in switch statement:", p.current)
			return nil
		}
	}
}

func (p *parser) parseSwitchBlock() *ast.Block {
	needBlockEnd := false
	if p.current.typ == token.BlockBegin {
		needBlockEnd = true
		p.next()
	}
	block := &ast.Block{
		Statements: make([]ast.Statement, 0),
	}
stmtLoop:
	for {
		switch p.current.typ {
		case token.BlockEnd:
			if needBlockEnd {
				needBlockEnd = false
				p.next()
			}
			fallthrough
		case token.Case, token.Default:
			break stmtLoop
		default:
			stmt := p.parseStmt()
			if stmt == nil {
				p.errorf("Invalid statement in switch block", p.current)
				break stmtLoop
			}
			block.Statements = append(block.Statements, stmt)
			p.next()
		}
	}
	if needBlockEnd {
		p.errorf("switch case needs block end")
	}
	return block
}
<commit_msg>Fixed parsing if statements with no else<commit_after>package php

import (
	"stephensearles.com/php/ast"
	"stephensearles.com/php/token"
)

func (p *parser) parseIf() *ast.IfStmt {
	p.expect(token.OpenParen)
	n := &ast.IfStmt{}
	p.next()
	n.Condition = p.parseExpression()
	p.expect(token.CloseParen)

	if p.peek().typ == token.TernaryOperator2 {
		p.expect(token.TernaryOperator2)
		n.TrueBranch = p.parseStatementsUntil(token.EndIf, token.ElseIf, token.Else)
	} else {
		p.next()
		n.TrueBranch = p.parseStmt()
		p.next()
	}

	switch p.current.typ {
	case token.ElseIf:
		n.FalseBranch = p.parseIf()
	case token.Else:
		p.next()
		if p.current.typ == token.TernaryOperator2 {
			n.FalseBranch = p.parseStatementsUntil(token.EndIf)
		} else {
			n.FalseBranch = p.parseStmt()
		}
	default:
		n.FalseBranch = ast.Block{}
		p.backup()
	}
	return n
}

func (p *parser) parseWhile() ast.Statement {
	p.expect(token.OpenParen)
	term := p.parseNextExpression()
	p.expect(token.CloseParen)
	p.next()
	block := p.parseStmt()
	return &ast.WhileStmt{
		Termination: term,
		LoopBlock:   block,
	}
}

func (p *parser) parseForeach() ast.Statement {
	stmt := &ast.ForeachStmt{}
	p.expect(token.OpenParen)
	stmt.Source = p.parseNextExpression()
	p.expect(token.AsOperator)
	if p.peek().typ == token.AmpersandOperator {
		p.expect(token.AmpersandOperator)
	}
	p.expect(token.VariableOperator)
	p.next()
	first := ast.NewVariable(p.current.val)
	if p.peek().typ == token.ArrayKeyOperator {
		stmt.Key = first
		p.expect(token.ArrayKeyOperator)
		if p.peek().typ == token.AmpersandOperator {
			p.expect(token.AmpersandOperator)
		}
		p.expect(token.VariableOperator)
		p.next()
		stmt.Value = ast.NewVariable(p.current.val)
	} else {
		stmt.Value = first
	}
	p.expect(token.CloseParen)
	p.next()
	stmt.LoopBlock = p.parseStmt()
	return stmt
}

func (p *parser) parseFor() ast.Statement {
	stmt := &ast.ForStmt{}
	p.expect(token.OpenParen)
	stmt.Initialization = p.parseNextExpression()
	p.expect(token.StatementEnd)
	stmt.Termination = p.parseNextExpression()
	p.expect(token.StatementEnd)
	stmt.Iteration = p.parseNextExpression()
	p.expect(token.CloseParen)
	p.next()
	stmt.LoopBlock = p.parseStmt()
	return stmt
}

func (p *parser) parseDo() ast.Statement {
	block := p.parseBlock()
	p.expect(token.While)
	p.expect(token.OpenParen)
	term := p.parseNextExpression()
	p.expect(token.CloseParen)
	p.expectStmtEnd()
	return &ast.DoWhileStmt{
		Termination: term,
		LoopBlock:   block,
	}
}

func (p *parser) parseSwitch() ast.Statement {
	stmt := ast.SwitchStmt{}
	p.expect(token.OpenParen)
	stmt.Expression = p.parseExpression()
	p.expectCurrent(token.CloseParen)
	p.expect(token.BlockBegin)
	p.next()
	for {
		switch p.current.typ {
		case token.Case:
			expr := p.parseNextExpression()
			p.expect(token.TernaryOperator2)
			p.next()
			stmt.Cases = append(stmt.Cases, &ast.SwitchCase{
				Expression: expr,
				Block:      *(p.parseSwitchBlock()),
			})
		case token.Default:
			p.expect(token.TernaryOperator2)
			p.next()
			stmt.DefaultCase = p.parseSwitchBlock()
		case token.BlockEnd:
			return stmt
		default:
			p.errorf("Unexpected token. in switch statement:", p.current)
			return nil
		}
	}
}

func (p *parser) parseSwitchBlock() *ast.Block {
	needBlockEnd := false
	if p.current.typ == token.BlockBegin {
		needBlockEnd = true
		p.next()
	}
	block := &ast.Block{
		Statements: make([]ast.Statement, 0),
	}
stmtLoop:
	for {
		switch p.current.typ {
		case token.BlockEnd:
			if needBlockEnd {
				needBlockEnd = false
				p.next()
			}
			fallthrough
		case token.Case, token.Default:
			break stmtLoop
		default:
			stmt := p.parseStmt()
			if stmt == nil {
				p.errorf("Invalid statement in switch block", p.current)
				break stmtLoop
			}
			block.Statements = append(block.Statements, stmt)
			p.next()
		}
	}
	if needBlockEnd {
		p.errorf("switch case needs block end")
	}
	return block
}
<|endoftext|>


<commit_before>//	 ,+---+
//	+---+´|    HASHBOX SOURCE
//	| # | |    Copyright 2015-2016
//	+---+´

// Hashbox core, version 0.1
package core

import (
	"bufio"
	"os"
)

//********************************************************************************//
//                                  BufferedFile                                  //
//********************************************************************************//

type BufferedReader struct {
	File *os.File
	*bufio.Reader
}

type BufferedUnserializer interface {
	Unserialize(r *BufferedReader) (size int)
}

func OpenBufferedReader(path string, buffersize int, flag int) (*BufferedReader, error) {
	file, err := os.OpenFile(path, flag|os.O_RDONLY, 0)
	if err != nil {
		return nil, err
	}
	return &BufferedReader{file, bufio.NewReaderSize(file, buffersize)}, nil
}
func (b *BufferedReader) Seek(offset int64, whence int) (ret int64, err error) {
	ret, err = b.File.Seek(offset, whence)
	if err == nil {
		b.Reset(b.File)
	}
	return ret, err
}

type BufferedWriter struct {
	File *os.File
	*bufio.Writer
}

type BufferedSerializer interface {
	Serialize(w *BufferedWriter) (size int)
}

func OpenBufferedWriter(path string, buffersize int, flag int, perm os.FileMode) (*BufferedWriter, error) {
	file, err := os.OpenFile(path, flag|os.O_WRONLY, perm)
	if err != nil {
		return nil, err
	}
	return &BufferedWriter{file, bufio.NewWriterSize(file, buffersize)}, nil
}
func (b *BufferedWriter) Seek(offset int64, whence int) (ret int64, err error) {
	b.Flush() // Always flush in case we want to read what we have written
	ret, err = b.File.Seek(offset, whence)
	if err == nil {
		b.Reset(b.File)
	}
	return ret, err
}

type BufferedFile struct {
	Path       string
	BufferSize int
	Flag       int
	Perm       os.FileMode
	Reader     *BufferedReader
	Writer     *BufferedWriter
}

func OpenBufferedFile(path string, buffersize int, flag int, perm os.FileMode) (*BufferedFile, error) {
	b := &BufferedFile{Path: path, BufferSize: buffersize, Flag: flag, Perm: perm}
	var err error
	if b.Writer, err = OpenBufferedWriter(b.Path, b.BufferSize, b.Flag, b.Perm); err != nil {
		return nil, err
	}
	if b.Reader, err = OpenBufferedReader(b.Path, b.BufferSize, b.Flag); err != nil {
		return nil, err
	}
	return b, nil
}
func (b *BufferedFile) Size() int64 {
	b.Writer.Flush() // Always flush in case we want to read what we have written
	if info, err := b.Reader.File.Stat(); err != nil {
		panic(err)
	} else {
		return info.Size()
	}
}
func (b *BufferedFile) Close() (err error) {
	b.Writer.Flush() // Always flush in case we want to read what we have written
	if e := b.Reader.File.Close(); e != nil {
		err = e
	}
	if e := b.Writer.File.Close(); e != nil {
		err = e
	}
	return err
}
<commit_msg>Using seek_end to get file size instead of doing a full File.Stat query<commit_after>//	 ,+---+
//	+---+´|    HASHBOX SOURCE
//	| # | |    Copyright 2015-2016
//	+---+´

// Hashbox core, version 0.1
package core

import (
	"bufio"
	"os"
)

//********************************************************************************//
//                                  BufferedFile                                  //
//********************************************************************************//

type BufferedReader struct {
	File *os.File
	*bufio.Reader
}

type BufferedUnserializer interface {
	Unserialize(r *BufferedReader) (size int)
}

func OpenBufferedReader(path string, buffersize int, flag int) (*BufferedReader, error) {
	file, err := os.OpenFile(path, flag|os.O_RDONLY, 0)
	if err != nil {
		return nil, err
	}
	return &BufferedReader{file, bufio.NewReaderSize(file, buffersize)}, nil
}
func (b *BufferedReader) Seek(offset int64, whence int) (ret int64, err error) {
	ret, err = b.File.Seek(offset, whence)
	if err == nil {
		b.Reset(b.File)
	}
	return ret, err
}

type BufferedWriter struct {
	File *os.File
	*bufio.Writer
}

type BufferedSerializer interface {
	Serialize(w *BufferedWriter) (size int)
}

func OpenBufferedWriter(path string, buffersize int, flag int, perm os.FileMode) (*BufferedWriter, error) {
	file, err := os.OpenFile(path, flag|os.O_WRONLY, perm)
	if err != nil {
		return nil, err
	}
	return &BufferedWriter{file, bufio.NewWriterSize(file, buffersize)}, nil
}
func (b *BufferedWriter) Seek(offset int64, whence int) (ret int64, err error) {
	b.Flush() // Always flush in case we want to read what we have written
	ret, err = b.File.Seek(offset, whence)
	if err == nil {
		b.Reset(b.File)
	}
	return ret, err
}

type BufferedFile struct {
	Path       string
	BufferSize int
	Flag       int
	Perm       os.FileMode
	Reader     *BufferedReader
	Writer     *BufferedWriter
}

func OpenBufferedFile(path string, buffersize int, flag int, perm os.FileMode) (*BufferedFile, error) {
	b := &BufferedFile{Path: path, BufferSize: buffersize, Flag: flag, Perm: perm}
	var err error
	if b.Writer, err = OpenBufferedWriter(b.Path, b.BufferSize, b.Flag, b.Perm); err != nil {
		return nil, err
	}
	if b.Reader, err = OpenBufferedReader(b.Path, b.BufferSize, b.Flag); err != nil {
		return nil, err
	}
	return b, nil
}
func (b *BufferedFile) Size() int64 {
	b.Writer.Flush() // Always flush in case we want to read what we have written
	size, err := b.Writer.Seek(0, os.SEEK_END)
	if err != nil {
		panic(err)
	}
	return size
}
func (b *BufferedFile) Close() (err error) {
	b.Writer.Flush() // Always flush in case we want to read what we have written
	if e := b.Reader.File.Close(); e != nil {
		err = e
	}
	if e := b.Writer.File.Close(); e != nil {
		err = e
	}
	return err
}
<|endoftext|>


<commit_before>package dht

import (
	"bytes"
	"testing"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"

	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
	ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"

	ci "github.com/jbenet/go-ipfs/crypto"
	spipe "github.com/jbenet/go-ipfs/crypto/spipe"
	inet "github.com/jbenet/go-ipfs/net"
	mux "github.com/jbenet/go-ipfs/net/mux"
	netservice "github.com/jbenet/go-ipfs/net/service"
	peer "github.com/jbenet/go-ipfs/peer"
	u "github.com/jbenet/go-ipfs/util"

	"fmt"
	"time"
)

func setupDHT(ctx context.Context, t *testing.T, p *peer.Peer) *IpfsDHT {
	peerstore := peer.NewPeerstore()

	dhts := netservice.NewService(nil) // nil handler for now, need to patch it
	if err := dhts.Start(ctx); err != nil {
		t.Fatal(err)
	}

	net, err := inet.NewIpfsNetwork(ctx, p, peerstore, &mux.ProtocolMap{
		mux.ProtocolID_Routing: dhts,
	})
	if err != nil {
		t.Fatal(err)
	}

	d := NewDHT(ctx, p, peerstore, net, dhts, ds.NewMapDatastore())
	dhts.SetHandler(d)
	return d
}

func setupDHTS(ctx context.Context, n int, t *testing.T) ([]ma.Multiaddr, []*peer.Peer, []*IpfsDHT) {
	var addrs []ma.Multiaddr
	for i := 0; i < n; i++ {
		a, err := ma.NewMultiaddr(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", 5000+i))
		if err != nil {
			t.Fatal(err)
		}
		addrs = append(addrs, a)
	}

	var peers []*peer.Peer
	for i := 0; i < n; i++ {
		p := makePeer(addrs[i])
		peers = append(peers, p)
	}

	dhts := make([]*IpfsDHT, n)
	for i := 0; i < n; i++ {
		dhts[i] = setupDHT(ctx, t, peers[i])
	}

	return addrs, peers, dhts
}

func makePeer(addr ma.Multiaddr) *peer.Peer {
	p := new(peer.Peer)
	p.AddAddress(addr)
	sk, pk, err := ci.GenerateKeyPair(ci.RSA, 512)
	if err != nil {
		panic(err)
	}
	p.PrivKey = sk
	p.PubKey = pk
	id, err := spipe.IDFromPubKey(pk)
	if err != nil {
		panic(err)
	}

	p.ID = id
	return p
}

func TestPing(t *testing.T) {
	// t.Skip("skipping test to debug another")
	ctx := context.Background()
	u.Debug = false
	addrA, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/2222")
	if err != nil {
		t.Fatal(err)
	}
	addrB, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/5678")
	if err != nil {
		t.Fatal(err)
	}

	peerA := makePeer(addrA)
	peerB := makePeer(addrB)

	dhtA := setupDHT(ctx, t, peerA)
	dhtB := setupDHT(ctx, t, peerB)

	defer dhtA.Halt()
	defer dhtB.Halt()
	defer dhtA.network.Close()
	defer dhtB.network.Close()

	_, err = dhtA.Connect(ctx, peerB)
	if err != nil {
		t.Fatal(err)
	}

	//Test that we can ping the node
	ctxT, _ := context.WithTimeout(ctx, 5*time.Millisecond)
	err = dhtA.Ping(ctxT, peerB)
	if err != nil {
		t.Fatal(err)
	}

	ctxT, _ = context.WithTimeout(ctx, 5*time.Millisecond)
	err = dhtB.Ping(ctxT, peerA)
	if err != nil {
		t.Fatal(err)
	}
}

func TestValueGetSet(t *testing.T) {
	// t.Skip("skipping test to debug another")

	ctx := context.Background()
	u.Debug = false
	addrA, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/1235")
	if err != nil {
		t.Fatal(err)
	}
	addrB, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/5679")
	if err != nil {
		t.Fatal(err)
	}

	peerA := makePeer(addrA)
	peerB := makePeer(addrB)

	dhtA := setupDHT(ctx, t, peerA)
	dhtB := setupDHT(ctx, t, peerB)

	defer dhtA.Halt()
	defer dhtB.Halt()
	defer dhtA.network.Close()
	defer dhtB.network.Close()

	_, err = dhtA.Connect(ctx, peerB)
	if err != nil {
		t.Fatal(err)
	}

	ctxT, _ := context.WithTimeout(ctx, time.Second)
	dhtA.PutValue(ctxT, "hello", []byte("world"))

	ctxT, _ = context.WithTimeout(ctx, time.Second*2)
	val, err := dhtA.GetValue(ctxT, "hello")
	if err != nil {
		t.Fatal(err)
	}

	if string(val) != "world" {
		t.Fatalf("Expected 'world' got '%s'", string(val))
	}

	ctxT, _ = context.WithTimeout(ctx, time.Second*2)
	val, err = dhtB.GetValue(ctxT, "hello")
	if err != nil {
		t.Fatal(err)
	}

	if string(val) != "world" {
		t.Fatalf("Expected 'world' got '%s'", string(val))
	}
}

func TestProvides(t *testing.T) {
	// t.Skip("skipping test to debug another")
	ctx := context.Background()

	u.Debug = false

	_, peers, dhts := setupDHTS(ctx, 4, t)
	defer func() {
		for i := 0; i < 4; i++ {
			dhts[i].Halt()
			defer dhts[i].network.Close()
		}
	}()

	_, err := dhts[0].Connect(ctx, peers[1])
	if err != nil {
		t.Fatal(err)
	}

	_, err = dhts[1].Connect(ctx, peers[2])
	if err != nil {
		t.Fatal(err)
	}

	_, err = dhts[1].Connect(ctx, peers[3])
	if err != nil {
		t.Fatal(err)
	}

	err = dhts[3].putLocal(u.Key("hello"), []byte("world"))
	if err != nil {
		t.Fatal(err)
	}

	bits, err := dhts[3].getLocal(u.Key("hello"))
	if err != nil && bytes.Equal(bits, []byte("world")) {
		t.Fatal(err)
	}

	err = dhts[3].Provide(ctx, u.Key("hello"))
	if err != nil {
		t.Fatal(err)
	}

	time.Sleep(time.Millisecond * 60)

	ctxT, _ := context.WithTimeout(ctx, time.Second)
	provchan := dhts[0].FindProvidersAsync(ctxT, u.Key("hello"), 1)

	after := time.After(time.Second)
	select {
	case prov := <-provchan:
		if prov == nil {
			t.Fatal("Got back nil provider")
		}
	case <-after:
		t.Fatal("Did not get a provider back.")
	}
}

func TestProvidesAsync(t *testing.T) {
	// t.Skip("skipping test to debug another")

	ctx := context.Background()
	u.Debug = false

	_, peers, dhts := setupDHTS(ctx, 4, t)
	defer func() {
		for i := 0; i < 4; i++ {
			dhts[i].Halt()
			defer dhts[i].network.Close()
		}
	}()

	_, err := dhts[0].Connect(ctx, peers[1])
	if err != nil {
		t.Fatal(err)
	}

	_, err = dhts[1].Connect(ctx, peers[2])
	if err != nil {
		t.Fatal(err)
	}

	_, err = dhts[1].Connect(ctx, peers[3])
	if err != nil {
		t.Fatal(err)
	}

	err = dhts[3].putLocal(u.Key("hello"), []byte("world"))
	if err != nil {
		t.Fatal(err)
	}

	bits, err := dhts[3].getLocal(u.Key("hello"))
	if err != nil && bytes.Equal(bits, []byte("world")) {
		t.Fatal(err)
	}

	err = dhts[3].Provide(ctx, u.Key("hello"))
	if err != nil {
		t.Fatal(err)
	}

	time.Sleep(time.Millisecond * 60)

	ctxT, _ := context.WithTimeout(ctx, time.Millisecond*300)
	provs := dhts[0].FindProvidersAsync(ctxT, u.Key("hello"), 5)
	select {
	case p := <-provs:
		if !p.ID.Equal(dhts[3].self.ID) {
			t.Fatalf("got a provider, but not the right one. %s", p)
		}
	case <-ctxT.Done():
		t.Fatal("Didnt get back providers")
	}
}

func TestLayeredGet(t *testing.T) {
	// t.Skip("skipping test to debug another")

	ctx := context.Background()
	u.Debug = false
	_, peers, dhts := setupDHTS(ctx, 4, t)
	defer func() {
		for i := 0; i < 4; i++ {
			dhts[i].Halt()
			defer dhts[i].network.Close()
		}
	}()

	_, err := dhts[0].Connect(ctx, peers[1])
	if err != nil {
		t.Fatalf("Failed to connect: %s", err)
	}

	_, err = dhts[1].Connect(ctx, peers[2])
	if err != nil {
		t.Fatal(err)
	}

	_, err = dhts[1].Connect(ctx, peers[3])
	if err != nil {
		t.Fatal(err)
	}

	err = dhts[3].putLocal(u.Key("hello"), []byte("world"))
	if err != nil {
		t.Fatal(err)
	}

	err = dhts[3].Provide(ctx, u.Key("hello"))
	if err != nil {
		t.Fatal(err)
	}

	time.Sleep(time.Millisecond * 60)

	ctxT, _ := context.WithTimeout(ctx, time.Second)
	val, err := dhts[0].GetValue(ctxT, u.Key("hello"))
	if err != nil {
		t.Fatal(err)
	}

	if string(val) != "world" {
		t.Fatal("Got incorrect value.")
	}

}

func TestFindPeer(t *testing.T) {
	// t.Skip("skipping test to debug another")

	ctx := context.Background()
	u.Debug = false

	_, peers, dhts := setupDHTS(ctx, 4, t)
	defer func() {
		for i := 0; i < 4; i++ {
			dhts[i].Halt()
			dhts[i].network.Close()
		}
	}()

	_, err := dhts[0].Connect(ctx, peers[1])
	if err != nil {
		t.Fatal(err)
	}

	_, err = dhts[1].Connect(ctx, peers[2])
	if err != nil {
		t.Fatal(err)
	}

	_, err = dhts[1].Connect(ctx, peers[3])
	if err != nil {
		t.Fatal(err)
	}

	ctxT, _ := context.WithTimeout(ctx, time.Second)
	p, err := dhts[0].FindPeer(ctxT, peers[2].ID)
	if err != nil {
		t.Fatal(err)
	}

	if p == nil {
		t.Fatal("Failed to find peer.")
	}

	if !p.ID.Equal(peers[2].ID) {
		t.Fatal("Didnt find expected peer.")
	}
}

func TestConnectCollision(t *testing.T) {
	// t.Skip("skipping test to debug another")

	ctx := context.Background()
	u.Debug = false
	addrA, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/1235")
	if err != nil {
		t.Fatal(err)
	}
	addrB, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/5679")
	if err != nil {
		t.Fatal(err)
	}

	peerA := makePeer(addrA)
	peerB := makePeer(addrB)

	dhtA := setupDHT(ctx, t, peerA)
	dhtB := setupDHT(ctx, t, peerB)

	defer dhtA.Halt()
	defer dhtB.Halt()
	defer dhtA.network.Close()
	defer dhtB.network.Close()

	done := make(chan struct{})
	go func() {
		_, err = dhtA.Connect(ctx, peerB)
		if err != nil {
			t.Fatal(err)
		}
		done <- struct{}{}
	}()
	go func() {
		_, err = dhtB.Connect(ctx, peerA)
		if err != nil {
			t.Fatal(err)
		}
		done <- struct{}{}
	}()

	timeout := time.After(time.Second * 5)
	select {
	case <-done:
	case <-timeout:
		t.Fatal("Timeout received!")
	}
	select {
	case <-done:
	case <-timeout:
		t.Fatal("Timeout received!")
	}
}
<commit_msg>move IDFromPubKey to peer pkg<commit_after>package dht

import (
	"bytes"
	"testing"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"

	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
	ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"

	ci "github.com/jbenet/go-ipfs/crypto"
	inet "github.com/jbenet/go-ipfs/net"
	mux "github.com/jbenet/go-ipfs/net/mux"
	netservice "github.com/jbenet/go-ipfs/net/service"
	peer "github.com/jbenet/go-ipfs/peer"
	u "github.com/jbenet/go-ipfs/util"

	"fmt"
	"time"
)

func setupDHT(ctx context.Context, t *testing.T, p *peer.Peer) *IpfsDHT {
	peerstore := peer.NewPeerstore()

	dhts := netservice.NewService(nil) // nil handler for now, need to patch it
	if err := dhts.Start(ctx); err != nil {
		t.Fatal(err)
	}

	net, err := inet.NewIpfsNetwork(ctx, p, peerstore, &mux.ProtocolMap{
		mux.ProtocolID_Routing: dhts,
	})
	if err != nil {
		t.Fatal(err)
	}

	d := NewDHT(ctx, p, peerstore, net, dhts, ds.NewMapDatastore())
	dhts.SetHandler(d)
	return d
}

func setupDHTS(ctx context.Context, n int, t *testing.T) ([]ma.Multiaddr, []*peer.Peer, []*IpfsDHT) {
	var addrs []ma.Multiaddr
	for i := 0; i < n; i++ {
		a, err := ma.NewMultiaddr(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", 5000+i))
		if err != nil {
			t.Fatal(err)
		}
		addrs = append(addrs, a)
	}

	var peers []*peer.Peer
	for i := 0; i < n; i++ {
		p := makePeer(addrs[i])
		peers = append(peers, p)
	}

	dhts := make([]*IpfsDHT, n)
	for i := 0; i < n; i++ {
		dhts[i] = setupDHT(ctx, t, peers[i])
	}

	return addrs, peers, dhts
}

func makePeer(addr ma.Multiaddr) *peer.Peer {
	p := new(peer.Peer)
	p.AddAddress(addr)
	sk, pk, err := ci.GenerateKeyPair(ci.RSA, 512)
	if err != nil {
		panic(err)
	}
	p.PrivKey = sk
	p.PubKey = pk
	id, err := peer.IDFromPubKey(pk)
	if err != nil {
		panic(err)
	}

	p.ID = id
	return p
}

func TestPing(t *testing.T) {
	// t.Skip("skipping test to debug another")
	ctx := context.Background()
	u.Debug = false
	addrA, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/2222")
	if err != nil {
		t.Fatal(err)
	}
	addrB, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/5678")
	if err != nil {
		t.Fatal(err)
	}

	peerA := makePeer(addrA)
	peerB := makePeer(addrB)

	dhtA := setupDHT(ctx, t, peerA)
	dhtB := setupDHT(ctx, t, peerB)

	defer dhtA.Halt()
	defer dhtB.Halt()
	defer dhtA.network.Close()
	defer dhtB.network.Close()

	_, err = dhtA.Connect(ctx, peerB)
	if err != nil {
		t.Fatal(err)
	}

	//Test that we can ping the node
	ctxT, _ := context.WithTimeout(ctx, 5*time.Millisecond)
	err = dhtA.Ping(ctxT, peerB)
	if err != nil {
		t.Fatal(err)
	}

	ctxT, _ = context.WithTimeout(ctx, 5*time.Millisecond)
	err = dhtB.Ping(ctxT, peerA)
	if err != nil {
		t.Fatal(err)
	}
}

func TestValueGetSet(t *testing.T) {
	// t.Skip("skipping test to debug another")

	ctx := context.Background()
	u.Debug = false
	addrA, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/1235")
	if err != nil {
		t.Fatal(err)
	}
	addrB, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/5679")
	if err != nil {
		t.Fatal(err)
	}

	peerA := makePeer(addrA)
	peerB := makePeer(addrB)

	dhtA := setupDHT(ctx, t, peerA)
	dhtB := setupDHT(ctx, t, peerB)

	defer dhtA.Halt()
	defer dhtB.Halt()
	defer dhtA.network.Close()
	defer dhtB.network.Close()

	_, err = dhtA.Connect(ctx, peerB)
	if err != nil {
		t.Fatal(err)
	}

	ctxT, _ := context.WithTimeout(ctx, time.Second)
	dhtA.PutValue(ctxT, "hello", []byte("world"))

	ctxT, _ = context.WithTimeout(ctx, time.Second*2)
	val, err := dhtA.GetValue(ctxT, "hello")
	if err != nil {
		t.Fatal(err)
	}

	if string(val) != "world" {
		t.Fatalf("Expected 'world' got '%s'", string(val))
	}

	ctxT, _ = context.WithTimeout(ctx, time.Second*2)
	val, err = dhtB.GetValue(ctxT, "hello")
	if err != nil {
		t.Fatal(err)
	}

	if string(val) != "world" {
		t.Fatalf("Expected 'world' got '%s'", string(val))
	}
}

func TestProvides(t *testing.T) {
	// t.Skip("skipping test to debug another")
	ctx := context.Background()

	u.Debug = false

	_, peers, dhts := setupDHTS(ctx, 4, t)
	defer func() {
		for i := 0; i < 4; i++ {
			dhts[i].Halt()
			defer dhts[i].network.Close()
		}
	}()

	_, err := dhts[0].Connect(ctx, peers[1])
	if err != nil {
		t.Fatal(err)
	}

	_, err = dhts[1].Connect(ctx, peers[2])
	if err != nil {
		t.Fatal(err)
	}

	_, err = dhts[1].Connect(ctx, peers[3])
	if err != nil {
		t.Fatal(err)
	}

	err = dhts[3].putLocal(u.Key("hello"), []byte("world"))
	if err != nil {
		t.Fatal(err)
	}

	bits, err := dhts[3].getLocal(u.Key("hello"))
	if err != nil && bytes.Equal(bits, []byte("world")) {
		t.Fatal(err)
	}

	err = dhts[3].Provide(ctx, u.Key("hello"))
	if err != nil {
		t.Fatal(err)
	}

	time.Sleep(time.Millisecond * 60)

	ctxT, _ := context.WithTimeout(ctx, time.Second)
	provchan := dhts[0].FindProvidersAsync(ctxT, u.Key("hello"), 1)

	after := time.After(time.Second)
	select {
	case prov := <-provchan:
		if prov == nil {
			t.Fatal("Got back nil provider")
		}
	case <-after:
		t.Fatal("Did not get a provider back.")
	}
}

func TestProvidesAsync(t *testing.T) {
	// t.Skip("skipping test to debug another")

	ctx := context.Background()
	u.Debug = false

	_, peers, dhts := setupDHTS(ctx, 4, t)
	defer func() {
		for i := 0; i < 4; i++ {
			dhts[i].Halt()
			defer dhts[i].network.Close()
		}
	}()

	_, err := dhts[0].Connect(ctx, peers[1])
	if err != nil {
		t.Fatal(err)
	}

	_, err = dhts[1].Connect(ctx, peers[2])
	if err != nil {
		t.Fatal(err)
	}

	_, err = dhts[1].Connect(ctx, peers[3])
	if err != nil {
		t.Fatal(err)
	}

	err = dhts[3].putLocal(u.Key("hello"), []byte("world"))
	if err != nil {
		t.Fatal(err)
	}

	bits, err := dhts[3].getLocal(u.Key("hello"))
	if err != nil && bytes.Equal(bits, []byte("world")) {
		t.Fatal(err)
	}

	err = dhts[3].Provide(ctx, u.Key("hello"))
	if err != nil {
		t.Fatal(err)
	}

	time.Sleep(time.Millisecond * 60)

	ctxT, _ := context.WithTimeout(ctx, time.Millisecond*300)
	provs := dhts[0].FindProvidersAsync(ctxT, u.Key("hello"), 5)
	select {
	case p := <-provs:
		if !p.ID.Equal(dhts[3].self.ID) {
			t.Fatalf("got a provider, but not the right one. %s", p)
		}
	case <-ctxT.Done():
		t.Fatal("Didnt get back providers")
	}
}

func TestLayeredGet(t *testing.T) {
	// t.Skip("skipping test to debug another")

	ctx := context.Background()
	u.Debug = false
	_, peers, dhts := setupDHTS(ctx, 4, t)
	defer func() {
		for i := 0; i < 4; i++ {
			dhts[i].Halt()
			defer dhts[i].network.Close()
		}
	}()

	_, err := dhts[0].Connect(ctx, peers[1])
	if err != nil {
		t.Fatalf("Failed to connect: %s", err)
	}

	_, err = dhts[1].Connect(ctx, peers[2])
	if err != nil {
		t.Fatal(err)
	}

	_, err = dhts[1].Connect(ctx, peers[3])
	if err != nil {
		t.Fatal(err)
	}

	err = dhts[3].putLocal(u.Key("hello"), []byte("world"))
	if err != nil {
		t.Fatal(err)
	}

	err = dhts[3].Provide(ctx, u.Key("hello"))
	if err != nil {
		t.Fatal(err)
	}

	time.Sleep(time.Millisecond * 60)

	ctxT, _ := context.WithTimeout(ctx, time.Second)
	val, err := dhts[0].GetValue(ctxT, u.Key("hello"))
	if err != nil {
		t.Fatal(err)
	}

	if string(val) != "world" {
		t.Fatal("Got incorrect value.")
	}

}

func TestFindPeer(t *testing.T) {
	// t.Skip("skipping test to debug another")

	ctx := context.Background()
	u.Debug = false

	_, peers, dhts := setupDHTS(ctx, 4, t)
	defer func() {
		for i := 0; i < 4; i++ {
			dhts[i].Halt()
			dhts[i].network.Close()
		}
	}()

	_, err := dhts[0].Connect(ctx, peers[1])
	if err != nil {
		t.Fatal(err)
	}

	_, err = dhts[1].Connect(ctx, peers[2])
	if err != nil {
		t.Fatal(err)
	}

	_, err = dhts[1].Connect(ctx, peers[3])
	if err != nil {
		t.Fatal(err)
	}

	ctxT, _ := context.WithTimeout(ctx, time.Second)
	p, err := dhts[0].FindPeer(ctxT, peers[2].ID)
	if err != nil {
		t.Fatal(err)
	}

	if p == nil {
		t.Fatal("Failed to find peer.")
	}

	if !p.ID.Equal(peers[2].ID) {
		t.Fatal("Didnt find expected peer.")
	}
}

func TestConnectCollision(t *testing.T) {
	// t.Skip("skipping test to debug another")

	ctx := context.Background()
	u.Debug = false
	addrA, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/1235")
	if err != nil {
		t.Fatal(err)
	}
	addrB, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/5679")
	if err != nil {
		t.Fatal(err)
	}

	peerA := makePeer(addrA)
	peerB := makePeer(addrB)

	dhtA := setupDHT(ctx, t, peerA)
	dhtB := setupDHT(ctx, t, peerB)

	defer dhtA.Halt()
	defer dhtB.Halt()
	defer dhtA.network.Close()
	defer dhtB.network.Close()

	done := make(chan struct{})
	go func() {
		_, err = dhtA.Connect(ctx, peerB)
		if err != nil {
			t.Fatal(err)
		}
		done <- struct{}{}
	}()
	go func() {
		_, err = dhtB.Connect(ctx, peerA)
		if err != nil {
			t.Fatal(err)
		}
		done <- struct{}{}
	}()

	timeout := time.After(time.Second * 5)
	select {
	case <-done:
	case <-timeout:
		t.Fatal("Timeout received!")
	}
	select {
	case <-done:
	case <-timeout:
		t.Fatal("Timeout received!")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/zenazn/goji/web"
	"gopkg.in/mgo.v2"
)

var AddDeviceMissingDeviceIdError = ErrorJson{"POST to /devices must be a JSON with a DeviceId property."}

type SidewinderDirector struct {
	MongoDB          string
	session          *mgo.Session
	ApnsCommunicator *APNSCommunicator
}

func NewSidewinderDirector(mongoDB string, communicator *APNSCommunicator) (*SidewinderDirector, error) {
	session, err := mgo.Dial("mongo,localhost")
	if err != nil {
		return nil, err
	}
	return &SidewinderDirector{mongoDB, session, communicator}, nil
}

func (self *SidewinderDirector) Store() *SidewinderStore {
	return &SidewinderStore{self.MongoDB, self.session.Copy()}
}

func (self *SidewinderDirector) DatastoreInfo(context web.C, writer http.ResponseWriter, request *http.Request) error {
	session := self.Store().session

	buildInfo, err := session.BuildInfo()
	if err != nil {
		return fmt.Errorf("Could not connect to MongoDB.\n%v", err.Error())
	}
	writer.Header().Set("Content-Type", "application/json")

	databases, err := session.DatabaseNames()
	if err != nil {
		return fmt.Errorf("Could not retrieve database names.\n%v", err.Error())
	}

	dataStoreInfo := DatastoreInfo{buildInfo, session.LiveServers(), databases}

	return json.NewEncoder(writer).Encode(&dataStoreInfo)
}

func (self *SidewinderDirector) postDevice(context web.C, writer http.ResponseWriter, request *http.Request) error {
	sentJSON := decodeDeviceDocument(request)
	if sentJSON == nil {
		return writeJson(400, AddDeviceMissingDeviceIdError, writer)
	}
	recordWasCreated, err := self.Store().AddDevice(sentJSON.DeviceId)
	if err != nil {
		return err
	} else if recordWasCreated {
		return writeJson(201, sentJSON, writer)
	} else {
		return writeJson(200, sentJSON, writer)
	}
}

func decodeDeviceDocument(request *http.Request) *DeviceDocument {
	var sentJSON DeviceDocument
	if decodeErr := json.NewDecoder(request.Body).Decode(&sentJSON); decodeErr == nil && sentJSON.DeviceId != "" {
		return &sentJSON
	} else {
		return nil
	}
}

type DeviceHandler func(id string, writer http.ResponseWriter, request *http.Request) error

func (self DeviceHandler) ServeHTTPC(context web.C, writer http.ResponseWriter, request *http.Request) {
	deviceId := context.URLParams["id"]
	err := self(deviceId, writer, request)
	if err != nil {
		writeJson(500, ErrorJson{err.Error()}, writer)
	}
}

func (self *SidewinderDirector) deleteDevice(deviceId string, writer http.ResponseWriter, request *http.Request) error {
	result, err := self.Store().FindDevice(deviceId)
	if err != nil {
		return err
	}

	if err := self.Store().DeleteDevice(deviceId); err != nil {
		return err
	}

	return writeJson(200, result, writer)
}

func (self *SidewinderDirector) PostNotification(deviceId string, writer http.ResponseWriter, request *http.Request) error {
	var notification map[string]string
	if decodeErr := json.NewDecoder(request.Body).Decode(&notification); decodeErr != nil {
		return decodeErr
	}

	if err := self.ApnsCommunicator.sendPushNotification(deviceId, notification["Alert"]); err != nil {
		return err
	}
	return writeJson(201, notification, writer)
}

func (self *SidewinderDirector) CircleNotify(context web.C, writer http.ResponseWriter, request *http.Request) error {
	var notification interface{}
	if decodeErr := json.NewDecoder(request.Body).Decode(&notification); decodeErr != nil {
		fmt.Printf("ERROR:  %v\n", decodeErr.Error())
		return decodeErr
	}
	fmt.Printf("Sent body:\n%v\n", notification)
	return nil
}
<commit_msg>Trying to write it out as JSON to log, because that'll be easier to read. I hope.<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"os"

	"github.com/zenazn/goji/web"
	"gopkg.in/mgo.v2"
)

var AddDeviceMissingDeviceIdError = ErrorJson{"POST to /devices must be a JSON with a DeviceId property."}

type SidewinderDirector struct {
	MongoDB          string
	session          *mgo.Session
	ApnsCommunicator *APNSCommunicator
}

func NewSidewinderDirector(mongoDB string, communicator *APNSCommunicator) (*SidewinderDirector, error) {
	session, err := mgo.Dial("mongo,localhost")
	if err != nil {
		return nil, err
	}
	return &SidewinderDirector{mongoDB, session, communicator}, nil
}

func (self *SidewinderDirector) Store() *SidewinderStore {
	return &SidewinderStore{self.MongoDB, self.session.Copy()}
}

func (self *SidewinderDirector) DatastoreInfo(context web.C, writer http.ResponseWriter, request *http.Request) error {
	session := self.Store().session

	buildInfo, err := session.BuildInfo()
	if err != nil {
		return fmt.Errorf("Could not connect to MongoDB.\n%v", err.Error())
	}
	writer.Header().Set("Content-Type", "application/json")

	databases, err := session.DatabaseNames()
	if err != nil {
		return fmt.Errorf("Could not retrieve database names.\n%v", err.Error())
	}

	dataStoreInfo := DatastoreInfo{buildInfo, session.LiveServers(), databases}

	return json.NewEncoder(writer).Encode(&dataStoreInfo)
}

func (self *SidewinderDirector) postDevice(context web.C, writer http.ResponseWriter, request *http.Request) error {
	sentJSON := decodeDeviceDocument(request)
	if sentJSON == nil {
		return writeJson(400, AddDeviceMissingDeviceIdError, writer)
	}
	recordWasCreated, err := self.Store().AddDevice(sentJSON.DeviceId)
	if err != nil {
		return err
	} else if recordWasCreated {
		return writeJson(201, sentJSON, writer)
	} else {
		return writeJson(200, sentJSON, writer)
	}
}

func decodeDeviceDocument(request *http.Request) *DeviceDocument {
	var sentJSON DeviceDocument
	if decodeErr := json.NewDecoder(request.Body).Decode(&sentJSON); decodeErr == nil && sentJSON.DeviceId != "" {
		return &sentJSON
	} else {
		return nil
	}
}

type DeviceHandler func(id string, writer http.ResponseWriter, request *http.Request) error

func (self DeviceHandler) ServeHTTPC(context web.C, writer http.ResponseWriter, request *http.Request) {
	deviceId := context.URLParams["id"]
	err := self(deviceId, writer, request)
	if err != nil {
		writeJson(500, ErrorJson{err.Error()}, writer)
	}
}

func (self *SidewinderDirector) deleteDevice(deviceId string, writer http.ResponseWriter, request *http.Request) error {
	result, err := self.Store().FindDevice(deviceId)
	if err != nil {
		return err
	}

	if err := self.Store().DeleteDevice(deviceId); err != nil {
		return err
	}

	return writeJson(200, result, writer)
}

func (self *SidewinderDirector) PostNotification(deviceId string, writer http.ResponseWriter, request *http.Request) error {
	var notification map[string]string
	if decodeErr := json.NewDecoder(request.Body).Decode(&notification); decodeErr != nil {
		return decodeErr
	}

	if err := self.ApnsCommunicator.sendPushNotification(deviceId, notification["Alert"]); err != nil {
		return err
	}
	return writeJson(201, notification, writer)
}

func (self *SidewinderDirector) CircleNotify(context web.C, writer http.ResponseWriter, request *http.Request) error {
	var notification interface{}
	if decodeErr := json.NewDecoder(request.Body).Decode(&notification); decodeErr != nil {
		fmt.Printf("ERROR:  %v\n", decodeErr.Error())
		return decodeErr
	}
	return json.NewEncoder(os.Stdout).Encode(notification)
}
<|endoftext|>


<commit_before>package main

import (
	"./dhcp"
	"bytes"
	"encoding/binary"
	"flag"
	"fmt"
	"net"
	"os"
	"time"
)

type option struct {
	Len  int
	Name string
}

var options map[byte]option

func init() {
	options = map[byte]option{
		dhcp.PadOption:          {0, "Pad Option"},
		dhcp.Router:             {-1, "Router"},
		dhcp.SubnetMask:         {4, "Subnet Mask"},
		dhcp.DomainNameServer:   {-1, "Domain Name Server"},
		dhcp.HostName:           {-1, "Host Name"},
		dhcp.DomainName:         {-1, "Domain Name"},
		dhcp.BroadcastAddress:   {4, "Broadcast Address"},
		dhcp.StaticRoute:        {-1, "Static Route"},
		dhcp.IPAddressLeaseTime: {4, "IP Address Lease Time"},
		dhcp.DHCPMessageType:    {1, "DHCP Message Type"},
		dhcp.ServerIdentifier:   {4, "Server Identifier"},
		dhcp.RenewalTimeValue:   {4, "Renewal Time Value"},
		dhcp.RebindingTimeValue: {4, "Rebinding Time Value"},
		dhcp.VendorSpecific:     {-1, "Vendor Specific"},
		dhcp.NetBIOSNameServer:  {-1, "NetBIOS Name Server"},
		dhcp.DomainSearch:       {-1, "Domain Search"},
		dhcp.WebProxyServer:     {-1, "Web Proxy Server"},
	}
}

func b32(data []byte) uint32 {
	buf := bytes.NewBuffer(data)
	var x uint32
	binary.Read(buf, binary.BigEndian, &x)
	return x
}

func ip4(data []byte) string {
	var ip dhcp.IPv4Address
	copy(ip[:], data[0:4])
	return ip.String()
}

func parseOptions(p *dhcp.Packet) {
	opts := p.Options
	fmt.Println("Options:")
loop:
	for i := 0; i < len(opts); i++ {
		o := opts[i]

		switch o {
		case dhcp.EndOption:
			fmt.Print("End Option")
			break loop
		case dhcp.PadOption:
			continue
		}

		length := int(opts[i+1])
		_, ok := options[o]
		if ok && options[o].Len >= 0 && options[o].Len != length {
			fmt.Printf("corrupted option (%d,%d)\n",
				options[o].Len, length)
			break loop
		}

		if name := options[o].Name; name != "" {
			fmt.Printf("%24s : ", options[o].Name)
		} else {
			fmt.Printf("%24d : ", o)
		}

		switch o {
		case dhcp.DHCPMessageType:
			t := opts[i+2]
			fmt.Print(t)
			break
		case dhcp.Router, dhcp.DomainNameServer, dhcp.NetBIOSNameServer:
			// Multiple IP addresses
			for n := 0; n < length; n += 4 {
				fmt.Print(ip4(opts[i+2+n:i+6+n]), " ")
			}
		case dhcp.ServerIdentifier, dhcp.SubnetMask, dhcp.BroadcastAddress:
			// Single IP address
			fmt.Print(ip4(opts[i+2:]))
			break
		case dhcp.IPAddressLeaseTime, dhcp.RenewalTimeValue, dhcp.RebindingTimeValue:
			// 32-bit integer
			fmt.Print(b32(opts[i+2:]))
			break
		case dhcp.HostName, dhcp.DomainName, dhcp.WebProxyServer:
			// String
			fmt.Print(string(opts[i+2 : i+2+length]))
			break
		case dhcp.DomainSearch:
			// Compressed domain names (RFC 1035)
			fmt.Print("[TODO RFC 1035 section 4.1.4]")
			break
		case dhcp.VendorSpecific:
			// Size only
			fmt.Printf("(%d bytes)", length)
			break
		}
		fmt.Println()

		i += 1 + length
	}
}

func showPacket(p *dhcp.Packet) {
	fmt.Println("Client IP address :", p.Ciaddr.String())
	fmt.Println("Your IP address   :", p.Yiaddr.String())
	fmt.Println("Server IP address :", p.Siaddr.String())
	fmt.Println("Relay IP address  :", p.Giaddr.String())
	parseOptions(p)
	fmt.Println()
}

func checkError(err error) {
	if err != nil {
		fmt.Fprintf(os.Stderr, "%s\n", err.Error())
		os.Exit(1)
	}
}

func getMAC(s string) (string, error) {
	ifaces, err := net.Interfaces()
	checkError(err)
	for _, i := range ifaces {
		if i.Name == s {
			return i.HardwareAddr.String(), nil
		}
	}
	return "", fmt.Errorf("%s: no such interface", s)
}

func usage() {
	fmt.Fprintf(os.Stderr, "usage: %s [options]\n", os.Args[0])
	flag.PrintDefaults()
}

func main() {
	var iface string
	var secs int

	flag.StringVar(&iface, "i", "", "network `interface` to use")
	flag.IntVar(&secs, "t", 5, "timeout in seconds")
	flag.Parse()

	if iface == "" {
		usage()
		os.Exit(1)
	}

	mac := ""
	timeout := time.Duration(secs) * time.Second

	mac, err := getMAC(iface)
	checkError(err)

	fmt.Printf("Interface: %s [%s]\n", iface, mac)

	// Set up server
	addr, err := net.ResolveUDPAddr("udp4", ":68")
	checkError(err)
	conn, err := net.ListenUDP("udp4", addr)
	checkError(err)
	defer conn.Close()

	// Send discover packet
	p := dhcp.NewDiscoverPacket()
	p.ParseMAC(mac)

	fmt.Println("\n>>> Send DHCP discover")
	showPacket(&p.Packet)
	err = p.Send()
	checkError(err)

	t := time.Now()
	for time.Since(t) < timeout {
		var o dhcp.Packet
		remote, err := o.Receive(conn, timeout)
		if err != nil {
			fmt.Fprintf(os.Stderr, "%s\n", err.Error())
			break
		}
		fmt.Println("\n<<< Receive DHCP offer from", remote.IP.String())
		showPacket(&o)
	}
	fmt.Println("No more offers.")
}
<commit_msg>Display message type names<commit_after>package main

import (
	"./dhcp"
	"bytes"
	"encoding/binary"
	"flag"
	"fmt"
	"net"
	"os"
	"time"
)

type option struct {
	Len  int
	Name string
}

var options map[byte]option
var messageType map[byte]string

func init() {
	options = map[byte]option{
		dhcp.PadOption:          {0, "Pad Option"},
		dhcp.Router:             {-1, "Router"},
		dhcp.SubnetMask:         {4, "Subnet Mask"},
		dhcp.DomainNameServer:   {-1, "Domain Name Server"},
		dhcp.HostName:           {-1, "Host Name"},
		dhcp.DomainName:         {-1, "Domain Name"},
		dhcp.BroadcastAddress:   {4, "Broadcast Address"},
		dhcp.StaticRoute:        {-1, "Static Route"},
		dhcp.IPAddressLeaseTime: {4, "IP Address Lease Time"},
		dhcp.DHCPMessageType:    {1, "DHCP Message Type"},
		dhcp.ServerIdentifier:   {4, "Server Identifier"},
		dhcp.RenewalTimeValue:   {4, "Renewal Time Value"},
		dhcp.RebindingTimeValue: {4, "Rebinding Time Value"},
		dhcp.VendorSpecific:     {-1, "Vendor Specific"},
		dhcp.NetBIOSNameServer:  {-1, "NetBIOS Name Server"},
		dhcp.DomainSearch:       {-1, "Domain Search"},
		dhcp.WebProxyServer:     {-1, "Web Proxy Server"},
	}

	messageType = map[byte]string{
		dhcp.DHCPDiscover: "DHCPDISCOVER",
		dhcp.DHCPOffer:    "DHCPOFFER",
		dhcp.DHCPRequest:  "DHCPREQUEST",
		dhcp.DHCPDecline:  "DHCPDECLINE",
		dhcp.DHCPAck:      "DHCPACK",
		dhcp.DHCPNack:     "DHCPNACK",
		dhcp.DHCPRelease:  "DHCPRELEASE",
	}
}

func b32(data []byte) uint32 {
	buf := bytes.NewBuffer(data)
	var x uint32
	binary.Read(buf, binary.BigEndian, &x)
	return x
}

func ip4(data []byte) string {
	var ip dhcp.IPv4Address
	copy(ip[:], data[0:4])
	return ip.String()
}

func parseOptions(p *dhcp.Packet) {
	opts := p.Options
	fmt.Println("Options:")
loop:
	for i := 0; i < len(opts); i++ {
		o := opts[i]

		switch o {
		case dhcp.EndOption:
			fmt.Print("End Option")
			break loop
		case dhcp.PadOption:
			continue
		}

		length := int(opts[i+1])
		_, ok := options[o]
		if ok && options[o].Len >= 0 && options[o].Len != length {
			fmt.Printf("corrupted option (%d,%d)\n",
				options[o].Len, length)
			break loop
		}

		if name := options[o].Name; name != "" {
			fmt.Printf("%24s : ", options[o].Name)
		} else {
			fmt.Printf("%24d : ", o)
		}

		switch o {
		case dhcp.DHCPMessageType:
			fmt.Print(messageType[opts[i+2]])
			break
		case dhcp.Router, dhcp.DomainNameServer, dhcp.NetBIOSNameServer:
			// Multiple IP addresses
			for n := 0; n < length; n += 4 {
				fmt.Print(ip4(opts[i+2+n:i+6+n]), " ")
			}
		case dhcp.ServerIdentifier, dhcp.SubnetMask, dhcp.BroadcastAddress:
			// Single IP address
			fmt.Print(ip4(opts[i+2:]))
			break
		case dhcp.IPAddressLeaseTime, dhcp.RenewalTimeValue, dhcp.RebindingTimeValue:
			// 32-bit integer
			fmt.Print(b32(opts[i+2:]))
			break
		case dhcp.HostName, dhcp.DomainName, dhcp.WebProxyServer:
			// String
			fmt.Print(string(opts[i+2 : i+2+length]))
			break
		case dhcp.DomainSearch:
			// Compressed domain names (RFC 1035)
			fmt.Print("[TODO RFC 1035 section 4.1.4]")
			break
		case dhcp.VendorSpecific:
			// Size only
			fmt.Printf("(%d bytes)", length)
			break
		}
		fmt.Println()

		i += 1 + length
	}
}

func showPacket(p *dhcp.Packet) {
	fmt.Println("Client IP address :", p.Ciaddr.String())
	fmt.Println("Your IP address   :", p.Yiaddr.String())
	fmt.Println("Server IP address :", p.Siaddr.String())
	fmt.Println("Relay IP address  :", p.Giaddr.String())
	parseOptions(p)
	fmt.Println()
}

func checkError(err error) {
	if err != nil {
		fmt.Fprintf(os.Stderr, "%s\n", err.Error())
		os.Exit(1)
	}
}

func getMAC(s string) (string, error) {
	ifaces, err := net.Interfaces()
	checkError(err)
	for _, i := range ifaces {
		if i.Name == s {
			return i.HardwareAddr.String(), nil
		}
	}
	return "", fmt.Errorf("%s: no such interface", s)
}

func usage() {
	fmt.Fprintf(os.Stderr, "usage: %s [options]\n", os.Args[0])
	flag.PrintDefaults()
}

func main() {
	var iface string
	var secs int

	flag.StringVar(&iface, "i", "", "network `interface` to use")
	flag.IntVar(&secs, "t", 5, "timeout in seconds")
	flag.Parse()

	if iface == "" {
		usage()
		os.Exit(1)
	}

	mac := ""
	timeout := time.Duration(secs) * time.Second

	mac, err := getMAC(iface)
	checkError(err)

	fmt.Printf("Interface: %s [%s]\n", iface, mac)

	// Set up server
	addr, err := net.ResolveUDPAddr("udp4", ":68")
	checkError(err)
	conn, err := net.ListenUDP("udp4", addr)
	checkError(err)
	defer conn.Close()

	// Send discover packet
	p := dhcp.NewDiscoverPacket()
	p.ParseMAC(mac)

	fmt.Println("\n>>> Send DHCP discover")
	showPacket(&p.Packet)
	err = p.Send()
	checkError(err)

	t := time.Now()
	for time.Since(t) < timeout {
		var o dhcp.Packet
		remote, err := o.Receive(conn, timeout)
		if err != nil {
			fmt.Fprintf(os.Stderr, "%s\n", err.Error())
			break
		}
		fmt.Println("\n<<< Receive DHCP offer from", remote.IP.String())
		showPacket(&o)
	}
	fmt.Println("No more offers.")
}
<|endoftext|>


<commit_before>package bslack

import (
	"context"
	"fmt"
	"regexp"
	"strings"
	"time"

	"github.com/42wim/matterbridge/bridge/config"
	"github.com/nlopes/slack"
)

func (b *Bslack) getUser(id string) *slack.User {
	b.usersMutex.RLock()
	defer b.usersMutex.RUnlock()

	return b.users[id]
}

func (b *Bslack) getUsername(id string) string {
	if user := b.getUser(id); user != nil {
		if user.Profile.DisplayName != "" {
			return user.Profile.DisplayName
		}
		return user.Name
	}
	b.Log.Warnf("Could not find user with ID '%s'", id)
	return ""
}

func (b *Bslack) getAvatar(id string) string {
	if user := b.getUser(id); user != nil {
		return user.Profile.Image48
	}
	return ""
}

func (b *Bslack) getChannel(channel string) (*slack.Channel, error) {
	if strings.HasPrefix(channel, "ID:") {
		return b.getChannelByID(strings.TrimPrefix(channel, "ID:"))
	}
	return b.getChannelByName(channel)
}

func (b *Bslack) getChannelByName(name string) (*slack.Channel, error) {
	return b.getChannelBy(name, b.channelsByName)
}

func (b *Bslack) getChannelByID(ID string) (*slack.Channel, error) {
	return b.getChannelBy(ID, b.channelsByID)
}

func (b *Bslack) getChannelBy(lookupKey string, lookupMap map[string]*slack.Channel) (*slack.Channel, error) {
	b.channelsMutex.RLock()
	defer b.channelsMutex.RUnlock()

	if channel, ok := lookupMap[lookupKey]; ok {
		return channel, nil
	}
	return nil, fmt.Errorf("%s: channel %s not found", b.Account, lookupKey)
}

const minimumRefreshInterval = 10 * time.Second

func (b *Bslack) populateUsers() {
	b.refreshMutex.Lock()
	if time.Now().Before(b.earliestUserRefresh) || b.refreshInProgress {
		b.Log.Debugf("Not refreshing user list as it was done less than %v ago.",
			minimumRefreshInterval)
		b.refreshMutex.Unlock()

		return
	}
	b.refreshInProgress = true
	b.refreshMutex.Unlock()

	newUsers := map[string]*slack.User{}
	pagination := b.sc.GetUsersPaginated(slack.GetUsersOptionLimit(200))
	for {
		var err error
		pagination, err = pagination.Next(context.Background())
		if err != nil {
			if pagination.Done(err) {
				break
			}

			if err = b.handleRateLimit(err); err != nil {
				b.Log.Errorf("Could not retrieve users: %#v", err)
				return
			}
			continue
		}

		for i := range pagination.Users {
			newUsers[pagination.Users[i].ID] = &pagination.Users[i]
		}
	}

	b.usersMutex.Lock()
	defer b.usersMutex.Unlock()
	b.users = newUsers

	b.refreshMutex.Lock()
	defer b.refreshMutex.Unlock()
	b.earliestUserRefresh = time.Now().Add(minimumRefreshInterval)
	b.refreshInProgress = false
}

func (b *Bslack) populateChannels() {
	b.refreshMutex.Lock()
	if time.Now().Before(b.earliestChannelRefresh) || b.refreshInProgress {
		b.Log.Debugf("Not refreshing channel list as it was done less than %v seconds ago.",
			minimumRefreshInterval)
		b.refreshMutex.Unlock()
		return
	}
	b.refreshInProgress = true
	b.refreshMutex.Unlock()

	newChannelsByID := map[string]*slack.Channel{}
	newChannelsByName := map[string]*slack.Channel{}

	// We only retrieve public and private channels, not IMs
	// and MPIMs as those do not have a channel name.
	queryParams := &slack.GetConversationsParameters{
		ExcludeArchived: "true",
		Types:           []string{"public_channel,private_channel"},
	}
	for {
		channels, nextCursor, err := b.sc.GetConversations(queryParams)
		if err != nil {
			if err = b.handleRateLimit(err); err != nil {
				b.Log.Errorf("Could not retrieve channels: %#v", err)
				return
			}
			continue
		}

		for i := range channels {
			newChannelsByID[channels[i].ID] = &channels[i]
			newChannelsByName[channels[i].Name] = &channels[i]
		}
		if nextCursor == "" {
			break
		}
		queryParams.Cursor = nextCursor
	}

	b.channelsMutex.Lock()
	defer b.channelsMutex.Unlock()
	b.channelsByID = newChannelsByID
	b.channelsByName = newChannelsByName

	b.refreshMutex.Lock()
	defer b.refreshMutex.Unlock()
	b.earliestChannelRefresh = time.Now().Add(minimumRefreshInterval)
	b.refreshInProgress = false
}

// populateReceivedMessage shapes the initial Matterbridge message that we will forward to the
// router before we apply message-dependent modifications.
func (b *Bslack) populateReceivedMessage(ev *slack.MessageEvent) (*config.Message, error) {
	// Use our own func because rtm.GetChannelInfo doesn't work for private channels.
	channel, err := b.getChannelByID(ev.Channel)
	if err != nil {
		return nil, err
	}

	rmsg := &config.Message{
		Text:     ev.Text,
		Channel:  channel.Name,
		Account:  b.Account,
		ID:       ev.Timestamp,
		Extra:    make(map[string][]interface{}),
		ParentID: ev.ThreadTimestamp,
		Protocol: b.Protocol,
	}
	if b.useChannelID {
		rmsg.Channel = "ID:" + channel.ID
	}

	// Handle 'edit' messages.
	if ev.SubMessage != nil && !b.GetBool(editDisableConfig) {
		rmsg.ID = ev.SubMessage.Timestamp
		if ev.SubMessage.ThreadTimestamp != ev.SubMessage.Timestamp {
			b.Log.Debugf("SubMessage %#v", ev.SubMessage)
			rmsg.Text = ev.SubMessage.Text + b.GetString(editSuffixConfig)
		}
	}

	if err = b.populateMessageWithUserInfo(ev, rmsg); err != nil {
		return nil, err
	}
	return rmsg, err
}

func (b *Bslack) populateMessageWithUserInfo(ev *slack.MessageEvent, rmsg *config.Message) error {
	if ev.SubType == sMessageDeleted || ev.SubType == sFileComment {
		return nil
	}

	// First, deal with bot-originating messages but only do so when not using webhooks: we
	// would not be able to distinguish which bot would be sending them.
	if err := b.populateMessageWithBotInfo(ev, rmsg); err != nil {
		return err
	}

	// Second, deal with "real" users if we have the necessary information.
	var userID string
	switch {
	case ev.User != "":
		userID = ev.User
	case ev.SubMessage != nil && ev.SubMessage.User != "":
		userID = ev.SubMessage.User
	default:
		return nil
	}

	user := b.getUser(userID)
	if user == nil {
		return fmt.Errorf("could not find information for user with id %s", ev.User)
	}

	rmsg.UserID = user.ID
	rmsg.Username = user.Name
	if user.Profile.DisplayName != "" {
		rmsg.Username = user.Profile.DisplayName
	}
	return nil
}

func (b *Bslack) populateMessageWithBotInfo(ev *slack.MessageEvent, rmsg *config.Message) error {
	if ev.BotID == "" || b.GetString(outgoingWebhookConfig) != "" {
		return nil
	}

	var err error
	var bot *slack.Bot
	for {
		bot, err = b.rtm.GetBotInfo(ev.BotID)
		if err == nil {
			break
		}

		if err = b.handleRateLimit(err); err != nil {
			b.Log.Errorf("Could not retrieve bot information: %#v", err)
			return err
		}
	}
	b.Log.Debugf("Found bot %#v", bot)

	if bot.Name != "" && bot.Name != "Slack API Tester" {
		rmsg.Username = bot.Name
		if ev.Username != "" {
			rmsg.Username = ev.Username
		}
		rmsg.UserID = bot.ID
	}
	return nil
}

var (
	mentionRE        = regexp.MustCompile(`<@([a-zA-Z0-9]+)>`)
	channelRE        = regexp.MustCompile(`<#[a-zA-Z0-9]+\|(.+?)>`)
	variableRE       = regexp.MustCompile(`<!((?:subteam\^)?[a-zA-Z0-9]+)(?:\|@?(.+?))?>`)
	urlRE            = regexp.MustCompile(`<(.*?)(\|.*?)?>`)
	topicOrPurposeRE = regexp.MustCompile(`(?s)(@.+) (cleared|set)(?: the)? channel (topic|purpose)(?:: (.*))?`)
)

func (b *Bslack) extractTopicOrPurpose(text string) (string, string) {
	r := topicOrPurposeRE.FindStringSubmatch(text)
	if len(r) == 5 {
		action, updateType, extracted := r[2], r[3], r[4]
		switch action {
		case "set":
			return updateType, extracted
		case "cleared":
			return updateType, ""
		}
	}
	b.Log.Warnf("Encountered channel topic or purpose change message with unexpected format: %s", text)
	return "unknown", ""
}

// @see https://api.slack.com/docs/message-formatting#linking_to_channels_and_users
func (b *Bslack) replaceMention(text string) string {
	replaceFunc := func(match string) string {
		userID := strings.Trim(match, "@<>")
		if username := b.getUsername(userID); userID != "" {
			return "@" + username
		}
		return match
	}
	return mentionRE.ReplaceAllStringFunc(text, replaceFunc)
}

// @see https://api.slack.com/docs/message-formatting#linking_to_channels_and_users
func (b *Bslack) replaceChannel(text string) string {
	for _, r := range channelRE.FindAllStringSubmatch(text, -1) {
		text = strings.Replace(text, r[0], "#"+r[1], 1)
	}
	return text
}

// @see https://api.slack.com/docs/message-formatting#variables
func (b *Bslack) replaceVariable(text string) string {
	for _, r := range variableRE.FindAllStringSubmatch(text, -1) {
		if r[2] != "" {
			text = strings.Replace(text, r[0], "@"+r[2], 1)
		} else {
			text = strings.Replace(text, r[0], "@"+r[1], 1)
		}
	}
	return text
}

// @see https://api.slack.com/docs/message-formatting#linking_to_urls
func (b *Bslack) replaceURL(text string) string {
	for _, r := range urlRE.FindAllStringSubmatch(text, -1) {
		if len(strings.TrimSpace(r[2])) == 1 { // A display text separator was found, but the text was blank
			text = strings.Replace(text, r[0], "", 1)
		} else {
			text = strings.Replace(text, r[0], r[1], 1)
		}
	}
	return text
}

func (b *Bslack) handleRateLimit(err error) error {
	rateLimit, ok := err.(*slack.RateLimitedError)
	if !ok {
		return err
	}
	b.Log.Infof("Rate-limited by Slack. Sleeping for %v", rateLimit.RetryAfter)
	time.Sleep(rateLimit.RetryAfter)
	return nil
}
<commit_msg>Fix bot (legacy token) messages not being send. Closes #571<commit_after>package bslack

import (
	"context"
	"fmt"
	"regexp"
	"strings"
	"time"

	"github.com/42wim/matterbridge/bridge/config"
	"github.com/nlopes/slack"
)

func (b *Bslack) getUser(id string) *slack.User {
	b.usersMutex.RLock()
	defer b.usersMutex.RUnlock()

	return b.users[id]
}

func (b *Bslack) getUsername(id string) string {
	if user := b.getUser(id); user != nil {
		if user.Profile.DisplayName != "" {
			return user.Profile.DisplayName
		}
		return user.Name
	}
	b.Log.Warnf("Could not find user with ID '%s'", id)
	return ""
}

func (b *Bslack) getAvatar(id string) string {
	if user := b.getUser(id); user != nil {
		return user.Profile.Image48
	}
	return ""
}

func (b *Bslack) getChannel(channel string) (*slack.Channel, error) {
	if strings.HasPrefix(channel, "ID:") {
		return b.getChannelByID(strings.TrimPrefix(channel, "ID:"))
	}
	return b.getChannelByName(channel)
}

func (b *Bslack) getChannelByName(name string) (*slack.Channel, error) {
	return b.getChannelBy(name, b.channelsByName)
}

func (b *Bslack) getChannelByID(ID string) (*slack.Channel, error) {
	return b.getChannelBy(ID, b.channelsByID)
}

func (b *Bslack) getChannelBy(lookupKey string, lookupMap map[string]*slack.Channel) (*slack.Channel, error) {
	b.channelsMutex.RLock()
	defer b.channelsMutex.RUnlock()

	if channel, ok := lookupMap[lookupKey]; ok {
		return channel, nil
	}
	return nil, fmt.Errorf("%s: channel %s not found", b.Account, lookupKey)
}

const minimumRefreshInterval = 10 * time.Second

func (b *Bslack) populateUsers() {
	b.refreshMutex.Lock()
	if time.Now().Before(b.earliestUserRefresh) || b.refreshInProgress {
		b.Log.Debugf("Not refreshing user list as it was done less than %v ago.",
			minimumRefreshInterval)
		b.refreshMutex.Unlock()

		return
	}
	b.refreshInProgress = true
	b.refreshMutex.Unlock()

	newUsers := map[string]*slack.User{}
	pagination := b.sc.GetUsersPaginated(slack.GetUsersOptionLimit(200))
	for {
		var err error
		pagination, err = pagination.Next(context.Background())
		if err != nil {
			if pagination.Done(err) {
				break
			}

			if err = b.handleRateLimit(err); err != nil {
				b.Log.Errorf("Could not retrieve users: %#v", err)
				return
			}
			continue
		}

		for i := range pagination.Users {
			newUsers[pagination.Users[i].ID] = &pagination.Users[i]
		}
	}

	b.usersMutex.Lock()
	defer b.usersMutex.Unlock()
	b.users = newUsers

	b.refreshMutex.Lock()
	defer b.refreshMutex.Unlock()
	b.earliestUserRefresh = time.Now().Add(minimumRefreshInterval)
	b.refreshInProgress = false
}

func (b *Bslack) populateChannels() {
	b.refreshMutex.Lock()
	if time.Now().Before(b.earliestChannelRefresh) || b.refreshInProgress {
		b.Log.Debugf("Not refreshing channel list as it was done less than %v seconds ago.",
			minimumRefreshInterval)
		b.refreshMutex.Unlock()
		return
	}
	b.refreshInProgress = true
	b.refreshMutex.Unlock()

	newChannelsByID := map[string]*slack.Channel{}
	newChannelsByName := map[string]*slack.Channel{}

	// We only retrieve public and private channels, not IMs
	// and MPIMs as those do not have a channel name.
	queryParams := &slack.GetConversationsParameters{
		ExcludeArchived: "true",
		Types:           []string{"public_channel,private_channel"},
	}
	for {
		channels, nextCursor, err := b.sc.GetConversations(queryParams)
		if err != nil {
			if err = b.handleRateLimit(err); err != nil {
				b.Log.Errorf("Could not retrieve channels: %#v", err)
				return
			}
			continue
		}

		for i := range channels {
			newChannelsByID[channels[i].ID] = &channels[i]
			newChannelsByName[channels[i].Name] = &channels[i]
		}
		if nextCursor == "" {
			break
		}
		queryParams.Cursor = nextCursor
	}

	b.channelsMutex.Lock()
	defer b.channelsMutex.Unlock()
	b.channelsByID = newChannelsByID
	b.channelsByName = newChannelsByName

	b.refreshMutex.Lock()
	defer b.refreshMutex.Unlock()
	b.earliestChannelRefresh = time.Now().Add(minimumRefreshInterval)
	b.refreshInProgress = false
}

// populateReceivedMessage shapes the initial Matterbridge message that we will forward to the
// router before we apply message-dependent modifications.
func (b *Bslack) populateReceivedMessage(ev *slack.MessageEvent) (*config.Message, error) {
	// Use our own func because rtm.GetChannelInfo doesn't work for private channels.
	channel, err := b.getChannelByID(ev.Channel)
	if err != nil {
		return nil, err
	}

	rmsg := &config.Message{
		Text:     ev.Text,
		Channel:  channel.Name,
		Account:  b.Account,
		ID:       ev.Timestamp,
		Extra:    make(map[string][]interface{}),
		ParentID: ev.ThreadTimestamp,
		Protocol: b.Protocol,
	}
	if b.useChannelID {
		rmsg.Channel = "ID:" + channel.ID
	}

	// Handle 'edit' messages.
	if ev.SubMessage != nil && !b.GetBool(editDisableConfig) {
		rmsg.ID = ev.SubMessage.Timestamp
		if ev.SubMessage.ThreadTimestamp != ev.SubMessage.Timestamp {
			b.Log.Debugf("SubMessage %#v", ev.SubMessage)
			rmsg.Text = ev.SubMessage.Text + b.GetString(editSuffixConfig)
		}
	}

	if err = b.populateMessageWithUserInfo(ev, rmsg); err != nil {
		return nil, err
	}
	return rmsg, err
}

func (b *Bslack) populateMessageWithUserInfo(ev *slack.MessageEvent, rmsg *config.Message) error {
	if ev.SubType == sMessageDeleted || ev.SubType == sFileComment {
		return nil
	}

	// First, deal with bot-originating messages but only do so when not using webhooks: we
	// would not be able to distinguish which bot would be sending them.
	if err := b.populateMessageWithBotInfo(ev, rmsg); err != nil {
		return err
	}

	// Second, deal with "real" users if we have the necessary information.
	var userID string
	switch {
	case ev.User != "":
		userID = ev.User
	case ev.SubMessage != nil && ev.SubMessage.User != "":
		userID = ev.SubMessage.User
	default:
		return nil
	}

	user := b.getUser(userID)
	if user == nil {
		return fmt.Errorf("could not find information for user with id %s", ev.User)
	}

	rmsg.UserID = user.ID
	rmsg.Username = user.Name
	if user.Profile.DisplayName != "" {
		rmsg.Username = user.Profile.DisplayName
	}
	return nil
}

func (b *Bslack) populateMessageWithBotInfo(ev *slack.MessageEvent, rmsg *config.Message) error {
	if ev.BotID == "" || b.GetString(outgoingWebhookConfig) != "" {
		return nil
	}

	var err error
	var bot *slack.Bot
	for {
		bot, err = b.rtm.GetBotInfo(ev.BotID)
		if err == nil {
			break
		}

		if err = b.handleRateLimit(err); err != nil {
			b.Log.Errorf("Could not retrieve bot information: %#v", err)
			return err
		}
	}
	b.Log.Debugf("Found bot %#v", bot)

	if bot.Name != "" {
		rmsg.Username = bot.Name
		if ev.Username != "" {
			rmsg.Username = ev.Username
		}
		rmsg.UserID = bot.ID
	}
	return nil
}

var (
	mentionRE        = regexp.MustCompile(`<@([a-zA-Z0-9]+)>`)
	channelRE        = regexp.MustCompile(`<#[a-zA-Z0-9]+\|(.+?)>`)
	variableRE       = regexp.MustCompile(`<!((?:subteam\^)?[a-zA-Z0-9]+)(?:\|@?(.+?))?>`)
	urlRE            = regexp.MustCompile(`<(.*?)(\|.*?)?>`)
	topicOrPurposeRE = regexp.MustCompile(`(?s)(@.+) (cleared|set)(?: the)? channel (topic|purpose)(?:: (.*))?`)
)

func (b *Bslack) extractTopicOrPurpose(text string) (string, string) {
	r := topicOrPurposeRE.FindStringSubmatch(text)
	if len(r) == 5 {
		action, updateType, extracted := r[2], r[3], r[4]
		switch action {
		case "set":
			return updateType, extracted
		case "cleared":
			return updateType, ""
		}
	}
	b.Log.Warnf("Encountered channel topic or purpose change message with unexpected format: %s", text)
	return "unknown", ""
}

// @see https://api.slack.com/docs/message-formatting#linking_to_channels_and_users
func (b *Bslack) replaceMention(text string) string {
	replaceFunc := func(match string) string {
		userID := strings.Trim(match, "@<>")
		if username := b.getUsername(userID); userID != "" {
			return "@" + username
		}
		return match
	}
	return mentionRE.ReplaceAllStringFunc(text, replaceFunc)
}

// @see https://api.slack.com/docs/message-formatting#linking_to_channels_and_users
func (b *Bslack) replaceChannel(text string) string {
	for _, r := range channelRE.FindAllStringSubmatch(text, -1) {
		text = strings.Replace(text, r[0], "#"+r[1], 1)
	}
	return text
}

// @see https://api.slack.com/docs/message-formatting#variables
func (b *Bslack) replaceVariable(text string) string {
	for _, r := range variableRE.FindAllStringSubmatch(text, -1) {
		if r[2] != "" {
			text = strings.Replace(text, r[0], "@"+r[2], 1)
		} else {
			text = strings.Replace(text, r[0], "@"+r[1], 1)
		}
	}
	return text
}

// @see https://api.slack.com/docs/message-formatting#linking_to_urls
func (b *Bslack) replaceURL(text string) string {
	for _, r := range urlRE.FindAllStringSubmatch(text, -1) {
		if len(strings.TrimSpace(r[2])) == 1 { // A display text separator was found, but the text was blank
			text = strings.Replace(text, r[0], "", 1)
		} else {
			text = strings.Replace(text, r[0], r[1], 1)
		}
	}
	return text
}

func (b *Bslack) handleRateLimit(err error) error {
	rateLimit, ok := err.(*slack.RateLimitedError)
	if !ok {
		return err
	}
	b.Log.Infof("Rate-limited by Slack. Sleeping for %v", rateLimit.RetryAfter)
	time.Sleep(rateLimit.RetryAfter)
	return nil
}
<|endoftext|>


<commit_before>// +build windows

package jibber_jabber

import (
	"errors"
	"syscall"
	"unsafe"
)

const LOCALE_NAME_MAX_LENGTH uint32 = 85

var SUPPORTED_LOCALES = map[uintptr]string{
	0x0407: "de-DE",
	0x0409: "en-US",
	0x0c0a: "es-ES", //or is it 0x040a
	0x040c: "fr-FR",
	0x0410: "it-IT",
	0x0411: "ja-JA",
	//0x0412: "ko_KO", - Will add support for Korean when nicksnyder/go-i18n supports Korean
	0x0416: "pt-BR",
	//0x0419: "ru_RU", - Will add support for Russian when nicksnyder/go-i18n supports Russian
	0x0804: "zh-CN",
	0x0c04: "zh-CK",
}

func getWindowsLocaleFrom(sysCall string) (locale string, err error) {
	buffer := make([]uint16, LOCALE_NAME_MAX_LENGTH)

	dll := syscall.MustLoadDLL("kernel32")
	proc := dll.MustFindProc(sysCall)
	r, _, dllError := proc.Call(uintptr(unsafe.Pointer(&buffer[0])), uintptr(LOCALE_NAME_MAX_LENGTH))
	if r == 0 {
		err = errors.New(COULD_NOT_DETECT_PACKAGE_ERROR_MESSAGE + ":\n" + dllError.Error())
		return
	}

	locale = syscall.UTF16ToString(buffer)

	return
}

func getAllWindowsLocaleFrom(sysCall string) (string, error) {
	dll, err := syscall.LoadDLL("kernel32")
	if err != nil {
		return "", errors.New("Could not find kernel32 dll")
	}

	proc, err := dll.FindProc(sysCall)
	if err != nil {
		return "", err
	}

	r, locale, dllError := proc.Call()
	if r == 0 {
		return "", errors.New(COULD_NOT_DETECT_PACKAGE_ERROR_MESSAGE + ":\n" + dllError.Error())
	}

	return SUPPORTED_LOCALES[locale], nil
}

func getWindowsLocale() (locale string, err error) {
	dll, err := syscall.LoadDLL("kernel32")
	if err != nil {
		return "", errors.New("Could not find kernel32 dll")
	}

	proc, err := dll.FindProc("GetVersion")
	if err != nil {
		return "", err
	}

	v, _, _ := p.Call()
	windowsVersion := byte(v)
	fmt.Printf("Windows version %d.%d (Build %d)\n", windowsVersion, uint8(v>>8), uint16(v>>16))
	isVistaOrGreater := (windowsVersion >= 6)

	if isVistaOrGreater {
		locale, err = getWindowsLocaleFrom("GetUserDefaultLocaleName")
		if err != nil {
			locale, err = getWindowsLocaleFrom("GetSystemDefaultLocaleName")
		}
	} else if !isVistaOrGreater {
		locale, err = getAllWindowsLocaleFrom("GetUserDefaultLCID")
		if err != nil {
			locale, err = getAllWindowsLocaleFrom("GetSystemDefaultLCID")
		}
	} else {
		panic(v)
	}
	return
}
func DetectIETF() (locale string, err error) {
	locale, err = getWindowsLocale()
	return
}

func DetectLanguage() (language string, err error) {
	windows_locale, err := getWindowsLocale()
	if err == nil {
		language, _ = splitLocale(windows_locale)
	}

	return
}

func DetectTerritory() (territory string, err error) {
	windows_locale, err := getWindowsLocale()
	if err == nil {
		_, territory = splitLocale(windows_locale)
	}

	return
}
<commit_msg>Fix build errors<commit_after>// +build windows

package jibber_jabber

import (
	"errors"
	"fmt"
	"syscall"
	"unsafe"
)

const LOCALE_NAME_MAX_LENGTH uint32 = 85

var SUPPORTED_LOCALES = map[uintptr]string{
	0x0407: "de-DE",
	0x0409: "en-US",
	0x0c0a: "es-ES", //or is it 0x040a
	0x040c: "fr-FR",
	0x0410: "it-IT",
	0x0411: "ja-JA",
	//0x0412: "ko_KO", - Will add support for Korean when nicksnyder/go-i18n supports Korean
	0x0416: "pt-BR",
	//0x0419: "ru_RU", - Will add support for Russian when nicksnyder/go-i18n supports Russian
	0x0804: "zh-CN",
	0x0c04: "zh-CK",
}

func getWindowsLocaleFrom(sysCall string) (locale string, err error) {
	buffer := make([]uint16, LOCALE_NAME_MAX_LENGTH)

	dll := syscall.MustLoadDLL("kernel32")
	proc := dll.MustFindProc(sysCall)
	r, _, dllError := proc.Call(uintptr(unsafe.Pointer(&buffer[0])), uintptr(LOCALE_NAME_MAX_LENGTH))
	if r == 0 {
		err = errors.New(COULD_NOT_DETECT_PACKAGE_ERROR_MESSAGE + ":\n" + dllError.Error())
		return
	}

	locale = syscall.UTF16ToString(buffer)

	return
}

func getAllWindowsLocaleFrom(sysCall string) (string, error) {
	dll, err := syscall.LoadDLL("kernel32")
	if err != nil {
		return "", errors.New("Could not find kernel32 dll")
	}

	proc, err := dll.FindProc(sysCall)
	if err != nil {
		return "", err
	}

	r, locale, dllError := proc.Call()
	if r == 0 {
		return "", errors.New(COULD_NOT_DETECT_PACKAGE_ERROR_MESSAGE + ":\n" + dllError.Error())
	}

	return SUPPORTED_LOCALES[locale], nil
}

func getWindowsLocale() (locale string, err error) {
	dll, err := syscall.LoadDLL("kernel32")
	if err != nil {
		return "", errors.New("Could not find kernel32 dll")
	}

	proc, err := dll.FindProc("GetVersion")
	if err != nil {
		return "", err
	}

	v, _, _ := proc.Call()
	windowsVersion := byte(v)
	fmt.Printf("Windows version %d.%d (Build %d)\n", windowsVersion, uint8(v>>8), uint16(v>>16))
	isVistaOrGreater := (windowsVersion >= 6)

	if isVistaOrGreater {
		locale, err = getWindowsLocaleFrom("GetUserDefaultLocaleName")
		if err != nil {
			locale, err = getWindowsLocaleFrom("GetSystemDefaultLocaleName")
		}
	} else if !isVistaOrGreater {
		locale, err = getAllWindowsLocaleFrom("GetUserDefaultLCID")
		if err != nil {
			locale, err = getAllWindowsLocaleFrom("GetSystemDefaultLCID")
		}
	} else {
		panic(v)
	}
	return
}
func DetectIETF() (locale string, err error) {
	locale, err = getWindowsLocale()
	return
}

func DetectLanguage() (language string, err error) {
	windows_locale, err := getWindowsLocale()
	if err == nil {
		language, _ = splitLocale(windows_locale)
	}

	return
}

func DetectTerritory() (territory string, err error) {
	windows_locale, err := getWindowsLocale()
	if err == nil {
		_, territory = splitLocale(windows_locale)
	}

	return
}
<|endoftext|>


<commit_before>package tools

import (
	"io/ioutil"
	"log"
	"net/http"
	"os"

	"github.com/artemnikitin/devicefarm-ci-tool/errors"
)

func sendRequest(url string, file *os.File, info os.FileInfo) *http.Response {
	client := &http.Client{}
	request, err := http.NewRequest("PUT", url, file)
	errors.Validate(err, "Failed to create HTTP request")
	request.Header.Add("Content-Type", "application/octet-stream")
	request.ContentLength = info.Size()
	resp, err := client.Do(request)
	errors.Validate(err, "Failed to upload file by S3 link")
	return resp
}

func prepareFile(path string) (*os.File, os.FileInfo) {
	file, err := os.Open(path)
	errors.Validate(err, "Failed to get file for upload")
	info, err := file.Stat()
	errors.Validate(err, "Failed to get info about file")
	return file, info
}

func getStatusOfUpload(response *http.Response) int {
	result := response.StatusCode
	body, err := ioutil.ReadAll(response.Body)
	errors.Validate(err, "Failed to get body of response")
	log.Println("Response code:", result)
	log.Println("Response body:", string(body))
	defer response.Body.Close()
	return result
}
<commit_msg>Fix for reusing connection<commit_after>package tools

import (
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"

	"github.com/artemnikitin/devicefarm-ci-tool/errors"
)

func sendRequest(url string, file *os.File, info os.FileInfo) *http.Response {
	client := &http.Client{}
	request, err := http.NewRequest("PUT", url, file)
	errors.Validate(err, "Failed to create HTTP request")
	request.Header.Add("Content-Type", "application/octet-stream")
	request.ContentLength = info.Size()
	resp, err := client.Do(request)
	errors.Validate(err, "Failed to upload file by S3 link")
	return resp
}

func prepareFile(path string) (*os.File, os.FileInfo) {
	file, err := os.Open(path)
	errors.Validate(err, "Failed to get file for upload")
	info, err := file.Stat()
	errors.Validate(err, "Failed to get info about file")
	return file, info
}

func getStatusOfUpload(response *http.Response) int {
	result := response.StatusCode
	body, err := ioutil.ReadAll(response.Body)
	errors.Validate(err, "Failed to get body of response")
	log.Println("Response code:", result)
	log.Println("Response body:", string(body))
	defer func() {
		// Drain and close the body to let the Transport reuse the connection
		io.Copy(ioutil.Discard, response.Body)
		response.Body.Close()
	}()
	return result
}
<|endoftext|>