<commit_before>package kv

import (
	"encoding/json"
	"fmt"
	"strings"

	"log"

	"github.com/willf/bloom"
	"golang.org/x/net/context"
	elastic "gopkg.in/olivere/elastic.v5"

	"github.com/movio/kasper/util"
)

const indexSettings = `{
	"index.translog.durability": "async",
	"index.translog.sync_interval": "60s",
	"index.translog.flush_threshold_size": "512m"
}`

const indexMapping = `{
	"_all" : {
		"enabled" : false
	},
	"dynamic_templates": [{
		"no_index": {
			"mapping": {
				"index": "no"
			},
			"match": "*"
		}
	}]
}`

type indexAndType struct {
	indexName string
	indexType string
}

// BloomFilterConfig contains estimates to configure the optional bloom filter.
// See https://godoc.org/github.com/willf/bloom#NewWithEstimates for more information.
type BloomFilterConfig struct {
	// An estimate size the entire data set
	SizeEstimate uint
	// An estimate of the desired false positive rate
	FalsePositiveRate float64
	// How many bloom filters can exist at one time
	MaxBloomFilters int
}

// ElasticsearchKeyValueStore is a key-value storage that uses ElasticSearch.
// In this key-value store, all keys must have the format "<index>/<type>/<_id>".
// For performance reasons, this implementation create indexes with async durability.
// You must call Flush() at appropriate times to ensure Elasticsearch syncs its translog to disk.
// See: https://www.elastic.co/products/elasticsearch
type ElasticsearchKeyValueStore struct {
	witness          *util.StructPtrWitness
	client           *elastic.Client
	context          context.Context
	existingIndexes  []indexAndType
	bloomFilters     map[string]map[string]*bloom.BloomFilter
	bloomFiltersList []indexAndType
	bfConfig         *BloomFilterConfig
}

// NewESKeyValueStore creates new ElasticsearchKeyValueStore instance.
// Host must of the format hostname:port.
// StructPtr should be a pointer to struct type that is used.
// for serialization and deserialization of store values.
func NewESKeyValueStore(url string, structPtr interface{}) *ElasticsearchKeyValueStore {
	return NewESKeyValueStoreWithBloomFilter(url, structPtr, nil)
}

// NewESKeyValueStoreWithBloomFilter enables an optional bloom filter to optimize Get() heavy workloads.
func NewESKeyValueStoreWithBloomFilter(url string, structPtr interface{}, bfConfig *BloomFilterConfig) *ElasticsearchKeyValueStore {
	client, err := elastic.NewClient(
		elastic.SetURL(url),
		elastic.SetSniff(false), // FIXME: workaround for issues with ES in docker
	)
	if err != nil {
		panic(fmt.Sprintf("Cannot create ElasticSearch Client to '%s': %s", url, err))
	}
	return &ElasticsearchKeyValueStore{
		witness:          util.NewStructPtrWitness(structPtr),
		client:           client,
		context:          context.Background(),
		existingIndexes:  nil,
		bloomFilters:     make(map[string]map[string]*bloom.BloomFilter),
		bloomFiltersList: []indexAndType{},
		bfConfig:         bfConfig,
	}
}

func (s *ElasticsearchKeyValueStore) ejectOldestBloomFilter() {
	i := s.bloomFiltersList[0]
	s.bloomFiltersList = s.bloomFiltersList[1:]
	s.removeBloomFilter(i.indexName, i.indexType)
}

func (s *ElasticsearchKeyValueStore) getBloomFilter(indexName, indexType string) *bloom.BloomFilter {
	if s.bloomFilters[indexName] == nil {
		return nil
	}
	return s.bloomFilters[indexName][indexType]
}

func (s *ElasticsearchKeyValueStore) setBloomFilter(indexName, indexType string, bf *bloom.BloomFilter) {
	if s.bfConfig == nil {
		return
	}
	if s.bloomFilters[indexName] == nil {
		s.bloomFilters[indexName] = make(map[string]*bloom.BloomFilter)
	}
	for len(s.bloomFiltersList) >= s.bfConfig.MaxBloomFilters {
		s.ejectOldestBloomFilter()
	}
	s.bloomFilters[indexName][indexType] = bf
	s.bloomFiltersList = append(s.bloomFiltersList, indexAndType{indexName, indexType})
}

func (s *ElasticsearchKeyValueStore) newBloomFilter() *bloom.BloomFilter {
	if s.bfConfig == nil {
		return nil
	}
	return bloom.NewWithEstimates(s.bfConfig.SizeEstimate, s.bfConfig.FalsePositiveRate)
}

func (s *ElasticsearchKeyValueStore) removeBloomFilter(indexName, indexType string) {
	if s.bloomFilters[indexName] == nil {
		return
	}
	delete(s.bloomFilters[indexName], indexType)
}

func (s *ElasticsearchKeyValueStore) provenAbsentByBloomFilter(indexName, indexType, id string) bool {
	bf := s.getBloomFilter(indexName, indexType)
	if bf == nil {
		return false
	}
	return bf.TestString(id) == false
}

func (s *ElasticsearchKeyValueStore) addToBloomFilter(indexName, indexType, id string) {
	bf := s.getBloomFilter(indexName, indexType)
	if bf == nil {
		return
	}
	bf.AddString(id)
}

func (s *ElasticsearchKeyValueStore) checkOrCreateIndex(indexName string, indexType string) {
	for _, existing := range s.existingIndexes {
		if existing.indexName == indexName && existing.indexType == indexType {
			return
		}
	}
	exists, err := s.client.IndexExists(indexName).Do(s.context)
	if err != nil {
		panic(fmt.Sprintf("Failed to check if index exists: %s", err))
	}
	if !exists {
		_, err = s.client.CreateIndex(indexName).BodyString(indexSettings).Do(s.context)
		if err != nil {
			panic(fmt.Sprintf("Failed to create index: %s", err))
		}
		s.putMapping(indexName, indexType)
		s.setBloomFilter(indexName, indexType, s.newBloomFilter())
	}

	s.existingIndexes = append(s.existingIndexes, indexAndType{indexName, indexType})
}

func (s *ElasticsearchKeyValueStore) putMapping(indexName string, indexType string) {
	resp, err := s.client.PutMapping().Index(indexName).Type(indexType).BodyString(indexMapping).Do(s.context)
	if err != nil {
		panic(fmt.Sprintf("Failed to put mapping for index: %s/%s: %s", indexName, indexType, err))
	}
	if resp == nil {
		panic(fmt.Sprintf("Expected put mapping response; got: %v", resp))
	}
	if !resp.Acknowledged {
		panic(fmt.Sprintf("Expected put mapping ack; got: %v", resp.Acknowledged))
	}
}

// Get gets value by key from store
func (s *ElasticsearchKeyValueStore) Get(key string) (interface{}, error) {
	keyParts := strings.Split(key, "/")
	if len(keyParts) != 3 {
		return nil, fmt.Errorf("invalid key: '%s'", key)
	}
	indexName := keyParts[0]
	indexType := keyParts[1]
	valueID := keyParts[2]

	s.checkOrCreateIndex(indexName, indexType)
	if s.provenAbsentByBloomFilter(indexName, indexType, valueID) {
		return s.witness.Nil(), nil
	}

	rawValue, err := s.client.Get().
		Index(indexName).
		Type(indexType).
		Id(valueID).
		Do(s.context)

	if fmt.Sprintf("%s", err) == "elastic: Error 404 (Not Found)" {
		return s.witness.Nil(), nil
	}

	if err != nil {
		return s.witness.Nil(), err
	}

	if !rawValue.Found {
		return s.witness.Nil(), nil
	}

	structPtr := s.witness.Allocate()
	err = json.Unmarshal(*rawValue.Source, structPtr)
	if err != nil {
		return s.witness.Nil(), err
	}
	return structPtr, nil
}

// Put updates key in store with serialized value
func (s *ElasticsearchKeyValueStore) Put(key string, structPtr interface{}) error {
	s.witness.Assert(structPtr)
	keyParts := strings.Split(key, "/")
	if len(keyParts) != 3 {
		return fmt.Errorf("invalid key: '%s'", key)
	}
	indexName := keyParts[0]
	indexType := keyParts[1]
	valueID := keyParts[2]

	s.checkOrCreateIndex(indexName, indexType)
	s.addToBloomFilter(indexName, indexType, valueID)

	_, err := s.client.Index().
		Index(indexName).
		Type(indexType).
		Id(valueID).
		BodyJson(structPtr).
		Do(s.context)

	return err
}

// PutAll bulk executes Put operation for several entries
func (s *ElasticsearchKeyValueStore) PutAll(entries []*Entry) error {
	if len(entries) == 0 {
		return nil
	}
	bulk := s.client.Bulk()
	for _, entry := range entries {
		keyParts := strings.Split(entry.key, "/")
		if len(keyParts) != 3 {
			return fmt.Errorf("invalid key: '%s'", entry.key)
		}
		indexName := keyParts[0]
		indexType := keyParts[1]
		valueID := keyParts[2]

		s.witness.Assert(entry.value)
		s.checkOrCreateIndex(indexName, indexType)
		s.addToBloomFilter(indexName, indexType, valueID)

		bulk.Add(elastic.NewBulkIndexRequest().
			Index(indexName).
			Type(indexType).
			Id(valueID).
			Doc(entry.value),
		)
	}
	_, err := bulk.Do(s.context)
	return err
}

// Delete removes key from store
func (s *ElasticsearchKeyValueStore) Delete(key string) error {
	keyParts := strings.Split(key, "/")
	if len(keyParts) != 3 {
		return fmt.Errorf("invalid key: '%s'", key)
	}
	indexName := keyParts[0]
	indexType := keyParts[1]
	valueID := keyParts[2]

	s.checkOrCreateIndex(indexName, indexType)
	s.removeBloomFilter(indexName, indexType)

	response, err := s.client.Delete().
		Index(indexName).
		Type(indexType).
		Id(valueID).
		Do(s.context)

	if response != nil && !response.Found {
		return nil
	}

	return err
}

// Flush the Elasticsearch translog to disk
func (s *ElasticsearchKeyValueStore) Flush() error {
	log.Println("Flusing ES indexes...")
	indexNames := []string{}
	for _, existing := range s.existingIndexes {
		indexNames = append(indexNames, existing.indexName)
	}
	_, err := s.client.Flush(indexNames...).
		WaitIfOngoing(true).
		Do(s.context)
	log.Println("Done flusing ES indexes.")
	return err
}
<commit_msg>use _all as index on Flush<commit_after>package kv

import (
	"encoding/json"
	"fmt"
	"strings"

	"log"

	"github.com/willf/bloom"
	"golang.org/x/net/context"
	elastic "gopkg.in/olivere/elastic.v5"

	"github.com/movio/kasper/util"
)

const indexSettings = `{
	"index.translog.durability": "async",
	"index.translog.sync_interval": "60s",
	"index.translog.flush_threshold_size": "512m"
}`

const indexMapping = `{
	"_all" : {
		"enabled" : false
	},
	"dynamic_templates": [{
		"no_index": {
			"mapping": {
				"index": "no"
			},
			"match": "*"
		}
	}]
}`

type indexAndType struct {
	indexName string
	indexType string
}

// BloomFilterConfig contains estimates to configure the optional bloom filter.
// See https://godoc.org/github.com/willf/bloom#NewWithEstimates for more information.
type BloomFilterConfig struct {
	// An estimate size the entire data set
	SizeEstimate uint
	// An estimate of the desired false positive rate
	FalsePositiveRate float64
	// How many bloom filters can exist at one time
	MaxBloomFilters int
}

// ElasticsearchKeyValueStore is a key-value storage that uses ElasticSearch.
// In this key-value store, all keys must have the format "<index>/<type>/<_id>".
// For performance reasons, this implementation create indexes with async durability.
// You must call Flush() at appropriate times to ensure Elasticsearch syncs its translog to disk.
// See: https://www.elastic.co/products/elasticsearch
type ElasticsearchKeyValueStore struct {
	witness          *util.StructPtrWitness
	client           *elastic.Client
	context          context.Context
	existingIndexes  []indexAndType
	bloomFilters     map[string]map[string]*bloom.BloomFilter
	bloomFiltersList []indexAndType
	bfConfig         *BloomFilterConfig
}

// NewESKeyValueStore creates new ElasticsearchKeyValueStore instance.
// Host must of the format hostname:port.
// StructPtr should be a pointer to struct type that is used.
// for serialization and deserialization of store values.
func NewESKeyValueStore(url string, structPtr interface{}) *ElasticsearchKeyValueStore {
	return NewESKeyValueStoreWithBloomFilter(url, structPtr, nil)
}

// NewESKeyValueStoreWithBloomFilter enables an optional bloom filter to optimize Get() heavy workloads.
func NewESKeyValueStoreWithBloomFilter(url string, structPtr interface{}, bfConfig *BloomFilterConfig) *ElasticsearchKeyValueStore {
	client, err := elastic.NewClient(
		elastic.SetURL(url),
		elastic.SetSniff(false), // FIXME: workaround for issues with ES in docker
	)
	if err != nil {
		panic(fmt.Sprintf("Cannot create ElasticSearch Client to '%s': %s", url, err))
	}
	return &ElasticsearchKeyValueStore{
		witness:          util.NewStructPtrWitness(structPtr),
		client:           client,
		context:          context.Background(),
		existingIndexes:  nil,
		bloomFilters:     make(map[string]map[string]*bloom.BloomFilter),
		bloomFiltersList: []indexAndType{},
		bfConfig:         bfConfig,
	}
}

func (s *ElasticsearchKeyValueStore) ejectOldestBloomFilter() {
	i := s.bloomFiltersList[0]
	s.bloomFiltersList = s.bloomFiltersList[1:]
	s.removeBloomFilter(i.indexName, i.indexType)
}

func (s *ElasticsearchKeyValueStore) getBloomFilter(indexName, indexType string) *bloom.BloomFilter {
	if s.bloomFilters[indexName] == nil {
		return nil
	}
	return s.bloomFilters[indexName][indexType]
}

func (s *ElasticsearchKeyValueStore) setBloomFilter(indexName, indexType string, bf *bloom.BloomFilter) {
	if s.bfConfig == nil {
		return
	}
	if s.bloomFilters[indexName] == nil {
		s.bloomFilters[indexName] = make(map[string]*bloom.BloomFilter)
	}
	for len(s.bloomFiltersList) >= s.bfConfig.MaxBloomFilters {
		s.ejectOldestBloomFilter()
	}
	s.bloomFilters[indexName][indexType] = bf
	s.bloomFiltersList = append(s.bloomFiltersList, indexAndType{indexName, indexType})
}

func (s *ElasticsearchKeyValueStore) newBloomFilter() *bloom.BloomFilter {
	if s.bfConfig == nil {
		return nil
	}
	return bloom.NewWithEstimates(s.bfConfig.SizeEstimate, s.bfConfig.FalsePositiveRate)
}

func (s *ElasticsearchKeyValueStore) removeBloomFilter(indexName, indexType string) {
	if s.bloomFilters[indexName] == nil {
		return
	}
	delete(s.bloomFilters[indexName], indexType)
}

func (s *ElasticsearchKeyValueStore) provenAbsentByBloomFilter(indexName, indexType, id string) bool {
	bf := s.getBloomFilter(indexName, indexType)
	if bf == nil {
		return false
	}
	return bf.TestString(id) == false
}

func (s *ElasticsearchKeyValueStore) addToBloomFilter(indexName, indexType, id string) {
	bf := s.getBloomFilter(indexName, indexType)
	if bf == nil {
		return
	}
	bf.AddString(id)
}

func (s *ElasticsearchKeyValueStore) checkOrCreateIndex(indexName string, indexType string) {
	for _, existing := range s.existingIndexes {
		if existing.indexName == indexName && existing.indexType == indexType {
			return
		}
	}
	exists, err := s.client.IndexExists(indexName).Do(s.context)
	if err != nil {
		panic(fmt.Sprintf("Failed to check if index exists: %s", err))
	}
	if !exists {
		_, err = s.client.CreateIndex(indexName).BodyString(indexSettings).Do(s.context)
		if err != nil {
			panic(fmt.Sprintf("Failed to create index: %s", err))
		}
		s.putMapping(indexName, indexType)
		s.setBloomFilter(indexName, indexType, s.newBloomFilter())
	}

	s.existingIndexes = append(s.existingIndexes, indexAndType{indexName, indexType})
}

func (s *ElasticsearchKeyValueStore) putMapping(indexName string, indexType string) {
	resp, err := s.client.PutMapping().Index(indexName).Type(indexType).BodyString(indexMapping).Do(s.context)
	if err != nil {
		panic(fmt.Sprintf("Failed to put mapping for index: %s/%s: %s", indexName, indexType, err))
	}
	if resp == nil {
		panic(fmt.Sprintf("Expected put mapping response; got: %v", resp))
	}
	if !resp.Acknowledged {
		panic(fmt.Sprintf("Expected put mapping ack; got: %v", resp.Acknowledged))
	}
}

// Get gets value by key from store
func (s *ElasticsearchKeyValueStore) Get(key string) (interface{}, error) {
	keyParts := strings.Split(key, "/")
	if len(keyParts) != 3 {
		return nil, fmt.Errorf("invalid key: '%s'", key)
	}
	indexName := keyParts[0]
	indexType := keyParts[1]
	valueID := keyParts[2]

	s.checkOrCreateIndex(indexName, indexType)
	if s.provenAbsentByBloomFilter(indexName, indexType, valueID) {
		return s.witness.Nil(), nil
	}

	rawValue, err := s.client.Get().
		Index(indexName).
		Type(indexType).
		Id(valueID).
		Do(s.context)

	if fmt.Sprintf("%s", err) == "elastic: Error 404 (Not Found)" {
		return s.witness.Nil(), nil
	}

	if err != nil {
		return s.witness.Nil(), err
	}

	if !rawValue.Found {
		return s.witness.Nil(), nil
	}

	structPtr := s.witness.Allocate()
	err = json.Unmarshal(*rawValue.Source, structPtr)
	if err != nil {
		return s.witness.Nil(), err
	}
	return structPtr, nil
}

// Put updates key in store with serialized value
func (s *ElasticsearchKeyValueStore) Put(key string, structPtr interface{}) error {
	s.witness.Assert(structPtr)
	keyParts := strings.Split(key, "/")
	if len(keyParts) != 3 {
		return fmt.Errorf("invalid key: '%s'", key)
	}
	indexName := keyParts[0]
	indexType := keyParts[1]
	valueID := keyParts[2]

	s.checkOrCreateIndex(indexName, indexType)
	s.addToBloomFilter(indexName, indexType, valueID)

	_, err := s.client.Index().
		Index(indexName).
		Type(indexType).
		Id(valueID).
		BodyJson(structPtr).
		Do(s.context)

	return err
}

// PutAll bulk executes Put operation for several entries
func (s *ElasticsearchKeyValueStore) PutAll(entries []*Entry) error {
	if len(entries) == 0 {
		return nil
	}
	bulk := s.client.Bulk()
	for _, entry := range entries {
		keyParts := strings.Split(entry.key, "/")
		if len(keyParts) != 3 {
			return fmt.Errorf("invalid key: '%s'", entry.key)
		}
		indexName := keyParts[0]
		indexType := keyParts[1]
		valueID := keyParts[2]

		s.witness.Assert(entry.value)
		s.checkOrCreateIndex(indexName, indexType)
		s.addToBloomFilter(indexName, indexType, valueID)

		bulk.Add(elastic.NewBulkIndexRequest().
			Index(indexName).
			Type(indexType).
			Id(valueID).
			Doc(entry.value),
		)
	}
	_, err := bulk.Do(s.context)
	return err
}

// Delete removes key from store
func (s *ElasticsearchKeyValueStore) Delete(key string) error {
	keyParts := strings.Split(key, "/")
	if len(keyParts) != 3 {
		return fmt.Errorf("invalid key: '%s'", key)
	}
	indexName := keyParts[0]
	indexType := keyParts[1]
	valueID := keyParts[2]

	s.checkOrCreateIndex(indexName, indexType)
	s.removeBloomFilter(indexName, indexType)

	response, err := s.client.Delete().
		Index(indexName).
		Type(indexType).
		Id(valueID).
		Do(s.context)

	if response != nil && !response.Found {
		return nil
	}

	return err
}

// Flush the Elasticsearch translog to disk
func (s *ElasticsearchKeyValueStore) Flush() error {
	log.Println("Flusing ES indexes...")
	_, err := s.client.Flush("_all").
		WaitIfOngoing(true).
		Do(s.context)
	log.Println("Done flusing ES indexes.")
	return err
}
<|endoftext|>


<commit_before>// Copyright 2014 Richard Lehane. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package frames describes the Frame interface.
// A set of standard frames are also defined in this package. These are: Fixed, Window, Wild and WildMin.
package frames

import (
	"errors"
	"strconv"

	"github.com/richardlehane/siegfried/internal/bytematcher/patterns"
	"github.com/richardlehane/siegfried/internal/persist"
)

// Frame encapsulates a pattern with offset information, mediating between the pattern and the bytestream.
type Frame struct {
	Min int
	Max int
	OffType
	patterns.Pattern
}

// OffType is the type of offset
type OffType uint8

// Four offset types are supported
const (
	BOF  OffType = iota // beginning of file offset
	PREV                // offset from previous frame
	SUCC                // offset from successive frame
	EOF                 // end of file offset
)

// OffString is an exported array of strings representing each of the four offset types
var OffString = [...]string{"B", "P", "S", "E"}

// Orientation returns the offset type of the frame which must be either BOF, PREV, SUCC or EOF
func (o OffType) Orientation() OffType {
	return o
}

// SwitchOff returns a new offset type according to a given set of rules. These are:
// 	- PREV -> SUCC
// 	- SUCC and EOF -> PREV
// This is helpful when changing the orientation of a frame (for example to allow right-left searching).
func (o OffType) SwitchOff() OffType {
	switch o {
	case PREV:
		return SUCC
	case SUCC, EOF:
		return PREV
	default:
		return o
	}
}

// NewFrame generates Fixed, Window, Wild and WildMin frames. The offsets argument controls what type of frame is created:
// 	- for a Wild frame, give no offsets or give a max offset of < 0 and a min of < 1
// 	- for a WildMin frame, give one offset, or give a max offset of < 0 and a min of > 0
// 	- for a Fixed frame, give two offsets that are both >= 0 and that are equal to each other
// 	- for a Window frame, give two offsets that are both >= 0 and that are not equal to each other.
func NewFrame(typ OffType, pat patterns.Pattern, offsets ...int) Frame {
	switch len(offsets) {
	case 0:
		return Frame{0, -1, typ, pat}
	case 1:
		if offsets[0] > 0 {
			return Frame{offsets[0], -1, typ, pat}
		}
		return Frame{0, -1, typ, pat}
	}
	if offsets[1] < 0 {
		if offsets[0] > 0 {
			return Frame{offsets[0], -1, typ, pat}
		}
		return Frame{0, -1, typ, pat}
	}
	if offsets[0] < 0 {
		offsets[0] = 0
	}
	return Frame{typ, offsets[0], offsets[1], pat}
}

// SwitchFrame returns a new frame with a different orientation (for example to allow right-left searching).
func SwitchFrame(f Frame, p patterns.Pattern) Frame {
	return NewFrame(f.SwitchOff(), p, f.Min, f.Max)
}

// BMHConvert converts the patterns within a slice of frames to BMH sequences if possible.
func BMHConvert(fs []Frame, rev bool) []Frame {
	nfs := make([]Frame, len(fs))
	for i, f := range fs {
		nfs[i] = NewFrame(f.Orientation(), patterns.BMH(f.Pat(), rev), f.Min, f.Max)
	}
	return nfs
}

// NonZero checks whether, when converted to simple byte sequences, this frame's pattern is all 0 bytes.
func NonZero(f Frame) bool {
	for _, seq := range f.Sequences() {
		allzeros := true
		for _, b := range seq {
			if b != 0 {
				allzeros = false
			}
		}
		if allzeros {
			return false
		}
	}
	return true
}

// TotalLength is sum of the maximum length of the enclosed pattern and the maximum offset.
func TotalLength(f Frame) int {
	// a wild frame has no total length
	if f.Max < 0 {
		return -1
	}
	_, l := f.Length()
	return l + f.Max
}

// Match the enclosed pattern against the byte slice in a L-R direction.
// Returns a slice of offsets for where a successive match by a related frame should begin.
func (f Frame) Match(b []byte) []int {
	ret := make([]int, 0, 1)
	min, max := f.Min, f.Max
	if max < 0 || max > len(b) {
		max = len(b)
	}
	for min <= max {
		length, adv := f.Test(b[min:])
		if length > -1 {
			ret = append(ret, min+length)
		}
		if adv < 1 {
			break
		}
		min += adv
	}
	return ret
}

// For the nth match (per above), return the offset for successive match by related frame and bytes that can advance to make a successive test by this frame.
func (f Frame) MatchN(b []byte, n int) (int, int) {
	var i int
	min, max := f.Min, f.Max
	if max < 0 || max > len(b) {
		max = len(b)
	}
	for min <= max {
		length, adv := f.Test(b[min:])
		if length > -1 {
			if i == n {
				return min + length, min + adv
			}
			i++
		}
		if adv < 1 {
			break
		}
		min += adv
	}
	return -1, 0
}

// Match the enclosed pattern against the byte slice in a reverse (R-L) direction. Returns a slice of offsets for where a successive match by a related frame should begin.
func (f Frame) MatchR(b []byte) []int {
	ret := make([]int, 0, 1)
	min, max := f.Min, f.Max
	if max < 0 || max > len(b) {
		max = len(b)
	}
	for min <= max {
		length, adv := f.TestR(b[:len(b)-min])
		if length > -1 {
			ret = append(ret, min+length)
		}
		if adv < 1 {
			break
		}
		min += adv
	}
	return ret
}

// For the nth match (per above), return the offset for successive match by related frame and bytes that can advance to make a successive test by this frame.
func (f Frame) MatchNR(b []byte, n int) (int, int) {
	var i int
	min, max := f.Min, w.Max
	if max < 0 || max > len(b) {
		max = len(b)
	}
	for min <= max {
		length, adv := f.TestR(b[:len(b)-min])
		if length > -1 {
			if i == n {
				return min + length, min + adv
			}
			i++
		}
		if adv < 1 {
			break
		}
		min += adv
	}
	return -1, 0
}

func (f Frame) Equals(f1 Frame) bool {
	if f.Min == f1.Min && f.Max == f1.Max && f.OffType == f1.OffType && f.Pattern.Equals(f1.Pattern) {
		return true
	}
	return false
}

func (f Frame) String() string {
	switch {
	case f.Min == f.Max && f.Min >= 0:
		"F " + OffString[f.OffType] + ":" + strconv.Itoa(f.Min) + " " + f.Pattern.String()
	case f.Min > 0 && f.Max < -1:
		"WM " + OffString[f.OffType] + ":" + strconv.Itoa(f.Min) + " " + f.Pattern.String()
	case f.Min <= 0 && f.Max < -1:
		"WL " + OffString[f.OffType] + " " + f.Pattern.String()
	default:
		"WW " + OffString[f.OffType] + ":" + strconv.Itoa(f.Min) + "-" + strconv.Itoa(f.Max) + " " + f.Pattern.String()
	}
}

// MaxMatches returns the max number of times a frame can match, the maximum remaining slice length, and the minimum length of the pattern, given a byte slice of length 'l'
func (f Frame) MaxMatches(l int) (int, int, int)

// MaxMatches returns the max number of times a frame can match, given a byte slice of length 'l', and the maximum remaining slice length
func (f Fixed) MaxMatches(l int) (int, int, int) {
	min, _ := f.Length()
	rem := l - min - f.Off
	if rem >= 0 {
		return 1, rem, min
	}
	return 0, 0, 0
}

// MaxMatches returns the max number of times a frame can match, given a byte slice of length 'l', and the maximum remaining slice length
// TODO: this is *wrong* because it presumes a pattern can't overlap i.e. in AAAAAA, the string AA can match at 5 positions, not 3
func (w Window) MaxMatches(l int) (int, int, int) {
	min, _ := w.Length()
	rem := l - min - w.MinOff
	if rem < 0 {
		return 0, 0, 0
	}
	if w.MaxOff+min > l {
		return rem/min + 1, rem, min
	}
	return (w.MaxOff + min - w.MinOff) / min, rem, min
}

// MaxMatches returns the max number of times a frame can match, given a byte slice of length 'l', and the maximum remaining slice length
func (w Wild) MaxMatches(l int) (int, int, int) {
	min, _ := w.Length()
	rem := l - min
	if rem < 0 {
		return 0, 0, 0
	}
	return rem/min + 1, rem, min
}

// MaxMatches returns the max number of times a frame can match, given a byte slice of length 'l', and the maximum remaining slice length
func (w WildMin) MaxMatches(l int) (int, int, int) {
	min, _ := w.Length()
	rem := l - min - w.MinOff
	if rem < 0 {
		return 0, 0, 0
	}
	return rem/min + 1, rem, min
}

// Linked tests whether a frame is linked to a preceding frame (by a preceding or succeding relationship) with an offset and range that is less than the supplied ints. Pass -1 as maxDistance to test linkage regardless of distance/range
func (f Frame) Linked(Frame, int, int) (bool, int, int) {}

// Linked tests whether a frame is linked to a preceding frame (by a preceding or succeding relationship) with an offset and range that is less than the supplied ints.
func (f Fixed) Linked(prev Frame, maxDistance, maxRange int) (bool, int, int) {
	switch f.OffType {
	case PREV:
		if maxDistance < 0 {
			return true, maxDistance, maxRange
		}
		if f.Off > maxDistance {
			return false, 0, 0
		}
		return true, maxDistance - f.Off, maxRange
	case SUCC, EOF:
		if prev.Orientation() != SUCC || prev.Max() < 0 {
			return false, 0, 0
		}
		if maxDistance < 0 {
			return true, maxDistance, maxRange
		}
		if prev.Max() > maxDistance || prev.Max()-prev.Min() > maxRange {
			return false, 0, 0
		}
		return true, maxDistance - prev.Max(), maxRange - (prev.Max() - prev.Min())
	default:
		return false, 0, 0
	}
}

// Linked tests whether a frame is linked to a preceding frame (by a preceding or succeding relationship) with an offset and range that is less than the supplied ints.
func (w Window) Linked(prev Frame, maxDistance, maxRange int) (bool, int, int) {
	switch w.OffType {
	case PREV:
		if maxDistance < 0 {
			return true, maxDistance, maxRange
		}
		if w.MaxOff > maxDistance || w.MaxOff-w.MinOff > maxRange {
			return false, 0, 0
		}
		return true, maxDistance - w.MaxOff, maxRange - (w.MaxOff - w.MinOff)
	case SUCC, EOF:
		if prev.Orientation() != SUCC || prev.Max() < 0 {
			return false, 0, 0
		}
		if maxDistance < 0 {
			return true, maxDistance, maxRange
		}
		if prev.Max() > maxDistance || prev.Max()-prev.Min() > maxRange {
			return false, 0, 0
		}
		return true, maxDistance - prev.Max(), maxRange - (prev.Max() - prev.Min())
	default:
		return false, 0, 0
	}
}

// Linked tests whether a frame is linked to a preceding frame (by a preceding or succeding relationship) with an offset and range that is less than the supplied ints.
func (w Wild) Linked(prev Frame, maxDistance, maxRange int) (bool, int, int) {
	switch w.OffType {
	case SUCC, EOF:
		if prev.Orientation() != SUCC || prev.Max() < 0 {
			return false, 0, 0
		}
		if maxDistance < 0 {
			return true, maxDistance, maxRange
		}
		if prev.Max() > maxDistance || prev.Max()-prev.Min() > maxRange {
			return false, 0, 0
		}
		return true, maxDistance - prev.Max(), maxRange - (prev.Max() - prev.Min())
	default:
		return false, 0, 0
	}
}

// Linked tests whether a frame is linked to a preceding frame (by a preceding or succeding relationship) with an offset and range that is less than the supplied ints.
func (w WildMin) Linked(prev Frame, maxDistance, maxRange int) (bool, int, int) {
	switch w.OffType {
	case SUCC, EOF:
		if prev.Orientation() != SUCC || prev.Max() < 0 {
			return false, 0, 0
		}
		if maxDistance < 0 {
			return true, maxDistance, maxRange
		}
		if prev.Max() > maxDistance || prev.Max()-prev.Min() > maxRange {
			return false, 0, 0
		}
		return true, maxDistance - prev.Max(), maxRange - (prev.Max() - prev.Min())
	default:
		return false, 0, 0
	}
}

func (f Frame) Save(*persist.LoadSaver) {
	ls.SaveInt(f.Min)
	ls.SaveInt(f.Max)
	ls.SaveByte(byte(f.OffType))
	f.Pattern.Save(ls)
}

func Load(ls *persist.LoadSaver) Frame {
	return Frame{
		ls.LoadInt(),
		ls.LoadInt(),
		OffType(ls.LoadByte()),
		patterns.Load(ls),
	}
}
<commit_msg>unify frames.Linked into a single func<commit_after>// Copyright 2014 Richard Lehane. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package frames describes the Frame interface.
// A set of standard frames are also defined in this package. These are: Fixed, Window, Wild and WildMin.
package frames

import (
	"errors"
	"strconv"

	"github.com/richardlehane/siegfried/internal/bytematcher/patterns"
	"github.com/richardlehane/siegfried/internal/persist"
)

// Frame encapsulates a pattern with offset information, mediating between the pattern and the bytestream.
type Frame struct {
	Min int
	Max int
	OffType
	patterns.Pattern
}

// OffType is the type of offset
type OffType uint8

// Four offset types are supported
const (
	BOF  OffType = iota // beginning of file offset
	PREV                // offset from previous frame
	SUCC                // offset from successive frame
	EOF                 // end of file offset
)

// OffString is an exported array of strings representing each of the four offset types
var OffString = [...]string{"B", "P", "S", "E"}

// Orientation returns the offset type of the frame which must be either BOF, PREV, SUCC or EOF
func (o OffType) Orientation() OffType {
	return o
}

// SwitchOff returns a new offset type according to a given set of rules. These are:
// 	- PREV -> SUCC
// 	- SUCC and EOF -> PREV
// This is helpful when changing the orientation of a frame (for example to allow right-left searching).
func (o OffType) SwitchOff() OffType {
	switch o {
	case PREV:
		return SUCC
	case SUCC, EOF:
		return PREV
	default:
		return o
	}
}

// NewFrame generates Fixed, Window, Wild and WildMin frames. The offsets argument controls what type of frame is created:
// 	- for a Wild frame, give no offsets or give a max offset of < 0 and a min of < 1
// 	- for a WildMin frame, give one offset, or give a max offset of < 0 and a min of > 0
// 	- for a Fixed frame, give two offsets that are both >= 0 and that are equal to each other
// 	- for a Window frame, give two offsets that are both >= 0 and that are not equal to each other.
func NewFrame(typ OffType, pat patterns.Pattern, offsets ...int) Frame {
	switch len(offsets) {
	case 0:
		return Frame{0, -1, typ, pat}
	case 1:
		if offsets[0] > 0 {
			return Frame{offsets[0], -1, typ, pat}
		}
		return Frame{0, -1, typ, pat}
	}
	if offsets[1] < 0 {
		if offsets[0] > 0 {
			return Frame{offsets[0], -1, typ, pat}
		}
		return Frame{0, -1, typ, pat}
	}
	if offsets[0] < 0 {
		offsets[0] = 0
	}
	return Frame{typ, offsets[0], offsets[1], pat}
}

// SwitchFrame returns a new frame with a different orientation (for example to allow right-left searching).
func SwitchFrame(f Frame, p patterns.Pattern) Frame {
	return NewFrame(f.SwitchOff(), p, f.Min, f.Max)
}

// BMHConvert converts the patterns within a slice of frames to BMH sequences if possible.
func BMHConvert(fs []Frame, rev bool) []Frame {
	nfs := make([]Frame, len(fs))
	for i, f := range fs {
		nfs[i] = NewFrame(f.Orientation(), patterns.BMH(f.Pat(), rev), f.Min, f.Max)
	}
	return nfs
}

// NonZero checks whether, when converted to simple byte sequences, this frame's pattern is all 0 bytes.
func NonZero(f Frame) bool {
	for _, seq := range f.Sequences() {
		allzeros := true
		for _, b := range seq {
			if b != 0 {
				allzeros = false
			}
		}
		if allzeros {
			return false
		}
	}
	return true
}

// TotalLength is sum of the maximum length of the enclosed pattern and the maximum offset.
func TotalLength(f Frame) int {
	// a wild frame has no total length
	if f.Max < 0 {
		return -1
	}
	_, l := f.Length()
	return l + f.Max
}

// Match the enclosed pattern against the byte slice in a L-R direction.
// Returns a slice of offsets for where a successive match by a related frame should begin.
func (f Frame) Match(b []byte) []int {
	ret := make([]int, 0, 1)
	min, max := f.Min, f.Max
	if max < 0 || max > len(b) {
		max = len(b)
	}
	for min <= max {
		length, adv := f.Test(b[min:])
		if length > -1 {
			ret = append(ret, min+length)
		}
		if adv < 1 {
			break
		}
		min += adv
	}
	return ret
}

// For the nth match (per above), return the offset for successive match by related frame and bytes that can advance to make a successive test by this frame.
func (f Frame) MatchN(b []byte, n int) (int, int) {
	var i int
	min, max := f.Min, f.Max
	if max < 0 || max > len(b) {
		max = len(b)
	}
	for min <= max {
		length, adv := f.Test(b[min:])
		if length > -1 {
			if i == n {
				return min + length, min + adv
			}
			i++
		}
		if adv < 1 {
			break
		}
		min += adv
	}
	return -1, 0
}

// Match the enclosed pattern against the byte slice in a reverse (R-L) direction. Returns a slice of offsets for where a successive match by a related frame should begin.
func (f Frame) MatchR(b []byte) []int {
	ret := make([]int, 0, 1)
	min, max := f.Min, f.Max
	if max < 0 || max > len(b) {
		max = len(b)
	}
	for min <= max {
		length, adv := f.TestR(b[:len(b)-min])
		if length > -1 {
			ret = append(ret, min+length)
		}
		if adv < 1 {
			break
		}
		min += adv
	}
	return ret
}

// For the nth match (per above), return the offset for successive match by related frame and bytes that can advance to make a successive test by this frame.
func (f Frame) MatchNR(b []byte, n int) (int, int) {
	var i int
	min, max := f.Min, w.Max
	if max < 0 || max > len(b) {
		max = len(b)
	}
	for min <= max {
		length, adv := f.TestR(b[:len(b)-min])
		if length > -1 {
			if i == n {
				return min + length, min + adv
			}
			i++
		}
		if adv < 1 {
			break
		}
		min += adv
	}
	return -1, 0
}

func (f Frame) Equals(f1 Frame) bool {
	if f.Min == f1.Min && f.Max == f1.Max && f.OffType == f1.OffType && f.Pattern.Equals(f1.Pattern) {
		return true
	}
	return false
}

func (f Frame) String() string {
	switch {
	case f.Min == f.Max && f.Min >= 0:
		"F " + OffString[f.OffType] + ":" + strconv.Itoa(f.Min) + " " + f.Pattern.String()
	case f.Min > 0 && f.Max < -1:
		"WM " + OffString[f.OffType] + ":" + strconv.Itoa(f.Min) + " " + f.Pattern.String()
	case f.Min <= 0 && f.Max < -1:
		"WL " + OffString[f.OffType] + " " + f.Pattern.String()
	default:
		"WW " + OffString[f.OffType] + ":" + strconv.Itoa(f.Min) + "-" + strconv.Itoa(f.Max) + " " + f.Pattern.String()
	}
}

// MaxMatches returns the max number of times a frame can match, given a byte slice of length 'l', and the maximum remaining slice length
func (f Frame) MaxMatches(l int) (int, int, int) {
	min, _ := f.Length()
	rem := l - min - f.Off
	if rem >= 0 {
		return 1, rem, min
	}
	return 0, 0, 0
}

// MaxMatches returns the max number of times a frame can match, given a byte slice of length 'l', and the maximum remaining slice length
// TODO: this is *wrong* because it presumes a pattern can't overlap i.e. in AAAAAA, the string AA can match at 5 positions, not 3
func (w Window) MaxMatches(l int) (int, int, int) {
	min, _ := w.Length()
	rem := l - min - w.MinOff
	if rem < 0 {
		return 0, 0, 0
	}
	if w.MaxOff+min > l {
		return rem/min + 1, rem, min
	}
	return (w.MaxOff + min - w.MinOff) / min, rem, min
}

// MaxMatches returns the max number of times a frame can match, given a byte slice of length 'l', and the maximum remaining slice length
func (w Wild) MaxMatches(l int) (int, int, int) {
	min, _ := w.Length()
	rem := l - min
	if rem < 0 {
		return 0, 0, 0
	}
	return rem/min + 1, rem, min
}

// MaxMatches returns the max number of times a frame can match, given a byte slice of length 'l', and the maximum remaining slice length
func (w WildMin) MaxMatches(l int) (int, int, int) {
	min, _ := w.Length()
	rem := l - min - w.MinOff
	if rem < 0 {
		return 0, 0, 0
	}
	return rem/min + 1, rem, min
}

// Linked tests whether a frame is linked to a preceding frame (by a preceding or succeding relationship) with an offset and range that is less than the supplied ints.
// If -1 is given for maxDistance, then will check if frame is linked to a preceding frame via a PREV or SUCC relationship.
func (f Frame) Linked(prev Frame, maxDistance, maxRange int) (bool, int, int) {
	switch f.OffType {
	case PREV:
		if maxDistance < 0 {
			return true, maxDistance, maxRange
		}
		if f.Max < 0 || f.Max > maxDistance || f.Max-w.Min > maxRange {
			return false, 0, 0
		}
		return true, maxDistance - f.Max, maxRange - (w.Max - w.Min)
	case SUCC, EOF:
		if prev.Orientation() != SUCC || prev.Max < 0 {
			return false, 0, 0
		}
		if maxDistance < 0 {
			return true, maxDistance, maxRange
		}
		if prev.Max > maxDistance || prev.Max-prev.Min > maxRange {
			return false, 0, 0
		}
		return true, maxDistance - prev.Max, maxRange - (prev.Max - prev.Min)
	default:
		return false, 0, 0
	}
}

func (f Frame) Save(*persist.LoadSaver) {
	ls.SaveInt(f.Min)
	ls.SaveInt(f.Max)
	ls.SaveByte(byte(f.OffType))
	f.Pattern.Save(ls)
}

func Load(ls *persist.LoadSaver) Frame {
	return Frame{
		ls.LoadInt(),
		ls.LoadInt(),
		OffType(ls.LoadByte()),
		patterns.Load(ls),
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 Landon Wainwright. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Starts up the blog system using the default values
package main

import (
	"flag"
	"github.com/landonia/simplegoblog/blog"
	"log"
	"os"
)

// Starts a new simple go blog server
func main() {

	// Define flags
	var postsdir, templatesdir, assetsdir, address string
	flag.StringVar(&postsdir, "pdir", "../posts", "the directory for storing the posts")
	flag.StringVar(&templatesdir, "tdir", "../templates", "the directory containing the templates")
	flag.StringVar(&assetsdir, "adir", "../assets", "the directory containing the assets")
	flag.StringVar(&address, "address", ":8080", "the host:port to run the blog on")
	flag.Parse()

	// Create a new configuration containing the info
	config := &blog.Configuration{Title: "Life thru a Lando", NoOfRecentPosts: 4, Postsdir: postsdir, Templatesdir: templatesdir, Assetsdir: assetsdir}

	// Create a new data structure for storing the data
	b := blog.New(config)

	// Start the blog server
	err := b.Start(address)
	if err != nil {
		log.Println(err)
		os.Exit(1)
	}
}
<commit_msg>New request throttle limiter<commit_after>// Copyright 2013 Landon Wainwright. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Starts up the blog system using the default values
package main

import (
	"flag"
	"github.com/landonia/simplegoblog/blog"
	"log"
	"os"
	"time"
)

// Starts a new simple go blog server
func main() {

	// Define flags
	var postsdir, templatesdir, assetsdir, address string
	flag.StringVar(&postsdir, "pdir", "../posts", "the directory for storing the posts")
	flag.StringVar(&templatesdir, "tdir", "../templates", "the directory containing the templates")
	flag.StringVar(&assetsdir, "adir", "../assets", "the directory containing the assets")
	flag.StringVar(&address, "address", ":8080", "the host:port to run the blog on")
	flag.Parse()

	// Create a new configuration containing the info
	config := &blog.Configuration{
		Title:           "Life thru a Lando",
		Postsdir:        postsdir,
		Templatesdir:    templatesdir,
		Assetsdir:       assetsdir,
		NoOfRecentPosts: 4,
		RequestHandlerLimit: blog.ThrottleLimit{
			Max: 1,
			Ttl: time.Second,
		},
		AssetHandlerLimit: blog.ThrottleLimit{
			Max: 500,
			Ttl: time.Millisecond,
		},
	}

	// Create a new data structure for storing the data
	b := blog.New(config)

	// Start the blog server
	err := b.Start(address)
	if err != nil {
		log.Println(err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>// SILVER - Service Wrapper
// Auto Updater
//
// Copyright (c) 2014 PaperCut Software http://www.papercut.com/
// Use of this source code is governed by an MIT or GPL Version 2 license.
// See the project's LICENSE file for more information.
//

// TODO:
//  - move and copy ops should support find best using same logic in service.
// FUTURE:
//  - support restart replace on Windows

package main

import (
	"archive/zip"
	"crypto/md5"
	"crypto/sha1"
	"crypto/sha256"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"hash"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"github.com/papercutsoftware/silver/lib/pathutils"
)

var (
	versionFile     = flag.String("f", ".version", "Set verison file")
	showVersion     = flag.Bool("v", false, "Display current installed version and exit")
	overrideVersion = flag.String("c", "", "Override current installed version")
	httpProxy       = flag.String("p", "", "Set HTTP proxy in format http://server:port")
	checkURL        = "" // First Arg
)

type UpgradeInfo struct {
	URL        string
	Version    string
	Md5        string
	Sha1       string
	Sha256     string
	Operations []Operation
}

type Operation struct {
	Action string
	Args   []string
}

func usage() {
	exeName := filepath.Base(os.Args[0])
	fmt.Fprintf(os.Stderr, "usage: %s [flags] [update url]\n", exeName)
	flag.PrintDefaults()
	os.Exit(2)
}

func main() {

	flag.Usage = usage
	flag.Parse()

	if *showVersion {
		fmt.Printf("Current version: %s\n", readCurrentVersion())
		return
	}

	if flag.NArg() == 0 {
		usage()
	}

	checkURL = flag.Arg(0)

	setupHTTPProxy()

	ok, err := upgradeIfRequired()
	if err != nil {
		fmt.Fprintf(os.Stderr, "ERROR: %v\n", err)
		os.Exit(1)
	}
	if !ok {
		fmt.Println("No upgrade required")
	} else {
		fmt.Printf("Upgrade successful at %s.\n", time.Now().Format(time.RFC822))
	}
}

func upgradeIfRequired() (upgraded bool, err error) {
	currentVer := readCurrentVersion()
	if len(*overrideVersion) > 0 {
		currentVer = *overrideVersion
	}

	// Ping update URL
	upgradeInfo, err := checkUpdate(checkURL, currentVer)
	if err != nil {
		return false, err
	}

	if upgradeInfo == nil || upgradeInfo.URL == "" {
		// No upgrade required
		return false, nil
	}

	// Download
	fmt.Printf("Downloading version %s update from %s ...\n",
		upgradeInfo.Version,
		upgradeInfo.URL)

	zipfile, err := download(upgradeInfo.URL)
	if err != nil {
		return false, err
	}
	defer os.Remove(zipfile)

	if size, err := fileSize(zipfile); err == nil {
		fmt.Printf("Download complete (%d bytes).\n", size)
	}

	// Validate checksum if provided
	var fileSum string
	var requiredSum string
	switch {
	case len(upgradeInfo.Sha256) > 0:
		requiredSum = upgradeInfo.Sha256
		fileSum = checksum("sha256", zipfile)
	case len(upgradeInfo.Sha1) > 0:
		requiredSum = upgradeInfo.Sha1
		fileSum = checksum("sha1", zipfile)
	case len(upgradeInfo.Md5) > 0:
		requiredSum = upgradeInfo.Md5
		fileSum = checksum("md5", zipfile)
	}

	if len(requiredSum) > 0 && fileSum != requiredSum {
		return false, errors.New("Download checksum failed!")
	}

	// Unzip
	fmt.Println("Unzipping update ...")
	err = extractZip(zipfile, ".")
	if err != nil {
		return false, err
	}
	fmt.Println("Unzip complete.")

	// Perform any operations
	for _, op := range upgradeInfo.Operations {
		action := strings.ToLower(op.Action)
		var fn func([]string) error
		switch action {
		case "exec", "run":
			fn = execOp
		case "batchrename", "batch-rename":
			fn = batchRenameOp
		case "move", "mv":
			fn = moveOp
		case "copy", "cp":
			fn = copyOp
		case "remove", "rm", "del", "delete":
			fn = removeOp
		default:
			msg := fmt.Sprintf("Invalid operation action: '%s'", action)
			return false, errors.New(msg)
		}
		fmt.Printf("Performing operation '%s (%s)' ...\n",
			action, strings.Join(op.Args, ", "))
		if err := fn(op.Args); err != nil {
			msg := fmt.Sprintf("Operation failed with error: %v", err)
			return false, errors.New(msg)
		}
	}

	// Write version file
	ioutil.WriteFile(*versionFile, []byte(upgradeInfo.Version+"\n"), 0644)

	// Request service restart by writing the reload file into our root
	ioutil.WriteFile(".reload", []byte(""), 0644)

	// Success
	return true, nil
}

func fileSize(file string) (size int64, err error) {
	f, err := os.Open(file)
	if err != nil {
		return
	}
	fi, err := f.Stat()
	if err != nil {
		return
	}
	return fi.Size(), nil
}

func checkUpdate(url string, currentVer string) (*UpgradeInfo, error) {
	client := &http.Client{}
	req, err := http.NewRequest("GET", url+"?version="+currentVer, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("User-Agent", "Update Check")

	res, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	if res.StatusCode == http.StatusNotModified {
		return nil, nil
	}

	dec := json.NewDecoder(res.Body)
	var info UpgradeInfo
	err = dec.Decode(&info)
	if err != nil {
		return nil, errors.New(fmt.Sprintf("Unable to parse JSON at %s : %v", url, err))
	}

	if info.Version != "" && info.Version == currentVer {
		// Same version!
		return nil, nil
	}

	return &info, nil
}

func download(url string) (string, error) {
	outfile, err := ioutil.TempFile("", "update-")
	if err != nil {
		return "", err
	}

	resp, err := http.Get(url)
	if err != nil {
		outfile.Close()
		os.Remove(outfile.Name())
		return "", err
	}
	defer resp.Body.Close()

	_, err = io.Copy(outfile, resp.Body)
	if err != nil {
		outfile.Close()
		os.Remove(outfile.Name())
		return "", err
	}
	outfile.Close()
	return outfile.Name(), nil
}

func checksum(hashType string, file string) string {
	var hasher hash.Hash

	switch {
	case hashType == "sha256":
		hasher = sha256.New()
	case hashType == "sha1":
		hasher = sha1.New()
	case hashType == "md5":
		hasher = md5.New()
	default:
		hasher = sha1.New()
	}
	f, err := os.Open(file)
	if err != nil {
		panic(err)
	}
	io.Copy(hasher, f)
	return fmt.Sprintf("%x", hasher.Sum(nil))
}

func extractZip(zipfile, dest string) error {
	r, err := zip.OpenReader(zipfile)
	if err != nil {
		return err
	}
	defer r.Close()
	for _, f := range r.File {
		if err := extractZipItem(f, dest); err != nil {
			return err
		}
	}
	return nil
}

func extractZipItem(f *zip.File, dest string) error {
	rc, err := f.Open()
	if err != nil {
		return err
	}
	defer rc.Close()

	path := filepath.Join(dest, f.Name)
	if f.FileInfo().IsDir() {
		os.MkdirAll(path, f.Mode())
	} else {
		f, err := os.OpenFile(
			path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
		if err != nil {
			return err
		}
		defer f.Close()

		_, err = io.Copy(f, rc)
		if err != nil {
			return err
		}
	}
	return nil
}

func readCurrentVersion() string {
	ver := "1"
	if dat, err := ioutil.ReadFile(*versionFile); err == nil {
		ver = strings.TrimSpace(string(dat))
	}
	return ver
}

func setupHTTPProxy() {
	if len(*httpProxy) > 0 {
		os.Setenv("HTTP_PROXY", *httpProxy)
		return
	}
	var proxy = ""
	if dat, err := ioutil.ReadFile("http-proxy.conf"); err == nil {
		proxy = strings.TrimSpace(string(dat))
	}
	if proxy != "" {
		os.Setenv("HTTP_PROXY", proxy)
		return
	}
}

func execOp(args []string) (err error) {
	if len(args) < 1 {
		return errors.New("Invalid exec operation format - arg expected.")
	}
	cmd := args[0]
	fmt.Printf("Running install command: %s\n", strings.Join(args, " "))
	os.Chmod(cmd, 0755)
	c := exec.Command(cmd, args[1:]...)
	c.Stdout = os.Stdout
	c.Stderr = os.Stderr
	err = c.Run()
	return err
}

func batchRenameOp(args []string) error {
	if len(args) != 3 {
		return errors.New("Invalid rename operation format - three args expected.")
	}
	root := args[0]
	find := args[1]
	replacement := args[2]
	fmt.Printf("Running batch rename operation on root %s ('%s' => '%s')\n", root, find, replacement)
	return batchRename(root, find, replacement)
}

func batchRename(root, find, replacement string) error {
	matches, err := filepath.Glob(root)
	if err != nil {
		return err
	}
	if len(matches) == 0 {
		return nil
	}

	re, err := regexp.Compile(find)
	if err != nil {
		return err
	}

	renameCnt := 0
	visitFn := func(path string, fi os.FileInfo, errin error) error {
		name := fi.Name()
		newName := re.ReplaceAllString(name, replacement)
		if name == newName {
			return nil
		}
		if !fileExists(path) {
			return nil
		}

		newPath := filepath.Join(filepath.Dir(path), newName)
		fmt.Printf("Renaming '%s' to '%s' ...\n", path, newPath)
		err := os.Rename(path, newPath)
		if err != nil {
			return err
		}
		renameCnt++
		return nil
	}

	for _, match := range matches {
		err := filepath.Walk(match, visitFn)
		if err != nil {
			return err
		}
	}
	fmt.Printf("Renamed %d files.\n", renameCnt)
	return nil
}

func moveOp(args []string) error {
	if len(args) != 2 {
		return errors.New("Invalid copy operation format - two args expected.")
	}
	src := pathutils.FindLastFile(args[0])
	fmt.Printf("Moving '%s' to '%s'...\n", src, args[1])
	return os.Rename(src, args[1])
}

func removeOp(args []string) error {
	if len(args) != 1 {
		return errors.New("Invalid remove operation format - one arg file expected.")
	}
	path := args[0]
	matches, err := filepath.Glob(path)
	if err != nil {
		return err
	}
	removeCnt := 0
	for _, match := range matches {
		fmt.Printf("Removing '%s' ...\n", match)
		if os.RemoveAll(match) != nil {
			return err
		}
		removeCnt++
	}
	fmt.Printf("Removed %d files.\n", removeCnt)
	return nil
}

func copyOp(args []string) error {
	if len(args) != 2 {
		return errors.New("Invalid copy operation format - two args expected.")
	}
	src := pathutils.FindLastFile(args[0])
	fmt.Printf("Copying '%s' to '%s'...\n", src, args[1])
	return copyFile(src, args[1])
}

func copyFile(src, dest string) error {
	s, err := os.Open(src)
	if err != nil {
		return err
	}
	defer s.Close()
	d, err := os.Create(dest)
	if err != nil {
		return err
	}
	if _, err := io.Copy(d, s); err != nil {
		d.Close()
		return err
	}
	return d.Close()
}

func fileExists(path string) bool {
	_, err := os.Stat(path)
	return !os.IsNotExist(err)
}
<commit_msg>Secure-by-default rather than by-convention<commit_after>// SILVER - Service Wrapper
// Auto Updater
//
// Copyright (c) 2014 PaperCut Software http://www.papercut.com/
// Use of this source code is governed by an MIT or GPL Version 2 license.
// See the project's LICENSE file for more information.
//

// TODO:
//  - move and copy ops should support find best using same logic in service.
// FUTURE:
//  - support restart replace on Windows

package main

import (
	"archive/zip"
	"crypto/md5"
	"crypto/sha1"
	"crypto/sha256"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"hash"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"github.com/papercutsoftware/silver/lib/pathutils"
)

var (
	versionFile     = flag.String("f", ".version", "Set verison file")
	showVersion     = flag.Bool("v", false, "Display current installed version and exit")
	overrideVersion = flag.String("c", "", "Override current installed version")
	httpProxy       = flag.String("p", "", "Set HTTP proxy in format http://server:port")
)

type UpgradeInfo struct {
	URL        string
	Version    string
	Md5        string
	Sha1       string
	Sha256     string
	Operations []Operation
}

type Operation struct {
	Action string
	Args   []string
}

func usage() {
	exeName := filepath.Base(os.Args[0])
	fmt.Fprintf(os.Stderr, "usage: %s [flags] [update url]\n", exeName)
	flag.PrintDefaults()
	os.Exit(2)
}

func main() {

	flag.Usage = usage
	flag.Parse()

	if *showVersion {
		fmt.Printf("Current version: %s\n", readCurrentVersion())
		return
	}

	if flag.NArg() == 0 {
		usage()
	}
	checkURL := flag.Arg(0)
	if !strings.HasPrefix(strings.ToLower(checkURL), "https") {
		fmt.Fprintf(os.Stderr, "ERROR: The update URL must be HTTPS for security reasons!\n")
		os.Exit(1)
	}

	setupHTTPProxy()
	ok, err := upgradeIfRequired(checkURL)
	if err != nil {
		fmt.Fprintf(os.Stderr, "ERROR: %v\n", err)
		os.Exit(1)
	}
	if !ok {
		fmt.Println("No upgrade required")
	} else {
		fmt.Printf("Upgrade successful at %s.\n", time.Now().Format(time.RFC822))
	}
}

func upgradeIfRequired(checkURL string) (upgraded bool, err error) {
	currentVer := readCurrentVersion()
	if len(*overrideVersion) > 0 {
		currentVer = *overrideVersion
	}

	// Ping update URL
	upgradeInfo, err := checkUpdate(checkURL, currentVer)
	if err != nil {
		return false, err
	}

	if upgradeInfo == nil || upgradeInfo.URL == "" {
		// No upgrade required
		return false, nil
	}

	// Download
	fmt.Printf("Downloading version %s update from %s ...\n",
		upgradeInfo.Version,
		upgradeInfo.URL)

	zipfile, err := download(upgradeInfo.URL)
	if err != nil {
		return false, err
	}
	defer os.Remove(zipfile)

	if size, err := fileSize(zipfile); err == nil {
		fmt.Printf("Download complete (%d bytes).\n", size)
	}

	// Validate checksum
	var fileSum string
	var requiredSum string
	switch {
	case len(upgradeInfo.Sha256) > 0:
		requiredSum = upgradeInfo.Sha256
		fileSum = checksum("sha256", zipfile)
	case len(upgradeInfo.Sha1) > 0:
		requiredSum = upgradeInfo.Sha1
		fileSum = checksum("sha1", zipfile)
	default:
		return false, errors.New("Upgrade failed: The upgrade URL did not provide a checksum!")
	}

	if fileSum != requiredSum {
		return false, errors.New("Download checksum failed!")
	}

	// Unzip
	fmt.Println("Unzipping update ...")
	err = extractZip(zipfile, ".")
	if err != nil {
		return false, err
	}
	fmt.Println("Unzip complete.")

	// Perform any operations
	for _, op := range upgradeInfo.Operations {
		action := strings.ToLower(op.Action)
		var fn func([]string) error
		switch action {
		case "exec", "run":
			fn = execOp
		case "batchrename", "batch-rename":
			fn = batchRenameOp
		case "move", "mv":
			fn = moveOp
		case "copy", "cp":
			fn = copyOp
		case "remove", "rm", "del", "delete":
			fn = removeOp
		default:
			msg := fmt.Sprintf("Invalid operation action: '%s'", action)
			return false, errors.New(msg)
		}
		fmt.Printf("Performing operation '%s (%s)' ...\n",
			action, strings.Join(op.Args, ", "))
		if err := fn(op.Args); err != nil {
			msg := fmt.Sprintf("Operation failed with error: %v", err)
			return false, errors.New(msg)
		}
	}

	// Write version file
	ioutil.WriteFile(*versionFile, []byte(upgradeInfo.Version+"\n"), 0644)

	// Request service restart by writing the reload file into our root
	ioutil.WriteFile(".reload", []byte(""), 0644)

	// Success
	return true, nil
}

func fileSize(file string) (size int64, err error) {
	f, err := os.Open(file)
	if err != nil {
		return
	}
	fi, err := f.Stat()
	if err != nil {
		return
	}
	return fi.Size(), nil
}

func checkUpdate(url string, currentVer string) (*UpgradeInfo, error) {
	client := &http.Client{}
	req, err := http.NewRequest("GET", url+"?version="+currentVer, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("User-Agent", "Update Check")

	res, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	if res.StatusCode == http.StatusNotModified {
		return nil, nil
	}

	dec := json.NewDecoder(res.Body)
	var info UpgradeInfo
	err = dec.Decode(&info)
	if err != nil {
		return nil, errors.New(fmt.Sprintf("Unable to parse JSON at %s : %v", url, err))
	}

	if info.Version != "" && info.Version == currentVer {
		// Same version!
		return nil, nil
	}

	return &info, nil
}

func download(url string) (string, error) {
	outfile, err := ioutil.TempFile("", "update-")
	if err != nil {
		return "", err
	}

	resp, err := http.Get(url)
	if err != nil {
		outfile.Close()
		os.Remove(outfile.Name())
		return "", err
	}
	defer resp.Body.Close()

	_, err = io.Copy(outfile, resp.Body)
	if err != nil {
		outfile.Close()
		os.Remove(outfile.Name())
		return "", err
	}
	outfile.Close()
	return outfile.Name(), nil
}

func checksum(hashType string, file string) string {
	var hasher hash.Hash

	switch {
	case hashType == "sha256":
		hasher = sha256.New()
	case hashType == "sha1":
		hasher = sha1.New()
	case hashType == "md5":
		hasher = md5.New()
	default:
		hasher = sha1.New()
	}
	f, err := os.Open(file)
	if err != nil {
		panic(err)
	}
	io.Copy(hasher, f)
	return fmt.Sprintf("%x", hasher.Sum(nil))
}

func extractZip(zipfile, dest string) error {
	r, err := zip.OpenReader(zipfile)
	if err != nil {
		return err
	}
	defer r.Close()
	for _, f := range r.File {
		if err := extractZipItem(f, dest); err != nil {
			return err
		}
	}
	return nil
}

func extractZipItem(f *zip.File, dest string) error {
	rc, err := f.Open()
	if err != nil {
		return err
	}
	defer rc.Close()

	path := filepath.Join(dest, f.Name)
	if f.FileInfo().IsDir() {
		os.MkdirAll(path, f.Mode())
	} else {
		f, err := os.OpenFile(
			path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
		if err != nil {
			return err
		}
		defer f.Close()

		_, err = io.Copy(f, rc)
		if err != nil {
			return err
		}
	}
	return nil
}

func readCurrentVersion() string {
	ver := "1"
	if dat, err := ioutil.ReadFile(*versionFile); err == nil {
		ver = strings.TrimSpace(string(dat))
	}
	return ver
}

func setupHTTPProxy() {
	if len(*httpProxy) > 0 {
		os.Setenv("HTTP_PROXY", *httpProxy)
		return
	}
	var proxy = ""
	if dat, err := ioutil.ReadFile("http-proxy.conf"); err == nil {
		proxy = strings.TrimSpace(string(dat))
	}
	if proxy != "" {
		os.Setenv("HTTP_PROXY", proxy)
		return
	}
}

func execOp(args []string) (err error) {
	if len(args) < 1 {
		return errors.New("Invalid exec operation format - arg expected.")
	}
	cmd := args[0]
	fmt.Printf("Running install command: %s\n", strings.Join(args, " "))
	os.Chmod(cmd, 0755)
	c := exec.Command(cmd, args[1:]...)
	c.Stdout = os.Stdout
	c.Stderr = os.Stderr
	err = c.Run()
	return err
}

func batchRenameOp(args []string) error {
	if len(args) != 3 {
		return errors.New("Invalid rename operation format - three args expected.")
	}
	root := args[0]
	find := args[1]
	replacement := args[2]
	fmt.Printf("Running batch rename operation on root %s ('%s' => '%s')\n", root, find, replacement)
	return batchRename(root, find, replacement)
}

func batchRename(root, find, replacement string) error {
	matches, err := filepath.Glob(root)
	if err != nil {
		return err
	}
	if len(matches) == 0 {
		return nil
	}

	re, err := regexp.Compile(find)
	if err != nil {
		return err
	}

	renameCnt := 0
	visitFn := func(path string, fi os.FileInfo, errin error) error {
		name := fi.Name()
		newName := re.ReplaceAllString(name, replacement)
		if name == newName {
			return nil
		}
		if !fileExists(path) {
			return nil
		}

		newPath := filepath.Join(filepath.Dir(path), newName)
		fmt.Printf("Renaming '%s' to '%s' ...\n", path, newPath)
		err := os.Rename(path, newPath)
		if err != nil {
			return err
		}
		renameCnt++
		return nil
	}

	for _, match := range matches {
		err := filepath.Walk(match, visitFn)
		if err != nil {
			return err
		}
	}
	fmt.Printf("Renamed %d files.\n", renameCnt)
	return nil
}

func moveOp(args []string) error {
	if len(args) != 2 {
		return errors.New("Invalid copy operation format - two args expected.")
	}
	src := pathutils.FindLastFile(args[0])
	fmt.Printf("Moving '%s' to '%s'...\n", src, args[1])
	return os.Rename(src, args[1])
}

func removeOp(args []string) error {
	if len(args) != 1 {
		return errors.New("Invalid remove operation format - one arg file expected.")
	}
	path := args[0]
	matches, err := filepath.Glob(path)
	if err != nil {
		return err
	}
	removeCnt := 0
	for _, match := range matches {
		fmt.Printf("Removing '%s' ...\n", match)
		if os.RemoveAll(match) != nil {
			return err
		}
		removeCnt++
	}
	fmt.Printf("Removed %d files.\n", removeCnt)
	return nil
}

func copyOp(args []string) error {
	if len(args) != 2 {
		return errors.New("Invalid copy operation format - two args expected.")
	}
	src := pathutils.FindLastFile(args[0])
	fmt.Printf("Copying '%s' to '%s'...\n", src, args[1])
	return copyFile(src, args[1])
}

func copyFile(src, dest string) error {
	s, err := os.Open(src)
	if err != nil {
		return err
	}
	defer s.Close()
	d, err := os.Create(dest)
	if err != nil {
		return err
	}
	if _, err := io.Copy(d, s); err != nil {
		d.Close()
		return err
	}
	return d.Close()
}

func fileExists(path string) bool {
	_, err := os.Stat(path)
	return !os.IsNotExist(err)
}
<|endoftext|>


<commit_before>package upload

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/meatballhat/artifacts/path"
	"github.com/mitchellh/goamz/aws"
	"github.com/mitchellh/goamz/s3"
)

type uploader struct {
	BucketName   string
	Paths        *path.PathSet
	TargetPath   string
	CacheControl string
}

// Upload does the deed!
func Upload(opts *Options) {
	newUploader(opts).Upload()
}

func newUploader(opts *Options) *uploader {
	u := &uploader{
		BucketName: opts.BucketName,
		TargetPath: opts.TargetPath,
		Paths:      path.NewPathSet(),
	}

	if opts.Private {
		opts.CacheControl = "private"
	} else if opts.CacheControl == "" {
		opts.CacheControl = "public, max-age=315360000"
	}

	u.CacheControl = opts.CacheControl

	for _, s := range opts.Paths {
		parts := strings.SplitN(s, ":", 2)
		if len(parts) < 2 {
			parts = append(parts, "")
		}
		u.Paths.Add(path.NewPath(opts.WorkingDir, parts[0], parts[1]))
	}

	return u
}

func (u *uploader) Upload() error {
	auth, err := aws.GetAuth("", "")
	if err != nil {
		return err
	}

	conn := s3.New(auth, aws.USEast)
	bucket := conn.Bucket(u.BucketName)

	if bucket == nil {
		return fmt.Errorf("failed to get bucket")
	}

	for artifact := range u.files() {
		u.uploadFile(bucket, artifact)
	}

	return nil
}

func (u *uploader) files() chan *artifact {
	artifacts := make(chan *artifact)

	go func() {
		for _, path := range u.Paths.All() {
			to, from, root := path.To, path.From, path.Root
			if path.IsDir() {
				root = filepath.Join(root, from)
				if strings.HasSuffix(root, "/") {
					root = root + "/"
				}
			}

			filepath.Walk(path.Fullpath(), func(f string, info os.FileInfo, err error) error {
				if info != nil && info.IsDir() {
					return nil
				}

				relPath := strings.Replace(strings.Replace(f, root, "", -1), root+"/", "", -1)
				destination := relPath
				if len(to) > 0 {
					if path.IsDir() {
						destination = filepath.Join(to, relPath)
					} else {
						destination = to
					}
				}

				artifacts <- &artifact{Source: f, Destination: destination}
				return nil
			})

		}
		close(artifacts)
	}()

	return artifacts
}

func (u *uploader) uploadFile(b *s3.Bucket, a *artifact) error {
	retries := 0

	for {
		fmt.Printf("uploading %q -> %q\n", a.Source, a.Destination)
		err := u.rawUpload(b, a)
		if err != nil {
			if retries < 2 {
				retries += 1
				continue
			} else {
				return err
			}
		}
		return nil
	}
}

func (u *uploader) rawUpload(b *s3.Bucket, a *artifact) error {
	destination := strings.TrimLeft(filepath.Join(u.TargetPath, a.Destination), "/")

	reader, err := a.Reader()
	if err != nil {
		return err
	}

	return b.PutReaderHeader(destination, reader, a.Size(),
		map[string][]string{
			"Content-Type":  []string{a.ContentType()},
			"Cache-Control": []string{u.CacheControl},
		}, s3.Private)
}
<commit_msg>Reporting a more correct-ish upload dest<commit_after>package upload

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/meatballhat/artifacts/path"
	"github.com/mitchellh/goamz/aws"
	"github.com/mitchellh/goamz/s3"
)

type uploader struct {
	BucketName   string
	Paths        *path.PathSet
	TargetPath   string
	CacheControl string
}

// Upload does the deed!
func Upload(opts *Options) {
	newUploader(opts).Upload()
}

func newUploader(opts *Options) *uploader {
	u := &uploader{
		BucketName: opts.BucketName,
		TargetPath: opts.TargetPath,
		Paths:      path.NewPathSet(),
	}

	if opts.Private {
		opts.CacheControl = "private"
	} else if opts.CacheControl == "" {
		opts.CacheControl = "public, max-age=315360000"
	}

	u.CacheControl = opts.CacheControl

	for _, s := range opts.Paths {
		parts := strings.SplitN(s, ":", 2)
		if len(parts) < 2 {
			parts = append(parts, "")
		}
		u.Paths.Add(path.NewPath(opts.WorkingDir, parts[0], parts[1]))
	}

	return u
}

func (u *uploader) Upload() error {
	auth, err := aws.GetAuth("", "")
	if err != nil {
		return err
	}

	conn := s3.New(auth, aws.USEast)
	bucket := conn.Bucket(u.BucketName)

	if bucket == nil {
		return fmt.Errorf("failed to get bucket")
	}

	for artifact := range u.files() {
		u.uploadFile(bucket, artifact)
	}

	return nil
}

func (u *uploader) files() chan *artifact {
	artifacts := make(chan *artifact)

	go func() {
		for _, path := range u.Paths.All() {
			to, from, root := path.To, path.From, path.Root
			if path.IsDir() {
				root = filepath.Join(root, from)
				if strings.HasSuffix(root, "/") {
					root = root + "/"
				}
			}

			filepath.Walk(path.Fullpath(), func(f string, info os.FileInfo, err error) error {
				if info != nil && info.IsDir() {
					return nil
				}

				relPath := strings.Replace(strings.Replace(f, root, "", -1), root+"/", "", -1)
				destination := relPath
				if len(to) > 0 {
					if path.IsDir() {
						destination = filepath.Join(to, relPath)
					} else {
						destination = to
					}
				}

				artifacts <- &artifact{Source: f, Destination: destination}
				return nil
			})

		}
		close(artifacts)
	}()

	return artifacts
}

func (u *uploader) uploadFile(b *s3.Bucket, a *artifact) error {
	retries := 0

	for {
		err := u.rawUpload(b, a)
		if err != nil {
			if retries < 2 {
				retries += 1
				continue
			} else {
				return err
			}
		}
		return nil
	}
}

func (u *uploader) rawUpload(b *s3.Bucket, a *artifact) error {
	destination := strings.TrimLeft(filepath.Join(u.TargetPath, a.Destination), "/")

	reader, err := a.Reader()
	if err != nil {
		return err
	}

	fmt.Printf("uploading %q -> %q\n", a.Source, destination)

	return b.PutReaderHeader(destination, reader, a.Size(),
		map[string][]string{
			"Content-Type":  []string{a.ContentType()},
			"Cache-Control": []string{u.CacheControl},
		}, s3.Private)
}
<|endoftext|>


<commit_before>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package signal

import (
	"bytes"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"syscall"
	"testing"
	"time"
)

func sendCtrlBreak(t *testing.T, pid int) {
	d, e := syscall.LoadDLL("kernel32.dll")
	if e != nil {
		t.Fatalf("LoadDLL: %v\n", e)
	}
	p, e := d.FindProc("GenerateConsoleCtrlEvent")
	if e != nil {
		t.Fatalf("FindProc: %v\n", e)
	}
	r, _, e := p.Call(syscall.CTRL_BREAK_EVENT, uintptr(pid))
	if r == 0 {
		t.Fatalf("GenerateConsoleCtrlEvent: %v\n", e)
	}
}

func TestCtrlBreak(t *testing.T) {
	// create source file
	const source = `
package main

import (
	"log"
	"os"
	"os/signal"
	"time"
)

func main() {
	c := make(chan os.Signal, 10)
	signal.Notify(c)
	select {
	case s := <-c:
		if s != os.Interrupt {
			log.Fatalf("Wrong signal received: got %q, want %q\n", s, os.Interrupt)
		}
	case <-time.After(3 * time.Second):
		log.Fatalf("Timeout waiting for Ctrl+Break\n")
	}
}
`
	tmp, err := ioutil.TempDir("", "TestCtrlBreak")
	if err != nil {
		t.Fatal("TempDir failed: ", err)
	}
	defer os.RemoveAll(tmp)

	// write ctrlbreak.go
	name := filepath.Join(tmp, "ctlbreak")
	src := name + ".go"
	f, err := os.Create(src)
	if err != nil {
		t.Fatalf("Failed to create %v: %v", src, err)
	}
	defer f.Close()
	f.Write([]byte(source))

	// compile it
	exe := name + ".exe"
	defer os.Remove(exe)
	o, err := exec.Command("go", "build", "-o", exe, src).CombinedOutput()
	if err != nil {
		t.Fatalf("Failed to compile: %v\n%v", err, string(o))
	}

	// run it
	cmd := exec.Command(exe)
	var b bytes.Buffer
	cmd.Stdout = &b
	cmd.Stderr = &b
	cmd.SysProcAttr = &syscall.SysProcAttr{
		CreationFlags: syscall.CREATE_NEW_PROCESS_GROUP,
	}
	err = cmd.Start()
	if err != nil {
		t.Fatalf("Start failed: %v", err)
	}
	go func() {
		time.Sleep(1 * time.Second)
		sendCtrlBreak(t, cmd.Process.Pid)
	}()
	err = cmd.Wait()
	if err != nil {
		t.Fatalf("Program exited with error: %v\n%v", err, string(b.Bytes()))
	}
}
<commit_msg>os/signal: skip TestCtrlBreak on windows/386 temporarily<commit_after>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package signal

import (
	"bytes"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"syscall"
	"testing"
	"time"
)

func sendCtrlBreak(t *testing.T, pid int) {
	d, e := syscall.LoadDLL("kernel32.dll")
	if e != nil {
		t.Fatalf("LoadDLL: %v\n", e)
	}
	p, e := d.FindProc("GenerateConsoleCtrlEvent")
	if e != nil {
		t.Fatalf("FindProc: %v\n", e)
	}
	r, _, e := p.Call(syscall.CTRL_BREAK_EVENT, uintptr(pid))
	if r == 0 {
		t.Fatalf("GenerateConsoleCtrlEvent: %v\n", e)
	}
}

func TestCtrlBreak(t *testing.T) {
	if runtime.GOARCH == "386" {
		t.Skip("known failing test on windows/386, see https://golang.org/issue/10215")
	}
	// create source file
	const source = `
package main

import (
	"log"
	"os"
	"os/signal"
	"time"
)

func main() {
	c := make(chan os.Signal, 10)
	signal.Notify(c)
	select {
	case s := <-c:
		if s != os.Interrupt {
			log.Fatalf("Wrong signal received: got %q, want %q\n", s, os.Interrupt)
		}
	case <-time.After(3 * time.Second):
		log.Fatalf("Timeout waiting for Ctrl+Break\n")
	}
}
`
	tmp, err := ioutil.TempDir("", "TestCtrlBreak")
	if err != nil {
		t.Fatal("TempDir failed: ", err)
	}
	defer os.RemoveAll(tmp)

	// write ctrlbreak.go
	name := filepath.Join(tmp, "ctlbreak")
	src := name + ".go"
	f, err := os.Create(src)
	if err != nil {
		t.Fatalf("Failed to create %v: %v", src, err)
	}
	defer f.Close()
	f.Write([]byte(source))

	// compile it
	exe := name + ".exe"
	defer os.Remove(exe)
	o, err := exec.Command("go", "build", "-o", exe, src).CombinedOutput()
	if err != nil {
		t.Fatalf("Failed to compile: %v\n%v", err, string(o))
	}

	// run it
	cmd := exec.Command(exe)
	var b bytes.Buffer
	cmd.Stdout = &b
	cmd.Stderr = &b
	cmd.SysProcAttr = &syscall.SysProcAttr{
		CreationFlags: syscall.CREATE_NEW_PROCESS_GROUP,
	}
	err = cmd.Start()
	if err != nil {
		t.Fatalf("Start failed: %v", err)
	}
	go func() {
		time.Sleep(1 * time.Second)
		sendCtrlBreak(t, cmd.Process.Pid)
	}()
	err = cmd.Wait()
	if err != nil {
		t.Fatalf("Program exited with error: %v\n%v", err, string(b.Bytes()))
	}
}
<|endoftext|>


<commit_before>package utils

import (
	"net/http"

	// have to use this context version to satisfy AccessController interface :-/
	"github.com/docker/distribution/context"
	"github.com/docker/distribution/registry/auth"
)

// TestingAccessController is for TEST USE ONLY!!!
// It allows you to configure an AccessController that always
// succeeds or fails.
type TestingAccessController struct {
	Err error
}

var _ auth.AccessController = TestingAccessController{}

// Authorized will either always error, or always succeed, depending on the
// configuration of the TestingAccessController
func (ac TestingAccessController) Authorized(ctx context.Context, access ...auth.Access) (context.Context, error) {
	return ctx, ac.Err
}

// TestingAuthChallenge is for TEST USE ONLY!!!
// It implements the auth.Challenge interface and allows a test to confirm
// the the SetHeaders method was called.
type TestingAuthChallenge struct {
	SetHeadersCalled bool
}

var _ auth.Challenge = &TestingAuthChallenge{}

func (c TestingAuthChallenge) Error() string {
	return "TestingAuthChallenge"
}

// SetHeaders just records that the function was called for the purpose of testing
func (c *TestingAuthChallenge) SetHeaders(w http.ResponseWriter) {
	c.SetHeadersCalled = true
}
<commit_msg>Remove use of distributions old wrapped context<commit_after>package utils

import (
	"context"
	"net/http"

	"github.com/docker/distribution/registry/auth"
)

// TestingAccessController is for TEST USE ONLY!!!
// It allows you to configure an AccessController that always
// succeeds or fails.
type TestingAccessController struct {
	Err error
}

var _ auth.AccessController = TestingAccessController{}

// Authorized will either always error, or always succeed, depending on the
// configuration of the TestingAccessController
func (ac TestingAccessController) Authorized(ctx context.Context, access ...auth.Access) (context.Context, error) {
	return ctx, ac.Err
}

// TestingAuthChallenge is for TEST USE ONLY!!!
// It implements the auth.Challenge interface and allows a test to confirm
// the the SetHeaders method was called.
type TestingAuthChallenge struct {
	SetHeadersCalled bool
}

var _ auth.Challenge = &TestingAuthChallenge{}

func (c TestingAuthChallenge) Error() string {
	return "TestingAuthChallenge"
}

// SetHeaders just records that the function was called for the purpose of testing
func (c *TestingAuthChallenge) SetHeaders(w http.ResponseWriter) {
	c.SetHeadersCalled = true
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"os/signal"

	"gitlab.com/gomidi/midi/v2"
	"gitlab.com/gomidi/midi/v2/drivers"

	_ "gitlab.com/gomidi/midi/v2/drivers/rtmididrv"
	"gitlab.com/metakeule/config"
)

var (
	cfg      = config.MustNew("midispy", "2.0.1", "spy on the MIDI data that is sent from a device to another.")
	inArg    = cfg.NewInt32("in", "number of the input device", config.Required, config.Shortflag('i'))
	outArg   = cfg.NewInt32("out", "number of the output device", config.Shortflag('o'))
	noLogArg = cfg.NewBool("nolog", "don't log, just connect in and out", config.Shortflag('n'))
	shortArg = cfg.NewBool("short", "log the short way", config.Shortflag('s'))
	listCmd  = cfg.MustCommand("list", "list devices").Relax("in").Relax("out")
)

func main() {
	if err := run(); err != nil {
		fmt.Fprintf(os.Stderr, "ERROR: %s\n", err.Error())
		os.Exit(1)
	}
}

func run() (err error) {
	defer midi.CloseDriver()

	if err := cfg.Run(); err != nil {
		listMIDIInDevices()
		return err
	}

	if cfg.ActiveCommand() == listCmd {
		listMIDIDevices()
		return nil
	}

	err = startSpying(!noLogArg.Get())

	if err != nil {
		return err
	}

	sigchan := make(chan os.Signal, 10)

	// listen for ctrl+c
	go signal.Notify(sigchan, os.Interrupt)

	// interrupt has happend
	<-sigchan
	fmt.Println("\n--interrupted!")

	return nil
}

func listMIDIDevices() {
	listMIDIInDevices()

	fmt.Print("\n--- MIDI output ports ---\n\n")

	for num, port := range midi.OutPorts() {
		fmt.Printf("[%d] %#v\n", num, port)
	}

	return
}

func listMIDIInDevices() {
	fmt.Print("\n--- MIDI input ports ---\n\n")

	for num, port := range midi.InPorts() {
		fmt.Printf("[%d] %#v\n", num, port)
	}
}

func startSpying(shouldlog bool) error {

	in := inArg.Get()

	inPort, err := drivers.InByNumber(int(in))
	if err != nil {
		return err
	}

	err = inPort.Open()

	if err != nil {
		return err
	}

	var outPort drivers.Out = nil
	var logfn func(...interface{})

	if outArg.IsSet() {

		out := outArg.Get()
		outPort, err = drivers.OutByNumber(int(out))
		if err != nil {
			return err
		}

		err = outPort.Open()

		if err != nil {
			return err
		}

		fmt.Printf("[%d] %#v\n->\n[%d] %#v\n-----------------------\n",
			inPort.Number(), inPort.String(), outPort.Number(), outPort.String())
		logfn = logger(in, out)
	} else {
		fmt.Printf("[%d] %#v\n-----------------------\n",
			inPort.Number(), inPort.String())
		logfn = logger(in, 0)
	}

	recv := func(m midi.Message, absmillisec int32) {}

	if shouldlog {
		recv = func(m midi.Message, absmillisec int32) {
			logfn(m)
		}
	}

	return Run(inPort, outPort, recv)
}

func logger(in, out int32) func(...interface{}) {
	if shortArg.Get() {
		return func(v ...interface{}) {
			fmt.Println(v...)
		}
	}
	if outArg.IsSet() {
		l := log.New(os.Stdout, fmt.Sprintf("[%d->%d] ", in, out), log.Lmicroseconds)
		return l.Println
	}

	l := log.New(os.Stdout, fmt.Sprintf("[%d] ", in), log.Lmicroseconds)
	return l.Println
}
<commit_msg>fix the version number of midispy<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"os/signal"

	"gitlab.com/gomidi/midi/v2"
	"gitlab.com/gomidi/midi/v2/drivers"

	_ "gitlab.com/gomidi/midi/v2/drivers/rtmididrv"
	"gitlab.com/metakeule/config"
)

var (
	cfg      = config.MustNew("midispy", "1.21.2", "spy on the MIDI data that is sent from a device to another.")
	inArg    = cfg.NewInt32("in", "number of the input device", config.Required, config.Shortflag('i'))
	outArg   = cfg.NewInt32("out", "number of the output device", config.Shortflag('o'))
	noLogArg = cfg.NewBool("nolog", "don't log, just connect in and out", config.Shortflag('n'))
	shortArg = cfg.NewBool("short", "log the short way", config.Shortflag('s'))
	listCmd  = cfg.MustCommand("list", "list devices").Relax("in").Relax("out")
)

func main() {
	if err := run(); err != nil {
		fmt.Fprintf(os.Stderr, "ERROR: %s\n", err.Error())
		os.Exit(1)
	}
}

func run() (err error) {
	defer midi.CloseDriver()

	if err := cfg.Run(); err != nil {
		listMIDIInDevices()
		return err
	}

	if cfg.ActiveCommand() == listCmd {
		listMIDIDevices()
		return nil
	}

	err = startSpying(!noLogArg.Get())

	if err != nil {
		return err
	}

	sigchan := make(chan os.Signal, 10)

	// listen for ctrl+c
	go signal.Notify(sigchan, os.Interrupt)

	// interrupt has happend
	<-sigchan
	fmt.Println("\n--interrupted!")

	return nil
}

func listMIDIDevices() {
	listMIDIInDevices()

	fmt.Print("\n--- MIDI output ports ---\n\n")

	for num, port := range midi.OutPorts() {
		fmt.Printf("[%d] %#v\n", num, port)
	}

	return
}

func listMIDIInDevices() {
	fmt.Print("\n--- MIDI input ports ---\n\n")

	for num, port := range midi.InPorts() {
		fmt.Printf("[%d] %#v\n", num, port)
	}
}

func startSpying(shouldlog bool) error {

	in := inArg.Get()

	inPort, err := drivers.InByNumber(int(in))
	if err != nil {
		return err
	}

	err = inPort.Open()

	if err != nil {
		return err
	}

	var outPort drivers.Out = nil
	var logfn func(...interface{})

	if outArg.IsSet() {

		out := outArg.Get()
		outPort, err = drivers.OutByNumber(int(out))
		if err != nil {
			return err
		}

		err = outPort.Open()

		if err != nil {
			return err
		}

		fmt.Printf("[%d] %#v\n->\n[%d] %#v\n-----------------------\n",
			inPort.Number(), inPort.String(), outPort.Number(), outPort.String())
		logfn = logger(in, out)
	} else {
		fmt.Printf("[%d] %#v\n-----------------------\n",
			inPort.Number(), inPort.String())
		logfn = logger(in, 0)
	}

	recv := func(m midi.Message, absmillisec int32) {}

	if shouldlog {
		recv = func(m midi.Message, absmillisec int32) {
			logfn(m)
		}
	}

	return Run(inPort, outPort, recv)
}

func logger(in, out int32) func(...interface{}) {
	if shortArg.Get() {
		return func(v ...interface{}) {
			fmt.Println(v...)
		}
	}
	if outArg.IsSet() {
		l := log.New(os.Stdout, fmt.Sprintf("[%d->%d] ", in, out), log.Lmicroseconds)
		return l.Println
	}

	l := log.New(os.Stdout, fmt.Sprintf("[%d] ", in), log.Lmicroseconds)
	return l.Println
}
<|endoftext|>


<commit_before>package mo_linkedapp

import (
	"encoding/json"
	"github.com/watermint/toolbox/app"
	"github.com/watermint/toolbox/domain/infra/api_parser"
	"github.com/watermint/toolbox/domain/model/mo_member"
	"go.uber.org/zap"
)

type LinkedApp struct {
	Raw          json.RawMessage
	TeamMemberId string `path:"-"`
	AppId        string `path:"app_id"`
	AppName      string `path:"app_name"`
	IsAppFolder  bool   `path:"is_app_folder"`
	Publisher    string `path:"publisher"`
	PublisherUrl string `path:"publisher_url"`
	Linked       string `path:"linked"`
}

type MemberLinkedApp struct {
	Raw             json.RawMessage
	TeamMemberId    string `path:"profile.team_member_id"`
	Email           string `path:"profile.email"`
	Status          string `path:"profile.status.\\.tag"`
	GivenName       string `path:"profile.name.given_name"`
	Surname         string `path:"profile.name.surname"`
	FamiliarName    string `path:"profile.name.familiar_name"`
	DisplayName     string `path:"profile.name.display_name"`
	AbbreviatedName string `path:"profile.name.abbreviated_name"`
	ExternalId      string `path:"profile.external_id"`
	AccountId       string `path:"profile.account_id"`
	AppId           string `path:"linked_app.app_id"`
	AppName         string `path:"linked_app.app_name"`
	IsAppFolder     bool   `path:"linked_app.is_app_folder"`
	Publisher       string `path:"linked_app.publisher"`
	PublisherUrl    string `path:"linked_app.publisher_url"`
	Linked          string `path:"linked_app.linked"`
}

func NewMemberLinkedApp(member *mo_member.Member, linkedApp *LinkedApp) (mla *MemberLinkedApp) {
	raws := make(map[string]json.RawMessage)
	raws["profile"] = member.Profile().Raw
	raws["linked_app"] = linkedApp.Raw
	raw := api_parser.CombineRaw(raws)

	mla = &MemberLinkedApp{}
	if err := api_parser.ParseModelRaw(mla, raw); err != nil {
		app.Root().Log().Error("unable to parse", zap.Error(err))
	}
	return mla
}
<commit_msg>fix model for linked app<commit_after>package mo_linkedapp

import (
	"encoding/json"
	"github.com/tidwall/gjson"
	"github.com/watermint/toolbox/app"
	"github.com/watermint/toolbox/domain/infra/api_parser"
	"github.com/watermint/toolbox/domain/model/mo_member"
	"go.uber.org/zap"
)

type LinkedApp struct {
	Raw          json.RawMessage
	TeamMemberId string `path:"-"`
	AppId        string `path:"app_id"`
	AppName      string `path:"app_name"`
	IsAppFolder  bool   `path:"is_app_folder"`
	Publisher    string `path:"publisher"`
	PublisherUrl string `path:"publisher_url"`
	Linked       string `path:"linked"`
}

type MemberLinkedApp struct {
	Raw             json.RawMessage
	TeamMemberId    string `path:"profile.team_member_id"`
	Email           string `path:"profile.email"`
	Status          string `path:"profile.status.\\.tag"`
	GivenName       string `path:"profile.name.given_name"`
	Surname         string `path:"profile.name.surname"`
	FamiliarName    string `path:"profile.name.familiar_name"`
	DisplayName     string `path:"profile.name.display_name"`
	AbbreviatedName string `path:"profile.name.abbreviated_name"`
	ExternalId      string `path:"profile.external_id"`
	AccountId       string `path:"profile.account_id"`
	AppId           string `path:"linked_app.app_id"`
	AppName         string `path:"linked_app.app_name"`
	IsAppFolder     bool   `path:"linked_app.is_app_folder"`
	Publisher       string `path:"linked_app.publisher"`
	PublisherUrl    string `path:"linked_app.publisher_url"`
	Linked          string `path:"linked_app.linked"`
}

func NewMemberLinkedApp(member *mo_member.Member, linkedApp *LinkedApp) (mla *MemberLinkedApp) {
	prof := gjson.ParseBytes(member.Raw).Get("profile")
	raws := make(map[string]json.RawMessage)
	raws["profile"] = json.RawMessage(prof.Raw)
	raws["linked_app"] = linkedApp.Raw
	raw := api_parser.CombineRaw(raws)

	mla = &MemberLinkedApp{}
	if err := api_parser.ParseModelRaw(mla, raw); err != nil {
		app.Root().Log().Error("unable to parse", zap.Error(err))
	}
	return mla
}
<|endoftext|>


<commit_before>// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements.  See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License.  You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package filter contains transformations for removing pipeline elements based on
// various conditions.
package filter

import (
	"reflect"

	"github.com/apache/beam/sdks/go/pkg/beam"
	"github.com/apache/beam/sdks/go/pkg/beam/core/funcx"
	"github.com/apache/beam/sdks/go/pkg/beam/core/typex"
)

var (
	sig = funcx.MakePredicate(beam.TType) // T -> bool
)

func init() {
	beam.RegisterType(reflect.TypeOf((*filterFn)(nil)).Elem())
}

// Include filters the elements of a PCollection<A> based on the given function,
// which must be of the form: A -> bool. Include removes all element for which
// the filter function returns false. It returns a PCollection of the same type
// as the input. For example:
//
//    words := beam.Create(s, "a", "b", "long", "alsolong")
//    short := filter.Include(s, words, func(s string) bool {
//        return len(s) < 3
//    })
//
// Here, "short" will contain "a" and "b" at runtime.
func Include(s beam.Scope, col beam.PCollection, fn interface{}) beam.PCollection {
	s = s.Scope("filter.Include")

	t := typex.SkipW(col.Type()).Type()
	funcx.MustSatisfy(fn, funcx.Replace(sig, beam.TType, t))

	return beam.ParDo(s, &filterFn{Predicate: beam.EncodedFn{Fn: reflect.ValueOf(fn)}, Include: true}, col)
}

// Exclude filters the elements of a PCollection<A> based on the given function,
// which must be of the form: A -> bool. Exclude removes all element for which
// the filter function returns true. It returns a PCollection of the same type
// as the input. For example:
//
//    words := beam.Create(s, "a", "b", "long", "alsolong")
//    long := filter.Exclude(s, words, func(s string) bool {
//        return len(s) < 3
//    })
//
// Here, "long" will contain "long" and "alsolong" at runtime.
func Exclude(s beam.Scope, col beam.PCollection, fn interface{}) beam.PCollection {
	s = s.Scope("filter.Exclude")

	t := typex.SkipW(col.Type()).Type()
	funcx.MustSatisfy(fn, funcx.Replace(sig, beam.TType, t))

	return beam.ParDo(s, &filterFn{Predicate: beam.EncodedFn{Fn: reflect.ValueOf(fn)}, Include: false}, col)
}

type filterFn struct {
	// Predicate is the encoded predicate.
	Predicate beam.EncodedFn `json:"predicate"`
	// Include indicates whether to include or exclude elements that satisfy the predicate.
	Include bool `json:"include"`
}

func (f *filterFn) ProcessElement(elm beam.T, emit func(beam.T)) {
	ret := f.Predicate.Fn.Call([]reflect.Value{reflect.ValueOf(elm)})
	if ret[0].Bool() == f.Include {
		emit(elm)
	}
}
<commit_msg>Use fast caller for filter transform predicate<commit_after>// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements.  See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License.  You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package filter contains transformations for removing pipeline elements based on
// various conditions.
package filter

import (
	"reflect"

	"github.com/apache/beam/sdks/go/pkg/beam"
	"github.com/apache/beam/sdks/go/pkg/beam/core/funcx"
	"github.com/apache/beam/sdks/go/pkg/beam/core/typex"
	"github.com/apache/beam/sdks/go/pkg/beam/core/util/reflectx"
)

var (
	sig = funcx.MakePredicate(beam.TType) // T -> bool
)

func init() {
	beam.RegisterType(reflect.TypeOf((*filterFn)(nil)).Elem())
}

// Include filters the elements of a PCollection<A> based on the given function,
// which must be of the form: A -> bool. Include removes all element for which
// the filter function returns false. It returns a PCollection of the same type
// as the input. For example:
//
//    words := beam.Create(s, "a", "b", "long", "alsolong")
//    short := filter.Include(s, words, func(s string) bool {
//        return len(s) < 3
//    })
//
// Here, "short" will contain "a" and "b" at runtime.
func Include(s beam.Scope, col beam.PCollection, fn interface{}) beam.PCollection {
	s = s.Scope("filter.Include")

	t := typex.SkipW(col.Type()).Type()
	funcx.MustSatisfy(fn, funcx.Replace(sig, beam.TType, t))

	return beam.ParDo(s, &filterFn{Predicate: beam.EncodedFn{Fn: reflect.ValueOf(fn)}, Include: true}, col)
}

// Exclude filters the elements of a PCollection<A> based on the given function,
// which must be of the form: A -> bool. Exclude removes all element for which
// the filter function returns true. It returns a PCollection of the same type
// as the input. For example:
//
//    words := beam.Create(s, "a", "b", "long", "alsolong")
//    long := filter.Exclude(s, words, func(s string) bool {
//        return len(s) < 3
//    })
//
// Here, "long" will contain "long" and "alsolong" at runtime.
func Exclude(s beam.Scope, col beam.PCollection, fn interface{}) beam.PCollection {
	s = s.Scope("filter.Exclude")

	t := typex.SkipW(col.Type()).Type()
	funcx.MustSatisfy(fn, funcx.Replace(sig, beam.TType, t))

	return beam.ParDo(s, &filterFn{Predicate: beam.EncodedFn{Fn: reflect.ValueOf(fn)}, Include: false}, col)
}

type filterFn struct {
	// Predicate is the encoded predicate.
	Predicate beam.EncodedFn `json:"predicate"`
	// Include indicates whether to include or exclude elements that satisfy the predicate.
	Include bool `json:"include"`

	fn reflectx.Caller1x1
}

func (f *filterFn) Setup() {
	f.fn = reflectx.MakeCaller1x1(f.Predicate.Fn.Interface())
}

func (f *filterFn) ProcessElement(elm beam.T, emit func(beam.T)) {
	match := f.fn.Call1x1(elm).(bool)
	if match == f.Include {
		emit(elm)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"net/http"
	"path/filepath"
	"text/template"
)

// templ represents a single template
type templateHandler struct {
	source string
	templ  *template.Template
}

// ServeHTTP handles the HTTP request.
func (t *templateHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if t.templ == nil {
		t.templ = template.Must(template.ParseFiles(filepath.Join("templates", t.source)))
	}
	t.templ.Execute(w, nil)
}

func main() {

	// root
	http.Handle("/", &templateHandler{source: "chat.html"})

	// start the web server
	if err := http.ListenAndServe(":8080", nil); err != nil {
		log.Fatal("ListenAndServe:", err)
	}

}
<commit_msg>changed 'source' to 'filename' to be clearer<commit_after>package main

import (
	"log"
	"net/http"
	"path/filepath"
	"text/template"
)

// templ represents a single template
type templateHandler struct {
	filename string
	templ    *template.Template
}

// ServeHTTP handles the HTTP request.
func (t *templateHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if t.templ == nil {
		t.templ = template.Must(template.ParseFiles(filepath.Join("templates", t.filename)))
	}
	t.templ.Execute(w, nil)
}

func main() {

	// root
	http.Handle("/", &templateHandler{filename: "chat.html"})

	// start the web server
	if err := http.ListenAndServe(":8080", nil); err != nil {
		log.Fatal("ListenAndServe:", err)
	}

}
<|endoftext|>