<commit_before>package main

import (
	"errors"
	"os"
	"os/signal"
	"runtime/pprof"
	"strings"
	"syscall"

	"github.com/cenkalti/log"
	"github.com/cenkalti/rain/client"
	"github.com/cenkalti/rain/internal/clientversion"
	"github.com/cenkalti/rain/internal/logger"
	"github.com/cenkalti/rain/resume/torrentresume"
	"github.com/cenkalti/rain/storage/filestorage"
	"github.com/cenkalti/rain/torrent"
	"github.com/mitchellh/go-homedir"
	"github.com/urfave/cli"
)

var (
	cfg = client.NewConfig()
	app = cli.NewApp()
)

func main() {
	app.Version = clientversion.Version
	app.Usage = "BitTorrent client"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "config, c",
			Usage: "read config from `FILE`",
		},
		cli.StringFlag{
			Name:  "cpuprofile",
			Usage: "write cpu profile to `FILE`",
		},
		cli.BoolFlag{
			Name:  "debug, d",
			Usage: "enable debug log",
		},
	}
	app.Before = handleBeforeCommand
	app.After = handleAfterCommand
	app.Commands = []cli.Command{
		{
			Name:      "download",
			Usage:     "download torrent or magnet",
			ArgsUsage: "[torrent path or magnet link]",
			Action:    handleDownload,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "dest",
					Usage: "save files under `DIR`",
					Value: ".",
				},
				cli.IntFlag{
					Name:  "port",
					Usage: "peer listen port",
				},
				cli.BoolFlag{
					Name:  "seed",
					Usage: "continue seeding after download finishes",
				},
			},
		},
	}
	err := app.Run(os.Args)
	if err != nil {
		log.Fatal(err)
	}
}

func handleBeforeCommand(c *cli.Context) error {
	cpuprofile := c.GlobalString("cpuprofile")
	if cpuprofile != "" {
		f, err := os.Create(cpuprofile)
		if err != nil {
			log.Fatal("could not create CPU profile: ", err)
		}
		if err := pprof.StartCPUProfile(f); err != nil {
			log.Fatal("could not start CPU profile: ", err)
		}
	}
	if c.GlobalBool("debug") {
		logger.SetLogLevel(log.DEBUG)
	}
	configPath := c.GlobalString("config")
	if configPath != "" {
		cp, err := homedir.Expand(configPath)
		if err != nil {
			log.Fatal(err)
		}
		err = cfg.LoadFile(cp)
		if err != nil {
			log.Fatal(err)
		}
	}
	return nil
}

func handleAfterCommand(c *cli.Context) error {
	if c.GlobalString("cpuprofile") != "" {
		pprof.StopCPUProfile()
	}
	return nil
}

func handleDownload(c *cli.Context) error {
	path := c.Args().Get(0)
	if path == "" {
		return errors.New("first argument must be a torrent file or magnet link")
	}
	sto, err := filestorage.New(c.String("dest"))
	if err != nil {
		log.Fatal(err)
	}
	var t *torrent.Torrent
	if strings.HasPrefix(path, "magnet:") {
		t, err = torrent.NewMagnet(path, c.Int("port"), sto)
	} else {
		f, err2 := os.Open(path) // nolint: gosec
		if err2 != nil {
			log.Fatal(err2)
		}
		t, err = torrent.New(f, c.Int("port"), sto)
		_ = f.Close()
	}
	if err != nil {
		log.Fatal(err)
	}
	defer t.Close()

	res, err := torrentresume.New(t.Name() + "." + t.InfoHash() + ".resume")
	if err != nil {
		log.Fatal(err)
	}
	err = t.SetResume(res)
	if err != nil {
		log.Fatal(err)
	}

	sigC := make(chan os.Signal, 1)
	signal.Notify(sigC, syscall.SIGINT, syscall.SIGTERM)

	t.Start()
	select {
	case <-t.NotifyComplete():
		if !c.Bool("seed") {
			return nil
		}
		select {
		case <-sigC:
		case err = <-t.NotifyError():
			return err
		}
	case <-sigC:
	case err = <-t.NotifyError():
		return err
	}
	return nil
}
<commit_msg>print stats periodically<commit_after>package main

import (
	"errors"
	"fmt"
	"os"
	"os/signal"
	"runtime/pprof"
	"strings"
	"syscall"
	"time"

	"github.com/cenkalti/log"
	"github.com/cenkalti/rain/client"
	"github.com/cenkalti/rain/internal/clientversion"
	"github.com/cenkalti/rain/internal/logger"
	"github.com/cenkalti/rain/resume/torrentresume"
	"github.com/cenkalti/rain/storage/filestorage"
	"github.com/cenkalti/rain/torrent"
	"github.com/mitchellh/go-homedir"
	"github.com/urfave/cli"
)

var (
	cfg = client.NewConfig()
	app = cli.NewApp()
)

func main() {
	app.Version = clientversion.Version
	app.Usage = "BitTorrent client"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "config, c",
			Usage: "read config from `FILE`",
		},
		cli.StringFlag{
			Name:  "cpuprofile",
			Usage: "write cpu profile to `FILE`",
		},
		cli.BoolFlag{
			Name:  "debug, d",
			Usage: "enable debug log",
		},
	}
	app.Before = handleBeforeCommand
	app.After = handleAfterCommand
	app.Commands = []cli.Command{
		{
			Name:      "download",
			Usage:     "download torrent or magnet",
			ArgsUsage: "[torrent path or magnet link]",
			Action:    handleDownload,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "dest",
					Usage: "save files under `DIR`",
					Value: ".",
				},
				cli.IntFlag{
					Name:  "port",
					Usage: "peer listen port",
				},
				cli.BoolFlag{
					Name:  "seed",
					Usage: "continue seeding after download finishes",
				},
			},
		},
	}
	err := app.Run(os.Args)
	if err != nil {
		log.Fatal(err)
	}
}

func handleBeforeCommand(c *cli.Context) error {
	cpuprofile := c.GlobalString("cpuprofile")
	if cpuprofile != "" {
		f, err := os.Create(cpuprofile)
		if err != nil {
			log.Fatal("could not create CPU profile: ", err)
		}
		if err := pprof.StartCPUProfile(f); err != nil {
			log.Fatal("could not start CPU profile: ", err)
		}
	}
	if c.GlobalBool("debug") {
		logger.SetLogLevel(log.DEBUG)
	}
	configPath := c.GlobalString("config")
	if configPath != "" {
		cp, err := homedir.Expand(configPath)
		if err != nil {
			log.Fatal(err)
		}
		err = cfg.LoadFile(cp)
		if err != nil {
			log.Fatal(err)
		}
	}
	return nil
}

func handleAfterCommand(c *cli.Context) error {
	if c.GlobalString("cpuprofile") != "" {
		pprof.StopCPUProfile()
	}
	return nil
}

func handleDownload(c *cli.Context) error {
	path := c.Args().Get(0)
	if path == "" {
		return errors.New("first argument must be a torrent file or magnet link")
	}
	sto, err := filestorage.New(c.String("dest"))
	if err != nil {
		log.Fatal(err)
	}
	var t *torrent.Torrent
	if strings.HasPrefix(path, "magnet:") {
		t, err = torrent.NewMagnet(path, c.Int("port"), sto)
	} else {
		f, err2 := os.Open(path) // nolint: gosec
		if err2 != nil {
			log.Fatal(err2)
		}
		t, err = torrent.New(f, c.Int("port"), sto)
		_ = f.Close()
	}
	if err != nil {
		log.Fatal(err)
	}
	defer t.Close()

	res, err := torrentresume.New(t.Name() + "." + t.InfoHash() + ".resume")
	if err != nil {
		log.Fatal(err)
	}
	err = t.SetResume(res)
	if err != nil {
		log.Fatal(err)
	}

	sigC := make(chan os.Signal, 1)
	signal.Notify(sigC, syscall.SIGINT, syscall.SIGTERM)

	t.Start()

	go func() {
		for range time.Tick(time.Second) {
			fmt.Printf("%#v\n", t.Stats())
		}
	}()

	select {
	case <-t.NotifyComplete():
		if !c.Bool("seed") {
			return nil
		}
		select {
		case <-sigC:
		case err = <-t.NotifyError():
			return err
		}
	case <-sigC:
	case err = <-t.NotifyError():
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package search

import (
	"gopkg.in/olivere/elastic.v5"

	"github.com/kevintavog/findaphoto/common"
)

type SearchOptions struct {
	Query            string
	Index            int
	Count            int
	CategoryOptions  *CategoryOptions
	DrilldownOptions *DrilldownOptions
}

//-------------------------------------------------------------------------------------------------
func NewSearchOptions(query string) *SearchOptions {
	return &SearchOptions{
		Query:            query,
		Index:            0,
		Count:            20,
		CategoryOptions:  NewCategoryOptions(),
		DrilldownOptions: NewDrilldownOptions(),
	}
}

func (so *SearchOptions) Search() (*SearchResult, error) {
	client := common.CreateClient()
	search := client.Search().
		Index(common.MediaIndexName).
		Type(common.MediaTypeName).
		Pretty(true)

	var query elastic.Query
	if so.Query == "" {
		query = elastic.NewMatchAllQuery()
	} else {
		query = elastic.NewQueryStringQuery(so.Query).
			Field("path"). // Folder name
			Field("monthname").
			Field("dayname").
			Field("keywords").
			Field("placename"). // Full reverse location lookup
			Field("tags")
	}

	search.Query(query)

	search.From(so.Index).Size(so.Count).Sort("datetime", false)
	return invokeSearch(search, &query, GroupByPath, so.CategoryOptions, so.DrilldownOptions, nil)
}
<commit_msg>Default text searches to group by date rather than folder<commit_after>package search

import (
	"gopkg.in/olivere/elastic.v5"

	"github.com/kevintavog/findaphoto/common"
)

type SearchOptions struct {
	Query            string
	Index            int
	Count            int
	CategoryOptions  *CategoryOptions
	DrilldownOptions *DrilldownOptions
}

//-------------------------------------------------------------------------------------------------
func NewSearchOptions(query string) *SearchOptions {
	return &SearchOptions{
		Query:            query,
		Index:            0,
		Count:            20,
		CategoryOptions:  NewCategoryOptions(),
		DrilldownOptions: NewDrilldownOptions(),
	}
}

func (so *SearchOptions) Search() (*SearchResult, error) {
	client := common.CreateClient()
	search := client.Search().
		Index(common.MediaIndexName).
		Type(common.MediaTypeName).
		Pretty(true)

	var query elastic.Query
	if so.Query == "" {
		query = elastic.NewMatchAllQuery()
	} else {
		query = elastic.NewQueryStringQuery(so.Query).
			Field("path"). // Folder name
			Field("monthname").
			Field("dayname").
			Field("keywords").
			Field("placename"). // Full reverse location lookup
			Field("tags")
	}

	search.Query(query)

	search.From(so.Index).Size(so.Count).Sort("datetime", false)
	return invokeSearch(search, &query, GroupByDate, so.CategoryOptions, so.DrilldownOptions, nil)
}
<|endoftext|>


<commit_before>// Package deque provides a double ended queue modeled after Python's
// collections.deque
package deque

import (
	"bytes"
	"fmt"
)

const (
	blockLen    = 64
	blockCenter = (blockLen - 1) / 2

	Version = "1.0.0"
)

type block struct {
	left, right *block
	data        []interface{}
}

func newBlock(left, right *block) *block {
	return &block{
		left:  left,
		right: right,
		data:  make([]interface{}, blockLen),
	}
}

// Deque is a double ended queue
type Deque struct {
	left, right       *block
	leftIdx, rightIdx int /* in range(BLOCKLEN) */
	size              int
	maxSize           int
}

// New returns a new unbounded Deque
func New() *Deque {
	block := newBlock(nil, nil)
	return &Deque{
		right:    block,
		left:     block,
		leftIdx:  blockCenter + 1,
		rightIdx: blockCenter,
		size:     0,
		maxSize:  -1,
	}
}

// NewBounded returns a new bounded Deque
// A bounded Deque will not grow over maxSize items
func NewBounded(maxSize int) (*Deque, error) {
	if maxSize <= 0 {
		return nil, fmt.Errorf("maxSize must be > 0 (got %d)", maxSize)
	}
	block := newBlock(nil, nil)
	dq := &Deque{
		right:    block,
		left:     block,
		leftIdx:  blockCenter + 1,
		rightIdx: blockCenter,
		size:     0,
		maxSize:  maxSize,
	}

	return dq, nil
}

// Len returns the number of items in the queue
func (dq *Deque) Len() int {
	return dq.size
}

// Append appends an item to the right of the deque
func (dq *Deque) Append(item interface{}) {
	if dq.rightIdx == blockLen-1 {
		block := newBlock(dq.right, nil)
		dq.right.right = block
		dq.right = block
		dq.rightIdx = -1
	}
	dq.size++
	dq.rightIdx++
	dq.right.data[dq.rightIdx] = item
	if dq.maxSize != -1 && dq.Len() > dq.maxSize {
		dq.PopLeft()
	}
}

// AppendLeft appends an item to the left of the deque
func (dq *Deque) AppendLeft(item interface{}) {
	if dq.leftIdx == 0 {
		block := newBlock(nil, dq.left)
		dq.left.left = block
		dq.left = block
		dq.leftIdx = blockLen
	}
	dq.size++
	dq.leftIdx--
	dq.left.data[dq.leftIdx] = item
	if dq.maxSize != -1 && dq.Len() > dq.maxSize {
		dq.Pop()
	}
}

// Pop removes and return the rightmost element
func (dq *Deque) Pop() (interface{}, error) {
	if dq.Len() == 0 {
		return nil, fmt.Errorf("Pop from an empty Deque")
	}

	item := dq.right.data[dq.rightIdx]
	dq.rightIdx--
	dq.size--
	if dq.rightIdx == -1 {
		if dq.Len() == 0 {
			// re-center instead of freeing a block
			dq.leftIdx = blockCenter + 1
			dq.rightIdx = blockCenter
		} else {
			prev := dq.right.left
			prev.right = nil
			dq.right = prev
			dq.rightIdx = blockLen - 1
		}
	}
	return item, nil
}

// PopLeft removes and return the leftmost element.
func (dq *Deque) PopLeft() (interface{}, error) {
	if dq.Len() == 0 {
		return nil, fmt.Errorf("PopLeft from an empty Deque")
	}

	item := dq.left.data[dq.leftIdx]
	dq.leftIdx++
	dq.size--

	if dq.leftIdx == blockLen {
		if dq.Len() == 0 {
			// re-center instead of freeing a block
			dq.leftIdx = blockCenter + 1
			dq.rightIdx = blockCenter
		} else {
			prev := dq.left.right
			prev.left = nil
			dq.left = prev
			dq.leftIdx = 0
		}
	}
	return item, nil
}

func (dq *Deque) locate(i int) (b *block, idx int) {
	if i == 0 {
		i = dq.leftIdx
		b = dq.left
	} else if i == dq.Len()-1 {
		i = dq.rightIdx
		b = dq.right
	} else {
		index := i
		i += dq.leftIdx
		n := i / blockLen
		i %= blockLen
		if index < (dq.Len() >> 1) {
			b = dq.right
			for ; n > 0; n-- {
				b = b.right
			}
		} else {
			n = (dq.leftIdx+dq.size-1)/blockLen - n
			b = dq.right
			for ; n > 0; n-- {
				b = b.left
			}
		}
	}
	return b, i
}

// Get return the item at position i
func (dq *Deque) Get(i int) (interface{}, error) {
	if i < 0 || i >= dq.Len() {
		return nil, fmt.Errorf("index %d out of range", i)
	}

	b, idx := dq.locate(i)

	return b.data[idx], nil
}

// Set sets the item at position i to val
func (dq *Deque) Set(i int, val interface{}) error {
	if i < 0 || i >= dq.Len() {
		return fmt.Errorf("index %d out of range", i)
	}

	b, idx := dq.locate(i)
	b.data[idx] = val
	return nil
}

func (dq *Deque) Rotate(n int) {
	if dq.Len() == 0 || n == 0 {
		return
	}

	var popfn func() (interface{}, error)
	var appendfn func(interface{})

	if n > 0 {
		popfn = dq.Pop
		appendfn = dq.AppendLeft
	} else {
		popfn = dq.PopLeft
		appendfn = dq.Append
		n = -n
	}

	for i := 0; i < n; i++ {
		val, _ := popfn()
		appendfn(val)
	}
}

func (dq *Deque) String() string {
	var buf bytes.Buffer

	fmt.Fprintf(&buf, "Deque{")
	n := dq.Len()
	chopped := false
	if n > 10 {
		n = 10
		chopped = true
	}
	for i := 0; i < n-1; i++ {
		val, _ := dq.Get(i)
		fmt.Fprintf(&buf, "%#v, ", val)
	}
	if chopped {
		fmt.Fprintf(&buf, "...")
	} else {
		val, _ := dq.Get(n - 1)
		fmt.Fprintf(&buf, "%#v", val)
	}
	fmt.Fprintf(&buf, "}")
	return buf.String()
}
<commit_msg>reuse New<commit_after>// Package deque provides a double ended queue modeled after Python's
// collections.deque
package deque

import (
	"bytes"
	"fmt"
)

const (
	blockLen    = 64
	blockCenter = (blockLen - 1) / 2

	Version = "1.0.0"
)

type block struct {
	left, right *block
	data        []interface{}
}

func newBlock(left, right *block) *block {
	return &block{
		left:  left,
		right: right,
		data:  make([]interface{}, blockLen),
	}
}

// Deque is a double ended queue
type Deque struct {
	left, right       *block
	leftIdx, rightIdx int /* in range(BLOCKLEN) */
	size              int
	maxSize           int
}

// New returns a new unbounded Deque
func New() *Deque {
	block := newBlock(nil, nil)
	return &Deque{
		right:    block,
		left:     block,
		leftIdx:  blockCenter + 1,
		rightIdx: blockCenter,
		size:     0,
		maxSize:  -1,
	}
}

// NewBounded returns a new bounded Deque
// A bounded Deque will not grow over maxSize items
func NewBounded(maxSize int) (*Deque, error) {
	if maxSize <= 0 {
		return nil, fmt.Errorf("maxSize must be > 0 (got %d)", maxSize)
	}
	dq := New()
	dq.maxSize = maxSize
	return dq, nil
}

// Len returns the number of items in the queue
func (dq *Deque) Len() int {
	return dq.size
}

// Append appends an item to the right of the deque
func (dq *Deque) Append(item interface{}) {
	if dq.rightIdx == blockLen-1 {
		block := newBlock(dq.right, nil)
		dq.right.right = block
		dq.right = block
		dq.rightIdx = -1
	}
	dq.size++
	dq.rightIdx++
	dq.right.data[dq.rightIdx] = item
	if dq.maxSize != -1 && dq.Len() > dq.maxSize {
		dq.PopLeft()
	}
}

// AppendLeft appends an item to the left of the deque
func (dq *Deque) AppendLeft(item interface{}) {
	if dq.leftIdx == 0 {
		block := newBlock(nil, dq.left)
		dq.left.left = block
		dq.left = block
		dq.leftIdx = blockLen
	}
	dq.size++
	dq.leftIdx--
	dq.left.data[dq.leftIdx] = item
	if dq.maxSize != -1 && dq.Len() > dq.maxSize {
		dq.Pop()
	}
}

// Pop removes and return the rightmost element
func (dq *Deque) Pop() (interface{}, error) {
	if dq.Len() == 0 {
		return nil, fmt.Errorf("Pop from an empty Deque")
	}

	item := dq.right.data[dq.rightIdx]
	dq.rightIdx--
	dq.size--
	if dq.rightIdx == -1 {
		if dq.Len() == 0 {
			// re-center instead of freeing a block
			dq.leftIdx = blockCenter + 1
			dq.rightIdx = blockCenter
		} else {
			prev := dq.right.left
			prev.right = nil
			dq.right = prev
			dq.rightIdx = blockLen - 1
		}
	}
	return item, nil
}

// PopLeft removes and return the leftmost element.
func (dq *Deque) PopLeft() (interface{}, error) {
	if dq.Len() == 0 {
		return nil, fmt.Errorf("PopLeft from an empty Deque")
	}

	item := dq.left.data[dq.leftIdx]
	dq.leftIdx++
	dq.size--

	if dq.leftIdx == blockLen {
		if dq.Len() == 0 {
			// re-center instead of freeing a block
			dq.leftIdx = blockCenter + 1
			dq.rightIdx = blockCenter
		} else {
			prev := dq.left.right
			prev.left = nil
			dq.left = prev
			dq.leftIdx = 0
		}
	}
	return item, nil
}

func (dq *Deque) locate(i int) (b *block, idx int) {
	if i == 0 {
		i = dq.leftIdx
		b = dq.left
	} else if i == dq.Len()-1 {
		i = dq.rightIdx
		b = dq.right
	} else {
		index := i
		i += dq.leftIdx
		n := i / blockLen
		i %= blockLen
		if index < (dq.Len() >> 1) {
			b = dq.right
			for ; n > 0; n-- {
				b = b.right
			}
		} else {
			n = (dq.leftIdx+dq.size-1)/blockLen - n
			b = dq.right
			for ; n > 0; n-- {
				b = b.left
			}
		}
	}
	return b, i
}

// Get return the item at position i
func (dq *Deque) Get(i int) (interface{}, error) {
	if i < 0 || i >= dq.Len() {
		return nil, fmt.Errorf("index %d out of range", i)
	}

	b, idx := dq.locate(i)

	return b.data[idx], nil
}

// Set sets the item at position i to val
func (dq *Deque) Set(i int, val interface{}) error {
	if i < 0 || i >= dq.Len() {
		return fmt.Errorf("index %d out of range", i)
	}

	b, idx := dq.locate(i)
	b.data[idx] = val
	return nil
}

func (dq *Deque) Rotate(n int) {
	if dq.Len() == 0 || n == 0 {
		return
	}

	var popfn func() (interface{}, error)
	var appendfn func(interface{})

	if n > 0 {
		popfn = dq.Pop
		appendfn = dq.AppendLeft
	} else {
		popfn = dq.PopLeft
		appendfn = dq.Append
		n = -n
	}

	for i := 0; i < n; i++ {
		val, _ := popfn()
		appendfn(val)
	}
}

func (dq *Deque) String() string {
	var buf bytes.Buffer

	fmt.Fprintf(&buf, "Deque{")
	n := dq.Len()
	chopped := false
	if n > 10 {
		n = 10
		chopped = true
	}
	for i := 0; i < n-1; i++ {
		val, _ := dq.Get(i)
		fmt.Fprintf(&buf, "%#v, ", val)
	}
	if chopped {
		fmt.Fprintf(&buf, "...")
	} else {
		val, _ := dq.Get(n - 1)
		fmt.Fprintf(&buf, "%#v", val)
	}
	fmt.Fprintf(&buf, "}")
	return buf.String()
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package s3

import (
	"fmt"
	"github.com/jacobsa/aws"
	"github.com/jacobsa/aws/s3/auth"
	"github.com/jacobsa/aws/s3/http"
	"github.com/jacobsa/aws/s3/time"
	"net/url"
	sys_time "time"
	"unicode/utf8"
)

// Bucket represents an S3 bucket, which is a collection of objects keyed on
// Unicode strings. The UTF-8 encoding of a key must be no more than 1024 bytes
// long.
//
// See here for more info:
//
//     http://goo.gl/Nd63t
//
type Bucket interface {
	// Retrieve data for the object with the given key.
	GetObject(key string) (data []byte, err error)

	// Store the supplied data with the given key, overwriting any previous
	// version.
	StoreObject(key string, data []byte) error
}

// OpenBucket returns a Bucket tied to a given name in whe given region. You
// must have previously created the bucket in the region, and the supplied
// access key must have access to it.
//
// To easily create a bucket, use the AWS Console:
//
//     http://aws.amazon.com/console/
//
func OpenBucket(name string, region Region, key aws.AccessKey) (Bucket, error) {
	// Create a connection to the given region's endpoint.
	endpoint := &url.URL{Scheme: "https", Host: string(region)}
	httpConn, err := http.NewConn(endpoint)
	if err != nil {
		return nil, fmt.Errorf("http.NewConn: %v", err)
	}

	// Create an appropriate request signer.
	signer, err := auth.NewSigner(&key)
	if err != nil {
		return nil, fmt.Errorf("auth.NewSigner: %v", err)
	}

	return openBucket(name, httpConn, signer, time.RealClock())
}

// A version of OpenBucket with the ability to inject dependencies, for
// testability.
func openBucket(
	name string,
	httpConn http.Conn,
	signer auth.Signer,
	clock time.Clock) (Bucket, error) {
	return &bucket{name, httpConn, signer, clock}, nil
}

type bucket struct {
	name string
	httpConn http.Conn
	signer auth.Signer
	clock time.Clock
}

func (b *bucket) GetObject(key string) (data []byte, err error) {
	return nil, fmt.Errorf("TODO: Implement bucket.GetObject.")
}

func (b *bucket) StoreObject(key string, data []byte) error {
	// Validate the key.
	if len(key) > 1024 {
		return fmt.Errorf("Keys may be no longer than 1024 bytes.")
	}

	if !utf8.ValidString(key) {
		return fmt.Errorf("Keys must be valid UTF-8.")
	}

	// Build an appropriate HTTP request.
	httpReq := &http.Request{
		Verb: "PUT",
		Path: fmt.Sprintf("/%s/%s", b.name, key),
		Body: data,
		Headers: map[string]string {
			"Date": b.clock.Now().UTC().Format(sys_time.RFC1123),
		},
	}

	// Sign the request.
	if err := b.signer.Sign(httpReq); err != nil {
		return fmt.Errorf("Sign: %v", err)
	}

	// Send the request.
	httpResp, err := b.httpConn.SendRequest(httpReq)
	if err != nil {
		return fmt.Errorf("SendRequest: %v", err)
	}

	// Check the response.
	if httpResp.StatusCode != 200 {
		return fmt.Errorf("Error from server: %d %s", httpResp.StatusCode, httpResp.Body)
	}

	return nil
}
<commit_msg>Added docs on ACLs.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package s3

import (
	"fmt"
	"github.com/jacobsa/aws"
	"github.com/jacobsa/aws/s3/auth"
	"github.com/jacobsa/aws/s3/http"
	"github.com/jacobsa/aws/s3/time"
	"net/url"
	sys_time "time"
	"unicode/utf8"
)

// Bucket represents an S3 bucket, which is a collection of objects keyed on
// Unicode strings. The UTF-8 encoding of a key must be no more than 1024 bytes
// long.
//
// See here for more info:
//
//     http://goo.gl/Nd63t
//
type Bucket interface {
	// Retrieve data for the object with the given key.
	GetObject(key string) (data []byte, err error)

	// Store the supplied data with the given key, overwriting any previous
	// version. The object is created with the default ACL of "private".
	StoreObject(key string, data []byte) error
}

// OpenBucket returns a Bucket tied to a given name in whe given region. You
// must have previously created the bucket in the region, and the supplied
// access key must have access to it.
//
// To easily create a bucket, use the AWS Console:
//
//     http://aws.amazon.com/console/
//
func OpenBucket(name string, region Region, key aws.AccessKey) (Bucket, error) {
	// Create a connection to the given region's endpoint.
	endpoint := &url.URL{Scheme: "https", Host: string(region)}
	httpConn, err := http.NewConn(endpoint)
	if err != nil {
		return nil, fmt.Errorf("http.NewConn: %v", err)
	}

	// Create an appropriate request signer.
	signer, err := auth.NewSigner(&key)
	if err != nil {
		return nil, fmt.Errorf("auth.NewSigner: %v", err)
	}

	return openBucket(name, httpConn, signer, time.RealClock())
}

// A version of OpenBucket with the ability to inject dependencies, for
// testability.
func openBucket(
	name string,
	httpConn http.Conn,
	signer auth.Signer,
	clock time.Clock) (Bucket, error) {
	return &bucket{name, httpConn, signer, clock}, nil
}

type bucket struct {
	name string
	httpConn http.Conn
	signer auth.Signer
	clock time.Clock
}

func (b *bucket) GetObject(key string) (data []byte, err error) {
	return nil, fmt.Errorf("TODO: Implement bucket.GetObject.")
}

func (b *bucket) StoreObject(key string, data []byte) error {
	// Validate the key.
	if len(key) > 1024 {
		return fmt.Errorf("Keys may be no longer than 1024 bytes.")
	}

	if !utf8.ValidString(key) {
		return fmt.Errorf("Keys must be valid UTF-8.")
	}

	// Build an appropriate HTTP request.
	httpReq := &http.Request{
		Verb: "PUT",
		Path: fmt.Sprintf("/%s/%s", b.name, key),
		Body: data,
		Headers: map[string]string {
			"Date": b.clock.Now().UTC().Format(sys_time.RFC1123),
		},
	}

	// Sign the request.
	if err := b.signer.Sign(httpReq); err != nil {
		return fmt.Errorf("Sign: %v", err)
	}

	// Send the request.
	httpResp, err := b.httpConn.SendRequest(httpReq)
	if err != nil {
		return fmt.Errorf("SendRequest: %v", err)
	}

	// Check the response.
	if httpResp.StatusCode != 200 {
		return fmt.Errorf("Error from server: %d %s", httpResp.StatusCode, httpResp.Body)
	}

	return nil
}
<|endoftext|>


<commit_before>package diskv

/*
type Key string
type Value []byte

type Store interface {
	Read(k Key) (Value, error)
	Write(k Key, v Value) error
	Erase(k Key) error
	Flush() error
}

type TransformFunc func(k Key) []string
*/
<commit_msg>Remove dead code file<commit_after><|endoftext|>


<commit_before>// Copyright (c) 2014-2017 Ludovic Fauvet
// Licensed under the MIT license

package scan

import (
	"errors"
	"fmt"
	. "github.com/etix/mirrorbits/config"
	"github.com/etix/mirrorbits/database"
	"github.com/etix/mirrorbits/filesystem"
	"github.com/etix/mirrorbits/utils"
	"github.com/garyburd/redigo/redis"
	"github.com/op/go-logging"
	"os"
	"path/filepath"
	"strconv"
	"time"
)

var (
	ScanAborted    = errors.New("scan aborted")
	ScanInProgress = errors.New("scan already in progress")

	log = logging.MustGetLogger("main")
)

type ScannerType int8

const (
	RSYNC ScannerType = iota
	FTP
)

type Scanner interface {
	Scan(url, identifier string, conn redis.Conn, stop chan bool) error
}

type filedata struct {
	path    string
	sha1    string
	sha256  string
	md5     string
	size    int64
	modTime time.Time
}

type scan struct {
	redis           *database.Redis
	walkSourceFiles []*filedata
	walkRedisConn   redis.Conn

	conn        redis.Conn
	identifier  string
	filesKey    string
	filesTmpKey string
	count       uint
}

func IsScanning(conn redis.Conn, identifier string) (bool, error) {
	return redis.Bool(conn.Do("EXISTS", fmt.Sprintf("SCANNING_%s", identifier)))
}

func Scan(typ ScannerType, r *database.Redis, url, identifier string, stop chan bool) error {
	s := &scan{
		redis:      r,
		identifier: identifier,
	}

	var scanner Scanner
	switch typ {
	case RSYNC:
		scanner = &RsyncScanner{
			scan: s,
		}
	case FTP:
		scanner = &FTPScanner{
			scan: s,
		}
	default:
		panic(fmt.Sprintf("Unknown scanner"))
	}

	// Connect to the database
	conn := s.redis.Get()
	defer conn.Close()

	s.conn = conn

	lockKey := fmt.Sprintf("SCANNING_%s", identifier)

	// Try to aquire a lock so we don't have a scanning race
	// from different nodes.
	lock, err := redis.Bool(conn.Do("SETNX", lockKey, 1))
	if err != nil {
		return err
	}
	if lock {
		// Lock aquired.
		defer conn.Do("DEL", lockKey)
		// Make the key expire automatically in case our process gets killed
		conn.Do("EXPIRE", lockKey, 600)
	} else {
		return ScanInProgress
	}

	s.setLastSync(conn, identifier, false)

	conn.Send("MULTI")

	s.filesKey = fmt.Sprintf("MIRROR_%s_FILES", identifier)
	s.filesTmpKey = fmt.Sprintf("MIRROR_%s_FILES_TMP", identifier)

	// Remove any left over
	conn.Send("DEL", s.filesTmpKey)

	err = scanner.Scan(url, identifier, conn, stop)
	if err != nil {
		// Discard MULTI
		s.ScannerDiscard()

		// Remove the temporary key
		conn.Do("DEL", s.filesTmpKey)

		log.Errorf("[%s] %s", identifier, err.Error())
		return err
	}

	// Exec multi
	s.ScannerCommit()

	// Get the list of files no more present on this mirror
	toremove, err := redis.Values(conn.Do("SDIFF", s.filesKey, s.filesTmpKey))
	if err != nil {
		return err
	}

	// Remove this mirror from the given file SET
	if len(toremove) > 0 {
		conn.Send("MULTI")
		for _, e := range toremove {
			log.Debugf("[%s] Removing %s from mirror", identifier, e)
			conn.Send("SREM", fmt.Sprintf("FILEMIRRORS_%s", e), identifier)
			conn.Send("DEL", fmt.Sprintf("FILEINFO_%s_%s", identifier, e))
			// Publish update
			database.SendPublish(conn, database.MIRROR_FILE_UPDATE, fmt.Sprintf("%s %s", identifier, e))

		}
		_, err = conn.Do("EXEC")
		if err != nil {
			return err
		}
	}

	// Finally rename the temporary sets containing the list
	// of files for this mirror to the production key
	_, err = conn.Do("RENAME", s.filesTmpKey, s.filesKey)
	if err != nil {
		return err
	}

	sinterKey := fmt.Sprintf("HANDLEDFILES_%s", identifier)

	// Count the number of files known on the remote end
	common, _ := redis.Int64(conn.Do("SINTERSTORE", sinterKey, "FILES", s.filesKey))

	if err != nil {
		return err
	}

	s.setLastSync(conn, identifier, true)
	log.Infof("[%s] Indexed %d files (%d known), %d removed", identifier, s.count, common, len(toremove))
	return nil
}

func (s *scan) ScannerAddFile(f filedata) {
	s.count++

	// Add all the files to a temporary key
	s.conn.Send("SADD", s.filesTmpKey, f.path)

	// Mark the file as being supported by this mirror
	rk := fmt.Sprintf("FILEMIRRORS_%s", f.path)
	s.conn.Send("SADD", rk, s.identifier)

	// Save the size of the current file found on this mirror
	ik := fmt.Sprintf("FILEINFO_%s_%s", s.identifier, f.path)
	s.conn.Send("HSET", ik, "size", f.size)

	// Publish update
	database.SendPublish(s.conn, database.MIRROR_FILE_UPDATE, fmt.Sprintf("%s %s", s.identifier, f.path))
}

func (s *scan) ScannerDiscard() {
	s.conn.Do("DISCARD")
}

func (s *scan) ScannerCommit() error {
	_, err := s.conn.Do("EXEC")
	return err
}

func (s *scan) setLastSync(conn redis.Conn, identifier string, successful bool) error {
	now := time.Now().UTC().Unix()

	conn.Send("MULTI")

	// Set the last sync time
	conn.Send("HSET", fmt.Sprintf("MIRROR_%s", identifier), "lastSync", now)

	// Set the last successful sync time
	if successful {
		conn.Send("HSET", fmt.Sprintf("MIRROR_%s", identifier), "lastSuccessfulSync", now)
	}

	_, err := conn.Do("EXEC")

	// Publish an update on redis
	database.Publish(conn, database.MIRROR_UPDATE, identifier)

	return err
}

// Walk inside the source/reference repository
func (s *scan) walkSource(path string, f os.FileInfo, err error) error {
	if f == nil || f.IsDir() || f.Mode()&os.ModeSymlink != 0 {
		return nil
	}

	d := new(filedata)
	d.path = path[len(GetConfig().Repository):]
	d.size = f.Size()
	d.modTime = f.ModTime()

	// Get the previous file properties
	properties, err := redis.Strings(s.walkRedisConn.Do("HMGET", fmt.Sprintf("FILE_%s", d.path), "size", "modTime", "sha1", "sha256", "md5"))
	if err != nil && err != redis.ErrNil {
		return err
	} else if len(properties) < 5 {
		// This will force a rehash
		properties = make([]string, 5)
	}

	size, _ := strconv.ParseInt(properties[0], 10, 64)
	modTime, _ := time.Parse("2006-01-02 15:04:05.999999999 -0700 MST", properties[1])
	sha1 := properties[2]
	sha256 := properties[3]
	md5 := properties[4]

	rehash := (GetConfig().Hashes.SHA1 && len(sha1) == 0) ||
		(GetConfig().Hashes.SHA256 && len(sha256) == 0) ||
		(GetConfig().Hashes.MD5 && len(md5) == 0)

	if rehash || size != d.size || !modTime.Equal(d.modTime) {
		h, err := filesystem.HashFile(GetConfig().Repository + d.path)
		if err != nil {
			log.Warningf("%s: hashing failed: %s", d.path, err.Error())
		} else {
			d.sha1 = h.Sha1
			d.sha256 = h.Sha256
			d.md5 = h.Md5
			if len(d.sha1) > 0 {
				log.Infof("%s: SHA1 %s", d.path, d.sha1)
			}
			if len(d.sha256) > 0 {
				log.Infof("%s: SHA256 %s", d.path, d.sha256)
			}
			if len(d.md5) > 0 {
				log.Infof("%s: MD5 %s", d.path, d.md5)
			}
		}
	} else {
		d.sha1 = sha1
		d.sha256 = sha256
		d.md5 = md5
	}

	s.walkSourceFiles = append(s.walkSourceFiles, d)
	return nil
}

func ScanSource(r *database.Redis, stop chan bool) (err error) {
	s := &scan{
		redis: r,
	}

	s.walkRedisConn = s.redis.Get()
	defer s.walkRedisConn.Close()

	if s.walkRedisConn.Err() != nil {
		return s.walkRedisConn.Err()
	}

	s.walkSourceFiles = make([]*filedata, 0, 1000)
	defer func() {
		// Reset the slice so it can be garbage collected
		s.walkSourceFiles = nil
	}()

	//TODO lock atomically inside redis to avoid two simultanous scan

	if _, err := os.Stat(GetConfig().Repository); os.IsNotExist(err) {
		return fmt.Errorf("%s: No such file or directory", GetConfig().Repository)
	}

	log.Info("[source] Scanning the filesystem...")
	err = filepath.Walk(GetConfig().Repository, s.walkSource)
	if utils.IsStopped(stop) {
		return ScanAborted
	}
	if err != nil {
		return err
	}
	log.Info("[source] Indexing the files...")

	s.walkRedisConn.Send("MULTI")

	// Remove any left over
	s.walkRedisConn.Send("DEL", "FILES_TMP")

	// Add all the files to a temporary key
	count := 0
	for _, e := range s.walkSourceFiles {
		s.walkRedisConn.Send("SADD", "FILES_TMP", e.path)
		count++
	}

	_, err = s.walkRedisConn.Do("EXEC")
	if err != nil {
		return err
	}

	// Do a diff between the sets to get the removed files
	toremove, err := redis.Values(s.walkRedisConn.Do("SDIFF", "FILES", "FILES_TMP"))

	// Create/Update the files' hash keys with the fresh infos
	s.walkRedisConn.Send("MULTI")
	for _, e := range s.walkSourceFiles {
		s.walkRedisConn.Send("HMSET", fmt.Sprintf("FILE_%s", e.path),
			"size", e.size,
			"modTime", e.modTime,
			"sha1", e.sha1,
			"sha256", e.sha256,
			"md5", e.md5)

		// Publish update
		database.SendPublish(s.walkRedisConn, database.FILE_UPDATE, e.path)
	}

	// Remove old keys
	if len(toremove) > 0 {
		for _, e := range toremove {
			s.walkRedisConn.Send("DEL", fmt.Sprintf("FILE_%s", e))

			// Publish update
			database.SendPublish(s.walkRedisConn, database.FILE_UPDATE, fmt.Sprintf("%s", e))
		}
	}

	// Finally rename the temporary sets containing the list
	// of files to the production key
	s.walkRedisConn.Send("RENAME", "FILES_TMP", "FILES")

	_, err = s.walkRedisConn.Do("EXEC")
	if err != nil {
		return err
	}

	log.Infof("[source] Scanned %d files", count)

	return nil
}
<commit_msg>scan: pass the db connection via a closure<commit_after>// Copyright (c) 2014-2017 Ludovic Fauvet
// Licensed under the MIT license

package scan

import (
	"errors"
	"fmt"
	. "github.com/etix/mirrorbits/config"
	"github.com/etix/mirrorbits/database"
	"github.com/etix/mirrorbits/filesystem"
	"github.com/etix/mirrorbits/utils"
	"github.com/garyburd/redigo/redis"
	"github.com/op/go-logging"
	"os"
	"path/filepath"
	"strconv"
	"time"
)

var (
	ScanAborted    = errors.New("scan aborted")
	ScanInProgress = errors.New("scan already in progress")

	log = logging.MustGetLogger("main")
)

type ScannerType int8

const (
	RSYNC ScannerType = iota
	FTP
)

type Scanner interface {
	Scan(url, identifier string, conn redis.Conn, stop chan bool) error
}

type filedata struct {
	path    string
	sha1    string
	sha256  string
	md5     string
	size    int64
	modTime time.Time
}

type scan struct {
	redis           *database.Redis
	walkSourceFiles []*filedata

	conn        redis.Conn
	identifier  string
	filesKey    string
	filesTmpKey string
	count       uint
}

func IsScanning(conn redis.Conn, identifier string) (bool, error) {
	return redis.Bool(conn.Do("EXISTS", fmt.Sprintf("SCANNING_%s", identifier)))
}

func Scan(typ ScannerType, r *database.Redis, url, identifier string, stop chan bool) error {
	s := &scan{
		redis:      r,
		identifier: identifier,
	}

	var scanner Scanner
	switch typ {
	case RSYNC:
		scanner = &RsyncScanner{
			scan: s,
		}
	case FTP:
		scanner = &FTPScanner{
			scan: s,
		}
	default:
		panic(fmt.Sprintf("Unknown scanner"))
	}

	// Connect to the database
	conn := s.redis.Get()
	defer conn.Close()

	s.conn = conn

	lockKey := fmt.Sprintf("SCANNING_%s", identifier)

	// Try to aquire a lock so we don't have a scanning race
	// from different nodes.
	lock, err := redis.Bool(conn.Do("SETNX", lockKey, 1))
	if err != nil {
		return err
	}
	if lock {
		// Lock aquired.
		defer conn.Do("DEL", lockKey)
		// Make the key expire automatically in case our process gets killed
		conn.Do("EXPIRE", lockKey, 600)
	} else {
		return ScanInProgress
	}

	s.setLastSync(conn, identifier, false)

	conn.Send("MULTI")

	s.filesKey = fmt.Sprintf("MIRROR_%s_FILES", identifier)
	s.filesTmpKey = fmt.Sprintf("MIRROR_%s_FILES_TMP", identifier)

	// Remove any left over
	conn.Send("DEL", s.filesTmpKey)

	err = scanner.Scan(url, identifier, conn, stop)
	if err != nil {
		// Discard MULTI
		s.ScannerDiscard()

		// Remove the temporary key
		conn.Do("DEL", s.filesTmpKey)

		log.Errorf("[%s] %s", identifier, err.Error())
		return err
	}

	// Exec multi
	s.ScannerCommit()

	// Get the list of files no more present on this mirror
	toremove, err := redis.Values(conn.Do("SDIFF", s.filesKey, s.filesTmpKey))
	if err != nil {
		return err
	}

	// Remove this mirror from the given file SET
	if len(toremove) > 0 {
		conn.Send("MULTI")
		for _, e := range toremove {
			log.Debugf("[%s] Removing %s from mirror", identifier, e)
			conn.Send("SREM", fmt.Sprintf("FILEMIRRORS_%s", e), identifier)
			conn.Send("DEL", fmt.Sprintf("FILEINFO_%s_%s", identifier, e))
			// Publish update
			database.SendPublish(conn, database.MIRROR_FILE_UPDATE, fmt.Sprintf("%s %s", identifier, e))

		}
		_, err = conn.Do("EXEC")
		if err != nil {
			return err
		}
	}

	// Finally rename the temporary sets containing the list
	// of files for this mirror to the production key
	_, err = conn.Do("RENAME", s.filesTmpKey, s.filesKey)
	if err != nil {
		return err
	}

	sinterKey := fmt.Sprintf("HANDLEDFILES_%s", identifier)

	// Count the number of files known on the remote end
	common, _ := redis.Int64(conn.Do("SINTERSTORE", sinterKey, "FILES", s.filesKey))

	if err != nil {
		return err
	}

	s.setLastSync(conn, identifier, true)
	log.Infof("[%s] Indexed %d files (%d known), %d removed", identifier, s.count, common, len(toremove))
	return nil
}

func (s *scan) ScannerAddFile(f filedata) {
	s.count++

	// Add all the files to a temporary key
	s.conn.Send("SADD", s.filesTmpKey, f.path)

	// Mark the file as being supported by this mirror
	rk := fmt.Sprintf("FILEMIRRORS_%s", f.path)
	s.conn.Send("SADD", rk, s.identifier)

	// Save the size of the current file found on this mirror
	ik := fmt.Sprintf("FILEINFO_%s_%s", s.identifier, f.path)
	s.conn.Send("HSET", ik, "size", f.size)

	// Publish update
	database.SendPublish(s.conn, database.MIRROR_FILE_UPDATE, fmt.Sprintf("%s %s", s.identifier, f.path))
}

func (s *scan) ScannerDiscard() {
	s.conn.Do("DISCARD")
}

func (s *scan) ScannerCommit() error {
	_, err := s.conn.Do("EXEC")
	return err
}

func (s *scan) setLastSync(conn redis.Conn, identifier string, successful bool) error {
	now := time.Now().UTC().Unix()

	conn.Send("MULTI")

	// Set the last sync time
	conn.Send("HSET", fmt.Sprintf("MIRROR_%s", identifier), "lastSync", now)

	// Set the last successful sync time
	if successful {
		conn.Send("HSET", fmt.Sprintf("MIRROR_%s", identifier), "lastSuccessfulSync", now)
	}

	_, err := conn.Do("EXEC")

	// Publish an update on redis
	database.Publish(conn, database.MIRROR_UPDATE, identifier)

	return err
}

// Walk inside the source/reference repository
func (s *scan) walkSource(conn redis.Conn, path string, f os.FileInfo, err error) error {
	if f == nil || f.IsDir() || f.Mode()&os.ModeSymlink != 0 {
		return nil
	}

	d := new(filedata)
	d.path = path[len(GetConfig().Repository):]
	d.size = f.Size()
	d.modTime = f.ModTime()

	// Get the previous file properties
	properties, err := redis.Strings(conn.Do("HMGET", fmt.Sprintf("FILE_%s", d.path), "size", "modTime", "sha1", "sha256", "md5"))
	if err != nil && err != redis.ErrNil {
		return err
	} else if len(properties) < 5 {
		// This will force a rehash
		properties = make([]string, 5)
	}

	size, _ := strconv.ParseInt(properties[0], 10, 64)
	modTime, _ := time.Parse("2006-01-02 15:04:05.999999999 -0700 MST", properties[1])
	sha1 := properties[2]
	sha256 := properties[3]
	md5 := properties[4]

	rehash := (GetConfig().Hashes.SHA1 && len(sha1) == 0) ||
		(GetConfig().Hashes.SHA256 && len(sha256) == 0) ||
		(GetConfig().Hashes.MD5 && len(md5) == 0)

	if rehash || size != d.size || !modTime.Equal(d.modTime) {
		h, err := filesystem.HashFile(GetConfig().Repository + d.path)
		if err != nil {
			log.Warningf("%s: hashing failed: %s", d.path, err.Error())
		} else {
			d.sha1 = h.Sha1
			d.sha256 = h.Sha256
			d.md5 = h.Md5
			if len(d.sha1) > 0 {
				log.Infof("%s: SHA1 %s", d.path, d.sha1)
			}
			if len(d.sha256) > 0 {
				log.Infof("%s: SHA256 %s", d.path, d.sha256)
			}
			if len(d.md5) > 0 {
				log.Infof("%s: MD5 %s", d.path, d.md5)
			}
		}
	} else {
		d.sha1 = sha1
		d.sha256 = sha256
		d.md5 = md5
	}

	s.walkSourceFiles = append(s.walkSourceFiles, d)
	return nil
}

func ScanSource(r *database.Redis, stop chan bool) (err error) {
	s := &scan{
		redis: r,
	}

	conn := s.redis.Get()
	defer conn.Close()

	if conn.Err() != nil {
		return conn.Err()
	}

	s.walkSourceFiles = make([]*filedata, 0, 1000)
	defer func() {
		// Reset the slice so it can be garbage collected
		s.walkSourceFiles = nil
	}()

	//TODO lock atomically inside redis to avoid two simultanous scan

	if _, err := os.Stat(GetConfig().Repository); os.IsNotExist(err) {
		return fmt.Errorf("%s: No such file or directory", GetConfig().Repository)
	}

	log.Info("[source] Scanning the filesystem...")
	err = filepath.Walk(GetConfig().Repository, func(path string, f os.FileInfo, err error) error {
		return s.walkSource(conn, path, f, err)
	})

	if utils.IsStopped(stop) {
		return ScanAborted
	}
	if err != nil {
		return err
	}
	log.Info("[source] Indexing the files...")

	conn.Send("MULTI")

	// Remove any left over
	conn.Send("DEL", "FILES_TMP")

	// Add all the files to a temporary key
	count := 0
	for _, e := range s.walkSourceFiles {
		conn.Send("SADD", "FILES_TMP", e.path)
		count++
	}

	_, err = conn.Do("EXEC")
	if err != nil {
		return err
	}

	// Do a diff between the sets to get the removed files
	toremove, err := redis.Values(conn.Do("SDIFF", "FILES", "FILES_TMP"))

	// Create/Update the files' hash keys with the fresh infos
	conn.Send("MULTI")
	for _, e := range s.walkSourceFiles {
		conn.Send("HMSET", fmt.Sprintf("FILE_%s", e.path),
			"size", e.size,
			"modTime", e.modTime,
			"sha1", e.sha1,
			"sha256", e.sha256,
			"md5", e.md5)

		// Publish update
		database.SendPublish(conn, database.FILE_UPDATE, e.path)
	}

	// Remove old keys
	if len(toremove) > 0 {
		for _, e := range toremove {
			conn.Send("DEL", fmt.Sprintf("FILE_%s", e))

			// Publish update
			database.SendPublish(conn, database.FILE_UPDATE, fmt.Sprintf("%s", e))
		}
	}

	// Finally rename the temporary sets containing the list
	// of files to the production key
	conn.Send("RENAME", "FILES_TMP", "FILES")

	_, err = conn.Do("EXEC")
	if err != nil {
		return err
	}

	log.Infof("[source] Scanned %d files", count)

	return nil
}
<|endoftext|>


<commit_before>package stackevents

import (
	"github.com/racker/perigee"
	"github.com/rackspace/gophercloud"
	"github.com/rackspace/gophercloud/pagination"
)

// Find retreives stack events for the given stack name.
func Find(c *gophercloud.ServiceClient, stackName string) FindResult {
	var res FindResult

	_, res.Err = perigee.Request("GET", findURL(c, stackName), perigee.Options{
		MoreHeaders: c.AuthenticatedHeaders(),
		Results:     &res.Body,
		OkCodes:     []int{200},
	})
	return res
}

// SortDir is a type for specifying in which direction to sort a list of events.
type SortDir string

// SortKey is a type for specifying by which key to sort a list of events.
type SortKey string

// ResourceStatus is a type for specifying by which resource status to filter a
// list of events.
type ResourceStatus string

// ResourceAction is a type for specifying by which resource action to filter a
// list of events.
type ResourceAction string

var (
	// SortAsc is used to sort a list of stacks in ascending order.
	SortAsc SortDir = "asc"
	// SortDesc is used to sort a list of stacks in descending order.
	SortDesc SortDir = "desc"

	// SortName is used to sort a list of stacks by name.
	SortName SortKey = "name"
	// SortResourceType is used to sort a list of stacks by resource type.
	SortResourceType SortKey = "resource_type"
	// SortCreatedAt is used to sort a list of stacks by date created.
	SortCreatedAt SortKey = "created_at"
)

// ListOptsBuilder allows extensions to add additional parameters to the
// List request.
type ListOptsBuilder interface {
	ToStackEventListQuery() (string, error)
}

// ListOpts allows the filtering and sorting of paginated collections through
// the API. Marker and Limit are used for pagination.
type ListOpts struct {
	// The stack resource ID with which to start the listing.
	Marker string `q:"marker"`
	// Integer value for the limit of values to return.
	Limit int `q:"limit"`
	// Filters the event list by the specified ResourceAction. You can use this
	// filter multiple times to filter by multiple resource actions: CREATE, DELETE,
	// UPDATE, ROLLBACK, SUSPEND, RESUME or ADOPT.
	ResourceActions []string `q:"resource_action"`
	// Filters the event list by the specified resource_status. You can use this
	// filter multiple times to filter by multiple resource statuses: IN_PROGRESS,
	// COMPLETE or FAILED.
	ResourceStatuses []string `q:"resource_status"`
	// Filters the event list by the specified resource_name. You can use this
	// filter multiple times to filter by multiple resource names.
	ResourceNames []string `q:"resource_name"`
	// Filters the event list by the specified resource_type. You can use this
	// filter multiple times to filter by multiple resource types: OS::Nova::Server,
	// OS::Cinder::Volume, and so on.
	ResourceTypes []string `q:"resource_type"`
	// Sorts the event list by: resource_type or created_at.
	SortKey SortKey `q:"sort_keys"`
	// The sort direction of the event list. Which is asc (ascending) or desc (descending).
	SortDir SortDir `q:"sort_dir"`
}

// ToStackEventListQuery formats a ListOpts into a query string.
func (opts ListOpts) ToStackEventListQuery() (string, error) {
	q, err := gophercloud.BuildQueryString(opts)
	if err != nil {
		return "", err
	}
	return q.String(), nil
}

// List makes a request against the API to list resources for the given stack.
func List(client *gophercloud.ServiceClient, stackName, stackID string, opts ListOptsBuilder) pagination.Pager {
	url := listURL(client, stackName, stackID)

	if opts != nil {
		query, err := opts.ToStackEventListQuery()
		if err != nil {
			return pagination.Pager{Err: err}
		}
		url += query
	}

	createPageFn := func(r pagination.PageResult) pagination.Page {
		p := EventPage{pagination.MarkerPageBase{PageResult: r}}
		p.MarkerPageBase.Owner = p
		return p
	}

	return pagination.NewPager(client, url, createPageFn)
}

// ListResourceEventsOptsBuilder allows extensions to add additional parameters to the
// ListResourceEvents request.
type ListResourceEventsOptsBuilder interface {
	ToResourceEventListQuery() (string, error)
}

// ListResourceEventsOpts allows the filtering and sorting of paginated resource events through
// the API. Marker and Limit are used for pagination.
type ListResourceEventsOpts struct {
	// The stack resource ID with which to start the listing.
	Marker string `q:"marker"`
	// Integer value for the limit of values to return.
	Limit int `q:"limit"`
	// Filters the event list by the specified ResourceAction. You can use this
	// filter multiple times to filter by multiple resource actions: CREATE, DELETE,
	// UPDATE, ROLLBACK, SUSPEND, RESUME or ADOPT.
	ResourceActions []string `q:"resource_action"`
	// Filters the event list by the specified resource_status. You can use this
	// filter multiple times to filter by multiple resource statuses: IN_PROGRESS,
	// COMPLETE or FAILED.
	ResourceStatuses []string `q:"resource_status"`
	// Filters the event list by the specified resource_name. You can use this
	// filter multiple times to filter by multiple resource names.
	ResourceNames []string `q:"resource_name"`
	// Filters the event list by the specified resource_type. You can use this
	// filter multiple times to filter by multiple resource types: OS::Nova::Server,
	// OS::Cinder::Volume, and so on.
	ResourceTypes []string `q:"resource_type"`
	// Sorts the event list by: resource_type or created_at.
	SortKey SortKey `q:"sort_keys"`
	// The sort direction of the event list. Which is asc (ascending) or desc (descending).
	SortDir SortDir `q:"sort_dir"`
}

// ToResourceEventsListQuery formats a ListOpts into a query string.
func (opts ListOpts) ToResourceEventsListQuery() (string, error) {
	q, err := gophercloud.BuildQueryString(opts)
	if err != nil {
		return "", err
	}
	return q.String(), nil
}

// ListResourceEvents makes a request against the API to list resources for the given stack.
func ListResourceEvents(client *gophercloud.ServiceClient, stackName, stackID, resourceName string, opts ListResourceEventsOptsBuilder) pagination.Pager {
	url := listResourceEventsURL(client, stackName, stackID, resourceName)

	if opts != nil {
		query, err := opts.ToResourceEventListQuery()
		if err != nil {
			return pagination.Pager{Err: err}
		}
		url += query
	}

	createPageFn := func(r pagination.PageResult) pagination.Page {
		p := EventPage{pagination.MarkerPageBase{PageResult: r}}
		p.MarkerPageBase.Owner = p
		return p
	}

	return pagination.NewPager(client, url, createPageFn)
}

// Get retreives data for the given stack resource.
func Get(c *gophercloud.ServiceClient, stackName, stackID, resourceName, eventID string) GetResult {
	var res GetResult
	_, res.Err = perigee.Request("GET", getURL(c, stackName, stackID, resourceName, eventID), perigee.Options{
		MoreHeaders: c.AuthenticatedHeaders(),
		Results:     &res.Body,
		OkCodes:     []int{200},
	})
	return res
}
<commit_msg>fix typo: retreives -> retrieves<commit_after>package stackevents

import (
	"github.com/racker/perigee"
	"github.com/rackspace/gophercloud"
	"github.com/rackspace/gophercloud/pagination"
)

// Find retrieves stack events for the given stack name.
func Find(c *gophercloud.ServiceClient, stackName string) FindResult {
	var res FindResult

	_, res.Err = perigee.Request("GET", findURL(c, stackName), perigee.Options{
		MoreHeaders: c.AuthenticatedHeaders(),
		Results:     &res.Body,
		OkCodes:     []int{200},
	})
	return res
}

// SortDir is a type for specifying in which direction to sort a list of events.
type SortDir string

// SortKey is a type for specifying by which key to sort a list of events.
type SortKey string

// ResourceStatus is a type for specifying by which resource status to filter a
// list of events.
type ResourceStatus string

// ResourceAction is a type for specifying by which resource action to filter a
// list of events.
type ResourceAction string

var (
	// SortAsc is used to sort a list of stacks in ascending order.
	SortAsc SortDir = "asc"
	// SortDesc is used to sort a list of stacks in descending order.
	SortDesc SortDir = "desc"

	// SortName is used to sort a list of stacks by name.
	SortName SortKey = "name"
	// SortResourceType is used to sort a list of stacks by resource type.
	SortResourceType SortKey = "resource_type"
	// SortCreatedAt is used to sort a list of stacks by date created.
	SortCreatedAt SortKey = "created_at"
)

// ListOptsBuilder allows extensions to add additional parameters to the
// List request.
type ListOptsBuilder interface {
	ToStackEventListQuery() (string, error)
}

// ListOpts allows the filtering and sorting of paginated collections through
// the API. Marker and Limit are used for pagination.
type ListOpts struct {
	// The stack resource ID with which to start the listing.
	Marker string `q:"marker"`
	// Integer value for the limit of values to return.
	Limit int `q:"limit"`
	// Filters the event list by the specified ResourceAction. You can use this
	// filter multiple times to filter by multiple resource actions: CREATE, DELETE,
	// UPDATE, ROLLBACK, SUSPEND, RESUME or ADOPT.
	ResourceActions []string `q:"resource_action"`
	// Filters the event list by the specified resource_status. You can use this
	// filter multiple times to filter by multiple resource statuses: IN_PROGRESS,
	// COMPLETE or FAILED.
	ResourceStatuses []string `q:"resource_status"`
	// Filters the event list by the specified resource_name. You can use this
	// filter multiple times to filter by multiple resource names.
	ResourceNames []string `q:"resource_name"`
	// Filters the event list by the specified resource_type. You can use this
	// filter multiple times to filter by multiple resource types: OS::Nova::Server,
	// OS::Cinder::Volume, and so on.
	ResourceTypes []string `q:"resource_type"`
	// Sorts the event list by: resource_type or created_at.
	SortKey SortKey `q:"sort_keys"`
	// The sort direction of the event list. Which is asc (ascending) or desc (descending).
	SortDir SortDir `q:"sort_dir"`
}

// ToStackEventListQuery formats a ListOpts into a query string.
func (opts ListOpts) ToStackEventListQuery() (string, error) {
	q, err := gophercloud.BuildQueryString(opts)
	if err != nil {
		return "", err
	}
	return q.String(), nil
}

// List makes a request against the API to list resources for the given stack.
func List(client *gophercloud.ServiceClient, stackName, stackID string, opts ListOptsBuilder) pagination.Pager {
	url := listURL(client, stackName, stackID)

	if opts != nil {
		query, err := opts.ToStackEventListQuery()
		if err != nil {
			return pagination.Pager{Err: err}
		}
		url += query
	}

	createPageFn := func(r pagination.PageResult) pagination.Page {
		p := EventPage{pagination.MarkerPageBase{PageResult: r}}
		p.MarkerPageBase.Owner = p
		return p
	}

	return pagination.NewPager(client, url, createPageFn)
}

// ListResourceEventsOptsBuilder allows extensions to add additional parameters to the
// ListResourceEvents request.
type ListResourceEventsOptsBuilder interface {
	ToResourceEventListQuery() (string, error)
}

// ListResourceEventsOpts allows the filtering and sorting of paginated resource events through
// the API. Marker and Limit are used for pagination.
type ListResourceEventsOpts struct {
	// The stack resource ID with which to start the listing.
	Marker string `q:"marker"`
	// Integer value for the limit of values to return.
	Limit int `q:"limit"`
	// Filters the event list by the specified ResourceAction. You can use this
	// filter multiple times to filter by multiple resource actions: CREATE, DELETE,
	// UPDATE, ROLLBACK, SUSPEND, RESUME or ADOPT.
	ResourceActions []string `q:"resource_action"`
	// Filters the event list by the specified resource_status. You can use this
	// filter multiple times to filter by multiple resource statuses: IN_PROGRESS,
	// COMPLETE or FAILED.
	ResourceStatuses []string `q:"resource_status"`
	// Filters the event list by the specified resource_name. You can use this
	// filter multiple times to filter by multiple resource names.
	ResourceNames []string `q:"resource_name"`
	// Filters the event list by the specified resource_type. You can use this
	// filter multiple times to filter by multiple resource types: OS::Nova::Server,
	// OS::Cinder::Volume, and so on.
	ResourceTypes []string `q:"resource_type"`
	// Sorts the event list by: resource_type or created_at.
	SortKey SortKey `q:"sort_keys"`
	// The sort direction of the event list. Which is asc (ascending) or desc (descending).
	SortDir SortDir `q:"sort_dir"`
}

// ToResourceEventsListQuery formats a ListOpts into a query string.
func (opts ListOpts) ToResourceEventsListQuery() (string, error) {
	q, err := gophercloud.BuildQueryString(opts)
	if err != nil {
		return "", err
	}
	return q.String(), nil
}

// ListResourceEvents makes a request against the API to list resources for the given stack.
func ListResourceEvents(client *gophercloud.ServiceClient, stackName, stackID, resourceName string, opts ListResourceEventsOptsBuilder) pagination.Pager {
	url := listResourceEventsURL(client, stackName, stackID, resourceName)

	if opts != nil {
		query, err := opts.ToResourceEventListQuery()
		if err != nil {
			return pagination.Pager{Err: err}
		}
		url += query
	}

	createPageFn := func(r pagination.PageResult) pagination.Page {
		p := EventPage{pagination.MarkerPageBase{PageResult: r}}
		p.MarkerPageBase.Owner = p
		return p
	}

	return pagination.NewPager(client, url, createPageFn)
}

// Get retreives data for the given stack resource.
func Get(c *gophercloud.ServiceClient, stackName, stackID, resourceName, eventID string) GetResult {
	var res GetResult
	_, res.Err = perigee.Request("GET", getURL(c, stackName, stackID, resourceName, eventID), perigee.Options{
		MoreHeaders: c.AuthenticatedHeaders(),
		Results:     &res.Body,
		OkCodes:     []int{200},
	})
	return res
}
<|endoftext|>


<commit_before>package tulib

type Rect struct {
	X int
	Y int
	Width int
	Height int
}

func (r Rect) IsValid() bool {
	return r.Width >= 0 && r.Height >= 0
}

func (r Rect) IsEmpty() bool {
	return r.Width <= 0 || r.Height <= 0
}

func (a Rect) Intersection(b Rect) Rect {
	// no intersection cases
	if a.X >= b.Width || a.Y >= b.Height {
		return Rect{0, 0, 0, 0}
	}

	if a.X + a.Width <= b.X || a.Y + a.Height <= b.Y {
		return Rect{0, 0, 0, 0}
	}

	// adjust X or Width
	if a.X + a.Width > b.X + b.Width {
		a.Width = (b.X + b.Width) - a.X
	}

	if a.X < b.X {
		a.Width -= b.X - a.X
		a.X = b.X
	}

	// adjust Y or Height
	if a.Y + a.Height > b.Y + b.Height {
		a.Height = (b.Y + b.Height) - a.Y
	}

	if a.Y < b.Y {
		a.Height -= b.Y - a.Y
		a.Y = b.Y
	}

	return a
}
<commit_msg>Add Rect.FitsIn method.<commit_after>package tulib

type Rect struct {
	X int
	Y int
	Width int
	Height int
}

func (r Rect) IsValid() bool {
	return r.Width >= 0 && r.Height >= 0
}

func (r Rect) IsEmpty() bool {
	return r.Width <= 0 || r.Height <= 0
}

func (a Rect) FitsIn(b Rect) bool {
	return a == a.Intersection(b)
}

func (a Rect) Intersection(b Rect) Rect {
	// no intersection cases
	if a.X >= b.Width || a.Y >= b.Height {
		return Rect{0, 0, 0, 0}
	}

	if a.X + a.Width <= b.X || a.Y + a.Height <= b.Y {
		return Rect{0, 0, 0, 0}
	}

	// adjust X or Width
	if a.X + a.Width > b.X + b.Width {
		a.Width = (b.X + b.Width) - a.X
	}

	if a.X < b.X {
		a.Width -= b.X - a.X
		a.X = b.X
	}

	// adjust Y or Height
	if a.Y + a.Height > b.Y + b.Height {
		a.Height = (b.Y + b.Height) - a.Y
	}

	if a.Y < b.Y {
		a.Height -= b.Y - a.Y
		a.Y = b.Y
	}

	return a
}
<|endoftext|>


<commit_before>package udpsender

import (
	"bytes"
	"crypto/aes"
	"crypto/cipher"
	"crypto/rand"
	"crypto/sha1"
	"encoding/binary"
	"encoding/hex"
	"errors"
	"fmt"
	"net"
	"os"
	"time"
)

type RecvService struct {
	key          string
	ch           chan *RecvBlock
	transactions map[string]*RecvTransaction
	sock         *net.UDPConn
}

type RecvTransaction struct {
	num         int
	blocks      map[int]*RecvBlock
	lastTime    int64
	blockSize   int64
	speed       float64
	completed   bool
	addr        *net.UDPAddr
	transaction []byte
}

type RecvBlock struct {
	transaction      []byte
	blockHash        []byte
	fileHash         []byte
	fileData         []byte
	num, id          int32
	start, end, size int64
	time1            int64
	addr             *net.UDPAddr
}

func InitRecvService(key string, url string) (service *RecvService, err error) {
	service = new(RecvService)
	service.key = key
	service.ch = make(chan *RecvBlock, 100)
	service.transactions = map[string]*RecvTransaction{}

	addr, err := net.ResolveUDPAddr("udp", url)
	if err != nil {
		return nil, err
	}
	service.sock, err = net.ListenUDP("udp", addr)
	if err != nil {
		return nil, err
	}
	go service.loop()

	return service, nil
}

func InitRecvTransaction(block *RecvBlock) *RecvTransaction {
	transaction := new(RecvTransaction)
	transaction.blocks = map[int]*RecvBlock{}
	transaction.lastTime = time.Now().UnixNano()
	transaction.blockSize = int64(len(block.fileData))
	transaction.num = int(block.num)
	transaction.addr = block.addr
	transaction.transaction = block.transaction
	return transaction
}
func (service *RecvService) processRecvBlock(block *RecvBlock) {
	stransaction := hex.EncodeToString(block.transaction)
	transaction, ok := service.transactions[stransaction]
	if !ok {
		fmt.Println("Creating transaction:", stransaction)
		transaction = InitRecvTransaction(block)
		service.transactions[stransaction] = transaction
	}
	if transaction.num != int(block.num) {
		// error
	}
	block.time1 = time.Now().UnixNano()
	fdtime := float64(block.time1 - transaction.lastTime)
	transaction.lastTime = block.time1
	speed := float64(len(block.fileData)) / fdtime * float64(time.Second)
	transaction.speed = movingExpAvg(speed, transaction.speed, fdtime, float64(time.Second))

	_, ok = transaction.blocks[int(block.id)]
	if ok {
		// dupe
		return
	}
	transaction.blocks[int(block.id)] = block
	// fmt.Printf("got %d %d %d\n%s\n", block.id, len(transaction.blocks), transaction.num, string(block.fileData))
	fmt.Printf("got %d %d %d %f\n", block.id, len(transaction.blocks), transaction.num, transaction.speed)
	if len(transaction.blocks) == transaction.num {
		transaction.completed = true
		fmt.Printf("Finish RecvTransaction %f\n", transaction.speed)
		// finish transaction
		// save file

		f1, err := os.Create("incomming/" + stransaction + "/file.out")
		if err != nil {
			panic(err)
		}
		// defer f1.Close()

		for i := 1; i <= transaction.num; i++ {
			block1, ok := transaction.blocks[i]
			if !ok {
				// error
				panic(errors.New(fmt.Sprintf("Not all blocks %d", i)))
			}
			f1.Write(block1.fileData)
		}
		f1.Close()
		// file recieved
	}
}
func (service *RecvService) Loop() {

	for {
		// len, remote, err := sock.ReadFromUDP(buf[:])
		var buf [1 << 16]byte
		len, remote, err := service.sock.ReadFromUDP(buf[:])
		if err != nil {
			panic(err)
		}
		service.processPacket(remote, buf[:len])
	}
}
func (service *RecvService) loop() {
	ticker := time.Tick(100 * time.Millisecond)
	for {

		select {
		case blck, ok := <-service.ch:
			if ok {
				service.processRecvBlock(blck)
			}
		case <-ticker:
			time1 := time.Now().UnixNano()
			for _, transaction := range service.transactions {
				if transaction.completed {
					continue
				}

				fdtime := float64(time1 - transaction.lastTime)
				transaction.lastTime = time1
				transaction.speed = movingExpAvg(0.0, transaction.speed, fdtime, float64(time.Second))

				//
				// find last_block
				last_block := -1
				var block *RecvBlock
				for i, _block := range transaction.blocks {
					if last_block < i {
						last_block = i
						block = _block
					}
				}
				if last_block == -1 {
					continue
				}
				// dtime  = Now - last_block.time
				dtime := time1 - block.time1
				// last_block + (dtime-100ms)*speed/blocksize
				shift := last_block + int((float64(dtime)-100.0*float64(time.Millisecond))*transaction.speed/float64(transaction.blockSize))
				// get first 100 un recieved blocks

				req := []int{}

				for i := 1; i <= transaction.num; i++ {
					if i > shift {
						break
					}
					_, ok := transaction.blocks[i]
					if !ok {
						req = append(req, i)
						if len(req) >= 100 {
							break
						}
					}
				}

				if len(req) > 0 {
					service.rerequestPackets(transaction, req)
				}
			}
		}

	}

}
func (service *RecvService) rerequestPackets(transaction *RecvTransaction, req []int) {
	r1 := make([]byte, 16)
	_, err := rand.Read(r1)
	if err != nil {
		panic(err)
	}

	data := append(transaction.transaction, r1...)

	buffer := new(bytes.Buffer)
	binary.Write(buffer, binary.LittleEndian, int32(1))
	binary.Write(buffer, binary.LittleEndian, int32(len(req)))
	for _, i := range req {
		binary.Write(buffer, binary.LittleEndian, int32(i))
	}

	block, err := aes.NewCipher([]byte(service.key))
	if err != nil {
		panic(err)
	}

	data1 := buffer.Bytes()

	stream := cipher.NewCFBEncrypter(block, r1)
	stream.XORKeyStream(data1, data1)

	data = append(data, data1...)

	service.sock.WriteToUDP(data, transaction.addr)
}
func (service *RecvService) processPacket(addr *net.UDPAddr, buf []byte) {
	blck := new(RecvBlock)
	blck.transaction = buf[:16]
	blck.addr = addr
	r1 := buf[16:32]
	data2 := buf[32:]

	block, err := aes.NewCipher([]byte(service.key))
	if err != nil {
		panic(err)
	}

	stream := cipher.NewCFBDecrypter(block, r1)

	stream.XORKeyStream(data2, data2)

	// fileHash := data2[0:20]

	buffer := bytes.NewBuffer(data2[20:52])
	binary.Read(buffer, binary.LittleEndian, &blck.id)
	binary.Read(buffer, binary.LittleEndian, &blck.num)
	binary.Read(buffer, binary.LittleEndian, &blck.start)
	binary.Read(buffer, binary.LittleEndian, &blck.end)
	binary.Read(buffer, binary.LittleEndian, &blck.size)
	blck.blockHash = data2[52:72]
	blck.fileData = data2[72:]
	blockHash := sha1.Sum(blck.fileData)
	verified := bytes.Equal(blck.blockHash, blockHash[:])

	fmt.Println("processPacker:", blck.id, hex.EncodeToString(r1))

	if verified {
		os.MkdirAll("incomming/"+hex.EncodeToString(blck.transaction), 0700)
		f1, err := os.Create("incomming/" + hex.EncodeToString(blck.transaction) + "/" + fmt.Sprintf("%d.data", blck.id))
		if err != nil {
			panic(err)
		}
		defer f1.Close()
		f1.Write(blck.fileData)
		service.ch <- blck
	} else {
		fmt.Println("wrong hash %#v %#v", blck.blockHash, blockHash[:])
	}

}
<commit_msg>double copy<commit_after>package udpsender

import (
	"bytes"
	"crypto/aes"
	"crypto/cipher"
	"crypto/rand"
	"crypto/sha1"
	"encoding/binary"
	"encoding/hex"
	"errors"
	"fmt"
	"net"
	"os"
	"time"
)

type RecvService struct {
	key          string
	ch           chan *RecvBlock
	transactions map[string]*RecvTransaction
	sock         *net.UDPConn
}

type RecvTransaction struct {
	num         int
	blocks      map[int]*RecvBlock
	lastTime    int64
	blockSize   int64
	speed       float64
	completed   bool
	addr        *net.UDPAddr
	transaction []byte
}

type RecvBlock struct {
	transaction      []byte
	blockHash        []byte
	fileHash         []byte
	fileData         []byte
	num, id          int32
	start, end, size int64
	time1            int64
	addr             *net.UDPAddr
}

func InitRecvService(key string, url string) (service *RecvService, err error) {
	service = new(RecvService)
	service.key = key
	service.ch = make(chan *RecvBlock, 100)
	service.transactions = map[string]*RecvTransaction{}

	addr, err := net.ResolveUDPAddr("udp", url)
	if err != nil {
		return nil, err
	}
	service.sock, err = net.ListenUDP("udp", addr)
	if err != nil {
		return nil, err
	}
	go service.loop()

	return service, nil
}

func InitRecvTransaction(block *RecvBlock) *RecvTransaction {
	transaction := new(RecvTransaction)
	transaction.blocks = map[int]*RecvBlock{}
	transaction.lastTime = time.Now().UnixNano()
	transaction.blockSize = int64(len(block.fileData))
	transaction.num = int(block.num)
	transaction.addr = block.addr
	transaction.transaction = block.transaction
	return transaction
}
func (service *RecvService) processRecvBlock(block *RecvBlock) {
	stransaction := hex.EncodeToString(block.transaction)
	transaction, ok := service.transactions[stransaction]
	if !ok {
		fmt.Println("Creating transaction:", stransaction)
		transaction = InitRecvTransaction(block)
		service.transactions[stransaction] = transaction
	}
	if transaction.num != int(block.num) {
		// error
	}
	block.time1 = time.Now().UnixNano()
	fdtime := float64(block.time1 - transaction.lastTime)
	transaction.lastTime = block.time1
	speed := float64(len(block.fileData)) / fdtime * float64(time.Second)
	transaction.speed = movingExpAvg(speed, transaction.speed, fdtime, float64(time.Second))

	_, ok = transaction.blocks[int(block.id)]
	if ok {
		// dupe
		return
	}
	transaction.blocks[int(block.id)] = block
	// fmt.Printf("got %d %d %d\n%s\n", block.id, len(transaction.blocks), transaction.num, string(block.fileData))
	fmt.Printf("got %d %d %d %f\n", block.id, len(transaction.blocks), transaction.num, transaction.speed)
	if len(transaction.blocks) == transaction.num {
		transaction.completed = true
		fmt.Printf("Finish RecvTransaction %f\n", transaction.speed)
		// finish transaction
		// save file

		f1, err := os.Create("incomming/" + stransaction + "/file.out")
		if err != nil {
			panic(err)
		}
		// defer f1.Close()

		for i := 1; i <= transaction.num; i++ {
			block1, ok := transaction.blocks[i]
			if !ok {
				// error
				panic(errors.New(fmt.Sprintf("Not all blocks %d", i)))
			}
			f1.Write(block1.fileData)
		}
		f1.Close()
		// file recieved
	}
}
func (service *RecvService) Loop() {

	for {
		// len, remote, err := sock.ReadFromUDP(buf[:])
		var buf [1 << 16]byte
		len, remote, err := service.sock.ReadFromUDP(buf[:])
		if err != nil {
			panic(err)
		}
		service.processPacket(remote, buf[:len])
	}
}
func (service *RecvService) loop() {
	ticker := time.Tick(100 * time.Millisecond)
	for {

		select {
		case blck, ok := <-service.ch:
			if ok {
				service.processRecvBlock(blck)
			}
		case <-ticker:
			time1 := time.Now().UnixNano()
			for _, transaction := range service.transactions {
				if transaction.completed {
					continue
				}

				fdtime := float64(time1 - transaction.lastTime)
				transaction.lastTime = time1
				transaction.speed = movingExpAvg(0.0, transaction.speed, fdtime, float64(time.Second))

				//
				// find last_block
				last_block := -1
				var block *RecvBlock
				for i, _block := range transaction.blocks {
					if last_block < i {
						last_block = i
						block = _block
					}
				}
				if last_block == -1 {
					continue
				}
				// dtime  = Now - last_block.time
				dtime := time1 - block.time1
				// last_block + (dtime-100ms)*speed/blocksize
				shift := last_block + int((float64(dtime)-100.0*float64(time.Millisecond))*transaction.speed/float64(transaction.blockSize))
				// get first 100 un recieved blocks

				req := []int{}

				for i := 1; i <= transaction.num; i++ {
					if i > shift {
						break
					}
					_, ok := transaction.blocks[i]
					if !ok {
						req = append(req, i)
						if len(req) >= 100 {
							break
						}
					}
				}

				if len(req) > 0 {
					service.rerequestPackets(transaction, req)
				}
			}
		}

	}

}
func (service *RecvService) rerequestPackets(transaction *RecvTransaction, req []int) {
	r1 := make([]byte, 16)
	_, err := rand.Read(r1)
	if err != nil {
		panic(err)
	}

	data := append(transaction.transaction, r1...)

	buffer := new(bytes.Buffer)
	binary.Write(buffer, binary.LittleEndian, int32(1))
	binary.Write(buffer, binary.LittleEndian, int32(len(req)))
	for _, i := range req {
		binary.Write(buffer, binary.LittleEndian, int32(i))
	}

	block, err := aes.NewCipher([]byte(service.key))
	if err != nil {
		panic(err)
	}

	data1 := buffer.Bytes()

	stream := cipher.NewCFBEncrypter(block, r1)
	stream.XORKeyStream(data1, data1)

	data = append(data, data1...)

	service.sock.WriteToUDP(data, transaction.addr)
}
func (service *RecvService) processPacket(addr *net.UDPAddr, buf []byte) {
	blck := new(RecvBlock)
	blck.transaction = buf[:16]
	blck.addr = addr
	r1 := buf[16:32]
	data2 := buf[32:]

	block, err := aes.NewCipher([]byte(service.key))
	if err != nil {
		panic(err)
	}

	stream := cipher.NewCFBDecrypter(block, r1)

	stream.XORKeyStream(data2, data2)

	// fileHash := data2[0:20]

	buffer := bytes.NewBuffer(data2[20:52])
	binary.Read(buffer, binary.LittleEndian, &blck.id)
	binary.Read(buffer, binary.LittleEndian, &blck.num)
	binary.Read(buffer, binary.LittleEndian, &blck.start)
	binary.Read(buffer, binary.LittleEndian, &blck.end)
	binary.Read(buffer, binary.LittleEndian, &blck.size)
	blck.blockHash = data2[52:72]
	bufCopy := make([]byte, len(data2[72:]))
	copy(bufCopy[:], data2[72:])
	blck.fileData = bufCopy
	blockHash := sha1.Sum(blck.fileData)
	verified := bytes.Equal(blck.blockHash, blockHash[:])

	fmt.Println("processPacker:", blck.id, hex.EncodeToString(r1))

	if verified {
		os.MkdirAll("incomming/"+hex.EncodeToString(blck.transaction), 0700)
		f1, err := os.Create("incomming/" + hex.EncodeToString(blck.transaction) + "/" + fmt.Sprintf("%d.data", blck.id))
		if err != nil {
			panic(err)
		}
		defer f1.Close()
		f1.Write(blck.fileData)
		service.ch <- blck
	} else {
		fmt.Println("wrong hash %#v %#v", blck.blockHash, blockHash[:])
	}

}
<|endoftext|>


<commit_before>// RTLAMR - An rtl-sdr receiver for smart meters operating in the 900MHz ISM band.
// Copyright (C) 2014 Douglas Hall
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published
// by the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package main

import (
	"encoding/gob"
	"encoding/json"
	"encoding/xml"
	"errors"
	"flag"
	"fmt"
	"log"
	"math"
	"net"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"time"

	"github.com/bemasher/rtltcp"

	"github.com/bemasher/rtlamr/bch"
	"github.com/bemasher/rtlamr/preamble"
)

const (
	BlockSize = 1 << 12

	SampleRate   = 2.4e6
	DataRate     = 32.768e3
	SymbolLength = SampleRate / DataRate

	PreambleSymbols = 42
	PreambleLength  = PreambleSymbols * SymbolLength

	PacketSymbols = 192
	PacketLength  = PacketSymbols * SymbolLength

	PreambleDFTSize = 8192

	CenterFreq    = 920299072
	RestrictLocal = false

	Preamble     = 0x1F2A60
	PreambleBits = "111110010101001100000"

	GenPoly = 0x16F63

	TimeFormat = "2006-01-02T15:04:05.000"
)

var SymLen = IntRound(SymbolLength)
var config Config

type Config struct {
	serverAddr     string
	logFilename    string
	sampleFilename string
	format         string

	ServerAddr *net.TCPAddr
	CenterFreq uint
	TimeLimit  time.Duration
	MeterID    uint

	Log     *log.Logger
	LogFile *os.File

	GobUnsafe bool
	Encoder   Encoder

	SampleFile *os.File

	Quiet bool
}

func (c *Config) Parse() (err error) {
	flag.StringVar(&c.serverAddr, "server", "127.0.0.1:1234", "address or hostname of rtl_tcp instance")
	flag.StringVar(&c.logFilename, "logfile", "/dev/stdout", "log statement dump file")
	flag.StringVar(&c.sampleFilename, "samplefile", os.DevNull, "received message signal dump file, offset and message length are displayed to log when enabled")
	flag.UintVar(&c.CenterFreq, "centerfreq", 920299072, "center frequency to receive on")
	flag.DurationVar(&c.TimeLimit, "duration", 0, "time to run for, 0 for infinite")
	flag.UintVar(&c.MeterID, "filterid", 0, "display only messages matching given id")
	flag.StringVar(&c.format, "format", "plain", "format to write log messages in: plain, json, xml or gob")
	flag.BoolVar(&c.GobUnsafe, "gobunsafe", false, "allow gob output to stdout")
	flag.BoolVar(&c.Quiet, "quiet", false, "suppress state information printed at startup")

	flag.Parse()

	// Parse and resolve rtl_tcp server address.
	c.ServerAddr, err = net.ResolveTCPAddr("tcp", c.serverAddr)
	if err != nil {
		return
	}

	// Open or create the log file.
	if c.logFilename == "/dev/stdout" {
		c.LogFile = os.Stdout
	} else {
		c.LogFile, err = os.Create(c.logFilename)
	}

	// Create a new logger with the log file as output.
	c.Log = log.New(c.LogFile, "", log.Lshortfile)
	if err != nil {
		return
	}

	// Create the sample file.
	c.SampleFile, err = os.Create(c.sampleFilename)
	if err != nil {
		return
	}

	// Create encoder for specified logging format.
	switch strings.ToLower(c.format) {
	case "plain":
		break
	case "json":
		c.Encoder = json.NewEncoder(c.LogFile)
	case "xml":
		c.Encoder = xml.NewEncoder(c.LogFile)
	case "gob":
		c.Encoder = gob.NewEncoder(c.LogFile)

		// Don't let the user output gob to stdout unless they really want to.
		if !c.GobUnsafe && c.logFilename == "/dev/stdout" {
			fmt.Println("Gob encoded messages are not stdout safe, specify logfile or use gobunsafe flag.")
			os.Exit(1)
		}
	default:
		// We didn't get a valid encoder, exit and say so.
		log.Fatal("Invalid log format:", c.format)
	}

	return
}

func (c Config) Close() {
	c.LogFile.Close()
	c.SampleFile.Close()
}

// JSON, XML and GOB all implement this interface so we can simplify log
// output formatting.
type Encoder interface {
	Encode(interface{}) error
}

type Receiver struct {
	rtltcp.SDR

	pd  preamble.PreambleDetector
	bch bch.BCH
}

func NewReceiver(blockSize int) (rcvr Receiver) {
	// Plan the preamble detector.
	rcvr.pd = preamble.NewPreambleDetector(PreambleDFTSize, SymbolLength, PreambleBits)

	// Create a new BCH for error detection.
	rcvr.bch = bch.NewBCH(GenPoly)
	if !config.Quiet {
		config.Log.Printf("BCH: %+v\n", rcvr.bch)
	}

	// Connect to rtl_tcp server.
	if err := rcvr.Connect(config.ServerAddr); err != nil {
		config.Log.Fatal(err)
	}

	// Tell the user how many gain settings were reported by rtl_tcp.
	if !config.Quiet {
		config.Log.Println("GainCount:", rcvr.SDR.Info.GainCount)
	}

	// Set some parameters for listening.
	rcvr.SetSampleRate(SampleRate)
	rcvr.SetCenterFreq(uint32(config.CenterFreq))
	rcvr.SetGainMode(true)

	return
}

// Clean up rtl_tcp connection and destroy preamble detection plan.
func (rcvr *Receiver) Close() {
	rcvr.SDR.Close()
	rcvr.pd.Close()
}

func (rcvr *Receiver) Run() {
	// Setup signal channel for interruption.
	sigint := make(chan os.Signal, 1)
	signal.Notify(sigint)

	// Allocate sample and demodulated signal buffers.
	raw := make([]byte, IntRound(PacketLength+BlockSize)<<1)
	amBuf := make([]float64, IntRound(PacketLength+BlockSize))

	// Setup time limit channel
	tLimit := make(<-chan time.Time, 1)
	if config.TimeLimit != 0 {
		tLimit = time.After(config.TimeLimit)
	}

	start := time.Now()
	for {
		// Exit on interrupt or time limit, otherwise receive.
		select {
		case <-sigint:
			return
		case <-tLimit:
			fmt.Println("Time Limit Reached:", time.Since(start))
			return
		default:
			copy(raw, raw[BlockSize<<1:])
			copy(amBuf, amBuf[BlockSize:])

			// Read new sample block.
			_, err := rcvr.Read(raw[IntRound(PacketLength)<<1:])
			if err != nil {
				config.Log.Fatal("Error reading samples:", err)
			}

			// AM Demodulate
			lower := IntRound(PacketLength)
			for i := 0; i < BlockSize; i++ {
				amBuf[lower+i] = Mag(raw[(lower+i)<<1], raw[((lower+i)<<1)+1])
			}

			// Detect preamble in first half of demod buffer.
			rcvr.pd.Execute(amBuf)
			align := rcvr.pd.ArgMax()

			// Bad framing, catch message on next block.
			if align > BlockSize {
				continue
			}

			// Filter signal and bit slice enough data to catch the preamble.
			filtered := MatchedFilter(amBuf[align:], PreambleSymbols>>1)
			bits := BitSlice(filtered)

			// If the preamble matches.
			if bits == PreambleBits {
				for i := BlockSize << 1; i < len(amBuf); i++ {
					amBuf[i] = Mag(raw[i<<1], raw[(i<<1)+1])
				}

				// Filter, slice and parse the rest of the buffered samples.
				filtered := MatchedFilter(amBuf[align:], PacketSymbols>>1)
				bits := BitSlice(filtered)

				// If the checksum fails, bail.
				if rcvr.bch.Encode(bits[16:]) != 0 {
					continue
				}

				// Parse SCM
				scm, err := ParseSCM(bits)
				if err != nil {
					config.Log.Fatal("Error parsing SCM:", err)
				}

				// If filtering by ID and ID doesn't match, bail.
				if config.MeterID != 0 && uint(scm.ID) != config.MeterID {
					continue
				}

				// Get current file offset.
				offset, err := config.SampleFile.Seek(0, os.SEEK_CUR)
				if err != nil {
					config.Log.Fatal("Error getting sample file offset:", err)
				}

				// Dump message to file.
				_, err = config.SampleFile.Write(raw)
				if err != nil {
					config.Log.Fatal("Error dumping samples:", err)
				}

				msg := Message{time.Now(), offset, len(raw), scm}

				// Write or encode message to log file.
				if config.Encoder == nil {
					// A nil encoder is just plain-text output.
					fmt.Fprintf(config.LogFile, "%+v", msg)
				} else {
					err = config.Encoder.Encode(msg)
					if err != nil {
						log.Fatal("Error encoding message:", err)
					}

					// The XML encoder doesn't write new lines after each
					// element, add them.
					if strings.ToLower(config.format) == "xml" {
						fmt.Fprintln(config.LogFile)
					}
				}
			}
		}
	}
}

// Shift sample from unsigned and normalize.
func Mag(i, q byte) float64 {
	j := (127.5 - float64(i)) / 127
	k := (127.5 - float64(q)) / 127
	return math.Hypot(j, k)
}

// Matched filter implemented as integrate and dump. Output array is equal to
// the number of manchester coded symbols per packet.
func MatchedFilter(input []float64, bits int) (output []float64) {
	output = make([]float64, bits)

	fIdx := 0
	for idx := 0.0; fIdx < bits; idx += SymbolLength * 2 {
		lower := IntRound(idx)
		upper := IntRound(idx + SymbolLength)

		for i := 0; i < upper-lower; i++ {
			output[fIdx] += input[lower+i] - input[upper+i]
		}
		fIdx++
	}
	return
}

func BitSlice(input []float64) (output string) {
	for _, v := range input {
		if v > 0.0 {
			output += "1"
		} else {
			output += "0"
		}
	}
	return
}

func ParseUint(raw string) uint64 {
	tmp, _ := strconv.ParseUint(raw, 2, 64)
	return tmp
}

// Message for logging.
type Message struct {
	Time   time.Time
	Offset int64
	Length int
	SCM    SCM
}

func (msg Message) String() string {
	// If we aren't dumping samples, omit offset and packet length.
	if config.sampleFilename == os.DevNull {
		return fmt.Sprintf("{Time:%s SCM:%+v}\n",
			msg.Time.Format(TimeFormat), msg.SCM,
		)
	}

	return fmt.Sprintf("{Time:%s Offset:%d Length:%d SCM:%+v}\n",
		msg.Time.Format(TimeFormat), msg.Offset, msg.Length, msg.SCM,
	)
}

// Standard Consumption Message
type SCM struct {
	ID          uint32
	Type        uint8
	Tamper      Tamper
	Consumption uint32
	Checksum    uint16
}

func (scm SCM) String() string {
	return fmt.Sprintf("{ID:%8d Type:%2d Tamper:%+v Consumption:%8d Checksum:0x%04X}",
		scm.ID, scm.Type, scm.Tamper, scm.Consumption, scm.Checksum,
	)
}

type Tamper struct {
	Phy uint8
	Enc uint8
}

func (t Tamper) String() string {
	return fmt.Sprintf("{Phy:%d Enc:%d}", t.Phy, t.Enc)
}

// Given a string of bits, parse the message.
func ParseSCM(data string) (scm SCM, err error) {
	if len(data) != 96 {
		return scm, errors.New("invalid input length")
	}

	scm.ID = uint32(ParseUint(data[21:23] + data[56:80]))
	scm.Type = uint8(ParseUint(data[26:30]))
	scm.Tamper.Phy = uint8(ParseUint(data[24:26]))
	scm.Tamper.Enc = uint8(ParseUint(data[30:32]))
	scm.Consumption = uint32(ParseUint(data[32:56]))
	scm.Checksum = uint16(ParseUint(data[80:96]))

	return scm, nil
}

func IntRound(i float64) int {
	return int(math.Floor(i + 0.5))
}

func init() {
	err := config.Parse()
	if err != nil {
		config.Log.Fatal("Error parsing flags:", err)
	}
}

func main() {
	if !config.Quiet {
		log.Println("Server:", config.ServerAddr)
		log.Println("BlockSize:", BlockSize)
		log.Println("SampleRate:", SampleRate)
		log.Println("DataRate:", DataRate)
		log.Println("SymbolLength:", SymbolLength)
		log.Println("PacketSymbols:", PacketSymbols)
		log.Println("PacketLength:", PacketLength)
		log.Println("CenterFreq:", config.CenterFreq)
		log.Println("TimeLimit:", config.TimeLimit)

		log.Println("Format:", config.format)
		log.Println("LogFile:", config.logFilename)
		log.Println("SampleFile:", config.sampleFilename)

		if config.MeterID != 0 {
			log.Println("FilterID:", config.MeterID)
		}
	}

	rcvr := NewReceiver(BlockSize)
	defer rcvr.Close()
	defer config.Close()

	if !config.Quiet {
		config.Log.Println("Running...")
	}

	rcvr.Run()
}
<commit_msg>Limited signals to exit on to Kill and Interrupt.<commit_after>// RTLAMR - An rtl-sdr receiver for smart meters operating in the 900MHz ISM band.
// Copyright (C) 2014 Douglas Hall
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published
// by the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package main

import (
	"encoding/gob"
	"encoding/json"
	"encoding/xml"
	"errors"
	"flag"
	"fmt"
	"log"
	"math"
	"net"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"time"

	"github.com/bemasher/rtltcp"

	"github.com/bemasher/rtlamr/bch"
	"github.com/bemasher/rtlamr/preamble"
)

const (
	BlockSize = 1 << 12

	SampleRate   = 2.4e6
	DataRate     = 32.768e3
	SymbolLength = SampleRate / DataRate

	PreambleSymbols = 42
	PreambleLength  = PreambleSymbols * SymbolLength

	PacketSymbols = 192
	PacketLength  = PacketSymbols * SymbolLength

	PreambleDFTSize = 8192

	CenterFreq    = 920299072
	RestrictLocal = false

	Preamble     = 0x1F2A60
	PreambleBits = "111110010101001100000"

	GenPoly = 0x16F63

	TimeFormat = "2006-01-02T15:04:05.000"
)

var SymLen = IntRound(SymbolLength)
var config Config

type Config struct {
	serverAddr     string
	logFilename    string
	sampleFilename string
	format         string

	ServerAddr *net.TCPAddr
	CenterFreq uint
	TimeLimit  time.Duration
	MeterID    uint

	Log     *log.Logger
	LogFile *os.File

	GobUnsafe bool
	Encoder   Encoder

	SampleFile *os.File

	Quiet bool
}

func (c *Config) Parse() (err error) {
	flag.StringVar(&c.serverAddr, "server", "127.0.0.1:1234", "address or hostname of rtl_tcp instance")
	flag.StringVar(&c.logFilename, "logfile", "/dev/stdout", "log statement dump file")
	flag.StringVar(&c.sampleFilename, "samplefile", os.DevNull, "received message signal dump file, offset and message length are displayed to log when enabled")
	flag.UintVar(&c.CenterFreq, "centerfreq", 920299072, "center frequency to receive on")
	flag.DurationVar(&c.TimeLimit, "duration", 0, "time to run for, 0 for infinite")
	flag.UintVar(&c.MeterID, "filterid", 0, "display only messages matching given id")
	flag.StringVar(&c.format, "format", "plain", "format to write log messages in: plain, json, xml or gob")
	flag.BoolVar(&c.GobUnsafe, "gobunsafe", false, "allow gob output to stdout")
	flag.BoolVar(&c.Quiet, "quiet", false, "suppress state information printed at startup")

	flag.Parse()

	// Parse and resolve rtl_tcp server address.
	c.ServerAddr, err = net.ResolveTCPAddr("tcp", c.serverAddr)
	if err != nil {
		return
	}

	// Open or create the log file.
	if c.logFilename == "/dev/stdout" {
		c.LogFile = os.Stdout
	} else {
		c.LogFile, err = os.Create(c.logFilename)
	}

	// Create a new logger with the log file as output.
	c.Log = log.New(c.LogFile, "", log.Lshortfile)
	if err != nil {
		return
	}

	// Create the sample file.
	c.SampleFile, err = os.Create(c.sampleFilename)
	if err != nil {
		return
	}

	// Create encoder for specified logging format.
	switch strings.ToLower(c.format) {
	case "plain":
		break
	case "json":
		c.Encoder = json.NewEncoder(c.LogFile)
	case "xml":
		c.Encoder = xml.NewEncoder(c.LogFile)
	case "gob":
		c.Encoder = gob.NewEncoder(c.LogFile)

		// Don't let the user output gob to stdout unless they really want to.
		if !c.GobUnsafe && c.logFilename == "/dev/stdout" {
			fmt.Println("Gob encoded messages are not stdout safe, specify logfile or use gobunsafe flag.")
			os.Exit(1)
		}
	default:
		// We didn't get a valid encoder, exit and say so.
		log.Fatal("Invalid log format:", c.format)
	}

	return
}

func (c Config) Close() {
	c.LogFile.Close()
	c.SampleFile.Close()
}

// JSON, XML and GOB all implement this interface so we can simplify log
// output formatting.
type Encoder interface {
	Encode(interface{}) error
}

type Receiver struct {
	rtltcp.SDR

	pd  preamble.PreambleDetector
	bch bch.BCH
}

func NewReceiver(blockSize int) (rcvr Receiver) {
	// Plan the preamble detector.
	rcvr.pd = preamble.NewPreambleDetector(PreambleDFTSize, SymbolLength, PreambleBits)

	// Create a new BCH for error detection.
	rcvr.bch = bch.NewBCH(GenPoly)
	if !config.Quiet {
		config.Log.Printf("BCH: %+v\n", rcvr.bch)
	}

	// Connect to rtl_tcp server.
	if err := rcvr.Connect(config.ServerAddr); err != nil {
		config.Log.Fatal(err)
	}

	// Tell the user how many gain settings were reported by rtl_tcp.
	if !config.Quiet {
		config.Log.Println("GainCount:", rcvr.SDR.Info.GainCount)
	}

	// Set some parameters for listening.
	rcvr.SetSampleRate(SampleRate)
	rcvr.SetCenterFreq(uint32(config.CenterFreq))
	rcvr.SetGainMode(true)

	return
}

// Clean up rtl_tcp connection and destroy preamble detection plan.
func (rcvr *Receiver) Close() {
	rcvr.SDR.Close()
	rcvr.pd.Close()
}

func (rcvr *Receiver) Run() {
	// Setup signal channel for interruption.
	sigint := make(chan os.Signal, 1)
	signal.Notify(sigint, os.Kill, os.Interrupt)

	// Allocate sample and demodulated signal buffers.
	raw := make([]byte, IntRound(PacketLength+BlockSize)<<1)
	amBuf := make([]float64, IntRound(PacketLength+BlockSize))

	// Setup time limit channel
	tLimit := make(<-chan time.Time, 1)
	if config.TimeLimit != 0 {
		tLimit = time.After(config.TimeLimit)
	}

	start := time.Now()
	for {
		// Exit on interrupt or time limit, otherwise receive.
		select {
		case <-sigint:
			return
		case <-tLimit:
			fmt.Println("Time Limit Reached:", time.Since(start))
			return
		default:
			copy(raw, raw[BlockSize<<1:])
			copy(amBuf, amBuf[BlockSize:])

			// Read new sample block.
			_, err := rcvr.Read(raw[IntRound(PacketLength)<<1:])
			if err != nil {
				config.Log.Fatal("Error reading samples:", err)
			}

			// AM Demodulate
			lower := IntRound(PacketLength)
			for i := 0; i < BlockSize; i++ {
				amBuf[lower+i] = Mag(raw[(lower+i)<<1], raw[((lower+i)<<1)+1])
			}

			// Detect preamble in first half of demod buffer.
			rcvr.pd.Execute(amBuf)
			align := rcvr.pd.ArgMax()

			// Bad framing, catch message on next block.
			if align > BlockSize {
				continue
			}

			// Filter signal and bit slice enough data to catch the preamble.
			filtered := MatchedFilter(amBuf[align:], PreambleSymbols>>1)
			bits := BitSlice(filtered)

			// If the preamble matches.
			if bits == PreambleBits {
				for i := BlockSize << 1; i < len(amBuf); i++ {
					amBuf[i] = Mag(raw[i<<1], raw[(i<<1)+1])
				}

				// Filter, slice and parse the rest of the buffered samples.
				filtered := MatchedFilter(amBuf[align:], PacketSymbols>>1)
				bits := BitSlice(filtered)

				// If the checksum fails, bail.
				if rcvr.bch.Encode(bits[16:]) != 0 {
					continue
				}

				// Parse SCM
				scm, err := ParseSCM(bits)
				if err != nil {
					config.Log.Fatal("Error parsing SCM:", err)
				}

				// If filtering by ID and ID doesn't match, bail.
				if config.MeterID != 0 && uint(scm.ID) != config.MeterID {
					continue
				}

				// Get current file offset.
				offset, err := config.SampleFile.Seek(0, os.SEEK_CUR)
				if err != nil {
					config.Log.Fatal("Error getting sample file offset:", err)
				}

				// Dump message to file.
				_, err = config.SampleFile.Write(raw)
				if err != nil {
					config.Log.Fatal("Error dumping samples:", err)
				}

				msg := Message{time.Now(), offset, len(raw), scm}

				// Write or encode message to log file.
				if config.Encoder == nil {
					// A nil encoder is just plain-text output.
					fmt.Fprintf(config.LogFile, "%+v", msg)
				} else {
					err = config.Encoder.Encode(msg)
					if err != nil {
						log.Fatal("Error encoding message:", err)
					}

					// The XML encoder doesn't write new lines after each
					// element, add them.
					if strings.ToLower(config.format) == "xml" {
						fmt.Fprintln(config.LogFile)
					}
				}
			}
		}
	}
}

// Shift sample from unsigned and normalize.
func Mag(i, q byte) float64 {
	j := (127.5 - float64(i)) / 127
	k := (127.5 - float64(q)) / 127
	return math.Hypot(j, k)
}

// Matched filter implemented as integrate and dump. Output array is equal to
// the number of manchester coded symbols per packet.
func MatchedFilter(input []float64, bits int) (output []float64) {
	output = make([]float64, bits)

	fIdx := 0
	for idx := 0.0; fIdx < bits; idx += SymbolLength * 2 {
		lower := IntRound(idx)
		upper := IntRound(idx + SymbolLength)

		for i := 0; i < upper-lower; i++ {
			output[fIdx] += input[lower+i] - input[upper+i]
		}
		fIdx++
	}
	return
}

func BitSlice(input []float64) (output string) {
	for _, v := range input {
		if v > 0.0 {
			output += "1"
		} else {
			output += "0"
		}
	}
	return
}

func ParseUint(raw string) uint64 {
	tmp, _ := strconv.ParseUint(raw, 2, 64)
	return tmp
}

// Message for logging.
type Message struct {
	Time   time.Time
	Offset int64
	Length int
	SCM    SCM
}

func (msg Message) String() string {
	// If we aren't dumping samples, omit offset and packet length.
	if config.sampleFilename == os.DevNull {
		return fmt.Sprintf("{Time:%s SCM:%+v}\n",
			msg.Time.Format(TimeFormat), msg.SCM,
		)
	}

	return fmt.Sprintf("{Time:%s Offset:%d Length:%d SCM:%+v}\n",
		msg.Time.Format(TimeFormat), msg.Offset, msg.Length, msg.SCM,
	)
}

// Standard Consumption Message
type SCM struct {
	ID          uint32
	Type        uint8
	Tamper      Tamper
	Consumption uint32
	Checksum    uint16
}

func (scm SCM) String() string {
	return fmt.Sprintf("{ID:%8d Type:%2d Tamper:%+v Consumption:%8d Checksum:0x%04X}",
		scm.ID, scm.Type, scm.Tamper, scm.Consumption, scm.Checksum,
	)
}

type Tamper struct {
	Phy uint8
	Enc uint8
}

func (t Tamper) String() string {
	return fmt.Sprintf("{Phy:%d Enc:%d}", t.Phy, t.Enc)
}

// Given a string of bits, parse the message.
func ParseSCM(data string) (scm SCM, err error) {
	if len(data) != 96 {
		return scm, errors.New("invalid input length")
	}

	scm.ID = uint32(ParseUint(data[21:23] + data[56:80]))
	scm.Type = uint8(ParseUint(data[26:30]))
	scm.Tamper.Phy = uint8(ParseUint(data[24:26]))
	scm.Tamper.Enc = uint8(ParseUint(data[30:32]))
	scm.Consumption = uint32(ParseUint(data[32:56]))
	scm.Checksum = uint16(ParseUint(data[80:96]))

	return scm, nil
}

func IntRound(i float64) int {
	return int(math.Floor(i + 0.5))
}

func init() {
	err := config.Parse()
	if err != nil {
		config.Log.Fatal("Error parsing flags:", err)
	}
}

func main() {
	if !config.Quiet {
		log.Println("Server:", config.ServerAddr)
		log.Println("BlockSize:", BlockSize)
		log.Println("SampleRate:", SampleRate)
		log.Println("DataRate:", DataRate)
		log.Println("SymbolLength:", SymbolLength)
		log.Println("PacketSymbols:", PacketSymbols)
		log.Println("PacketLength:", PacketLength)
		log.Println("CenterFreq:", config.CenterFreq)
		log.Println("TimeLimit:", config.TimeLimit)

		log.Println("Format:", config.format)
		log.Println("LogFile:", config.logFilename)
		log.Println("SampleFile:", config.sampleFilename)

		if config.MeterID != 0 {
			log.Println("FilterID:", config.MeterID)
		}
	}

	rcvr := NewReceiver(BlockSize)
	defer rcvr.Close()
	defer config.Close()

	if !config.Quiet {
		config.Log.Println("Running...")
	}

	rcvr.Run()
}
<|endoftext|>