<commit_before>package putio

import "fmt"

// File represents a Put.io file.
type File struct {
	ID                int64  `json:"id"`
	Name              string `json:"name"`
	Size              int64  `json:"size"`
	ContentType       string `json:"content_type"`
	CreatedAt         *Time  `json:"created_at"`
	FirstAccessedAt   *Time  `json:"first_accessed_at"`
	ParentID          int64  `json:"parent_id"`
	Screenshot        string `json:"screenshot"`
	OpensubtitlesHash string `json:"opensubtitles_hash"`
	IsMP4Available    bool   `json:"is_mp4_available"`
	Icon              string `json:"icon"`
	CRC32             string `json:"crc32"`
	IsShared          bool   `json:"is_shared"`
}

func (f *File) String() string {
	return fmt.Sprintf("<ID: %v Name: %q Size: %v>", f.ID, f.Name, f.Size)
}

// IsDir reports whether the file is a directory.
func (f *File) IsDir() bool {
	return f.ContentType == "application/x-directory"
}

// Upload represents a Put.io upload. If the uploaded file is a torrent file,
// Transfer field will represent the status of the transfer.
type Upload struct {
	File     *File     `json:"file"`
	Transfer *Transfer `json:"transfer"`
}

// Search represents a search response.
type Search struct {
	Files []File `json:"files"`
	Next  string `json:"next"`
}

// Transfer represents a Put.io transfer state.
type Transfer struct {
	Availability   int    `json:"availability"`
	CallbackURL    string `json:"callback_url"`
	CreatedAt      *Time  `json:"created_at"`
	CreatedTorrent bool   `json:"created_torrent"`
	ClientIP       string `json:"client_ip"`

	// FIXME: API returns either string or float non-deterministically.
	// CurrentRatio       float32 `json:"current_ratio"`

	DownloadSpeed      int    `json:"down_speed"`
	Downloaded         int64  `json:"downloaded"`
	DownloadID         int64  `json:"download_id"`
	ErrorMessage       string `json:"error_message"`
	EstimatedTime      int64  `json:"estimated_time"`
	Extract            bool   `json:"extract"`
	FileID             int64  `json:"file_id"`
	FinishedAt         *Time  `json:"finished_at"`
	ID                 int64  `json:"id"`
	IsPrivate          bool   `json:"is_private"`
	MagnetURI          string `json:"magneturi"`
	Name               string `json:"name"`
	PeersConnected     int    `json:"peers_connected"`
	PeersGettingFromUs int    `json:"peers_getting_from_us"`
	PeersSendingToUs   int    `json:"peers_sending_to_us"`
	PercentDone        int    `json:"percent_done"`
	SaveParentID       int64  `json:"save_parent_id"`
	SecondsSeeding     int    `json:"seconds_seeding"`
	Size               int    `json:"size"`
	Source             string `json:"source"`
	Status             string `json:"status"`
	StatusMessage      string `json:"status_message"`
	SubscriptionID     int    `json:"subscription_id"`
	TorrentLink        string `json:"torrent_link"`
	TrackerMessage     string `json:"tracker_message"`
	Trackers           string `json:"tracker"`
	Type               string `json:"type"`
	UploadSpeed        int    `json:"up_speed"`
	Uploaded           int64  `json:"uploaded"`
}

// AccountInfo represents user's account information.
type AccountInfo struct {
	AccountActive           bool   `json:"account_active"`
	AvatarURL               string `json:"avatar_url"`
	DaysUntilFilesDeletion  int    `json:"days_until_files_deletion"`
	DefaultSubtitleLanguage string `json:"default_subtitle_language"`
	Disk                    struct {
		Avail int64 `json:"avail"`
		Size  int64 `json:"size"`
		Used  int64 `json:"used"`
	} `json:"disk"`
	HasVoucher                int      `json:"has_voucher"`
	Mail                      string   `json:"mail"`
	PlanExpirationDate        string   `json:"plan_expiration_date"`
	Settings                  Settings `json:"settings"`
	SimultaneousDownloadLimit int      `json:"simultaneous_download_limit"`
	SubtitleLanguages         []string `json:"subtitle_languages"`
	UserID                    int64    `json:"user_id"`
	Username                  string   `json:"username"`
}

// Settings represents user's personal settings.
type Settings struct {
	CallbackURL             string      `json:"callback_url"`
	DefaultDownloadFolder   int64       `json:"default_download_folder"`
	DefaultSubtitleLanguage string      `json:"default_subtitle_language"`
	DownloadFolderUnset     bool        `json:"download_folder_unset"`
	IsInvisible             bool        `json:"is_invisible"`
	Nextepisode             bool        `json:"nextepisode"`
	PrivateDownloadHostIP   interface{} `json:"private_download_host_ip"`
	PushoverToken           string      `json:"pushover_token"`
	Routing                 string      `json:"routing"`
	Sorting                 string      `json:"sorting"`
	SSLEnabled              bool        `json:"ssl_enabled"`
	StartFrom               bool        `json:"start_from"`
	SubtitleLanguages       []string    `json:"subtitle_languages"`
}

// Friend represents Put.io user's friend.
type Friend struct {
	ID        int64  `json:"id"`
	Name      string `json:"name"`
	AvatarURL string `json:"avatar_url"`
}

// Zip represents Put.io zip file.
type Zip struct {
	ID        int64 `json:"id"`
	CreatedAt *Time `json:"created_at"`

	Size   int64  `json:"size"`
	Status string `json:"status"`
	URL    string `json:"url"`

	// FIXME: missing_files field is missin
	missingFiles string
}

// Subtitle represents a subtitle.
type Subtitle struct {
	Key      string
	Language string
	Name     string
	Source   string
}

// Event represents a Put.io event. It could be a transfer or a shared file.
type Event struct {
	ID           int64  `json:"id"`
	FileID       int64  `json:"file_id"`
	Source       string `json:"source"`
	Type         string `json:"type"`
	TransferName string `json:"transfer_name"`
	TransferSize int64  `json:"transfer_size"`
	CreatedAt    *Time  `json:"created_at"`
}

type share struct {
	FileID   int64  `json:"file_id"`
	Filename string `json:"file_name"`
	// Number of friends the file is shared with
	SharedWith int64 `json:"shared_with"`
}

// errorResponse represents a common error message that Put.io v2 API sends on
// error.
type errorResponse struct {
	ErrorMessage string `json:"error_message"`
	ErrorType    string `json:"error_type"`
	ErrorURI     string `json:"error_uri"`
	Status       string `json:"status"`
	StatusCode   int    `json:"status_code"`
}

func (e errorResponse) Error() string {
	return fmt.Sprintf("StatusCode: %v ErrorType: %v ErrorMsg: %v", e.StatusCode, e.ErrorType, e.ErrorMessage)
}
<commit_msg>types: fix incorrect type for has_voucher field<commit_after>package putio

import "fmt"

// File represents a Put.io file.
type File struct {
	ID                int64  `json:"id"`
	Name              string `json:"name"`
	Size              int64  `json:"size"`
	ContentType       string `json:"content_type"`
	CreatedAt         *Time  `json:"created_at"`
	FirstAccessedAt   *Time  `json:"first_accessed_at"`
	ParentID          int64  `json:"parent_id"`
	Screenshot        string `json:"screenshot"`
	OpensubtitlesHash string `json:"opensubtitles_hash"`
	IsMP4Available    bool   `json:"is_mp4_available"`
	Icon              string `json:"icon"`
	CRC32             string `json:"crc32"`
	IsShared          bool   `json:"is_shared"`
}

func (f *File) String() string {
	return fmt.Sprintf("<ID: %v Name: %q Size: %v>", f.ID, f.Name, f.Size)
}

// IsDir reports whether the file is a directory.
func (f *File) IsDir() bool {
	return f.ContentType == "application/x-directory"
}

// Upload represents a Put.io upload. If the uploaded file is a torrent file,
// Transfer field will represent the status of the transfer.
type Upload struct {
	File     *File     `json:"file"`
	Transfer *Transfer `json:"transfer"`
}

// Search represents a search response.
type Search struct {
	Files []File `json:"files"`
	Next  string `json:"next"`
}

// Transfer represents a Put.io transfer state.
type Transfer struct {
	Availability   int    `json:"availability"`
	CallbackURL    string `json:"callback_url"`
	CreatedAt      *Time  `json:"created_at"`
	CreatedTorrent bool   `json:"created_torrent"`
	ClientIP       string `json:"client_ip"`

	// FIXME: API returns either string or float non-deterministically.
	// CurrentRatio       float32 `json:"current_ratio"`

	DownloadSpeed      int    `json:"down_speed"`
	Downloaded         int64  `json:"downloaded"`
	DownloadID         int64  `json:"download_id"`
	ErrorMessage       string `json:"error_message"`
	EstimatedTime      int64  `json:"estimated_time"`
	Extract            bool   `json:"extract"`
	FileID             int64  `json:"file_id"`
	FinishedAt         *Time  `json:"finished_at"`
	ID                 int64  `json:"id"`
	IsPrivate          bool   `json:"is_private"`
	MagnetURI          string `json:"magneturi"`
	Name               string `json:"name"`
	PeersConnected     int    `json:"peers_connected"`
	PeersGettingFromUs int    `json:"peers_getting_from_us"`
	PeersSendingToUs   int    `json:"peers_sending_to_us"`
	PercentDone        int    `json:"percent_done"`
	SaveParentID       int64  `json:"save_parent_id"`
	SecondsSeeding     int    `json:"seconds_seeding"`
	Size               int    `json:"size"`
	Source             string `json:"source"`
	Status             string `json:"status"`
	StatusMessage      string `json:"status_message"`
	SubscriptionID     int    `json:"subscription_id"`
	TorrentLink        string `json:"torrent_link"`
	TrackerMessage     string `json:"tracker_message"`
	Trackers           string `json:"tracker"`
	Type               string `json:"type"`
	UploadSpeed        int    `json:"up_speed"`
	Uploaded           int64  `json:"uploaded"`
}

// AccountInfo represents user's account information.
type AccountInfo struct {
	AccountActive           bool   `json:"account_active"`
	AvatarURL               string `json:"avatar_url"`
	DaysUntilFilesDeletion  int    `json:"days_until_files_deletion"`
	DefaultSubtitleLanguage string `json:"default_subtitle_language"`
	Disk                    struct {
		Avail int64 `json:"avail"`
		Size  int64 `json:"size"`
		Used  int64 `json:"used"`
	} `json:"disk"`
	HasVoucher                bool     `json:"has_voucher"`
	Mail                      string   `json:"mail"`
	PlanExpirationDate        string   `json:"plan_expiration_date"`
	Settings                  Settings `json:"settings"`
	SimultaneousDownloadLimit int      `json:"simultaneous_download_limit"`
	SubtitleLanguages         []string `json:"subtitle_languages"`
	UserID                    int64    `json:"user_id"`
	Username                  string   `json:"username"`
}

// Settings represents user's personal settings.
type Settings struct {
	CallbackURL             string      `json:"callback_url"`
	DefaultDownloadFolder   int64       `json:"default_download_folder"`
	DefaultSubtitleLanguage string      `json:"default_subtitle_language"`
	DownloadFolderUnset     bool        `json:"download_folder_unset"`
	IsInvisible             bool        `json:"is_invisible"`
	Nextepisode             bool        `json:"nextepisode"`
	PrivateDownloadHostIP   interface{} `json:"private_download_host_ip"`
	PushoverToken           string      `json:"pushover_token"`
	Routing                 string      `json:"routing"`
	Sorting                 string      `json:"sorting"`
	SSLEnabled              bool        `json:"ssl_enabled"`
	StartFrom               bool        `json:"start_from"`
	SubtitleLanguages       []string    `json:"subtitle_languages"`
}

// Friend represents Put.io user's friend.
type Friend struct {
	ID        int64  `json:"id"`
	Name      string `json:"name"`
	AvatarURL string `json:"avatar_url"`
}

// Zip represents Put.io zip file.
type Zip struct {
	ID        int64 `json:"id"`
	CreatedAt *Time `json:"created_at"`

	Size   int64  `json:"size"`
	Status string `json:"status"`
	URL    string `json:"url"`

	// FIXME: missing_files field is missin
	missingFiles string
}

// Subtitle represents a subtitle.
type Subtitle struct {
	Key      string
	Language string
	Name     string
	Source   string
}

// Event represents a Put.io event. It could be a transfer or a shared file.
type Event struct {
	ID           int64  `json:"id"`
	FileID       int64  `json:"file_id"`
	Source       string `json:"source"`
	Type         string `json:"type"`
	TransferName string `json:"transfer_name"`
	TransferSize int64  `json:"transfer_size"`
	CreatedAt    *Time  `json:"created_at"`
}

type share struct {
	FileID   int64  `json:"file_id"`
	Filename string `json:"file_name"`
	// Number of friends the file is shared with
	SharedWith int64 `json:"shared_with"`
}

// errorResponse represents a common error message that Put.io v2 API sends on
// error.
type errorResponse struct {
	ErrorMessage string `json:"error_message"`
	ErrorType    string `json:"error_type"`
	ErrorURI     string `json:"error_uri"`
	Status       string `json:"status"`
	StatusCode   int    `json:"status_code"`
}

func (e errorResponse) Error() string {
	return fmt.Sprintf("StatusCode: %v ErrorType: %v ErrorMsg: %v", e.StatusCode, e.ErrorType, e.ErrorMessage)
}
<|endoftext|>


<commit_before>package main

import (
	"time"

	"github.com/davecgh/go-spew/spew"
	"github.com/ninjasphere/go-zigbee/nwkmgr"
)

type OnOffSwitchCluster struct {
	Channel
	SendEvent func(event string, payload interface{}) error
}

func (c *OnOffSwitchCluster) SetEventHandler(handler func(event string, payload interface{}) error) {
	c.SendEvent = handler
}

func (c *OnOffSwitchCluster) GetProtocol() string {
	return "button-momentary"
}

func (c *OnOffSwitchCluster) init() error {
	log.Debugf("Initialising on/off button cluster of device %d", *c.device.deviceInfo.IeeeAddress)

	clusterID := uint32(0x06)

	dstEndpoint := uint32(5)

	bindReq := &nwkmgr.NwkSetBindingEntryReq{
		SrcAddr: &nwkmgr.NwkAddressStructT{
			AddressType: nwkmgr.NwkAddressTypeT_UNICAST.Enum(),
			IeeeAddr:    c.device.deviceInfo.IeeeAddress,
			EndpointId:  c.endpoint.EndpointId,
		},
		ClusterId: &clusterID,
		DstAddr: &nwkmgr.NwkAddressStructT{
			AddressType: nwkmgr.NwkAddressTypeT_UNICAST.Enum(),
			IeeeAddr:    c.device.driver.localDevice.IeeeAddress,
			EndpointId:  &dstEndpoint,
		},
		BindingMode: nwkmgr.NwkBindingModeT_BIND.Enum(),
	}

	log.Infof("Binding on-off cluster %v", bindReq)

	bindRes := &nwkmgr.NwkSetBindingEntryRspInd{}

	err := c.device.driver.nwkmgrConn.SendAsyncCommand(bindReq, bindRes, time.Second*10)
	if err != nil {
		log.Errorf("Error binding on/off cluster: %s", err)
	} else if bindRes.Status.String() != "STATUS_SUCCESS" {
		log.Errorf("Failed to bind on/off cluster. status: %s", bindRes.Status.String())
	}

	update := c.device.driver.gatewayConn.OnBoundCluster(*c.device.deviceInfo.IeeeAddress, *c.endpoint.EndpointId, clusterID)

	go func() {
		for {
			state := <-update

			spew.Dump("Incoming on/off state:", state)

			c.SendEvent("state", true)
		}
	}()

	err = c.device.driver.Conn.ExportChannel(c.device, c, c.ID)
	if err != nil {
		log.Fatalf("Failed to announce on/off switch channel: %s", err)
	}

	return nil

}
<commit_msg>Use "pressed" instead of "state" for outgoing event in OnOffSwitchCluster<commit_after>package main

import (
	"time"

	"github.com/davecgh/go-spew/spew"
	"github.com/ninjasphere/go-zigbee/nwkmgr"
)

type OnOffSwitchCluster struct {
	Channel
	SendEvent func(event string, payload interface{}) error
}

func (c *OnOffSwitchCluster) SetEventHandler(handler func(event string, payload interface{}) error) {
	c.SendEvent = handler
}

func (c *OnOffSwitchCluster) GetProtocol() string {
	return "button-momentary"
}

func (c *OnOffSwitchCluster) init() error {
	log.Debugf("Initialising on/off button cluster of device %d", *c.device.deviceInfo.IeeeAddress)

	clusterID := uint32(0x06)

	dstEndpoint := uint32(5)

	bindReq := &nwkmgr.NwkSetBindingEntryReq{
		SrcAddr: &nwkmgr.NwkAddressStructT{
			AddressType: nwkmgr.NwkAddressTypeT_UNICAST.Enum(),
			IeeeAddr:    c.device.deviceInfo.IeeeAddress,
			EndpointId:  c.endpoint.EndpointId,
		},
		ClusterId: &clusterID,
		DstAddr: &nwkmgr.NwkAddressStructT{
			AddressType: nwkmgr.NwkAddressTypeT_UNICAST.Enum(),
			IeeeAddr:    c.device.driver.localDevice.IeeeAddress,
			EndpointId:  &dstEndpoint,
		},
		BindingMode: nwkmgr.NwkBindingModeT_BIND.Enum(),
	}

	log.Infof("Binding on-off cluster %v", bindReq)

	bindRes := &nwkmgr.NwkSetBindingEntryRspInd{}

	err := c.device.driver.nwkmgrConn.SendAsyncCommand(bindReq, bindRes, time.Second*10)
	if err != nil {
		log.Errorf("Error binding on/off cluster: %s", err)
	} else if bindRes.Status.String() != "STATUS_SUCCESS" {
		log.Errorf("Failed to bind on/off cluster. status: %s", bindRes.Status.String())
	}

	update := c.device.driver.gatewayConn.OnBoundCluster(*c.device.deviceInfo.IeeeAddress, *c.endpoint.EndpointId, clusterID)

	go func() {
		for {
			state := <-update

			spew.Dump("Incoming on/off state:", state)

			c.SendEvent("pressed", true)
		}
	}()

	err = c.device.driver.Conn.ExportChannel(c.device, c, c.ID)
	if err != nil {
		log.Fatalf("Failed to announce on/off switch channel: %s", err)
	}

	return nil

}
<|endoftext|>


<commit_before>// Copyright 2018 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package scanner

import (
	"context"
	"errors"
	"sync"
	"time"

	"github.com/golang/glog"
	ct "github.com/google/certificate-transparency-go"
	"github.com/google/certificate-transparency-go/client"
	"github.com/google/trillian/client/backoff"
)

// FetcherOptions holds configuration options for the Fetcher.
type FetcherOptions struct {
	// Number of entries to request in one batch from the Log.
	BatchSize int

	// Number of concurrent fetcher workers to run.
	ParallelFetch int

	// [StartIndex, EndIndex) is a log entry range to fetch. If EndIndex == 0,
	// then it gets reassigned to sth.TreeSize.
	StartIndex int64
	EndIndex   int64

	// Continuous determines whether Fetcher should run indefinitely after
	// reaching EndIndex.
	Continuous bool
}

// DefaultFetcherOptions returns new FetcherOptions with sensible defaults.
func DefaultFetcherOptions() *FetcherOptions {
	return &FetcherOptions{
		BatchSize:     1000,
		ParallelFetch: 1,
		StartIndex:    0,
		EndIndex:      0,
		Continuous:    false,
	}
}

// Fetcher is a tool that fetches entries from a CT Log.
type Fetcher struct {
	// Client used to talk to the CT log instance.
	client *client.LogClient
	// Configuration options for this Fetcher instance.
	opts *FetcherOptions

	// Current STH of the Log this Fetcher sends queries to.
	sth *ct.SignedTreeHead
	// The STH retrieval backoff state. Used only in Continuous fetch mode.
	sthBackoff *backoff.Backoff
}

// EntryBatch represents a contiguous range of entries of the Log.
type EntryBatch struct {
	Start   int64          // LeafIndex of the first entry in the range.
	Entries []ct.LeafEntry // Entries of the range.
}

// fetchRange represents a range of certs to fetch from a CT log.
type fetchRange struct {
	start int64 // inclusive
	end   int64 // inclusive
}

// NewFetcher creates a Fetcher instance using client to talk to the log,
// taking configuration options from opts.
func NewFetcher(client *client.LogClient, opts *FetcherOptions) *Fetcher {
	return &Fetcher{client: client, opts: opts}
}

// Prepare caches the latest Log's STH if not present and returns it. It also
// adjusts the entry range to fit the size of the tree.
func (f *Fetcher) Prepare(ctx context.Context) (*ct.SignedTreeHead, error) {
	if f.sth != nil {
		return f.sth, nil
	}

	sth, err := f.client.GetSTH(ctx)
	if err != nil {
		glog.Errorf("GetSTH() failed: %v", err)
		return nil, err
	}
	glog.Infof("Got STH with %d certs", sth.TreeSize)

	if size := int64(sth.TreeSize); f.opts.EndIndex == 0 || f.opts.EndIndex > size {
		glog.Warningf("Reset EndIndex from %d to %d", f.opts.EndIndex, size)
		f.opts.EndIndex = size
	}
	f.sth = sth
	return sth, nil
}

// Run performs fetching of the Log. Blocks until scanning is complete or
// context is cancelled. For each successfully fetched batch, runs the fn
// callback.
func (f *Fetcher) Run(ctx context.Context, fn func(EntryBatch)) error {
	glog.V(1).Info("Starting up Fetcher...")
	if _, err := f.Prepare(ctx); err != nil {
		return err
	}

	ranges := f.genRanges(ctx)

	// Run fetcher workers.
	var wg sync.WaitGroup
	for w, cnt := 0, f.opts.ParallelFetch; w < cnt; w++ {
		wg.Add(1)
		go func(idx int) {
			defer wg.Done()
			glog.V(1).Infof("Starting up Fetcher worker %d...", idx)
			f.runWorker(ctx, ranges, fn)
			glog.V(1).Infof("Fetcher worker %d finished", idx)
		}(w)
	}
	wg.Wait()

	glog.V(1).Info("Fetcher terminated")
	return nil
}

// genRanges returns a channel of ranges to fetch, and starts a goroutine that
// sends things down this channel. The goroutine terminates when all ranges
// have been generated, or if context is cancelled.
func (f *Fetcher) genRanges(ctx context.Context) <-chan fetchRange {
	batch := int64(f.opts.BatchSize)
	ranges := make(chan fetchRange)

	go func() {
		defer close(ranges)
		start, end := f.opts.StartIndex, f.opts.EndIndex

		for start < end {
			batchEnd := start + min(end-start, batch)
			next := fetchRange{start, batchEnd - 1}
			select {
			case <-ctx.Done():
				glog.Warningf("Cancelling genRanges: %v", ctx.Err())
				return
			case ranges <- next:
			}
			start = batchEnd

			if start == end && f.opts.Continuous {
				if err := f.updateSTH(ctx); err != nil {
					glog.Warningf("STH update cancelled: %v", err)
					return
				}
				end = f.opts.EndIndex
			}
		}
	}()

	return ranges
}

// updateSTH waits until a bigger STH is discovered, and updates the Fetcher
// accordingly. It is optimized for both bulk-load (new STH is way bigger then
// the last one) and keep-up (STH grows slowly) modes of operation. Waits for
// some time until the STH grows enough to request a full batch, but falls back
// to *any* STH bigger than the old one if it takes too long.
// Returns error only if the context is cancelled.
func (f *Fetcher) updateSTH(ctx context.Context) error {
	// TODO(pavelkalinnikov): Make these parameters tunable.
	const quickDur = 45 * time.Second
	if f.sthBackoff == nil {
		f.sthBackoff = &backoff.Backoff{
			Min:    1 * time.Second,
			Max:    30 * time.Second,
			Factor: 2,
			Jitter: true,
		}
	}

	lastSize := uint64(f.opts.EndIndex)
	targetSize := lastSize + uint64(f.opts.BatchSize)
	quickDeadline := time.Now().Add(quickDur)

	return f.sthBackoff.Retry(ctx, func() error {
		sth, err := f.client.GetSTH(ctx)
		if err != nil {
			return err
		}
		glog.V(2).Infof("Got STH with %d certs", sth.TreeSize)

		quick := time.Now().Before(quickDeadline)
		if sth.TreeSize <= lastSize || quick && sth.TreeSize < targetSize {
			return errors.New("waiting for bigger STH")
		}

		if quick {
			f.sthBackoff.Reset() // Growth is presumably fast, set next pause to Min.
		}
		f.sth = sth
		f.opts.EndIndex = int64(sth.TreeSize)
		return nil
	})
}

// runWorker is a worker function for handling fetcher ranges.
// Accepts cert ranges to fetch over the ranges channel, and if the fetch is
// successful sends the corresponding EntryBatch through the fn callback. Will
// retry failed attempts to retrieve ranges until the context is cancelled.
func (f *Fetcher) runWorker(ctx context.Context, ranges <-chan fetchRange, fn func(EntryBatch)) {
	for r := range ranges {
		// Logs MAY return fewer than the number of leaves requested. Only complete
		// if we actually got all the leaves we were expecting.
		for r.start <= r.end {
			// Fetcher.Run() can be cancelled while we are looping over this job.
			if err := ctx.Err(); err != nil {
				glog.Warningf("Worker context closed: %v", err)
				return
			}
			resp, err := f.client.GetRawEntries(ctx, r.start, r.end)
			if err != nil {
				glog.Errorf("GetRawEntries() failed: %v", err)
				// TODO(pavelkalinnikov): Introduce backoff policy and pause here.
				continue
			}
			fn(EntryBatch{Start: r.start, Entries: resp.Entries})
			r.start += int64(len(resp.Entries))
		}
	}
}

func min(a, b int64) int64 {
	if a < b {
		return a
	}
	return b
}
<commit_msg>[Fetcher] Add graceful shutdown (#330)<commit_after>// Copyright 2018 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package scanner

import (
	"context"
	"errors"
	"sync"
	"time"

	"github.com/golang/glog"
	ct "github.com/google/certificate-transparency-go"
	"github.com/google/certificate-transparency-go/client"
	"github.com/google/trillian/client/backoff"
)

// FetcherOptions holds configuration options for the Fetcher.
type FetcherOptions struct {
	// Number of entries to request in one batch from the Log.
	BatchSize int

	// Number of concurrent fetcher workers to run.
	ParallelFetch int

	// [StartIndex, EndIndex) is a log entry range to fetch. If EndIndex == 0,
	// then it gets reassigned to sth.TreeSize.
	StartIndex int64
	EndIndex   int64

	// Continuous determines whether Fetcher should run indefinitely after
	// reaching EndIndex.
	Continuous bool
}

// DefaultFetcherOptions returns new FetcherOptions with sensible defaults.
func DefaultFetcherOptions() *FetcherOptions {
	return &FetcherOptions{
		BatchSize:     1000,
		ParallelFetch: 1,
		StartIndex:    0,
		EndIndex:      0,
		Continuous:    false,
	}
}

// Fetcher is a tool that fetches entries from a CT Log.
type Fetcher struct {
	// Client used to talk to the CT log instance.
	client *client.LogClient
	// Configuration options for this Fetcher instance.
	opts *FetcherOptions

	// Current STH of the Log this Fetcher sends queries to.
	sth *ct.SignedTreeHead
	// The STH retrieval backoff state. Used only in Continuous fetch mode.
	sthBackoff *backoff.Backoff

	// Stops range generator, which causes the Fetcher to terminate gracefully.
	mu     sync.Mutex
	cancel context.CancelFunc
}

// EntryBatch represents a contiguous range of entries of the Log.
type EntryBatch struct {
	Start   int64          // LeafIndex of the first entry in the range.
	Entries []ct.LeafEntry // Entries of the range.
}

// fetchRange represents a range of certs to fetch from a CT log.
type fetchRange struct {
	start int64 // inclusive
	end   int64 // inclusive
}

// NewFetcher creates a Fetcher instance using client to talk to the log,
// taking configuration options from opts.
func NewFetcher(client *client.LogClient, opts *FetcherOptions) *Fetcher {
	cancel := func() {} // Protect against calling Stop before Run.
	return &Fetcher{client: client, opts: opts, cancel: cancel}
}

// Prepare caches the latest Log's STH if not present and returns it. It also
// adjusts the entry range to fit the size of the tree.
func (f *Fetcher) Prepare(ctx context.Context) (*ct.SignedTreeHead, error) {
	if f.sth != nil {
		return f.sth, nil
	}

	sth, err := f.client.GetSTH(ctx)
	if err != nil {
		glog.Errorf("GetSTH() failed: %v", err)
		return nil, err
	}
	glog.Infof("Got STH with %d certs", sth.TreeSize)

	if size := int64(sth.TreeSize); f.opts.EndIndex == 0 || f.opts.EndIndex > size {
		glog.Warningf("Reset EndIndex from %d to %d", f.opts.EndIndex, size)
		f.opts.EndIndex = size
	}
	f.sth = sth
	return sth, nil
}

// Run performs fetching of the Log. Blocks until scanning is complete, the
// passed in context is canceled, or Stop is called (and pending work is
// finished). For each successfully fetched batch, runs the fn callback.
func (f *Fetcher) Run(ctx context.Context, fn func(EntryBatch)) error {
	glog.V(1).Info("Starting up Fetcher...")
	if _, err := f.Prepare(ctx); err != nil {
		return err
	}

	cctx, cancel := context.WithCancel(ctx)
	defer cancel()

	f.mu.Lock()
	f.cancel = cancel
	f.mu.Unlock()

	// Use a separately-cancelable context for the range generator, so we can
	// close it down (in Stop) but still let the fetchers below run to
	// completion.
	ranges := f.genRanges(cctx)

	// Run fetcher workers.
	var wg sync.WaitGroup
	for w, cnt := 0, f.opts.ParallelFetch; w < cnt; w++ {
		wg.Add(1)
		go func(idx int) {
			defer wg.Done()
			glog.V(1).Infof("Starting up Fetcher worker %d...", idx)
			f.runWorker(ctx, ranges, fn)
			glog.V(1).Infof("Fetcher worker %d finished", idx)
		}(w)
	}
	wg.Wait()

	glog.V(1).Info("Fetcher terminated")
	return nil
}

// Stop causes the Fetcher to terminate gracefully. After this call Run will
// try to finish all the started fetches, and then return. Does nothing if
// there was no preceding Run invocation.
func (f *Fetcher) Stop() {
	f.mu.Lock()
	defer f.mu.Unlock()
	f.cancel()
}

// genRanges returns a channel of ranges to fetch, and starts a goroutine that
// sends things down this channel. The goroutine terminates when all ranges
// have been generated, or if context is cancelled.
func (f *Fetcher) genRanges(ctx context.Context) <-chan fetchRange {
	batch := int64(f.opts.BatchSize)
	ranges := make(chan fetchRange)

	go func() {
		defer close(ranges)
		start, end := f.opts.StartIndex, f.opts.EndIndex

		for start < end {
			batchEnd := start + min(end-start, batch)
			next := fetchRange{start, batchEnd - 1}
			select {
			case <-ctx.Done():
				glog.Warningf("Cancelling genRanges: %v", ctx.Err())
				return
			case ranges <- next:
			}
			start = batchEnd

			if start == end && f.opts.Continuous {
				if err := f.updateSTH(ctx); err != nil {
					glog.Warningf("STH update cancelled: %v", err)
					return
				}
				end = f.opts.EndIndex
			}
		}
	}()

	return ranges
}

// updateSTH waits until a bigger STH is discovered, and updates the Fetcher
// accordingly. It is optimized for both bulk-load (new STH is way bigger then
// the last one) and keep-up (STH grows slowly) modes of operation. Waits for
// some time until the STH grows enough to request a full batch, but falls back
// to *any* STH bigger than the old one if it takes too long.
// Returns error only if the context is cancelled.
func (f *Fetcher) updateSTH(ctx context.Context) error {
	// TODO(pavelkalinnikov): Make these parameters tunable.
	const quickDur = 45 * time.Second
	if f.sthBackoff == nil {
		f.sthBackoff = &backoff.Backoff{
			Min:    1 * time.Second,
			Max:    30 * time.Second,
			Factor: 2,
			Jitter: true,
		}
	}

	lastSize := uint64(f.opts.EndIndex)
	targetSize := lastSize + uint64(f.opts.BatchSize)
	quickDeadline := time.Now().Add(quickDur)

	return f.sthBackoff.Retry(ctx, func() error {
		sth, err := f.client.GetSTH(ctx)
		if err != nil {
			return err
		}
		glog.V(2).Infof("Got STH with %d certs", sth.TreeSize)

		quick := time.Now().Before(quickDeadline)
		if sth.TreeSize <= lastSize || quick && sth.TreeSize < targetSize {
			return errors.New("waiting for bigger STH")
		}

		if quick {
			f.sthBackoff.Reset() // Growth is presumably fast, set next pause to Min.
		}
		f.sth = sth
		f.opts.EndIndex = int64(sth.TreeSize)
		return nil
	})
}

// runWorker is a worker function for handling fetcher ranges.
// Accepts cert ranges to fetch over the ranges channel, and if the fetch is
// successful sends the corresponding EntryBatch through the fn callback. Will
// retry failed attempts to retrieve ranges until the context is cancelled.
func (f *Fetcher) runWorker(ctx context.Context, ranges <-chan fetchRange, fn func(EntryBatch)) {
	for r := range ranges {
		// Logs MAY return fewer than the number of leaves requested. Only complete
		// if we actually got all the leaves we were expecting.
		for r.start <= r.end {
			// Fetcher.Run() can be cancelled while we are looping over this job.
			if err := ctx.Err(); err != nil {
				glog.Warningf("Worker context closed: %v", err)
				return
			}
			resp, err := f.client.GetRawEntries(ctx, r.start, r.end)
			if err != nil {
				glog.Errorf("GetRawEntries() failed: %v", err)
				// TODO(pavelkalinnikov): Introduce backoff policy and pause here.
				continue
			}
			fn(EntryBatch{Start: r.start, Entries: resp.Entries})
			r.start += int64(len(resp.Entries))
		}
	}
}

func min(a, b int64) int64 {
	if a < b {
		return a
	}
	return b
}
<|endoftext|>


<commit_before>package rewrite

import (
	"fmt"
	"os"
	"strings"

	rewriterpc "github.com/codelingo/codelingo/flows/codelingo/rewrite/rpc"

	flowutil "github.com/codelingo/codelingo/sdk/flow"
	jc "github.com/juju/testing/checkers"
	gc "gopkg.in/check.v1"
)

func (s *cmdSuite) TestWrite(c *gc.C) {
	c.Skip("reason")
	results := []*flowutil.DecoratedResult{
		{
			Ctx: nil,
			Payload: &rewriterpc.Hunk{
				Filename:    "test/mock.go",
				StartOffset: int32(19),
				EndOffset:   int32(23),
				SRC:         "newName",
			}},
	}

	err := Write(results)
	c.Assert(err, jc.ErrorIsNil)

}

// TODO: implement once rewrite fname is implemented.
func (s *cmdSuite) TestRewriteFileName(c *gc.C) {

}

func (s *cmdSuite) TestNewFile(c *gc.C) {

	newFile := "new_test.go"

	ctx, err := flowutil.NewCtx(&DecoratorApp.App, "--new-file", newFile, "--new-file-perm", "0755")
	c.Assert(err, jc.ErrorIsNil)

	results := []*flowutil.DecoratedResult{
		{

			Ctx: ctx,
			Payload: &rewriterpc.Hunk{
				SRC:         "package rewrite_test",
				StartOffset: int32(19),
				EndOffset:   int32(23),
				Filename:    "flows/codelingo/rewrite/rewrite/writer_test.go",
			},
		},
	}

	c.Assert(Write(results), jc.ErrorIsNil)

	_, err = os.Stat(newFile)
	c.Assert(os.IsNotExist(err), jc.IsFalse)
	c.Assert(os.Remove(newFile), jc.ErrorIsNil)
}

func (s *cmdSuite) TestNewFileSRC(c *gc.C) {

	for _, data := range testData {

		hunk := &rewriterpc.Hunk{
			SRC:              "<NEW CODE>",
			StartOffset:      int32(19),
			EndOffset:        int32(23),
			DecoratorOptions: data.decOpts,
			Filename:         "not_used",
			Comment:          "<ALT CODE>",
		}

		ctx, err := flowutil.NewCtx(&DecoratorApp.App, strings.Split(hunk.DecoratorOptions, " ")[1:]...)
		c.Assert(err, jc.ErrorIsNil)

		newCode, comment, err := newFileSRC(ctx, hunk, []byte(oldSRC))
		c.Assert(err, jc.ErrorIsNil)
		c.Assert(string(newCode), gc.Equals, string(data.newSRC))
		c.Assert(comment, gc.DeepEquals, data.comment)
		fmt.Println("PASS:", data.decOpts)

	}
}

var oldSRC string = `
package test

func main() {

}
`[1:]

var testData = []struct {
	decOpts string
	newSRC  []byte
	comment *comment
}{
	{
		decOpts: "rewrite \"<NEW CODE>\"",
		comment: &comment{
			line:     2,
			content:  "func <ALT CODE>() {",
			original: "func main() {",
		},
		newSRC: []byte(`
package test

func <NEW CODE>() {

}
`[1:]),
	}, {
		decOpts: "rewrite name",
		comment: &comment{
			line:     2,
			content:  "func <ALT CODE>() {",
			original: "func main() {",
		},
		newSRC: []byte(`
package test

func <NEW CODE>() {

}
`[1:]),
	}, {
		decOpts: "rewrite --replace name",
		comment: &comment{
			line:     2,
			content:  "func <ALT CODE>() {",
			original: "func main() {",
		},
		newSRC: []byte(`
package test

func <NEW CODE>() {

}
`[1:]),
	}, {
		decOpts: "rewrite --replace --start-to-end-offset name",
		comment: &comment{
			line:     2,
			content:  "func <ALT CODE>() {",
			original: "func main() {",
		},
		newSRC: []byte(`
package test

func <NEW CODE>() {

}
`[1:]),
	}, {
		decOpts: "rewrite --start-to-end-offset name",
		comment: &comment{
			line:     2,
			content:  "func <ALT CODE>() {",
			original: "func main() {",
		},
		newSRC: []byte(`
package test

func <NEW CODE>() {

}
`[1:]),
	}, {
		decOpts: "rewrite --start-offset name",
		comment: &comment{
			line:     2,
			content:  "func <ALT CODE>ain() {",
			original: "func main() {",
		},
		newSRC: []byte(`
package test

func <NEW CODE>ain() {

}
`[1:]),
	}, {
		decOpts: "rewrite --line name",
		comment: &comment{
			line:     2,
			content:  "<ALT CODE>",
			original: "func main() {",
		},
		newSRC: []byte(`
package test

<NEW CODE>

}
`[1:]),
	}, {
		decOpts: "rewrite --start-offset --line name",
		comment: &comment{
			line:     2,
			content:  "<ALT CODE>",
			original: "func main() {",
		},
		newSRC: []byte(`
package test

<NEW CODE>

}
`[1:]),
	}, {
		decOpts: "rewrite --end-offset --line name",
		comment: &comment{
			line:     2,
			content:  "<ALT CODE>",
			original: "func main() {",
		},
		newSRC: []byte(`
package test

<NEW CODE>

}
`[1:]),
	}, {
		decOpts: "rewrite --start-to-end-offset --prepend name",
		comment: &comment{
			line:     2,
			content:  "func <ALT CODE>main() {",
			original: "func main() {",
		},
		newSRC: []byte(`
package test

func <NEW CODE>main() {

}
`[1:]),
	}, {
		decOpts: "rewrite --start-offset --prepend name",
		comment: &comment{
			line:     2,
			content:  "func <ALT CODE>main() {",
			original: "func main() {",
		},
		newSRC: []byte(`
package test

func <NEW CODE>main() {

}
`[1:]),
	}, {
		decOpts: "rewrite --prepend name",
		comment: &comment{
			line:     2,
			content:  "func <ALT CODE>main() {",
			original: "func main() {",
		},
		newSRC: []byte(`
package test

func <NEW CODE>main() {

}
`[1:]),
	}, {
		decOpts: "rewrite --end-offset --prepend name",
		comment: &comment{
			line:     2,
			content:  "func mai<ALT CODE>n() {",
			original: "func main() {",
		},
		newSRC: []byte(`
package test

func mai<NEW CODE>n() {

}
`[1:]),
	}, {
		decOpts: "rewrite --prepend --line name",
		comment: &comment{
			line:     2,
			content:  "<ALT CODE>",
			original: "func main() {",
		},
		newSRC: []byte(`
package test

<NEW CODE>
func main() {

}
`[1:]),
	}, {
		decOpts: "rewrite --start-to-end-offset --prepend --line name",
		comment: &comment{
			line:     2,
			content:  "<ALT CODE>",
			original: "func main() {",
		},
		newSRC: []byte(`
package test

<NEW CODE>
func main() {

}
`[1:]),
	}, {
		decOpts: "rewrite --start-offset --prepend --line name",
		comment: &comment{
			line:     2,
			content:  "<ALT CODE>",
			original: "func main() {",
		},
		newSRC: []byte(`
package test

<NEW CODE>
func main() {

}
`[1:]),
	}, {
		decOpts: "rewrite --end-offset --prepend --line name",
		comment: &comment{
			line:     2,
			content:  "<ALT CODE>",
			original: "func main() {",
		},
		newSRC: []byte(`
package test

<NEW CODE>
func main() {

}
`[1:]),
	}, {
		decOpts: "rewrite --append name",
		comment: &comment{
			line:     2,
			content:  "func main<ALT CODE>() {",
			original: "func main() {",
		},
		newSRC: []byte(`
package test

func main<NEW CODE>() {

}
`[1:]),
	}, {
		decOpts: "rewrite --start-to-end-offset --append name",
		comment: &comment{
			line:     2,
			content:  "func main<ALT CODE>() {",
			original: "func main() {",
		},
		newSRC: []byte(`
package test

func main<NEW CODE>() {

}
`[1:]),
	}, {
		decOpts: "rewrite --start-offset --append name",
		comment: &comment{
			line:     2,
			content:  "func m<ALT CODE>ain() {",
			original: "func main() {",
		},
		newSRC: []byte(`
package test

func m<NEW CODE>ain() {

}
`[1:]),
	}, {
		decOpts: "rewrite --end-offset --append name",
		comment: &comment{
			line:     2,
			content:  "func main<ALT CODE>() {",
			original: "func main() {",
		},
		newSRC: []byte(`
package test

func main<NEW CODE>() {

}
`[1:]),
	}, {
		decOpts: "rewrite --append --line name",
		comment: &comment{
			line:     3,
			content:  "<ALT CODE>",
			original: "func main() {",
		},
		newSRC: []byte(`
package test

func main() {
<NEW CODE>

}
`[1:]),
	}, {
		decOpts: "rewrite --start-to-end-offset --append --line name",
		comment: &comment{
			line:     3,
			content:  "<ALT CODE>",
			original: "func main() {",
		},
		newSRC: []byte(`
package test

func main() {
<NEW CODE>

}
`[1:]),
	}, {
		decOpts: "rewrite --end-offset --append --line name",
		comment: &comment{
			line:     3,
			content:  "<ALT CODE>",
			original: "func main() {",
		},
		newSRC: []byte(`
package test

func main() {
<NEW CODE>

}
`[1:]),
	}, {
		decOpts: "rewrite --start-offset --append --line name",
		comment: &comment{
			line:     3,
			content:  "<ALT CODE>",
			original: "func main() {",
		},
		newSRC: []byte(`
package test

func main() {
<NEW CODE>

}
`[1:]),
	},
}

// TODO(waigani) test replace first line
<commit_msg>Capitalise comment properties to pass tests<commit_after>package rewrite

import (
	"fmt"
	"os"
	"strings"

	rewriterpc "github.com/codelingo/codelingo/flows/codelingo/rewrite/rpc"

	flowutil "github.com/codelingo/codelingo/sdk/flow"
	jc "github.com/juju/testing/checkers"
	gc "gopkg.in/check.v1"
)

func (s *cmdSuite) TestWrite(c *gc.C) {
	c.Skip("reason")
	results := []*flowutil.DecoratedResult{
		{
			Ctx: nil,
			Payload: &rewriterpc.Hunk{
				Filename:    "test/mock.go",
				StartOffset: int32(19),
				EndOffset:   int32(23),
				SRC:         "newName",
			}},
	}

	err := Write(results)
	c.Assert(err, jc.ErrorIsNil)

}

// TODO: implement once rewrite fname is implemented.
func (s *cmdSuite) TestRewriteFileName(c *gc.C) {

}

func (s *cmdSuite) TestNewFile(c *gc.C) {

	newFile := "new_test.go"

	ctx, err := flowutil.NewCtx(&DecoratorApp.App, "--new-file", newFile, "--new-file-perm", "0755")
	c.Assert(err, jc.ErrorIsNil)

	results := []*flowutil.DecoratedResult{
		{

			Ctx: ctx,
			Payload: &rewriterpc.Hunk{
				SRC:         "package rewrite_test",
				StartOffset: int32(19),
				EndOffset:   int32(23),
				Filename:    "flows/codelingo/rewrite/rewrite/writer_test.go",
			},
		},
	}

	c.Assert(Write(results), jc.ErrorIsNil)

	_, err = os.Stat(newFile)
	c.Assert(os.IsNotExist(err), jc.IsFalse)
	c.Assert(os.Remove(newFile), jc.ErrorIsNil)
}

func (s *cmdSuite) TestNewFileSRC(c *gc.C) {

	for _, data := range testData {

		hunk := &rewriterpc.Hunk{
			SRC:              "<NEW CODE>",
			StartOffset:      int32(19),
			EndOffset:        int32(23),
			DecoratorOptions: data.decOpts,
			Filename:         "not_used",
			Comment:          "<ALT CODE>",
		}

		ctx, err := flowutil.NewCtx(&DecoratorApp.App, strings.Split(hunk.DecoratorOptions, " ")[1:]...)
		c.Assert(err, jc.ErrorIsNil)

		newCode, comment, err := newFileSRC(ctx, hunk, []byte(oldSRC))
		c.Assert(err, jc.ErrorIsNil)
		c.Assert(string(newCode), gc.Equals, string(data.newSRC))
		c.Assert(comment, gc.DeepEquals, data.comment)
		fmt.Println("PASS:", data.decOpts)

	}
}

var oldSRC = `
package test

func main() {

}
`[1:]

var testData = []struct {
	decOpts string
	newSRC  []byte
	comment *comment
}{
	{
		decOpts: "rewrite \"<NEW CODE>\"",
		comment: &comment{
			Line:     2,
			Content:  "func <ALT CODE>() {",
			Original: "func main() {",
		},
		newSRC: []byte(`
package test

func <NEW CODE>() {

}
`[1:]),
	}, {
		decOpts: "rewrite name",
		comment: &comment{
			Line:     2,
			Content:  "func <ALT CODE>() {",
			Original: "func main() {",
		},
		newSRC: []byte(`
package test

func <NEW CODE>() {

}
`[1:]),
	}, {
		decOpts: "rewrite --replace name",
		comment: &comment{
			Line:     2,
			Content:  "func <ALT CODE>() {",
			Original: "func main() {",
		},
		newSRC: []byte(`
package test

func <NEW CODE>() {

}
`[1:]),
	}, {
		decOpts: "rewrite --replace --start-to-end-offset name",
		comment: &comment{
			Line:     2,
			Content:  "func <ALT CODE>() {",
			Original: "func main() {",
		},
		newSRC: []byte(`
package test

func <NEW CODE>() {

}
`[1:]),
	}, {
		decOpts: "rewrite --start-to-end-offset name",
		comment: &comment{
			Line:     2,
			Content:  "func <ALT CODE>() {",
			Original: "func main() {",
		},
		newSRC: []byte(`
package test

func <NEW CODE>() {

}
`[1:]),
	}, {
		decOpts: "rewrite --start-offset name",
		comment: &comment{
			Line:     2,
			Content:  "func <ALT CODE>ain() {",
			Original: "func main() {",
		},
		newSRC: []byte(`
package test

func <NEW CODE>ain() {

}
`[1:]),
	}, {
		decOpts: "rewrite --line name",
		comment: &comment{
			Line:     2,
			Content:  "<ALT CODE>",
			Original: "func main() {",
		},
		newSRC: []byte(`
package test

<NEW CODE>

}
`[1:]),
	}, {
		decOpts: "rewrite --start-offset --line name",
		comment: &comment{
			Line:     2,
			Content:  "<ALT CODE>",
			Original: "func main() {",
		},
		newSRC: []byte(`
package test

<NEW CODE>

}
`[1:]),
	}, {
		decOpts: "rewrite --end-offset --line name",
		comment: &comment{
			Line:     2,
			Content:  "<ALT CODE>",
			Original: "func main() {",
		},
		newSRC: []byte(`
package test

<NEW CODE>

}
`[1:]),
	}, {
		decOpts: "rewrite --start-to-end-offset --prepend name",
		comment: &comment{
			Line:     2,
			Content:  "func <ALT CODE>main() {",
			Original: "func main() {",
		},
		newSRC: []byte(`
package test

func <NEW CODE>main() {

}
`[1:]),
	}, {
		decOpts: "rewrite --start-offset --prepend name",
		comment: &comment{
			Line:     2,
			Content:  "func <ALT CODE>main() {",
			Original: "func main() {",
		},
		newSRC: []byte(`
package test

func <NEW CODE>main() {

}
`[1:]),
	}, {
		decOpts: "rewrite --prepend name",
		comment: &comment{
			Line:     2,
			Content:  "func <ALT CODE>main() {",
			Original: "func main() {",
		},
		newSRC: []byte(`
package test

func <NEW CODE>main() {

}
`[1:]),
	}, {
		decOpts: "rewrite --end-offset --prepend name",
		comment: &comment{
			Line:     2,
			Content:  "func mai<ALT CODE>n() {",
			Original: "func main() {",
		},
		newSRC: []byte(`
package test

func mai<NEW CODE>n() {

}
`[1:]),
	}, {
		decOpts: "rewrite --prepend --line name",
		comment: &comment{
			Line:     2,
			Content:  "<ALT CODE>",
			Original: "func main() {",
		},
		newSRC: []byte(`
package test

<NEW CODE>
func main() {

}
`[1:]),
	}, {
		decOpts: "rewrite --start-to-end-offset --prepend --line name",
		comment: &comment{
			Line:     2,
			Content:  "<ALT CODE>",
			Original: "func main() {",
		},
		newSRC: []byte(`
package test

<NEW CODE>
func main() {

}
`[1:]),
	}, {
		decOpts: "rewrite --start-offset --prepend --line name",
		comment: &comment{
			Line:     2,
			Content:  "<ALT CODE>",
			Original: "func main() {",
		},
		newSRC: []byte(`
package test

<NEW CODE>
func main() {

}
`[1:]),
	}, {
		decOpts: "rewrite --end-offset --prepend --line name",
		comment: &comment{
			Line:     2,
			Content:  "<ALT CODE>",
			Original: "func main() {",
		},
		newSRC: []byte(`
package test

<NEW CODE>
func main() {

}
`[1:]),
	}, {
		decOpts: "rewrite --append name",
		comment: &comment{
			Line:     2,
			Content:  "func main<ALT CODE>() {",
			Original: "func main() {",
		},
		newSRC: []byte(`
package test

func main<NEW CODE>() {

}
`[1:]),
	}, {
		decOpts: "rewrite --start-to-end-offset --append name",
		comment: &comment{
			Line:     2,
			Content:  "func main<ALT CODE>() {",
			Original: "func main() {",
		},
		newSRC: []byte(`
package test

func main<NEW CODE>() {

}
`[1:]),
	}, {
		decOpts: "rewrite --start-offset --append name",
		comment: &comment{
			Line:     2,
			Content:  "func m<ALT CODE>ain() {",
			Original: "func main() {",
		},
		newSRC: []byte(`
package test

func m<NEW CODE>ain() {

}
`[1:]),
	}, {
		decOpts: "rewrite --end-offset --append name",
		comment: &comment{
			Line:     2,
			Content:  "func main<ALT CODE>() {",
			Original: "func main() {",
		},
		newSRC: []byte(`
package test

func main<NEW CODE>() {

}
`[1:]),
	}, {
		decOpts: "rewrite --append --line name",
		comment: &comment{
			Line:     3,
			Content:  "<ALT CODE>",
			Original: "func main() {",
		},
		newSRC: []byte(`
package test

func main() {
<NEW CODE>

}
`[1:]),
	}, {
		decOpts: "rewrite --start-to-end-offset --append --line name",
		comment: &comment{
			Line:     3,
			Content:  "<ALT CODE>",
			Original: "func main() {",
		},
		newSRC: []byte(`
package test

func main() {
<NEW CODE>

}
`[1:]),
	}, {
		decOpts: "rewrite --end-offset --append --line name",
		comment: &comment{
			Line:     3,
			Content:  "<ALT CODE>",
			Original: "func main() {",
		},
		newSRC: []byte(`
package test

func main() {
<NEW CODE>

}
`[1:]),
	}, {
		decOpts: "rewrite --start-offset --append --line name",
		comment: &comment{
			Line:     3,
			Content:  "<ALT CODE>",
			Original: "func main() {",
		},
		newSRC: []byte(`
package test

func main() {
<NEW CODE>

}
`[1:]),
	},
}

// TODO(waigani) test replace first line
<|endoftext|>


<commit_before>package virtualbox

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"github.com/mitchellh/multistep"
	"github.com/mitchellh/packer/packer"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"
)

// This step downloads the ISO specified.
//
// Uses:
//   cache packer.Cache
//   config *config
//   ui     packer.Ui
//
// Produces:
//   iso_path string
type stepDownloadISO struct{}

func (s stepDownloadISO) Run(state map[string]interface{}) multistep.StepAction {
	cache := state["cache"].(packer.Cache)
	config := state["config"].(*config)
	ui := state["ui"].(packer.Ui)

	log.Printf("Acquiring lock to download the ISO.")
	cachePath := cache.Lock(config.ISOUrl)
	defer cache.Unlock(config.ISOUrl)

	err := s.checkMD5(cachePath, config.ISOMD5)
	haveFile := err == nil
	if err != nil {
		if !os.IsNotExist(err) {
			ui.Say(fmt.Sprintf("Error validating MD5 of ISO: %s", err))
			return multistep.ActionHalt
		}
	}

	if !haveFile {
		url, err := url.Parse(config.ISOUrl)
		if err != nil {
			ui.Error(fmt.Sprintf("Error parsing iso_url: %s", err))
			return multistep.ActionHalt
		}

		// Start the download in a goroutine so that we cancel it and such.
		var progress uint
		downloadComplete := make(chan bool, 1)
		go func() {
			ui.Say("Copying or downloading ISO. Progress will be shown periodically.")
			cachePath, err = s.downloadUrl(cachePath, url, &progress)
			downloadComplete <- true
		}()

		progressTimer := time.NewTicker(15 * time.Second)
		defer progressTimer.Stop()

	DownloadWaitLoop:
		for {
			select {
			case <-downloadComplete:
				log.Println("Download of ISO completed.")
				break DownloadWaitLoop
			case <-progressTimer.C:
				ui.Say(fmt.Sprintf("Download progress: %d%%", progress))
			case <-time.After(1 * time.Second):
				if _, ok := state[multistep.StateCancelled]; ok {
					ui.Say("Interrupt received. Cancelling download...")
					return multistep.ActionHalt
				}
			}
		}

		if err != nil {
			ui.Error(fmt.Sprintf("Error downloading ISO: %s", err))
			return multistep.ActionHalt
		}

		if err = s.checkMD5(cachePath, config.ISOMD5); err != nil {
			ui.Say(fmt.Sprintf("Error validating MD5 of ISO: %s", err))
			return multistep.ActionHalt
		}
	}

	log.Printf("Path to ISO on disk: %s", cachePath)
	state["iso_path"] = cachePath

	return multistep.ActionContinue
}

func (stepDownloadISO) Cleanup(map[string]interface{}) {}

func (stepDownloadISO) checkMD5(path string, expected string) error {
	f, err := os.Open(path)
	if err != nil {
		return err
	}

	hash := md5.New()
	io.Copy(hash, f)
	result := strings.ToLower(hex.EncodeToString(hash.Sum(nil)))
	if result != expected {
		return fmt.Errorf("result != expected: %s != %s", result, expected)
	}

	return nil
}

func (stepDownloadISO) downloadUrl(path string, url *url.URL, progress *uint) (string, error) {
	if url.Scheme == "file" {
		// If it is just a file URL, then we already have the ISO
		return url.Path, nil
	}

	// Otherwise, it is an HTTP URL, and we must download it.
	f, err := os.Create(path)
	if err != nil {
		return "", err
	}
	defer f.Close()

	log.Printf("Beginning download of ISO: %s", url.String())
	resp, err := http.Get(url.String())
	if err != nil {
		return "", err
	}

	var buffer [4096]byte
	var totalRead int64
	for {
		n, err := resp.Body.Read(buffer[:])
		if err != nil && err != io.EOF {
			return "", err
		}

		totalRead += int64(n)
		*progress = uint((float64(totalRead) / float64(resp.ContentLength)) * 100)

		if _, werr := f.Write(buffer[:n]); werr != nil {
			return "", werr
		}

		if err == io.EOF {
			break
		}
	}

	return path, nil
}
<commit_msg>builder/virtualbox: Use the common downloader<commit_after>package virtualbox

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"github.com/mitchellh/multistep"
	"github.com/mitchellh/packer/builder/common"
	"github.com/mitchellh/packer/packer"
	"log"
	"time"
)

// This step downloads the ISO specified.
//
// Uses:
//   cache packer.Cache
//   config *config
//   ui     packer.Ui
//
// Produces:
//   iso_path string
type stepDownloadISO struct{}

func (s stepDownloadISO) Run(state map[string]interface{}) multistep.StepAction {
	cache := state["cache"].(packer.Cache)
	config := state["config"].(*config)
	ui := state["ui"].(packer.Ui)

	checksum, err := hex.DecodeString(config.ISOMD5)
	if err != nil {
		ui.Error(fmt.Sprintf("Error parsing checksum: %s", err))
		return multistep.ActionHalt
	}

	log.Printf("Acquiring lock to download the ISO.")
	cachePath := cache.Lock(config.ISOUrl)
	defer cache.Unlock(config.ISOUrl)

	downloadConfig := &common.DownloadConfig{
		Url:        config.ISOUrl,
		TargetPath: cachePath,
		CopyFile:   false,
		Hash:       md5.New(),
		Checksum:   checksum,
	}

	download := common.NewDownloadClient(downloadConfig)

	downloadCompleteCh := make(chan error, 1)
	go func() {
		ui.Say("Copying or downloading ISO. Progress will be reported periodically.")
		cachePath, err = download.Get()
		downloadCompleteCh <- err
	}()

	progressTicker := time.NewTicker(5 * time.Second)
	defer progressTicker.Stop()

DownloadWaitLoop:
	for {
		select {
		case err := <-downloadCompleteCh:
			if err != nil {
				ui.Error(fmt.Sprintf("Error downloading ISO: %s", err))
			}

			break DownloadWaitLoop
		case <-progressTicker.C:
			ui.Say(fmt.Sprintf("Download progress: %d%%", download.PercentProgress()))
		case <-time.After(1 * time.Second):
			if _, ok := state[multistep.StateCancelled]; ok {
				ui.Say("Interrupt received. Cancelling download...")
				return multistep.ActionHalt
			}
		}
	}

	log.Printf("Path to ISO on disk: %s", cachePath)
	state["iso_path"] = cachePath

	return multistep.ActionContinue
}

func (stepDownloadISO) Cleanup(map[string]interface{}) {}
<|endoftext|>


<commit_before>package testing

import (
	"fmt"
	"log"

	"github.com/awslabs/aws-sdk-go/gen/swf"
	"github.com/sclasen/swfsm/fsm"
	. "github.com/sclasen/swfsm/sugar"
)

const (
	StubWorkflow      = "stub"
	ShortStubWorkflow = "stub"
	StubVersion       = "1"
)

var (
	StubTaskList      = &swf.TaskList{Name: S(fmt.Sprintf("%s->%s", StubWorkflow, StubVersion))}
	ShortStubTaskList = &swf.TaskList{Name: S(fmt.Sprintf("%s->%s", ShortStubWorkflow, StubVersion))}
)

type DecisionOutcome struct {
	DecisionTask *swf.DecisionTask
	State        string
	Decisions    []swf.Decision
}

func StubFSM(domain string, client fsm.SWFOps) *fsm.FSM {
	f := &fsm.FSM{
		SWF:        client,
		DataType:   make(map[string]interface{}),
		Domain:     domain,
		Name:       StubWorkflow,
		Serializer: fsm.JSONStateSerializer{},
		TaskList:   *StubTaskList.Name,
	}

	f.AddInitialState(&fsm.FSMState{Name: "Initial", Decider: StubState()})
	return f
}

func StubState() fsm.Decider {
	return func(ctx *fsm.FSMContext, h swf.HistoryEvent, data interface{}) fsm.Outcome {
		log.Printf("at=stub-event event=%+v", PrettyHistoryEvent(h))
		return ctx.Stay(data, ctx.EmptyDecisions())
	}
}

func ShortStubFSM(domain string, client fsm.SWFOps) *fsm.FSM {
	f := &fsm.FSM{
		SWF:        client,
		DataType:   make(map[string]interface{}),
		Domain:     domain,
		Name:       ShortStubWorkflow,
		Serializer: fsm.JSONStateSerializer{},
		TaskList:   *StubTaskList.Name,
	}

	f.AddInitialState(&fsm.FSMState{Name: "Initial", Decider: ShortStubState()})
	return f
}

func ShortStubState() fsm.Decider {
	return func(ctx *fsm.FSMContext, h swf.HistoryEvent, data interface{}) fsm.Outcome {
		log.Printf("at=short-stub-event event=%+v", PrettyHistoryEvent(h))
		return ctx.CompleteWorkflow(data)
	}
}

//intercept any attempts to start a workflow and launch the stub workflow instead.
func TestInterceptor(testID string, stubbedWorkflows, stubbedShortWorkflows []string) *fsm.FuncInterceptor {
	stubbed := make(map[string]struct{})
	stubbedShort := make(map[string]struct{})
	v := struct{}{}
	for _, s := range stubbedWorkflows {
		stubbed[s] = v
	}
	for _, s := range stubbedShortWorkflows {
		stubbedShort[s] = v
	}
	return &fsm.FuncInterceptor{
		AfterDecisionFn: func(decision *swf.DecisionTask, ctx *fsm.FSMContext, outcome *fsm.Outcome) {
			for _, d := range outcome.Decisions {
				switch *d.DecisionType {
				case swf.DecisionTypeStartChildWorkflowExecution:
					if _, ok := stubbed[*d.StartChildWorkflowExecutionDecisionAttributes.WorkflowType.Name]; ok {
						d.StartChildWorkflowExecutionDecisionAttributes.WorkflowType.Name = S(StubWorkflow)
						d.StartChildWorkflowExecutionDecisionAttributes.WorkflowType.Version = S(StubVersion)
						d.StartChildWorkflowExecutionDecisionAttributes.ExecutionStartToCloseTimeout = S("360")
						d.StartChildWorkflowExecutionDecisionAttributes.TaskList = StubTaskList
					}
					if _, ok := stubbedShort[*d.StartChildWorkflowExecutionDecisionAttributes.WorkflowType.Name]; ok {
						d.StartChildWorkflowExecutionDecisionAttributes.WorkflowType.Name = S(ShortStubWorkflow)
						d.StartChildWorkflowExecutionDecisionAttributes.WorkflowType.Version = S(StubVersion)
						d.StartChildWorkflowExecutionDecisionAttributes.ExecutionStartToCloseTimeout = S("360")
						d.StartChildWorkflowExecutionDecisionAttributes.TaskList = ShortStubTaskList
					}
				case swf.DecisionTypeScheduleActivityTask:
					d.ScheduleActivityTaskDecisionAttributes.TaskList = &swf.TaskList{Name: S(*d.ScheduleActivityTaskDecisionAttributes.TaskList.Name + testID)}
				case swf.DecisionTypeContinueAsNewWorkflowExecution:
					d.ContinueAsNewWorkflowExecutionDecisionAttributes.TaskList = &swf.TaskList{Name: S(*d.ContinueAsNewWorkflowExecutionDecisionAttributes.TaskList.Name + testID)}
				}
			}
		},
	}
}

func TestReplicator(decisionOutcomes chan DecisionOutcome) fsm.ReplicationHandler {
	return func(ctx *fsm.FSMContext, task *swf.DecisionTask, outcome *swf.RespondDecisionTaskCompletedInput, state *fsm.SerializedState) error {
		decisionOutcomes <- DecisionOutcome{State: state.StateName, DecisionTask: task, Decisions: outcome.Decisions}
		return nil
	}
}

func TestSWF(client fsm.ClientSWFOps, stubbedWorkflow ...string) fsm.ClientSWFOps {
	stubbed := make(map[string]struct{})
	v := struct{}{}
	for _, s := range stubbedWorkflow {
		stubbed[s] = v
	}
	return &StubSWFClient{
		ClientSWFOps:     client,
		stubbedWorkflows: stubbed,
	}
}

//intercept any attempts to start a workflow and launch the stub workflow instead.
type StubSWFClient struct {
	fsm.ClientSWFOps
	stubbedWorkflows map[string]struct{}
}

func (s *StubSWFClient) StartWorkflowExecution(req *swf.StartWorkflowExecutionInput) (resp *swf.Run, err error) {
	if _, ok := s.stubbedWorkflows[*req.WorkflowType.Name]; ok {
		req.WorkflowType.Name = S(StubWorkflow)
		req.WorkflowType.Version = S(StubVersion)
		req.ExecutionStartToCloseTimeout = S("360")
		req.TaskList = StubTaskList
	}
	return s.ClientSWFOps.StartWorkflowExecution(req)
}
<commit_msg>Revert "intercept continueAsNew and set the task list"<commit_after>package testing

import (
	"fmt"
	"log"

	"github.com/awslabs/aws-sdk-go/gen/swf"
	"github.com/sclasen/swfsm/fsm"
	. "github.com/sclasen/swfsm/sugar"
)

const (
	StubWorkflow      = "stub"
	ShortStubWorkflow = "stub"
	StubVersion       = "1"
)

var (
	StubTaskList      = &swf.TaskList{Name: S(fmt.Sprintf("%s->%s", StubWorkflow, StubVersion))}
	ShortStubTaskList = &swf.TaskList{Name: S(fmt.Sprintf("%s->%s", ShortStubWorkflow, StubVersion))}
)

type DecisionOutcome struct {
	DecisionTask *swf.DecisionTask
	State        string
	Decisions    []swf.Decision
}

func StubFSM(domain string, client fsm.SWFOps) *fsm.FSM {
	f := &fsm.FSM{
		SWF:        client,
		DataType:   make(map[string]interface{}),
		Domain:     domain,
		Name:       StubWorkflow,
		Serializer: fsm.JSONStateSerializer{},
		TaskList:   *StubTaskList.Name,
	}

	f.AddInitialState(&fsm.FSMState{Name: "Initial", Decider: StubState()})
	return f
}

func StubState() fsm.Decider {
	return func(ctx *fsm.FSMContext, h swf.HistoryEvent, data interface{}) fsm.Outcome {
		log.Printf("at=stub-event event=%+v", PrettyHistoryEvent(h))
		return ctx.Stay(data, ctx.EmptyDecisions())
	}
}

func ShortStubFSM(domain string, client fsm.SWFOps) *fsm.FSM {
	f := &fsm.FSM{
		SWF:        client,
		DataType:   make(map[string]interface{}),
		Domain:     domain,
		Name:       ShortStubWorkflow,
		Serializer: fsm.JSONStateSerializer{},
		TaskList:   *StubTaskList.Name,
	}

	f.AddInitialState(&fsm.FSMState{Name: "Initial", Decider: ShortStubState()})
	return f
}

func ShortStubState() fsm.Decider {
	return func(ctx *fsm.FSMContext, h swf.HistoryEvent, data interface{}) fsm.Outcome {
		log.Printf("at=short-stub-event event=%+v", PrettyHistoryEvent(h))
		return ctx.CompleteWorkflow(data)
	}
}

//intercept any attempts to start a workflow and launch the stub workflow instead.
func TestInterceptor(testID string, stubbedWorkflows, stubbedShortWorkflows []string) *fsm.FuncInterceptor {
	stubbed := make(map[string]struct{})
	stubbedShort := make(map[string]struct{})
	v := struct{}{}
	for _, s := range stubbedWorkflows {
		stubbed[s] = v
	}
	for _, s := range stubbedShortWorkflows {
		stubbedShort[s] = v
	}
	return &fsm.FuncInterceptor{
		AfterDecisionFn: func(decision *swf.DecisionTask, ctx *fsm.FSMContext, outcome *fsm.Outcome) {
			for _, d := range outcome.Decisions {
				switch *d.DecisionType {
				case swf.DecisionTypeStartChildWorkflowExecution:
					if _, ok := stubbed[*d.StartChildWorkflowExecutionDecisionAttributes.WorkflowType.Name]; ok {
						d.StartChildWorkflowExecutionDecisionAttributes.WorkflowType.Name = S(StubWorkflow)
						d.StartChildWorkflowExecutionDecisionAttributes.WorkflowType.Version = S(StubVersion)
						d.StartChildWorkflowExecutionDecisionAttributes.ExecutionStartToCloseTimeout = S("360")
						d.StartChildWorkflowExecutionDecisionAttributes.TaskList = StubTaskList
					}
					if _, ok := stubbedShort[*d.StartChildWorkflowExecutionDecisionAttributes.WorkflowType.Name]; ok {
						d.StartChildWorkflowExecutionDecisionAttributes.WorkflowType.Name = S(ShortStubWorkflow)
						d.StartChildWorkflowExecutionDecisionAttributes.WorkflowType.Version = S(StubVersion)
						d.StartChildWorkflowExecutionDecisionAttributes.ExecutionStartToCloseTimeout = S("360")
						d.StartChildWorkflowExecutionDecisionAttributes.TaskList = ShortStubTaskList
					}
				case swf.DecisionTypeScheduleActivityTask:
					d.ScheduleActivityTaskDecisionAttributes.TaskList = &swf.TaskList{Name: S(*d.ScheduleActivityTaskDecisionAttributes.TaskList.Name + testID)}
				}
			}
		},
	}
}

func TestReplicator(decisionOutcomes chan DecisionOutcome) fsm.ReplicationHandler {
	return func(ctx *fsm.FSMContext, task *swf.DecisionTask, outcome *swf.RespondDecisionTaskCompletedInput, state *fsm.SerializedState) error {
		decisionOutcomes <- DecisionOutcome{State: state.StateName, DecisionTask: task, Decisions: outcome.Decisions}
		return nil
	}
}

func TestSWF(client fsm.ClientSWFOps, stubbedWorkflow ...string) fsm.ClientSWFOps {
	stubbed := make(map[string]struct{})
	v := struct{}{}
	for _, s := range stubbedWorkflow {
		stubbed[s] = v
	}
	return &StubSWFClient{
		ClientSWFOps:     client,
		stubbedWorkflows: stubbed,
	}
}

//intercept any attempts to start a workflow and launch the stub workflow instead.
type StubSWFClient struct {
	fsm.ClientSWFOps
	stubbedWorkflows map[string]struct{}
}

func (s *StubSWFClient) StartWorkflowExecution(req *swf.StartWorkflowExecutionInput) (resp *swf.Run, err error) {
	if _, ok := s.stubbedWorkflows[*req.WorkflowType.Name]; ok {
		req.WorkflowType.Name = S(StubWorkflow)
		req.WorkflowType.Version = S(StubVersion)
		req.ExecutionStartToCloseTimeout = S("360")
		req.TaskList = StubTaskList
	}
	return s.ClientSWFOps.StartWorkflowExecution(req)
}
<|endoftext|>


<commit_before>package serial

import (
	"testing"
)

func TestConnection(t *testing.T) {
	c0 := &Config{Name: "COM5", Baud: 115200}

	/*
		c1 := new(Config)
		c1.Name = "COM5"
		c1.Baud = 115200
	*/

	s, err := OpenPort(c0)
	if err != nil {
		t.Fatal(err)
	}

	_, err = s.Write([]byte("test"))
	if err != nil {
		t.Fatal(err)
	}

	buf := make([]byte, 128)
	_, err = s.Read(buf)
	if err != nil {
		t.Fatal(err)
	}
}

// BUG(tarmigan): Add loopback test
func TestLoopback(t *testing.T) {

}
<commit_msg>Skip the connection test if test.short is used.<commit_after>package serial

import (
	"testing"
)

func TestConnection(t *testing.T) {
	if testing.Short() {
		return
	}
	c0 := &Config{Name: "COM5", Baud: 115200}

	/*
		c1 := new(Config)
		c1.Name = "COM5"
		c1.Baud = 115200
	*/

	s, err := OpenPort(c0)
	if err != nil {
		t.Fatal(err)
	}

	_, err = s.Write([]byte("test"))
	if err != nil {
		t.Fatal(err)
	}

	buf := make([]byte, 128)
	_, err = s.Read(buf)
	if err != nil {
		t.Fatal(err)
	}
}

// BUG(tarmigan): Add loopback test
func TestLoopback(t *testing.T) {

}
<|endoftext|>


<commit_before>package aviator

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/JulzDiverse/aviator/spruce"

	"gopkg.in/yaml.v2"
)

type Aviator struct {
	Spruce []SpruceConfig `yaml:"spruce"`
	Fly    FlyConfig      `yaml:"fly"`
}

type SpruceConfig struct {
	Base           string   `yaml:"base"`
	Prune          []string `yaml:"prune"`
	Chain          []Chain  `yaml:"merge"`
	WithIn         string   `yaml:"with_in"`
	Folder         string   `yaml:"dir"`
	ForEach        []string `yaml:"for_each"`
	ForEachIn      string   `yaml:"for_each_in"`
	Walk           string   `yaml:"walk_through"`
	ForAll         string   `yaml:"for_all"`
	CopyParents    bool     `yaml:"copy_parents"`
	EnableMatching bool     `yaml:"enable_matching"`
	DestFile       string   `yaml:"to"`
	DestDir        string   `yaml:"to_dir"`
	Regexp         string   `yaml:"regexp"`
}

type Chain struct {
	With   With   `yaml:"with"`
	WithIn string `yaml:"with_in"`
	Regexp string `yaml:"regexp"`
}

type With struct {
	Files    []string `yaml:"files"`
	InDir    string   `yaml:"in_dir"`
	Existing bool     `yaml:"skip_non_existing"`
}

type FlyConfig struct {
	Name   string   `yaml:"name"`
	Target string   `yaml:"target"`
	Config string   `yaml:"config"`
	Vars   []string `yaml:"vars"`
}

func ReadYaml(ymlBytes []byte) Aviator {
	var yml Aviator

	ymlBytes = quoteBraces(ymlBytes)
	err := yaml.Unmarshal(ymlBytes, &yml)
	if err != nil {
		panic(err)
	}

	return yml
}

var quoteRegex = `\{\{([-\w\p{L}]+)\}\}`
var re = regexp.MustCompile("(" + quoteRegex + ")")

func quoteBraces(input []byte) []byte {
	return re.ReplaceAll(input, []byte("\"$1\""))
}

func FlyPipeline(fly FlyConfig) {

	flyCmd := []string{"-t", fly.Target, "set-pipeline", "-p", fly.Name, "-c", fly.Config}
	for _, val := range fly.Vars {
		flyCmd = append(flyCmd, "-l", val)
	}

	cmd := exec.Command("fly", flyCmd...)

	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin

	err := cmd.Run()
	if err != nil {
		fmt.Printf("Failed to run fly. %s\n", err.Error())
		os.Exit(1)
	}
}

func ProcessSprucePlan(spruce []SpruceConfig) error {
	for _, conf := range spruce {

		verifySpruceConfig(conf)

		if conf.ForEachIn == "" && len(conf.ForEach) == 0 && conf.Walk == "" {
			err := simpleMerge(conf)
			if err != nil {
				return err
			}
		}
		if len(conf.ForEach) != 0 {
			err := ForEachFile(conf)
			if err != nil {
				return err
			}
		}
		if conf.ForEachIn != "" {
			err := ForEachIn(conf)
			if err != nil {
				return err
			}
		}
		if conf.Walk != "" {
			if conf.ForAll != "" {
				err := ForAll(conf)
				if err != nil {
					return err
				}
			} else {
				err := Walk(conf, "")
				if err != nil {
					return err
				}
			}
		}
	}

	return nil
}

func simpleMerge(conf SpruceConfig) error {
	files := collectFiles(conf)
	mergeConf := spruce.MergeOpts{
		Files: files,
		Prune: conf.Prune,
	}
	err := spruceToFile(mergeConf, conf.DestFile)
	if err != nil {
		return err
	}
	return nil
}

func collectFiles(conf SpruceConfig) []string {
	files := []string{conf.Base}
	for _, val := range conf.Chain {
		tmp := collectFromMergeSection(val)
		for _, str := range tmp {
			files = append(files, str)
		}
	}
	return files
}

func ForEachFile(conf SpruceConfig) error {
	for _, val := range conf.ForEach {
		files := collectFiles(conf)
		fileName, _ := ConcatFileName(val)
		files = append(files, val)
		mergeConf := spruce.MergeOpts{
			Files: files,
			Prune: conf.Prune,
		}
		err := spruceToFile(mergeConf, conf.DestDir+fileName)
		if err != nil {
			return err
		}

	}
	return nil
}

func ForEachIn(conf SpruceConfig) error {
	filePaths, _ := ioutil.ReadDir(conf.ForEachIn)
	regex := getRegexp(conf)
	for _, f := range filePaths {
		files := collectFiles(conf)
		matched, _ := regexp.MatchString(regex, f.Name())
		if matched {
			prefix := Chunk(conf.ForEachIn)
			files = append(files, conf.ForEachIn+f.Name())
			mergeConf := spruce.MergeOpts{
				Files: files,
				Prune: conf.Prune,
			}
			err := spruceToFile(mergeConf, conf.DestDir+prefix+"_"+f.Name())
			if err != nil {
				return err
			}

		}
	}
	return nil
}

func ForEachInner(conf SpruceConfig, outer string) error {
	filePaths, _ := ioutil.ReadDir(conf.ForEachIn)
	regex := getRegexp(conf)
	for _, f := range filePaths {
		files := collectFiles(conf)
		matched, _ := regexp.MatchString(regex, f.Name())
		if matched {
			prefix := Chunk(conf.ForEachIn)
			files = append(files, conf.ForEachIn+f.Name())
			files = append(files, outer)
			mergeConf := spruce.MergeOpts{
				Files: files,
				Prune: conf.Prune,
			}
			err := spruceToFile(mergeConf, conf.DestDir+prefix+"_"+f.Name())
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func ForAll(conf SpruceConfig) error {
	if conf.ForAll != "" {
		files, _ := ioutil.ReadDir(conf.ForAll)
		for _, f := range files {
			err := Walk(conf, conf.ForAll+f.Name())
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func Walk(conf SpruceConfig, outer string) error {
	sl := getAllFilesInSubDirs(conf.Walk)
	regex := getRegexp(conf)

	for _, f := range sl {
		filename, parent := ConcatFileName(f)
		match := isMatchingEnabled(conf, parent)
		if strings.Contains(outer, match) {
			matched, _ := regexp.MatchString(regex, filename)
			if matched {
				files := collectFiles(conf)
				files = append(files, f)
				files = append(files, outer)
				if conf.CopyParents {
					CreateDir(conf.DestDir + parent)
				} else {
					parent = ""
				}
				mergeConf := spruce.MergeOpts{
					Files: files,
					Prune: conf.Prune,
				}
				err := spruceToFile(mergeConf, conf.DestDir+parent+"/"+filename)
				if err != nil {
					return err
				}

			}
		}
	}
	return nil
}

func collectFromMergeSection(chain Chain) []string {
	var result []string
	for _, file := range chain.With.Files {
		if chain.With.InDir != "" {
			dir := chain.With.InDir
			file = dir + file
		}
		if !chain.With.Existing || fileExists(file) {
			result = append(result, file)
		}
	}

	if chain.WithIn != "" {
		within := chain.WithIn
		files, _ := ioutil.ReadDir(within)
		regex := getChainRegexp(chain)
		for _, f := range files {
			matched, _ := regexp.MatchString(regex, f.Name())
			if matched {
				result = append(result, within+f.Name())
			}
		}
	}

	return result
}

func spruceToFile(opts spruce.MergeOpts, fileName string) error {
	beautifyPrint(opts, fileName)
	rawYml, err := spruce.CmdMergeEval(opts)
	if err != nil {
		return err
	}

	resultYml, err := yaml.Marshal(rawYml)
	if err != nil {
		return err
	}

	spruce.WriteYamlToPathOrStore(fileName, resultYml)
	return nil
}

func Cleanup(path string) {
	d, err := os.Open(path)
	if err != nil {
		panic(err)
	}
	defer d.Close()
	names, err := d.Readdirnames(-1)
	if err != nil {
		panic(err)
	}
	for _, name := range names {
		err = os.RemoveAll(filepath.Join(path, name))
		if err != nil {
			panic(err)
		}
	}
}
<commit_msg>add except param for with_in<commit_after>package aviator

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/JulzDiverse/aviator/spruce"

	"gopkg.in/yaml.v2"
)

type Aviator struct {
	Spruce []SpruceConfig `yaml:"spruce"`
	Fly    FlyConfig      `yaml:"fly"`
}

type SpruceConfig struct {
	Base           string   `yaml:"base"`
	Prune          []string `yaml:"prune"`
	Chain          []Chain  `yaml:"merge"`
	WithIn         string   `yaml:"with_in"`
	Folder         string   `yaml:"dir"`
	ForEach        []string `yaml:"for_each"`
	ForEachIn      string   `yaml:"for_each_in"`
	Walk           string   `yaml:"walk_through"`
	ForAll         string   `yaml:"for_all"`
	CopyParents    bool     `yaml:"copy_parents"`
	EnableMatching bool     `yaml:"enable_matching"`
	DestFile       string   `yaml:"to"`
	DestDir        string   `yaml:"to_dir"`
	Regexp         string   `yaml:"regexp"`
}

type Chain struct {
	With   With     `yaml:"with"`
	WithIn string   `yaml:"with_in"`
	Except []string `yaml:"except"`
	Regexp string   `yaml:"regexp"`
}

type With struct {
	Files    []string `yaml:"files"`
	InDir    string   `yaml:"in_dir"`
	Existing bool     `yaml:"skip_non_existing"`
}

type FlyConfig struct {
	Name   string   `yaml:"name"`
	Target string   `yaml:"target"`
	Config string   `yaml:"config"`
	Vars   []string `yaml:"vars"`
}

func ReadYaml(ymlBytes []byte) Aviator {
	var yml Aviator

	ymlBytes = quoteBraces(ymlBytes)
	err := yaml.Unmarshal(ymlBytes, &yml)
	if err != nil {
		panic(err)
	}

	return yml
}

var quoteRegex = `\{\{([-\w\p{L}]+)\}\}`
var re = regexp.MustCompile("(" + quoteRegex + ")")

func quoteBraces(input []byte) []byte {
	return re.ReplaceAll(input, []byte("\"$1\""))
}

func FlyPipeline(fly FlyConfig) {

	flyCmd := []string{"-t", fly.Target, "set-pipeline", "-p", fly.Name, "-c", fly.Config}
	for _, val := range fly.Vars {
		flyCmd = append(flyCmd, "-l", val)
	}

	cmd := exec.Command("fly", flyCmd...)

	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin

	err := cmd.Run()
	if err != nil {
		fmt.Printf("Failed to run fly. %s\n", err.Error())
		os.Exit(1)
	}
}

func ProcessSprucePlan(spruce []SpruceConfig) error {
	for _, conf := range spruce {

		verifySpruceConfig(conf)

		if conf.ForEachIn == "" && len(conf.ForEach) == 0 && conf.Walk == "" {
			err := simpleMerge(conf)
			if err != nil {
				return err
			}
		}
		if len(conf.ForEach) != 0 {
			err := ForEachFile(conf)
			if err != nil {
				return err
			}
		}
		if conf.ForEachIn != "" {
			err := ForEachIn(conf)
			if err != nil {
				return err
			}
		}
		if conf.Walk != "" {
			if conf.ForAll != "" {
				err := ForAll(conf)
				if err != nil {
					return err
				}
			} else {
				err := Walk(conf, "")
				if err != nil {
					return err
				}
			}
		}
	}

	return nil
}

func simpleMerge(conf SpruceConfig) error {
	files := collectFiles(conf)
	mergeConf := spruce.MergeOpts{
		Files: files,
		Prune: conf.Prune,
	}
	err := spruceToFile(mergeConf, conf.DestFile)
	if err != nil {
		return err
	}
	return nil
}

func collectFiles(conf SpruceConfig) []string {
	files := []string{conf.Base}
	for _, val := range conf.Chain {
		tmp := collectFromMergeSection(val)
		for _, str := range tmp {
			files = append(files, str)
		}
	}
	return files
}

func ForEachFile(conf SpruceConfig) error {
	for _, val := range conf.ForEach {
		files := collectFiles(conf)
		fileName, _ := ConcatFileName(val)
		files = append(files, val)
		mergeConf := spruce.MergeOpts{
			Files: files,
			Prune: conf.Prune,
		}
		err := spruceToFile(mergeConf, conf.DestDir+fileName)
		if err != nil {
			return err
		}

	}
	return nil
}

func ForEachIn(conf SpruceConfig) error {
	filePaths, _ := ioutil.ReadDir(conf.ForEachIn)
	regex := getRegexp(conf)
	for _, f := range filePaths {
		files := collectFiles(conf)
		matched, _ := regexp.MatchString(regex, f.Name())
		if matched {
			prefix := Chunk(conf.ForEachIn)
			files = append(files, conf.ForEachIn+f.Name())
			mergeConf := spruce.MergeOpts{
				Files: files,
				Prune: conf.Prune,
			}
			err := spruceToFile(mergeConf, conf.DestDir+prefix+"_"+f.Name())
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func ForEachInner(conf SpruceConfig, outer string) error {
	filePaths, _ := ioutil.ReadDir(conf.ForEachIn)
	regex := getRegexp(conf)
	for _, f := range filePaths {
		files := collectFiles(conf)
		matched, _ := regexp.MatchString(regex, f.Name())
		if matched {
			prefix := Chunk(conf.ForEachIn)
			files = append(files, conf.ForEachIn+f.Name())
			files = append(files, outer)
			mergeConf := spruce.MergeOpts{
				Files: files,
				Prune: conf.Prune,
			}
			err := spruceToFile(mergeConf, conf.DestDir+prefix+"_"+f.Name())
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func ForAll(conf SpruceConfig) error {
	if conf.ForAll != "" {
		files, _ := ioutil.ReadDir(conf.ForAll)
		for _, f := range files {
			err := Walk(conf, conf.ForAll+f.Name())
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func Walk(conf SpruceConfig, outer string) error {
	sl := getAllFilesInSubDirs(conf.Walk)
	regex := getRegexp(conf)

	for _, f := range sl {
		filename, parent := ConcatFileName(f)
		match := isMatchingEnabled(conf, parent)
		if strings.Contains(outer, match) {
			matched, _ := regexp.MatchString(regex, filename)
			if matched {
				files := collectFiles(conf)
				files = append(files, f)
				files = append(files, outer)
				if conf.CopyParents {
					CreateDir(conf.DestDir + parent)
				} else {
					parent = ""
				}
				mergeConf := spruce.MergeOpts{
					Files: files,
					Prune: conf.Prune,
				}
				err := spruceToFile(mergeConf, conf.DestDir+parent+"/"+filename)
				if err != nil {
					return err
				}

			}
		}
	}
	return nil
}

func collectFromMergeSection(chain Chain) []string {
	var result []string
	for _, file := range chain.With.Files {
		if chain.With.InDir != "" {
			dir := chain.With.InDir
			file = dir + file
		}
		if !chain.With.Existing || fileExists(file) {
			result = append(result, file)
		}
	}

	if chain.WithIn != "" {
		within := chain.WithIn
		files, _ := ioutil.ReadDir(within)
		regex := getChainRegexp(chain)
		for _, f := range files {
			if except(chain.Except, f.Name()) {
				continue
			}
			matched, _ := regexp.MatchString(regex, f.Name())
			if matched {
				result = append(result, within+f.Name())
			}
		}
	}

	return result
}

func except(except []string, file string) bool {
	for _, f := range except {
		if f == file {
			return true
		}
	}
	return false
}

func spruceToFile(opts spruce.MergeOpts, fileName string) error {
	beautifyPrint(opts, fileName)
	rawYml, err := spruce.CmdMergeEval(opts)
	if err != nil {
		return err
	}

	resultYml, err := yaml.Marshal(rawYml)
	if err != nil {
		return err
	}

	spruce.WriteYamlToPathOrStore(fileName, resultYml)
	return nil
}

func Cleanup(path string) {
	d, err := os.Open(path)
	if err != nil {
		panic(err)
	}
	defer d.Close()
	names, err := d.Readdirnames(-1)
	if err != nil {
		panic(err)
	}
	for _, name := range names {
		err = os.RemoveAll(filepath.Join(path, name))
		if err != nil {
			panic(err)
		}
	}
}
<|endoftext|>


<commit_before>package miniredis

import (
	"fmt"
	"net"
	"sync"

	"github.com/bsm/redeo"
)

type redisDB struct {
	sync.Mutex
	keys       map[string]string            // Master map of keys with their type
	stringKeys map[string]string            // GET/SET &c. keys
	hashKeys   map[string]map[string]string // MGET/MSET &c. keys
	expire     map[string]int               // EXPIRE values
}

// Miniredis is a Redis server implementation.
type Miniredis struct {
	sync.Mutex
	closed   chan struct{}
	listen   net.Listener
	info     *redeo.ServerInfo
	dbs      map[int]*redisDB
	clientDB int            // DB id used in the direct Get(), Set() &c.
	selectDB map[uint64]int // Current DB per connection id
}

// NewMiniRedis makes a new, non-started, Miniredis object.
func NewMiniRedis() *Miniredis {
	return &Miniredis{
		closed:   make(chan struct{}),
		dbs:      map[int]*redisDB{},
		selectDB: map[uint64]int{},
	}
}

func newRedisDB() redisDB {
	return redisDB{
		keys:       map[string]string{},
		stringKeys: map[string]string{},
		hashKeys:   map[string]map[string]string{},
		expire:     map[string]int{},
	}
}

// Run creates and Start()s a Miniredis.
func Run() (*Miniredis, error) {
	m := NewMiniRedis()
	return m, m.Start()
}

// Start starts a server. It listens on a random port on localhost. See also Addr().
func (m *Miniredis) Start() error {
	m.Lock()
	defer m.Unlock()
	l, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		if l, err = net.Listen("tcp6", "[::1]:0"); err != nil {
			return fmt.Errorf("failed to listen on a port: %v", err)
		}
	}
	m.listen = l
	srv := redeo.NewServer(&redeo.Config{Addr: "localhost:0"})

	m.info = srv.Info()

	commandsConnection(m, srv)
	commandsGeneric(m, srv)
	commandsString(m, srv)
	commandsHash(m, srv)

	go func() {
		e := make(chan error)
		go srv.Serve(e, m.listen)
		<-e
		m.closed <- struct{}{}
	}()
	return nil
}

// Close shuts down a Miniredis.
func (m *Miniredis) Close() {
	m.Lock()
	defer m.Unlock()
	if m.listen == nil {
		return
	}
	if m.listen.Close() != nil {
		return
	}
	<-m.closed
	m.listen = nil
}

// DB returns a DB by ID.
func (m *Miniredis) DB(i int) *redisDB {
	m.Lock()
	defer m.Unlock()
	return m.db(i)
}

// get DB. No locks!
func (m *Miniredis) db(i int) *redisDB {
	if db, ok := m.dbs[i]; ok {
		return db
	}
	db := newRedisDB()
	m.dbs[i] = &db
	return &db
}

// dbFor gets the DB for a connection id.
func (m *Miniredis) dbFor(connID uint64) *redisDB {
	m.Lock()
	defer m.Unlock()
	return m.db(m.selectDB[connID])
}

// Addr returns '127.0.0.1:12345'. Can be given to a Dial(). See also Host()
// and Port(), which return the same things.
func (m *Miniredis) Addr() string {
	m.Lock()
	defer m.Unlock()
	return m.listen.Addr().String()
}

// Host returns the host part of Addr()
func (m *Miniredis) Host() string {
	m.Lock()
	defer m.Unlock()
	host, _, _ := net.SplitHostPort(m.listen.Addr().String())
	return host
}

// Port returns the (random) port part of Addr().
func (m *Miniredis) Port() string {
	m.Lock()
	defer m.Unlock()
	_, port, _ := net.SplitHostPort(m.listen.Addr().String())
	return port
}

// CommandCount returns the number of processed commands.
func (m *Miniredis) CommandCount() int {
	m.Lock()
	defer m.Unlock()
	return int(m.info.TotalProcessed())
}

// CurrentConnectionCount returns the number of currently connected clients.
func (m *Miniredis) CurrentConnectionCount() int {
	m.Lock()
	defer m.Unlock()
	return m.info.ClientsLen()
}

// TotalConnectionCount returns the number of client connections since server start.
func (m *Miniredis) TotalConnectionCount() int {
	m.Lock()
	defer m.Unlock()
	return int(m.info.TotalConnections())
}
<commit_msg>Minimal godoc docs.<commit_after>// Miniredis is a pure Go Redis test server, for use in Go unittests. There are
// no dependencies on system binaries, and every server you start will be empty.
//
// Start a server with `s, err := miniredis.Run()`.
// Stop it with `defer s.Close()`.
//
// Point your Redis client to `s.Addr()` or `s.Host(), s.Port()`.
//
// Set keys directly via s.Set(...) and similar commands, or use a Redis client.
//
// For direct use you can select a Redis database with either `s.Select(12); s.Get("foo")` or `s.DB(12).Get("foo")`.
package miniredis

import (
	"fmt"
	"net"
	"sync"

	"github.com/bsm/redeo"
)

type redisDB struct {
	sync.Mutex
	keys       map[string]string            // Master map of keys with their type
	stringKeys map[string]string            // GET/SET &c. keys
	hashKeys   map[string]map[string]string // MGET/MSET &c. keys
	expire     map[string]int               // EXPIRE values
}

// Miniredis is a Redis server implementation.
type Miniredis struct {
	sync.Mutex
	closed   chan struct{}
	listen   net.Listener
	info     *redeo.ServerInfo
	dbs      map[int]*redisDB
	clientDB int            // DB id used in the direct Get(), Set() &c.
	selectDB map[uint64]int // Current DB per connection id
}

// NewMiniRedis makes a new, non-started, Miniredis object.
func NewMiniRedis() *Miniredis {
	return &Miniredis{
		closed:   make(chan struct{}),
		dbs:      map[int]*redisDB{},
		selectDB: map[uint64]int{},
	}
}

func newRedisDB() redisDB {
	return redisDB{
		keys:       map[string]string{},
		stringKeys: map[string]string{},
		hashKeys:   map[string]map[string]string{},
		expire:     map[string]int{},
	}
}

// Run creates and Start()s a Miniredis.
func Run() (*Miniredis, error) {
	m := NewMiniRedis()
	return m, m.Start()
}

// Start starts a server. It listens on a random port on localhost. See also Addr().
func (m *Miniredis) Start() error {
	m.Lock()
	defer m.Unlock()
	l, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		if l, err = net.Listen("tcp6", "[::1]:0"); err != nil {
			return fmt.Errorf("failed to listen on a port: %v", err)
		}
	}
	m.listen = l
	srv := redeo.NewServer(&redeo.Config{Addr: "localhost:0"})

	m.info = srv.Info()

	commandsConnection(m, srv)
	commandsGeneric(m, srv)
	commandsString(m, srv)
	commandsHash(m, srv)

	go func() {
		e := make(chan error)
		go srv.Serve(e, m.listen)
		<-e
		m.closed <- struct{}{}
	}()
	return nil
}

// Close shuts down a Miniredis.
func (m *Miniredis) Close() {
	m.Lock()
	defer m.Unlock()
	if m.listen == nil {
		return
	}
	if m.listen.Close() != nil {
		return
	}
	<-m.closed
	m.listen = nil
}

// DB returns a DB by ID.
func (m *Miniredis) DB(i int) *redisDB {
	m.Lock()
	defer m.Unlock()
	return m.db(i)
}

// get DB. No locks!
func (m *Miniredis) db(i int) *redisDB {
	if db, ok := m.dbs[i]; ok {
		return db
	}
	db := newRedisDB()
	m.dbs[i] = &db
	return &db
}

// dbFor gets the DB for a connection id.
func (m *Miniredis) dbFor(connID uint64) *redisDB {
	m.Lock()
	defer m.Unlock()
	return m.db(m.selectDB[connID])
}

// Addr returns '127.0.0.1:12345'. Can be given to a Dial(). See also Host()
// and Port(), which return the same things.
func (m *Miniredis) Addr() string {
	m.Lock()
	defer m.Unlock()
	return m.listen.Addr().String()
}

// Host returns the host part of Addr()
func (m *Miniredis) Host() string {
	m.Lock()
	defer m.Unlock()
	host, _, _ := net.SplitHostPort(m.listen.Addr().String())
	return host
}

// Port returns the (random) port part of Addr().
func (m *Miniredis) Port() string {
	m.Lock()
	defer m.Unlock()
	_, port, _ := net.SplitHostPort(m.listen.Addr().String())
	return port
}

// CommandCount returns the number of processed commands.
func (m *Miniredis) CommandCount() int {
	m.Lock()
	defer m.Unlock()
	return int(m.info.TotalProcessed())
}

// CurrentConnectionCount returns the number of currently connected clients.
func (m *Miniredis) CurrentConnectionCount() int {
	m.Lock()
	defer m.Unlock()
	return m.info.ClientsLen()
}

// TotalConnectionCount returns the number of client connections since server start.
func (m *Miniredis) TotalConnectionCount() int {
	m.Lock()
	defer m.Unlock()
	return int(m.info.TotalConnections())
}
<|endoftext|>


<commit_before>package pigosat

import (
	"reflect"
	"testing"
	"fmt"
)

// TestMinimize will test optimal values from `from` to `to`.
const (
	from = -32
	to   = -from
)

func init() {
	if from >= to {
		panic("from >= to")
	}
}

type parameters struct {
	lower, upper, optimal int
}

type arguments struct {
	k        int
	status   Status
	solution Solution
}

type minimizer struct {
	t      *testing.T
	args   []arguments
	params parameters
}

func newMinimizer(lo, hi, opt int, t *testing.T) *minimizer {
	m := &minimizer{params: parameters{lower: lo, upper: hi, optimal: opt}, t: t}
	// A little testing by hand suggests 2 is faster than 0 or (to - from)
	m.args = make([]arguments, 2)
	return m
}

func (m *minimizer) LowerBound() int { return m.params.lower }

func (m *minimizer) UpperBound() int { return m.params.upper }

func (m *minimizer) IsFeasible(k int) (solution Solution, status Status) {
	if k < from {
		m.t.Errorf("k too low: %d", k)
	}
	if k > to {
		m.t.Errorf("k too hi: %d", k)
	}
	status = Satisfiable
	if k < m.params.optimal {
		status = Unsatisfiable
	}
	m.args = append(m.args, arguments{k, status, solution})
	return
}

func (m *minimizer) RecordSolution(k int, solution Solution, status Status) {
	m.args = append(m.args, arguments{k, status, solution})
}

// Check that RecordSolution is called with IsFeasible's output every time.
func checkFeasibleRecord(t *testing.T, v parameters, args []arguments) {
	var last arguments
	if len(args)%2 != 0 {
		t.Fatalf("Entries in 'args' not recorded in IsFeasible/RecordSolution pairs")
	}
	for count, arg := range args {
		// Each call to IsFeasible is paried with a RecordSolution. Thus we're
		// looking for pairs of arguments.
		if count%2 == 0 {
			last = arg
			continue
		}
		if arg.k != last.k || arg.status != last.status ||
			!reflect.DeepEqual(arg.solution, last.solution) {
			t.Errorf("%+v: feasible=%+v record=%+v", v, last, arg)
		}
	}
}

// TestMinimize tests that the bisection search that Minimize does correctly
// finds the optimal value within the lower and upper bounds, that optimal and
// feasible flags are set correctly, Minimizer.RecordSolution is always called
// after Minimizer.IsFeasible.
func TestMinimize(t *testing.T) {
	for hi := from; hi <= to; hi++ {
		for lo := from; lo <= hi; lo++ {
			for opt := lo; opt <= hi+1; opt++ {
				t.Run(fmt.Sprintf("hi=%d,lo=%d,opt=%d", hi, lo, opt),
					  func (t *testing.T) {
					m := newMinimizer(lo, hi, opt, t)
					min, optimal, feasible := Minimize(m)
					checkFeasibleRecord(t, m.params, m.args)
					if opt <= hi && min != opt {
						t.Errorf("%+v: min=%d", m.params, min)
					}
					if opt > lo && opt <= hi && !optimal {
						t.Errorf("%+v: Should have been optimal", m.params)
					} else if opt <= lo && optimal {
						t.Errorf("%+v: Should not have been optimal", m.params)
					}
					if opt <= hi && !feasible {
						t.Errorf("%+v: Should have been feasible", m.params)
					} else if opt > hi && feasible {
						t.Errorf("%+v: Should not have been feasible", m.params)
					}
				})
			} // opt
		} // lo
	} // hi
} // func
<commit_msg>Test panic in Minimize when UpperBound < LowerBound<commit_after>package pigosat

import (
	"reflect"
	"testing"
	"fmt"
)

// TestMinimize will test optimal values from `from` to `to`.
const (
	from = -32
	to   = -from
)

func init() {
	if from >= to {
		panic("from >= to")
	}
}

type parameters struct {
	lower, upper, optimal int
}

type arguments struct {
	k        int
	status   Status
	solution Solution
}

type minimizer struct {
	t      *testing.T
	args   []arguments
	params parameters
}

func newMinimizer(lo, hi, opt int, t *testing.T) *minimizer {
	m := &minimizer{params: parameters{lower: lo, upper: hi, optimal: opt}, t: t}
	// A little testing by hand suggests 2 is faster than 0 or (to - from)
	m.args = make([]arguments, 2)
	return m
}

func (m *minimizer) LowerBound() int { return m.params.lower }

func (m *minimizer) UpperBound() int { return m.params.upper }

func (m *minimizer) IsFeasible(k int) (solution Solution, status Status) {
	if k < from {
		m.t.Errorf("k too low: %d", k)
	}
	if k > to {
		m.t.Errorf("k too hi: %d", k)
	}
	status = Satisfiable
	if k < m.params.optimal {
		status = Unsatisfiable
	}
	m.args = append(m.args, arguments{k, status, solution})
	return
}

func (m *minimizer) RecordSolution(k int, solution Solution, status Status) {
	m.args = append(m.args, arguments{k, status, solution})
}

// Check that RecordSolution is called with IsFeasible's output every time.
func checkFeasibleRecord(t *testing.T, v parameters, args []arguments) {
	var last arguments
	if len(args)%2 != 0 {
		t.Fatalf("Entries in 'args' not recorded in IsFeasible/RecordSolution pairs")
	}
	for count, arg := range args {
		// Each call to IsFeasible is paried with a RecordSolution. Thus we're
		// looking for pairs of arguments.
		if count%2 == 0 {
			last = arg
			continue
		}
		if arg.k != last.k || arg.status != last.status ||
			!reflect.DeepEqual(arg.solution, last.solution) {
			t.Errorf("%+v: feasible=%+v record=%+v", v, last, arg)
		}
	}
}

// TestMinimize tests that the bisection search that Minimize does correctly
// finds the optimal value within the lower and upper bounds, that optimal and
// feasible flags are set correctly, Minimizer.RecordSolution is always called
// after Minimizer.IsFeasible.
func TestMinimize(t *testing.T) {
	for hi := from; hi <= to; hi++ {
		for lo := from; lo <= hi; lo++ {
			for opt := lo; opt <= hi+1; opt++ {
				t.Run(fmt.Sprintf("hi=%d,lo=%d,opt=%d", hi, lo, opt),
					  func (t *testing.T) {
					m := newMinimizer(lo, hi, opt, t)
					min, optimal, feasible := Minimize(m)
					checkFeasibleRecord(t, m.params, m.args)
					if opt <= hi && min != opt {
						t.Errorf("%+v: min=%d", m.params, min)
					}
					if opt > lo && opt <= hi && !optimal {
						t.Errorf("%+v: Should have been optimal", m.params)
					} else if opt <= lo && optimal {
						t.Errorf("%+v: Should not have been optimal", m.params)
					}
					if opt <= hi && !feasible {
						t.Errorf("%+v: Should have been feasible", m.params)
					} else if opt > hi && feasible {
						t.Errorf("%+v: Should not have been feasible", m.params)
					}
				})
			} // opt
		} // lo
	} // hi

	// Test the error when UpperBound() < LowerBound()
	t.Run("UpperBound < LowerBound", func (t *testing.T) {
		m := newMinimizer(to, from, to, t)
		assertPanics(t, "Minimize", func () { Minimize(m) })
	})
} // func
<|endoftext|>


<commit_before>// +build !windows,!appengine

package maxminddb

import (
	"syscall"

	"golang.org/x/sys/unix"
)

func mmap(fd int, length int) (data []byte, err error) {
	return unix.Mmap(fd, 0, length, syscall.PROT_READ, syscall.MAP_SHARED)
}

func munmap(b []byte) (err error) {
	return unix.Munmap(b)
}
<commit_msg>Remove leftover syscall usage<commit_after>// +build !windows,!appengine

package maxminddb

import (
	"golang.org/x/sys/unix"
)

func mmap(fd int, length int) (data []byte, err error) {
	return unix.Mmap(fd, 0, length, unix.PROT_READ, unix.MAP_SHARED)
}

func munmap(b []byte) (err error) {
	return unix.Munmap(b)
}
<|endoftext|>