<commit_before>package libkbfs

import (
	"container/heap"
	"sync"

	"golang.org/x/net/context"
)

const (
	defaultBlockRetrievalWorkerQueueSize int = 100
)

// blockRetrievalRequest represents one consumer's request for a block.
type blockRetrievalRequest struct {
	ctx    context.Context
	block  Block
	doneCh chan error
}

// blockRetrieval contains the metadata for a given block retrieval. May
// represent many requests, all of which will be handled at once.
type blockRetrieval struct {
	// The block being retrieved
	blockPtr BlockPointer
	// The index of the retrieval in the heap
	index int
	// The priority of the retrieval
	priority int
	// The count of elements in the heap when this element was inserted.
	// Maintains FIFO.
	insertionOrder uint64
	// All the individual requests for this block. They must be notified once
	// the block is returned.
	requests []*blockRetrievalRequest
}

// blockRetrievalQueue manages block retrieval requests. Higher priority
// requests are executed first. Requests are executed in FIFO order within a
// given priority level.
type blockRetrievalQueue struct {
	mtx sync.RWMutex
	// queued or in progress retrievals
	ptrs map[BlockPointer]*blockRetrieval
	// global counter of insertions to queue
	// capacity: ~584 years at 1 billion requests/sec
	insertionCount uint64

	heap *blockRetrievalHeap
	// This is a channel of channels to maximize the time that each request is
	// in the heap, allowing preemption as long as possible. This way, a
	// request only exits the heap once a worker is ready.
	workerQueue chan chan *blockRetrieval
}

// newBlockRetrievalQueue creates a new block retrieval queue. The numWorkers
// parameter determines how many workers can concurrently call WorkOnRequest
// (more than numWorkers will block).
func newBlockRetrievalQueue(numWorkers int) *blockRetrievalQueue {
	return &blockRetrievalQueue{
		ptrs:        make(map[BlockPointer]*blockRetrieval),
		heap:        &blockRetrievalHeap{},
		workerQueue: make(chan chan *blockRetrieval, numWorkers),
	}
}

// notifyWorker notifies workers that there is a new request for processing.
func (brq *blockRetrievalQueue) notifyWorker() {
	go func() {
		// Get the next queued worker
		ch := <-brq.workerQueue
		// Prevent interference with the heap while we're retrieving from it
		brq.mtx.Lock()
		defer brq.mtx.Unlock()
		// Pop from the heap
		ch <- heap.Pop(brq.heap).(*blockRetrieval)
	}()
}

// Request submits a block request to the queue.
func (brq *blockRetrievalQueue) Request(ctx context.Context, priority int, ptr BlockPointer, block Block) <-chan error {
	brq.mtx.Lock()
	defer brq.mtx.Unlock()
	var br *blockRetrieval
	var exists bool
	if br, exists = brq.ptrs[ptr]; !exists {
		// Add to the heap
		br = &blockRetrieval{
			blockPtr:       ptr,
			index:          -1,
			priority:       priority,
			insertionOrder: brq.insertionCount,
			requests:       []*blockRetrievalRequest{},
		}
		brq.insertionCount++
		brq.ptrs[ptr] = br
		heap.Push(brq.heap, br)
		defer brq.notifyWorker()
	}
	ch := make(chan error, 1)
	br.requests = append(br.requests, &blockRetrievalRequest{ctx, block, ch})
	// If the new request priority is higher, elevate the request in the queue
	// Request might not be in the heap any more (could be processing)
	if br.index != -1 && priority > br.priority {
		br.priority = priority
		heap.Fix(brq.heap, br.index)
	}
	return ch
}

// WorkOnRequest returns a new channel for a worker to obtain a blockRetrieval.
func (brq *blockRetrievalQueue) WorkOnRequest() <-chan *blockRetrieval {
	ch := make(chan *blockRetrieval, 1)
	brq.workerQueue <- ch

	return ch
}

// FinalizeRequest communicates that any subsequent requestors for this block
// won't be notified by the current worker processing it.  This must be called
// before sending out the responses to the blockRetrievalRequests for a given
// blockRetrieval.
func (brq *blockRetrievalQueue) FinalizeRequest(ptr BlockPointer) {
	brq.mtx.Lock()
	defer brq.mtx.Unlock()

	delete(brq.ptrs, ptr)
}
<commit_msg>Added KeyMetadata parameter to the retrieval<commit_after>package libkbfs

import (
	"container/heap"
	"sync"

	"golang.org/x/net/context"
)

const (
	defaultBlockRetrievalWorkerQueueSize int = 100
)

// blockRetrievalRequest represents one consumer's request for a block.
type blockRetrievalRequest struct {
	ctx    context.Context
	block  Block
	doneCh chan error
}

// blockRetrieval contains the metadata for a given block retrieval. May
// represent many requests, all of which will be handled at once.
type blockRetrieval struct {
	// The block being retrieved
	blockPtr BlockPointer
	// The key metadata for the request
	kmd KeyMetadata
	// The index of the retrieval in the heap
	index int
	// The priority of the retrieval
	priority int
	// The count of elements in the heap when this element was inserted.
	// Maintains FIFO.
	insertionOrder uint64
	// All the individual requests for this block. They must be notified once
	// the block is returned.
	requests []*blockRetrievalRequest
}

// blockRetrievalQueue manages block retrieval requests. Higher priority
// requests are executed first. Requests are executed in FIFO order within a
// given priority level.
type blockRetrievalQueue struct {
	mtx sync.RWMutex
	// queued or in progress retrievals
	ptrs map[BlockPointer]*blockRetrieval
	// global counter of insertions to queue
	// capacity: ~584 years at 1 billion requests/sec
	insertionCount uint64

	heap *blockRetrievalHeap
	// This is a channel of channels to maximize the time that each request is
	// in the heap, allowing preemption as long as possible. This way, a
	// request only exits the heap once a worker is ready.
	workerQueue chan chan *blockRetrieval
}

// newBlockRetrievalQueue creates a new block retrieval queue. The numWorkers
// parameter determines how many workers can concurrently call WorkOnRequest
// (more than numWorkers will block).
func newBlockRetrievalQueue(numWorkers int) *blockRetrievalQueue {
	return &blockRetrievalQueue{
		ptrs:        make(map[BlockPointer]*blockRetrieval),
		heap:        &blockRetrievalHeap{},
		workerQueue: make(chan chan *blockRetrieval, numWorkers),
	}
}

// notifyWorker notifies workers that there is a new request for processing.
func (brq *blockRetrievalQueue) notifyWorker() {
	go func() {
		// Get the next queued worker
		ch := <-brq.workerQueue
		// Prevent interference with the heap while we're retrieving from it
		brq.mtx.Lock()
		defer brq.mtx.Unlock()
		// Pop from the heap
		ch <- heap.Pop(brq.heap).(*blockRetrieval)
	}()
}

// Request submits a block request to the queue.
func (brq *blockRetrievalQueue) Request(ctx context.Context, priority int, ptr BlockPointer, block Block) <-chan error {
	brq.mtx.Lock()
	defer brq.mtx.Unlock()
	var br *blockRetrieval
	var exists bool
	if br, exists = brq.ptrs[ptr]; !exists {
		// Add to the heap
		br = &blockRetrieval{
			blockPtr:       ptr,
			index:          -1,
			priority:       priority,
			insertionOrder: brq.insertionCount,
			requests:       []*blockRetrievalRequest{},
		}
		brq.insertionCount++
		brq.ptrs[ptr] = br
		heap.Push(brq.heap, br)
		defer brq.notifyWorker()
	}
	ch := make(chan error, 1)
	br.requests = append(br.requests, &blockRetrievalRequest{ctx, block, ch})
	// If the new request priority is higher, elevate the request in the queue
	// Request might not be in the heap any more (could be processing)
	if br.index != -1 && priority > br.priority {
		br.priority = priority
		heap.Fix(brq.heap, br.index)
	}
	return ch
}

// WorkOnRequest returns a new channel for a worker to obtain a blockRetrieval.
func (brq *blockRetrievalQueue) WorkOnRequest() <-chan *blockRetrieval {
	ch := make(chan *blockRetrieval, 1)
	brq.workerQueue <- ch

	return ch
}

// FinalizeRequest communicates that any subsequent requestors for this block
// won't be notified by the current worker processing it.  This must be called
// before sending out the responses to the blockRetrievalRequests for a given
// blockRetrieval.
func (brq *blockRetrievalQueue) FinalizeRequest(ptr BlockPointer) {
	brq.mtx.Lock()
	defer brq.mtx.Unlock()

	delete(brq.ptrs, ptr)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"strings"

	"github.com/imdario/mergo"
	"github.com/mitchellh/go-homedir"
	"github.com/zackbloom/go-ini"
	"github.com/zackbloom/goamz/aws"
	"github.com/zackbloom/goamz/cloudfront"
	"github.com/zackbloom/goamz/iam"
	"github.com/zackbloom/goamz/route53"
	"github.com/zackbloom/goamz/s3"
	"gopkg.in/yaml.v1"
)

const (
	LIMITED = 60
	FOREVER = 31556926
)

var s3Session *s3.S3
var iamSession *iam.IAM
var r53Session *route53.Route53
var cfSession *cloudfront.CloudFront

func getRegion(region string) aws.Region {
	regionS, ok := aws.Regions[region]
	if !ok {
		panic("Region not found")
	}
	return regionS
}

func openS3(key, secret, region string) *s3.S3 {
	regionS := getRegion(region)

	auth := aws.Auth{
		AccessKey: key,
		SecretKey: secret,
	}
	return s3.New(auth, regionS)
}

func openIAM(key, secret, region string) *iam.IAM {
	regionS := getRegion(region)

	auth := aws.Auth{
		AccessKey: key,
		SecretKey: secret,
	}
	return iam.New(auth, regionS)
}

func openCloudFront(key, secret string) *cloudfront.CloudFront {
	auth := aws.Auth{
		AccessKey: key,
		SecretKey: secret,
	}
	return cloudfront.NewCloudFront(auth)
}

func openRoute53(key, secret string) *route53.Route53 {
	auth := aws.Auth{
		AccessKey: key,
		SecretKey: secret,
	}

	r53, _ := route53.NewRoute53(auth)
	return r53
}

func panicIf(err error) {
	if err != nil {
		panic(err)
	}
}
func must(val interface{}, err error) interface{} {
	if err != nil {
		panic(err)
	}

	return val
}
func mustString(val string, err error) string {
	panicIf(err)
	return val
}
func mustInt(val int, err error) int {
	panicIf(err)
	return val
}

type Options struct {
	Files      string `yaml:"files"`
	Root       string `yaml:"root"`
	Dest       string `yaml:"dest"`
	ConfigFile string `yaml:"-"`
	Env        string `yaml:"-"`
	Bucket     string `yaml:"bucket"`
	AWSKey     string `yaml:"key"`
	AWSSecret  string `yaml:"secret"`
	AWSRegion  string `yaml:"region"`
	NoUser     bool   `yaml:"-"`
}

func parseOptions() (o Options, set *flag.FlagSet) {
	set = flag.NewFlagSet(os.Args[1], flag.ExitOnError)
	//TODO: Set set.Usage

	set.StringVar(&o.Files, "files", "*", "Comma-seperated glob patterns of files to deploy (within root)")
	set.StringVar(&o.Root, "root", "./", "The local directory to deploy")
	set.StringVar(&o.Dest, "dest", "./", "The destination directory to write files to in the S3 bucket")
	set.StringVar(&o.ConfigFile, "config", "", "A yaml file to read configuration from")
	set.StringVar(&o.Env, "env", "", "The env to read from the config file")
	set.StringVar(&o.Bucket, "bucket", "", "The bucket to deploy to")
	set.StringVar(&o.AWSKey, "key", "", "The AWS key to use")
	set.StringVar(&o.AWSSecret, "secret", "", "The AWS secret of the provided key")
	set.StringVar(&o.AWSRegion, "region", "us-east-1", "The AWS region the S3 bucket is in")
	set.BoolVar(&o.NoUser, "no-user", false, "When creating, should we make a user account?")

	set.Parse(os.Args[2:])

	return
}

type ConfigFile map[string]Options

func loadConfigFile(o *Options) {
	isDefault := false
	configPath := o.ConfigFile
	if o.ConfigFile == "" {
		isDefault = true
		configPath = "./deploy.yaml"
	}

	data, err := ioutil.ReadFile(configPath)
	if err != nil {
		if os.IsNotExist(err) && isDefault {
			return
		}

		panic(err)
	}

	var file ConfigFile
	err = yaml.Unmarshal(data, &file)
	panicIf(err)

	var envCfg Options
	if o.Env != "" {
		var ok bool
		envCfg, ok = file[o.Env]
		if !ok {
			panic("Config for specified env not found")
		}
	}

	defCfg, _ := file["default"]

	panicIf(mergo.Merge(o, defCfg))
	panicIf(mergo.Merge(o, envCfg))
}

func addAWSConfig(o *Options) {
	if o.AWSKey == "" && o.AWSSecret == "" {
		o.AWSKey, o.AWSSecret = loadAWSConfig()
	}
}

type AWSConfig struct {
	Default struct {
		AccessKey string `ini:"aws_access_key_id"`
		SecretKey string `ini:"aws_secret_access_key"`
	} `ini:"[default]"`
}

func loadAWSConfig() (access string, secret string) {
	cfg := AWSConfig{}

	path, err := homedir.Expand("~/.aws/config")
	if err != nil {
		return
	}

	content, err := ioutil.ReadFile(path)
	if err != nil {
		return
	}

	ini.Unmarshal(content, &cfg)

	return cfg.Default.AccessKey, cfg.Default.SecretKey
}

func copyFile(bucket *s3.Bucket, from string, to string, contentType string, maxAge int) {
	copyOpts := s3.CopyOptions{
		MetadataDirective: "REPLACE",
		ContentType:       contentType,
		Options: s3.Options{
			CacheControl:    fmt.Sprintf("public, max-age=%d", maxAge),
			ContentEncoding: "gzip",
		},
	}

	_, err := bucket.PutCopy(to, s3.PublicRead, copyOpts, joinPath(bucket.Name, from))
	if err != nil {
		panic(err)
	}
}

var pathRe = regexp.MustCompile("/{2,}")

func joinPath(parts ...string) string {
	// Like filepath.Join, but always uses '/'
	return pathRe.ReplaceAllLiteralString(strings.Join(parts, "/"), "/")
}
<commit_msg>Fix filepath joining to cleanup paths<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/imdario/mergo"
	"github.com/mitchellh/go-homedir"
	"github.com/zackbloom/go-ini"
	"github.com/zackbloom/goamz/aws"
	"github.com/zackbloom/goamz/cloudfront"
	"github.com/zackbloom/goamz/iam"
	"github.com/zackbloom/goamz/route53"
	"github.com/zackbloom/goamz/s3"
	"gopkg.in/yaml.v1"
)

const (
	LIMITED = 60
	FOREVER = 31556926
)

var s3Session *s3.S3
var iamSession *iam.IAM
var r53Session *route53.Route53
var cfSession *cloudfront.CloudFront

func getRegion(region string) aws.Region {
	regionS, ok := aws.Regions[region]
	if !ok {
		panic("Region not found")
	}
	return regionS
}

func openS3(key, secret, region string) *s3.S3 {
	regionS := getRegion(region)

	auth := aws.Auth{
		AccessKey: key,
		SecretKey: secret,
	}
	return s3.New(auth, regionS)
}

func openIAM(key, secret, region string) *iam.IAM {
	regionS := getRegion(region)

	auth := aws.Auth{
		AccessKey: key,
		SecretKey: secret,
	}
	return iam.New(auth, regionS)
}

func openCloudFront(key, secret string) *cloudfront.CloudFront {
	auth := aws.Auth{
		AccessKey: key,
		SecretKey: secret,
	}
	return cloudfront.NewCloudFront(auth)
}

func openRoute53(key, secret string) *route53.Route53 {
	auth := aws.Auth{
		AccessKey: key,
		SecretKey: secret,
	}

	r53, _ := route53.NewRoute53(auth)
	return r53
}

func panicIf(err error) {
	if err != nil {
		panic(err)
	}
}
func must(val interface{}, err error) interface{} {
	if err != nil {
		panic(err)
	}

	return val
}
func mustString(val string, err error) string {
	panicIf(err)
	return val
}
func mustInt(val int, err error) int {
	panicIf(err)
	return val
}

type Options struct {
	Files      string `yaml:"files"`
	Root       string `yaml:"root"`
	Dest       string `yaml:"dest"`
	ConfigFile string `yaml:"-"`
	Env        string `yaml:"-"`
	Bucket     string `yaml:"bucket"`
	AWSKey     string `yaml:"key"`
	AWSSecret  string `yaml:"secret"`
	AWSRegion  string `yaml:"region"`
	NoUser     bool   `yaml:"-"`
}

func parseOptions() (o Options, set *flag.FlagSet) {
	set = flag.NewFlagSet(os.Args[1], flag.ExitOnError)
	//TODO: Set set.Usage

	set.StringVar(&o.Files, "files", "*", "Comma-seperated glob patterns of files to deploy (within root)")
	set.StringVar(&o.Root, "root", "./", "The local directory to deploy")
	set.StringVar(&o.Dest, "dest", "./", "The destination directory to write files to in the S3 bucket")
	set.StringVar(&o.ConfigFile, "config", "", "A yaml file to read configuration from")
	set.StringVar(&o.Env, "env", "", "The env to read from the config file")
	set.StringVar(&o.Bucket, "bucket", "", "The bucket to deploy to")
	set.StringVar(&o.AWSKey, "key", "", "The AWS key to use")
	set.StringVar(&o.AWSSecret, "secret", "", "The AWS secret of the provided key")
	set.StringVar(&o.AWSRegion, "region", "us-east-1", "The AWS region the S3 bucket is in")
	set.BoolVar(&o.NoUser, "no-user", false, "When creating, should we make a user account?")

	set.Parse(os.Args[2:])

	return
}

type ConfigFile map[string]Options

func loadConfigFile(o *Options) {
	isDefault := false
	configPath := o.ConfigFile
	if o.ConfigFile == "" {
		isDefault = true
		configPath = "./deploy.yaml"
	}

	data, err := ioutil.ReadFile(configPath)
	if err != nil {
		if os.IsNotExist(err) && isDefault {
			return
		}

		panic(err)
	}

	var file ConfigFile
	err = yaml.Unmarshal(data, &file)
	panicIf(err)

	var envCfg Options
	if o.Env != "" {
		var ok bool
		envCfg, ok = file[o.Env]
		if !ok {
			panic("Config for specified env not found")
		}
	}

	defCfg, _ := file["default"]

	panicIf(mergo.Merge(o, defCfg))
	panicIf(mergo.Merge(o, envCfg))
}

func addAWSConfig(o *Options) {
	if o.AWSKey == "" && o.AWSSecret == "" {
		o.AWSKey, o.AWSSecret = loadAWSConfig()
	}
}

type AWSConfig struct {
	Default struct {
		AccessKey string `ini:"aws_access_key_id"`
		SecretKey string `ini:"aws_secret_access_key"`
	} `ini:"[default]"`
}

func loadAWSConfig() (access string, secret string) {
	cfg := AWSConfig{}

	path, err := homedir.Expand("~/.aws/config")
	if err != nil {
		return
	}

	content, err := ioutil.ReadFile(path)
	if err != nil {
		return
	}

	ini.Unmarshal(content, &cfg)

	return cfg.Default.AccessKey, cfg.Default.SecretKey
}

func copyFile(bucket *s3.Bucket, from string, to string, contentType string, maxAge int) {
	copyOpts := s3.CopyOptions{
		MetadataDirective: "REPLACE",
		ContentType:       contentType,
		Options: s3.Options{
			CacheControl:    fmt.Sprintf("public, max-age=%d", maxAge),
			ContentEncoding: "gzip",
		},
	}

	_, err := bucket.PutCopy(to, s3.PublicRead, copyOpts, joinPath(bucket.Name, from))
	if err != nil {
		panic(err)
	}
}

var pathRe = regexp.MustCompile("/{2,}")

func joinPath(parts ...string) string {
	// Like filepath.Join, but always uses '/'
	out := filepath.Join(parts...)

	if os.PathSeparator != '/' {
		out = strings.Replace(out, string(os.PathSeparator), "/", -1)
	}

	return out
}
<|endoftext|>


<commit_before>package backendutil

import (
	"bytes"
	"io"
	"io/ioutil"
	"strings"
	"time"

	"github.com/emersion/go-imap"
	"github.com/emersion/go-message"
	"github.com/emersion/go-message/mail"
)

func matchString(s, substr string) bool {
	return strings.Contains(strings.ToLower(s), strings.ToLower(substr))
}

// Match returns true if a message matches the provided criteria. Flag, sequence
// number and UID contrainsts are not checked.
func Match(e *message.Entity, c *imap.SearchCriteria) (bool, error) {
	// TODO: support encoded header fields for Bcc, Cc, From, To
	// TODO: add header size for Larger and Smaller

	h := mail.Header{e.Header}

	// TODO: optimize this
	b, err := ioutil.ReadAll(e.Body)
	if err != nil {
		return false, err
	}
	s := string(b)
	br := bytes.NewReader(b)
	e.Body = br

	if c.Bcc != "" && !matchString(h.Get("Bcc"), c.Bcc) {
		return false, nil
	}
	if !c.Before.IsZero() {
		if t, err := h.Date(); err != nil {
			return false, err
		} else if !t.Before(c.Before) {
			return false, nil
		}
	}
	if c.Body != "" && !matchString(s, c.Body) {
		return false, nil
	}
	if c.Cc != "" && !matchString(h.Get("Cc"), c.Cc) {
		return false, nil
	}
	if c.From != "" && !matchString(h.Get("From"), c.From) {
		return false, nil
	}
	if c.Header[0] != "" {
		key, value := c.Header[0], c.Header[1]
		values, ok := e.Header[key]
		if value == "" && !ok {
			return false, nil
		}
		if value != "" {
			ok := false
			for _, v := range values {
				if matchString(v, value) {
					ok = true
					break
				}
			}
			if !ok {
				return false, nil
			}
		}
	}
	if c.Larger > 0 && uint32(len(b)) < c.Larger {
		return false, nil
	}
	if c.Not != nil {
		ok, err := Match(e, c.Not)
		br.Seek(0, io.SeekStart)
		if err != nil || ok {
			return false, err
		}
	}
	if !c.On.IsZero() {
		if t, err := h.Date(); err != nil {
			return false, err
		} else if !t.Round(24 * time.Hour).Equal(c.On) {
			return false, nil
		}
	}
	if c.Or[0] != nil && c.Or[1] != nil {
		ok1, err := Match(e, c.Or[0])
		br.Seek(0, io.SeekStart)
		if err != nil {
			return ok1, err
		}

		ok2, err := Match(e, c.Or[1])
		br.Seek(0, io.SeekStart)
		if err != nil || (!ok1 && !ok2) {
			return false, err
		}
	}
	if !c.Since.IsZero() {
		if t, err := h.Date(); err != nil {
			return false, err
		} else if !t.After(c.Since) {
			return false, nil
		}
	}
	if c.Smaller > 0 && uint32(len(b)) > c.Smaller {
		return false, nil
	}
	if c.Subject != "" {
		if subject, err := h.Subject(); err != nil {
			return false, err
		} else if !matchString(subject, c.Subject) {
			return false, nil
		}
	}
	if c.Text != "" && !matchString(s, c.Text) {
		return false, nil // TODO: also search in header fields
	}
	if c.To != "" && !matchString(h.Get("To"), c.To) {
		return false, nil
	}

	return true, nil
}

func criteriaFlags(c *imap.SearchCriteria) (want, dontWant []string) {
	if c.Answered {
		want = append(want, imap.AnsweredFlag)
	}
	if c.Deleted {
		want = append(want, imap.DeletedFlag)
	}
	if c.Draft {
		want = append(want, imap.DraftFlag)
	}
	if c.Flagged {
		want = append(want, imap.FlaggedFlag)
	}
	if c.Keyword != "" {
		want = append(want, c.Keyword)
	}
	if c.New {
		want = append(want, imap.RecentFlag)
		dontWant = append(dontWant, imap.SeenFlag)
	}
	if c.Old {
		dontWant = append(dontWant, imap.RecentFlag)
	}
	if c.Recent {
		want = append(want, imap.RecentFlag)
	}
	if c.Seen {
		want = append(want, imap.SeenFlag)
	}
	if c.Unanswered {
		dontWant = append(dontWant, imap.AnsweredFlag)
	}
	if c.Undeleted {
		dontWant = append(dontWant, imap.DeletedFlag)
	}
	if c.Undraft {
		dontWant = append(dontWant, imap.DraftFlag)
	}
	if c.Unflagged {
		dontWant = append(dontWant, imap.FlaggedFlag)
	}
	if c.Unkeyword != "" {
		dontWant = append(dontWant, c.Unkeyword)
	}
	if c.Unseen {
		dontWant = append(dontWant, imap.SeenFlag)
	}
	return
}

func matchFlags(flags map[string]bool, c *imap.SearchCriteria) bool {
	want, dontWant := criteriaFlags(c)
	for _, f := range want {
		if !flags[f] {
			return false
		}
	}
	for _, f := range dontWant {
		if flags[f] {
			return false
		}
	}

	if c.Not != nil {
		if matchFlags(flags, c.Not) {
			return false
		}
	}

	if c.Or[0] != nil && c.Or[1] != nil {
		if !matchFlags(flags, c.Or[0]) && !matchFlags(flags, c.Or[1]) {
			return false
		}
	}

	return true
}

// MatchFlags returns true if a flag list matches the provided criteria.
func MatchFlags(flags []string, c *imap.SearchCriteria) bool {
	flagsMap := make(map[string]bool)
	for _, f := range flags {
		flagsMap[f] = true
	}

	return matchFlags(flagsMap, c)
}
<commit_msg>backendutil: adds MatchSeqNumAndUid<commit_after>package backendutil

import (
	"bytes"
	"io"
	"io/ioutil"
	"strings"
	"time"

	"github.com/emersion/go-imap"
	"github.com/emersion/go-message"
	"github.com/emersion/go-message/mail"
)

func matchString(s, substr string) bool {
	return strings.Contains(strings.ToLower(s), strings.ToLower(substr))
}

// Match returns true if a message matches the provided criteria. Flag, sequence
// number and UID contrainsts are not checked.
func Match(e *message.Entity, c *imap.SearchCriteria) (bool, error) {
	// TODO: support encoded header fields for Bcc, Cc, From, To
	// TODO: add header size for Larger and Smaller

	h := mail.Header{e.Header}

	// TODO: optimize this
	b, err := ioutil.ReadAll(e.Body)
	if err != nil {
		return false, err
	}
	s := string(b)
	br := bytes.NewReader(b)
	e.Body = br

	if c.Bcc != "" && !matchString(h.Get("Bcc"), c.Bcc) {
		return false, nil
	}
	if !c.Before.IsZero() {
		if t, err := h.Date(); err != nil {
			return false, err
		} else if !t.Before(c.Before) {
			return false, nil
		}
	}
	if c.Body != "" && !matchString(s, c.Body) {
		return false, nil
	}
	if c.Cc != "" && !matchString(h.Get("Cc"), c.Cc) {
		return false, nil
	}
	if c.From != "" && !matchString(h.Get("From"), c.From) {
		return false, nil
	}
	if c.Header[0] != "" {
		key, value := c.Header[0], c.Header[1]
		values, ok := e.Header[key]
		if value == "" && !ok {
			return false, nil
		}
		if value != "" {
			ok := false
			for _, v := range values {
				if matchString(v, value) {
					ok = true
					break
				}
			}
			if !ok {
				return false, nil
			}
		}
	}
	if c.Larger > 0 && uint32(len(b)) < c.Larger {
		return false, nil
	}
	if c.Not != nil {
		ok, err := Match(e, c.Not)
		br.Seek(0, io.SeekStart)
		if err != nil || ok {
			return false, err
		}
	}
	if !c.On.IsZero() {
		if t, err := h.Date(); err != nil {
			return false, err
		} else if !t.Round(24 * time.Hour).Equal(c.On) {
			return false, nil
		}
	}
	if c.Or[0] != nil && c.Or[1] != nil {
		ok1, err := Match(e, c.Or[0])
		br.Seek(0, io.SeekStart)
		if err != nil {
			return ok1, err
		}

		ok2, err := Match(e, c.Or[1])
		br.Seek(0, io.SeekStart)
		if err != nil || (!ok1 && !ok2) {
			return false, err
		}
	}
	if !c.Since.IsZero() {
		if t, err := h.Date(); err != nil {
			return false, err
		} else if !t.After(c.Since) {
			return false, nil
		}
	}
	if c.Smaller > 0 && uint32(len(b)) > c.Smaller {
		return false, nil
	}
	if c.Subject != "" {
		if subject, err := h.Subject(); err != nil {
			return false, err
		} else if !matchString(subject, c.Subject) {
			return false, nil
		}
	}
	if c.Text != "" && !matchString(s, c.Text) {
		return false, nil // TODO: also search in header fields
	}
	if c.To != "" && !matchString(h.Get("To"), c.To) {
		return false, nil
	}

	return true, nil
}

func criteriaFlags(c *imap.SearchCriteria) (want, dontWant []string) {
	if c.Answered {
		want = append(want, imap.AnsweredFlag)
	}
	if c.Deleted {
		want = append(want, imap.DeletedFlag)
	}
	if c.Draft {
		want = append(want, imap.DraftFlag)
	}
	if c.Flagged {
		want = append(want, imap.FlaggedFlag)
	}
	if c.Keyword != "" {
		want = append(want, c.Keyword)
	}
	if c.New {
		want = append(want, imap.RecentFlag)
		dontWant = append(dontWant, imap.SeenFlag)
	}
	if c.Old {
		dontWant = append(dontWant, imap.RecentFlag)
	}
	if c.Recent {
		want = append(want, imap.RecentFlag)
	}
	if c.Seen {
		want = append(want, imap.SeenFlag)
	}
	if c.Unanswered {
		dontWant = append(dontWant, imap.AnsweredFlag)
	}
	if c.Undeleted {
		dontWant = append(dontWant, imap.DeletedFlag)
	}
	if c.Undraft {
		dontWant = append(dontWant, imap.DraftFlag)
	}
	if c.Unflagged {
		dontWant = append(dontWant, imap.FlaggedFlag)
	}
	if c.Unkeyword != "" {
		dontWant = append(dontWant, c.Unkeyword)
	}
	if c.Unseen {
		dontWant = append(dontWant, imap.SeenFlag)
	}
	return
}

func matchFlags(flags map[string]bool, c *imap.SearchCriteria) bool {
	want, dontWant := criteriaFlags(c)
	for _, f := range want {
		if !flags[f] {
			return false
		}
	}
	for _, f := range dontWant {
		if flags[f] {
			return false
		}
	}

	if c.Not != nil {
		if matchFlags(flags, c.Not) {
			return false
		}
	}

	if c.Or[0] != nil && c.Or[1] != nil {
		if !matchFlags(flags, c.Or[0]) && !matchFlags(flags, c.Or[1]) {
			return false
		}
	}

	return true
}

// MatchFlags returns true if a flag list matches the provided criteria.
func MatchFlags(flags []string, c *imap.SearchCriteria) bool {
	flagsMap := make(map[string]bool)
	for _, f := range flags {
		flagsMap[f] = true
	}

	return matchFlags(flagsMap, c)
}

// MatchSeqNumAndUid returns true if a sequence number and a UID matches the
// provided criteria.
func MatchSeqNumAndUid(seqNum uint32, uid uint32, c *imap.SearchCriteria) bool {
	if c.SeqSet != nil && !c.SeqSet.Contains(seqNum) {
		return false
	}
	if c.Uid != nil && !c.Uid.Contains(uid) {
		return false
	}

	if c.Not != nil && MatchSeqNumAndUid(seqNum, uid, c.Not) {
		return false
	}

	if c.Or[0] != nil && c.Or[1] != nil {
		if !MatchSeqNumAndUid(seqNum, uid, c.Or[0]) && !MatchSeqNumAndUid(seqNum, uid, c.Or[1]) {
			return false
		}
	}

	return true
}
<|endoftext|>


<commit_before>package docker

import (
	"bufio"
	"context"
	"github.com/docker/docker/api/types"
	"github.com/gorilla/websocket"
	"log"
	"net/http"
	"regexp"
)

const ignoredByteLogSize = 8

var logWebsocketRequest = regexp.MustCompile(`/containers/([^/]+)/logs`)
var hostCheck = regexp.MustCompile(`vibioh\.fr$`)

var upgrader = websocket.Upgrader{
	ReadBufferSize:  1024,
	WriteBufferSize: 1024,
	CheckOrigin: func(r *http.Request) bool {
		return hostCheck.MatchString(r.Host)
	},
}

func logsContainerWebsocketHandler(w http.ResponseWriter, r *http.Request, containerID []byte) {
	ws, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		log.Print(err)
		return
	}

	defer func() {
		ws.Close()
		log.Print(`Websocket logging ended`)
	}()

	_, basicAuth, err := ws.ReadMessage()
	if err != nil {
		log.Print(err)
		return
	}

	if _, err := isAuthenticatedByBasicAuth(string(basicAuth)); err != nil {
		ws.WriteMessage(websocket.TextMessage, []byte(err.Error()))
		return
	}

	logs, err := docker.ContainerLogs(context.Background(), string(containerID), types.ContainerLogsOptions{ShowStdout: true, ShowStderr: true, Follow: true})
	if err != nil {
		log.Print(err)
		return
	}

	defer logs.Close()

	go func() {
		scanner := bufio.NewScanner(logs)
		for scanner.Scan() {
			if err = ws.WriteMessage(websocket.TextMessage, scanner.Bytes()[ignoredByteLogSize:]); err != nil {
				log.Print(err)
				return
			}
		}
	}()

	for {
		_, action, err := ws.ReadMessage()
		if err != nil {
			log.Print(err)
			return
		}

		if string(action) == `close` {
			return
		}
	}
}

func handleWebsocket(w http.ResponseWriter, r *http.Request) {
	urlPath := []byte(r.URL.Path)

	if logWebsocketRequest.Match(urlPath) {
		logsContainerWebsocketHandler(w, r, logWebsocketRequest.FindSubmatch(urlPath)[1])
	}
}
<commit_msg>Update websocket.go<commit_after>package docker

import (
	"bufio"
	"context"
	"github.com/docker/docker/api/types"
	"github.com/gorilla/websocket"
	"log"
	"net/http"
	"regexp"
)

const ignoredByteLogSize = 8
const closeMessage = `close`

var logWebsocketRequest = regexp.MustCompile(`/containers/([^/]+)/logs`)
var hostCheck = regexp.MustCompile(`vibioh\.fr$`)

var upgrader = websocket.Upgrader{
	ReadBufferSize:  1024,
	WriteBufferSize: 1024,
	CheckOrigin: func(r *http.Request) bool {
		return hostCheck.MatchString(r.Host)
	},
}

func logsContainerWebsocketHandler(w http.ResponseWriter, r *http.Request, containerID []byte) {
	ws, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		log.Print(err)
		return
	}

	defer ws.Close()

	_, basicAuth, err := ws.ReadMessage()
	if err != nil {
		log.Print(err)
		return
	}

	if _, err := isAuthenticatedByBasicAuth(string(basicAuth)); err != nil {
		ws.WriteMessage(websocket.TextMessage, []byte(err.Error()))
		return
	}

	logs, err := docker.ContainerLogs(context.Background(), string(containerID), types.ContainerLogsOptions{ShowStdout: true, ShowStderr: true, Follow: true})
	if err != nil {
		log.Print(err)
		return
	}

	defer logs.Close()

	go func() {
		scanner := bufio.NewScanner(logs)
		for scanner.Scan() {
			if err = ws.WriteMessage(websocket.TextMessage, scanner.Bytes()[ignoredByteLogSize:]); err != nil {
				log.Print(err)
				return
			}
		}
	}()

	for {
		_, action, err := ws.ReadMessage()
		if err != nil {
			log.Print(err)
			return
		}

		if string(action) == closeMessage {
			return
		}
	}
}

func handleWebsocket(w http.ResponseWriter, r *http.Request) {
	urlPath := []byte(r.URL.Path)

	if logWebsocketRequest.Match(urlPath) {
		logsContainerWebsocketHandler(w, r, logWebsocketRequest.FindSubmatch(urlPath)[1])
	}
}
<|endoftext|>


<commit_before>package mozillaEvaluationWorker

var ServerSideTLSConfiguration = `{
    "configurations": {
        "modern": {
            "openssl_ciphersuites": "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256",
            "ciphersuites": [
                "ECDHE-ECDSA-AES256-GCM-SHA384",
                "ECDHE-RSA-AES256-GCM-SHA384",
                "ECDHE-ECDSA-CHACHA20-POLY1305",
                "ECDHE-RSA-CHACHA20-POLY1305",
                "ECDHE-ECDSA-AES128-GCM-SHA256",
                "ECDHE-RSA-AES128-GCM-SHA256",
                "ECDHE-ECDSA-AES256-SHA384",
                "ECDHE-RSA-AES256-SHA384",
                "ECDHE-ECDSA-AES128-SHA256",
                "ECDHE-RSA-AES128-SHA256"
            ],
            "tls_versions": ["TLSv1.2" ],
            "tls_curves": [ "prime256v1", "secp384r1", "secp521r1" ],
            "certificate_types": ["ecdsa"],
            "certificate_curves": ["prime256v1", "secp384r1", "secp521r1"],
            "certificate_signatures": ["sha256WithRSAEncryption", "ecdsa-with-SHA256", "ecdsa-with-SHA384", "ecdsa-with-SHA512"],
            "rsa_key_size": 2048,
            "dh_param_size": null,
            "ecdh_param_size": 256,
            "hsts_min_age": 15768000,
            "oldest_clients": [ "Firefox 27", "Chrome 22", "IE 11", "Opera 14", "Safari 7", "Android 4.4", "Java 8", "Windows Vista"]
        },
        "intermediate": {
            "openssl_ciphersuites": "ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS",
            "ciphersuites": [
                "ECDHE-ECDSA-CHACHA20-POLY1305",
                "ECDHE-RSA-CHACHA20-POLY1305",
                "ECDHE-ECDSA-AES128-GCM-SHA256",
                "ECDHE-RSA-AES128-GCM-SHA256",
                "ECDHE-ECDSA-AES256-GCM-SHA384",
                "ECDHE-RSA-AES256-GCM-SHA384",
                "DHE-RSA-AES128-GCM-SHA256",
                "DHE-RSA-AES256-GCM-SHA384",
                "ECDHE-ECDSA-AES128-SHA256",
                "ECDHE-RSA-AES128-SHA256",
                "ECDHE-ECDSA-AES128-SHA",
                "ECDHE-RSA-AES256-SHA384",
                "ECDHE-RSA-AES128-SHA",
                "ECDHE-ECDSA-AES256-SHA384",
                "ECDHE-ECDSA-AES256-SHA",
                "ECDHE-RSA-AES256-SHA",
                "DHE-RSA-AES128-SHA256",
                "DHE-RSA-AES128-SHA",
                "DHE-RSA-AES256-SHA256",
                "DHE-RSA-AES256-SHA",
                "ECDHE-ECDSA-DES-CBC3-SHA",
                "ECDHE-RSA-DES-CBC3-SHA",
                "EDH-RSA-DES-CBC3-SHA",
                "AES128-GCM-SHA256",
                "AES256-GCM-SHA384",
                "AES128-SHA256",
                "AES256-SHA256",
                "AES128-SHA",
                "AES256-SHA",
                "DES-CBC3-SHA"
            ],
            "tls_versions": ["TLSv1.2", "TLSv1.1", "TLSv1" ],
            "tls_curves": [ "secp256r1", "secp384r1", "secp521r1" ],
            "certificate_types": ["rsa"],
            "certificate_curves": null,
            "certificate_signatures": ["sha256WithRSAEncryption"],
            "rsa_key_size": 2048,
            "dh_param_size": 2048,
            "ecdh_param_size": 256,
            "hsts_min_age": 15768000,
            "oldest_clients": [ "Firefox 1", "Chrome 1", "IE 7", "Opera 5", "Safari 1", "Windows XP IE8", "Android 2.3", "Java 7" ]
        },
        "old": {
            "openssl_ciphersuites": "ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:ECDHE-RSA-DES-CBC3-SHA:ECDHE-ECDSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:DES-CBC3-SHA:HIGH:SEED:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!RSAPSK:!aDH:!aECDH:!EDH-DSS-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA:!SRP",
            "ciphersuites": [
                "ECDHE-ECDSA-CHACHA20-POLY1305",
                "ECDHE-RSA-CHACHA20-POLY1305",
                "ECDHE-RSA-AES128-GCM-SHA256",
                "ECDHE-ECDSA-AES128-GCM-SHA256",
                "ECDHE-RSA-AES256-GCM-SHA384",
                "ECDHE-ECDSA-AES256-GCM-SHA384",
                "DHE-RSA-AES128-GCM-SHA256",
                "DHE-DSS-AES128-GCM-SHA256",
                "DHE-DSS-AES256-GCM-SHA384",
                "DHE-RSA-AES256-GCM-SHA384",
                "ECDHE-RSA-AES128-SHA256",
                "ECDHE-ECDSA-AES128-SHA256",
                "ECDHE-RSA-AES128-SHA",
                "ECDHE-ECDSA-AES128-SHA",
                "ECDHE-RSA-AES256-SHA384",
                "ECDHE-ECDSA-AES256-SHA384",
                "ECDHE-RSA-AES256-SHA",
                "ECDHE-ECDSA-AES256-SHA",
                "DHE-RSA-AES128-SHA256",
                "DHE-RSA-AES128-SHA",
                "DHE-DSS-AES128-SHA256",
                "DHE-RSA-AES256-SHA256",
                "DHE-DSS-AES256-SHA",
                "DHE-RSA-AES256-SHA",
                "ECDHE-RSA-DES-CBC3-SHA",
                "ECDHE-ECDSA-DES-CBC3-SHA",
                "EDH-RSA-DES-CBC3-SHA",
                "AES128-GCM-SHA256",
                "AES256-GCM-SHA384",
                "AES128-SHA256",
                "AES256-SHA256",
                "AES128-SHA",
                "AES256-SHA",
                "DHE-DSS-AES256-SHA256",
                "DHE-DSS-AES128-SHA",
                "DES-CBC3-SHA",
                "DHE-RSA-CHACHA20-POLY1305",
                "ECDHE-RSA-CAMELLIA256-SHA384",
                "ECDHE-ECDSA-CAMELLIA256-SHA384",
                "DHE-RSA-CAMELLIA256-SHA256",
                "DHE-DSS-CAMELLIA256-SHA256",
                "DHE-RSA-CAMELLIA256-SHA",
                "DHE-DSS-CAMELLIA256-SHA",
                "CAMELLIA256-SHA256",
                "CAMELLIA256-SHA",
                "ECDHE-RSA-CAMELLIA128-SHA256",
                "ECDHE-ECDSA-CAMELLIA128-SHA256",
                "DHE-RSA-CAMELLIA128-SHA256",
                "DHE-DSS-CAMELLIA128-SHA256",
                "DHE-RSA-CAMELLIA128-SHA",
                "DHE-DSS-CAMELLIA128-SHA",
                "CAMELLIA128-SHA256",
                "CAMELLIA128-SHA",
                "DHE-RSA-SEED-SHA",
                "DHE-DSS-SEED-SHA",
                "SEED-SHA"
            ],
            "tls_versions": ["TLSv1.2", "TLSv1.1", "TLSv1", "SSLv3" ],
            "tls_curves": [ "secp256r1", "secp384r1", "secp521r1" ],
            "certificate_types": ["rsa"],
            "certificate_curves": null,
            "certificate_signatures": ["sha1WithRSAEncryption"],
            "rsa_key_size": 2048,
            "dh_param_size": 1024,
            "ecdh_param_size": 160,
            "hsts_min_age": 15768000,
            "oldest_clients": [ "Firefox 1", "Chrome 1", "Windows XP IE 6", "Opera 4", "Safari 1", "Java 6" ]
        }
    },
    "version": 4.0
}`
<commit_msg>updated local tls configurations ( to be used in the bad case of fallback )<commit_after>package mozillaEvaluationWorker

//saved TLS configurations
var ServerSideTLSConfiguration = `{
    "configurations": {
        "modern": {
            "openssl_ciphersuites": "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256",
            "ciphersuites": [
                "ECDHE-ECDSA-AES256-GCM-SHA384",
                "ECDHE-RSA-AES256-GCM-SHA384",
                "ECDHE-ECDSA-CHACHA20-POLY1305",
                "ECDHE-RSA-CHACHA20-POLY1305",
                "ECDHE-ECDSA-AES128-GCM-SHA256",
                "ECDHE-RSA-AES128-GCM-SHA256",
                "ECDHE-ECDSA-AES256-SHA384",
                "ECDHE-RSA-AES256-SHA384",
                "ECDHE-ECDSA-AES128-SHA256",
                "ECDHE-RSA-AES128-SHA256"
            ],
            "tls_versions": ["TLSv1.2" ],
            "tls_curves": [ "prime256v1", "secp384r1", "secp521r1" ],
            "certificate_types": ["ecdsa"],
            "certificate_curves": ["prime256v1", "secp384r1", "secp521r1"],
            "certificate_signatures": ["sha256WithRSAEncryption", "ecdsa-with-SHA256", "ecdsa-with-SHA384", "ecdsa-with-SHA512"],
            "rsa_key_size": 2048,
            "dh_param_size": null,
            "ecdh_param_size": 256,
            "hsts_min_age": 15768000,
            "oldest_clients": [ "Firefox 27", "Chrome 30", "IE 11 on Windows 7", "Edge 1", "Opera 17", "Safari 9", "Android 5.0", "Java 8"]
        },
        "intermediate": {
            "openssl_ciphersuites": "ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS",
            "ciphersuites": [
                "ECDHE-ECDSA-CHACHA20-POLY1305",
                "ECDHE-RSA-CHACHA20-POLY1305",
                "ECDHE-ECDSA-AES128-GCM-SHA256",
                "ECDHE-RSA-AES128-GCM-SHA256",
                "ECDHE-ECDSA-AES256-GCM-SHA384",
                "ECDHE-RSA-AES256-GCM-SHA384",
                "DHE-RSA-AES128-GCM-SHA256",
                "DHE-RSA-AES256-GCM-SHA384",
                "ECDHE-ECDSA-AES128-SHA256",
                "ECDHE-RSA-AES128-SHA256",
                "ECDHE-ECDSA-AES128-SHA",
                "ECDHE-RSA-AES256-SHA384",
                "ECDHE-RSA-AES128-SHA",
                "ECDHE-ECDSA-AES256-SHA384",
                "ECDHE-ECDSA-AES256-SHA",
                "ECDHE-RSA-AES256-SHA",
                "DHE-RSA-AES128-SHA256",
                "DHE-RSA-AES128-SHA",
                "DHE-RSA-AES256-SHA256",
                "DHE-RSA-AES256-SHA",
                "ECDHE-ECDSA-DES-CBC3-SHA",
                "ECDHE-RSA-DES-CBC3-SHA",
                "EDH-RSA-DES-CBC3-SHA",
                "AES128-GCM-SHA256",
                "AES256-GCM-SHA384",
                "AES128-SHA256",
                "AES256-SHA256",
                "AES128-SHA",
                "AES256-SHA",
                "DES-CBC3-SHA"
            ],
            "tls_versions": ["TLSv1.2", "TLSv1.1", "TLSv1" ],
            "tls_curves": [ "secp256r1", "secp384r1", "secp521r1" ],
            "certificate_types": ["rsa"],
            "certificate_curves": null,
            "certificate_signatures": ["sha256WithRSAEncryption"],
            "rsa_key_size": 2048,
            "dh_param_size": 2048,
            "ecdh_param_size": 256,
            "hsts_min_age": 15768000,
            "oldest_clients": [ "Firefox 1", "Chrome 1", "IE 7", "Opera 5", "Safari 1", "Windows XP IE8", "Android 2.3", "Java 7" ]
        },
        "old": {
            "openssl_ciphersuites": "ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:ECDHE-RSA-DES-CBC3-SHA:ECDHE-ECDSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:DES-CBC3-SHA:HIGH:SEED:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!RSAPSK:!aDH:!aECDH:!EDH-DSS-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA:!SRP",
            "ciphersuites": [
                "ECDHE-ECDSA-CHACHA20-POLY1305",
                "ECDHE-RSA-CHACHA20-POLY1305",
                "ECDHE-RSA-AES128-GCM-SHA256",
                "ECDHE-ECDSA-AES128-GCM-SHA256",
                "ECDHE-RSA-AES256-GCM-SHA384",
                "ECDHE-ECDSA-AES256-GCM-SHA384",
                "DHE-RSA-AES128-GCM-SHA256",
                "DHE-DSS-AES128-GCM-SHA256",
                "DHE-DSS-AES256-GCM-SHA384",
                "DHE-RSA-AES256-GCM-SHA384",
                "ECDHE-RSA-AES128-SHA256",
                "ECDHE-ECDSA-AES128-SHA256",
                "ECDHE-RSA-AES128-SHA",
                "ECDHE-ECDSA-AES128-SHA",
                "ECDHE-RSA-AES256-SHA384",
                "ECDHE-ECDSA-AES256-SHA384",
                "ECDHE-RSA-AES256-SHA",
                "ECDHE-ECDSA-AES256-SHA",
                "DHE-RSA-AES128-SHA256",
                "DHE-RSA-AES128-SHA",
                "DHE-DSS-AES128-SHA256",
                "DHE-RSA-AES256-SHA256",
                "DHE-DSS-AES256-SHA",
                "DHE-RSA-AES256-SHA",
                "ECDHE-RSA-DES-CBC3-SHA",
                "ECDHE-ECDSA-DES-CBC3-SHA",
                "EDH-RSA-DES-CBC3-SHA",
                "AES128-GCM-SHA256",
                "AES256-GCM-SHA384",
                "AES128-SHA256",
                "AES256-SHA256",
                "AES128-SHA",
                "AES256-SHA",
                "DHE-DSS-AES256-SHA256",
                "DHE-DSS-AES128-SHA",
                "DES-CBC3-SHA",
                "DHE-RSA-CHACHA20-POLY1305",
                "ECDHE-RSA-CAMELLIA256-SHA384",
                "ECDHE-ECDSA-CAMELLIA256-SHA384",
                "DHE-RSA-CAMELLIA256-SHA256",
                "DHE-DSS-CAMELLIA256-SHA256",
                "DHE-RSA-CAMELLIA256-SHA",
                "DHE-DSS-CAMELLIA256-SHA",
                "CAMELLIA256-SHA256",
                "CAMELLIA256-SHA",
                "ECDHE-RSA-CAMELLIA128-SHA256",
                "ECDHE-ECDSA-CAMELLIA128-SHA256",
                "DHE-RSA-CAMELLIA128-SHA256",
                "DHE-DSS-CAMELLIA128-SHA256",
                "DHE-RSA-CAMELLIA128-SHA",
                "DHE-DSS-CAMELLIA128-SHA",
                "CAMELLIA128-SHA256",
                "CAMELLIA128-SHA",
                "DHE-RSA-SEED-SHA",
                "DHE-DSS-SEED-SHA",
                "SEED-SHA"
            ],
            "tls_versions": ["TLSv1.2", "TLSv1.1", "TLSv1", "SSLv3" ],
            "tls_curves": [ "secp256r1", "secp384r1", "secp521r1" ],
            "certificate_types": ["rsa"],
            "certificate_curves": null,
            "certificate_signatures": ["sha1WithRSAEncryption"],
            "rsa_key_size": 2048,
            "dh_param_size": 1024,
            "ecdh_param_size": 160,
            "hsts_min_age": 15768000,
            "oldest_clients": [ "Firefox 1", "Chrome 1", "Windows XP IE 6", "Opera 4", "Safari 1", "Java 6" ]
        }
    },
    "version": 4.0
}`
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"os/exec"
	"os/signal"
	"syscall"

	"github.com/squaremo/ambergreen/balancer"
	"github.com/squaremo/ambergreen/balancer/fatal"
)

func iptables(args []string) ([]byte, error) {
	return exec.Command("iptables", args...).CombinedOutput()
}

func main() {
	// Catch some signals for whcih we want to clean up on exit
	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)

	fatalSink := fatal.New()
	i := balancer.Start(os.Args, fatalSink, iptables)

	exitCode := 0
	var exitSignal os.Signal

	select {
	case err := <-fatalSink:
		fmt.Fprintln(os.Stderr, err)
		exitCode = 1
	case exitSignal = <-sigs:
		exitCode = 2
	}

	i.Stop()

	if sig, ok := exitSignal.(syscall.Signal); ok {
		// Now we have cleaned up, re-kill the process with
		// the signal in order to produce a signal exit
		// status:
		signal.Reset(sig)
		syscall.Kill(syscall.Getpid(), sig)
	}

	os.Exit(exitCode)
}
<commit_msg>Fix comment typo<commit_after>package main

import (
	"fmt"
	"os"
	"os/exec"
	"os/signal"
	"syscall"

	"github.com/squaremo/ambergreen/balancer"
	"github.com/squaremo/ambergreen/balancer/fatal"
)

func iptables(args []string) ([]byte, error) {
	return exec.Command("iptables", args...).CombinedOutput()
}

func main() {
	// Catch some signals for which we want to clean up on exit
	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)

	fatalSink := fatal.New()
	i := balancer.Start(os.Args, fatalSink, iptables)

	exitCode := 0
	var exitSignal os.Signal

	select {
	case err := <-fatalSink:
		fmt.Fprintln(os.Stderr, err)
		exitCode = 1
	case exitSignal = <-sigs:
		exitCode = 2
	}

	i.Stop()

	if sig, ok := exitSignal.(syscall.Signal); ok {
		// Now we have cleaned up, re-kill the process with
		// the signal in order to produce a signal exit
		// status:
		signal.Reset(sig)
		syscall.Kill(syscall.Getpid(), sig)
	}

	os.Exit(exitCode)
}
<|endoftext|>


<commit_before>package parse

import (
	"bufio"
	"bytes"
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"io"
	"os"
	"strings"
	"unicode"

	"code.google.com/p/go.tools/imports"
)

type isExported bool

var header = []byte(`

// This file was automatically generated by genny.
// Any changes will be lost if this file is regenerated.
// see https://github.com/metabition/genny

`)

var (
	packageKeyword = []byte("package")
	importKeyword  = []byte("import")
	openBrace      = []byte("(")
	closeBrace     = []byte(")")
	space          = " "
	genericPackage = "generic"
	genericType    = "generic.Type"
	genericNumber  = "generic.Number"
	linefeed       = "\r\n"
)
var unwantedLinePrefixes = [][]byte{
	[]byte("//go:generate genny"),
}

func generateSpecific(filename string, in io.ReadSeeker, typeSet map[string]string) ([]byte, error) {

	// ensure we are at the beginning of the file
	in.Seek(0, os.SEEK_SET)

	// parse the source file
	fs := token.NewFileSet()
	file, err := parser.ParseFile(fs, filename, in, 0)
	if err != nil {
		return nil, &errSource{Err: err}
	}

	// make sure every generic.Type is represented in the types
	// argument.
	for _, decl := range file.Decls {
		switch it := decl.(type) {
		case *ast.GenDecl:
			for _, spec := range it.Specs {
				ts, ok := spec.(*ast.TypeSpec)
				if !ok {
					continue
				}
				switch tt := ts.Type.(type) {
				case *ast.SelectorExpr:
					if name, ok := tt.X.(*ast.Ident); ok {
						if name.Name == genericPackage {
							if _, ok := typeSet[ts.Name.Name]; !ok {
								return nil, &errMissingSpecificType{GenericType: ts.Name.Name}
							}
						}
					}
				}
			}
		}
	}

	// go back to the start of the file
	in.Seek(0, os.SEEK_SET)

	var buf bytes.Buffer

	scanner := bufio.NewScanner(in)
	for scanner.Scan() {

		l := scanner.Text()

		// does this line contain generic.Type?
		if strings.Contains(l, genericType) || strings.Contains(l, genericNumber) {
			continue
		}

		for t, specificType := range typeSet {

			// does the line contain our type
			if strings.Contains(l, t) {

				var newLine string
				// check each word
				for _, word := range strings.Fields(l) {

					i := strings.Index(word, t) // find out where

					if i > -1 {

						// if this isn't an exact match
						if i > 0 && isAlphaNumeric(rune(word[i-1])) || i < len(word)-len(t) && isAlphaNumeric(rune(word[i+len(t)])) {
							// replace the word with a capitolized version
							word = strings.Replace(word, t, wordify(specificType, unicode.IsUpper(rune(strings.TrimLeft(word, "*&")[0]))), 1)
						} else {
							// replace the word as is
							word = strings.Replace(word, t, specificType, 1)
						}

					}

					newLine = newLine + word + space
				}
				l = newLine

			}
		}

		// write the line
		buf.WriteString(line(l))
	}

	// write it out
	return buf.Bytes(), nil
}

// Generics parses the source file and generates the bytes replacing the
// generic types for the keys map with the specific types (its value).
func Generics(filename string, in io.ReadSeeker, typeSets []map[string]string) ([]byte, error) {

	totalOutput := header

	for _, typeSet := range typeSets {

		// generate the specifics
		parsed, err := generateSpecific(filename, in, typeSet)
		if err != nil {
			return nil, err
		}

		totalOutput = append(totalOutput, parsed...)

	}

	// clean up the code line by line
	packageFound := false
	insideImportBlock := false
	var cleanOutputLines []string
	scanner := bufio.NewScanner(bytes.NewReader(totalOutput))
	for scanner.Scan() {

		// end of imports block?
		if insideImportBlock {
			if bytes.HasSuffix(scanner.Bytes(), closeBrace) {
				insideImportBlock = false
			}
			continue
		}

		if bytes.HasPrefix(scanner.Bytes(), packageKeyword) {
			if packageFound {
				continue
			} else {
				packageFound = true
			}
		} else if bytes.HasPrefix(scanner.Bytes(), importKeyword) {
			if bytes.HasSuffix(scanner.Bytes(), openBrace) {
				insideImportBlock = true
			}
			continue
		}

		// check all unwantedLinePrefixes - and skip them
		for _, prefix := range unwantedLinePrefixes {
			if bytes.HasPrefix(scanner.Bytes(), prefix) {
				continue
			}
		}

		cleanOutputLines = append(cleanOutputLines, line(scanner.Text()))
	}

	cleanOutput := strings.Join(cleanOutputLines, "")

	// fix the imports
	output, err := imports.Process(filename, []byte(cleanOutput), nil)
	if err != nil {
		return nil, &errImports{Err: err}
	}

	return output, nil
}

func line(s string) string {
	return fmt.Sprintln(strings.TrimRight(s, linefeed))
}

// isAlphaNumeric gets whether the rune is alphanumeric or _.
func isAlphaNumeric(r rune) bool {
	return r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r)
}

// wordify turns a type into a nice word for function and type
// names etc.
func wordify(s string, exported bool) string {
	s = strings.TrimLeft(s, "*&")
	if !exported {
		return s
	}
	return strings.ToUpper(string(s[0])) + s[1:]
}
<commit_msg>fixed bug<commit_after>package parse

import (
	"bufio"
	"bytes"
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"io"
	"os"
	"strings"
	"unicode"

	"code.google.com/p/go.tools/imports"
)

type isExported bool

var header = []byte(`

// This file was automatically generated by genny.
// Any changes will be lost if this file is regenerated.
// see https://github.com/metabition/genny

`)

var (
	packageKeyword = []byte("package")
	importKeyword  = []byte("import")
	openBrace      = []byte("(")
	closeBrace     = []byte(")")
	space          = " "
	genericPackage = "generic"
	genericType    = "generic.Type"
	genericNumber  = "generic.Number"
	linefeed       = "\r\n"
)
var unwantedLinePrefixes = [][]byte{
	[]byte("//go:generate genny "),
}

func generateSpecific(filename string, in io.ReadSeeker, typeSet map[string]string) ([]byte, error) {

	// ensure we are at the beginning of the file
	in.Seek(0, os.SEEK_SET)

	// parse the source file
	fs := token.NewFileSet()
	file, err := parser.ParseFile(fs, filename, in, 0)
	if err != nil {
		return nil, &errSource{Err: err}
	}

	// make sure every generic.Type is represented in the types
	// argument.
	for _, decl := range file.Decls {
		switch it := decl.(type) {
		case *ast.GenDecl:
			for _, spec := range it.Specs {
				ts, ok := spec.(*ast.TypeSpec)
				if !ok {
					continue
				}
				switch tt := ts.Type.(type) {
				case *ast.SelectorExpr:
					if name, ok := tt.X.(*ast.Ident); ok {
						if name.Name == genericPackage {
							if _, ok := typeSet[ts.Name.Name]; !ok {
								return nil, &errMissingSpecificType{GenericType: ts.Name.Name}
							}
						}
					}
				}
			}
		}
	}

	// go back to the start of the file
	in.Seek(0, os.SEEK_SET)

	var buf bytes.Buffer

	scanner := bufio.NewScanner(in)
	for scanner.Scan() {

		l := scanner.Text()

		// does this line contain generic.Type?
		if strings.Contains(l, genericType) || strings.Contains(l, genericNumber) {
			continue
		}

		for t, specificType := range typeSet {

			// does the line contain our type
			if strings.Contains(l, t) {

				var newLine string
				// check each word
				for _, word := range strings.Fields(l) {

					i := strings.Index(word, t) // find out where

					if i > -1 {

						// if this isn't an exact match
						if i > 0 && isAlphaNumeric(rune(word[i-1])) || i < len(word)-len(t) && isAlphaNumeric(rune(word[i+len(t)])) {
							// replace the word with a capitolized version
							word = strings.Replace(word, t, wordify(specificType, unicode.IsUpper(rune(strings.TrimLeft(word, "*&")[0]))), 1)
						} else {
							// replace the word as is
							word = strings.Replace(word, t, specificType, 1)
						}

					}

					newLine = newLine + word + space
				}
				l = newLine

			}
		}

		// write the line
		buf.WriteString(line(l))
	}

	// write it out
	return buf.Bytes(), nil
}

// Generics parses the source file and generates the bytes replacing the
// generic types for the keys map with the specific types (its value).
func Generics(filename string, in io.ReadSeeker, typeSets []map[string]string) ([]byte, error) {

	totalOutput := header

	for _, typeSet := range typeSets {

		// generate the specifics
		parsed, err := generateSpecific(filename, in, typeSet)
		if err != nil {
			return nil, err
		}

		totalOutput = append(totalOutput, parsed...)

	}

	// clean up the code line by line
	packageFound := false
	insideImportBlock := false
	var cleanOutputLines []string
	scanner := bufio.NewScanner(bytes.NewReader(totalOutput))
	for scanner.Scan() {

		// end of imports block?
		if insideImportBlock {
			if bytes.HasSuffix(scanner.Bytes(), closeBrace) {
				insideImportBlock = false
			}
			continue
		}

		if bytes.HasPrefix(scanner.Bytes(), packageKeyword) {
			if packageFound {
				continue
			} else {
				packageFound = true
			}
		} else if bytes.HasPrefix(scanner.Bytes(), importKeyword) {
			if bytes.HasSuffix(scanner.Bytes(), openBrace) {
				insideImportBlock = true
			}
			continue
		}

		// check all unwantedLinePrefixes - and skip them
		skipline := false
		for _, prefix := range unwantedLinePrefixes {
			if bytes.HasPrefix(scanner.Bytes(), prefix) {
				skipline = true
				continue
			}
		}

		if skipline {
			continue
		}

		cleanOutputLines = append(cleanOutputLines, line(scanner.Text()))
	}

	cleanOutput := strings.Join(cleanOutputLines, "")

	// fix the imports
	output, err := imports.Process(filename, []byte(cleanOutput), nil)
	if err != nil {
		return nil, &errImports{Err: err}
	}

	return output, nil
}

func line(s string) string {
	return fmt.Sprintln(strings.TrimRight(s, linefeed))
}

// isAlphaNumeric gets whether the rune is alphanumeric or _.
func isAlphaNumeric(r rune) bool {
	return r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r)
}

// wordify turns a type into a nice word for function and type
// names etc.
func wordify(s string, exported bool) string {
	s = strings.TrimLeft(s, "*&")
	if !exported {
		return s
	}
	return strings.ToUpper(string(s[0])) + s[1:]
}
<|endoftext|>


<commit_before>// go-rst - A reStructuredText parser for Go
// 2014 (c) The go-rst Authors
// MIT Licensed. See LICENSE for details.

package parse

import (
	"fmt"
	"github.com/demizer/go-elog"
)

type systemMessageLevel int

const (
	levelInfo systemMessageLevel = iota
	levelWarning
	levelError
	levelSevere
)

var systemMessageLevels = [...]string{
	"INFO",
	"WARNING",
	"ERROR",
	"SEVERE",
}

func (s systemMessageLevel) String() string {
	return systemMessageLevels[s]
}

type systemMessage struct {
	level  systemMessageLevel
	line   int
	source string
	items  []item
}

type sectionLevel struct {
	char     rune // The adornment character used to describe the section
	overline bool // The section contains an overline
	length   int  // The length of the adornment lines
}

type sectionLevels []sectionLevel

func (s *sectionLevels) String() string {
	var out string
	for lvl, sec := range *s {
		out += fmt.Sprintf("level: %d, rune: %q, overline: %t, length: %d\n",
			lvl+1, sec.char, sec.overline, sec.length)
	}
	return out
}

func (s *sectionLevels) Add(adornChar rune, overline bool, length int) int {
	lvl := s.Find(adornChar)
	if lvl > 0 {
		return lvl
	}
	*s = append(*s, sectionLevel{char: adornChar, overline: overline, length: length})
	return len(*s)
}

// Returns -1 if not found
func (s *sectionLevels) Find(adornChar rune) int {
	for lvl, sec := range *s {
		if sec.char == adornChar {
			return lvl + 1
		}
	}
	return -1
}

func (s *sectionLevels) Level() int {
	return len(*s)
}

func Parse(name, text string) (t *Tree, err error) {
	t = New(name)
	t.text = text
	_, err = t.Parse(text, t)
	return
}

func New(name string) *Tree {
	return &Tree{Name: name, sectionLevels: new(sectionLevels)}
}

type Tree struct {
	Name          string
	Document      *NodeList // The root node list
	text          string
	branch        *NodeList // The current branch to add nodes to
	lex           *lexer
	peekCount     int
	token         [3]item        // three-token look-ahead for parser.
	sectionLevel  int            // The current section level of parsing
	sectionLevels *sectionLevels // Encountered section levels
}

// startParse initializes the parser, using the lexer.
func (t *Tree) startParse(lex *lexer) {
	t.Document = nil
	t.lex = lex
}

// stopParse terminates parsing.
func (t *Tree) stopParse() {
	t.lex = nil
}

func (t *Tree) Parse(text string, treeSet *Tree) (tree *Tree, err error) {
	log.Debugln("Start")
	t.startParse(lex(t.Name, text))
	t.text = text
	t.parse(treeSet)
	log.Debugln("End")
	return t, nil
}

func (t *Tree) parse(tree *Tree) (next Node) {
	log.Debugln("Start")
	t.Document = newList()
	for t.peek().ElementType != itemEOF {
		switch token := t.next(); token.ElementType {
		case itemBlankLine:
			log.Debugln("Found itemBlankLine")
		case itemSectionAdornment:
			log.Debugln("Found itemSectionAdornment")
		case itemTitle:
			log.Debugln("Found itemTitle")
		case itemParagraph:
			log.Debugln("Found itemParagraph")
		}

		if len([]Node(*t.Document)) == 0 {
			t.Document.append(n)
		} else {
			t.branch.append(n)
		}
	}
	log.Debugln("End")
	return nil
}

// peek returns but does not consume the next token.
func (t *Tree) peek() item {
	if t.peekCount > 0 {
		return t.token[t.peekCount-1]

	}
	t.peekCount = 1
	t.token[0] = t.lex.nextItem()
	return t.token[0]
}
<commit_msg>parse.go: Add error methods to the Tree type<commit_after>// go-rst - A reStructuredText parser for Go
// 2014 (c) The go-rst Authors
// MIT Licensed. See LICENSE for details.

package parse

import (
	"fmt"
	"github.com/demizer/go-elog"
)

type systemMessageLevel int

const (
	levelInfo systemMessageLevel = iota
	levelWarning
	levelError
	levelSevere
)

var systemMessageLevels = [...]string{
	"INFO",
	"WARNING",
	"ERROR",
	"SEVERE",
}

func (s systemMessageLevel) String() string {
	return systemMessageLevels[s]
}

type systemMessage struct {
	level  systemMessageLevel
	line   int
	source string
	items  []item
}

type sectionLevel struct {
	char     rune // The adornment character used to describe the section
	overline bool // The section contains an overline
	length   int  // The length of the adornment lines
}

type sectionLevels []sectionLevel

func (s *sectionLevels) String() string {
	var out string
	for lvl, sec := range *s {
		out += fmt.Sprintf("level: %d, rune: %q, overline: %t, length: %d\n",
			lvl+1, sec.char, sec.overline, sec.length)
	}
	return out
}

func (s *sectionLevels) Add(adornChar rune, overline bool, length int) int {
	lvl := s.Find(adornChar)
	if lvl > 0 {
		return lvl
	}
	*s = append(*s, sectionLevel{char: adornChar, overline: overline, length: length})
	return len(*s)
}

// Returns -1 if not found
func (s *sectionLevels) Find(adornChar rune) int {
	for lvl, sec := range *s {
		if sec.char == adornChar {
			return lvl + 1
		}
	}
	return -1
}

func (s *sectionLevels) Level() int {
	return len(*s)
}

func Parse(name, text string) (t *Tree, err error) {
	t = New(name)
	t.text = text
	_, err = t.Parse(text, t)
	return
}

func New(name string) *Tree {
	return &Tree{Name: name, sectionLevels: new(sectionLevels)}
}

type Tree struct {
	Name          string
	Document      *NodeList // The root node list
	text          string
	branch        *NodeList // The current branch to add nodes to
	lex           *lexer
	peekCount     int
	token         [3]item        // three-token look-ahead for parser.
	sectionLevel  int            // The current section level of parsing
	sectionLevels *sectionLevels // Encountered section levels
}

func (t *Tree) errorf(format string, args ...interface{}) {
	t.Document = nil
	format = fmt.Sprintf("go-rst: %s:%d: %s", t.Name, t.lex.lineNumber(), format)
	panic(fmt.Errorf(format, args...))
}

func (t *Tree) error(err error) {
	t.errorf("%s", err)
}

// startParse initializes the parser, using the lexer.
func (t *Tree) startParse(lex *lexer) {
	t.Document = nil
	t.lex = lex
}

// stopParse terminates parsing.
func (t *Tree) stopParse() {
	t.lex = nil
}

func (t *Tree) Parse(text string, treeSet *Tree) (tree *Tree, err error) {
	log.Debugln("Start")
	t.startParse(lex(t.Name, text))
	t.text = text
	t.parse(treeSet)
	log.Debugln("End")
	return t, nil
}

func (t *Tree) parse(tree *Tree) (next Node) {
	log.Debugln("Start")
	t.Document = newList()
	for t.peek().ElementType != itemEOF {
		switch token := t.next(); token.ElementType {
		case itemBlankLine:
			log.Debugln("Found itemBlankLine")
		case itemSectionAdornment:
			log.Debugln("Found itemSectionAdornment")
		case itemTitle:
			log.Debugln("Found itemTitle")
		case itemParagraph:
			log.Debugln("Found itemParagraph")
		}

		if len([]Node(*t.Document)) == 0 {
			t.Document.append(n)
		} else {
			t.branch.append(n)
		}
	}
	log.Debugln("End")
	return nil
}

// peek returns but does not consume the next token.
func (t *Tree) peek() item {
	if t.peekCount > 0 {
		return t.token[t.peekCount-1]

	}
	t.peekCount = 1
	t.token[0] = t.lex.nextItem()
	return t.token[0]
}
<|endoftext|>


<commit_before>// Copyright 2015 Peter Mattis.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.

package linguist

import (
	"log"
	"regexp"
	"strings"

	"gopkg.in/yaml.v1"
)

// ShouldIgnoreFilename returns true if filename matches known files which
// typically should not be passed to LanguageByFilename.
//
// Right now, this simply calls IsVendored and IsDocumentation.
func ShouldIgnoreFilename(filename string) bool {
	return IsVendored(filename) || IsDocumentation(filename)
}

// ShouldIgnoreContents returns true if contents match known files which
// typically should not be passed to LangugeByContents.
//
// Right now, this simply calls IsBinary.
func ShouldIgnoreContents(contents []byte) bool {
	return IsBinary(contents)
}

var vendorRE *regexp.Regexp
var doxRE *regexp.Regexp

func init() {
	var regexps []string
	bytes := []byte(files["data/vendor.yaml"])
	if err := yaml.Unmarshal(bytes, &regexps); err != nil {
		log.Fatal(err)
		return
	}
	vendorRE = regexp.MustCompile(strings.Join(regexps, "|"))

	var moreregex []string
	bytes = []byte(files["data/documentation.yaml"])
	if err := yaml.Unmarshal(bytes, &moreregex); err != nil {
		log.Fatal(err)
		return
	}
	doxRE = regexp.MustCompile(strings.Join(moreregex, "|"))
}

// IsVendored returns true if path is considered "vendored" and should
// be excluded from statistics.
//
// See also the data/vendor.yaml file distributed with this package.
func IsVendored(path string) bool {
	return vendorRE.MatchString(path)
}

// IsDocumentation returns true if path is considered documentation.
//
// See also the data/documentation.yaml file distributed with this package.
func IsDocumentation(path string) bool {
	return doxRE.MatchString(path)
}

// IsBinary checks contents for known character escape codes which
// frequently show up in binary files but rarely (if ever) in text.
//
// Use this check before using DetectFromContents to reduce likelihood
// of passing binary data into it.
//
// NOTE(tso): preliminary testing on this method of checking for binary
// contents were promising, having fed a document consisting of all
// utf-8 codepoints from 0000 to FFFF with satisfactory results. Thanks
// to robpike.io/cmd/unicode:
// ```
// unicode -c $(seq 0 65535 | xargs printf "%04x ") | tr -d '\n' > unicode_test
// ```
//
// However, the intentional presence of character escape codes to throw
// this function off is entirely possible, as is, potentially, a binary
// file consisting entirely of the 4 exceptions to the rule for the first
// 512 bytes. It is also possible that more character escape codes need
// to be added.
//
// Further analysis and real world testing of this is required.
func IsBinary(contents []byte) (probably bool) {
	for _, b := range contents[:512] {
		if b < 32 {
			switch b {
			case 0:
				fallthrough
			case 9:
				fallthrough
			case 10:
				fallthrough
			case 13:
				continue
			default:
				return true
			}
		}
	}
	return false
}
<commit_msg>Fix index out of range error<commit_after>// Copyright 2015 Peter Mattis.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.

package linguist

import (
	"log"
	"regexp"
	"strings"

	"gopkg.in/yaml.v1"
)

// ShouldIgnoreFilename returns true if filename matches known files which
// typically should not be passed to LanguageByFilename.
//
// Right now, this simply calls IsVendored and IsDocumentation.
func ShouldIgnoreFilename(filename string) bool {
	return IsVendored(filename) || IsDocumentation(filename)
}

// ShouldIgnoreContents returns true if contents match known files which
// typically should not be passed to LangugeByContents.
//
// Right now, this simply calls IsBinary.
func ShouldIgnoreContents(contents []byte) bool {
	return IsBinary(contents)
}

var vendorRE *regexp.Regexp
var doxRE *regexp.Regexp

func init() {
	var regexps []string
	bytes := []byte(files["data/vendor.yaml"])
	if err := yaml.Unmarshal(bytes, &regexps); err != nil {
		log.Fatal(err)
		return
	}
	vendorRE = regexp.MustCompile(strings.Join(regexps, "|"))

	var moreregex []string
	bytes = []byte(files["data/documentation.yaml"])
	if err := yaml.Unmarshal(bytes, &moreregex); err != nil {
		log.Fatal(err)
		return
	}
	doxRE = regexp.MustCompile(strings.Join(moreregex, "|"))
}

// IsVendored returns true if path is considered "vendored" and should
// be excluded from statistics.
//
// See also the data/vendor.yaml file distributed with this package.
func IsVendored(path string) bool {
	return vendorRE.MatchString(path)
}

// IsDocumentation returns true if path is considered documentation.
//
// See also the data/documentation.yaml file distributed with this package.
func IsDocumentation(path string) bool {
	return doxRE.MatchString(path)
}

// IsBinary checks contents for known character escape codes which
// frequently show up in binary files but rarely (if ever) in text.
//
// Use this check before using DetectFromContents to reduce likelihood
// of passing binary data into it.
//
// NOTE(tso): preliminary testing on this method of checking for binary
// contents were promising, having fed a document consisting of all
// utf-8 codepoints from 0000 to FFFF with satisfactory results. Thanks
// to robpike.io/cmd/unicode:
// ```
// unicode -c $(seq 0 65535 | xargs printf "%04x ") | tr -d '\n' > unicode_test
// ```
//
// However, the intentional presence of character escape codes to throw
// this function off is entirely possible, as is, potentially, a binary
// file consisting entirely of the 4 exceptions to the rule for the first
// 512 bytes. It is also possible that more character escape codes need
// to be added.
//
// Further analysis and real world testing of this is required.
func IsBinary(contents []byte) (probably bool) {
	for n, b := range contents {
		if n >= 512 {
			break
		}
		if b < 32 {
			switch b {
			case 0:
				fallthrough
			case 9:
				fallthrough
			case 10:
				fallthrough
			case 13:
				continue
			default:
				return true
			}
		}
	}
	return false
}
<|endoftext|>


<commit_before>package hostess_test

import (
	"fmt"
	"github.com/cbednarski/hostess"
	"net"
	"testing"
)

func TestMakeSurrogateIP(t *testing.T) {
	original := net.ParseIP("127.0.0.1")
	expected1 := net.ParseIP("0.0.0.1")
	IP1 := hostess.MakeSurrogateIP(original)
	if !IP1.Equal(expected1) {
		t.Errorf("Expected %s to convert to %s; got %s", original, expected1, IP1)
	}

	expected2 := net.ParseIP("10.20.30.40")
	IP2 := hostess.MakeSurrogateIP(expected2)
	if !IP2.Equal(expected2) {
		t.Errorf("Expected %s to remain unchanged; got %s", expected2, IP2)
	}
}

func TestContainsDomainIp(t *testing.T) {
	hosts := hostess.NewHostlist()
	hosts.Add(hostess.NewHostname(domain, ip, false))
	hosts.Add(hostess.NewHostname("google.com", "8.8.8.8", true))

	if !hosts.ContainsDomain(domain) {
		t.Errorf("Expected to find %s", domain)
	}

	const extra_domain = "yahoo.com"
	if hosts.ContainsDomain(extra_domain) {
		t.Errorf("Did not expect to find %s", extra_domain)
	}

	var first_ip = net.ParseIP(ip)
	if !hosts.ContainsIP(first_ip) {
		t.Errorf("Expected to find %s", ip)
	}

	var extra_ip = net.ParseIP("1.2.3.4")
	if hosts.ContainsIP(extra_ip) {
		t.Errorf("Did not expect to find %s", extra_ip)
	}

	hostname := hostess.NewHostname(domain, ip, true)
	if !hosts.Contains(hostname) {
		t.Errorf("Expected to find %s", hostname)
	}

	extra_hostname := hostess.NewHostname("yahoo.com", "4.3.2.1", false)
	if hosts.Contains(extra_hostname) {
		t.Errorf("Did not expect to find %s", extra_hostname)
	}
}

func TestFormat(t *testing.T) {
	hosts := hostess.NewHostlist()
	hosts.Add(hostess.NewHostname(domain, ip, false))
	hosts.Add(hostess.NewHostname("google.com", "8.8.8.8", true))

	expected := `# 127.0.0.1 localhost
8.8.8.8 google.com
`
	if string(hosts.Format()) != expected {
		t.Error("Formatted hosts list is not formatted correctly")
	}
}

func TestRemove(t *testing.T) {
	hosts := hostess.NewHostlist()
	hosts.Add(hostess.NewHostname(domain, ip, false))
	hosts.Add(hostess.NewHostname("google.com", "8.8.8.8", true))

	hosts.Remove(1)
	if len(*hosts) > 1 {
		t.Errorf("Expected hostlist to have 1 item, found %d", len(*hosts))
	}

	if hosts.ContainsDomain("google.com") {
		t.Errorf("Expected not to find google.com")
	}
}

func TestRemoveDomain(t *testing.T) {
	hosts := hostess.NewHostlist()
	h1 := hostess.NewHostname("google.com", "127.0.0.1", false)
	h2 := hostess.NewHostname("google.com", "::1", true)
	hosts.Add(h1)
	hosts.Add(h2)

	hosts.RemoveDomainV("google.com", 4)
	if hosts.Contains(h1) {
		t.Error("Should not contain ipv4 hostname")
	}
	if !hosts.Contains(h2) {
		t.Error("Should still contain ipv6 hostname")
	}

	hosts.RemoveDomainV("google.com", 6)
	if len(*hosts) != 0 {
		t.Error("Should no longer contain any hostnames")
	}
}

func CheckIndexDomain(t *testing.T, index int, domain string, hosts *hostess.Hostlist) {
	if (*hosts)[index].Domain != domain {
		t.Errorf("Expected %s to be in position %d. Found: %s", domain, index, (*hosts)[index].FormatHuman())
	}
}

func TestSort(t *testing.T) {
	// Getting 100% coverage on this is kinda tricky. It's pretty close and
	// this is already too long.

	hosts := hostess.NewHostlist()
	hosts.Add(hostess.NewHostname("google.com", "8.8.8.8", true))
	hosts.Add(hostess.NewHostname("google3.com", "::1", true))
	hosts.Add(hostess.NewHostname(domain, ip, false))
	hosts.Add(hostess.NewHostname("google2.com", "8.8.4.4", true))
	hosts.Add(hostess.NewHostname("blah2", "10.20.1.1", true))
	hosts.Add(hostess.NewHostname("blah3", "10.20.1.1", true))
	hosts.Add(hostess.NewHostname("blah33", "10.20.1.1", true))
	hosts.Add(hostess.NewHostname("blah", "10.20.1.1", true))
	hosts.Add(hostess.NewHostname("hostname", "127.0.1.1", true))
	hosts.Add(hostess.NewHostname("devsite", "127.0.0.1", true))

	hosts.Sort()

	CheckIndexDomain(t, 0, "localhost", hosts)
	CheckIndexDomain(t, 1, "devsite", hosts)
	CheckIndexDomain(t, 2, "hostname", hosts)
	CheckIndexDomain(t, 3, "google2.com", hosts)
	CheckIndexDomain(t, 4, "google.com", hosts)
	CheckIndexDomain(t, 5, "blah", hosts)
	CheckIndexDomain(t, 6, "blah2", hosts)
	CheckIndexDomain(t, 7, "blah3", hosts)
	CheckIndexDomain(t, 8, "blah33", hosts)
	CheckIndexDomain(t, 9, "google3.com", hosts)

	fmt.Println(string(hosts.Format()))
}

func ExampleHostlist_1() {
	hosts := hostess.NewHostlist()
	hosts.Add(hostess.NewHostname("google.com", "127.0.0.1", false))
	hosts.Add(hostess.NewHostname("google.com", "::1", true))

	fmt.Printf("%s\n", hosts.Format())
	// Output:
	// # 127.0.0.1 google.com
	// ::1 google.com
}
<commit_msg>Removed debug output<commit_after>package hostess_test

import (
	"fmt"
	"github.com/cbednarski/hostess"
	"net"
	"testing"
)

func TestMakeSurrogateIP(t *testing.T) {
	original := net.ParseIP("127.0.0.1")
	expected1 := net.ParseIP("0.0.0.1")
	IP1 := hostess.MakeSurrogateIP(original)
	if !IP1.Equal(expected1) {
		t.Errorf("Expected %s to convert to %s; got %s", original, expected1, IP1)
	}

	expected2 := net.ParseIP("10.20.30.40")
	IP2 := hostess.MakeSurrogateIP(expected2)
	if !IP2.Equal(expected2) {
		t.Errorf("Expected %s to remain unchanged; got %s", expected2, IP2)
	}
}

func TestContainsDomainIp(t *testing.T) {
	hosts := hostess.NewHostlist()
	hosts.Add(hostess.NewHostname(domain, ip, false))
	hosts.Add(hostess.NewHostname("google.com", "8.8.8.8", true))

	if !hosts.ContainsDomain(domain) {
		t.Errorf("Expected to find %s", domain)
	}

	const extra_domain = "yahoo.com"
	if hosts.ContainsDomain(extra_domain) {
		t.Errorf("Did not expect to find %s", extra_domain)
	}

	var first_ip = net.ParseIP(ip)
	if !hosts.ContainsIP(first_ip) {
		t.Errorf("Expected to find %s", ip)
	}

	var extra_ip = net.ParseIP("1.2.3.4")
	if hosts.ContainsIP(extra_ip) {
		t.Errorf("Did not expect to find %s", extra_ip)
	}

	hostname := hostess.NewHostname(domain, ip, true)
	if !hosts.Contains(hostname) {
		t.Errorf("Expected to find %s", hostname)
	}

	extra_hostname := hostess.NewHostname("yahoo.com", "4.3.2.1", false)
	if hosts.Contains(extra_hostname) {
		t.Errorf("Did not expect to find %s", extra_hostname)
	}
}

func TestFormat(t *testing.T) {
	hosts := hostess.NewHostlist()
	hosts.Add(hostess.NewHostname(domain, ip, false))
	hosts.Add(hostess.NewHostname("google.com", "8.8.8.8", true))

	expected := `# 127.0.0.1 localhost
8.8.8.8 google.com
`
	if string(hosts.Format()) != expected {
		t.Error("Formatted hosts list is not formatted correctly")
	}
}

func TestRemove(t *testing.T) {
	hosts := hostess.NewHostlist()
	hosts.Add(hostess.NewHostname(domain, ip, false))
	hosts.Add(hostess.NewHostname("google.com", "8.8.8.8", true))

	hosts.Remove(1)
	if len(*hosts) > 1 {
		t.Errorf("Expected hostlist to have 1 item, found %d", len(*hosts))
	}

	if hosts.ContainsDomain("google.com") {
		t.Errorf("Expected not to find google.com")
	}
}

func TestRemoveDomain(t *testing.T) {
	hosts := hostess.NewHostlist()
	h1 := hostess.NewHostname("google.com", "127.0.0.1", false)
	h2 := hostess.NewHostname("google.com", "::1", true)
	hosts.Add(h1)
	hosts.Add(h2)

	hosts.RemoveDomainV("google.com", 4)
	if hosts.Contains(h1) {
		t.Error("Should not contain ipv4 hostname")
	}
	if !hosts.Contains(h2) {
		t.Error("Should still contain ipv6 hostname")
	}

	hosts.RemoveDomainV("google.com", 6)
	if len(*hosts) != 0 {
		t.Error("Should no longer contain any hostnames")
	}
}

func CheckIndexDomain(t *testing.T, index int, domain string, hosts *hostess.Hostlist) {
	if (*hosts)[index].Domain != domain {
		t.Errorf("Expected %s to be in position %d. Found: %s", domain, index, (*hosts)[index].FormatHuman())
	}
}

func TestSort(t *testing.T) {
	// Getting 100% coverage on this is kinda tricky. It's pretty close and
	// this is already too long.

	hosts := hostess.NewHostlist()
	hosts.Add(hostess.NewHostname("google.com", "8.8.8.8", true))
	hosts.Add(hostess.NewHostname("google3.com", "::1", true))
	hosts.Add(hostess.NewHostname(domain, ip, false))
	hosts.Add(hostess.NewHostname("google2.com", "8.8.4.4", true))
	hosts.Add(hostess.NewHostname("blah2", "10.20.1.1", true))
	hosts.Add(hostess.NewHostname("blah3", "10.20.1.1", true))
	hosts.Add(hostess.NewHostname("blah33", "10.20.1.1", true))
	hosts.Add(hostess.NewHostname("blah", "10.20.1.1", true))
	hosts.Add(hostess.NewHostname("hostname", "127.0.1.1", true))
	hosts.Add(hostess.NewHostname("devsite", "127.0.0.1", true))

	hosts.Sort()

	CheckIndexDomain(t, 0, "localhost", hosts)
	CheckIndexDomain(t, 1, "devsite", hosts)
	CheckIndexDomain(t, 2, "hostname", hosts)
	CheckIndexDomain(t, 3, "google2.com", hosts)
	CheckIndexDomain(t, 4, "google.com", hosts)
	CheckIndexDomain(t, 5, "blah", hosts)
	CheckIndexDomain(t, 6, "blah2", hosts)
	CheckIndexDomain(t, 7, "blah3", hosts)
	CheckIndexDomain(t, 8, "blah33", hosts)
	CheckIndexDomain(t, 9, "google3.com", hosts)
}

func ExampleHostlist_1() {
	hosts := hostess.NewHostlist()
	hosts.Add(hostess.NewHostname("google.com", "127.0.0.1", false))
	hosts.Add(hostess.NewHostname("google.com", "::1", true))

	fmt.Printf("%s\n", hosts.Format())
	// Output:
	// # 127.0.0.1 google.com
	// ::1 google.com
}
<|endoftext|>


<commit_before>package jo

import (
	"fmt"
	"testing"
)

type event struct {
	where int
	what  Event
}

type parseTest struct {
	json   string
	events []event
}

var legalLiterals = []parseTest{
	{`""`, []event{{1, StringStart}, {1, StringEnd}}},
	{`"abc"`, []event{{1, StringStart}, {4, StringEnd}}},
	{`"\u8bA0"`, []event{{1, StringStart}, {7, StringEnd}}},
	{`"\u12345"`, []event{{1, StringStart}, {8, StringEnd}}},
	{"\"\\b\\f\\n\\r\\t\\\\\"", []event{{1, StringStart}, {13, StringEnd}}},

	{`0`, []event{{1, NumberStart}, {0, NumberEnd}}},
	{`123`, []event{{1, NumberStart}, {2, None}, {0, NumberEnd}}},
	{`-10`, []event{{1, NumberStart}, {2, None}, {0, NumberEnd}}},
	{`0.10`, []event{{1, NumberStart}, {3, None}, {0, NumberEnd}}},
	{`12.03e+1`, []event{{1, NumberStart}, {7, None}, {0, NumberEnd}}},
	{`0.1e2`, []event{{1, NumberStart}, {4, None}, {0, NumberEnd}}},
	{`1e1`, []event{{1, NumberStart}, {2, None}, {0, NumberEnd}}},
	{`3.141569`, []event{{1, NumberStart}, {7, None}, {0, NumberEnd}}},
	{`10000000000000e-10`, []event{{1, NumberStart}, {17, None}, {0, NumberEnd}}},
	{`9223372036854775808`, []event{{1, NumberStart}, {18, None}, {0, NumberEnd}}},
	{`6E-06`, []event{{1, NumberStart}, {4, None}, {0, NumberEnd}}},
	{`1E-06`, []event{{1, NumberStart}, {4, None}, {0, NumberEnd}}},

	{`false`, []event{{1, BoolStart}, {4, BoolEnd}}},
	{`true`, []event{{1, BoolStart}, {3, BoolEnd}}},
	{`null`, []event{{1, NullStart}, {3, NullEnd}}},
}

var illegalLiterals = []parseTest{
	{`"`, []event{{1, StringStart}, {0, SyntaxError}}},
	{`"foo`, []event{{1, StringStart}, {3, None}, {0, SyntaxError}}},
	{`'single'`, []event{{0, SyntaxError}}},
	{`"\u12g8"`, []event{{1, StringStart}, {4, SyntaxError}}},
	{`"\u"`, []event{{1, StringStart}, {2, SyntaxError}}},
	{`"you can\'t do this"`, []event{{1, StringStart}, {8, SyntaxError}}},

	{`-`, []event{{1, NumberStart}, {0, SyntaxError}}},
	{`0.`, []event{{1, NumberStart}, {1, None}, {0, SyntaxError}}},
	{`123.456.789`, []event{{1, NumberStart}, {6, SyntaxError}}},
	{`10e`, []event{{1, NumberStart}, {2, None}, {0, SyntaxError}}},
	{`10e+`, []event{{1, NumberStart}, {3, None}, {0, SyntaxError}}},
	{`10e-`, []event{{1, NumberStart}, {3, None}, {0, SyntaxError}}},
	{`0e1x`, []event{{1, NumberStart}, {2, NumberEnd}, {0, SyntaxError}}},
	{`0e+13.`, []event{{1, NumberStart}, {4, NumberEnd}, {0, SyntaxError}}},
	{`0e+-0`, []event{{1, NumberStart}, {2, SyntaxError}}},

	{`tr`, []event{{1, BoolStart}, {1, None}, {0, SyntaxError}}},
	{`truE`, []event{{1, BoolStart}, {2, SyntaxError}}},
	{`fals`, []event{{1, BoolStart}, {3, None}, {0, SyntaxError}}},
	{`fALSE`, []event{{1, BoolStart}, {0, SyntaxError}}},
	{`n`, []event{{1, NullStart}, {0, SyntaxError}}},
	{`NULL`, []event{{0, SyntaxError}}},
}

var legalWhitespace = []parseTest{
	{` 17`, []event{{2, NumberStart}, {1, None}, {0, NumberEnd}}},
	{`38 `, []event{{1, NumberStart}, {1, NumberEnd}, {1, None}, {0, None}}},
	{`  " what ? "  `, []event{{3, StringStart}, {9, StringEnd}, {2, None}, {0, None}}},
	{"\nnull", []event{{2, NullStart}, {3, NullEnd}, {0, None}}},
	{"\n\r\t true \r\n\t", []event{{5, BoolStart}, {3, BoolEnd}, {4, None}, {0, None}}},
}

func TestParsing(t *testing.T) {
	tests := make([]parseTest, 0)
	tests = append(tests, legalLiterals...)
	tests = append(tests, illegalLiterals...)
	tests = append(tests, legalWhitespace...)

	for _, test := range tests {
		input := []byte(test.json)
		pos := 0

		// instantiate a new Parser and reset the log output
		// before each test
		p := Parser{}
		log := []string{"p := Parser{}"}

		// evaluate all expected events
		for i := 0; i < len(test.events); i++ {
			event := test.events[i]

			var where int
			var what Event

			if len(input)-pos == 0 {
				log = append(log, "  .Eof()")
				where, what = 0, p.Eof()
			} else {
				line := fmt.Sprintf("  .Parse(%#q)", input[pos:])
				log = append(log, line)

				where, what = p.Parse(input[pos:])
			}

			if where != event.where || what != event.what {
				// dump the log output we've accumulated
				for _, line := range log {
					t.Logf(line)
				}

				t.Fatalf("want %s at index %d, got %s at index %d",
					event.what, event.where, what, where)
			}

			// skip the bytes consumed during the last call
			// to parser.Parse()
			pos += where
		}
	}
}
<commit_msg>Add object test cases<commit_after>package jo

import (
	"fmt"
	"testing"
)

type event struct {
	where int
	what  Event
}

type parseTest struct {
	json   string
	events []event
}

var legalLiterals = []parseTest{
	{`""`, []event{{1, StringStart}, {1, StringEnd}}},
	{`"abc"`, []event{{1, StringStart}, {4, StringEnd}}},
	{`"\u8bA0"`, []event{{1, StringStart}, {7, StringEnd}}},
	{`"\u12345"`, []event{{1, StringStart}, {8, StringEnd}}},
	{"\"\\b\\f\\n\\r\\t\\\\\"", []event{{1, StringStart}, {13, StringEnd}}},

	{`0`, []event{{1, NumberStart}, {0, NumberEnd}}},
	{`123`, []event{{1, NumberStart}, {2, None}, {0, NumberEnd}}},
	{`-10`, []event{{1, NumberStart}, {2, None}, {0, NumberEnd}}},
	{`0.10`, []event{{1, NumberStart}, {3, None}, {0, NumberEnd}}},
	{`12.03e+1`, []event{{1, NumberStart}, {7, None}, {0, NumberEnd}}},
	{`0.1e2`, []event{{1, NumberStart}, {4, None}, {0, NumberEnd}}},
	{`1e1`, []event{{1, NumberStart}, {2, None}, {0, NumberEnd}}},
	{`3.141569`, []event{{1, NumberStart}, {7, None}, {0, NumberEnd}}},
	{`10000000000000e-10`, []event{{1, NumberStart}, {17, None}, {0, NumberEnd}}},
	{`9223372036854775808`, []event{{1, NumberStart}, {18, None}, {0, NumberEnd}}},
	{`6E-06`, []event{{1, NumberStart}, {4, None}, {0, NumberEnd}}},
	{`1E-06`, []event{{1, NumberStart}, {4, None}, {0, NumberEnd}}},

	{`false`, []event{{1, BoolStart}, {4, BoolEnd}}},
	{`true`, []event{{1, BoolStart}, {3, BoolEnd}}},
	{`null`, []event{{1, NullStart}, {3, NullEnd}}},
}

var illegalLiterals = []parseTest{
	{`"`, []event{{1, StringStart}, {0, SyntaxError}}},
	{`"foo`, []event{{1, StringStart}, {3, None}, {0, SyntaxError}}},
	{`'single'`, []event{{0, SyntaxError}}},
	{`"\u12g8"`, []event{{1, StringStart}, {4, SyntaxError}}},
	{`"\u"`, []event{{1, StringStart}, {2, SyntaxError}}},
	{`"you can\'t do this"`, []event{{1, StringStart}, {8, SyntaxError}}},

	{`-`, []event{{1, NumberStart}, {0, SyntaxError}}},
	{`0.`, []event{{1, NumberStart}, {1, None}, {0, SyntaxError}}},
	{`123.456.789`, []event{{1, NumberStart}, {6, SyntaxError}}},
	{`10e`, []event{{1, NumberStart}, {2, None}, {0, SyntaxError}}},
	{`10e+`, []event{{1, NumberStart}, {3, None}, {0, SyntaxError}}},
	{`10e-`, []event{{1, NumberStart}, {3, None}, {0, SyntaxError}}},
	{`0e1x`, []event{{1, NumberStart}, {2, NumberEnd}, {0, SyntaxError}}},
	{`0e+13.`, []event{{1, NumberStart}, {4, NumberEnd}, {0, SyntaxError}}},
	{`0e+-0`, []event{{1, NumberStart}, {2, SyntaxError}}},

	{`tr`, []event{{1, BoolStart}, {1, None}, {0, SyntaxError}}},
	{`truE`, []event{{1, BoolStart}, {2, SyntaxError}}},
	{`fals`, []event{{1, BoolStart}, {3, None}, {0, SyntaxError}}},
	{`fALSE`, []event{{1, BoolStart}, {0, SyntaxError}}},
	{`n`, []event{{1, NullStart}, {0, SyntaxError}}},
	{`NULL`, []event{{0, SyntaxError}}},
}

var legalObjects = []parseTest{
	{
		`{}`,
		[]event{
			{1, ObjectStart},
			{1, ObjectEnd},
			{0, None},
		},
	},
	{
		`{"foo":"bar"}`,
		[]event{
			{1, ObjectStart},
			{1, KeyStart},
			{4, KeyEnd},
			{2, StringStart},
			{4, StringEnd},
			{1, ObjectEnd},
			{0, None},
		},
	},
	{
		`{"1":1,"2":2}`,
		[]event{
			{1, ObjectStart},
			{1, KeyStart},
			{2, KeyEnd},
			{2, NumberStart},
			{0, NumberEnd},
			{2, KeyStart},
			{2, KeyEnd},
			{2, NumberStart},
			{0, NumberEnd},
			{1, ObjectEnd},
			{0, None},
		},
	},
	{
		`{"\u1234\t\n\b\u8BbF\"":0}`,
		[]event{
			{1, ObjectStart},
			{1, KeyStart},
			{21, KeyEnd},
			{2, NumberStart},
			{0, NumberEnd},
			{1, ObjectEnd},
			{0, None},
		},
	},
	{
		`{"{":"}"}`,
		[]event{
			{1, ObjectStart},
			{1, KeyStart},
			{2, KeyEnd},
			{2, StringStart},
			{2, StringEnd},
			{1, ObjectEnd},
			{0, None},
		},
	},
	{
		`{"foo":{"bar":{"baz":{}}}}`,
		[]event{
			{1, ObjectStart},
			{1, KeyStart},
			{4, KeyEnd},
			{2, ObjectStart},
			{1, KeyStart},
			{4, KeyEnd},
			{2, ObjectStart},
			{1, KeyStart},
			{4, KeyEnd},
			{2, ObjectStart},
			{1, ObjectEnd},
			{1, ObjectEnd},
			{1, ObjectEnd},
			{1, ObjectEnd},
			{0, None},
		},
	},
	{
		`{"true":true,"false":false,"null":null}`,
		[]event{
			{1, ObjectStart},
			{1, KeyStart},
			{5, KeyEnd},
			{2, BoolStart},
			{3, BoolEnd},
			{2, KeyStart},
			{6, KeyEnd},
			{2, BoolStart},
			{4, BoolEnd},
			{2, KeyStart},
			{5, KeyEnd},
			{2, NullStart},
			{3, NullEnd},
			{1, ObjectEnd},
			{0, None},
		},
	},
}

var illegalObjects = []parseTest{
	{
		`{0:1}`,
		[]event{
			{1, ObjectStart},
			{0, SyntaxError},
		},
	},
	{
		`{"foo":"bar"`,
		[]event{
			{1, ObjectStart},
			{1, KeyStart},
			{4, KeyEnd},
			{2, StringStart},
			{4, StringEnd},
			{0, SyntaxError},
		},
	},
	{
		`{{`,
		[]event{
			{1, ObjectStart},
			{0, SyntaxError},
		},
	},
	{
		`{"a":1,}`,
		[]event{
			{1, ObjectStart},
			{1, KeyStart},
			{2, KeyEnd},
			{2, NumberStart},
			{0, NumberEnd},
			{1, SyntaxError},
		},
	},
	{
		`{"a":1,,`,
		[]event{
			{1, ObjectStart},
			{1, KeyStart},
			{2, KeyEnd},
			{2, NumberStart},
			{0, NumberEnd},
			{1, SyntaxError},
		},
	},
	{
		`{"a"}`,
		[]event{
			{1, ObjectStart},
			{1, KeyStart},
			{2, KeyEnd},
			{0, SyntaxError},
		},
	},
	{
		`{"a":"1}`,
		[]event{
			{1, ObjectStart},
			{1, KeyStart},
			{2, KeyEnd},
			{2, StringStart},
			{2, None},
			{0, SyntaxError},
		},
	},
	{
		`{"a":1"b":2}`,
		[]event{
			{1, ObjectStart},
			{1, KeyStart},
			{2, KeyEnd},
			{2, NumberStart},
			{0, NumberEnd},
			{0, SyntaxError},
		},
	},
}

var legalWhitespace = []parseTest{
	{` 17`, []event{{2, NumberStart}, {1, None}, {0, NumberEnd}}},
	{`38 `, []event{{1, NumberStart}, {1, NumberEnd}, {1, None}, {0, None}}},
	{`  " what ? "  `, []event{{3, StringStart}, {9, StringEnd}, {2, None}, {0, None}}},
	{"\nnull", []event{{2, NullStart}, {3, NullEnd}, {0, None}}},
	{"\n\r\t true \r\n\t", []event{{5, BoolStart}, {3, BoolEnd}, {4, None}, {0, None}}},
}

func TestParsing(t *testing.T) {
	tests := make([]parseTest, 0)
	tests = append(tests, legalLiterals...)
	tests = append(tests, legalObjects...)
	tests = append(tests, illegalLiterals...)
	tests = append(tests, illegalObjects...)
	tests = append(tests, legalWhitespace...)

	for _, test := range tests {
		input := []byte(test.json)
		pos := 0

		// instantiate a new Parser and reset the log output
		// before each test
		p := Parser{}
		log := []string{"p := Parser{}"}

		// evaluate all expected events
		for i := 0; i < len(test.events); i++ {
			event := test.events[i]

			var where int
			var what Event

			if len(input)-pos == 0 {
				log = append(log, "  .Eof()")
				where, what = 0, p.Eof()
			} else {
				line := fmt.Sprintf("  .Parse(%#q)", input[pos:])
				log = append(log, line)

				where, what = p.Parse(input[pos:])
			}

			if where != event.where || what != event.what {
				// dump the log output we've accumulated
				for _, line := range log {
					t.Logf(line)
				}

				t.Fatalf("want %s at index %d, got %s at index %d",
					event.what, event.where, what, where)
			}

			// skip the bytes consumed during the last call
			// to parser.Parse()
			pos += where
		}
	}
}
<|endoftext|>


<commit_before>package stats

import (
	"fmt"
	"sort"
	"strings"
)

type Map map[string]*Value

func (m Map) String() string {
	return m.ReversedValues().String()
}

func (m Map) Sum() (sum int) {
	for _, v := range m {
		sum += v.Value
	}
	return sum

}

func (m Map) ReversedValues() Values {
	v := m.Values()
	sort.Sort(v)
	return v
}

func (m Map) SortedValues() Values {
	v := m.Values()
	sort.Sort(v)
	return v
}

func (m Map) Values() Values {
	values := make(Values, 0, len(m))
	for _, v := range m {
		values = append(values, v)
	}
	return values
}

func (m Map) Inc(key string) int {
	return m.IncBy(key, 1)
}

func (m Map) IncBy(key string, value int) int {
	if m[key] == nil {
		m[key] = &Value{Key: key}
	}
	m[key].Value += value
	return m[key].Value
}

type Values []*Value

func (v Values) String() string {
	out := []string{}
	for _, k := range v {
		out = append(out, fmt.Sprintf("%s: %d", k.Key, k.Value))
	}
	return strings.Join(out, ",")
}

func (list Values) TopN(n int) Values {
	sort.Sort(sort.Reverse(list))
	out := make(Values, 0, n)
	for _, v := range list {
		out = append(out, v)
		if len(out) >= n {
			break
		}
	}
	return out
}

func (list Values) Len() int {
	return len(list)
}

func (list Values) Swap(a, b int) {
	list[a], list[b] = list[b], list[a]
}

func (list Values) Less(a, b int) bool {
	diff := list[a].Value - list[b].Value
	switch {
	case diff < 0:
		return false
	case diff > 0:
		return true
	default:
		return list[a].Key < list[b].Key
	}
}

type Value struct {
	Key   string
	Value int
}
<commit_msg>skip sorting every time<commit_after>package stats

import (
	"fmt"
	"sort"
	"strings"
)

type Map map[string]*Value

func (m Map) String() string {
	return m.ReversedValues().String()
}

func (m Map) Sum() (sum int) {
	for _, v := range m {
		sum += v.Value
	}
	return sum

}

func (m Map) ReversedValues() Values {
	v := m.Values()
	sort.Sort(v)
	return v
}

func (m Map) SortedValues() Values {
	v := m.Values()
	sort.Sort(v)
	return v
}

func (m Map) Values() Values {
	values := make(Values, 0, len(m))
	for _, v := range m {
		values = append(values, v)
	}
	return values
}

func (m Map) Inc(key string) int {
	return m.IncBy(key, 1)
}

func (m Map) IncBy(key string, value int) int {
	if m[key] == nil {
		m[key] = &Value{Key: key}
	}
	m[key].Value += value
	return m[key].Value
}

type Values []*Value

func (v Values) String() string {
	out := []string{}
	for _, k := range v {
		out = append(out, fmt.Sprintf("%s: %d", k.Key, k.Value))
	}
	return strings.Join(out, ",")
}

func (list Values) TopN(n int) Values {
	out := make(Values, 0, n)
	for _, v := range list {
		out = append(out, v)
		if len(out) >= n {
			break
		}
	}
	return out
}

func (list Values) Len() int {
	return len(list)
}

func (list Values) Swap(a, b int) {
	list[a], list[b] = list[b], list[a]
}

func (list Values) Less(a, b int) bool {
	diff := list[a].Value - list[b].Value
	switch {
	case diff < 0:
		return false
	case diff > 0:
		return true
	default:
		return list[a].Key < list[b].Key
	}
}

type Value struct {
	Key   string
	Value int
}
<|endoftext|>