<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package garbagecollector

import (
	"fmt"

	"k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/client-go/util/retry"
	"k8s.io/klog"
)

// cluster scoped resources don't have namespaces.  Default to the item's namespace, but clear it for cluster scoped resources
func resourceDefaultNamespace(namespaced bool, defaultNamespace string) string {
	if namespaced {
		return defaultNamespace
	}
	return ""
}

// apiResource consults the REST mapper to translate an <apiVersion, kind,
// namespace> tuple to a unversioned.APIResource struct.
func (gc *GarbageCollector) apiResource(apiVersion, kind string) (schema.GroupVersionResource, bool, error) {
	fqKind := schema.FromAPIVersionAndKind(apiVersion, kind)
	mapping, err := gc.restMapper.RESTMapping(fqKind.GroupKind(), fqKind.Version)
	if err != nil {
		return schema.GroupVersionResource{}, false, newRESTMappingError(kind, apiVersion)
	}
	return mapping.Resource, mapping.Scope == meta.RESTScopeNamespace, nil
}

func (gc *GarbageCollector) deleteObject(item objectReference, policy *metav1.DeletionPropagation) error {
	resource, namespaced, err := gc.apiResource(item.APIVersion, item.Kind)
	if err != nil {
		return err
	}
	uid := item.UID
	preconditions := metav1.Preconditions{UID: &uid}
	deleteOptions := metav1.DeleteOptions{Preconditions: &preconditions, PropagationPolicy: policy}
	return gc.dynamicClient.Resource(resource).Namespace(resourceDefaultNamespace(namespaced, item.Namespace)).Delete(item.Name, &deleteOptions)
}

func (gc *GarbageCollector) getObject(item objectReference) (*unstructured.Unstructured, error) {
	resource, namespaced, err := gc.apiResource(item.APIVersion, item.Kind)
	if err != nil {
		return nil, err
	}
	return gc.dynamicClient.Resource(resource).Namespace(resourceDefaultNamespace(namespaced, item.Namespace)).Get(item.Name, metav1.GetOptions{})
}

func (gc *GarbageCollector) updateObject(item objectReference, obj *unstructured.Unstructured) (*unstructured.Unstructured, error) {
	resource, namespaced, err := gc.apiResource(item.APIVersion, item.Kind)
	if err != nil {
		return nil, err
	}
	return gc.dynamicClient.Resource(resource).Namespace(resourceDefaultNamespace(namespaced, item.Namespace)).Update(obj, metav1.UpdateOptions{})
}

func (gc *GarbageCollector) patchObject(item objectReference, patch []byte, pt types.PatchType) (*unstructured.Unstructured, error) {
	resource, namespaced, err := gc.apiResource(item.APIVersion, item.Kind)
	if err != nil {
		return nil, err
	}
	return gc.dynamicClient.Resource(resource).Namespace(resourceDefaultNamespace(namespaced, item.Namespace)).Patch(item.Name, pt, patch, metav1.PatchOptions{})
}

// TODO: Using Patch when strategicmerge supports deleting an entry from a
// slice of a base type.
func (gc *GarbageCollector) removeFinalizer(owner *node, targetFinalizer string) error {
	err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
		ownerObject, err := gc.getObject(owner.identity)
		if errors.IsNotFound(err) {
			return nil
		}
		if err != nil {
			return fmt.Errorf("cannot finalize owner %s, because cannot get it: %v. The garbage collector will retry later", owner.identity, err)
		}
		accessor, err := meta.Accessor(ownerObject)
		if err != nil {
			return fmt.Errorf("cannot access the owner object %v: %v. The garbage collector will retry later", ownerObject, err)
		}
		finalizers := accessor.GetFinalizers()
		var newFinalizers []string
		found := false
		for _, f := range finalizers {
			if f == targetFinalizer {
				found = true
				continue
			}
			newFinalizers = append(newFinalizers, f)
		}
		if !found {
			klog.V(5).Infof("the %s finalizer is already removed from object %s", targetFinalizer, owner.identity)
			return nil
		}
		// remove the owner from dependent's OwnerReferences
		ownerObject.SetFinalizers(newFinalizers)
		_, err = gc.updateObject(owner.identity, ownerObject)
		return err
	})
	if errors.IsConflict(err) {
		return fmt.Errorf("updateMaxRetries(%d) has reached. The garbage collector will retry later for owner %v", retry.DefaultBackoff.Steps, owner.identity)
	}
	return err
}
<commit_msg>Garbage collector should use a merge patch instead of Update<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package garbagecollector

import (
	"encoding/json"
	"fmt"

	"k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/client-go/util/retry"
	"k8s.io/klog"
)

// cluster scoped resources don't have namespaces.  Default to the item's namespace, but clear it for cluster scoped resources
func resourceDefaultNamespace(namespaced bool, defaultNamespace string) string {
	if namespaced {
		return defaultNamespace
	}
	return ""
}

// apiResource consults the REST mapper to translate an <apiVersion, kind,
// namespace> tuple to a unversioned.APIResource struct.
func (gc *GarbageCollector) apiResource(apiVersion, kind string) (schema.GroupVersionResource, bool, error) {
	fqKind := schema.FromAPIVersionAndKind(apiVersion, kind)
	mapping, err := gc.restMapper.RESTMapping(fqKind.GroupKind(), fqKind.Version)
	if err != nil {
		return schema.GroupVersionResource{}, false, newRESTMappingError(kind, apiVersion)
	}
	return mapping.Resource, mapping.Scope == meta.RESTScopeNamespace, nil
}

func (gc *GarbageCollector) deleteObject(item objectReference, policy *metav1.DeletionPropagation) error {
	resource, namespaced, err := gc.apiResource(item.APIVersion, item.Kind)
	if err != nil {
		return err
	}
	uid := item.UID
	preconditions := metav1.Preconditions{UID: &uid}
	deleteOptions := metav1.DeleteOptions{Preconditions: &preconditions, PropagationPolicy: policy}
	return gc.dynamicClient.Resource(resource).Namespace(resourceDefaultNamespace(namespaced, item.Namespace)).Delete(item.Name, &deleteOptions)
}

func (gc *GarbageCollector) getObject(item objectReference) (*unstructured.Unstructured, error) {
	resource, namespaced, err := gc.apiResource(item.APIVersion, item.Kind)
	if err != nil {
		return nil, err
	}
	return gc.dynamicClient.Resource(resource).Namespace(resourceDefaultNamespace(namespaced, item.Namespace)).Get(item.Name, metav1.GetOptions{})
}

func (gc *GarbageCollector) patchObject(item objectReference, patch []byte, pt types.PatchType) (*unstructured.Unstructured, error) {
	resource, namespaced, err := gc.apiResource(item.APIVersion, item.Kind)
	if err != nil {
		return nil, err
	}
	return gc.dynamicClient.Resource(resource).Namespace(resourceDefaultNamespace(namespaced, item.Namespace)).Patch(item.Name, pt, patch, metav1.PatchOptions{})
}

func (gc *GarbageCollector) removeFinalizer(owner *node, targetFinalizer string) error {
	err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
		ownerObject, err := gc.getObject(owner.identity)
		if errors.IsNotFound(err) {
			return nil
		}
		if err != nil {
			return fmt.Errorf("cannot finalize owner %s, because cannot get it: %v. The garbage collector will retry later", owner.identity, err)
		}
		accessor, err := meta.Accessor(ownerObject)
		if err != nil {
			return fmt.Errorf("cannot access the owner object %v: %v. The garbage collector will retry later", ownerObject, err)
		}
		finalizers := accessor.GetFinalizers()
		var newFinalizers []string
		found := false
		for _, f := range finalizers {
			if f == targetFinalizer {
				found = true
				continue
			}
			newFinalizers = append(newFinalizers, f)
		}
		if !found {
			klog.V(5).Infof("the %s finalizer is already removed from object %s", targetFinalizer, owner.identity)
			return nil
		}

		// remove the owner from dependent's OwnerReferences
		patch, err := json.Marshal(map[string]interface{}{
			"metadata": map[string]interface{}{
				"resourceVersion": accessor.GetResourceVersion(),
				"finalizers":      newFinalizers,
			},
		})
		if err != nil {
			return fmt.Errorf("unable to finalize %s due to an error serializing patch: %v", owner.identity, err)
		}
		_, err = gc.patchObject(owner.identity, patch, types.MergePatchType)
		return err
	})
	if errors.IsConflict(err) {
		return fmt.Errorf("updateMaxRetries(%d) has reached. The garbage collector will retry later for owner %v", retry.DefaultBackoff.Steps, owner.identity)
	}
	return err
}
<|endoftext|>


<commit_before>/*

   Copyright 2016 Wenhui Shen <www.webx.top>

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

*/
package session

import (
	"github.com/admpub/sessions"
	"github.com/webx-top/echo"
)

func Sessions(options *echo.SessionOptions, store sessions.Store) echo.MiddlewareFuncd {
	var newSession func(ctx echo.Context) echo.Sessioner
	if options == nil {
		newSession = func(ctx echo.Context) echo.Sessioner {
			options = ctx.SessionOptions()
			return NewMySession(store, options.Name, ctx)
		}
	} else {
		newSession = func(ctx echo.Context) echo.Sessioner {
			ctx.SetSessionOptions(options)
			return NewMySession(store, options.Name, ctx)
		}
	}
	return func(h echo.Handler) echo.HandlerFunc {
		return func(c echo.Context) error {
			s := newSession(c)
			c.SetSessioner(s)
			c.AddPreResponseHook(func() error {
				if options.Engine == `cookie` {
					return s.Save()
				}
				return nil
			})
			err := h.Handle(c)
			if e := s.Save(); e != nil {
				c.Logger().Error(e)
			}
			return err
		}
	}
}

func Middleware(options *echo.SessionOptions) echo.MiddlewareFuncd {
	store := StoreEngine(options)
	return Sessions(options, store)
}
<commit_msg>improved: session<commit_after>/*

   Copyright 2016 Wenhui Shen <www.webx.top>

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

*/
package session

import (
	"errors"

	"github.com/admpub/sessions"
	"github.com/webx-top/echo"
)

func Sessions(options *echo.SessionOptions, store sessions.Store) echo.MiddlewareFuncd {
	var newSession func(ctx echo.Context) echo.Sessioner
	if options == nil {
		newSession = func(ctx echo.Context) echo.Sessioner {
			options = ctx.SessionOptions()
			return NewMySession(store, options.Name, ctx)
		}
	} else {
		newSession = func(ctx echo.Context) echo.Sessioner {
			ctx.SetSessionOptions(options)
			return NewMySession(store, options.Name, ctx)
		}
	}
	return func(h echo.Handler) echo.HandlerFunc {
		return func(c echo.Context) error {
			s := newSession(c)
			c.SetSessioner(s)
			c.AddPreResponseHook(func() error {
				if options.Engine == `cookie` {
					return s.Save()
				}
				return nil
			})
			err := h.Handle(c)
			if e := s.Save(); e != nil {
				if err != nil {
					err = errors.New("Multiple errors:\n1. " + err.Error() + "\n2. " + e.Error())
				} else {
					err = e
				}
				c.Logger().Error(err)
			}
			return err
		}
	}
}

func Middleware(options *echo.SessionOptions) echo.MiddlewareFuncd {
	store := StoreEngine(options)
	return Sessions(options, store)
}
<|endoftext|>


<commit_before>//  Copyright (c) 2012 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package db

import (
	"encoding/json"
	"math"

	"github.com/couchbaselabs/go-couchbase"

	"github.com/couchbaselabs/sync_gateway/base"
	"github.com/couchbaselabs/sync_gateway/channels"
)

// Options for Database.getChanges
type ChangesOptions struct {
	Since          uint64
	Limit          int
	Conflicts      bool
	IncludeDocs    bool
	includeDocMeta bool
	Wait           bool
}

// A changes entry; Database.getChanges returns an array of these.
// Marshals into the standard CouchDB _changes format.
type ChangeEntry struct {
	Seq     uint64       `json:"seq"`
	ID      string       `json:"id"`
	Deleted bool         `json:"deleted,omitempty"`
	Removed channels.Set `json:"removed,omitempty"`
	Doc     Body         `json:"doc,omitempty"`
	Changes []ChangeRev  `json:"changes"`
	docMeta *document
}

type ChangeRev map[string]string

type ViewDoc struct {
	Json json.RawMessage // should be type 'document', but that fails to unmarshal correctly
}

type ViewRow struct {
	ID    string
	Key   interface{}
	Value interface{}
	Doc   *ViewDoc
}

type ViewResult struct {
	TotalRows int `json:"total_rows"`
	Rows      []ViewRow
	Errors    []couchbase.ViewError
}

const kChangesPageSize = 200

// Returns a list of all the changes made on a channel. Does NOT check authorization.
func (db *Database) ChangesFeed(channel string, options ChangesOptions) (<-chan *ChangeEntry, error) {
	lastSequence := options.Since
	endkey := []interface{}{channel, map[string]interface{}{}}
	totalLimit := options.Limit
	usingDocs := options.Conflicts || options.IncludeDocs || options.includeDocMeta
	opts := Body{"stale": false, "update_seq": true,
		"endkey":       endkey,
		"include_docs": usingDocs}

	feed := make(chan *ChangeEntry, kChangesPageSize)

	lastSeq := db.LastSequence()
	if options.Since >= lastSeq && !options.Wait {
		close(feed)
		return feed, nil
	}

	// Generate the output in a new goroutine, writing to 'feed':
	go func() {
		defer close(feed)
		for {
			// Query the 'channels' view:
			opts["startkey"] = []interface{}{channel, lastSequence + 1}
			limit := totalLimit
			if limit == 0 || limit > kChangesPageSize {
				limit = kChangesPageSize
			}
			opts["limit"] = limit

			var vres ViewResult
			var err error
			for len(vres.Rows) == 0 {
				vres = ViewResult{}
				err = db.Bucket.ViewCustom("sync_gateway", "channels", opts, &vres)
				if err != nil {
					base.Log("Error from 'channels' view: %v", err)
					return
				}
				if len(vres.Rows) == 0 {
					if !options.Wait || !db.WaitForRevision() {
						return
					}
				}
			}

			for _, row := range vres.Rows {
				key := row.Key.([]interface{})
				lastSequence = uint64(key[1].(float64))
				value := row.Value.([]interface{})
				docID := value[0].(string)
				revID := value[1].(string)
				entry := &ChangeEntry{
					Seq:     lastSequence,
					ID:      docID,
					Changes: []ChangeRev{{"rev": revID}},
					Deleted: (len(value) >= 3 && value[2].(bool)),
				}
				if len(value) >= 3 && !value[2].(bool) {
					entry.Removed = channels.SetOf(channel)
				}
				if usingDocs {
					doc, err := unmarshalDocument(docID, row.Doc.Json)
					if err == nil {
						if options.Conflicts {
							for _, leafID := range doc.History.getLeaves() {
								if leafID != revID {
									entry.Changes = append(entry.Changes, ChangeRev{"rev": leafID})
								}
							}
						}
						if options.IncludeDocs {
							entry.Doc, _ = db.getRevFromDoc(doc, revID, false)
						}
						if options.includeDocMeta {
							entry.docMeta = doc
						}
					}
				}
				feed <- entry
			}

			// Step to the next page of results:
			nRows := len(vres.Rows)
			if nRows < kChangesPageSize || options.Wait {
				break
			}
			if totalLimit > 0 {
				totalLimit -= nRows
				if totalLimit <= 0 {
					break
				}
			}
			delete(opts, "stale") // we only need to update the index once
		}
	}()
	return feed, nil
}

// Returns of all the changes made to multiple channels.
func (db *Database) MultiChangesFeed(chans channels.Set, options ChangesOptions) (<-chan *ChangeEntry, error) {
	base.LogTo("Changes", "MultiChangesFeed(%s, %v) ...", chans, options)
	if len(chans) == 0 {
		return nil, nil
	} else if len(chans) == 1 && !chans.Contains("*") {
		for channel, _ := range chans {
			return db.ChangesFeed(channel, options)
		}
	}

	waitMode := options.Wait
	options.Wait = false

	output := make(chan *ChangeEntry, kChangesPageSize)
	go func() {
		defer close(output)

		for {
			// Restrict to available channels, expand wild-card, and find since when these channels
			// have been available to the user:
			var channelsSince channels.TimedSet
			if db.user != nil {
				channelsSince = db.user.FilterToAvailableChannels(chans)
			} else {
				channelsSince = chans.AtSequence(1)
			}
			base.LogTo("Changes", "MultiChangesFeed: channels expand to %s ...", channelsSince)

			feeds := make([]<-chan *ChangeEntry, 0, len(channelsSince))
			for name, _ := range channelsSince {
				feedOpts := options
				if channelsSince[name] > options.Since {
					feedOpts.Since = 0 // channel wasn't available before, so start from beginning
				}
				feed, err := db.ChangesFeed(name, feedOpts)
				if err != nil {
					base.Warn("Error reading changes feed %q: %v", name, err)
					return
				}
				feeds = append(feeds, feed)
			}
			current := make([]*ChangeEntry, len(feeds))

			var lastSeqSent uint64
			for {
				//FIX: This assumes Reverse or Limit aren't set in the options
				// Read more entries to fill up the current[] array:
				for i, cur := range current {
					if cur == nil && feeds[i] != nil {
						var ok bool
						current[i], ok = <-feeds[i]
						if !ok {
							feeds[i] = nil
						}
					}
				}

				// Find the current entry with the minimum sequence:
				var minSeq uint64 = math.MaxUint64
				var minEntry *ChangeEntry
				for _, cur := range current {
					if cur != nil && cur.Seq < minSeq {
						minSeq = cur.Seq
						minEntry = cur
					}
				}
				if minEntry == nil {
					break // Exit the loop when there are no more entries
				}

				// Clear the current entries for the sequence just sent:
				for i, cur := range current {
					if cur != nil && cur.Seq == minEntry.Seq {
						current[i] = nil
						// Also concatenate the matching entries' Removed arrays:
						if cur != minEntry && cur.Removed != nil {
							if minEntry.Removed == nil {
								minEntry.Removed = cur.Removed
							} else {
								minEntry.Removed = minEntry.Removed.Union(cur.Removed)
							}
						}
					}
				}

				// Send the entry, and repeat the loop:
				output <- minEntry
				lastSeqSent = minEntry.Seq
			}

			// If nothing found, and in wait mode: wait for the db to change, then run again
			if lastSeqSent > 0 || !waitMode || !db.WaitForRevision() {
				break
			}
		}
	}()

	return output, nil
}

// Synchronous convenience function that returns all changes as a simple array.
func (db *Database) GetChanges(channels channels.Set, options ChangesOptions) ([]*ChangeEntry, error) {
	var changes = make([]*ChangeEntry, 0, 50)
	feed, err := db.MultiChangesFeed(channels, options)
	if err == nil && feed != nil {
		for entry := range feed {
			changes = append(changes, entry)
		}
	}
	return changes, err
}

func (db *Database) WaitForRevision() bool {
	base.Log("\twaiting for a revision...")
	waitFor("")
	base.Log("\t...done waiting")
	return true
}

func (db *Database) NotifyRevision() {
	notify("")
}

func (context *DatabaseContext) LastSequence() uint64 {
	return context.sequences.lastSequence()
}

func (db *Database) ReserveSequences(numToReserve uint64) error {
	return db.sequences.reserveSequences(numToReserve)
}
<commit_msg>Better changes-feed logging<commit_after>//  Copyright (c) 2012 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package db

import (
	"encoding/json"
	"math"

	"github.com/couchbaselabs/go-couchbase"

	"github.com/couchbaselabs/sync_gateway/base"
	"github.com/couchbaselabs/sync_gateway/channels"
)

// Options for Database.getChanges
type ChangesOptions struct {
	Since          uint64
	Limit          int
	Conflicts      bool
	IncludeDocs    bool
	includeDocMeta bool
	Wait           bool
}

// A changes entry; Database.getChanges returns an array of these.
// Marshals into the standard CouchDB _changes format.
type ChangeEntry struct {
	Seq     uint64       `json:"seq"`
	ID      string       `json:"id"`
	Deleted bool         `json:"deleted,omitempty"`
	Removed channels.Set `json:"removed,omitempty"`
	Doc     Body         `json:"doc,omitempty"`
	Changes []ChangeRev  `json:"changes"`
	docMeta *document
}

type ChangeRev map[string]string

type ViewDoc struct {
	Json json.RawMessage // should be type 'document', but that fails to unmarshal correctly
}

type ViewRow struct {
	ID    string
	Key   interface{}
	Value interface{}
	Doc   *ViewDoc
}

type ViewResult struct {
	TotalRows int `json:"total_rows"`
	Rows      []ViewRow
	Errors    []couchbase.ViewError
}

const kChangesPageSize = 200

// Returns a list of all the changes made on a channel. Does NOT check authorization.
func (db *Database) ChangesFeed(channel string, options ChangesOptions) (<-chan *ChangeEntry, error) {
	lastSequence := options.Since
	endkey := []interface{}{channel, map[string]interface{}{}}
	totalLimit := options.Limit
	usingDocs := options.Conflicts || options.IncludeDocs || options.includeDocMeta
	opts := Body{"stale": false, "update_seq": true,
		"endkey":       endkey,
		"include_docs": usingDocs}

	feed := make(chan *ChangeEntry, kChangesPageSize)

	lastSeq := db.LastSequence()
	if options.Since >= lastSeq && !options.Wait {
		close(feed)
		return feed, nil
	}

	// Generate the output in a new goroutine, writing to 'feed':
	go func() {
		defer close(feed)
		for {
			// Query the 'channels' view:
			opts["startkey"] = []interface{}{channel, lastSequence + 1}
			limit := totalLimit
			if limit == 0 || limit > kChangesPageSize {
				limit = kChangesPageSize
			}
			opts["limit"] = limit

			var vres ViewResult
			var err error
			for len(vres.Rows) == 0 {
				vres = ViewResult{}
				err = db.Bucket.ViewCustom("sync_gateway", "channels", opts, &vres)
				if err != nil {
					base.Log("Error from 'channels' view: %v", err)
					return
				}
				if len(vres.Rows) == 0 {
					if !options.Wait || !db.WaitForRevision() {
						return
					}
				}
			}

			for _, row := range vres.Rows {
				key := row.Key.([]interface{})
				lastSequence = uint64(key[1].(float64))
				value := row.Value.([]interface{})
				docID := value[0].(string)
				revID := value[1].(string)
				entry := &ChangeEntry{
					Seq:     lastSequence,
					ID:      docID,
					Changes: []ChangeRev{{"rev": revID}},
					Deleted: (len(value) >= 3 && value[2].(bool)),
				}
				if len(value) >= 3 && !value[2].(bool) {
					entry.Removed = channels.SetOf(channel)
				}
				if usingDocs {
					doc, err := unmarshalDocument(docID, row.Doc.Json)
					if err == nil {
						if options.Conflicts {
							for _, leafID := range doc.History.getLeaves() {
								if leafID != revID {
									entry.Changes = append(entry.Changes, ChangeRev{"rev": leafID})
								}
							}
						}
						if options.IncludeDocs {
							entry.Doc, _ = db.getRevFromDoc(doc, revID, false)
						}
						if options.includeDocMeta {
							entry.docMeta = doc
						}
					}
				}
				feed <- entry
			}

			// Step to the next page of results:
			nRows := len(vres.Rows)
			if nRows < kChangesPageSize || options.Wait {
				break
			}
			if totalLimit > 0 {
				totalLimit -= nRows
				if totalLimit <= 0 {
					break
				}
			}
			delete(opts, "stale") // we only need to update the index once
		}
	}()
	return feed, nil
}

// Returns of all the changes made to multiple channels.
func (db *Database) MultiChangesFeed(chans channels.Set, options ChangesOptions) (<-chan *ChangeEntry, error) {
	if len(chans) == 0 {
		return nil, nil
	} else if len(chans) == 1 && !chans.Contains("*") {
		for channel, _ := range chans {
			base.LogTo("Changes", "ChangesFeed(%s, %+v) ...", channel, options)
			return db.ChangesFeed(channel, options)
		}
	}
	base.LogTo("Changes", "MultiChangesFeed(%s, %+v) ...", chans, options)

	waitMode := options.Wait
	options.Wait = false

	output := make(chan *ChangeEntry, kChangesPageSize)
	go func() {
		defer close(output)

		for {
			// Restrict to available channels, expand wild-card, and find since when these channels
			// have been available to the user:
			var channelsSince channels.TimedSet
			if db.user != nil {
				channelsSince = db.user.FilterToAvailableChannels(chans)
			} else {
				channelsSince = chans.AtSequence(1)
			}
			base.LogTo("Changes", "MultiChangesFeed: channels expand to %s ...", channelsSince)

			feeds := make([]<-chan *ChangeEntry, 0, len(channelsSince))
			for name, _ := range channelsSince {
				feedOpts := options
				if channelsSince[name] > options.Since {
					feedOpts.Since = 0 // channel wasn't available before, so start from beginning
				}
				feed, err := db.ChangesFeed(name, feedOpts)
				if err != nil {
					base.Warn("Error reading changes feed %q: %v", name, err)
					return
				}
				feeds = append(feeds, feed)
			}
			current := make([]*ChangeEntry, len(feeds))

			var lastSeqSent uint64
			for {
				//FIX: This assumes Reverse or Limit aren't set in the options
				// Read more entries to fill up the current[] array:
				for i, cur := range current {
					if cur == nil && feeds[i] != nil {
						var ok bool
						current[i], ok = <-feeds[i]
						if !ok {
							feeds[i] = nil
						}
					}
				}

				// Find the current entry with the minimum sequence:
				var minSeq uint64 = math.MaxUint64
				var minEntry *ChangeEntry
				for _, cur := range current {
					if cur != nil && cur.Seq < minSeq {
						minSeq = cur.Seq
						minEntry = cur
					}
				}
				if minEntry == nil {
					break // Exit the loop when there are no more entries
				}

				// Clear the current entries for the sequence just sent:
				for i, cur := range current {
					if cur != nil && cur.Seq == minEntry.Seq {
						current[i] = nil
						// Also concatenate the matching entries' Removed arrays:
						if cur != minEntry && cur.Removed != nil {
							if minEntry.Removed == nil {
								minEntry.Removed = cur.Removed
							} else {
								minEntry.Removed = minEntry.Removed.Union(cur.Removed)
							}
						}
					}
				}

				// Send the entry, and repeat the loop:
				output <- minEntry
				lastSeqSent = minEntry.Seq
			}

			// If nothing found, and in wait mode: wait for the db to change, then run again
			if lastSeqSent > 0 || !waitMode || !db.WaitForRevision() {
				break
			}
		}
		base.LogTo("Changes", "MultiChangesFeed done")
	}()

	return output, nil
}

// Synchronous convenience function that returns all changes as a simple array.
func (db *Database) GetChanges(channels channels.Set, options ChangesOptions) ([]*ChangeEntry, error) {
	var changes = make([]*ChangeEntry, 0, 50)
	feed, err := db.MultiChangesFeed(channels, options)
	if err == nil && feed != nil {
		for entry := range feed {
			changes = append(changes, entry)
		}
	}
	return changes, err
}

func (db *Database) WaitForRevision() bool {
	base.LogTo("Changes", "\twaiting for a revision...")
	waitFor("")
	base.LogTo("Changes", "\t...done waiting")
	return true
}

func (db *Database) NotifyRevision() {
	notify("")
}

func (context *DatabaseContext) LastSequence() uint64 {
	return context.sequences.lastSequence()
}

func (db *Database) ReserveSequences(numToReserve uint64) error {
	return db.sequences.reserveSequences(numToReserve)
}
<|endoftext|>


<commit_before>package air

import (
	"bytes"
	"compress/gzip"
	"encoding/binary"
	"io/ioutil"
	"mime"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/VictoriaMetrics/fastcache"
	"github.com/aofei/mimesniffer"
	"github.com/cespare/xxhash"
	"github.com/fsnotify/fsnotify"
)

// coffer is a binary asset file manager that uses runtime memory to reduce disk
// I/O pressure.
type coffer struct {
	a         *Air
	loadOnce  *sync.Once
	loadError error
	watcher   *fsnotify.Watcher
	assets    *sync.Map
	cache     *fastcache.Cache
}

// newCoffer returns a new instance of the `coffer` with the a.
func newCoffer(a *Air) *coffer {
	return &coffer{
		a:        a,
		loadOnce: &sync.Once{},
	}
}

// load loads the stuff of the c up.
func (c *coffer) load() {
	defer func() {
		if c.loadError != nil {
			c.loadOnce = &sync.Once{}
		}
	}()

	if c.watcher == nil {
		c.watcher, c.loadError = fsnotify.NewWatcher()
		if c.loadError != nil {
			return
		}

		go func() {
			for {
				select {
				case e := <-c.watcher.Events:
					ai, ok := c.assets.Load(e.Name)
					if !ok {
						break
					}

					a := ai.(*asset)
					c.assets.Delete(a.name)
					c.cache.Del(a.digest)
					if a.gzippedDigest != nil {
						c.cache.Del(a.gzippedDigest)
					}
				case err := <-c.watcher.Errors:
					c.a.errorLogger.Printf(
						"air: coffer watcher error: %v",
						err,
					)
				}
			}
		}()
	}

	c.assets = &sync.Map{}
	c.cache = fastcache.New(c.a.CofferMaxMemoryBytes)
}

// asset returns an `asset` from the c for the name.
func (c *coffer) asset(name string) (*asset, error) {
	if c.loadOnce.Do(c.load); c.loadError != nil {
		return nil, c.loadError
	} else if ai, ok := c.assets.Load(name); ok {
		return ai.(*asset), nil
	} else if ar, err := filepath.Abs(c.a.CofferAssetRoot); err != nil {
		return nil, err
	} else if !strings.HasPrefix(name, ar) {
		return nil, nil
	}

	ext := filepath.Ext(name)
	if !stringSliceContainsCIly(c.a.CofferAssetExts, ext) {
		return nil, nil
	}

	fi, err := os.Stat(name)
	if err != nil {
		return nil, err
	}

	b, err := ioutil.ReadFile(name)
	if err != nil {
		return nil, err
	}

	var (
		mt       = mime.TypeByExtension(ext)
		minified bool
		gb       []byte
	)

	if mt == "" {
		mt = mimesniffer.Sniff(b)
	}

	pmt, _, err := mime.ParseMediaType(mt)
	if err != nil {
		return nil, err
	}

	if c.a.MinifierEnabled &&
		stringSliceContainsCIly(c.a.MinifierMIMETypes, pmt) {
		if b, err = c.a.minifier.minify(pmt, b); err != nil {
			return nil, err
		}

		minified = true
	}

	if c.a.GzipEnabled && stringSliceContainsCIly(c.a.GzipMIMETypes, pmt) {
		buf := bytes.Buffer{}
		if gw, err := gzip.NewWriterLevel(
			&buf,
			c.a.GzipCompressionLevel,
		); err != nil {
			return nil, err
		} else if _, err = gw.Write(b); err != nil {
			return nil, err
		} else if err = gw.Close(); err != nil {
			return nil, err
		}

		gb = buf.Bytes()
	}

	if err := c.watcher.Add(name); err != nil {
		return nil, err
	}

	a := &asset{
		coffer:   c,
		name:     name,
		mimeType: mt,
		modTime:  fi.ModTime(),
		minified: minified,
		digest:   make([]byte, 8),
	}

	binary.BigEndian.PutUint64(a.digest, xxhash.Sum64(b))
	c.cache.Set(a.digest, b)

	if gb != nil {
		a.gzippedDigest = make([]byte, 8)
		binary.BigEndian.PutUint64(a.gzippedDigest, xxhash.Sum64(gb))
		c.cache.Set(a.gzippedDigest, gb)
	}

	c.assets.Store(name, a)

	return a, nil
}

// asset is a binary asset file.
type asset struct {
	coffer        *coffer
	name          string
	mimeType      string
	modTime       time.Time
	minified      bool
	digest        []byte
	gzippedDigest []byte
}

// content returns the content of the a with the gzipped.
func (a *asset) content(gzipped bool) []byte {
	var c []byte
	if gzipped {
		c = a.coffer.cache.Get(nil, a.gzippedDigest)
	} else {
		c = a.coffer.cache.Get(nil, a.digest)
	}

	if len(c) == 0 {
		a.coffer.assets.Delete(a.name)
		a.coffer.cache.Del(a.digest)
		if a.gzippedDigest != nil {
			a.coffer.cache.Del(a.gzippedDigest)
		}

		return nil
	}

	return c
}
<commit_msg>fix: correct `coffer`<commit_after>package air

import (
	"bytes"
	"compress/gzip"
	"encoding/binary"
	"io/ioutil"
	"mime"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/VictoriaMetrics/fastcache"
	"github.com/aofei/mimesniffer"
	"github.com/cespare/xxhash"
	"github.com/fsnotify/fsnotify"
)

// coffer is a binary asset file manager that uses runtime memory to reduce disk
// I/O pressure.
type coffer struct {
	a         *Air
	loadOnce  *sync.Once
	loadError error
	watcher   *fsnotify.Watcher
	assets    *sync.Map
	cache     *fastcache.Cache
}

// newCoffer returns a new instance of the `coffer` with the a.
func newCoffer(a *Air) *coffer {
	return &coffer{
		a:        a,
		loadOnce: &sync.Once{},
	}
}

// load loads the stuff of the c up.
func (c *coffer) load() {
	defer func() {
		if c.loadError != nil {
			c.loadOnce = &sync.Once{}
		}
	}()

	if c.watcher == nil {
		c.watcher, c.loadError = fsnotify.NewWatcher()
		if c.loadError != nil {
			return
		}

		go func() {
			for {
				select {
				case e := <-c.watcher.Events:
					ai, ok := c.assets.Load(e.Name)
					if !ok {
						break
					}

					a := ai.(*asset)
					c.assets.Delete(a.name)
					c.cache.Del(a.digest)
					if a.gzippedDigest != nil {
						c.cache.Del(a.gzippedDigest)
					}
				case err := <-c.watcher.Errors:
					c.a.errorLogger.Printf(
						"air: coffer watcher error: %v",
						err,
					)
				}
			}
		}()
	}

	c.assets = &sync.Map{}
	c.cache = fastcache.New(c.a.CofferMaxMemoryBytes)
}

// asset returns an `asset` from the c for the name.
func (c *coffer) asset(name string) (*asset, error) {
	if c.loadOnce.Do(c.load); c.loadError != nil {
		return nil, c.loadError
	} else if ai, ok := c.assets.Load(name); ok {
		return ai.(*asset), nil
	} else if ar, err := filepath.Abs(c.a.CofferAssetRoot); err != nil {
		return nil, err
	} else if !strings.HasPrefix(name, ar) {
		return nil, nil
	}

	ext := filepath.Ext(name)
	if !stringSliceContainsCIly(c.a.CofferAssetExts, ext) {
		return nil, nil
	}

	fi, err := os.Stat(name)
	if err != nil {
		return nil, err
	}

	b, err := ioutil.ReadFile(name)
	if err != nil {
		return nil, err
	}

	var (
		mt       = mime.TypeByExtension(ext)
		minified bool
		gb       []byte
	)

	if mt == "" {
		mt = mimesniffer.Sniff(b)
	}

	pmt, _, err := mime.ParseMediaType(mt)
	if err != nil {
		return nil, err
	}

	if c.a.MinifierEnabled &&
		stringSliceContainsCIly(c.a.MinifierMIMETypes, pmt) {
		if b, err = c.a.minifier.minify(pmt, b); err != nil {
			return nil, err
		}

		minified = true
	}

	if c.a.GzipEnabled && stringSliceContainsCIly(c.a.GzipMIMETypes, pmt) {
		buf := bytes.Buffer{}
		if gw, err := gzip.NewWriterLevel(
			&buf,
			c.a.GzipCompressionLevel,
		); err != nil {
			return nil, err
		} else if _, err = gw.Write(b); err != nil {
			return nil, err
		} else if err = gw.Close(); err != nil {
			return nil, err
		}

		gb = buf.Bytes()
	}

	if err := c.watcher.Add(name); err != nil {
		return nil, err
	}

	a := &asset{
		coffer:   c,
		name:     name,
		mimeType: mt,
		modTime:  fi.ModTime(),
		minified: minified,
		digest:   make([]byte, 8),
	}

	binary.BigEndian.PutUint64(a.digest, xxhash.Sum64(b))
	c.cache.SetBig(a.digest, b)

	if gb != nil {
		a.gzippedDigest = make([]byte, 8)
		binary.BigEndian.PutUint64(a.gzippedDigest, xxhash.Sum64(gb))
		c.cache.SetBig(a.gzippedDigest, gb)
	}

	c.assets.Store(name, a)

	return a, nil
}

// asset is a binary asset file.
type asset struct {
	coffer        *coffer
	name          string
	mimeType      string
	modTime       time.Time
	minified      bool
	digest        []byte
	gzippedDigest []byte
}

// content returns the content of the a with the gzipped.
func (a *asset) content(gzipped bool) []byte {
	var c []byte
	if gzipped {
		c = a.coffer.cache.GetBig(nil, a.gzippedDigest)
	} else {
		c = a.coffer.cache.GetBig(nil, a.digest)
	}

	if len(c) == 0 {
		a.coffer.assets.Delete(a.name)
		a.coffer.cache.Del(a.digest)
		if a.gzippedDigest != nil {
			a.coffer.cache.Del(a.gzippedDigest)
		}

		return nil
	}

	return c
}
<|endoftext|>


<commit_before>package app

import (
	"errors"
	"fmt"
	"net/http"
	"time"

	"appengine"
	"appengine/datastore"
	"appengine/user"
)

type uLogData struct {
	Time time.Time
}

func logUser(c appengine.Context) error {
	u := user.Current(c)
	if u == nil {
		return errors.New("User should be logged in before calling logUser.")
	}
	key := datastore.NewKey(c, "userLog", u.ID, 0, nil)
	now := uLogData{time.Now()}
	_, err := datastore.Put(c, key, &now)
	return err
}

func vanity(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	since := time.Now().Add(-30 * 24 * time.Hour)
	q := datastore.NewQuery("userLog").Filter("Time>", since).Order("-Time")
	count, err := q.Count(c)
	if err != nil {
		handleError(w, err)
		return
	}
	fmt.Fprintf(w, "%d unique visitors in the last 30 days. First 100:\n", count)
	q = q.Limit(100)
	times := make([]uLogData, 0, 100)
	ks, err := q.GetAll(c, &times)
	if err != nil {
		handleError(w, err)
		return
	}
	for i, k := range ks {
		fmt.Fprintf(w, "%v: %s\n", times[i].Time, k.StringID())
	}
	fmt.Fprintf(w, "I am %s\n", user.Current(c).ID)
}
<commit_msg>Content type for vanity page<commit_after>package app

import (
	"errors"
	"fmt"
	"net/http"
	"time"

	"appengine"
	"appengine/datastore"
	"appengine/user"
)

type uLogData struct {
	Time time.Time
}

func logUser(c appengine.Context) error {
	u := user.Current(c)
	if u == nil {
		return errors.New("User should be logged in before calling logUser.")
	}
	key := datastore.NewKey(c, "userLog", u.ID, 0, nil)
	now := uLogData{time.Now()}
	_, err := datastore.Put(c, key, &now)
	return err
}

func vanity(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain")
	c := appengine.NewContext(r)
	since := time.Now().Add(-30 * 24 * time.Hour)
	q := datastore.NewQuery("userLog").Filter("Time>", since).Order("-Time")
	count, err := q.Count(c)
	if err != nil {
		handleError(w, err)
		return
	}
	fmt.Fprintf(w, "%d unique visitors in the last 30 days. First 100:\n", count)
	q = q.Limit(100)
	times := make([]uLogData, 0, 100)
	ks, err := q.GetAll(c, &times)
	if err != nil {
		handleError(w, err)
		return
	}
	for i, k := range ks {
		fmt.Fprintf(w, "%v: %s\n", times[i].Time, k.StringID())
	}
	fmt.Fprintf(w, "I am %s\n", user.Current(c).ID)
}
<|endoftext|>


<commit_before>package dnsr

var root = `
;       This file holds the information on root name servers needed to
;       initialize cache of Internet domain name servers
;       (e.g. reference this file in the "cache  .  <file>"
;       configuration file of BIND domain name servers).
;
;       This file is made available by InterNIC
;       under anonymous FTP as
;           file                /domain/named.cache
;           on server           FTP.INTERNIC.NET
;       -OR-                    RS.INTERNIC.NET
;
;       last update:     February 16, 2022
;       related version of root zone:     2022021601
;
; FORMERLY NS.INTERNIC.NET
;
.                        3600000      NS    A.ROOT-SERVERS.NET.
A.ROOT-SERVERS.NET.      3600000      A     198.41.0.4
A.ROOT-SERVERS.NET.      3600000      AAAA  2001:503:ba3e::2:30
;
; FORMERLY NS1.ISI.EDU
;
.                        3600000      NS    B.ROOT-SERVERS.NET.
B.ROOT-SERVERS.NET.      3600000      A     199.9.14.201
B.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:200::b
;
; FORMERLY C.PSI.NET
;
.                        3600000      NS    C.ROOT-SERVERS.NET.
C.ROOT-SERVERS.NET.      3600000      A     192.33.4.12
C.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2::c
;
; FORMERLY TERP.UMD.EDU
;
.                        3600000      NS    D.ROOT-SERVERS.NET.
D.ROOT-SERVERS.NET.      3600000      A     199.7.91.13
D.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2d::d
;
; FORMERLY NS.NASA.GOV
;
.                        3600000      NS    E.ROOT-SERVERS.NET.
E.ROOT-SERVERS.NET.      3600000      A     192.203.230.10
E.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:a8::e
;
; FORMERLY NS.ISC.ORG
;
.                        3600000      NS    F.ROOT-SERVERS.NET.
F.ROOT-SERVERS.NET.      3600000      A     192.5.5.241
F.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2f::f
;
; FORMERLY NS.NIC.DDN.MIL
;
.                        3600000      NS    G.ROOT-SERVERS.NET.
G.ROOT-SERVERS.NET.      3600000      A     192.112.36.4
G.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:12::d0d
;
; FORMERLY AOS.ARL.ARMY.MIL
;
.                        3600000      NS    H.ROOT-SERVERS.NET.
H.ROOT-SERVERS.NET.      3600000      A     198.97.190.53
H.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:1::53
;
; FORMERLY NIC.NORDU.NET
;
.                        3600000      NS    I.ROOT-SERVERS.NET.
I.ROOT-SERVERS.NET.      3600000      A     192.36.148.17
I.ROOT-SERVERS.NET.      3600000      AAAA  2001:7fe::53
;
; OPERATED BY VERISIGN, INC.
;
.                        3600000      NS    J.ROOT-SERVERS.NET.
J.ROOT-SERVERS.NET.      3600000      A     192.58.128.30
J.ROOT-SERVERS.NET.      3600000      AAAA  2001:503:c27::2:30
;
; OPERATED BY RIPE NCC
;
.                        3600000      NS    K.ROOT-SERVERS.NET.
K.ROOT-SERVERS.NET.      3600000      A     193.0.14.129
K.ROOT-SERVERS.NET.      3600000      AAAA  2001:7fd::1
;
; OPERATED BY ICANN
;
.                        3600000      NS    L.ROOT-SERVERS.NET.
L.ROOT-SERVERS.NET.      3600000      A     199.7.83.42
L.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:9f::42
;
; OPERATED BY WIDE
;
.                        3600000      NS    M.ROOT-SERVERS.NET.
M.ROOT-SERVERS.NET.      3600000      A     202.12.27.33
M.ROOT-SERVERS.NET.      3600000      AAAA  2001:dc3::35
; End of file`
<commit_msg>Automatic update for Thu Mar 17 08:19:42 UTC 2022<commit_after>package dnsr

var root = `
;       This file holds the information on root name servers needed to
;       initialize cache of Internet domain name servers
;       (e.g. reference this file in the "cache  .  <file>"
;       configuration file of BIND domain name servers).
;
;       This file is made available by InterNIC
;       under anonymous FTP as
;           file                /domain/named.cache
;           on server           FTP.INTERNIC.NET
;       -OR-                    RS.INTERNIC.NET
;
;       last update:     March 16, 2022
;       related version of root zone:     2022031601
;
; FORMERLY NS.INTERNIC.NET
;
.                        3600000      NS    A.ROOT-SERVERS.NET.
A.ROOT-SERVERS.NET.      3600000      A     198.41.0.4
A.ROOT-SERVERS.NET.      3600000      AAAA  2001:503:ba3e::2:30
;
; FORMERLY NS1.ISI.EDU
;
.                        3600000      NS    B.ROOT-SERVERS.NET.
B.ROOT-SERVERS.NET.      3600000      A     199.9.14.201
B.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:200::b
;
; FORMERLY C.PSI.NET
;
.                        3600000      NS    C.ROOT-SERVERS.NET.
C.ROOT-SERVERS.NET.      3600000      A     192.33.4.12
C.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2::c
;
; FORMERLY TERP.UMD.EDU
;
.                        3600000      NS    D.ROOT-SERVERS.NET.
D.ROOT-SERVERS.NET.      3600000      A     199.7.91.13
D.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2d::d
;
; FORMERLY NS.NASA.GOV
;
.                        3600000      NS    E.ROOT-SERVERS.NET.
E.ROOT-SERVERS.NET.      3600000      A     192.203.230.10
E.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:a8::e
;
; FORMERLY NS.ISC.ORG
;
.                        3600000      NS    F.ROOT-SERVERS.NET.
F.ROOT-SERVERS.NET.      3600000      A     192.5.5.241
F.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2f::f
;
; FORMERLY NS.NIC.DDN.MIL
;
.                        3600000      NS    G.ROOT-SERVERS.NET.
G.ROOT-SERVERS.NET.      3600000      A     192.112.36.4
G.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:12::d0d
;
; FORMERLY AOS.ARL.ARMY.MIL
;
.                        3600000      NS    H.ROOT-SERVERS.NET.
H.ROOT-SERVERS.NET.      3600000      A     198.97.190.53
H.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:1::53
;
; FORMERLY NIC.NORDU.NET
;
.                        3600000      NS    I.ROOT-SERVERS.NET.
I.ROOT-SERVERS.NET.      3600000      A     192.36.148.17
I.ROOT-SERVERS.NET.      3600000      AAAA  2001:7fe::53
;
; OPERATED BY VERISIGN, INC.
;
.                        3600000      NS    J.ROOT-SERVERS.NET.
J.ROOT-SERVERS.NET.      3600000      A     192.58.128.30
J.ROOT-SERVERS.NET.      3600000      AAAA  2001:503:c27::2:30
;
; OPERATED BY RIPE NCC
;
.                        3600000      NS    K.ROOT-SERVERS.NET.
K.ROOT-SERVERS.NET.      3600000      A     193.0.14.129
K.ROOT-SERVERS.NET.      3600000      AAAA  2001:7fd::1
;
; OPERATED BY ICANN
;
.                        3600000      NS    L.ROOT-SERVERS.NET.
L.ROOT-SERVERS.NET.      3600000      A     199.7.83.42
L.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:9f::42
;
; OPERATED BY WIDE
;
.                        3600000      NS    M.ROOT-SERVERS.NET.
M.ROOT-SERVERS.NET.      3600000      A     202.12.27.33
M.ROOT-SERVERS.NET.      3600000      AAAA  2001:dc3::35
; End of file`
<|endoftext|>


<commit_before>package httpsnoop

import (
	"bytes"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"testing"
	"time"
)

func TestSnoopResponserWriter_interfaces(t *testing.T) {
	// @TODO(fg) is there a better way to test this? Perhaps using reflection?
	tests := []struct {
		W                 http.ResponseWriter
		WantFlusher       bool
		WantCloseNotifier bool
		WantReaderFrom    bool
		WantHijacker      bool
	}{
		{
			W: struct {
				http.ResponseWriter
				http.Hijacker
			}{},
			WantHijacker:      true,
			WantFlusher:       false,
			WantCloseNotifier: false,
			WantReaderFrom:    false,
		},
		{
			W: struct {
				http.ResponseWriter
				http.Hijacker
				http.Flusher
			}{},
			WantHijacker:      true,
			WantFlusher:       true,
			WantCloseNotifier: false,
			WantReaderFrom:    false,
		},
		{
			W: struct {
				http.ResponseWriter
				http.Hijacker
				http.CloseNotifier
			}{},
			WantHijacker:      true,
			WantFlusher:       false,
			WantCloseNotifier: true,
			WantReaderFrom:    false,
		},
		{
			W: struct {
				http.ResponseWriter
				http.Hijacker
				http.Flusher
				http.CloseNotifier
			}{},
			WantHijacker:      true,
			WantFlusher:       true,
			WantCloseNotifier: true,
			WantReaderFrom:    false,
		},
		{
			W: struct {
				http.ResponseWriter
				http.Hijacker
				io.ReaderFrom
			}{},
			WantHijacker:      true,
			WantFlusher:       false,
			WantCloseNotifier: false,
			WantReaderFrom:    true,
		},
		{
			W: struct {
				http.ResponseWriter
				http.Hijacker
				http.Flusher
				io.ReaderFrom
			}{},
			WantHijacker:      true,
			WantFlusher:       true,
			WantCloseNotifier: false,
			WantReaderFrom:    true,
		},
		{
			W: struct {
				http.ResponseWriter
				http.Hijacker
				http.CloseNotifier
				io.ReaderFrom
			}{},
			WantHijacker:      true,
			WantFlusher:       false,
			WantCloseNotifier: true,
			WantReaderFrom:    true,
		},
		{
			W: struct {
				http.ResponseWriter
				http.Flusher
				http.CloseNotifier
				io.ReaderFrom
			}{},
			WantHijacker:      true,
			WantFlusher:       true,
			WantCloseNotifier: true,
			WantReaderFrom:    true,
		},
		{
			W:                 struct{ http.ResponseWriter }{},
			WantHijacker:      false,
			WantFlusher:       false,
			WantCloseNotifier: false,
			WantReaderFrom:    false,
		},
		{
			W: struct {
				http.ResponseWriter
				http.Flusher
			}{},
			WantHijacker:      false,
			WantFlusher:       true,
			WantCloseNotifier: false,
			WantReaderFrom:    false,
		},
		{
			W: struct {
				http.ResponseWriter
				http.CloseNotifier
			}{},
			WantHijacker:      false,
			WantFlusher:       false,
			WantCloseNotifier: true,
			WantReaderFrom:    false,
		},
		{
			W: struct {
				http.ResponseWriter
				http.Flusher
				http.CloseNotifier
			}{},
			WantHijacker:      false,
			WantFlusher:       true,
			WantCloseNotifier: true,
			WantReaderFrom:    false,
		},
		{
			W: struct {
				http.ResponseWriter
				io.ReaderFrom
			}{},
			WantHijacker:      false,
			WantFlusher:       false,
			WantCloseNotifier: false,
			WantReaderFrom:    true,
		},
		{
			W: struct {
				http.ResponseWriter
				http.Flusher
				io.ReaderFrom
			}{},
			WantHijacker:      false,
			WantFlusher:       true,
			WantCloseNotifier: false,
			WantReaderFrom:    true,
		},
		{
			W: struct {
				http.ResponseWriter
				http.CloseNotifier
				io.ReaderFrom
			}{},
			WantHijacker:      false,
			WantFlusher:       false,
			WantCloseNotifier: true,
			WantReaderFrom:    true,
		},
		{
			W: struct {
				http.ResponseWriter
				http.Flusher
				http.CloseNotifier
				io.ReaderFrom
			}{},
			WantHijacker:      false,
			WantFlusher:       true,
			WantCloseNotifier: true,
			WantReaderFrom:    true,
		},
	}
	for i, test := range tests {
		sw := Wrap(test.W, Hooks{})
		if _, got := sw.(http.CloseNotifier); got != test.WantCloseNotifier {
			t.Errorf("#%d: got=%t want=%t", i, got, test.WantCloseNotifier)
		}
		if _, got := sw.(http.Flusher); got != test.WantFlusher {
			t.Errorf("#%d: got=%t want=%t", i, got, test.WantFlusher)
		}
	}
}

func TestSnoopResponserWriter_integration(t *testing.T) {
	tests := []struct {
		Name     string
		Handler  http.Handler
		Hooks    Hooks
		WantCode int
		WantBody []byte
	}{
		{
			Name: "WriteHeader (no hook)",
			Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				w.WriteHeader(http.StatusNotFound)
			}),
			WantCode: http.StatusNotFound,
		},
		{
			Name: "WriteHeader",
			Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				w.WriteHeader(http.StatusNotFound)
			}),
			Hooks: Hooks{
				WriteHeader: func(next WriteHeaderFunc) WriteHeaderFunc {
					return func(code int) {
						if code != http.StatusNotFound {
							t.Errorf("got=%d want=%d", code, http.StatusNotFound)
						}
						next(http.StatusForbidden)
					}
				},
			},
			WantCode: http.StatusForbidden,
		},

		{
			Name: "Write (no hook)",
			Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				w.Write([]byte("foo"))
			}),
			WantCode: http.StatusOK,
			WantBody: []byte("foo"),
		},
		{
			Name: "Write (rewrite hook)",
			Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				if n, err := w.Write([]byte("foo")); err != nil {
					t.Errorf("got=%s", err)
				} else if got, want := n, len("foobar"); got != want {
					t.Errorf("got=%d want=%d", got, want)
				}
			}),
			Hooks: Hooks{
				Write: func(next WriteFunc) WriteFunc {
					return func(p []byte) (int, error) {
						if string(p) != "foo" {
							t.Errorf("%s", p)
						}
						return next([]byte("foobar"))
					}
				},
			},
			WantCode: http.StatusOK,
			WantBody: []byte("foobar"),
		},
		{
			Name: "Write (error hook)",
			Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				if n, err := w.Write([]byte("foo")); n != 0 {
					t.Errorf("got=%d want=%d", n, 0)
				} else if err != io.EOF {
					t.Errorf("got=%s want=%s", err, io.EOF)
				}
			}),
			Hooks: Hooks{
				Write: func(next WriteFunc) WriteFunc {
					return func(p []byte) (int, error) {
						if string(p) != "foo" {
							t.Errorf("%s", p)
						}
						return 0, io.EOF
					}
				},
			},
			WantCode: http.StatusOK,
		},
	}

	for _, test := range tests {
		func() {
			h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				sw := Wrap(w, test.Hooks)
				test.Handler.ServeHTTP(sw, r)
			})
			s := httptest.NewServer(h)
			defer s.Close()
			res, err := http.Get(s.URL)
			if err != nil {
				t.Fatal(err)
			}
			defer res.Body.Close()
			gotBody, err := ioutil.ReadAll(res.Body)
			if res.StatusCode != test.WantCode {
				t.Errorf("got=%d want=%d", res.StatusCode, test.WantCode)
			} else if !bytes.Equal(gotBody, test.WantBody) {
				t.Errorf("got=%s want=%s", gotBody, test.WantBody)
			}
		}()
	}
}

func TestSnoopHandlerMetrics(t *testing.T) {
	tests := []struct {
		Handler      http.Handler
		WantDuration time.Duration
		WantWritten  int64
		WantCode     int
	}{
		{
			Handler:  http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}),
			WantCode: http.StatusOK,
		},
		{
			Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				w.WriteHeader(http.StatusBadRequest)
				w.WriteHeader(http.StatusNotFound)
				w.Write([]byte("foo"))
				w.Write([]byte("bar"))
				time.Sleep(25 * time.Millisecond)
			}),
			WantCode:     http.StatusBadRequest,
			WantWritten:  6,
			WantDuration: 25 * time.Millisecond,
		},
	}

	for _, test := range tests {
		func() {
			ch := make(chan Metrics, 1)
			h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				ch <- CaptureMetrics(test.Handler, w, r)
			})
			s := httptest.NewServer(h)
			defer s.Close()
			res, err := http.Get(s.URL)
			if err != nil {
				t.Fatal(err)
			}
			defer res.Body.Close()
			m := <-ch
			if m.Code != test.WantCode {
				t.Errorf("got=%d want=%d", m.Code, test.WantCode)
			} else if m.Duration < test.WantDuration {
				t.Errorf("got=%s want=%s", m.Duration, test.WantDuration)
			} else if m.Written < test.WantWritten {
				t.Errorf("got=%d want=%d", m.Written, test.WantWritten)
			}
		}()
	}
}
<commit_msg>fix method name<commit_after>package httpsnoop

import (
	"bytes"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"testing"
	"time"
)

func TestSnoopResponserWriter_interfaces(t *testing.T) {
	// @TODO(fg) is there a better way to test this? Perhaps using reflection?
	tests := []struct {
		W                 http.ResponseWriter
		WantFlusher       bool
		WantCloseNotifier bool
		WantReaderFrom    bool
		WantHijacker      bool
	}{
		{
			W: struct {
				http.ResponseWriter
				http.Hijacker
			}{},
			WantHijacker:      true,
			WantFlusher:       false,
			WantCloseNotifier: false,
			WantReaderFrom:    false,
		},
		{
			W: struct {
				http.ResponseWriter
				http.Hijacker
				http.Flusher
			}{},
			WantHijacker:      true,
			WantFlusher:       true,
			WantCloseNotifier: false,
			WantReaderFrom:    false,
		},
		{
			W: struct {
				http.ResponseWriter
				http.Hijacker
				http.CloseNotifier
			}{},
			WantHijacker:      true,
			WantFlusher:       false,
			WantCloseNotifier: true,
			WantReaderFrom:    false,
		},
		{
			W: struct {
				http.ResponseWriter
				http.Hijacker
				http.Flusher
				http.CloseNotifier
			}{},
			WantHijacker:      true,
			WantFlusher:       true,
			WantCloseNotifier: true,
			WantReaderFrom:    false,
		},
		{
			W: struct {
				http.ResponseWriter
				http.Hijacker
				io.ReaderFrom
			}{},
			WantHijacker:      true,
			WantFlusher:       false,
			WantCloseNotifier: false,
			WantReaderFrom:    true,
		},
		{
			W: struct {
				http.ResponseWriter
				http.Hijacker
				http.Flusher
				io.ReaderFrom
			}{},
			WantHijacker:      true,
			WantFlusher:       true,
			WantCloseNotifier: false,
			WantReaderFrom:    true,
		},
		{
			W: struct {
				http.ResponseWriter
				http.Hijacker
				http.CloseNotifier
				io.ReaderFrom
			}{},
			WantHijacker:      true,
			WantFlusher:       false,
			WantCloseNotifier: true,
			WantReaderFrom:    true,
		},
		{
			W: struct {
				http.ResponseWriter
				http.Flusher
				http.CloseNotifier
				io.ReaderFrom
			}{},
			WantHijacker:      true,
			WantFlusher:       true,
			WantCloseNotifier: true,
			WantReaderFrom:    true,
		},
		{
			W:                 struct{ http.ResponseWriter }{},
			WantHijacker:      false,
			WantFlusher:       false,
			WantCloseNotifier: false,
			WantReaderFrom:    false,
		},
		{
			W: struct {
				http.ResponseWriter
				http.Flusher
			}{},
			WantHijacker:      false,
			WantFlusher:       true,
			WantCloseNotifier: false,
			WantReaderFrom:    false,
		},
		{
			W: struct {
				http.ResponseWriter
				http.CloseNotifier
			}{},
			WantHijacker:      false,
			WantFlusher:       false,
			WantCloseNotifier: true,
			WantReaderFrom:    false,
		},
		{
			W: struct {
				http.ResponseWriter
				http.Flusher
				http.CloseNotifier
			}{},
			WantHijacker:      false,
			WantFlusher:       true,
			WantCloseNotifier: true,
			WantReaderFrom:    false,
		},
		{
			W: struct {
				http.ResponseWriter
				io.ReaderFrom
			}{},
			WantHijacker:      false,
			WantFlusher:       false,
			WantCloseNotifier: false,
			WantReaderFrom:    true,
		},
		{
			W: struct {
				http.ResponseWriter
				http.Flusher
				io.ReaderFrom
			}{},
			WantHijacker:      false,
			WantFlusher:       true,
			WantCloseNotifier: false,
			WantReaderFrom:    true,
		},
		{
			W: struct {
				http.ResponseWriter
				http.CloseNotifier
				io.ReaderFrom
			}{},
			WantHijacker:      false,
			WantFlusher:       false,
			WantCloseNotifier: true,
			WantReaderFrom:    true,
		},
		{
			W: struct {
				http.ResponseWriter
				http.Flusher
				http.CloseNotifier
				io.ReaderFrom
			}{},
			WantHijacker:      false,
			WantFlusher:       true,
			WantCloseNotifier: true,
			WantReaderFrom:    true,
		},
	}
	for i, test := range tests {
		sw := Wrap(test.W, Hooks{})
		if _, got := sw.(http.CloseNotifier); got != test.WantCloseNotifier {
			t.Errorf("#%d: got=%t want=%t", i, got, test.WantCloseNotifier)
		}
		if _, got := sw.(http.Flusher); got != test.WantFlusher {
			t.Errorf("#%d: got=%t want=%t", i, got, test.WantFlusher)
		}
	}
}

func TestSnoopResponserWriter_integration(t *testing.T) {
	tests := []struct {
		Name     string
		Handler  http.Handler
		Hooks    Hooks
		WantCode int
		WantBody []byte
	}{
		{
			Name: "WriteHeader (no hook)",
			Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				w.WriteHeader(http.StatusNotFound)
			}),
			WantCode: http.StatusNotFound,
		},
		{
			Name: "WriteHeader",
			Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				w.WriteHeader(http.StatusNotFound)
			}),
			Hooks: Hooks{
				WriteHeader: func(next WriteHeaderFunc) WriteHeaderFunc {
					return func(code int) {
						if code != http.StatusNotFound {
							t.Errorf("got=%d want=%d", code, http.StatusNotFound)
						}
						next(http.StatusForbidden)
					}
				},
			},
			WantCode: http.StatusForbidden,
		},

		{
			Name: "Write (no hook)",
			Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				w.Write([]byte("foo"))
			}),
			WantCode: http.StatusOK,
			WantBody: []byte("foo"),
		},
		{
			Name: "Write (rewrite hook)",
			Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				if n, err := w.Write([]byte("foo")); err != nil {
					t.Errorf("got=%s", err)
				} else if got, want := n, len("foobar"); got != want {
					t.Errorf("got=%d want=%d", got, want)
				}
			}),
			Hooks: Hooks{
				Write: func(next WriteFunc) WriteFunc {
					return func(p []byte) (int, error) {
						if string(p) != "foo" {
							t.Errorf("%s", p)
						}
						return next([]byte("foobar"))
					}
				},
			},
			WantCode: http.StatusOK,
			WantBody: []byte("foobar"),
		},
		{
			Name: "Write (error hook)",
			Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				if n, err := w.Write([]byte("foo")); n != 0 {
					t.Errorf("got=%d want=%d", n, 0)
				} else if err != io.EOF {
					t.Errorf("got=%s want=%s", err, io.EOF)
				}
			}),
			Hooks: Hooks{
				Write: func(next WriteFunc) WriteFunc {
					return func(p []byte) (int, error) {
						if string(p) != "foo" {
							t.Errorf("%s", p)
						}
						return 0, io.EOF
					}
				},
			},
			WantCode: http.StatusOK,
		},
	}

	for _, test := range tests {
		func() {
			h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				sw := Wrap(w, test.Hooks)
				test.Handler.ServeHTTP(sw, r)
			})
			s := httptest.NewServer(h)
			defer s.Close()
			res, err := http.Get(s.URL)
			if err != nil {
				t.Fatal(err)
			}
			defer res.Body.Close()
			gotBody, err := ioutil.ReadAll(res.Body)
			if res.StatusCode != test.WantCode {
				t.Errorf("got=%d want=%d", res.StatusCode, test.WantCode)
			} else if !bytes.Equal(gotBody, test.WantBody) {
				t.Errorf("got=%s want=%s", gotBody, test.WantBody)
			}
		}()
	}
}

func TestCaptureMetrics(t *testing.T) {
	tests := []struct {
		Handler      http.Handler
		WantDuration time.Duration
		WantWritten  int64
		WantCode     int
	}{
		{
			Handler:  http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}),
			WantCode: http.StatusOK,
		},
		{
			Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				w.WriteHeader(http.StatusBadRequest)
				w.WriteHeader(http.StatusNotFound)
				w.Write([]byte("foo"))
				w.Write([]byte("bar"))
				time.Sleep(25 * time.Millisecond)
			}),
			WantCode:     http.StatusBadRequest,
			WantWritten:  6,
			WantDuration: 25 * time.Millisecond,
		},
	}

	for _, test := range tests {
		func() {
			ch := make(chan Metrics, 1)
			h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				ch <- CaptureMetrics(test.Handler, w, r)
			})
			s := httptest.NewServer(h)
			defer s.Close()
			res, err := http.Get(s.URL)
			if err != nil {
				t.Fatal(err)
			}
			defer res.Body.Close()
			m := <-ch
			if m.Code != test.WantCode {
				t.Errorf("got=%d want=%d", m.Code, test.WantCode)
			} else if m.Duration < test.WantDuration {
				t.Errorf("got=%s want=%s", m.Duration, test.WantDuration)
			} else if m.Written < test.WantWritten {
				t.Errorf("got=%d want=%d", m.Written, test.WantWritten)
			}
		}()
	}
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"log"

	"github.com/fatih/color"
)

func failure(err error) {
	c := color.New(color.FgRed)
	if _, err := c.Fprintf(writer, "%s\n", err.Error()); err != nil {
		log.Fatal(err)
	}
}

func printWithNewLine(str string) {
	if _, err := fmt.Fprintf(writer, "%s\n", str); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>cmd : fix variable shadowed<commit_after>package cmd

import (
	"fmt"
	"log"

	"github.com/fatih/color"
)

func failure(err error) {
	c := color.New(color.FgRed)
	if _, ferr := c.Fprintf(writer, "%s\n", err.Error()); ferr != nil {
		log.Fatal(ferr)
	}
}

func printWithNewLine(str string) {
	if _, err := fmt.Fprintf(writer, "%s\n", str); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package ice

import (
	"regexp"
	"sync"
	"testing"
)

func TestMathRandomGenerator(t *testing.T) {
	g := newMathRandomGenerator()
	isLetter := regexp.MustCompile(`^[a-zA-Z]+$`).MatchString

	for i := 0; i < 10000; i++ {
		s := g.GenerateString(10, runesAlpha)
		if len(s) != 10 {
			t.Error("Generator returned invalid length")
		}
		if !isLetter(s) {
			t.Errorf("Generator returned unexpected character: %s", s)
		}
	}
}

func TestCryptoRandomGenerator(t *testing.T) {
	isLetter := regexp.MustCompile(`^[a-zA-Z]+$`).MatchString

	for i := 0; i < 10000; i++ {
		s, err := generateCryptoRandomString(10, runesAlpha)
		if err != nil {
			t.Error(err)
		}
		if len(s) != 10 {
			t.Error("Generator returned invalid length")
		}
		if !isLetter(s) {
			t.Errorf("Generator returned unexpected character: %s", s)
		}
	}
}

func TestRandomGeneratorCollision(t *testing.T) {
	candidateIDGen := newCandidateIDGenerator()

	testCases := map[string]struct {
		gen func(t *testing.T) string
	}{
		"CandidateID": {
			gen: func(t *testing.T) string {
				return candidateIDGen.Generate()
			},
		},
		"PWD": {
			gen: func(t *testing.T) string {
				s, err := generatePwd()
				if err != nil {
					t.Fatal(err)
				}
				return s
			},
		},
		"Ufrag": {
			gen: func(t *testing.T) string {
				s, err := generateUFrag()
				if err != nil {
					t.Fatal(err)
				}
				return s
			},
		},
	}

	const N = 1000
	const iteration = 100

	for name, testCase := range testCases {
		testCase := testCase
		t.Run(name, func(t *testing.T) {
			for iter := 0; iter < iteration; iter++ {
				var wg sync.WaitGroup
				var mu sync.Mutex

				rands := make([]string, 0, N)

				for i := 0; i < N; i++ {
					wg.Add(1)
					go func() {
						r := testCase.gen(t)
						mu.Lock()
						rands = append(rands, r)
						mu.Unlock()
						wg.Done()
					}()
				}
				wg.Wait()

				if len(rands) != N {
					t.Fatal("Failed to generate randoms")
				}

				for i := 0; i < N; i++ {
					for j := i + 1; j < N; j++ {
						if rands[i] == rands[j] {
							t.Fatalf("generateRandString caused collision: %s == %s", rands[i], rands[j])
						}
					}
				}
			}
		})
	}
}
<commit_msg>Reduce iteration in TestRandomGeneratorCollision<commit_after>package ice

import (
	"regexp"
	"sync"
	"testing"
)

func TestMathRandomGenerator(t *testing.T) {
	g := newMathRandomGenerator()
	isLetter := regexp.MustCompile(`^[a-zA-Z]+$`).MatchString

	for i := 0; i < 10000; i++ {
		s := g.GenerateString(10, runesAlpha)
		if len(s) != 10 {
			t.Error("Generator returned invalid length")
		}
		if !isLetter(s) {
			t.Errorf("Generator returned unexpected character: %s", s)
		}
	}
}

func TestCryptoRandomGenerator(t *testing.T) {
	isLetter := regexp.MustCompile(`^[a-zA-Z]+$`).MatchString

	for i := 0; i < 10000; i++ {
		s, err := generateCryptoRandomString(10, runesAlpha)
		if err != nil {
			t.Error(err)
		}
		if len(s) != 10 {
			t.Error("Generator returned invalid length")
		}
		if !isLetter(s) {
			t.Errorf("Generator returned unexpected character: %s", s)
		}
	}
}

func TestRandomGeneratorCollision(t *testing.T) {
	candidateIDGen := newCandidateIDGenerator()

	testCases := map[string]struct {
		gen func(t *testing.T) string
	}{
		"CandidateID": {
			gen: func(t *testing.T) string {
				return candidateIDGen.Generate()
			},
		},
		"PWD": {
			gen: func(t *testing.T) string {
				s, err := generatePwd()
				if err != nil {
					t.Fatal(err)
				}
				return s
			},
		},
		"Ufrag": {
			gen: func(t *testing.T) string {
				s, err := generateUFrag()
				if err != nil {
					t.Fatal(err)
				}
				return s
			},
		},
	}

	const N = 100
	const iteration = 100

	for name, testCase := range testCases {
		testCase := testCase
		t.Run(name, func(t *testing.T) {
			for iter := 0; iter < iteration; iter++ {
				var wg sync.WaitGroup
				var mu sync.Mutex

				rands := make([]string, 0, N)

				for i := 0; i < N; i++ {
					wg.Add(1)
					go func() {
						r := testCase.gen(t)
						mu.Lock()
						rands = append(rands, r)
						mu.Unlock()
						wg.Done()
					}()
				}
				wg.Wait()

				if len(rands) != N {
					t.Fatal("Failed to generate randoms")
				}

				for i := 0; i < N; i++ {
					for j := i + 1; j < N; j++ {
						if rands[i] == rands[j] {
							t.Fatalf("generateRandString caused collision: %s == %s", rands[i], rands[j])
						}
					}
				}
			}
		})
	}
}
<|endoftext|>


<commit_before>package main

import (
"encoding/json"
	"github.com/iambc/xerrors"
	"github.com/golang/glog"
	"strconv"
	)

func getBoards(apiKey string)  ([]byte, error) {

    rows, err := dbh.Query("select b.id, b.name, b.descr from boards b join image_board_clusters ibc on ibc.id = b.image_board_cluster_id where api_key = $1;", apiKey)
    if err != nil {
	return []byte{}, xerrors.NewUIErr(err.Error(), err.Error(), `002`, true)
    }
    defer rows.Close()

    var curr_boards []boards
    for rows.Next() {
	var board boards
	err = rows.Scan(&board.Id, &board.Name, &board.Descr)
	if err != nil {
	    return []byte{}, xerrors.NewUIErr(err.Error(), err.Error(), `003`, true)
	}
	curr_boards = append(curr_boards, board)
    }
    bytes, err1 := json.Marshal(api_request{"ok", nil, &curr_boards})
    if err1 != nil {
	return []byte{}, xerrors.NewUIErr(err1.Error(), err1.Error(), `004`, true)
    }
    return bytes, nil
}

func getActiveThreadsForBoard(apiKey string, boardId int)  ([]byte, error) {

    rows, err := dbh.Query(`select t.id, t.name, count(*), (select count(*) from thread_posts where thread_id = t.id and attachment_url is not null) from threads t
				join boards b on b.id = t.board_id
				join image_board_clusters ibc on ibc.id = b.image_board_cluster_id
				left join thread_posts tp on tp.thread_id = t.id
			    where t.is_active = TRUE and t.board_id = $1 and ibc.api_key = $2 group by 1,2 order by t.id;`, boardId, apiKey)
    if err != nil {
        return []byte{}, xerrors.NewUIErr(err.Error(), err.Error(), `006`, true)
    }
    defer rows.Close()

    var active_threads []threads
    for rows.Next() {
	glog.Info("Popped new thread")
        var thread threads
        err = rows.Scan(&thread.Id, &thread.Name, &thread.PostCount, &thread.PostCountWithAttachment)
        if err != nil {
            return []byte{}, xerrors.NewUIErr(err.Error(), err.Error(), `007`, true)
        }
        active_threads = append(active_threads, thread)
    }
    var bytes []byte
    var err1 error
    if(len(active_threads) == 0){
        errMsg := "No objects returned."
        bytes, err1 = json.Marshal(api_request{"error", &errMsg, &active_threads})
    }else {
        bytes, err1 = json.Marshal(api_request{"ok", nil, &active_threads})
    }

    if err1 != nil {
        return []byte{}, xerrors.NewUIErr(err1.Error(), err1.Error(), `008`, true)
    }

    return bytes, nil
}

func getPostsForThread(apiKey string, threadId int)  ([]byte, error) {
    rows, err := dbh.Query(`select tp.id, tp.body, tp.attachment_url, tp.inserted_at, tp.source_ip
			    from thread_posts tp join threads t on t.id = tp.thread_id
						 join boards b on b.id = t.board_id
						 join image_board_clusters ibc on ibc.id = b.image_board_cluster_id
			    where tp.thread_id = $1 and ibc.api_key = $2 and t.is_active = true;`, threadId, apiKey)
    if err != nil {
	glog.Error(err)
        return []byte{}, xerrors.NewSysErr()
    }
    defer rows.Close()

    var curr_posts []thread_posts
    for rows.Next() {
	glog.Info("new post for thread with id: ", threadId)
        var curr_post thread_posts
        err = rows.Scan(&curr_post.Id, &curr_post.Body, &curr_post.AttachmentUrl, &curr_post.InsertedAt, &curr_post.SourceIp)
        if err != nil {
	    glog.Error(err)
            return []byte{}, xerrors.NewSysErr()
        }
        curr_posts = append(curr_posts, curr_post)
    }

    var bytes []byte
    var err1 error
    if(len(curr_posts) == 0){
	errMsg := "No objects returned."
	bytes, err1 = json.Marshal(api_request{"error", &errMsg, &curr_posts})
    }else {
	bytes, err1 = json.Marshal(api_request{"ok", nil, &curr_posts})
    }

    if err1 != nil {
        return []byte{}, xerrors.NewSysErr()
    }

    return bytes, nil
}

func addPostToThread(threadId int, threadBodyPost string, attachmentUrl *string, clientRemoteAddr string) ([]byte,error) {
    var is_limit_reached bool
    var max_post_length int
    var min_post_length int
    err := dbh.QueryRow("select (select count(*) from thread_posts  where thread_id = $1) > max_posts_per_thread, min_post_length, max_post_length  from threads where id = $1;", threadId).Scan(&is_limit_reached, &min_post_length, &max_post_length)
    if err != nil {
	return []byte{}, xerrors.NewUIErr(err.Error(), err.Error(), `009`, true)
    }

    if is_limit_reached {
	dbh.QueryRow("UPDATE threads set is_active = false where id = $1", threadId).Scan()
	return []byte{}, xerrors.NewUIErr(`Thread post limit reached!`, `Thread post limit reached!`, `010`, true)
    }

   if(min_post_length > len(threadBodyPost)  && min_post_length != -1){
	return []byte{}, xerrors.NewUIErr(`Post length is less than minimum length!`, `Post length is less than minimum length! post length: ` + strconv.Itoa(len(threadBodyPost))  +` min length: ` + strconv.Itoa(min_post_length) , `020`, false)
    }
   if(max_post_length < len(threadBodyPost)  && max_post_length != -1){
	return []byte{}, xerrors.NewUIErr(`Post length is more than maximum length!`, `Post length is more than maximum length! post length: ` + strconv.Itoa(len(threadBodyPost))  +` max length: ` + strconv.Itoa(max_post_length) , `021`, false)
    }

    _, err = dbh.Query("INSERT INTO thread_posts(body, thread_id, attachment_url, source_ip) VALUES($1, $2, $3, $4)", threadBodyPost, threadId, attachmentUrl, clientRemoteAddr)

    if err != nil {
	glog.Error(err)
        return []byte{}, xerrors.NewUIErr(err.Error(), err.Error(), `011`, true)
    }

    bytes, err1 := json.Marshal(api_request{"ok", nil, nil})
    if err1 != nil {
        return []byte{}, xerrors.NewUIErr(err1.Error(), err1.Error(), `012`, true)
    }

    return bytes, nil
}

func addThread(boardId int, threadName string) ([]byte, error) {
    var is_limit_reached bool
    err := dbh.QueryRow("select (select count(*) from threads  where board_id = $1) > thread_setting_max_thread_count  from boards where id = $1;", boardId).Scan(&is_limit_reached)
    if err != nil {
	glog.Error("COULD NOT SELECT thread_count")
	return []byte{}, xerrors.NewUIErr(err.Error(), err.Error(), `015`, true)
    }
    if is_limit_reached {
	return []byte{}, xerrors.NewUIErr(`Thread limit reached!`, `Thread limit reached!`, `016`, true)
    }

    var threadId int
    err = dbh.QueryRow("INSERT INTO threads(name, board_id, limits_reached_action_id, max_posts_per_thread) VALUES($1, $2, 1, 10)  RETURNING id, name", threadName, boardId).Scan(&threadId, &threadName)

    if err != nil {
	glog.Error("INSERT FAILED")
        return []byte{}, xerrors.NewUIErr(err.Error(), err.Error(), `017`, true)
    }

    a := struct{
		    Id int
		    Name string
		}{
		    threadId,
		    threadName,
		}
    bytes, err1 := json.Marshal(api_request{`ok`, nil, a })
    if err1 != nil {
        return []byte{}, xerrors.NewUIErr(err1.Error(), err1.Error(), `018`, true)
    }

    return bytes, nil
}

<commit_msg>docs: file purpose<commit_after>package main

/*
This file contains all of the API functions.
*/

import (
"encoding/json"
	"github.com/iambc/xerrors"
	"github.com/golang/glog"
	"strconv"
	)

func getBoards(apiKey string)  ([]byte, error) {

    rows, err := dbh.Query("select b.id, b.name, b.descr from boards b join image_board_clusters ibc on ibc.id = b.image_board_cluster_id where api_key = $1;", apiKey)
    if err != nil {
	return []byte{}, xerrors.NewUIErr(err.Error(), err.Error(), `002`, true)
    }
    defer rows.Close()

    var curr_boards []boards
    for rows.Next() {
	var board boards
	err = rows.Scan(&board.Id, &board.Name, &board.Descr)
	if err != nil {
	    return []byte{}, xerrors.NewUIErr(err.Error(), err.Error(), `003`, true)
	}
	curr_boards = append(curr_boards, board)
    }
    bytes, err1 := json.Marshal(api_request{"ok", nil, &curr_boards})
    if err1 != nil {
	return []byte{}, xerrors.NewUIErr(err1.Error(), err1.Error(), `004`, true)
    }
    return bytes, nil
}

func getActiveThreadsForBoard(apiKey string, boardId int)  ([]byte, error) {

    rows, err := dbh.Query(`select t.id, t.name, count(*), (select count(*) from thread_posts where thread_id = t.id and attachment_url is not null) from threads t
				join boards b on b.id = t.board_id
				join image_board_clusters ibc on ibc.id = b.image_board_cluster_id
				left join thread_posts tp on tp.thread_id = t.id
			    where t.is_active = TRUE and t.board_id = $1 and ibc.api_key = $2 group by 1,2 order by t.id;`, boardId, apiKey)
    if err != nil {
        return []byte{}, xerrors.NewUIErr(err.Error(), err.Error(), `006`, true)
    }
    defer rows.Close()

    var active_threads []threads
    for rows.Next() {
	glog.Info("Popped new thread")
        var thread threads
        err = rows.Scan(&thread.Id, &thread.Name, &thread.PostCount, &thread.PostCountWithAttachment)
        if err != nil {
            return []byte{}, xerrors.NewUIErr(err.Error(), err.Error(), `007`, true)
        }
        active_threads = append(active_threads, thread)
    }
    var bytes []byte
    var err1 error
    if(len(active_threads) == 0){
        errMsg := "No objects returned."
        bytes, err1 = json.Marshal(api_request{"error", &errMsg, &active_threads})
    }else {
        bytes, err1 = json.Marshal(api_request{"ok", nil, &active_threads})
    }

    if err1 != nil {
        return []byte{}, xerrors.NewUIErr(err1.Error(), err1.Error(), `008`, true)
    }

    return bytes, nil
}

func getPostsForThread(apiKey string, threadId int)  ([]byte, error) {
    rows, err := dbh.Query(`select tp.id, tp.body, tp.attachment_url, tp.inserted_at, tp.source_ip
			    from thread_posts tp join threads t on t.id = tp.thread_id
						 join boards b on b.id = t.board_id
						 join image_board_clusters ibc on ibc.id = b.image_board_cluster_id
			    where tp.thread_id = $1 and ibc.api_key = $2 and t.is_active = true;`, threadId, apiKey)
    if err != nil {
	glog.Error(err)
        return []byte{}, xerrors.NewSysErr()
    }
    defer rows.Close()

    var curr_posts []thread_posts
    for rows.Next() {
	glog.Info("new post for thread with id: ", threadId)
        var curr_post thread_posts
        err = rows.Scan(&curr_post.Id, &curr_post.Body, &curr_post.AttachmentUrl, &curr_post.InsertedAt, &curr_post.SourceIp)
        if err != nil {
	    glog.Error(err)
            return []byte{}, xerrors.NewSysErr()
        }
        curr_posts = append(curr_posts, curr_post)
    }

    var bytes []byte
    var err1 error
    if(len(curr_posts) == 0){
	errMsg := "No objects returned."
	bytes, err1 = json.Marshal(api_request{"error", &errMsg, &curr_posts})
    }else {
	bytes, err1 = json.Marshal(api_request{"ok", nil, &curr_posts})
    }

    if err1 != nil {
        return []byte{}, xerrors.NewSysErr()
    }

    return bytes, nil
}

func addPostToThread(threadId int, threadBodyPost string, attachmentUrl *string, clientRemoteAddr string) ([]byte,error) {
    var is_limit_reached bool
    var max_post_length int
    var min_post_length int
    err := dbh.QueryRow("select (select count(*) from thread_posts  where thread_id = $1) > max_posts_per_thread, min_post_length, max_post_length  from threads where id = $1;", threadId).Scan(&is_limit_reached, &min_post_length, &max_post_length)
    if err != nil {
	return []byte{}, xerrors.NewUIErr(err.Error(), err.Error(), `009`, true)
    }

    if is_limit_reached {
	dbh.QueryRow("UPDATE threads set is_active = false where id = $1", threadId).Scan()
	return []byte{}, xerrors.NewUIErr(`Thread post limit reached!`, `Thread post limit reached!`, `010`, true)
    }

   if(min_post_length > len(threadBodyPost)  && min_post_length != -1){
	return []byte{}, xerrors.NewUIErr(`Post length is less than minimum length!`, `Post length is less than minimum length! post length: ` + strconv.Itoa(len(threadBodyPost))  +` min length: ` + strconv.Itoa(min_post_length) , `020`, false)
    }
   if(max_post_length < len(threadBodyPost)  && max_post_length != -1){
	return []byte{}, xerrors.NewUIErr(`Post length is more than maximum length!`, `Post length is more than maximum length! post length: ` + strconv.Itoa(len(threadBodyPost))  +` max length: ` + strconv.Itoa(max_post_length) , `021`, false)
    }

    _, err = dbh.Query("INSERT INTO thread_posts(body, thread_id, attachment_url, source_ip) VALUES($1, $2, $3, $4)", threadBodyPost, threadId, attachmentUrl, clientRemoteAddr)

    if err != nil {
	glog.Error(err)
        return []byte{}, xerrors.NewUIErr(err.Error(), err.Error(), `011`, true)
    }

    bytes, err1 := json.Marshal(api_request{"ok", nil, nil})
    if err1 != nil {
        return []byte{}, xerrors.NewUIErr(err1.Error(), err1.Error(), `012`, true)
    }

    return bytes, nil
}

func addThread(boardId int, threadName string) ([]byte, error) {
    var is_limit_reached bool
    err := dbh.QueryRow("select (select count(*) from threads  where board_id = $1) > thread_setting_max_thread_count  from boards where id = $1;", boardId).Scan(&is_limit_reached)
    if err != nil {
	glog.Error("COULD NOT SELECT thread_count")
	return []byte{}, xerrors.NewUIErr(err.Error(), err.Error(), `015`, true)
    }
    if is_limit_reached {
	return []byte{}, xerrors.NewUIErr(`Thread limit reached!`, `Thread limit reached!`, `016`, true)
    }

    var threadId int
    err = dbh.QueryRow("INSERT INTO threads(name, board_id, limits_reached_action_id, max_posts_per_thread) VALUES($1, $2, 1, 10)  RETURNING id, name", threadName, boardId).Scan(&threadId, &threadName)

    if err != nil {
	glog.Error("INSERT FAILED")
        return []byte{}, xerrors.NewUIErr(err.Error(), err.Error(), `017`, true)
    }

    a := struct{
		    Id int
		    Name string
		}{
		    threadId,
		    threadName,
		}
    bytes, err1 := json.Marshal(api_request{`ok`, nil, a })
    if err1 != nil {
        return []byte{}, xerrors.NewUIErr(err1.Error(), err1.Error(), `018`, true)
    }

    return bytes, nil
}

<|endoftext|>


<commit_before>package youtubedl

import (
	"bufio"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"strings"
)

// Error youtubedl specific error
type Error string

func (e Error) Error() string {
	return string(e)
}

// Info youtubedl json, thumbnail bytes and raw JSON
type Info struct {
	Artist   string `json:"artist"`
	Uploader string `json:"uploader"`
	Creator  string `json:"creator"`

	Title       string   `json:"title"`
	Description string   `json:"description"`
	Duration    float64  `json:"duration"`
	Thumbnail   string   `json:"thumbnail"`
	Formats     []Format `json:"formats"`

	// not unmarshalled, populated from image thumbnail file
	ThumbnailBytes []byte `json:"-"`

	// private, save raw json to be used later when downloading
	rawJSON []byte
}

// Format youtubedl downloadable format
type Format struct {
	FormatID string  `json:"format_id"`
	Protocol string  `json:"protocol"`
	Ext      string  `json:"ext"`
	ACodec   string  `json:"acodec"`
	VCodec   string  `json:"vcodec"`
	TBR      float64 `json:"tbr"`
	ABR      float64 `json:"abr"`
	VBR      float64 `json:"vbr"`

	NormBR     float64
	NormACodec string
	NormVCodec string
}

func (f Format) String() string {
	return fmt.Sprintf("%s:%s:%s:%s:%s:%f",
		f.FormatID,
		f.Protocol,
		f.Ext,
		f.NormACodec,
		f.NormVCodec,
		f.NormBR,
	)
}

// guess codec from fuzzy codec name
func normalizeCodecName(c string) string {
	codecNameNormalizeMap := map[string]string{
		"none": "",
		"avc1": "h264",
		"mp4a": "aac",
		"mp4v": "h264",
		"h265": "hevc",
	}

	// "  NAME.something  " -> "name"
	c = strings.Trim(c, " ")
	c = strings.ToLower(c)
	p := strings.SplitN(c, ".", 2)
	c = p[0]

	if n, ok := codecNameNormalizeMap[c]; ok {
		return n
	}

	return c
}

// guess codecs based on ext
func codecFromExt(ext string) (acodec string, vcodec string) {
	switch strings.ToLower(ext) {
	case "mp3":
		return "mp3", ""
	case "mp4":
		return "aac", "h264"
	case "flv":
		return "aac", "h264"
	default:
		return "", ""
	}
}

func parseInfo(r io.Reader) (info Info, err error) {
	info = Info{}

	info.rawJSON, err = ioutil.ReadAll(r)
	if err != nil {
		return Info{}, err
	}

	if err := json.Unmarshal(info.rawJSON, &info); err != nil {
		return Info{}, err
	}

	for i, _ := range info.Formats {
		f := &info.Formats[i]

		f.NormACodec = normalizeCodecName(f.ACodec)
		f.NormVCodec = normalizeCodecName(f.VCodec)

		extACodec, extVCodec := codecFromExt(f.Ext)
		if f.ACodec == "" {
			f.NormACodec = extACodec
		}
		if f.VCodec == "" {
			f.NormVCodec = extVCodec
		}

		if f.TBR != 0 {
			f.NormBR = f.TBR
		} else {
			f.NormBR = f.ABR + f.VBR
		}
	}

	return info, nil
}

// NewFromURL new Info downloaded from URL using context
func NewFromURL(ctx context.Context, url string, stdout io.Writer) (info Info, err error) {
	tempPath, _ := ioutil.TempDir("", "ydls-youtubedl")
	defer os.RemoveAll(tempPath)

	cmd := exec.CommandContext(
		ctx,
		"youtube-dl",
		"--no-call-home",
		"--no-cache-dir",
		"--skip-download",
		"--write-info-json",
		"--write-thumbnail",
		"--restrict-filenames",
		// don't base output filename on source info
		"--output", "source",
		// provide URL via stdin for security, youtube-dl has some run command args
		"--batch-file", "-",
	)
	cmd.Dir = tempPath
	cmd.Stdout = stdout
	cmdStderr, cmdStderrErr := cmd.StderrPipe()
	if cmdStderrErr != nil {
		return Info{}, cmdStderrErr
	}
	cmdStdin, cmdStdinErr := cmd.StdinPipe()
	if cmdStdinErr != nil {
		return Info{}, cmdStdinErr
	}

	if err := cmd.Start(); err != nil {
		return Info{}, err
	}
	defer cmd.Wait()

	fmt.Fprintln(cmdStdin, url)
	cmdStdin.Close()

	stderrLineScanner := bufio.NewScanner(cmdStderr)
	for stderrLineScanner.Scan() {
		const errorPrefix = "ERROR: "
		line := stderrLineScanner.Text()
		if strings.HasPrefix(line, errorPrefix) {
			return Info{}, Error(line[len(errorPrefix):])
		}
	}

	return NewFromPath(tempPath)
}

// NewFromPath new Info from path with JSON and optional image
func NewFromPath(infoPath string) (info Info, err error) {
	files, err := ioutil.ReadDir(infoPath)
	if err != nil {
		return Info{}, err
	}

	infoJSONPath := ""
	thumbnailPath := ""
	for _, f := range files {
		ext := path.Ext(f.Name())
		if ext == ".json" {
			infoJSONPath = path.Join(infoPath, f.Name())
		} else if ext == ".png" || ext == ".jpg" || ext == ".jpeg" {
			thumbnailPath = path.Join(infoPath, f.Name())
		}
	}

	if infoPath == "" {
		return Info{}, fmt.Errorf("no info json found")
	}

	file, err := os.Open(infoJSONPath)
	if err != nil {
		return Info{}, err
	}
	defer file.Close()
	info, err = parseInfo(file)
	if err != nil {
		return Info{}, err
	}

	if thumbnailPath != "" {
		info.ThumbnailBytes, err = ioutil.ReadFile(thumbnailPath)
		if err != nil {
			return Info{}, fmt.Errorf("failed to read thumbnail")
		}
	}

	return info, nil
}

// DownloadResult download result
type DownloadResult struct {
	Reader io.ReadCloser
	waitCh chan struct{}
}

// Download format matched by filter
func (info Info) Download(ctx context.Context, filter string, stderr io.Writer) (*DownloadResult, error) {
	tempPath, tempErr := ioutil.TempDir("", "ydls-youtubedl")
	if tempErr != nil {
		return nil, tempErr
	}
	jsonTempPath := path.Join(tempPath, "info.json")
	if err := ioutil.WriteFile(jsonTempPath, info.rawJSON, 0644); err != nil {
		os.RemoveAll(tempPath)
		return nil, err
	}

	dr := &DownloadResult{
		waitCh: make(chan struct{}, 1),
	}

	cmd := exec.CommandContext(
		ctx,
		"youtube-dl",
		"--no-call-home",
		"--no-cache-dir",
		"--restrict-filenames",
		"--load-info", jsonTempPath,
		"-f", filter,
		"-o", "-",
	)
	cmd.Dir = tempPath
	var w io.WriteCloser
	dr.Reader, w = io.Pipe()
	cmd.Stdout = w
	cmd.Stderr = stderr

	if err := cmd.Start(); err != nil {
		return nil, err
	}

	go func() {
		cmd.Wait()
		w.Close()
		os.RemoveAll(tempPath)
		close(dr.waitCh)
	}()

	return dr, nil
}

// Wait for resource cleanup
func (dr *DownloadResult) Wait() {
	<-dr.waitCh
}
<commit_msg>youtubedl.Info String() include ABR and VBR<commit_after>package youtubedl

import (
	"bufio"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"strings"
)

// Error youtubedl specific error
type Error string

func (e Error) Error() string {
	return string(e)
}

// Info youtubedl json, thumbnail bytes and raw JSON
type Info struct {
	Artist   string `json:"artist"`
	Uploader string `json:"uploader"`
	Creator  string `json:"creator"`

	Title       string   `json:"title"`
	Description string   `json:"description"`
	Duration    float64  `json:"duration"`
	Thumbnail   string   `json:"thumbnail"`
	Formats     []Format `json:"formats"`

	// not unmarshalled, populated from image thumbnail file
	ThumbnailBytes []byte `json:"-"`

	// private, save raw json to be used later when downloading
	rawJSON []byte
}

// Format youtubedl downloadable format
type Format struct {
	FormatID string  `json:"format_id"`
	Protocol string  `json:"protocol"`
	Ext      string  `json:"ext"`
	ACodec   string  `json:"acodec"`
	VCodec   string  `json:"vcodec"`
	TBR      float64 `json:"tbr"`
	ABR      float64 `json:"abr"`
	VBR      float64 `json:"vbr"`

	NormBR     float64
	NormACodec string
	NormVCodec string
}

func (f Format) String() string {
	return fmt.Sprintf("%s:%s:%s a:%s:%f v:%s:%f %f",
		f.FormatID,
		f.Protocol,
		f.Ext,
		f.NormACodec,
		f.ABR,
		f.NormVCodec,
		f.VBR,
		f.NormBR,
	)
}

// guess codec from fuzzy codec name
func normalizeCodecName(c string) string {
	codecNameNormalizeMap := map[string]string{
		"none": "",
		"avc1": "h264",
		"mp4a": "aac",
		"mp4v": "h264",
		"h265": "hevc",
	}

	// "  NAME.something  " -> "name"
	c = strings.Trim(c, " ")
	c = strings.ToLower(c)
	p := strings.SplitN(c, ".", 2)
	c = p[0]

	if n, ok := codecNameNormalizeMap[c]; ok {
		return n
	}

	return c
}

// guess codecs based on ext
func codecFromExt(ext string) (acodec string, vcodec string) {
	switch strings.ToLower(ext) {
	case "mp3":
		return "mp3", ""
	case "mp4":
		return "aac", "h264"
	case "flv":
		return "aac", "h264"
	default:
		return "", ""
	}
}

func parseInfo(r io.Reader) (info Info, err error) {
	info = Info{}

	info.rawJSON, err = ioutil.ReadAll(r)
	if err != nil {
		return Info{}, err
	}

	if err := json.Unmarshal(info.rawJSON, &info); err != nil {
		return Info{}, err
	}

	for i, _ := range info.Formats {
		f := &info.Formats[i]

		f.NormACodec = normalizeCodecName(f.ACodec)
		f.NormVCodec = normalizeCodecName(f.VCodec)

		extACodec, extVCodec := codecFromExt(f.Ext)
		if f.ACodec == "" {
			f.NormACodec = extACodec
		}
		if f.VCodec == "" {
			f.NormVCodec = extVCodec
		}

		if f.TBR != 0 {
			f.NormBR = f.TBR
		} else {
			f.NormBR = f.ABR + f.VBR
		}
	}

	return info, nil
}

// NewFromURL new Info downloaded from URL using context
func NewFromURL(ctx context.Context, url string, stdout io.Writer) (info Info, err error) {
	tempPath, _ := ioutil.TempDir("", "ydls-youtubedl")
	defer os.RemoveAll(tempPath)

	cmd := exec.CommandContext(
		ctx,
		"youtube-dl",
		"--no-call-home",
		"--no-cache-dir",
		"--skip-download",
		"--write-info-json",
		"--write-thumbnail",
		"--restrict-filenames",
		// don't base output filename on source info
		"--output", "source",
		// provide URL via stdin for security, youtube-dl has some run command args
		"--batch-file", "-",
	)
	cmd.Dir = tempPath
	cmd.Stdout = stdout
	cmdStderr, cmdStderrErr := cmd.StderrPipe()
	if cmdStderrErr != nil {
		return Info{}, cmdStderrErr
	}
	cmdStdin, cmdStdinErr := cmd.StdinPipe()
	if cmdStdinErr != nil {
		return Info{}, cmdStdinErr
	}

	if err := cmd.Start(); err != nil {
		return Info{}, err
	}
	defer cmd.Wait()

	fmt.Fprintln(cmdStdin, url)
	cmdStdin.Close()

	stderrLineScanner := bufio.NewScanner(cmdStderr)
	for stderrLineScanner.Scan() {
		const errorPrefix = "ERROR: "
		line := stderrLineScanner.Text()
		if strings.HasPrefix(line, errorPrefix) {
			return Info{}, Error(line[len(errorPrefix):])
		}
	}

	return NewFromPath(tempPath)
}

// NewFromPath new Info from path with JSON and optional image
func NewFromPath(infoPath string) (info Info, err error) {
	files, err := ioutil.ReadDir(infoPath)
	if err != nil {
		return Info{}, err
	}

	infoJSONPath := ""
	thumbnailPath := ""
	for _, f := range files {
		ext := path.Ext(f.Name())
		if ext == ".json" {
			infoJSONPath = path.Join(infoPath, f.Name())
		} else if ext == ".png" || ext == ".jpg" || ext == ".jpeg" {
			thumbnailPath = path.Join(infoPath, f.Name())
		}
	}

	if infoPath == "" {
		return Info{}, fmt.Errorf("no info json found")
	}

	file, err := os.Open(infoJSONPath)
	if err != nil {
		return Info{}, err
	}
	defer file.Close()
	info, err = parseInfo(file)
	if err != nil {
		return Info{}, err
	}

	if thumbnailPath != "" {
		info.ThumbnailBytes, err = ioutil.ReadFile(thumbnailPath)
		if err != nil {
			return Info{}, fmt.Errorf("failed to read thumbnail")
		}
	}

	return info, nil
}

// DownloadResult download result
type DownloadResult struct {
	Reader io.ReadCloser
	waitCh chan struct{}
}

// Download format matched by filter
func (info Info) Download(ctx context.Context, filter string, stderr io.Writer) (*DownloadResult, error) {
	tempPath, tempErr := ioutil.TempDir("", "ydls-youtubedl")
	if tempErr != nil {
		return nil, tempErr
	}
	jsonTempPath := path.Join(tempPath, "info.json")
	if err := ioutil.WriteFile(jsonTempPath, info.rawJSON, 0644); err != nil {
		os.RemoveAll(tempPath)
		return nil, err
	}

	dr := &DownloadResult{
		waitCh: make(chan struct{}, 1),
	}

	cmd := exec.CommandContext(
		ctx,
		"youtube-dl",
		"--no-call-home",
		"--no-cache-dir",
		"--restrict-filenames",
		"--load-info", jsonTempPath,
		"-f", filter,
		"-o", "-",
	)
	cmd.Dir = tempPath
	var w io.WriteCloser
	dr.Reader, w = io.Pipe()
	cmd.Stdout = w
	cmd.Stderr = stderr

	if err := cmd.Start(); err != nil {
		return nil, err
	}

	go func() {
		cmd.Wait()
		w.Close()
		os.RemoveAll(tempPath)
		close(dr.waitCh)
	}()

	return dr, nil
}

// Wait for resource cleanup
func (dr *DownloadResult) Wait() {
	<-dr.waitCh
}
<|endoftext|>