<commit_before>// Copyright 2018 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// stdlib builds the standard library in the appropriate mode into a new goroot.
package main

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
)

type replicateMode int

const (
	copyMode replicateMode = iota
	hardlinkMode
	softlinkMode
)

type replicateOption func(*replicateConfig)
type replicateConfig struct {
	removeFirst bool
	fileMode    replicateMode
	dirMode     replicateMode
	paths       []string
}

func replicatePaths(paths ...string) replicateOption {
	return func(config *replicateConfig) {
		config.paths = append(config.paths, paths...)
	}
}

// replicatePrepare is the common preparation steps for a replication entry
func replicatePrepare(dst string, config *replicateConfig) error {
	dir := filepath.Dir(dst)
	if err := os.MkdirAll(dir, 0755); err != nil {
		return fmt.Errorf("Failed to make %s: %v", dir, err)
	}
	if config.removeFirst {
		_ = os.Remove(dst)
	}
	return nil
}

// replicateFile is called internally by replicate to map a single file from src into dst.
func replicateFile(src, dst string, config *replicateConfig) error {
	if err := replicatePrepare(dst, config); err != nil {
		return err
	}
	switch config.fileMode {
	case copyMode:
		in, err := os.Open(src)
		if err != nil {
			return err
		}
		defer in.Close()
		out, err := os.Create(dst)
		if err != nil {
			return err
		}
		_, err = io.Copy(out, in)
		closeerr := out.Close()
		if err != nil {
			return err
		}
		if closeerr != nil {
			return closeerr
		}
		s, err := os.Stat(src)
		if err != nil {
			return err
		}
		if err := os.Chmod(dst, s.Mode()); err != nil {
			return err
		}
		return nil
	case hardlinkMode:
		return os.Link(src, dst)
	case softlinkMode:
		return os.Symlink(src, dst)
	default:
		return fmt.Errorf("Invalid replication mode %d", config.fileMode)
	}
}

// replicateDir makes a tree of files visible in a new location.
// It is allowed to take any efficient method of doing so.
func replicateDir(src, dst string, config *replicateConfig) error {
	if err := replicatePrepare(dst, config); err != nil {
		return err
	}
	switch config.dirMode {
	case copyMode:
		return filepath.Walk(src, func(path string, f os.FileInfo, err error) error {
			if f.IsDir() {
				return nil
			}
			relative, err := filepath.Rel(src, path)
			if err != nil {
				return err
			}
			return replicateFile(path, filepath.Join(dst, relative), config)
		})
	case hardlinkMode:
		return os.Link(src, dst)
	case softlinkMode:
		return os.Symlink(src, dst)
	default:
		return fmt.Errorf("Invalid replication mode %d", config.fileMode)
	}
}

// replicateTree is called for each single src dst pair.
func replicateTree(src, dst string, config *replicateConfig) error {
	if err := os.RemoveAll(dst); err != nil {
		return fmt.Errorf("Failed to remove file at destination %s: %v", dst, err)
	}
	if l, err := filepath.EvalSymlinks(src); err == nil {
		src = l
	}
	s, err := os.Stat(src)
	if err != nil && !os.IsNotExist(err) {
		return err
	}
	if s.IsDir() {
		return replicateDir(src, dst, config)
	}
	return replicateFile(src, dst, config)
}

// replicate makes a tree of files visible in a new location.
// You control how it does so using options, by default it presumes the entire tree
// of files rooted at src must be visible at dst, and that it should do so by copying.
// src is allowed to be a file, in which case just the one file is copied.
func replicate(src, dst string, options ...replicateOption) error {
	config := replicateConfig{
		removeFirst: true,
	}
	for _, option := range options {
		option(&config)
	}
	if len(config.paths) == 0 {
		return replicateTree(src, dst, &config)
	}
	for _, base := range config.paths {
		from := filepath.Join(src, base)
		to := filepath.Join(dst, base)
		if err := replicateTree(from, to, &config); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>Fix error check in stdlib builder (#1363)<commit_after>// Copyright 2018 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// stdlib builds the standard library in the appropriate mode into a new goroot.
package main

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
)

type replicateMode int

const (
	copyMode replicateMode = iota
	hardlinkMode
	softlinkMode
)

type replicateOption func(*replicateConfig)
type replicateConfig struct {
	removeFirst bool
	fileMode    replicateMode
	dirMode     replicateMode
	paths       []string
}

func replicatePaths(paths ...string) replicateOption {
	return func(config *replicateConfig) {
		config.paths = append(config.paths, paths...)
	}
}

// replicatePrepare is the common preparation steps for a replication entry
func replicatePrepare(dst string, config *replicateConfig) error {
	dir := filepath.Dir(dst)
	if err := os.MkdirAll(dir, 0755); err != nil {
		return fmt.Errorf("Failed to make %s: %v", dir, err)
	}
	if config.removeFirst {
		_ = os.Remove(dst)
	}
	return nil
}

// replicateFile is called internally by replicate to map a single file from src into dst.
func replicateFile(src, dst string, config *replicateConfig) error {
	if err := replicatePrepare(dst, config); err != nil {
		return err
	}
	switch config.fileMode {
	case copyMode:
		in, err := os.Open(src)
		if err != nil {
			return err
		}
		defer in.Close()
		out, err := os.Create(dst)
		if err != nil {
			return err
		}
		_, err = io.Copy(out, in)
		closeerr := out.Close()
		if err != nil {
			return err
		}
		if closeerr != nil {
			return closeerr
		}
		s, err := os.Stat(src)
		if err != nil {
			return err
		}
		if err := os.Chmod(dst, s.Mode()); err != nil {
			return err
		}
		return nil
	case hardlinkMode:
		return os.Link(src, dst)
	case softlinkMode:
		return os.Symlink(src, dst)
	default:
		return fmt.Errorf("Invalid replication mode %d", config.fileMode)
	}
}

// replicateDir makes a tree of files visible in a new location.
// It is allowed to take any efficient method of doing so.
func replicateDir(src, dst string, config *replicateConfig) error {
	if err := replicatePrepare(dst, config); err != nil {
		return err
	}
	switch config.dirMode {
	case copyMode:
		return filepath.Walk(src, func(path string, f os.FileInfo, err error) error {
			if f.IsDir() {
				return nil
			}
			relative, err := filepath.Rel(src, path)
			if err != nil {
				return err
			}
			return replicateFile(path, filepath.Join(dst, relative), config)
		})
	case hardlinkMode:
		return os.Link(src, dst)
	case softlinkMode:
		return os.Symlink(src, dst)
	default:
		return fmt.Errorf("Invalid replication mode %d", config.fileMode)
	}
}

// replicateTree is called for each single src dst pair.
func replicateTree(src, dst string, config *replicateConfig) error {
	if err := os.RemoveAll(dst); err != nil {
		return fmt.Errorf("Failed to remove file at destination %s: %v", dst, err)
	}
	if l, err := filepath.EvalSymlinks(src); err != nil {
		return err
	} else {
		src = l
	}
	if s, err := os.Stat(src); err != nil {
		return err
	} else if s.IsDir() {
		return replicateDir(src, dst, config)
	}
	return replicateFile(src, dst, config)
}

// replicate makes a tree of files visible in a new location.
// You control how it does so using options, by default it presumes the entire tree
// of files rooted at src must be visible at dst, and that it should do so by copying.
// src is allowed to be a file, in which case just the one file is copied.
func replicate(src, dst string, options ...replicateOption) error {
	config := replicateConfig{
		removeFirst: true,
	}
	for _, option := range options {
		option(&config)
	}
	if len(config.paths) == 0 {
		return replicateTree(src, dst, &config)
	}
	for _, base := range config.paths {
		from := filepath.Join(src, base)
		to := filepath.Join(dst, base)
		if err := replicateTree(from, to, &config); err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2013, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package worker

import (
	"flag"
	"fmt"
	"html/template"
	"net/http"
	"strings"
	"sync"

	"github.com/youtube/vitess/go/vt/concurrency"
	"github.com/youtube/vitess/go/vt/topo/topoproto"
	"github.com/youtube/vitess/go/vt/wrangler"
	"golang.org/x/net/context"
)

const splitDiffHTML = `
<!DOCTYPE html>

  <title>Split Diff Action</title>

Split Diff Action

    {{if .Error}}
      Error: {{.Error}}

    {{else}}
      {{range $i, $si := .Shards}}

{{$si.Keyspace}}/{{$si.Shard}}

      {{end}}
    {{end}}

`

const splitDiffHTML2 = `
<!DOCTYPE html>

  <title>Split Diff Action</title>

Shard involved: {{.Keyspace}}/{{.Shard}}

Split Diff Action

    <form action="/Diffs/SplitDiff" method="post">
      Exclude Tables:
        <INPUT type="text" id="excludeTables" name="excludeTables" value="">

      <INPUT type="hidden" name="keyspace" value="{{.Keyspace}}"/>
      <INPUT type="hidden" name="shard" value="{{.Shard}}"/>
      <INPUT type="submit" name="submit" value="Split Diff"/>
    </form>

`

var splitDiffTemplate = mustParseTemplate("splitDiff", splitDiffHTML)
var splitDiffTemplate2 = mustParseTemplate("splitDiff2", splitDiffHTML2)

func commandSplitDiff(wi *Instance, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) (Worker, error) {
	excludeTables := subFlags.String("exclude_tables", "", "comma separated list of tables to exclude")
	if err := subFlags.Parse(args); err != nil {
		return nil, err
	}
	if subFlags.NArg() != 1 {
		subFlags.Usage()
		return nil, fmt.Errorf("command SplitDiff requires <keyspace/shard>")
	}
	keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0))
	if err != nil {
		return nil, err
	}
	var excludeTableArray []string
	if *excludeTables != "" {
		excludeTableArray = strings.Split(*excludeTables, ",")
	}
	return NewSplitDiffWorker(wr, wi.cell, keyspace, shard, excludeTableArray), nil
}

// shardsWithSources returns all the shards that have SourceShards set
// with no Tables list.
func shardsWithSources(ctx context.Context, wr *wrangler.Wrangler) ([]map[string]string, error) {
	shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
	keyspaces, err := wr.TopoServer().GetKeyspaces(shortCtx)
	cancel()
	if err != nil {
		return nil, err
	}

	wg := sync.WaitGroup{}
	mu := sync.Mutex{} // protects result
	result := make([]map[string]string, 0, len(keyspaces))
	rec := concurrency.AllErrorRecorder{}
	for _, keyspace := range keyspaces {
		wg.Add(1)
		go func(keyspace string) {
			defer wg.Done()
			shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
			shards, err := wr.TopoServer().GetShardNames(shortCtx, keyspace)
			cancel()
			if err != nil {
				rec.RecordError(err)
				return
			}
			for _, shard := range shards {
				wg.Add(1)
				go func(keyspace, shard string) {
					defer wg.Done()
					shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
					si, err := wr.TopoServer().GetShard(shortCtx, keyspace, shard)
					cancel()
					if err != nil {
						rec.RecordError(err)
						return
					}

					if len(si.SourceShards) > 0 && len(si.SourceShards[0].Tables) == 0 {
						mu.Lock()
						result = append(result, map[string]string{
							"Keyspace": keyspace,
							"Shard":    shard,
						})
						mu.Unlock()
					}
				}(keyspace, shard)
			}
		}(keyspace)
	}
	wg.Wait()

	if rec.HasErrors() {
		return nil, rec.Error()
	}
	if len(result) == 0 {
		return nil, fmt.Errorf("There are no shards with SourceShards")
	}
	return result, nil
}

func interactiveSplitDiff(ctx context.Context, wi *Instance, wr *wrangler.Wrangler, w http.ResponseWriter, r *http.Request) (Worker, *template.Template, map[string]interface{}, error) {
	if err := r.ParseForm(); err != nil {
		return nil, nil, nil, fmt.Errorf("cannot parse form: %s", err)
	}
	keyspace := r.FormValue("keyspace")
	shard := r.FormValue("shard")

	if keyspace == "" || shard == "" {
		// display the list of possible shards to chose from
		result := make(map[string]interface{})
		shards, err := shardsWithSources(ctx, wr)
		if err != nil {
			result["Error"] = err.Error()
		} else {
			result["Shards"] = shards
		}
		return nil, splitDiffTemplate, result, nil
	}

	submitButtonValue := r.FormValue("submit")
	if submitButtonValue == "" {
		// display the input form
		result := make(map[string]interface{})
		result["Keyspace"] = keyspace
		result["Shard"] = shard
		return nil, splitDiffTemplate2, result, nil
	}

	// Process input form.
	excludeTables := r.FormValue("excludeTables")
	var excludeTableArray []string
	if excludeTables != "" {
		excludeTableArray = strings.Split(excludeTables, ",")
	}

	// start the diff job
	wrk := NewSplitDiffWorker(wr, wi.cell, keyspace, shard, excludeTableArray)
	if _, err := wi.setAndStartWorker(wrk, nil); err != nil {
		return nil, nil, nil, fmt.Errorf("cannot set worker: %s", err)
	}
	return wrk, nil, nil, nil
}

func init() {
	AddCommand("Diffs", Command{"SplitDiff",
		commandSplitDiff, interactiveSplitDiff,
		"[--exclude_tables=''] <keyspace/shard>",
		"Diffs a rdonly destination shard against its SourceShards"})
}
<commit_msg>worker: Remove duplicate call to setAndStartWorker() when calling SplitDiff through the webinterface.<commit_after>// Copyright 2013, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package worker

import (
	"flag"
	"fmt"
	"html/template"
	"net/http"
	"strings"
	"sync"

	"github.com/youtube/vitess/go/vt/concurrency"
	"github.com/youtube/vitess/go/vt/topo/topoproto"
	"github.com/youtube/vitess/go/vt/wrangler"
	"golang.org/x/net/context"
)

const splitDiffHTML = `
<!DOCTYPE html>

  <title>Split Diff Action</title>

Split Diff Action

    {{if .Error}}
      Error: {{.Error}}

    {{else}}
      {{range $i, $si := .Shards}}

{{$si.Keyspace}}/{{$si.Shard}}

      {{end}}
    {{end}}

`

const splitDiffHTML2 = `
<!DOCTYPE html>

  <title>Split Diff Action</title>

Shard involved: {{.Keyspace}}/{{.Shard}}

Split Diff Action

    <form action="/Diffs/SplitDiff" method="post">
      Exclude Tables:
        <INPUT type="text" id="excludeTables" name="excludeTables" value="">

      <INPUT type="hidden" name="keyspace" value="{{.Keyspace}}"/>
      <INPUT type="hidden" name="shard" value="{{.Shard}}"/>
      <INPUT type="submit" name="submit" value="Split Diff"/>
    </form>

`

var splitDiffTemplate = mustParseTemplate("splitDiff", splitDiffHTML)
var splitDiffTemplate2 = mustParseTemplate("splitDiff2", splitDiffHTML2)

func commandSplitDiff(wi *Instance, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) (Worker, error) {
	excludeTables := subFlags.String("exclude_tables", "", "comma separated list of tables to exclude")
	if err := subFlags.Parse(args); err != nil {
		return nil, err
	}
	if subFlags.NArg() != 1 {
		subFlags.Usage()
		return nil, fmt.Errorf("command SplitDiff requires <keyspace/shard>")
	}
	keyspace, shard, err := topoproto.ParseKeyspaceShard(subFlags.Arg(0))
	if err != nil {
		return nil, err
	}
	var excludeTableArray []string
	if *excludeTables != "" {
		excludeTableArray = strings.Split(*excludeTables, ",")
	}
	return NewSplitDiffWorker(wr, wi.cell, keyspace, shard, excludeTableArray), nil
}

// shardsWithSources returns all the shards that have SourceShards set
// with no Tables list.
func shardsWithSources(ctx context.Context, wr *wrangler.Wrangler) ([]map[string]string, error) {
	shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
	keyspaces, err := wr.TopoServer().GetKeyspaces(shortCtx)
	cancel()
	if err != nil {
		return nil, err
	}

	wg := sync.WaitGroup{}
	mu := sync.Mutex{} // protects result
	result := make([]map[string]string, 0, len(keyspaces))
	rec := concurrency.AllErrorRecorder{}
	for _, keyspace := range keyspaces {
		wg.Add(1)
		go func(keyspace string) {
			defer wg.Done()
			shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
			shards, err := wr.TopoServer().GetShardNames(shortCtx, keyspace)
			cancel()
			if err != nil {
				rec.RecordError(err)
				return
			}
			for _, shard := range shards {
				wg.Add(1)
				go func(keyspace, shard string) {
					defer wg.Done()
					shortCtx, cancel := context.WithTimeout(ctx, *remoteActionsTimeout)
					si, err := wr.TopoServer().GetShard(shortCtx, keyspace, shard)
					cancel()
					if err != nil {
						rec.RecordError(err)
						return
					}

					if len(si.SourceShards) > 0 && len(si.SourceShards[0].Tables) == 0 {
						mu.Lock()
						result = append(result, map[string]string{
							"Keyspace": keyspace,
							"Shard":    shard,
						})
						mu.Unlock()
					}
				}(keyspace, shard)
			}
		}(keyspace)
	}
	wg.Wait()

	if rec.HasErrors() {
		return nil, rec.Error()
	}
	if len(result) == 0 {
		return nil, fmt.Errorf("There are no shards with SourceShards")
	}
	return result, nil
}

func interactiveSplitDiff(ctx context.Context, wi *Instance, wr *wrangler.Wrangler, w http.ResponseWriter, r *http.Request) (Worker, *template.Template, map[string]interface{}, error) {
	if err := r.ParseForm(); err != nil {
		return nil, nil, nil, fmt.Errorf("cannot parse form: %s", err)
	}
	keyspace := r.FormValue("keyspace")
	shard := r.FormValue("shard")

	if keyspace == "" || shard == "" {
		// display the list of possible shards to chose from
		result := make(map[string]interface{})
		shards, err := shardsWithSources(ctx, wr)
		if err != nil {
			result["Error"] = err.Error()
		} else {
			result["Shards"] = shards
		}
		return nil, splitDiffTemplate, result, nil
	}

	submitButtonValue := r.FormValue("submit")
	if submitButtonValue == "" {
		// display the input form
		result := make(map[string]interface{})
		result["Keyspace"] = keyspace
		result["Shard"] = shard
		return nil, splitDiffTemplate2, result, nil
	}

	// Process input form.
	excludeTables := r.FormValue("excludeTables")
	var excludeTableArray []string
	if excludeTables != "" {
		excludeTableArray = strings.Split(excludeTables, ",")
	}

	// start the diff job
	wrk := NewSplitDiffWorker(wr, wi.cell, keyspace, shard, excludeTableArray)
	return wrk, nil, nil, nil
}

func init() {
	AddCommand("Diffs", Command{"SplitDiff",
		commandSplitDiff, interactiveSplitDiff,
		"[--exclude_tables=''] <keyspace/shard>",
		"Diffs a rdonly destination shard against its SourceShards"})
}
<|endoftext|>


<commit_before>package contrib

import (
	"encoding/base64"
	"encoding/binary"
	"fmt"
	"io"
	"time"

	"github.com/gin-gonic/gin"
)

var pid = uint32(time.Now().UnixNano() % 4294967291)

func GenReqId() string {
	var b [12]byte
	binary.LittleEndian.PutUint32(b[:], pid)
	binary.LittleEndian.PutUint64(b[4:], uint64(time.Now().UnixNano()))
	return base64.URLEncoding.EncodeToString(b[:])
}

var (
	green   = string([]byte{27, 91, 57, 55, 59, 52, 50, 109})
	white   = string([]byte{27, 91, 57, 48, 59, 52, 55, 109})
	yellow  = string([]byte{27, 91, 57, 55, 59, 52, 51, 109})
	red     = string([]byte{27, 91, 57, 55, 59, 52, 49, 109})
	blue    = string([]byte{27, 91, 57, 55, 59, 52, 52, 109})
	magenta = string([]byte{27, 91, 57, 55, 59, 52, 53, 109})
	cyan    = string([]byte{27, 91, 57, 55, 59, 52, 54, 109})
	reset   = string([]byte{27, 91, 48, 109})
)

func ErrorLogger() gin.HandlerFunc {
	return ErrorLoggerT(gin.ErrorTypeAny)
}

func ErrorLoggerT(typ gin.ErrorType) gin.HandlerFunc {
	return func(c *gin.Context) {
		c.Next()
		// avoid writting if we already wrote into the response body
		if !c.Writer.Written() {
			errors := c.Errors.ByType(typ)
			if len(errors) > 0 {
				c.JSON(-1, errors)
			}
		}
	}
}

// Instances a Logger middleware that will write the logs to gin.DefaultWriter
// By default gin.DefaultWriter = os.Stdout
func Logger() gin.HandlerFunc {
	return LoggerWithWriter(gin.DefaultWriter)
}

// Instance a Logger middleware with the specified writter buffer.
// Example: os.Stdout, a file opened in write mode, a socket...
func LoggerWithWriter(out io.Writer, notlogged ...string) gin.HandlerFunc {
	var skip map[string]struct{}

	if length := len(notlogged); length > 0 {
		skip = make(map[string]struct{}, length)

		for _, path := range notlogged {
			skip[path] = struct{}{}
		}
	}

	return func(c *gin.Context) {
		// Start timer
		start := time.Now()

		xReqid := c.Request.Header.Get("X-Reqid")

		if xReqid == "" {
			xReqid = GenReqId()
		}

		c.Header("X-Reqid", xReqid)

		path := c.Request.URL.RequestURI()

		clientIP := c.ClientIP()
		method := c.Request.Method
		methodColor := colorForMethod(method)

		// Log only when path is not being skipped
		if _, ok := skip[path]; !ok {
			fmt.Fprintf(out, "[GIN] [%s] [Started]\t%v |%s  %s %-7s %s\n",
				xReqid,
				start.Format("2006/01/02 - 15:04:05"),
				methodColor, reset, method,
				path,
			)
			fmt.Printf("%+v\n", c.Request)
		}

		// Process request
		c.Next()

		// Log only when path is not being skipped
		if _, ok := skip[path]; !ok {
			// Stop timer
			end := time.Now()
			latency := end.Sub(start)

			comment := c.Errors.ByType(gin.ErrorTypePrivate).String()

			statusCode := c.Writer.Status()
			statusColor := colorForStatus(statusCode)
			methodColor = colorForMethod(method)

			fmt.Fprintf(out, "[GIN] [%s] [Ended]\t%v |%s %3d %s| %13v | %s |%s  %s %-7s %s\n%s",
				xReqid,
				end.Format("2006/01/02 - 15:04:05"),
				statusColor, statusCode, reset,
				latency,
				clientIP,
				methodColor, reset, method,
				path,
				comment,
			)
		}
	}
}

func colorForStatus(code int) string {
	switch {
	case code >= 200 && code < 300:
		return green
	case code >= 300 && code < 400:
		return white
	case code >= 400 && code < 500:
		return yellow
	default:
		return red
	}
}

func colorForMethod(method string) string {
	switch method {
	case "GET":
		return blue
	case "POST":
		return cyan
	case "PUT":
		return yellow
	case "DELETE":
		return red
	case "PATCH":
		return green
	case "HEAD":
		return magenta
	case "OPTIONS":
		return white
	default:
		return reset
	}
}
<commit_msg>Update logger<commit_after>package contrib

import (
	"encoding/base64"
	"encoding/binary"
	"fmt"
	"io"
	"time"

	"github.com/gin-gonic/gin"
)

var pid = uint32(time.Now().UnixNano() % 4294967291)

// GenReqID is a random generate string func
func GenReqID() string {
	var b [12]byte
	binary.LittleEndian.PutUint32(b[:], pid)
	binary.LittleEndian.PutUint64(b[4:], uint64(time.Now().UnixNano()))
	return base64.URLEncoding.EncodeToString(b[:])
}

var (
	green   = string([]byte{27, 91, 57, 55, 59, 52, 50, 109})
	white   = string([]byte{27, 91, 57, 48, 59, 52, 55, 109})
	yellow  = string([]byte{27, 91, 57, 55, 59, 52, 51, 109})
	red     = string([]byte{27, 91, 57, 55, 59, 52, 49, 109})
	blue    = string([]byte{27, 91, 57, 55, 59, 52, 52, 109})
	magenta = string([]byte{27, 91, 57, 55, 59, 52, 53, 109})
	cyan    = string([]byte{27, 91, 57, 55, 59, 52, 54, 109})
	reset   = string([]byte{27, 91, 48, 109})
)

func ErrorLogger() gin.HandlerFunc {
	return ErrorLoggerT(gin.ErrorTypeAny)
}

func ErrorLoggerT(typ gin.ErrorType) gin.HandlerFunc {
	return func(c *gin.Context) {
		c.Next()
		// avoid writting if we already wrote into the response body
		if !c.Writer.Written() {
			errors := c.Errors.ByType(typ)
			if len(errors) > 0 {
				c.JSON(-1, errors)
			}
		}
	}
}

// Logger instances a Logger middleware that will write the logs to gin.DefaultWriter
// By default gin.DefaultWriter = os.Stdout
func Logger() gin.HandlerFunc {
	return LoggerWithWriter(gin.DefaultWriter)
}

// LoggerWithWriter instance a Logger middleware with the specified writter buffer.
// Example: os.Stdout, a file opened in write mode, a socket...
func LoggerWithWriter(out io.Writer, notlogged ...string) gin.HandlerFunc {
	var skip map[string]struct{}

	if length := len(notlogged); length > 0 {
		skip = make(map[string]struct{}, length)

		for _, path := range notlogged {
			skip[path] = struct{}{}
		}
	}

	return func(c *gin.Context) {
		// Start timer
		start := time.Now()

		xReqid := c.Request.Header.Get("X-Reqid")
		if xReqid == "" {
			xReqid = GenReqID()
		}

		c.Header("X-Reqid", xReqid)

		path := c.Request.URL.RequestURI()

		// Log only when path is not being skipped
		if _, ok := skip[path]; !ok {
			fmt.Fprintf(out, "[GIN] [%s] [Started]\t%v |\t  %#v\n",
				xReqid,
				start.Format("2006/01/02 - 15:04:05"),
				c.Request.Header,
			)
		}

		// Process request
		c.Next()

		// Log only when path is not being skipped
		if _, ok := skip[path]; !ok {
			// Stop timer
			end := time.Now()
			latency := end.Sub(start)

			statusCode := c.Writer.Status()
			statusColor := colorForStatus(statusCode)
			clientIP := c.ClientIP()
			method := c.Request.Method
			methodColor := colorForMethod(method)
			comment := c.Errors.ByType(gin.ErrorTypePrivate).String()

			fmt.Fprintf(out, "[GIN] [%s] [Completed]\t%v |%s %3d %s| %13v | %s | %s %s %s|\t %s\n%s",
				xReqid,
				end.Format("2006/01/02 - 15:04:05"),
				statusColor, statusCode, reset,
				latency,
				clientIP,
				methodColor, method, reset,
				c.Request.RequestURI,
				comment,
			)
		}
	}
}

func colorForStatus(code int) string {
	switch {
	case code >= 200 && code < 300:
		return green
	case code >= 300 && code < 400:
		return white
	case code >= 400 && code < 500:
		return yellow
	default:
		return red
	}
}

func colorForMethod(method string) string {
	switch method {
	case "GET":
		return blue
	case "POST":
		return cyan
	case "PUT":
		return yellow
	case "DELETE":
		return red
	case "PATCH":
		return green
	case "HEAD":
		return magenta
	case "OPTIONS":
		return white
	default:
		return reset
	}
}
<|endoftext|>


<commit_before>package handler

import (
	"fmt"
	"os"
)

// A Logger interface is used by handlers to when some kind of output needs to be provided
type Logger interface {
	Print(v ...interface{})
}

type NopLogger struct{}

func (l NopLogger) Print(v ...interface{}) {}

type OutLogger struct{}

func (l OutLogger) Print(v ...interface{}) {
	fmt.Fprint(os.Stdout, v...)
}

type ErrLogger struct{}

func (l ErrLogger) Print(v ...interface{}) {
	fmt.Fprint(os.Stderr, v...)
}
<commit_msg>Add docs to the various simple loggers<commit_after>package handler

import (
	"fmt"
	"os"
)

// A Logger interface is used by handlers to when some kind of output needs to be provided
type Logger interface {
	Print(v ...interface{})
}

// NopLogger is a Logger implementation that does nothing
type NopLogger struct{}

func (l NopLogger) Print(v ...interface{}) {}

// OutLogger is a Logger implementation that outputs to os.Stdout
type OutLogger struct{}

func (l OutLogger) Print(v ...interface{}) {
	fmt.Fprint(os.Stdout, v...)
}

// ErrLogger is a Logger implementation that outputs to os.Stderr
type ErrLogger struct{}

func (l ErrLogger) Print(v ...interface{}) {
	fmt.Fprint(os.Stderr, v...)
}
<|endoftext|>


<commit_before>package queue

import (
	"sync"
	"time"
)

// -----------------------------------------------------------------------------

// TODO
func (q Queue) Shift() *Message {
	q.Lock()
	defer q.Unlock()

	if len(q.msgs) != 0 {
		msg := q.msgs[0]
		q.msgs = q.msgs[1:]
		return msg
	}
	return nil
}

// TODO
func (q Queue) Push(msg *Message) {
	q.Lock()
	defer q.Unlock()

	q.msgs = append(q.msgs, msg)
}

// TODO
// launched with go
// Specify that it is a long running routine
func (q Queue) Poll(IDChan <-chan int, msgChan chan<- *Message, d time.Duration) {
	for {
		select {
		case ID := <-IDChan:
			q.Discard(ID)
		case <-time.After(d):
			msgChan <- q.Purge(d)
		}
	}
}

// TODO
// NOTE: No leak
func (q Queue) Discard(ID int) bool {
	q.Lock()
	defer q.Unlock()

	for i := 0; i < len(q.msgs); i++ {
		if q.msgs[i].ID == ID {
			copy(q.msgs[i:], q.msgs[i+1:])
			q.msgs[len(q.msgs)-1] = nil
			q.msgs = q.msgs[:len(q.msgs)-1]
			return true
		}
	}
	return false
}

// TODO
// NOTE: No leak
// TODO BETTER LOCKING HERE
func (q Queue) Purge(d time.Duration) *Message {
	q.Lock()
	defer q.Unlock()

	for i := 0; i < len(q.msgs); i++ {
		if q.msgs[i].TimeoutReached(d) == true {
			msg := q.msgs[i]
			copy(q.msgs[i:], q.msgs[i+1:])
			q.msgs[len(q.msgs)-1] = nil
			q.msgs = q.msgs[:len(q.msgs)-1]
			return msg
		}
	}
	return nil
}

// TODO + explanation first in last out container
type Queue struct {
	*sync.RWMutex

	msgs []*Message
}

// NewQueue returns a new `queue.Queue`.
func NewQueue(size int) Queue { return Queue{RWMutex: &sync.RWMutex{}, msgs: make([]*Message, size)} }
<commit_msg>docs(queue): add documentation<commit_after>package queue

import (
	"sync"
	"time"
)

// -----------------------------------------------------------------------------

// Shift returns the `queue.Message` contained at the index 0 of `q.msgs` if
// the length of `q.msgs` is greater than 0. It returns `nil` otherwise.
//
// NOTE: This function is thread-safe.
func (q Queue) Shift() *Message {
	q.Lock()
	defer q.Unlock()

	if len(q.msgs) != 0 {
		msg := q.msgs[0]
		q.msgs = q.msgs[1:]
		return msg
	}
	return nil
}

// Push appends `msg` at the end of `q.msgs`.
//
// NOTE: This function is thread-safe.
func (q Queue) Push(msg *Message) {
	q.Lock()
	defer q.Unlock()

	q.msgs = append(q.msgs, msg)
}

// Poll purges `q.msgs` from `queue.Message` no longer relevant.
// In the case where an `ID` is received on `IDChan`, the corresponding
// `queue.Message` is deleted from `q.msgs` and `q.Purge()` is called.
// When no `ID` is received on `IDChan` after `d` time, `q.Purge()` is called.
//
// NOTE: This function is an infinite loop.
func (q Queue) Poll(IDChan <-chan int, msgChan chan<- *Message, d time.Duration) {
	for {
		select {
		case ID := <-IDChan:
			q.Discard(ID)
			msgChan <- q.Purge(d)
		case <-time.After(d):
			msgChan <- q.Purge(d)
		}
	}
}

// Discard deletes the first `queue.Message` contained in `q.msgs` where
// `queue.Message.ID` matches `ID`.
//
// NOTE: This function is thread-safe.
func (q Queue) Discard(ID int) bool {
	q.Lock()
	defer q.Unlock()

	for i := 0; i < len(q.msgs); i++ {
		if q.msgs[i].ID == ID {
			copy(q.msgs[i:], q.msgs[i+1:])
			q.msgs[len(q.msgs)-1] = nil
			q.msgs = q.msgs[:len(q.msgs)-1]
			return true
		}
	}
	return false
}

// Purge deletes all `queue.Message`s from `q.msgs` where the time elapsed
// since `queue.Message.Timeout` is greater or equal to `d`.
//
// NOTE: This function is thread-safe.
func (q Queue) Purge(d time.Duration) *Message {
	q.Lock()
	defer q.Unlock()
	// TODO BETTER LOCKING HERE
	for i := 0; i < len(q.msgs); i++ {
		if q.msgs[i].TimeoutReached(d) == true {
			msg := q.msgs[i]
			copy(q.msgs[i:], q.msgs[i+1:])
			q.msgs[len(q.msgs)-1] = nil
			q.msgs = q.msgs[:len(q.msgs)-1]
			return msg
		}
	}
	return nil
}

// Queue is a structure representing a first-in first-out container used by
// `queue.ZMQBroker` for re-emitting a given `queue.Message`s in the case
// where a `queue.ZMQWorker` crashes while processing this `queue.Message`.
type Queue struct {
	*sync.RWMutex

	msgs []*Message
}

// NewQueue returns a new `queue.Queue`.
func NewQueue(size int) Queue { return Queue{RWMutex: &sync.RWMutex{}, msgs: make([]*Message, size)} }
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	inet "flunky/net"
	daemon "flunky/daemon"
)

var Usage = func() {
	fmt.Fprintf(os.Stderr, "Usage of %s\n", os.Args[0])
	flag.PrintDefaults()
	os.Exit(0)
}

var dump bool
var help bool
var fileDir string
var comStatDaemon *daemon.Daemon

func init() {
	fileDir = "../../../etc/ComStat/"
	comStatDaemon = daemon.New("ComStat", fileDir)
	flag.BoolVar(&dump, "d", false, "Print state information")
	flag.BoolVar(&help, "h", false, "Print help message")
}

func GetDump(daemonName string) {

	server := comStatDaemon.Cfg.Data[daemonName]

	query := inet.NewBuildServer("http://"+server, true)

	resp, err := query.Get("dump")
	if err != nil {
		comStatDaemon.DaemonLog.LogError(fmt.Sprintf("Cannot read data from %s", daemonName), err)
	} else {
	        fmt.Println(string(resp))
		return
	}
}

func main() {
	flag.Parse()

	if help {
		Usage()
		os.Exit(0)
	}
	if dump {
		daemonName := flag.Args()
		for _, name := range daemonName {
		       GetDump(name)
		}
	}

}
<commit_msg>comstat cleanups<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	inet "flunky/net"
	daemon "flunky/daemon"
)

var Usage = func() {
	fmt.Fprintf(os.Stderr, "Usage of %s\n", os.Args[0])
	flag.PrintDefaults()
	os.Exit(0)
}

var dump bool
var help bool
var fileDir string
var comStatDaemon *daemon.Daemon

func init() {
	fileDir = "../../../etc/ComStat/"
	comStatDaemon = daemon.New("ComStat", fileDir)
	flag.BoolVar(&help, "h", false, "Print help message")
}

func GetDump(daemonName string) {

	server, ok := comStatDaemon.Cfg.Data[daemonName]

	if ! ok {
		fmt.Println(fmt.Sprintf("Cannot find URL for component %s", daemonName))
		os.Exit(1)
	}

	query := inet.NewBuildServer("http://"+server, true)

	resp, err := query.Get("dump")
	if err != nil {
		fmt.Println(fmt.Sprintf("Cannot read data from %s", daemonName), err)
	} else {
	        fmt.Println(string(resp))
		return
	}
}

func main() {
	flag.Parse()

	if help {
		Usage()
		os.Exit(0)
	}

	for _, name := range flag.Args() {
		GetDump(name)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"doozer"
	"doozer/util"
	"flag"
	"fmt"
	"net"
	"os"
)

// Flags
var (
	listenAddr  = flag.String("l", "127.0.0.1:8046", "The address to bind to.")
	attachAddr  = flag.String("a", "", "The address of another node to attach to.")
	webAddr     = flag.String("w", ":8080", "Serve web requests on this address.")
	clusterName = flag.String("c", "local", "The non-empty cluster name.")
)

func Usage() {
	fmt.Fprintf(os.Stderr, "Usage: %s [OPTIONS]\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "\nOptions:\n")
	flag.PrintDefaults()
}
func main() {
	util.LogWriter = os.Stderr
	flag.Usage = Usage
	flag.Parse()

	if *listenAddr == "" {
		fmt.Fprintln(os.Stderr, "require a listen address")
		flag.Usage()
		os.Exit(1)
	}

	listener, err := net.Listen("tcp", *listenAddr)
	if err != nil {
		panic(err)
	}

	conn, err := net.ListenPacket("udp", *listenAddr)
	if err != nil {
		panic(err)
	}

	var wl net.Listener
	if *webAddr != "" {
		wl, err = net.Listen("tcp", *webAddr)
		if err != nil {
			panic(err)
		}
	}

	doozer.Main(*clusterName, *attachAddr, conn, listener, wl)
}
<commit_msg>nicer go formating<commit_after>package main

import (
	"doozer"
	"doozer/util"
	"flag"
	"fmt"
	"net"
	"os"
)

var (
	listenAddr  = flag.String("l", "127.0.0.1:8046", "The address to bind to.")
	attachAddr  = flag.String("a", "", "The address of another node to attach to.")
	webAddr     = flag.String("w", ":8080", "Serve web requests on this address.")
	clusterName = flag.String("c", "local", "The non-empty cluster name.")
)

func Usage() {
	fmt.Fprintf(os.Stderr, "Usage: %s [OPTIONS]\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "\nOptions:\n")
	flag.PrintDefaults()
}

func main() {
	util.LogWriter = os.Stderr
	flag.Usage = Usage
	flag.Parse()

	if *listenAddr == "" {
		fmt.Fprintln(os.Stderr, "require a listen address")
		flag.Usage()
		os.Exit(1)
	}

	listener, err := net.Listen("tcp", *listenAddr)
	if err != nil {
		panic(err)
	}

	conn, err := net.ListenPacket("udp", *listenAddr)
	if err != nil {
		panic(err)
	}

	var wl net.Listener
	if *webAddr != "" {
		wl, err = net.Listen("tcp", *webAddr)
		if err != nil {
			panic(err)
		}
	}

	doozer.Main(*clusterName, *attachAddr, conn, listener, wl)
}
<|endoftext|>


<commit_before>/*
Copyright 2014 Google Inc. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// evolution of: https://github.com/GoogleCloudPlatform/kubernetes/blob/release-0.6/pkg/client/cache/fifo.go
package queue

import (
	"fmt"
	"reflect"
	"sync"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
)

type EventType int

const (
	ADD_EVENT EventType = 1 << iota
	UPDATE_EVENT
	DELETE_EVENT
	POP_EVENT
)

type Entry interface {
	Copyable
	Value() UniqueCopyable
	// types is a logically OR'd combination of EventType, e.g. ADD_EVENT|UPDATE_EVENT
	Is(types EventType) bool
}

type entry struct {
	value UniqueCopyable
	event EventType
}

type deletedEntry struct {
	*entry
	expiration time.Time
}

type Copyable interface {
	// return an independent copy (deep clone) of the current object
	Copy() Copyable
}

type UniqueID interface {
	GetUID() string
}

type UniqueCopyable interface {
	Copyable
	UniqueID
}

func (e *entry) Value() UniqueCopyable {
	return e.value
}

func (e *entry) Copy() Copyable {
	if e == nil {
		return nil
	}
	return &entry{e.value.Copy().(UniqueCopyable), e.event}
}

func (e *entry) Is(types EventType) bool {
	return types&e.event != 0
}

func (e *deletedEntry) Copy() Copyable {
	if e == nil {
		return nil
	}
	return &deletedEntry{e.entry.Copy().(*entry), e.expiration}
}

// deliver a message
type pigeon func(msg Entry)

func dead(msg Entry) {
	// intentionally blank
}

// HistoricalFIFO receives adds and updates from a Reflector, and puts them in a queue for
// FIFO order processing. If multiple adds/updates of a single item happen while
// an item is in the queue before it has been processed, it will only be
// processed once, and when it is processed, the most recent version will be
// processed. This can't be done with a channel.
// TODO(jdef): I used to think that I'd need history of state changes recorded
// in `history` but it's turning out that I really only need to maintain the
// the current state, so `history` should revert back into `items`
type HistoricalFIFO struct {
	lock      sync.RWMutex
	cond      sync.Cond
	items     map[string]Entry // We depend on the property that items in the queue are in the set.
	queue     []string
	carrier   pigeon // may be dead, but never nil
	gcc       int
	lingerTTL time.Duration
}

// panics if obj doesn't implement UniqueCopyable; otherwise returns the same, typecast object
func checkType(obj interface{}) UniqueCopyable {
	if v, ok := obj.(UniqueCopyable); !ok {
		panic(fmt.Sprintf("Illegal object type, expected UniqueCopyable: %T", obj))
	} else {
		return v
	}
}

// Add inserts an item, and puts it in the queue. The item is only enqueued
// if it doesn't already exist in the set.
func (f *HistoricalFIFO) Add(id string, v interface{}) {
	obj := checkType(v)
	notifications := []Entry(nil)
	defer func() {
		for _, e := range notifications {
			f.carrier(e)
		}
	}()

	f.lock.Lock()
	defer f.lock.Unlock()

	if entry, exists := f.items[id]; !exists {
		f.queue = append(f.queue, id)
	} else {
		if entry.Is(DELETE_EVENT | POP_EVENT) {
			f.queue = append(f.queue, id)
		}
	}
	notifications = f.merge(id, obj, nil)
	f.cond.Broadcast()
}

// Update is the same as Add in this implementation.
func (f *HistoricalFIFO) Update(id string, obj interface{}) {
	f.Add(id, obj)
}

// Add the item to the store, but only if there exists a prior entry for
// for the object in the store whose event type matches that given, and then
// only enqueued if it doesn't already exist in the set.
func (f *HistoricalFIFO) Readd(id string, v interface{}, t EventType) {
	obj := checkType(v)
	notifications := []Entry(nil)
	defer func() {
		for _, e := range notifications {
			f.carrier(e)
		}
	}()

	f.lock.Lock()
	defer f.lock.Unlock()

	if entry, exists := f.items[id]; exists {
		if !entry.Is(t) {
			return
		} else if entry.Is(DELETE_EVENT | POP_EVENT) {
			f.queue = append(f.queue, id)
		}
	}
	notifications = f.merge(id, obj)
	f.cond.Broadcast()
}

// Delete removes an item. It doesn't add it to the queue, because
// this implementation assumes the consumer only cares about the objects,
// not the order in which they were created/added.
func (f *HistoricalFIFO) Delete(id string) {
	deleteEvent := (Entry)(nil)
	defer func() {
		f.carrier(deleteEvent)
	}()

	f.lock.Lock()
	defer f.lock.Unlock()
	item, exists := f.items[id]
	if exists && !item.Is(DELETE_EVENT) {
		e := item.(*entry)
		e.event = DELETE_EVENT
		deleteEvent = &deletedEntry{e, time.Now().Add(f.lingerTTL)}
		f.items[id] = deleteEvent
	}
}

// List returns a list of all the items.
func (f *HistoricalFIFO) List() []interface{} {
	f.lock.RLock()
	defer f.lock.RUnlock()

	// TODO(jdef): slightly overallocates b/c of deleted items
	list := make([]interface{}, 0, len(f.queue))

	for _, entry := range f.items {
		if entry.Is(DELETE_EVENT | POP_EVENT) {
			continue
		}
		list = append(list, entry.Value().Copy())
	}
	return list
}

// ContainedIDs returns a util.StringSet containing all IDs of the stored items.
// This is a snapshot of a moment in time, and one should keep in mind that
// other go routines can add or remove items after you call this.
func (c *HistoricalFIFO) ContainedIDs() util.StringSet {
	c.lock.RLock()
	defer c.lock.RUnlock()
	set := util.StringSet{}
	for id, entry := range c.items {
		if entry.Is(DELETE_EVENT | POP_EVENT) {
			continue
		}
		set.Insert(id)
	}
	return set
}

// Get returns the requested item, or sets exists=false.
func (f *HistoricalFIFO) Get(id string) (interface{}, bool) {
	f.lock.RLock()
	defer f.lock.RUnlock()
	entry, exists := f.items[id]
	if exists && !entry.Is(DELETE_EVENT|POP_EVENT) {
		return entry.Value().Copy(), true
	}
	return nil, false
}

// Pop waits until an item is ready and returns it. If multiple items are
// ready, they are returned in the order in which they were added/updated.
// The item is removed from the queue (and the store) before it is returned,
// so if you don't succesfully process it, you need to add it back with Add().
func (f *HistoricalFIFO) Pop() interface{} {
	popEvent := (Entry)(nil)
	defer func() {
		f.carrier(popEvent)
	}()

	f.lock.Lock()
	defer f.lock.Unlock()
	for {
		for len(f.queue) == 0 {
			f.cond.Wait()
		}
		id := f.queue[0]
		f.queue = f.queue[1:]
		item, ok := f.items[id]
		if !ok || item.Is(DELETE_EVENT|POP_EVENT) {
			// Item may have been deleted subsequently.
			continue
		}
		value := item.Value()
		popEvent = &entry{value, POP_EVENT}
		f.items[id] = popEvent
		return value.Copy()
	}
}

// Replace will delete the contents of 'f', using instead the given map.
// 'f' takes ownersip of the map, you should not reference the map again
// after calling this function. f's queue is reset, too; upon return, it
// will contain the items in the map, in no particular order.
func (f *HistoricalFIFO) Replace(idToObj map[string]interface{}) {
	notifications := make([]Entry, 0, len(idToObj))
	defer func() {
		for _, e := range notifications {
			f.carrier(e)
		}
	}()

	f.lock.Lock()
	defer f.lock.Unlock()

	f.queue = f.queue[:0]
	now := time.Now()
	for id, v := range f.items {
		if _, exists := idToObj[id]; !exists && !v.Is(DELETE_EVENT) {
			// a non-deleted entry in the items list that doesn't show up in the
			// new list: mark it as deleted
			ent := v.(*entry)
			ent.event = DELETE_EVENT
			e := &deletedEntry{ent, now.Add(f.lingerTTL)}
			f.items[id] = e
			notifications = append(notifications, e)
		}
	}
	for id, v := range idToObj {
		obj := checkType(v)
		f.queue = append(f.queue, id)
		n := f.merge(id, obj, nil)
		notifications = append(notifications, n...)
	}
	if len(f.queue) > 0 {
		f.cond.Broadcast()
	}
}

// garbage collect DELETEd items whose TTL has expired; the IDs of such items are removed
// from the queue. This impl assumes that caller has acquired state lock.
func (f *HistoricalFIFO) gc() {
	now := time.Now()
	deleted := make(map[string]struct{})
	for id, v := range f.items {
		if v.Is(DELETE_EVENT) {
			ent := v.(*deletedEntry)
			if ent.expiration.Before(now) {
				delete(f.items, id)
				deleted[id] = struct{}{}
			}
		}
	}
	// remove deleted items from the queue, will likely (slightly) overallocate here
	queue := make([]string, 0, len(f.queue))
	for _, id := range f.queue {
		if _, exists := deleted[id]; !exists {
			queue = append(queue, id)
		}
	}
	f.queue = queue
}

// Assumes that the caller has acquired the state lock.
func (f *HistoricalFIFO) merge(id string, obj UniqueCopyable) (notifications []Entry) {
	item, exists := f.items[id]
	now := time.Now()
	if !exists {
		e := &entry{obj.Copy().(UniqueCopyable), ADD_EVENT}
		f.items[id] = e
		notifications = append(notifications, e)
	} else {
		if !item.Is(DELETE_EVENT) && item.Value().GetUID() != obj.GetUID() {
			// hidden DELETE!
			// (1) append a DELETE
			// (2) append an ADD
			// .. and notify listeners in that order
			ent := item.(*entry)
			ent.event = DELETE_EVENT
			e1 := &deletedEntry{ent, now.Add(f.lingerTTL)}
			e2 := &entry{obj.Copy().(UniqueCopyable), ADD_EVENT}
			f.items[id] = e2
			notifications = append(notifications, e1, e2)
		} else if !reflect.DeepEqual(obj, item.Value()) {
			//TODO(jdef): it would be nice if we could rely on resource versions
			//instead of doing a DeepEqual. Maybe someday we'll be able to.
			e := &entry{obj.Copy().(UniqueCopyable), UPDATE_EVENT}
			f.items[id] = e
			notifications = append(notifications, e)
		}
	}
	// check for garbage collection
	f.gcc++
	if f.gcc%256 == 0 { //TODO(jdef): extract constant
		f.gcc = 0
		f.gc()
	}
	return
}

// NewFIFO returns a Store which can be used to queue up items to
// process. If a non-nil Mux is provided, then modifications to the
// the FIFO are delivered on a channel specific to this fifo.
func NewFIFO(ch chan<- Entry) *HistoricalFIFO {
	carrier := dead
	if ch != nil {
		carrier = func(msg Entry) {
			if msg != nil {
				ch <- msg.Copy().(Entry)
			}
		}
	}
	f := &HistoricalFIFO{
		items:     map[string]Entry{},
		queue:     []string{},
		carrier:   carrier,
		lingerTTL: 5 * time.Minute, // TODO(jdef): extract constant
	}
	f.cond.L = &f.lock
	return f
}
<commit_msg>revert history to items; update Replace to consider items not present in the replacment map as deleted<commit_after>/*
Copyright 2014 Google Inc. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// evolution of: https://github.com/GoogleCloudPlatform/kubernetes/blob/release-0.6/pkg/client/cache/fifo.go
package queue

import (
	"fmt"
	"reflect"
	"sync"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
)

type EventType int

const (
	ADD_EVENT EventType = 1 << iota
	UPDATE_EVENT
	DELETE_EVENT
	POP_EVENT
)

type Entry interface {
	Copyable
	Value() UniqueCopyable
	// types is a logically OR'd combination of EventType, e.g. ADD_EVENT|UPDATE_EVENT
	Is(types EventType) bool
}

type entry struct {
	value UniqueCopyable
	event EventType
}

type deletedEntry struct {
	*entry
	expiration time.Time
}

type Copyable interface {
	// return an independent copy (deep clone) of the current object
	Copy() Copyable
}

type UniqueID interface {
	GetUID() string
}

type UniqueCopyable interface {
	Copyable
	UniqueID
}

func (e *entry) Value() UniqueCopyable {
	return e.value
}

func (e *entry) Copy() Copyable {
	if e == nil {
		return nil
	}
	return &entry{e.value.Copy().(UniqueCopyable), e.event}
}

func (e *entry) Is(types EventType) bool {
	return types&e.event != 0
}

func (e *deletedEntry) Copy() Copyable {
	if e == nil {
		return nil
	}
	return &deletedEntry{e.entry.Copy().(*entry), e.expiration}
}

// deliver a message
type pigeon func(msg Entry)

func dead(msg Entry) {
	// intentionally blank
}

// HistoricalFIFO receives adds and updates from a Reflector, and puts them in a queue for
// FIFO order processing. If multiple adds/updates of a single item happen while
// an item is in the queue before it has been processed, it will only be
// processed once, and when it is processed, the most recent version will be
// processed. This can't be done with a channel.
type HistoricalFIFO struct {
	lock      sync.RWMutex
	cond      sync.Cond
	items     map[string]Entry // We depend on the property that items in the queue are in the set.
	queue     []string
	carrier   pigeon // may be dead, but never nil
	gcc       int
	lingerTTL time.Duration
}

// panics if obj doesn't implement UniqueCopyable; otherwise returns the same, typecast object
func checkType(obj interface{}) UniqueCopyable {
	if v, ok := obj.(UniqueCopyable); !ok {
		panic(fmt.Sprintf("Illegal object type, expected UniqueCopyable: %T", obj))
	} else {
		return v
	}
}

// Add inserts an item, and puts it in the queue. The item is only enqueued
// if it doesn't already exist in the set.
func (f *HistoricalFIFO) Add(id string, v interface{}) {
	obj := checkType(v)
	notifications := []Entry(nil)
	defer func() {
		for _, e := range notifications {
			f.carrier(e)
		}
	}()

	f.lock.Lock()
	defer f.lock.Unlock()

	if entry, exists := f.items[id]; !exists {
		f.queue = append(f.queue, id)
	} else {
		if entry.Is(DELETE_EVENT | POP_EVENT) {
			f.queue = append(f.queue, id)
		}
	}
	notifications = f.merge(id, obj)
	f.cond.Broadcast()
}

// Update is the same as Add in this implementation.
func (f *HistoricalFIFO) Update(id string, obj interface{}) {
	f.Add(id, obj)
}

// Add the item to the store, but only if there exists a prior entry for
// for the object in the store whose event type matches that given, and then
// only enqueued if it doesn't already exist in the set.
func (f *HistoricalFIFO) Readd(id string, v interface{}, t EventType) {
	obj := checkType(v)
	notifications := []Entry(nil)
	defer func() {
		for _, e := range notifications {
			f.carrier(e)
		}
	}()

	f.lock.Lock()
	defer f.lock.Unlock()

	if entry, exists := f.items[id]; exists {
		if !entry.Is(t) {
			return
		} else if entry.Is(DELETE_EVENT | POP_EVENT) {
			f.queue = append(f.queue, id)
		}
	}
	notifications = f.merge(id, obj)
	f.cond.Broadcast()
}

// Delete removes an item. It doesn't add it to the queue, because
// this implementation assumes the consumer only cares about the objects,
// not the order in which they were created/added.
func (f *HistoricalFIFO) Delete(id string) {
	deleteEvent := (Entry)(nil)
	defer func() {
		f.carrier(deleteEvent)
	}()

	f.lock.Lock()
	defer f.lock.Unlock()
	item, exists := f.items[id]
	if exists && !item.Is(DELETE_EVENT) {
		e := item.(*entry)
		e.event = DELETE_EVENT
		deleteEvent = &deletedEntry{e, time.Now().Add(f.lingerTTL)}
		f.items[id] = deleteEvent
	}
}

// List returns a list of all the items.
func (f *HistoricalFIFO) List() []interface{} {
	f.lock.RLock()
	defer f.lock.RUnlock()

	// TODO(jdef): slightly overallocates b/c of deleted items
	list := make([]interface{}, 0, len(f.queue))

	for _, entry := range f.items {
		if entry.Is(DELETE_EVENT | POP_EVENT) {
			continue
		}
		list = append(list, entry.Value().Copy())
	}
	return list
}

// ContainedIDs returns a util.StringSet containing all IDs of the stored items.
// This is a snapshot of a moment in time, and one should keep in mind that
// other go routines can add or remove items after you call this.
func (c *HistoricalFIFO) ContainedIDs() util.StringSet {
	c.lock.RLock()
	defer c.lock.RUnlock()
	set := util.StringSet{}
	for id, entry := range c.items {
		if entry.Is(DELETE_EVENT | POP_EVENT) {
			continue
		}
		set.Insert(id)
	}
	return set
}

// Get returns the requested item, or sets exists=false.
func (f *HistoricalFIFO) Get(id string) (interface{}, bool) {
	f.lock.RLock()
	defer f.lock.RUnlock()
	entry, exists := f.items[id]
	if exists && !entry.Is(DELETE_EVENT|POP_EVENT) {
		return entry.Value().Copy(), true
	}
	return nil, false
}

// Pop waits until an item is ready and returns it. If multiple items are
// ready, they are returned in the order in which they were added/updated.
// The item is removed from the queue (and the store) before it is returned,
// so if you don't succesfully process it, you need to add it back with Add().
func (f *HistoricalFIFO) Pop() interface{} {
	popEvent := (Entry)(nil)
	defer func() {
		f.carrier(popEvent)
	}()

	f.lock.Lock()
	defer f.lock.Unlock()
	for {
		for len(f.queue) == 0 {
			f.cond.Wait()
		}
		id := f.queue[0]
		f.queue = f.queue[1:]
		item, ok := f.items[id]
		if !ok || item.Is(DELETE_EVENT|POP_EVENT) {
			// Item may have been deleted subsequently.
			continue
		}
		value := item.Value()
		popEvent = &entry{value, POP_EVENT}
		f.items[id] = popEvent
		return value.Copy()
	}
}

// Replace will delete the contents of 'f', using instead the given map.
// 'f' takes ownersip of the map, you should not reference the map again
// after calling this function. f's queue is reset, too; upon return, it
// will contain the items in the map, in no particular order.
func (f *HistoricalFIFO) Replace(idToObj map[string]interface{}) {
	notifications := make([]Entry, 0, len(idToObj))
	defer func() {
		for _, e := range notifications {
			f.carrier(e)
		}
	}()

	f.lock.Lock()
	defer f.lock.Unlock()

	f.queue = f.queue[:0]
	now := time.Now()
	for id, v := range f.items {
		if _, exists := idToObj[id]; !exists && !v.Is(DELETE_EVENT) {
			// a non-deleted entry in the items list that doesn't show up in the
			// new list: mark it as deleted
			ent := v.(*entry)
			ent.event = DELETE_EVENT
			e := &deletedEntry{ent, now.Add(f.lingerTTL)}
			f.items[id] = e
			notifications = append(notifications, e)
		}
	}
	for id, v := range idToObj {
		obj := checkType(v)
		f.queue = append(f.queue, id)
		n := f.merge(id, obj)
		notifications = append(notifications, n...)
	}
	if len(f.queue) > 0 {
		f.cond.Broadcast()
	}
}

// garbage collect DELETEd items whose TTL has expired; the IDs of such items are removed
// from the queue. This impl assumes that caller has acquired state lock.
func (f *HistoricalFIFO) gc() {
	now := time.Now()
	deleted := make(map[string]struct{})
	for id, v := range f.items {
		if v.Is(DELETE_EVENT) {
			ent := v.(*deletedEntry)
			if ent.expiration.Before(now) {
				delete(f.items, id)
				deleted[id] = struct{}{}
			}
		}
	}
	// remove deleted items from the queue, will likely (slightly) overallocate here
	queue := make([]string, 0, len(f.queue))
	for _, id := range f.queue {
		if _, exists := deleted[id]; !exists {
			queue = append(queue, id)
		}
	}
	f.queue = queue
}

// Assumes that the caller has acquired the state lock.
func (f *HistoricalFIFO) merge(id string, obj UniqueCopyable) (notifications []Entry) {
	item, exists := f.items[id]
	now := time.Now()
	if !exists {
		e := &entry{obj.Copy().(UniqueCopyable), ADD_EVENT}
		f.items[id] = e
		notifications = append(notifications, e)
	} else {
		if !item.Is(DELETE_EVENT) && item.Value().GetUID() != obj.GetUID() {
			// hidden DELETE!
			// (1) append a DELETE
			// (2) append an ADD
			// .. and notify listeners in that order
			ent := item.(*entry)
			ent.event = DELETE_EVENT
			e1 := &deletedEntry{ent, now.Add(f.lingerTTL)}
			e2 := &entry{obj.Copy().(UniqueCopyable), ADD_EVENT}
			f.items[id] = e2
			notifications = append(notifications, e1, e2)
		} else if !reflect.DeepEqual(obj, item.Value()) {
			//TODO(jdef): it would be nice if we could rely on resource versions
			//instead of doing a DeepEqual. Maybe someday we'll be able to.
			e := &entry{obj.Copy().(UniqueCopyable), UPDATE_EVENT}
			f.items[id] = e
			notifications = append(notifications, e)
		}
	}
	// check for garbage collection
	f.gcc++
	if f.gcc%256 == 0 { //TODO(jdef): extract constant
		f.gcc = 0
		f.gc()
	}
	return
}

// NewFIFO returns a Store which can be used to queue up items to
// process. If a non-nil Mux is provided, then modifications to the
// the FIFO are delivered on a channel specific to this fifo.
func NewFIFO(ch chan<- Entry) *HistoricalFIFO {
	carrier := dead
	if ch != nil {
		carrier = func(msg Entry) {
			if msg != nil {
				ch <- msg.Copy().(Entry)
			}
		}
	}
	f := &HistoricalFIFO{
		items:     map[string]Entry{},
		queue:     []string{},
		carrier:   carrier,
		lingerTTL: 5 * time.Minute, // TODO(jdef): extract constant
	}
	f.cond.L = &f.lock
	return f
}
<|endoftext|>


<commit_before>// Copyright (c) 2013-2017, Peter H. Froehlich. All rights reserved.
// Use of this source code is governed by a BSD-style license
// that can be found in the LICENSE file.

package queue

import "testing"
import "container/list"
import "math/rand"

func ensureEmpty(t *testing.T, q *Queue) {
	if l := q.Len(); l != 0 {
		t.Errorf("q.Len() = %d, want %d", l, 0)
	}
	if e := q.Front(); e != nil {
		t.Errorf("q.Front() = %v, want %v", e, nil)
	}
	if e := q.Back(); e != nil {
		t.Errorf("q.Back() = %v, want %v", e, nil)
	}
	if e := q.PopFront(); e != nil {
		t.Errorf("q.PopFront() = %v, want %v", e, nil)
	}
	if e := q.PopBack(); e != nil {
		t.Errorf("q.PopBack() = %v, want %v", e, nil)
	}
}

func TestNew(t *testing.T) {
	q := New()
	ensureEmpty(t, q)
}

func ensureSingleton(t *testing.T, q *Queue) {
	if l := q.Len(); l != 1 {
		t.Errorf("q.Len() = %d, want %d", l, 1)
	}
	if e := q.Front(); e != 42 {
		t.Errorf("q.Front() = %v, want %v", e, 42)
	}
	if e := q.Back(); e != 42 {
		t.Errorf("q.Back() = %v, want %v", e, 42)
	}
}

func TestSingleton(t *testing.T) {
	q := New()
	ensureEmpty(t, q)
	q.PushFront(42)
	ensureSingleton(t, q)
	q.PopFront()
	ensureEmpty(t, q)
	q.PushBack(42)
	ensureSingleton(t, q)
	q.PopBack()
	ensureEmpty(t, q)
	q.PushFront(42)
	ensureSingleton(t, q)
	q.PopBack()
	ensureEmpty(t, q)
	q.PushBack(42)
	ensureSingleton(t, q)
	q.PopFront()
	ensureEmpty(t, q)
}

func TestDuos(t *testing.T) {
	q := New()
	ensureEmpty(t, q)
	q.PushFront(42)
	ensureSingleton(t, q)
	q.PushBack(43)
	if l := q.Len(); l != 2 {
		t.Errorf("q.Len() = %d, want %d", l, 2)
	}
	if e := q.Front(); e != 42 {
		t.Errorf("q.Front() = %v, want %v", e, 42)
	}
	if e := q.Back(); e != 43 {
		t.Errorf("q.Back() = %v, want %v", e, 43)
	}
}

func ensureLength(t *testing.T, q *Queue, len int) {
	if l := q.Len(); l != len {
		t.Errorf("q.Len() = %d, want %d", l, len)
	}
}

func TestZeroValue(t *testing.T) {
	var q Queue
	q.PushFront(1)
	ensureLength(t, &q, 1)
	q.PushFront(2)
	ensureLength(t, &q, 2)
	q.PushFront(3)
	ensureLength(t, &q, 3)
	q.PushFront(4)
	ensureLength(t, &q, 4)
	q.PushFront(5)
	ensureLength(t, &q, 5)
	q.PushBack(6)
	ensureLength(t, &q, 6)
	q.PushBack(7)
	ensureLength(t, &q, 7)
	q.PushBack(8)
	ensureLength(t, &q, 8)
	q.PushBack(9)
	ensureLength(t, &q, 9)
	const want = "[5, 4, 3, 2, 1, 6, 7, 8, 9]"
	if s := q.String(); s != want {
		t.Errorf("q.String() = %s, want %s", s, want)
	}
}

const size = 1024

func BenchmarkPushFrontQueue(b *testing.B) {
	for i := 0; i < b.N; i++ {
		var q Queue
		for n := 0; n < size; n++ {
			q.PushFront(n)
		}
	}
}
func BenchmarkPushFrontList(b *testing.B) {
	for i := 0; i < b.N; i++ {
		var q list.List
		for n := 0; n < size; n++ {
			q.PushFront(n)
		}
	}
}

func BenchmarkPushBackQueue(b *testing.B) {
	for i := 0; i < b.N; i++ {
		var q Queue
		for n := 0; n < size; n++ {
			q.PushBack(n)
		}
	}
}
func BenchmarkPushBackList(b *testing.B) {
	for i := 0; i < b.N; i++ {
		var q list.List
		for n := 0; n < size; n++ {
			q.PushBack(n)
		}
	}
}
func BenchmarkPushBackChannel(b *testing.B) {
	for i := 0; i < b.N; i++ {
		q := make(chan interface{}, size)
		for n := 0; n < size; n++ {
			q <- n
		}
		close(q)
	}
}

func BenchmarkRandomQueue(b *testing.B) {
	rand.Seed(64738)
	for i := 0; i < b.N; i++ {
		var q Queue
		for n := 0; n < size; n++ {
			if rand.Float32() < 0.8 {
				q.PushBack(n)
			}
			if rand.Float32() < 0.8 {
				q.PushFront(n)
			}
			if rand.Float32() < 0.5 {
				q.PopFront()
			}
			if rand.Float32() < 0.5 {
				q.PopBack()
			}
		}
	}
}
func BenchmarkRandomList(b *testing.B) {
	rand.Seed(64738)
	for i := 0; i < b.N; i++ {
		var q list.List
		for n := 0; n < size; n++ {
			if rand.Float32() < 0.8 {
				q.PushBack(n)
			}
			if rand.Float32() < 0.8 {
				q.PushFront(n)
			}
			if rand.Float32() < 0.5 {
				if e := q.Front(); e != nil {
					q.Remove(e)
				}
			}
			if rand.Float32() < 0.5 {
				if e := q.Back(); e != nil {
					q.Remove(e)
				}
			}
		}
	}
}
<commit_msg>Generate random numbers ahead of time.<commit_after>// Copyright (c) 2013-2017, Peter H. Froehlich. All rights reserved.
// Use of this source code is governed by a BSD-style license
// that can be found in the LICENSE file.

package queue

import "testing"
import "container/list"
import "math/rand"

func ensureEmpty(t *testing.T, q *Queue) {
	if l := q.Len(); l != 0 {
		t.Errorf("q.Len() = %d, want %d", l, 0)
	}
	if e := q.Front(); e != nil {
		t.Errorf("q.Front() = %v, want %v", e, nil)
	}
	if e := q.Back(); e != nil {
		t.Errorf("q.Back() = %v, want %v", e, nil)
	}
	if e := q.PopFront(); e != nil {
		t.Errorf("q.PopFront() = %v, want %v", e, nil)
	}
	if e := q.PopBack(); e != nil {
		t.Errorf("q.PopBack() = %v, want %v", e, nil)
	}
}

func TestNew(t *testing.T) {
	q := New()
	ensureEmpty(t, q)
}

func ensureSingleton(t *testing.T, q *Queue) {
	if l := q.Len(); l != 1 {
		t.Errorf("q.Len() = %d, want %d", l, 1)
	}
	if e := q.Front(); e != 42 {
		t.Errorf("q.Front() = %v, want %v", e, 42)
	}
	if e := q.Back(); e != 42 {
		t.Errorf("q.Back() = %v, want %v", e, 42)
	}
}

func TestSingleton(t *testing.T) {
	q := New()
	ensureEmpty(t, q)
	q.PushFront(42)
	ensureSingleton(t, q)
	q.PopFront()
	ensureEmpty(t, q)
	q.PushBack(42)
	ensureSingleton(t, q)
	q.PopBack()
	ensureEmpty(t, q)
	q.PushFront(42)
	ensureSingleton(t, q)
	q.PopBack()
	ensureEmpty(t, q)
	q.PushBack(42)
	ensureSingleton(t, q)
	q.PopFront()
	ensureEmpty(t, q)
}

func TestDuos(t *testing.T) {
	q := New()
	ensureEmpty(t, q)
	q.PushFront(42)
	ensureSingleton(t, q)
	q.PushBack(43)
	if l := q.Len(); l != 2 {
		t.Errorf("q.Len() = %d, want %d", l, 2)
	}
	if e := q.Front(); e != 42 {
		t.Errorf("q.Front() = %v, want %v", e, 42)
	}
	if e := q.Back(); e != 43 {
		t.Errorf("q.Back() = %v, want %v", e, 43)
	}
}

func ensureLength(t *testing.T, q *Queue, len int) {
	if l := q.Len(); l != len {
		t.Errorf("q.Len() = %d, want %d", l, len)
	}
}

func TestZeroValue(t *testing.T) {
	var q Queue
	q.PushFront(1)
	ensureLength(t, &q, 1)
	q.PushFront(2)
	ensureLength(t, &q, 2)
	q.PushFront(3)
	ensureLength(t, &q, 3)
	q.PushFront(4)
	ensureLength(t, &q, 4)
	q.PushFront(5)
	ensureLength(t, &q, 5)
	q.PushBack(6)
	ensureLength(t, &q, 6)
	q.PushBack(7)
	ensureLength(t, &q, 7)
	q.PushBack(8)
	ensureLength(t, &q, 8)
	q.PushBack(9)
	ensureLength(t, &q, 9)
	const want = "[5, 4, 3, 2, 1, 6, 7, 8, 9]"
	if s := q.String(); s != want {
		t.Errorf("q.String() = %s, want %s", s, want)
	}
}

const size = 1024

func BenchmarkPushFrontQueue(b *testing.B) {
	for i := 0; i < b.N; i++ {
		var q Queue
		for n := 0; n < size; n++ {
			q.PushFront(n)
		}
	}
}
func BenchmarkPushFrontList(b *testing.B) {
	for i := 0; i < b.N; i++ {
		var q list.List
		for n := 0; n < size; n++ {
			q.PushFront(n)
		}
	}
}

func BenchmarkPushBackQueue(b *testing.B) {
	for i := 0; i < b.N; i++ {
		var q Queue
		for n := 0; n < size; n++ {
			q.PushBack(n)
		}
	}
}
func BenchmarkPushBackList(b *testing.B) {
	for i := 0; i < b.N; i++ {
		var q list.List
		for n := 0; n < size; n++ {
			q.PushBack(n)
		}
	}
}
func BenchmarkPushBackChannel(b *testing.B) {
	for i := 0; i < b.N; i++ {
		q := make(chan interface{}, size)
		for n := 0; n < size; n++ {
			q <- n
		}
		close(q)
	}
}

var rands []float32
func makeRands() {
	if rands != nil {
		return
	}
	rand.Seed(64738)
	for i := 0; i < 4*size; i++ {
		rands = append(rands, rand.Float32())
	}
}
func BenchmarkRandomQueue(b *testing.B) {
	makeRands()
	for i := 0; i < b.N; i++ {
		var q Queue
		for n := 0; n < size; n++ {
			if rands[n*4] < 0.8 {
				q.PushBack(n)
			}
			if rands[n*4+1] < 0.8 {
				q.PushFront(n)
			}
			if rands[n*4+2] < 0.5 {
				q.PopFront()
			}
			if rands[n*4+3] < 0.5 {
				q.PopBack()
			}
		}
	}
}
func BenchmarkRandomList(b *testing.B) {
	makeRands()
	for i := 0; i < b.N; i++ {
		var q list.List
		for n := 0; n < size; n++ {
			if rands[n*4] < 0.8 {
				q.PushBack(n)
			}
			if rands[n*4+1] < 0.8 {
				q.PushFront(n)
			}
			if rands[n*4+2] < 0.5 {
				if e := q.Front(); e != nil {
					q.Remove(e)
				}
			}
			if rands[n*4+3] < 0.5 {
				if e := q.Back(); e != nil {
					q.Remove(e)
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package rat

import (
	"fmt"
	"strings"

	termbox "github.com/nsf/termbox-go"
)

type Pager interface {
	Widget
	AddEventListener(keyStr string, handler func())
	AddAnnotationEventListener(keyStr string, annotationTypes []string, handler func(Context))
	Reload()
	CursorUp()
	CursorDown()
	CursorFirstLine()
	CursorLastLine()
	ScrollUp()
	ScrollDown()
	PageUp()
	PageDown()
}

type cmdPager struct {
	modes                    []Mode
	cmd                      string
	ctx                      Context
	command                  ShellCommand
	buffer                   Buffer
	scrollOffsetY            int
	cursorY                  int
	stop                     chan bool
	eventListeners           map[keyEvent]func()
	annotationEventListeners map[keyEvent]map[string]func(Context)

	box        Box
	headerBox  Box
	contentBox Box
}

func NewCmdPager(modeNames string, cmd string, ctx Context) Pager {
	p := &cmdPager{}

	p.cmd = cmd
	p.ctx = ctx

	p.eventListeners = make(map[keyEvent]func())
	p.annotationEventListeners = make(map[keyEvent]map[string]func(Context))

	p.addDefaultListeners()

	splitModeNames := strings.Split(modeNames, ",")
	p.modes = make([]Mode, 0, len(splitModeNames))

	for _, modeName := range splitModeNames {
		if mode, ok := modes[modeName]; ok {
			p.modes = append(p.modes, mode)

			mode.AddEventListeners(ctx)(p)
		}
	}

	p.RunCommand()

	return p
}

func (p *cmdPager) AddEventListener(keyStr string, handler func()) {
	p.eventListeners[KeyEventFromString(keyStr)] = handler
}

func (p *cmdPager) AddAnnotationEventListener(keyStr string, annotationTypes []string, handler func(Context)) {
	ke := KeyEventFromString(keyStr)

	if _, ok := p.annotationEventListeners[ke]; !ok {
		p.annotationEventListeners[ke] = make(map[string]func(Context))
	}

	for _, annotationType := range annotationTypes {
		p.annotationEventListeners[KeyEventFromString(keyStr)][annotationType] = handler
	}
}

func (p *cmdPager) Destroy() {
	p.Stop()
}

func (p *cmdPager) Stop() {
	p.command.Close()
	p.buffer.Close()
}

func (p *cmdPager) Reload() {
	p.Stop()
	p.RunCommand()
}

func (p *cmdPager) RunCommand() {
	var err error

	if p.command, err = NewShellCommand(p.InterpolatedCmd()); err != nil {
		panic(err)
	}

	p.buffer = NewBuffer(p.command)

	for _, m := range p.modes {
		for _, a := range m.InitAnnotators(p.ctx)() {
			go p.buffer.AnnotateWith(a)
		}
	}
}

func (p *cmdPager) HandleEvent(ke keyEvent) bool {
	p.buffer.Lock()
	defer p.buffer.Unlock()

	annotations := p.buffer.AnnotationsForLine(p.cursorY)
	if handlers, ok := p.annotationEventListeners[ke]; ok && len(annotations) > 0 {
		for _, a := range annotations {
			if handler, ok := handlers[a.Class()]; ok {
				handler(Context{a.Class(): a.Val()})
				return true
			}
		}
	}

	if handler, ok := p.eventListeners[ke]; ok {
		handler()
		return true
	}

	return false
}

func (p *cmdPager) SetBox(box Box) {
	p.box = box
	p.layout()
}

func (p *cmdPager) GetBox() Box {
	return p.box
}

func (p *cmdPager) layout() {
	p.headerBox = NewBox(p.box.Left(), p.box.Top(), p.box.Width(), 1)
	p.contentBox = NewBox(p.box.Left(), p.box.Top()+1, p.box.Width(), p.box.Height()-1)
}

func (p *cmdPager) drawHeader() {
	p.headerBox.DrawStyledRunes(1, 0, StyledRunesFromString(p.InterpolatedCmd(), gTermStyles.Get(termbox.AttrUnderline, termbox.ColorDefault)))

	pagerInfo := StyledRunesFromString(fmt.Sprintf(" %d %d/%d ", p.buffer.NumAnnotations(), p.cursorY+1, p.buffer.NumLines()), gTermStyles.Get(termbox.AttrBold, termbox.ColorDefault))
	p.headerBox.DrawStyledRunes(p.headerBox.Width()-len(pagerInfo), 0, pagerInfo)
}

func (p *cmdPager) drawContent() {
	p.contentBox.DrawStyledRune(1, p.cursorY-p.scrollOffsetY, NewStyledRune('▶', gTermStyles.Get(termbox.ColorRed, termbox.ColorDefault)))

	for y, line := range p.buffer.StyledLines(p.scrollOffsetY, p.contentBox.Height()) {
		p.contentBox.DrawStyledRunes(3, y, []StyledRune(line))
	}
}

func (p *cmdPager) Render() {
	p.buffer.Lock()
	p.drawHeader()
	p.drawContent()
	p.buffer.Unlock()
}

func (p *cmdPager) MoveCursorToY(cursorY int) {
	if cursorY < 0 {
		p.cursorY = 0
	} else if cursorY >= p.buffer.NumLines() {
		p.cursorY = p.buffer.NumLines() - 1
	} else {
		p.cursorY = cursorY
	}

	if p.cursorY < p.scrollOffsetY {
		p.ScrollToY(p.cursorY)
	} else if p.cursorY > p.scrollOffsetY+p.contentBox.Height()-1 {
		p.ScrollToY(p.cursorY - (p.contentBox.Height() - 1))
	}
}

func (p *cmdPager) MoveCursorY(delta int) {
	p.MoveCursorToY(p.cursorY + delta)
}

func (p *cmdPager) ScrollToY(scrollY int) {
	if scrollY < 0 {
		p.scrollOffsetY = 0
	} else if scrollY >= p.buffer.NumLines()-p.contentBox.Height() {
		if p.buffer.NumLines() > p.contentBox.Height() {
			p.scrollOffsetY = p.buffer.NumLines() - p.contentBox.Height()
		} else {
			p.scrollOffsetY = 0
		}
	} else {
		p.scrollOffsetY = scrollY
	}

	if p.cursorY < p.scrollOffsetY {
		p.MoveCursorToY(p.scrollOffsetY)
	} else if p.cursorY > p.scrollOffsetY+p.contentBox.Height()-1 {
		p.MoveCursorToY(p.scrollOffsetY + p.contentBox.Height() - 1)
	}
}

func (p *cmdPager) ScrollY(delta int) {
	p.ScrollToY(p.scrollOffsetY + delta)
}

func (p *cmdPager) CursorY() int {
	return p.cursorY
}

func (p *cmdPager) addDefaultListeners() {
	p.AddEventListener("C-r", p.Reload)
	p.AddEventListener("j", p.CursorDown)
	p.AddEventListener("k", p.CursorUp)
	p.AddEventListener("down", p.CursorDown)
	p.AddEventListener("up", p.CursorUp)
	p.AddEventListener("C-e", p.ScrollDown)
	p.AddEventListener("C-y", p.ScrollUp)
	p.AddEventListener("pgdn", p.PageDown)
	p.AddEventListener("pgup", p.PageUp)
	p.AddEventListener("g", p.CursorFirstLine)
	p.AddEventListener("S-g", p.CursorLastLine)
}

func (p *cmdPager) CursorUp() {
	p.MoveCursorY(-1)
}

func (p *cmdPager) CursorDown() {
	p.MoveCursorY(1)
}

func (p *cmdPager) CursorFirstLine() {
	p.MoveCursorToY(0)
}

func (p *cmdPager) CursorLastLine() {
	p.MoveCursorToY(p.buffer.NumLines())
}

func (p *cmdPager) ScrollUp() {
	p.ScrollY(-1)
}

func (p *cmdPager) ScrollDown() {
	p.ScrollY(1)
}

func (p *cmdPager) PageUp() {
	p.ScrollY(-p.contentBox.Height())
}

func (p *cmdPager) PageDown() {
	p.ScrollY(p.contentBox.Height())
}

func (p *cmdPager) InterpolatedCmd() string {
	return InterpolateContext(p.cmd, p.ctx)
}
<commit_msg>Add the annotations on the current line to context passed to handler<commit_after>package rat

import (
	"fmt"
	"strings"

	termbox "github.com/nsf/termbox-go"
)

type Pager interface {
	Widget
	AddEventListener(keyStr string, handler func())
	AddAnnotationEventListener(keyStr string, annotationTypes []string, handler func(Context))
	Reload()
	CursorUp()
	CursorDown()
	CursorFirstLine()
	CursorLastLine()
	ScrollUp()
	ScrollDown()
	PageUp()
	PageDown()
}

type cmdPager struct {
	modes                    []Mode
	cmd                      string
	ctx                      Context
	command                  ShellCommand
	buffer                   Buffer
	scrollOffsetY            int
	cursorY                  int
	stop                     chan bool
	eventListeners           map[keyEvent]func()
	annotationEventListeners map[keyEvent]map[string]func(Context)

	box        Box
	headerBox  Box
	contentBox Box
}

func NewCmdPager(modeNames string, cmd string, ctx Context) Pager {
	p := &cmdPager{}

	p.cmd = cmd
	p.ctx = ctx

	p.eventListeners = make(map[keyEvent]func())
	p.annotationEventListeners = make(map[keyEvent]map[string]func(Context))

	p.addDefaultListeners()

	splitModeNames := strings.Split(modeNames, ",")
	p.modes = make([]Mode, 0, len(splitModeNames))

	for _, modeName := range splitModeNames {
		if mode, ok := modes[modeName]; ok {
			p.modes = append(p.modes, mode)

			mode.AddEventListeners(ctx)(p)
		}
	}

	p.RunCommand()

	return p
}

func (p *cmdPager) AddEventListener(keyStr string, handler func()) {
	p.eventListeners[KeyEventFromString(keyStr)] = handler
}

func (p *cmdPager) AddAnnotationEventListener(keyStr string, annotationTypes []string, handler func(Context)) {
	ke := KeyEventFromString(keyStr)

	if _, ok := p.annotationEventListeners[ke]; !ok {
		p.annotationEventListeners[ke] = make(map[string]func(Context))
	}

	for _, annotationType := range annotationTypes {
		p.annotationEventListeners[KeyEventFromString(keyStr)][annotationType] = handler
	}
}

func (p *cmdPager) Destroy() {
	p.Stop()
}

func (p *cmdPager) Stop() {
	p.command.Close()
	p.buffer.Close()
}

func (p *cmdPager) Reload() {
	p.Stop()
	p.RunCommand()
}

func (p *cmdPager) RunCommand() {
	var err error

	if p.command, err = NewShellCommand(p.InterpolatedCmd()); err != nil {
		panic(err)
	}

	p.buffer = NewBuffer(p.command)

	for _, m := range p.modes {
		for _, a := range m.InitAnnotators(p.ctx)() {
			go p.buffer.AnnotateWith(a)
		}
	}
}

func (p *cmdPager) HandleEvent(ke keyEvent) bool {
	p.buffer.Lock()
	defer p.buffer.Unlock()

	annotations := p.buffer.AnnotationsForLine(p.cursorY)

	ctx := Context{}
	for _, a := range annotations {
		ctx[a.Class()] = a.Val()
	}

	if handlers, ok := p.annotationEventListeners[ke]; ok && len(annotations) > 0 {
		for _, a := range annotations {
			if handler, ok := handlers[a.Class()]; ok {
				handler(ctx)
				return true
			}
		}
	}

	if handler, ok := p.eventListeners[ke]; ok {
		handler()
		return true
	}

	return false
}

func (p *cmdPager) SetBox(box Box) {
	p.box = box
	p.layout()
}

func (p *cmdPager) GetBox() Box {
	return p.box
}

func (p *cmdPager) layout() {
	p.headerBox = NewBox(p.box.Left(), p.box.Top(), p.box.Width(), 1)
	p.contentBox = NewBox(p.box.Left(), p.box.Top()+1, p.box.Width(), p.box.Height()-1)
}

func (p *cmdPager) drawHeader() {
	p.headerBox.DrawStyledRunes(1, 0, StyledRunesFromString(p.InterpolatedCmd(), gTermStyles.Get(termbox.AttrUnderline, termbox.ColorDefault)))

	pagerInfo := StyledRunesFromString(fmt.Sprintf(" %d %d/%d ", p.buffer.NumAnnotations(), p.cursorY+1, p.buffer.NumLines()), gTermStyles.Get(termbox.AttrBold, termbox.ColorDefault))
	p.headerBox.DrawStyledRunes(p.headerBox.Width()-len(pagerInfo), 0, pagerInfo)
}

func (p *cmdPager) drawContent() {
	p.contentBox.DrawStyledRune(1, p.cursorY-p.scrollOffsetY, NewStyledRune('▶', gTermStyles.Get(termbox.ColorRed, termbox.ColorDefault)))

	for y, line := range p.buffer.StyledLines(p.scrollOffsetY, p.contentBox.Height()) {
		p.contentBox.DrawStyledRunes(3, y, []StyledRune(line))
	}
}

func (p *cmdPager) Render() {
	p.buffer.Lock()
	p.drawHeader()
	p.drawContent()
	p.buffer.Unlock()
}

func (p *cmdPager) MoveCursorToY(cursorY int) {
	if cursorY < 0 {
		p.cursorY = 0
	} else if cursorY >= p.buffer.NumLines() {
		p.cursorY = p.buffer.NumLines() - 1
	} else {
		p.cursorY = cursorY
	}

	if p.cursorY < p.scrollOffsetY {
		p.ScrollToY(p.cursorY)
	} else if p.cursorY > p.scrollOffsetY+p.contentBox.Height()-1 {
		p.ScrollToY(p.cursorY - (p.contentBox.Height() - 1))
	}
}

func (p *cmdPager) MoveCursorY(delta int) {
	p.MoveCursorToY(p.cursorY + delta)
}

func (p *cmdPager) ScrollToY(scrollY int) {
	if scrollY < 0 {
		p.scrollOffsetY = 0
	} else if scrollY >= p.buffer.NumLines()-p.contentBox.Height() {
		if p.buffer.NumLines() > p.contentBox.Height() {
			p.scrollOffsetY = p.buffer.NumLines() - p.contentBox.Height()
		} else {
			p.scrollOffsetY = 0
		}
	} else {
		p.scrollOffsetY = scrollY
	}

	if p.cursorY < p.scrollOffsetY {
		p.MoveCursorToY(p.scrollOffsetY)
	} else if p.cursorY > p.scrollOffsetY+p.contentBox.Height()-1 {
		p.MoveCursorToY(p.scrollOffsetY + p.contentBox.Height() - 1)
	}
}

func (p *cmdPager) ScrollY(delta int) {
	p.ScrollToY(p.scrollOffsetY + delta)
}

func (p *cmdPager) CursorY() int {
	return p.cursorY
}

func (p *cmdPager) addDefaultListeners() {
	p.AddEventListener("C-r", p.Reload)
	p.AddEventListener("j", p.CursorDown)
	p.AddEventListener("k", p.CursorUp)
	p.AddEventListener("down", p.CursorDown)
	p.AddEventListener("up", p.CursorUp)
	p.AddEventListener("C-e", p.ScrollDown)
	p.AddEventListener("C-y", p.ScrollUp)
	p.AddEventListener("pgdn", p.PageDown)
	p.AddEventListener("pgup", p.PageUp)
	p.AddEventListener("g", p.CursorFirstLine)
	p.AddEventListener("S-g", p.CursorLastLine)
}

func (p *cmdPager) CursorUp() {
	p.MoveCursorY(-1)
}

func (p *cmdPager) CursorDown() {
	p.MoveCursorY(1)
}

func (p *cmdPager) CursorFirstLine() {
	p.MoveCursorToY(0)
}

func (p *cmdPager) CursorLastLine() {
	p.MoveCursorToY(p.buffer.NumLines())
}

func (p *cmdPager) ScrollUp() {
	p.ScrollY(-1)
}

func (p *cmdPager) ScrollDown() {
	p.ScrollY(1)
}

func (p *cmdPager) PageUp() {
	p.ScrollY(-p.contentBox.Height())
}

func (p *cmdPager) PageDown() {
	p.ScrollY(p.contentBox.Height())
}

func (p *cmdPager) InterpolatedCmd() string {
	return InterpolateContext(p.cmd, p.ctx)
}
<|endoftext|>


<commit_before>package cache

import (
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/vinaymayar/sgit/utils"
)

var targets = map[string]string{
	"sbt": "target",
	"":    "bin",
}

func SaveCache() {
	settings, err := getCacheSettings()
	if err != nil && os.IsNotExist(err) {
		log.Print(err)
		return
	}

	targetDirs, err := getTargetDirs(settings)
	if err != nil {
		log.Print(err)
		return
	}

	for _, dir := range targetDirs {
		if 0 == len(dir) {
			continue
		}

		relDir, err := filepath.Rel(settings.rootDir, dir)
		if err != nil {
			log.Print(err)
			continue
		}

		log.Printf("Saving %v.\n", relDir)

		_, err = utils.Execute("rsync", "-Rr", relDir, settings.cachePrefix)
		if err != nil {
			log.Print(err)
		}
	}
}

func RestoreCache() {
	settings, err := getCacheSettings()
	if err != nil {
		log.Print(err)
		return
	}

	err = os.Chdir(settings.cachePrefix)
	if err != nil {
		log.Print(noCacheError(settings.branch))
	}
	cacheDirs, err := getCacheDirs(settings)
	if err != nil {
		log.Print(err)
		return
	}

	for _, dir := range cacheDirs {
		if 0 == len(dir) {
			continue
		}

		relDir, err := filepath.Rel(settings.cachePrefix, dir)
		if err != nil {
			log.Print(err)
			continue
		}

		log.Printf("Restoring %v.\n", relDir)

		_, err = utils.Execute("rsync", "-Rr", relDir, settings.rootDir)
		if err != nil {
			log.Print(err)
		}
	}
}

func ClearCache(args []string) {
	sgitDir, err := utils.GetSgitRootDir()
	if err != nil {
		log.Fatal(err)
	}

	if len(args) == 1 {

		utils.Execute(
			"find",
			sgitDir,
			"-d",
			"-mindepth", "1",
			"-type", "d",
			"-exec", "rm", "-r", "{}", ";")

	} else {

		for _, branch := range args[1:] {
			cachePrefix := filepath.Join(sgitDir, branch)
			err := utils.Execute("rm", "-r", cachePrefix)

			if err != nil && os.IsNotExist(err) {
				log.Print(noCacheError(branch))
			}
		}

	}
}

func getCacheDirs(settings CacheSettings) ([]string, error) {
	cacheDirs, err := utils.Execute(
		"find",
		settings.cachePrefix,
		"-name",
		targets[settings.config.ProjectType])
	return strings.Split(strings.TrimSpace(cacheDirs), "\n"), err
}

func getTargetDirs(settings CacheSettings) ([]string, error) {
	targetDirs, err := utils.Execute(
		"find",
		settings.rootDir,
		"-name",
		targets[settings.config.ProjectType],
		"-not",
		"-path",
		filepath.Join(settings.sgitDir, "*"))

	return strings.Split(strings.TrimSpace(targetDirs), "\n"), err
}
<commit_msg>Added command clear-cache to clear the cache.<commit_after>package cache

import (
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/vinaymayar/sgit/utils"
)

var targets = map[string]string{
	"sbt": "target",
	"":    "bin",
}

func SaveCache() {
	settings, err := getCacheSettings()
	if err != nil && os.IsNotExist(err) {
		log.Print(err)
		return
	}

	targetDirs, err := getTargetDirs(settings)
	if err != nil {
		log.Print(err)
		return
	}

	for _, dir := range targetDirs {
		if 0 == len(dir) {
			continue
		}

		relDir, err := filepath.Rel(settings.rootDir, dir)
		if err != nil {
			log.Print(err)
			continue
		}

		log.Printf("Saving %v.\n", relDir)

		_, err = utils.Execute("rsync", "-Rr", relDir, settings.cachePrefix)
		if err != nil {
			log.Print(err)
		}
	}
}

func RestoreCache() {
	settings, err := getCacheSettings()
	if err != nil {
		log.Print(err)
		return
	}

	err = os.Chdir(settings.cachePrefix)
	if err != nil {
		log.Print(noCacheError(settings.branch))
	}
	cacheDirs, err := getCacheDirs(settings)
	if err != nil {
		log.Print(err)
		return
	}

	for _, dir := range cacheDirs {
		if 0 == len(dir) {
			continue
		}

		relDir, err := filepath.Rel(settings.cachePrefix, dir)
		if err != nil {
			log.Print(err)
			continue
		}

		log.Printf("Restoring %v.\n", relDir)

		_, err = utils.Execute("rsync", "-Rr", relDir, settings.rootDir)
		if err != nil {
			log.Print(err)
		}
	}
}

func ClearCache(args []string) {
	sgitDir, err := utils.GetSgitRootDir()
	if err != nil {
		log.Fatal(err)
	}

	if len(args) == 1 {

		utils.Execute(
			"find",
			sgitDir,
			"-d",
			"-mindepth", "1",
			"-type", "d",
			"-exec", "rm", "-r", "{}", ";")

	} else {

		for _, branch := range args[1:] {
			cachePrefix := filepath.Join(sgitDir, branch)
			_, err = utils.Execute("rm", "-r", cachePrefix)

			if err != nil && os.IsNotExist(err) {
				log.Print(noCacheError(branch))
			}
		}

	}
}

func getCacheDirs(settings CacheSettings) ([]string, error) {
	cacheDirs, err := utils.Execute(
		"find",
		settings.cachePrefix,
		"-name",
		targets[settings.config.ProjectType])
	return strings.Split(strings.TrimSpace(cacheDirs), "\n"), err
}

func getTargetDirs(settings CacheSettings) ([]string, error) {
	targetDirs, err := utils.Execute(
		"find",
		settings.rootDir,
		"-name",
		targets[settings.config.ProjectType],
		"-not",
		"-path",
		filepath.Join(settings.sgitDir, "*"))

	return strings.Split(strings.TrimSpace(targetDirs), "\n"), err
}
<|endoftext|>


<commit_before>package cache

import (
	"time"

	"github.com/koding/cache"
	"golang.org/x/net/context"
)

type Cache interface {
	Get(string) (interface{}, error)
	Set(string, interface{}) error
}

func Get(c context.Context, key string) (interface{}, error) {
	return FromContext(c).Get(key)
}

func Set(c context.Context, key string, value interface{}) error {
	return FromContext(c).Set(key, value)
}

// Default creates an in-memory cache with the default
// 24 hour expiration period.
func Default() Cache {
	return cache.NewMemoryWithTTL(time.Hour * 24)
}

// NewTTL returns an in-memory cache with the specified
// ttl expiration period.
func NewTTL(t time.Duration) Cache {
	return cache.NewMemoryWithTTL(t)
}

// NewTTL returns an in-memory cache with the specified
// ttl expiration period.
func NewLRU(size int) Cache {
	return cache.NewLRU(size)
}
<commit_msg>Update cache.go<commit_after>package cache

import (
	"time"

	"github.com/koding/cache"
	"golang.org/x/net/context"
)

type Cache interface {
	Get(string) (interface{}, error)
	Set(string, interface{}) error
}

func Get(c context.Context, key string) (interface{}, error) {
	return FromContext(c).Get(key)
}

func Set(c context.Context, key string, value interface{}) error {
	return FromContext(c).Set(key, value)
}

// Default creates an in-memory cache with the default
// 30 minute expiration period.
func Default() Cache {
	return cache.NewMemoryWithTTL(time.Minute * 30)
}

// NewTTL returns an in-memory cache with the specified
// ttl expiration period.
func NewTTL(t time.Duration) Cache {
	return cache.NewMemoryWithTTL(t)
}

// NewTTL returns an in-memory cache with the specified
// ttl expiration period.
func NewLRU(size int) Cache {
	return cache.NewLRU(size)
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Fission Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cache

import (
	"time"

	"fmt"
	"github.com/platform9/fission"
)

type requestType int

const (
	GET requestType = iota
	SET
	DELETE
	EXPIRE
	COPY
)

type (
	Value struct {
		ctime time.Time
		atime time.Time
		value interface{}
	}
	Cache struct {
		cache          map[interface{}]Value
		expiryTime     time.Duration
		requestChannel chan *request
	}

	request struct {
		requestType
		key             interface{}
		value           interface{}
		responseChannel chan *response
	}
	response struct {
		error
		mapCopy map[interface{}]interface{}
		value   interface{}
	}
)

func (c *Cache) IsOld(v *Value) bool {
	if c.expiryTime == time.Duration(0) {
		return false
	}
	if time.Now().Sub(v.ctime) > c.expiryTime {
		return true
	}
	return false
}

func MakeCache(expiryTime time.Duration) *Cache {
	c := &Cache{
		cache:          make(map[interface{}]Value),
		expiryTime:     expiryTime,
		requestChannel: make(chan *request),
	}
	go c.service()
	if expiryTime != time.Duration(0) {
		go c.expiryService()
	}
	return c
}

func (c *Cache) service() {
	for {
		req := <-c.requestChannel
		resp := &response{}
		switch req.requestType {
		case GET:
			val, ok := c.cache[req.key]
			if !ok {
				resp.error = fission.MakeError(fission.ErrorNotFound,
					fmt.Sprintf("key '%v' not found", req.key))
			} else if c.IsOld(&val) {
				resp.error = fission.MakeError(fission.ErrorNotFound,
					fmt.Sprintf("key '%v' expired (atime %v)", req.key, val.atime))
				delete(c.cache, req.key)
			} else {
				// update atime
				val.atime = time.Now()
				c.cache[req.key] = val
				resp.value = val.value
			}
			req.responseChannel <- resp
		case SET:
			now := time.Now()
			c.cache[req.key] = Value{
				value: req.value,
				ctime: now,
				atime: now,
			}
			req.responseChannel <- resp
		case DELETE:
			delete(c.cache, req.key)
			req.responseChannel <- resp
		case EXPIRE:
			for k, v := range c.cache {
				if c.IsOld(&v) {
					delete(c.cache, k)
				}
			}
			// no response
		case COPY:
			resp.mapCopy = make(map[interface{}]interface{})
			for k, v := range c.cache {
				resp.mapCopy[k] = v
			}
			req.responseChannel <- resp
		default:
			resp.error = fission.MakeError(fission.ErrorInvalidArgument,
				fmt.Sprintf("invalid request type: %v", req.requestType))
			req.responseChannel <- resp
		}
	}
}

func (c *Cache) Get(key interface{}) (interface{}, error) {
	respChannel := make(chan *response)
	c.requestChannel <- &request{
		requestType:     GET,
		key:             key,
		responseChannel: respChannel,
	}
	resp := <-respChannel
	return resp.value, resp.error
}

func (c *Cache) Set(key interface{}, value interface{}) error {
	respChannel := make(chan *response)
	c.requestChannel <- &request{
		requestType:     SET,
		key:             key,
		value:           value,
		responseChannel: respChannel,
	}
	resp := <-respChannel
	return resp.error
}

func (c *Cache) Delete(key interface{}) error {
	respChannel := make(chan *response)
	c.requestChannel <- &request{
		requestType:     DELETE,
		key:             key,
		responseChannel: respChannel,
	}
	resp := <-respChannel
	return resp.error
}

func (c *Cache) Copy() map[interface{}]interface{} {
	respChannel := make(chan *response)
	c.requestChannel <- &request{
		requestType:     COPY,
		responseChannel: respChannel,
	}
	resp := <-respChannel
	return resp.mapCopy
}

func (c *Cache) expiryService() {
	for {
		time.Sleep(time.Minute)
		c.requestChannel <- &request{
			requestType: EXPIRE,
		}
	}
}
<commit_msg>Support cache expiry based on create/last access time<commit_after>/*
Copyright 2016 The Fission Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cache

import (
	"time"

	"errors"
	"fmt"
	"github.com/platform9/fission"
)

type requestType int

const (
	GET requestType = iota
	SET
	DELETE
	EXPIRE
	COPY
)

type (
	Value struct {
		ctime time.Time
		atime time.Time
		value interface{}
	}
	Cache struct {
		cache          map[interface{}]*Value
		ctimeExpiry    time.Duration
		atimeExpiry    time.Duration
		requestChannel chan *request
	}

	request struct {
		requestType
		key             interface{}
		value           interface{}
		responseChannel chan *response
	}
	response struct {
		error
		existingValue interface{}
		mapCopy       map[interface{}]interface{}
		value         interface{}
	}
)

func (c *Cache) IsOld(v *Value) bool {
	if (c.ctimeExpiry != time.Duration(0)) && (time.Now().Sub(v.ctime) > c.ctimeExpiry) {
		return true
	}

	if (c.atimeExpiry != time.Duration(0)) && (time.Now().Sub(v.atime) > c.atimeExpiry) {
		return true
	}

	return false
}

func MakeCache(ctimeExpiry, atimeExpiry time.Duration) *Cache {
	c := &Cache{
		cache:          make(map[interface{}]*Value),
		ctimeExpiry:    ctimeExpiry,
		atimeExpiry:    atimeExpiry,
		requestChannel: make(chan *request),
	}
	go c.service()
	if ctimeExpiry != time.Duration(0) || atimeExpiry != time.Duration(0) {
		go c.expiryService()
	}
	return c
}

func (c *Cache) service() {
	for {
		req := <-c.requestChannel
		resp := &response{}
		switch req.requestType {
		case GET:
			val, ok := c.cache[req.key]
			if !ok {
				resp.error = fission.MakeError(fission.ErrorNotFound,
					fmt.Sprintf("key '%v' not found", req.key))
			} else if c.IsOld(val) {
				resp.error = fission.MakeError(fission.ErrorNotFound,
					fmt.Sprintf("key '%v' expired (atime %v)", req.key, val.atime))
				delete(c.cache, req.key)
			} else {
				// update atime
				val.atime = time.Now()
				c.cache[req.key] = val
				resp.value = val.value
			}
			req.responseChannel <- resp
		case SET:
			now := time.Now()
			if _, ok := c.cache[req.key]; ok {
				val := c.cache[req.key]
				val.atime = time.Now()
				resp.existingValue = val.value
				resp.error = errors.New("value already exists")
			} else {
				c.cache[req.key] = &Value{
					value: req.value,
					ctime: now,
					atime: now,
				}
			}
			req.responseChannel <- resp
		case DELETE:
			delete(c.cache, req.key)
			req.responseChannel <- resp
		case EXPIRE:
			for k, v := range c.cache {
				if c.IsOld(v) {
					delete(c.cache, k)
				}
			}
			// no response
		case COPY:
			resp.mapCopy = make(map[interface{}]interface{})
			for k, v := range c.cache {
				resp.mapCopy[k] = v.value
			}
			req.responseChannel <- resp
		default:
			resp.error = fission.MakeError(fission.ErrorInvalidArgument,
				fmt.Sprintf("invalid request type: %v", req.requestType))
			req.responseChannel <- resp
		}
	}
}

func (c *Cache) Get(key interface{}) (interface{}, error) {
	respChannel := make(chan *response)
	c.requestChannel <- &request{
		requestType:     GET,
		key:             key,
		responseChannel: respChannel,
	}
	resp := <-respChannel
	return resp.value, resp.error
}

// if key exists in the cache, the new value is NOT set; instead an
// error and the old value are returned
func (c *Cache) Set(key interface{}, value interface{}) (error, interface{}) {
	respChannel := make(chan *response)
	c.requestChannel <- &request{
		requestType:     SET,
		key:             key,
		value:           value,
		responseChannel: respChannel,
	}
	resp := <-respChannel
	return resp.error, resp.existingValue
}

func (c *Cache) Delete(key interface{}) error {
	respChannel := make(chan *response)
	c.requestChannel <- &request{
		requestType:     DELETE,
		key:             key,
		responseChannel: respChannel,
	}
	resp := <-respChannel
	return resp.error
}

func (c *Cache) Copy() map[interface{}]interface{} {
	respChannel := make(chan *response)
	c.requestChannel <- &request{
		requestType:     COPY,
		responseChannel: respChannel,
	}
	resp := <-respChannel
	return resp.mapCopy
}

func (c *Cache) expiryService() {
	for {
		time.Sleep(time.Minute)
		c.requestChannel <- &request{
			requestType: EXPIRE,
		}
	}
}
<|endoftext|>