<commit_before>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package managedwriter

import (
	"errors"
	"io"
	"math/rand"
	"strings"
	"time"

	"github.com/googleapis/gax-go/v2"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

var (
	defaultRetryAttempts = 4
)

// This retry predicate is used for higher level retries, enqueing appends onto to a bidi
// channel and evaluating whether an append should be retried (re-enqueued).
func retryPredicate(err error) (shouldRetry, aggressiveBackoff bool) {
	if err == nil {
		return
	}

	s, ok := status.FromError(err)
	// non-status based error conditions.
	if !ok {
		// EOF can happen in the case of connection close.
		if errors.Is(err, io.EOF) {
			shouldRetry = true
			return
		}
		// All other non-status errors are treated as non-retryable (including context errors).
		return
	}
	switch s.Code() {
	case codes.Aborted,
		codes.Canceled,
		codes.DeadlineExceeded,
		codes.Internal,
		codes.Unavailable:
		shouldRetry = true
		return
	case codes.ResourceExhausted:
		if strings.HasPrefix(s.Message(), "Exceeds 'AppendRows throughput' quota") {
			// Note: internal b/246031522 opened to give this a structured error
			// and avoid string parsing.  Should be a QuotaFailure or similar.
			shouldRetry = true
			return
		}
	}
	return
}

// unaryRetryer is for retrying a unary-style operation, like (re)-opening the bidi connection.
type unaryRetryer struct {
	bo gax.Backoff
}

func (ur *unaryRetryer) Retry(err error) (time.Duration, bool) {
	shouldRetry, _ := retryPredicate(err)
	return ur.bo.Pause(), shouldRetry
}

// statelessRetryer is used for backing off within a continuous process, like processing the responses
// from the receive side of the bidi stream.  An individual item in that process has a notion of an attempt
// count, and we use maximum retries as a way of evicting bad items.
type statelessRetryer struct {
	r                *rand.Rand
	minBackoff       time.Duration
	jitter           time.Duration
	aggressiveFactor int
	maxAttempts      int
}

func newStatelessRetryer() *statelessRetryer {
	return &statelessRetryer{
		r:           rand.New(rand.NewSource(time.Now().UnixNano())),
		minBackoff:  50 * time.Millisecond,
		jitter:      time.Second,
		maxAttempts: defaultRetryAttempts,
	}
}

func (sr *statelessRetryer) pause(aggressiveBackoff bool) time.Duration {
	jitter := sr.jitter.Nanoseconds()
	if jitter > 0 {
		jitter = sr.r.Int63n(jitter)
	}
	pause := sr.minBackoff.Nanoseconds() + jitter
	if aggressiveBackoff {
		pause = pause * int64(sr.aggressiveFactor)
	}
	return time.Duration(pause)
}

func (sr *statelessRetryer) Retry(err error, attemptCount int) (time.Duration, bool) {
	if attemptCount >= sr.maxAttempts {
		return 0, false
	}
	shouldRetry, aggressive := retryPredicate(err)
	if shouldRetry {
		return sr.pause(aggressive), true
	}
	return 0, false
}

// shouldReconnect is akin to a retry predicate, in that it evaluates whether we should force
// our bidi stream to close/reopen based on the responses error.  Errors here signal that no
// further appends will succeed.
func shouldReconnect(err error) bool {
	var knownErrors = []error{
		io.EOF,
		status.Error(codes.Unavailable, "the connection is draining"), // errStreamDrain in gRPC transport
	}
	for _, ke := range knownErrors {
		if errors.Is(err, ke) {
			return true
		}
	}
	return false
}
<commit_msg>feat(bigquery/storage/managedwriter): retry on FailedPrecondition (#6761)<commit_after>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package managedwriter

import (
	"errors"
	"io"
	"math/rand"
	"strings"
	"time"

	"github.com/googleapis/gax-go/v2"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

var (
	defaultRetryAttempts = 4
)

// This retry predicate is used for higher level retries, enqueing appends onto to a bidi
// channel and evaluating whether an append should be retried (re-enqueued).
func retryPredicate(err error) (shouldRetry, aggressiveBackoff bool) {
	if err == nil {
		return
	}

	s, ok := status.FromError(err)
	// non-status based error conditions.
	if !ok {
		// EOF can happen in the case of connection close.
		if errors.Is(err, io.EOF) {
			shouldRetry = true
			return
		}
		// All other non-status errors are treated as non-retryable (including context errors).
		return
	}
	switch s.Code() {
	case codes.Aborted,
		codes.Canceled,
		codes.DeadlineExceeded,
		codes.FailedPrecondition,
		codes.Internal,
		codes.Unavailable:
		shouldRetry = true
		return
	case codes.ResourceExhausted:
		if strings.HasPrefix(s.Message(), "Exceeds 'AppendRows throughput' quota") {
			// Note: internal b/246031522 opened to give this a structured error
			// and avoid string parsing.  Should be a QuotaFailure or similar.
			shouldRetry = true
			return
		}
	}
	return
}

// unaryRetryer is for retrying a unary-style operation, like (re)-opening the bidi connection.
type unaryRetryer struct {
	bo gax.Backoff
}

func (ur *unaryRetryer) Retry(err error) (time.Duration, bool) {
	shouldRetry, _ := retryPredicate(err)
	return ur.bo.Pause(), shouldRetry
}

// statelessRetryer is used for backing off within a continuous process, like processing the responses
// from the receive side of the bidi stream.  An individual item in that process has a notion of an attempt
// count, and we use maximum retries as a way of evicting bad items.
type statelessRetryer struct {
	r                *rand.Rand
	minBackoff       time.Duration
	jitter           time.Duration
	aggressiveFactor int
	maxAttempts      int
}

func newStatelessRetryer() *statelessRetryer {
	return &statelessRetryer{
		r:           rand.New(rand.NewSource(time.Now().UnixNano())),
		minBackoff:  50 * time.Millisecond,
		jitter:      time.Second,
		maxAttempts: defaultRetryAttempts,
	}
}

func (sr *statelessRetryer) pause(aggressiveBackoff bool) time.Duration {
	jitter := sr.jitter.Nanoseconds()
	if jitter > 0 {
		jitter = sr.r.Int63n(jitter)
	}
	pause := sr.minBackoff.Nanoseconds() + jitter
	if aggressiveBackoff {
		pause = pause * int64(sr.aggressiveFactor)
	}
	return time.Duration(pause)
}

func (sr *statelessRetryer) Retry(err error, attemptCount int) (time.Duration, bool) {
	if attemptCount >= sr.maxAttempts {
		return 0, false
	}
	shouldRetry, aggressive := retryPredicate(err)
	if shouldRetry {
		return sr.pause(aggressive), true
	}
	return 0, false
}

// shouldReconnect is akin to a retry predicate, in that it evaluates whether we should force
// our bidi stream to close/reopen based on the responses error.  Errors here signal that no
// further appends will succeed.
func shouldReconnect(err error) bool {
	var knownErrors = []error{
		io.EOF,
		status.Error(codes.Unavailable, "the connection is draining"), // errStreamDrain in gRPC transport
	}
	for _, ke := range knownErrors {
		if errors.Is(err, ke) {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package flagparse

import (
	"math"
	"regexp"
)

// HelpFormatter for generating usage messages and argument help strings.
type HelpFormatter struct {
	prog              string
	indentIncrement   int
	maxHelpPosition   int
	width             int
	currentIndent     int
	level             int
	actionMaxLength   int
	rootSection       *section
	currentSection    *section
	whitespaceMatcher *regexp.Regexp
	longBreakMatcher  *regexp.Regexp
}

type section struct{}

// NewHelpFormatter returns new HelpFormatter struct.
func NewHelpFormatter(prog string, indentIncrement, maxHelpPosition, width int) HelpFormatter {
	hf := HelpFormatter{}
	hf.prog = prog
	hf.indentIncrement = indentIncrement
	hf.maxHelpPosition = int(math.Min(float64(maxHelpPosition), math.Max(float64(width-20), float64(indentIncrement*2))))
	hf.width = width
	hf.currentIndent = 0
	hf.level = 0
	hf.actionMaxLength = 0
	hf.rootSection = &section{}
	hf.currentSection = hf.rootSection
	hf.whitespaceMatcher = regexp.MustCompile(`\s+`)
	hf.longBreakMatcher = regexp.MustCompile(`\n\n\n+`)

	return hf
}
<commit_msg>Update help_fomatter struct<commit_after>package flagparse

import (
	"log"
	"math"
	"regexp"
)

// HelpFormatter for generating usage messages and argument help strings.
type HelpFormatter struct {
	prog              string
	indentIncrement   int
	maxHelpPosition   int
	width             int
	currentIndent     int
	level             int
	actionMaxLength   int
	rootSection       *section
	currentSection    *section
	whitespaceMatcher *regexp.Regexp
	longBreakMatcher  *regexp.Regexp
}

// NewHelpFormatter returns new HelpFormatter struct.
func NewHelpFormatter(prog string, indentIncrement, maxHelpPosition, width int) *HelpFormatter {
	hf := &HelpFormatter{}
	hf.prog = prog
	hf.indentIncrement = indentIncrement
	hf.maxHelpPosition = int(math.Min(float64(maxHelpPosition), math.Max(float64(width-20), float64(indentIncrement*2))))
	hf.width = width
	hf.currentIndent = 0
	hf.level = 0
	hf.actionMaxLength = 0
	hf.rootSection = &section{}
	hf.currentSection = hf.rootSection
	hf.whitespaceMatcher = regexp.MustCompile(`\s+`)
	hf.longBreakMatcher = regexp.MustCompile(`\n\n\n+`)

	return hf
}

func (h *HelpFormatter) indent() {
	h.currentIndent += h.indentIncrement
	h.level++
}

func (h *HelpFormatter) dedent() {
	h.currentIndent -= h.indentIncrement
	if h.currentIndent < 0 {
		log.Println("Indent decreased below 0.")
	}
	h.level--
}

type section struct {
	formatter string
	parent    string
	heading   string
	items     []item
}

type item struct {
	function func(s string) string
	args     []string
}

func newSection(formatter string, parent string, heading string) *section {
	var items []item
	return &section{
		formatter,
		parent,
		heading,
		items,
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fstesting

import (
	"fmt"
	"os"
	"os/signal"
	"reflect"
	"strings"
	"sync"

	"github.com/googlecloudplatform/gcsfuse/fs"
	"github.com/jacobsa/ogletest"
	"github.com/jacobsa/ogletest/srcutil"
)

// Dependencies and configration needed by the tests registered by
// RegisterFSTests.
type FSTestConfig struct {
	// Configuration for the fuse server to be created.
	ServerConfig fs.ServerConfig
}

// An interface that all FS tests must implement.
type fsTestInterface interface {
	setUpFSTest(config FSTestConfig)
	tearDownFsTest()
}

func getSuiteName(suiteType reflect.Type) string {
	return strings.Title(suiteType.Name())
}

func isExported(name string) bool {
	return len(name) > 0 && name[0] >= 'A' && name[0] <= 'Z'
}

func getTestMethods(suitePointerType reflect.Type) []reflect.Method {
	var exportedMethods []reflect.Method
	for _, m := range srcutil.GetMethodsInSourceOrder(suitePointerType) {
		if isExported(m.Name) {
			exportedMethods = append(exportedMethods, m)
		}
	}

	return exportedMethods
}

func registerTestSuite(
	conditionName string,
	makeConfig func() FSTestConfig,
	prototype fsTestInterface) {
	suitePointerType := reflect.TypeOf(prototype)
	suiteType := suitePointerType.Elem()

	// We don't need anything fancy at the suite level.
	var ts ogletest.TestSuite
	ts.Name = fmt.Sprintf("%s.%s", conditionName, getSuiteName(suiteType))

	// For each method, we create a test function.
	for _, method := range getTestMethods(suitePointerType) {
		var tf ogletest.TestFunction
		tf.Name = method.Name

		// Create an instance to be shared among SetUp and the test function itself.
		var instance reflect.Value = reflect.New(suiteType)

		// SetUp should create a bucket and then initialize the suite object,
		// remembering that the suite implements fsTestInterface.
		tf.SetUp = func(*ogletest.TestInfo) {
			instance.Interface().(fsTestInterface).setUpFSTest(makeConfig())
		}

		// The test function itself should simply invoke the method.
		methodCopy := method
		tf.Run = func() {
			methodCopy.Func.Call([]reflect.Value{instance})
		}

		// TearDown should work much like SetUp.
		tf.TearDown = func() {
			instance.Interface().(fsTestInterface).tearDownFsTest()

			// If we took a Ctrl-C during this test, prevent further tests.
			select {
			default:
			case <-sigintReceived:
				os.Exit(1)
			}
		}

		// Save the test function.
		ts.TestFunctions = append(ts.TestFunctions, tf)
	}

	// Register the suite.
	ogletest.Register(ts)
}

// Given a function that returns appropriate test config, register test suites
// that exercise a file system created from that config. The condition name
// should be something like "RealGCS" or "FakeGCS".
func RegisterFSTests(conditionName string, makeConfig func() FSTestConfig) {
	ensureSignalHandler()

	// A list of empty instances of the test suites we want to register.
	suitePrototypes := []fsTestInterface{
		&foreignModsTest{},
		&implicitDirsTest{},
		&openTest{},
		&modesTest{},
		&directoryTest{},
		&fileTest{},
	}

	// Register each.
	for _, suitePrototype := range suitePrototypes {
		registerTestSuite(conditionName, makeConfig, suitePrototype)
	}
}

var signalHandlerOnce sync.Once

// A channel that is closed after SIGINT is received.
var sigintReceived = make(chan struct{})

// Make sure the user doesn't freeze the program by hitting Ctrl-C, which is
// frustrating.
//
// Why does the freeze happen? When the kernel goes to clean up after the
// process, it attempts to close its files. That involves telling the kernel
// fuse module to flush, which in turn attempts to contact the process. But
// the user mode end of fuse in the process is already dead, so we have a
// deadlock with a kernel stack that looks something like this:
//
//     [<ffffffff812aa77a>] wait_answer_interruptible+0x6a/0xa0
//     [<ffffffff812aab7b>] __fuse_request_send+0x1fb/0x280
//     [<ffffffff812aac12>] fuse_request_send+0x12/0x20
//     [<ffffffff812b3837>] fuse_flush+0xd7/0x120
//     [<ffffffff811ba79f>] filp_close+0x2f/0x70
//     [<ffffffff811daad8>] put_files_struct+0x88/0xe0
//     [<ffffffff811dabd7>] exit_files+0x47/0x50
//     [<ffffffff81069c86>] do_exit+0x296/0xa50
//     [<ffffffff8106a4bf>] do_group_exit+0x3f/0xa0
//     [<ffffffff8106a534>] SyS_exit_group+0x14/0x20
//     [<ffffffff8172f82d>] system_call_fastpath+0x1a/0x1f
//     [<ffffffffffffffff>] 0xffffffffffffffff
//
// SIGKILL probably does the same thing, but we can't catch that. If a
// process gets into this state, the only way to kill it for sure is to write
// to the 'abort' file in the appropriate sub-directory of
// /sys/fs/fuse/connections, as documented here:
//
//     https://www.kernel.org/doc/Documentation/filesystems/fuse.txt
//
func ensureSignalHandler() {
	signalHandlerOnce.Do(func() {
		sigChan := make(chan os.Signal, 1)
		signal.Notify(sigChan, os.Interrupt)

		go func() {
			for {
				<-sigChan
				fmt.Println("SIGINT received. Halting tests after this one completes.")
				close(sigintReceived)
			}
		}()
	})
}
<commit_msg>Don't close sigintReceived multiple times.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fstesting

import (
	"fmt"
	"os"
	"os/signal"
	"reflect"
	"strings"
	"sync"

	"github.com/googlecloudplatform/gcsfuse/fs"
	"github.com/jacobsa/ogletest"
	"github.com/jacobsa/ogletest/srcutil"
)

// Dependencies and configration needed by the tests registered by
// RegisterFSTests.
type FSTestConfig struct {
	// Configuration for the fuse server to be created.
	ServerConfig fs.ServerConfig
}

// An interface that all FS tests must implement.
type fsTestInterface interface {
	setUpFSTest(config FSTestConfig)
	tearDownFsTest()
}

func getSuiteName(suiteType reflect.Type) string {
	return strings.Title(suiteType.Name())
}

func isExported(name string) bool {
	return len(name) > 0 && name[0] >= 'A' && name[0] <= 'Z'
}

func getTestMethods(suitePointerType reflect.Type) []reflect.Method {
	var exportedMethods []reflect.Method
	for _, m := range srcutil.GetMethodsInSourceOrder(suitePointerType) {
		if isExported(m.Name) {
			exportedMethods = append(exportedMethods, m)
		}
	}

	return exportedMethods
}

func registerTestSuite(
	conditionName string,
	makeConfig func() FSTestConfig,
	prototype fsTestInterface) {
	suitePointerType := reflect.TypeOf(prototype)
	suiteType := suitePointerType.Elem()

	// We don't need anything fancy at the suite level.
	var ts ogletest.TestSuite
	ts.Name = fmt.Sprintf("%s.%s", conditionName, getSuiteName(suiteType))

	// For each method, we create a test function.
	for _, method := range getTestMethods(suitePointerType) {
		var tf ogletest.TestFunction
		tf.Name = method.Name

		// Create an instance to be shared among SetUp and the test function itself.
		var instance reflect.Value = reflect.New(suiteType)

		// SetUp should create a bucket and then initialize the suite object,
		// remembering that the suite implements fsTestInterface.
		tf.SetUp = func(*ogletest.TestInfo) {
			instance.Interface().(fsTestInterface).setUpFSTest(makeConfig())
		}

		// The test function itself should simply invoke the method.
		methodCopy := method
		tf.Run = func() {
			methodCopy.Func.Call([]reflect.Value{instance})
		}

		// TearDown should work much like SetUp.
		tf.TearDown = func() {
			instance.Interface().(fsTestInterface).tearDownFsTest()

			// If we took a Ctrl-C during this test, prevent further tests.
			select {
			default:
			case <-sigintReceived:
				os.Exit(1)
			}
		}

		// Save the test function.
		ts.TestFunctions = append(ts.TestFunctions, tf)
	}

	// Register the suite.
	ogletest.Register(ts)
}

// Given a function that returns appropriate test config, register test suites
// that exercise a file system created from that config. The condition name
// should be something like "RealGCS" or "FakeGCS".
func RegisterFSTests(conditionName string, makeConfig func() FSTestConfig) {
	ensureSignalHandler()

	// A list of empty instances of the test suites we want to register.
	suitePrototypes := []fsTestInterface{
		&foreignModsTest{},
		&implicitDirsTest{},
		&openTest{},
		&modesTest{},
		&directoryTest{},
		&fileTest{},
	}

	// Register each.
	for _, suitePrototype := range suitePrototypes {
		registerTestSuite(conditionName, makeConfig, suitePrototype)
	}
}

var signalHandlerOnce sync.Once

// A channel that is closed after SIGINT is received.
var sigintReceived = make(chan struct{})

// Make sure the user doesn't freeze the program by hitting Ctrl-C, which is
// frustrating.
//
// Why does the freeze happen? When the kernel goes to clean up after the
// process, it attempts to close its files. That involves telling the kernel
// fuse module to flush, which in turn attempts to contact the process. But
// the user mode end of fuse in the process is already dead, so we have a
// deadlock with a kernel stack that looks something like this:
//
//     [<ffffffff812aa77a>] wait_answer_interruptible+0x6a/0xa0
//     [<ffffffff812aab7b>] __fuse_request_send+0x1fb/0x280
//     [<ffffffff812aac12>] fuse_request_send+0x12/0x20
//     [<ffffffff812b3837>] fuse_flush+0xd7/0x120
//     [<ffffffff811ba79f>] filp_close+0x2f/0x70
//     [<ffffffff811daad8>] put_files_struct+0x88/0xe0
//     [<ffffffff811dabd7>] exit_files+0x47/0x50
//     [<ffffffff81069c86>] do_exit+0x296/0xa50
//     [<ffffffff8106a4bf>] do_group_exit+0x3f/0xa0
//     [<ffffffff8106a534>] SyS_exit_group+0x14/0x20
//     [<ffffffff8172f82d>] system_call_fastpath+0x1a/0x1f
//     [<ffffffffffffffff>] 0xffffffffffffffff
//
// SIGKILL probably does the same thing, but we can't catch that. If a
// process gets into this state, the only way to kill it for sure is to write
// to the 'abort' file in the appropriate sub-directory of
// /sys/fs/fuse/connections, as documented here:
//
//     https://www.kernel.org/doc/Documentation/filesystems/fuse.txt
//
func ensureSignalHandler() {
	signalHandlerOnce.Do(func() {
		sigChan := make(chan os.Signal, 1)
		signal.Notify(sigChan, os.Interrupt)

		go func() {
			<-sigChan
			fmt.Println("SIGINT received. Halting tests after this one completes.")
			close(sigintReceived)
		}()
	})
}
<|endoftext|>


<commit_before>package conn

import "github.com/garyburd/redigo/redis"

var createProductLua = `local product_id = redis.call("INCR", "ProductIncrId")
redis.call("HMSET", "product:"..product_id,
"Name", KEYS[1],
"CreateTime", KEYS[2],
"ReviewTime", KEYS[3],
"PublishTime", KEYS[4],
"InterestStart", KEYS[5],
"InterestEnd", KEYS[6],
"Period", KEYS[7],
"FundsAmount", KEYS[8],
"MinPurchase", KEYS[9],
"SalseAmount",KEYS[10],
"InterestYear", KEYS[11],
"RepaymentType", KEYS[12],
"Category", KEYS[13],
"Status", KEYS[14],
"IntegerRequired", KEYS[15],
"RequiredLv", KEYS[16],
"BorrowId", KEYS[17],
"CreateStuffId", KEYS[18],
"ReviewStuffId", KEYS[19])
return product_id
`
var CreateProductScript = redis.NewScript(19, createProductLua)

var multiGetLua = `
	local data = redis.call("SMEMBERS", "language")
	local ret = {}
  	for idx=1,#data do
  		ret[idx] = redis.call("GET","l:"..data[idx])
  	end
  	return ret
   `

var multiGetScript = redis.NewScript(0, multiGetLua)
<commit_msg>test author<commit_after>package conn

import "github.com/garyburd/redigo/redis"

var createProductLua = `local product_id = redis.call("INCR", "ProductIncrId")
redis.call("HMSET", "product:"..product_id,
"Name", KEYS[1],
"CreateTime", KEYS[2],
"ReviewTime", KEYS[3],
"PublishTime", KEYS[4],
"InterestStart", KEYS[5],
"InterestEnd", KEYS[6],
"Period", KEYS[7],
"FundsAmount", KEYS[8],
"MinPurchase", KEYS[9],
"SalseAmount",KEYS[10],
"InterestYear", KEYS[11],
"RepaymentType", KEYS[12],
"Category", KEYS[13],
"Status", KEYS[14],
"IntegerRequired", KEYS[15],
"RequiredLv", KEYS[16],
"BorrowId", KEYS[17],
"CreateStuffId", KEYS[18],
"ReviewStuffId", KEYS[19])
return product_id
`
var CreateProductScript = redis.NewScript(19, createProductLua)

var multiGetLua = `
	local data = redis.call("SMEMBERS", "language")
	local ret = {}
  	for idx=1,#data do
  		ret[idx] = redis.call("GET","l:"..data[idx])
  	end
  	return ret
   `

var multiGetScript = redis.NewScript(0, multiGetLua)
var test string
<|endoftext|>


<commit_before>/*
 *  Copyright (c) 2011 NeuStar, Inc.
 *  All rights reserved.
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 *
 *  NeuStar, the Neustar logo and related names and logos are registered
 *  trademarks, service marks or tradenames of NeuStar, Inc. All other
 *  product names, company names, marks, logos and symbols may be trademarks
 *  of their respective owners.
 */

package kafka

import (
	"encoding/binary"
	"io"
	"log"
	"net"
	"time"
)

// BrokerConsumer holds a Kafka broker instance and the consumer settings
type BrokerConsumer struct {
	broker  *Broker
	offset  uint64
	maxSize uint32
	codecs  map[byte]PayloadCodec
}

// NewBrokerConsumer creates a new broker consumer
// * hostname - host and optionally port, delimited by ':'
// * topic to consume
// * partition to consume from
// * offset to start consuming from
// * maxSize (in bytes) of the message to consume (this should be at least as big as the biggest message to be published)
func NewBrokerConsumer(hostname string, topic string, partition int, offset uint64, maxSize uint32) *BrokerConsumer {
	return &BrokerConsumer{broker: newBroker(hostname, topic, partition),
		offset:  offset,
		maxSize: maxSize,
		codecs:  DefaultCodecsMap}
}

// NewBrokerOffsetConsumer creates a simplified consumer that defaults the offset and maxSize to 0.
// * hostname - host and optionally port, delimited by ':'
// * topic to consume
// * partition to consume from
func NewBrokerOffsetConsumer(hostname string, topic string, partition int) *BrokerConsumer {
	return &BrokerConsumer{broker: newBroker(hostname, topic, partition),
		offset:  0,
		maxSize: 0,
		codecs:  DefaultCodecsMap}
}

// AddCodecs is a utility method to add Custom Payload Codecs for Consumer Decoding
// payloadCodecs - an array of PayloadCodec implementations
func (consumer *BrokerConsumer) AddCodecs(payloadCodecs []PayloadCodec) {
	// merge to the default map, so one 'could' override the default codecs..
	for k, v := range codecsMap(payloadCodecs) {
		consumer.codecs[k] = v
	}
}

// ConsumeOnChannel fetches messages from kafka and enqueues them in a channel
func (consumer *BrokerConsumer) ConsumeOnChannel(msgChan chan *Message, pollTimeoutMs int64, quit chan struct{}) (int, error) {
	conn, err := consumer.broker.connect()
	if err != nil {
		return -1, err
	}

	forceQuit := make(chan struct{})

	num := 0
	done := make(chan bool, 1)
	go func() {
	Loop:
		for {
			_, err := consumer.consumeWithConn(conn, func(msg *Message) {
				//msg.Print()
				msgChan <- msg
				num++
			}, quit)

			if err != nil {
				if err != io.EOF && err.Error() != "use of closed network connection" { //
					log.Println("Fatal Error: ", err)
					panic(err)
				}
				close(forceQuit)
				break
			}

			// time.Sleep(time.Millisecond * time.Duration(pollTimeoutMs))
			select {
			case <-quit:
				log.Println("Kafka consumer - received request to stop")
				break Loop
			case <-time.After(time.Millisecond * time.Duration(pollTimeoutMs)):
				// carry on after the polling interval
			}
		}
		done <- true
	}()

	// wait to be told to stop...
	select {
	case <-forceQuit:
		// stopping from within this function
	case <-quit:
		// we were told to stop from outside
	}

	conn.Close()
	<-done
	close(msgChan)
	return num, err
}

// MessageHandlerFunc defines the interface for message handlers accepted by Consume()
type MessageHandlerFunc func(msg *Message)

// Consume makes a single fetch request and sends the messages in the message set to a handler function
func (consumer *BrokerConsumer) Consume(handlerFunc MessageHandlerFunc, stop <-chan struct{}) (int, error) {
	conn, err := consumer.broker.connect()
	if err != nil {
		return -1, err
	}
	defer conn.Close()

	num, err := consumer.consumeWithConn(conn, handlerFunc, stop)

	if err != nil {
		log.Println("Fatal Error: ", err)
	}

	return num, err
}

func (consumer *BrokerConsumer) consumeWithConn(conn *net.TCPConn, handlerFunc MessageHandlerFunc, stop <-chan struct{}) (int, error) {
	_, err := conn.Write(consumer.broker.EncodeConsumeRequest(consumer.offset, consumer.maxSize))
	if err != nil {
		log.Println("Failed kafka fetch request:", err.Error())
		return -1, err
	}

	length, payload, err := consumer.broker.readResponse(conn)
	//log.Println("kafka fetch request of", length, "bytes starting from offset", consumer.offset)

	if err != nil {
		return -1, err
	}

	num := 0
	if length > 2 {
		// parse out the messages
		currentOffset := uint64(0)
		for currentOffset <= uint64(length-4) {
			totalLength, msgs, err1 := Decode(payload[currentOffset:], consumer.codecs)
			if ErrIncompletePacket == err1 {
				// Reached the end of the current packet and the last message is incomplete.
				if 0 == num {
					// This is the very first message in the batch => we need to request a larger packet
					// or the consumer will get stuck here indefinitely
					log.Printf("ERROR: Incomplete message at offset %d %d, change the configuration to a larger max fetch size\n",
						consumer.offset,
						currentOffset)
				} else {
					// Partial message at end of current batch, need a new Fetch Request from a newer offset
					log.Printf("DEBUG: Incomplete message at offset %d %d for topic '%s' (%s, partition %d), fetching new batch from offset %d\n",
						consumer.offset,
						currentOffset,
						consumer.broker.topic,
						consumer.broker.hostname,
						consumer.broker.partition,
						consumer.offset+currentOffset)
				}
				break
			}
			msgOffset := consumer.offset + currentOffset
			for _, msg := range msgs {
				// do a non-blocking select to see whether we received a request to stop reading
				select {
				case <-stop:
					//fmt.Println("received request to stop whilst iterating message set")
					return num, err
				default:
					// update the offset of whole message set
					// multiple messages can be at the same offset (compressed for example)
					msg.offset = msgOffset
					handlerFunc(&msg)
					num++
				}

			}
			currentOffset += uint64(4 + totalLength)
		}
		// update the broker's offset for next consumption
		consumer.offset += currentOffset
	}

	return num, err
}

// GetOffsets returns a list of valid offsets (up to maxNumOffsets) before the given time, where
// time is in milliseconds (-1, from the latest offset available, -2 from the smallest offset available)
// The result is a list of offsets, in descending order.
func (consumer *BrokerConsumer) GetOffsets(time int64, maxNumOffsets uint32) ([]uint64, error) {
	var offsets []uint64

	conn, err := consumer.broker.connect()
	if err != nil {
		return offsets, err
	}

	defer conn.Close()

	_, err = conn.Write(consumer.broker.EncodeOffsetRequest(time, maxNumOffsets))
	if err != nil {
		return offsets, err
	}

	length, payload, err := consumer.broker.readResponse(conn)
	if err != nil {
		return offsets, err
	}

	if length > 4 {
		// get the number of offsets
		numOffsets := binary.BigEndian.Uint32(payload[0:])
		var currentOffset uint64 = 4
		for currentOffset < uint64(length-4) && uint32(len(offsets)) < numOffsets {
			offset := binary.BigEndian.Uint64(payload[currentOffset:])
			offsets = append(offsets, offset)
			currentOffset += 8 // offset size
		}
	}

	return offsets, err
}

// GetOffset returns the current offset for a broker.
func (consumer *BrokerConsumer) GetOffset() uint64 {
	return consumer.offset
}
<commit_msg>also print hex payload in case of fatal error<commit_after>/*
 *  Copyright (c) 2011 NeuStar, Inc.
 *  All rights reserved.
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 *
 *  NeuStar, the Neustar logo and related names and logos are registered
 *  trademarks, service marks or tradenames of NeuStar, Inc. All other
 *  product names, company names, marks, logos and symbols may be trademarks
 *  of their respective owners.
 */

package kafka

import (
	"encoding/binary"
	"io"
	"log"
	"net"
	"time"
)

// BrokerConsumer holds a Kafka broker instance and the consumer settings
type BrokerConsumer struct {
	broker  *Broker
	offset  uint64
	maxSize uint32
	codecs  map[byte]PayloadCodec
}

// NewBrokerConsumer creates a new broker consumer
// * hostname - host and optionally port, delimited by ':'
// * topic to consume
// * partition to consume from
// * offset to start consuming from
// * maxSize (in bytes) of the message to consume (this should be at least as big as the biggest message to be published)
func NewBrokerConsumer(hostname string, topic string, partition int, offset uint64, maxSize uint32) *BrokerConsumer {
	return &BrokerConsumer{broker: newBroker(hostname, topic, partition),
		offset:  offset,
		maxSize: maxSize,
		codecs:  DefaultCodecsMap}
}

// NewBrokerOffsetConsumer creates a simplified consumer that defaults the offset and maxSize to 0.
// * hostname - host and optionally port, delimited by ':'
// * topic to consume
// * partition to consume from
func NewBrokerOffsetConsumer(hostname string, topic string, partition int) *BrokerConsumer {
	return &BrokerConsumer{broker: newBroker(hostname, topic, partition),
		offset:  0,
		maxSize: 0,
		codecs:  DefaultCodecsMap}
}

// AddCodecs is a utility method to add Custom Payload Codecs for Consumer Decoding
// payloadCodecs - an array of PayloadCodec implementations
func (consumer *BrokerConsumer) AddCodecs(payloadCodecs []PayloadCodec) {
	// merge to the default map, so one 'could' override the default codecs..
	for k, v := range codecsMap(payloadCodecs) {
		consumer.codecs[k] = v
	}
}

// ConsumeOnChannel fetches messages from kafka and enqueues them in a channel
func (consumer *BrokerConsumer) ConsumeOnChannel(msgChan chan *Message, pollTimeoutMs int64, quit chan struct{}) (int, error) {
	conn, err := consumer.broker.connect()
	if err != nil {
		return -1, err
	}

	forceQuit := make(chan struct{})

	num := 0
	done := make(chan bool, 1)
	go func() {
	Loop:
		for {
			_, err := consumer.consumeWithConn(conn, func(msg *Message) {
				//msg.Print()
				msgChan <- msg
				num++
			}, quit)

			if err != nil {
				if err != io.EOF && err.Error() != "use of closed network connection" { //
					log.Println("Fatal Error: ", err)
					panic(err)
				}
				close(forceQuit)
				break
			}

			// time.Sleep(time.Millisecond * time.Duration(pollTimeoutMs))
			select {
			case <-quit:
				log.Println("Kafka consumer - received request to stop")
				break Loop
			case <-time.After(time.Millisecond * time.Duration(pollTimeoutMs)):
				// carry on after the polling interval
			}
		}
		done <- true
	}()

	// wait to be told to stop...
	select {
	case <-forceQuit:
		// stopping from within this function
	case <-quit:
		// we were told to stop from outside
	}

	conn.Close()
	<-done
	close(msgChan)
	return num, err
}

// MessageHandlerFunc defines the interface for message handlers accepted by Consume()
type MessageHandlerFunc func(msg *Message)

// Consume makes a single fetch request and sends the messages in the message set to a handler function
func (consumer *BrokerConsumer) Consume(handlerFunc MessageHandlerFunc, stop <-chan struct{}) (int, error) {
	conn, err := consumer.broker.connect()
	if err != nil {
		return -1, err
	}
	defer conn.Close()

	num, err := consumer.consumeWithConn(conn, handlerFunc, stop)

	if err != nil {
		log.Println("Fatal Error: ", err)
	}

	return num, err
}

func (consumer *BrokerConsumer) consumeWithConn(conn *net.TCPConn, handlerFunc MessageHandlerFunc, stop <-chan struct{}) (int, error) {
	_, err := conn.Write(consumer.broker.EncodeConsumeRequest(consumer.offset, consumer.maxSize))
	if err != nil {
		log.Println("Failed kafka fetch request:", err.Error())
		return -1, err
	}

	length, payload, err := consumer.broker.readResponse(conn)
	//log.Println("kafka fetch request of", length, "bytes starting from offset", consumer.offset)

	if err != nil {
		return -1, err
	}

	num := 0
	if length > 2 {
		// parse out the messages
		currentOffset := uint64(0)
		for currentOffset <= uint64(length-4) {
			totalLength, msgs, err1 := Decode(payload[currentOffset:], consumer.codecs)
			if ErrIncompletePacket == err1 {
				// Reached the end of the current packet and the last message is incomplete.
				if 0 == num {
					// This is the very first message in the batch => we need to request a larger packet
					// or the consumer will get stuck here indefinitely
					log.Printf("ERROR: Incomplete message at offset %d %d, change the configuration to a larger max fetch size\n",
						consumer.offset,
						currentOffset)
					log.Printf("Payload length: %d, currentOffset: %d, payload: [%x]", length, currentOffset, payload)
				} else {
					// Partial message at end of current batch, need a new Fetch Request from a newer offset
					log.Printf("DEBUG: Incomplete message at offset %d %d for topic '%s' (%s, partition %d), fetching new batch from offset %d\n",
						consumer.offset,
						currentOffset,
						consumer.broker.topic,
						consumer.broker.hostname,
						consumer.broker.partition,
						consumer.offset+currentOffset)
				}
				break
			}
			msgOffset := consumer.offset + currentOffset
			for _, msg := range msgs {
				// do a non-blocking select to see whether we received a request to stop reading
				select {
				case <-stop:
					//fmt.Println("received request to stop whilst iterating message set")
					return num, err
				default:
					// update the offset of whole message set
					// multiple messages can be at the same offset (compressed for example)
					msg.offset = msgOffset
					handlerFunc(&msg)
					num++
				}

			}
			currentOffset += uint64(4 + totalLength)
		}
		// update the broker's offset for next consumption
		consumer.offset += currentOffset
	}

	return num, err
}

// GetOffsets returns a list of valid offsets (up to maxNumOffsets) before the given time, where
// time is in milliseconds (-1, from the latest offset available, -2 from the smallest offset available)
// The result is a list of offsets, in descending order.
func (consumer *BrokerConsumer) GetOffsets(time int64, maxNumOffsets uint32) ([]uint64, error) {
	var offsets []uint64

	conn, err := consumer.broker.connect()
	if err != nil {
		return offsets, err
	}

	defer conn.Close()

	_, err = conn.Write(consumer.broker.EncodeOffsetRequest(time, maxNumOffsets))
	if err != nil {
		return offsets, err
	}

	length, payload, err := consumer.broker.readResponse(conn)
	if err != nil {
		return offsets, err
	}

	if length > 4 {
		// get the number of offsets
		numOffsets := binary.BigEndian.Uint32(payload[0:])
		var currentOffset uint64 = 4
		for currentOffset < uint64(length-4) && uint32(len(offsets)) < numOffsets {
			offset := binary.BigEndian.Uint64(payload[currentOffset:])
			offsets = append(offsets, offset)
			currentOffset += 8 // offset size
		}
	}

	return offsets, err
}

// GetOffset returns the current offset for a broker.
func (consumer *BrokerConsumer) GetOffset() uint64 {
	return consumer.offset
}
<|endoftext|>


<commit_before>package jsval

import (
	"errors"
	"fmt"
	"reflect"

	"github.com/lestrrat/go-pdebug"
)

// Array creates a new ArrayConstraint
func Array() *ArrayConstraint {
	return &ArrayConstraint{
		additionalItems: EmptyConstraint,
		maxItems:        -1,
		minItems:        -1,
	}
}

// Validate validates the given value against this Constraint
func (c *ArrayConstraint) Validate(v interface{}) (err error) {
	if pdebug.Enabled {
		g := pdebug.IPrintf("START ArrayConstraint.Validate")
		defer func() {
			if err == nil {
				g.IRelease("END ArrayConstraint.Validate (PASS)")
			} else {
				g.IRelease("END ArrayConstraint.Validate (FAIL): %s", err)
			}
		}()
	}

	rv := reflect.ValueOf(v)
	switch rv.Kind() {
	case reflect.Slice:
	default:
		return errors.New("value must be a slice (was: " + rv.Type().String() + ")")
	}

	l := rv.Len()

	if mi := c.minItems; mi > -1 && l < mi {
		return errors.New("fewer items than minItems")
	}

	if mi := c.maxItems; mi > -1 && l > mi {
		return errors.New("more items than maxItems")
	}

	var uitems map[string]struct{}
	if c.uniqueItems {
		pdebug.Printf("Check for unique items enabled")
		uitems = make(map[string]struct{})
		for i := 0; i < l; i++ {
			iv := rv.Index(i).Interface()
			kv := fmt.Sprintf("%s", iv)
pdebug.Printf("unique? -> %s", kv)
			if _, ok := uitems[kv]; ok {
				return errors.New("duplicate element found")
			}
			uitems[kv] = struct{}{}
		}
	}

	if celem := c.items; celem != nil {
		if pdebug.Enabled {
			pdebug.Printf("Checking if all items match a spec")
		}
		// if this is set, then all items must fulfill this.
		// additional items are ignored
		for i := 0; i < l; i++ {
			iv := rv.Index(i).Interface()
			if err := celem.Validate(iv); err != nil {
				return err
			}
		}
	} else {
		// otherwise, check the positional specs, and apply the
		// additionalItems constraint
		lv := rv.Len()
		for i, cpos := range c.positionalItems {
			if lv <= i {
				break
			}
			if pdebug.Enabled {
				pdebug.Printf("Checking positional item at '%d'", i)
			}
			iv := rv.Index(i).Interface()
			if err := cpos.Validate(iv); err != nil {
				return err
			}
		}

		lp := len(c.positionalItems)
		if lp > 0 && l > lp { // we got more than positional schemas
			cadd := c.additionalItems
			if cadd == nil { // you can't have additionalItems!
				return errors.New("additional elements found in array")
			}
			for i := lp - 1; i < l; i++ {
				iv := rv.Index(i).Interface()
				if err := cadd.Validate(iv); err != nil {
					return err
				}
			}
		}
	}
	return nil
}

// AdditionalItems specifies the constraint that additional items
// must be validated against. Note that if you specify `Items`
// with this constraint, `AdditionalItems` has no effect. This
// constraint only makes sense when used along side with the
// `PositionalItems` constraint.
//
// If you want to allow any additional items, pass `EmptyConstraint`
// to this method. If unspecified, no extra items are allowed.
func (c *ArrayConstraint) AdditionalItems(ac Constraint) *ArrayConstraint {
	c.additionalItems = ac
	return c
}

// Items specifies the constraint that all items in the array
// must be validated against
func (c *ArrayConstraint) Items(ac Constraint) *ArrayConstraint {
	c.items = ac
	return c
}

// MinItems specifies the minimum number of items in the value.
// If unspecified, the check is not performed.
func (c *ArrayConstraint) MinItems(i int) *ArrayConstraint {
	c.minItems = i
	return c
}

// MaxItems specifies the maximum number of items in the value.
// If unspecified, the check is not performed.
func (c *ArrayConstraint) MaxItems(i int) *ArrayConstraint {
	c.maxItems = i
	return c
}

// PositionalItems specifies the constraint that elements in
// each position of the array being validated. Extra items in
// the array are validated against the constraint specified for
// `AdditionalItems`.
func (c *ArrayConstraint) PositionalItems(ac []Constraint) *ArrayConstraint {
	c.positionalItems = ac
	return c
}

// UniqueItems specifies if the array can hold non-unique items.
// When set to true, the validation will fail unless all of your
// elements are unique.
//
// Caveat: Currently we stringify the elements before comparing
// for uniqueness. This could/should be fixed.
func (c *ArrayConstraint) UniqueItems(b bool) *ArrayConstraint {
	if pdebug.Enabled {
		pdebug.Printf("Setting uniqueItems = %t", b)
	}
	c.uniqueItems = b
	return c
}<commit_msg>check for invalid values<commit_after>package jsval

import (
	"errors"
	"fmt"
	"reflect"

	"github.com/lestrrat/go-pdebug"
)

// Array creates a new ArrayConstraint
func Array() *ArrayConstraint {
	return &ArrayConstraint{
		additionalItems: EmptyConstraint,
		maxItems:        -1,
		minItems:        -1,
	}
}

// Validate validates the given value against this Constraint
func (c *ArrayConstraint) Validate(v interface{}) (err error) {
	if pdebug.Enabled {
		g := pdebug.IPrintf("START ArrayConstraint.Validate")
		defer func() {
			if err == nil {
				g.IRelease("END ArrayConstraint.Validate (PASS)")
			} else {
				g.IRelease("END ArrayConstraint.Validate (FAIL): %s", err)
			}
		}()
	}

	rv := reflect.ValueOf(v)
	switch rv.Kind() {
	case reflect.Slice:
	default:
		var typ string
		if rv == zeroval {
			typ = "invalid"
		} else {
			typ = rv.Type().String()
		}
		return errors.New("value must be a slice (was: " + typ + ")")
	}

	l := rv.Len()

	if mi := c.minItems; mi > -1 && l < mi {
		return errors.New("fewer items than minItems")
	}

	if mi := c.maxItems; mi > -1 && l > mi {
		return errors.New("more items than maxItems")
	}

	var uitems map[string]struct{}
	if c.uniqueItems {
		pdebug.Printf("Check for unique items enabled")
		uitems = make(map[string]struct{})
		for i := 0; i < l; i++ {
			iv := rv.Index(i).Interface()
			kv := fmt.Sprintf("%s", iv)
pdebug.Printf("unique? -> %s", kv)
			if _, ok := uitems[kv]; ok {
				return errors.New("duplicate element found")
			}
			uitems[kv] = struct{}{}
		}
	}

	if celem := c.items; celem != nil {
		if pdebug.Enabled {
			pdebug.Printf("Checking if all items match a spec")
		}
		// if this is set, then all items must fulfill this.
		// additional items are ignored
		for i := 0; i < l; i++ {
			iv := rv.Index(i).Interface()
			if err := celem.Validate(iv); err != nil {
				return err
			}
		}
	} else {
		// otherwise, check the positional specs, and apply the
		// additionalItems constraint
		lv := rv.Len()
		for i, cpos := range c.positionalItems {
			if lv <= i {
				break
			}
			if pdebug.Enabled {
				pdebug.Printf("Checking positional item at '%d'", i)
			}
			iv := rv.Index(i).Interface()
			if err := cpos.Validate(iv); err != nil {
				return err
			}
		}

		lp := len(c.positionalItems)
		if lp > 0 && l > lp { // we got more than positional schemas
			cadd := c.additionalItems
			if cadd == nil { // you can't have additionalItems!
				return errors.New("additional elements found in array")
			}
			for i := lp - 1; i < l; i++ {
				iv := rv.Index(i).Interface()
				if err := cadd.Validate(iv); err != nil {
					return err
				}
			}
		}
	}
	return nil
}

// AdditionalItems specifies the constraint that additional items
// must be validated against. Note that if you specify `Items`
// with this constraint, `AdditionalItems` has no effect. This
// constraint only makes sense when used along side with the
// `PositionalItems` constraint.
//
// If you want to allow any additional items, pass `EmptyConstraint`
// to this method. If unspecified, no extra items are allowed.
func (c *ArrayConstraint) AdditionalItems(ac Constraint) *ArrayConstraint {
	c.additionalItems = ac
	return c
}

// Items specifies the constraint that all items in the array
// must be validated against
func (c *ArrayConstraint) Items(ac Constraint) *ArrayConstraint {
	c.items = ac
	return c
}

// MinItems specifies the minimum number of items in the value.
// If unspecified, the check is not performed.
func (c *ArrayConstraint) MinItems(i int) *ArrayConstraint {
	c.minItems = i
	return c
}

// MaxItems specifies the maximum number of items in the value.
// If unspecified, the check is not performed.
func (c *ArrayConstraint) MaxItems(i int) *ArrayConstraint {
	c.maxItems = i
	return c
}

// PositionalItems specifies the constraint that elements in
// each position of the array being validated. Extra items in
// the array are validated against the constraint specified for
// `AdditionalItems`.
func (c *ArrayConstraint) PositionalItems(ac []Constraint) *ArrayConstraint {
	c.positionalItems = ac
	return c
}

// UniqueItems specifies if the array can hold non-unique items.
// When set to true, the validation will fail unless all of your
// elements are unique.
//
// Caveat: Currently we stringify the elements before comparing
// for uniqueness. This could/should be fixed.
func (c *ArrayConstraint) UniqueItems(b bool) *ArrayConstraint {
	if pdebug.Enabled {
		pdebug.Printf("Setting uniqueItems = %t", b)
	}
	c.uniqueItems = b
	return c
}<|endoftext|>


<commit_before>package generator

import (
	"encoding/xml"

	"socialapi/config"
	"socialapi/workers/helper"
	"socialapi/workers/sitemap/common"
	"socialapi/workers/sitemap/models"

	"github.com/jinzhu/gorm"
	"github.com/koding/logging"
	"github.com/koding/redis"
	"github.com/robfig/cron"
)

type Controller struct {
	log          logging.Logger
	fileSelector FileSelector
	fileName     string
	redisConn    *redis.RedisSession
}

const (
	// before sending this interval, beware that you have to change
	// TIMERANGE in cache key file
	SCHEDULE = "0 0-59/30 * * * *"
)

var (
	cronJob *cron.Cron
)

func New(log logging.Logger) (*Controller, error) {
	conf := *config.Get()
	conf.Redis.DB = conf.Sitemap.RedisDB

	redisConn := helper.MustInitRedisConn(&conf)
	c := &Controller{
		log:          log,
		fileSelector: CachedFileSelector{},
		redisConn:    redisConn,
	}

	return c, c.initCron()
}

func (c *Controller) initCron() error {
	cronJob = cron.New()
	if err := cronJob.AddFunc(SCHEDULE, c.generate); err != nil {
		return err
	}
	cronJob.Start()

	return nil
}

func (c *Controller) Shutdown() {
	cronJob.Stop()
}

func (c *Controller) generate() {
	c.log.Info("Sitemap update started")
	for {
		name, err := c.fileSelector.Select()
		if err == redis.ErrNil {
			return
		}

		if err != nil {
			c.log.Error("Could not fetch file name: %s", err)
			return
		}
		c.log.Info("Updating sitemap: %s", name)
		// there is not any waiting sitemap updates
		if name == "" {
			return
		}

		c.fileName = name

		els, err := c.fetchElements()
		if err != nil {
			c.log.Error("Could not fetch updated elements: %s", err)
			continue
		}

		if len(els) == 0 {
			c.log.Info("Items are already added")
			continue
		}

		container := c.buildContainer(els)

		if err := c.updateFile(container); err != nil {
			c.log.Critical("Could not update file: %s", err)
			continue
		}

	}
}

func (c *Controller) fetchElements() ([]*models.SitemapItem, error) {
	key := common.PrepareCurrentFileCacheKey(c.fileName)
	els := make([]*models.SitemapItem, 0)

	for {
		item, err := c.redisConn.PopSetMember(key)
		if err != nil && err != redis.ErrNil {
			return els, err
		}

		if item == "" {
			return els, nil
		}

		i := &models.SitemapItem{}

		if err := i.Populate(item); err != nil {
			c.log.Error("Could not update item %s: %s", item, err)
			continue
		}

		els = append(els, i)
	}
}

func (c *Controller) updateFile(container *models.ItemContainer) error {
	sf := new(models.SitemapFile)
	newItem := false
	err := sf.ByName(c.fileName)
	if err == gorm.RecordNotFound {
		newItem = true
	}

	if err != nil && !newItem {
		return err
	}

	s := models.NewItemSet()
	if !newItem && len(sf.Blob) > 0 {
		if err := xml.Unmarshal(sf.Blob, &s); err != nil {
			return err
		}
	}

	s.Populate(container)
	v, err := xml.Marshal(s)
	if err != nil {
		return err
	}
	sf.Blob = v

	if newItem {
		return sf.Create()
	}

	return sf.Update()
}

func (c *Controller) buildContainer(items []*models.SitemapItem) *models.ItemContainer {
	container := models.NewItemContainer()
	for _, v := range items {
		item := v.Definition(config.Get().Uri)
		switch v.Status {
		case models.STATUS_ADD:
			container.Add = append(container.Add, item)
		case models.STATUS_DELETE:
			container.Delete = append(container.Delete, item)
		case models.STATUS_UPDATE:
			container.Update = append(container.Update, item)
		}
	}

	return container
}
<commit_msg>Sitemap: Set file name before creating<commit_after>package generator

import (
	"encoding/xml"

	"socialapi/config"
	"socialapi/workers/helper"
	"socialapi/workers/sitemap/common"
	"socialapi/workers/sitemap/models"

	"github.com/jinzhu/gorm"
	"github.com/koding/logging"
	"github.com/koding/redis"
	"github.com/robfig/cron"
)

type Controller struct {
	log          logging.Logger
	fileSelector FileSelector
	fileName     string
	redisConn    *redis.RedisSession
}

const (
	// before sending this interval, beware that you have to change
	// TIMERANGE in cache key file
	SCHEDULE = "0 0-59/30 * * * *"
)

var (
	cronJob *cron.Cron
)

func New(log logging.Logger) (*Controller, error) {
	conf := *config.Get()
	conf.Redis.DB = conf.Sitemap.RedisDB

	redisConn := helper.MustInitRedisConn(&conf)
	c := &Controller{
		log:          log,
		fileSelector: CachedFileSelector{},
		redisConn:    redisConn,
	}

	return c, c.initCron()
}

func (c *Controller) initCron() error {
	cronJob = cron.New()
	if err := cronJob.AddFunc(SCHEDULE, c.generate); err != nil {
		return err
	}
	cronJob.Start()

	return nil
}

func (c *Controller) Shutdown() {
	cronJob.Stop()
}

func (c *Controller) generate() {
	c.log.Info("Sitemap update started")
	for {
		name, err := c.fileSelector.Select()
		if err == redis.ErrNil {
			return
		}

		if err != nil {
			c.log.Error("Could not fetch file name: %s", err)
			return
		}
		c.log.Info("Updating sitemap: %s", name)
		// there is not any waiting sitemap updates
		if name == "" {
			return
		}

		c.fileName = name

		els, err := c.fetchElements()
		if err != nil {
			c.log.Error("Could not fetch updated elements: %s", err)
			continue
		}

		if len(els) == 0 {
			c.log.Info("Items are already added")
			continue
		}

		container := c.buildContainer(els)

		if err := c.updateFile(container); err != nil {
			c.log.Critical("Could not update file: %s", err)
			continue
		}

	}
}

func (c *Controller) fetchElements() ([]*models.SitemapItem, error) {
	key := common.PrepareCurrentFileCacheKey(c.fileName)
	els := make([]*models.SitemapItem, 0)

	for {
		item, err := c.redisConn.PopSetMember(key)
		if err != nil && err != redis.ErrNil {
			return els, err
		}

		if item == "" {
			return els, nil
		}

		i := &models.SitemapItem{}

		if err := i.Populate(item); err != nil {
			c.log.Error("Could not update item %s: %s", item, err)
			continue
		}

		els = append(els, i)
	}
}

func (c *Controller) updateFile(container *models.ItemContainer) error {
	sf := new(models.SitemapFile)
	newItem := false
	err := sf.ByName(c.fileName)
	if err == gorm.RecordNotFound {
		newItem = true
	}

	if err != nil && !newItem {
		return err
	}

	s := models.NewItemSet()
	if !newItem && len(sf.Blob) > 0 {
		if err := xml.Unmarshal(sf.Blob, &s); err != nil {
			return err
		}
	}

	s.Populate(container)
	v, err := xml.Marshal(s)
	if err != nil {
		return err
	}
	sf.Blob = v

	if newItem {
		sf.Name = c.fileName
		return sf.Create()
	}

	return sf.Update()
}

func (c *Controller) buildContainer(items []*models.SitemapItem) *models.ItemContainer {
	container := models.NewItemContainer()
	for _, v := range items {
		item := v.Definition(config.Get().Uri)
		switch v.Status {
		case models.STATUS_ADD:
			container.Add = append(container.Add, item)
		case models.STATUS_DELETE:
			container.Delete = append(container.Delete, item)
		case models.STATUS_UPDATE:
			container.Update = append(container.Update, item)
		}
	}

	return container
}
<|endoftext|>


<commit_before>/*
Copyright 2015 Gravitational, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package httplib implements common utility functions for writing
// classic HTTP handlers
package httplib

import (
	"net/http"
	"strings"
)

// SetNoCacheHeaders tells proxies and browsers do not cache the content
func SetNoCacheHeaders(h http.Header) {
	h.Set("Cache-Control", "no-cache, no-store, must-revalidate")
	h.Set("Pragma", "no-cache")
	h.Set("Expires", "0")
}

// SetStaticFileHeaders sets security header flags for static non-html resources
func SetStaticFileHeaders(h http.Header) {
	SetSameOriginIFrame(h)
	SetNoSniff(h)
}

// SetIndexHTMLHeaders sets security header flags for main index.html page
func SetIndexHTMLHeaders(h http.Header) {
	SetNoCacheHeaders(h)
	SetSameOriginIFrame(h)
	SetNoSniff(h)

	// Only send the origin of the document as the referrer in all cases.
	// The document https://example.com/page.html will send the referrer https://example.com/.
	h.Set("Referrer-Policy", "origin")

	// X-Frame-Options indicates that the page can only be displayed in iframe on the same origin as the page itself
	h.Set("X-Frame-Options", "SAMEORIGIN")

	// X-XSS-Protection is a feature of Internet Explorer, Chrome and Safari that stops pages
	// from loading when they detect reflected cross-site scripting (XSS) attacks.
	h.Set("X-XSS-Protection", "1; mode=block")

	// Once a supported browser receives this header that browser will prevent any communications from
	// being sent over HTTP to the specified domain and will instead send all communications over HTTPS.
	// It also prevents HTTPS click through prompts on browsers
	h.Set("Strict-Transport-Security", "max-age=31536000; includeSubDomains")

	// Prevent web browsers from using content sniffing to discover a file’s MIME type
	h.Set("X-Content-Type-Options", "nosniff")

	// Set content policy flags
	var cspValue = strings.Join([]string{
		"default-src 'self'",
		// cloud version uses stripe.com to update billing information
		"script-src 'self' https://js.stripe.com",
		"frame-src https://js.stripe.com",
		"frame-ancestors 'none'",
		// 'unsafe-inline' is required by CSS-in-JS to work
		"style-src 'self' 'unsafe-inline'",
		"object-src 'none'",
		"img-src 'self' data: blob:",
		"font-src 'self' data:",
		"base-uri 'self'",
		"form-action 'self'",
	}, ";")

	h.Set("Content-Security-Policy", cspValue)
}

// SetSameOriginIFrame sets X-Frame-Options flag
func SetSameOriginIFrame(h http.Header) {
	// X-Frame-Options indicates that the page can only be displayed in iframe on the same origin as the page itself
	h.Set("X-Frame-Options", "SAMEORIGIN")
}

// SetNoSniff sets X-Content-Type-Options flag
func SetNoSniff(h http.Header) {
	// Prevent web browsers from using content sniffing to discover a file’s MIME type
	h.Set("X-Content-Type-Options", "nosniff")
}

// SetWebConfigHeaders sets headers for webConfig.js
func SetWebConfigHeaders(h http.Header) {
	SetStaticFileHeaders(h)
	h.Set("Content-Type", "application/javascript")
}
<commit_msg>allow  websocket connections to the same host (csp) (#7929)<commit_after>/*
Copyright 2015 Gravitational, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package httplib implements common utility functions for writing
// classic HTTP handlers
package httplib

import (
	"net/http"
	"strings"
)

// SetNoCacheHeaders tells proxies and browsers do not cache the content
func SetNoCacheHeaders(h http.Header) {
	h.Set("Cache-Control", "no-cache, no-store, must-revalidate")
	h.Set("Pragma", "no-cache")
	h.Set("Expires", "0")
}

// SetStaticFileHeaders sets security header flags for static non-html resources
func SetStaticFileHeaders(h http.Header) {
	SetSameOriginIFrame(h)
	SetNoSniff(h)
}

// SetIndexHTMLHeaders sets security header flags for main index.html page
func SetIndexHTMLHeaders(h http.Header) {
	SetNoCacheHeaders(h)
	SetSameOriginIFrame(h)
	SetNoSniff(h)

	// Only send the origin of the document as the referrer in all cases.
	// The document https://example.com/page.html will send the referrer https://example.com/.
	h.Set("Referrer-Policy", "origin")

	// X-Frame-Options indicates that the page can only be displayed in iframe on the same origin as the page itself
	h.Set("X-Frame-Options", "SAMEORIGIN")

	// X-XSS-Protection is a feature of Internet Explorer, Chrome and Safari that stops pages
	// from loading when they detect reflected cross-site scripting (XSS) attacks.
	h.Set("X-XSS-Protection", "1; mode=block")

	// Once a supported browser receives this header that browser will prevent any communications from
	// being sent over HTTP to the specified domain and will instead send all communications over HTTPS.
	// It also prevents HTTPS click through prompts on browsers
	h.Set("Strict-Transport-Security", "max-age=31536000; includeSubDomains")

	// Prevent web browsers from using content sniffing to discover a file’s MIME type
	h.Set("X-Content-Type-Options", "nosniff")

	// Set content policy flags
	var cspValue = strings.Join([]string{
		"default-src 'self'",
		// cloud version uses stripe.com to update billing information
		"script-src 'self' https://js.stripe.com",
		"frame-src https://js.stripe.com",
		"frame-ancestors 'none'",
		// 'unsafe-inline' is required by CSS-in-JS to work
		"style-src 'self' 'unsafe-inline'",
		"object-src 'none'",
		"img-src 'self' data: blob:",
		"font-src 'self' data:",
		"base-uri 'self'",
		"form-action 'self'",
		"connect-src 'self' wss:",
	}, ";")

	h.Set("Content-Security-Policy", cspValue)
}

// SetSameOriginIFrame sets X-Frame-Options flag
func SetSameOriginIFrame(h http.Header) {
	// X-Frame-Options indicates that the page can only be displayed in iframe on the same origin as the page itself
	h.Set("X-Frame-Options", "SAMEORIGIN")
}

// SetNoSniff sets X-Content-Type-Options flag
func SetNoSniff(h http.Header) {
	// Prevent web browsers from using content sniffing to discover a file’s MIME type
	h.Set("X-Content-Type-Options", "nosniff")
}

// SetWebConfigHeaders sets headers for webConfig.js
func SetWebConfigHeaders(h http.Header) {
	SetStaticFileHeaders(h)
	h.Set("Content-Type", "application/javascript")
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"os"
	"time"

	log "github.com/golang/glog"
	flag "github.com/spf13/pflag"

	"k8s.io/contrib/addon-resizer/nanny"
	resource "k8s.io/kubernetes/pkg/api/resource"

	client "k8s.io/kubernetes/pkg/client/clientset_generated/release_1_3"
	"k8s.io/kubernetes/pkg/client/restclient"
)

const noValue = "MISSING"

var (
	// Flags to define the resource requirements.
	baseCPU        = flag.String("cpu", noValue, "The base CPU resource requirement.")
	cpuPerNode     = flag.String("extra-cpu", "0", "The amount of CPU to add per node.")
	baseMemory     = flag.String("memory", noValue, "The base memory resource requirement.")
	memoryPerNode  = flag.String("extra-memory", "0Mi", "The amount of memory to add per node.")
	baseStorage    = flag.String("storage", noValue, "The base storage resource requirement.")
	storagePerNode = flag.String("extra-storage", "0Gi", "The amount of storage to add per node.")
	threshold      = flag.Int("threshold", 0, "A number between 0-100. The dependent's resources are rewritten when they deviate from expected by more than threshold.")
	// Flags to identify the container to nanny.
	podNamespace  = flag.String("namespace", os.Getenv("MY_POD_NAMESPACE"), "The namespace of the ward. This defaults to the nanny pod's own namespace.")
	deployment    = flag.String("deployment", "", "The name of the deployment being monitored. This is required.")
	podName       = flag.String("pod", os.Getenv("MY_POD_NAME"), "The name of the pod to watch. This defaults to the nanny's own pod.")
	containerName = flag.String("container", "pod-nanny", "The name of the container to watch. This defaults to the nanny itself.")
	// Flags to control runtime behavior.
	pollPeriod = time.Millisecond * time.Duration(*flag.Int("poll-period", 10000, "The time, in milliseconds, to poll the dependent container."))
	estimator  = flag.String("estimator", "linear", "The estimator to use. Currently supported: linear, exponential")
)

func main() {
	// First log our starting config, and then set up.
	log.Infof("Invoked by %v", os.Args)
	flag.Parse()

	// Perform further validation of flags.
	if *deployment == "" {
		log.Fatal("Must specify a deployment.")
	}

	if *threshold < 0 || *threshold > 100 {
		log.Fatalf("Threshold must be between 0 and 100 inclusively, was %d.", threshold)
	}

	log.Infof("Watching namespace: %s, pod: %s, container: %s.", *podNamespace, *podName, *containerName)
	log.Infof("cpu: %s, extra_cpu: %s, memory: %s, extra_memory: %s, storage: %s, extra_storage: %s", *baseCPU, *cpuPerNode, *baseMemory, *memoryPerNode, *baseStorage, *storagePerNode)

	// Set up work objects.
	config, err := restclient.InClusterConfig()
	if err != nil {
		log.Fatal(err)
	}

	clientset, err := client.NewForConfig(config)
	if err != nil {
		log.Fatal(err)
	}
	k8s := nanny.NewKubernetesClient(*podNamespace, *deployment, *podName, *containerName, clientset)

	var resources []nanny.Resource

	// Monitor only the resources specified.
	if *baseCPU != noValue {
		resources = append(resources, nanny.Resource{
			Base:         resource.MustParse(*baseCPU),
			ExtraPerNode: resource.MustParse(*cpuPerNode),
			Name:         "cpu",
		})
	}

	if *baseMemory != noValue {
		resources = append(resources, nanny.Resource{
			Base:         resource.MustParse(*baseMemory),
			ExtraPerNode: resource.MustParse(*memoryPerNode),
			Name:         "memory",
		})
	}

	if *baseStorage != noValue {
		resources = append(resources, nanny.Resource{
			Base:         resource.MustParse(*baseStorage),
			ExtraPerNode: resource.MustParse(*memoryPerNode),
			Name:         "storage",
		})
	}

	log.Infof("Resources: %+v", resources)

	var est nanny.ResourceEstimator
	if *estimator == "linear" {
		est = nanny.LinearEstimator{
			Resources: resources,
		}
	} else if *estimator == "exponential" {
		est = nanny.ExponentialEstimator{
			Resources:   resources,
			ScaleFactor: 1.5,
		}
	} else {
		log.Fatalf("Estimator %s not supported", *estimator)
	}

	// Begin nannying.
	nanny.PollAPIServer(k8s, est, *containerName, pollPeriod, uint64(*threshold))
}
<commit_msg>Fix nanny to actually use poll-period flag.<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"os"
	"time"

	log "github.com/golang/glog"
	flag "github.com/spf13/pflag"

	"k8s.io/contrib/addon-resizer/nanny"
	resource "k8s.io/kubernetes/pkg/api/resource"

	client "k8s.io/kubernetes/pkg/client/clientset_generated/release_1_3"
	"k8s.io/kubernetes/pkg/client/restclient"
)

const noValue = "MISSING"

var (
	// Flags to define the resource requirements.
	baseCPU        = flag.String("cpu", noValue, "The base CPU resource requirement.")
	cpuPerNode     = flag.String("extra-cpu", "0", "The amount of CPU to add per node.")
	baseMemory     = flag.String("memory", noValue, "The base memory resource requirement.")
	memoryPerNode  = flag.String("extra-memory", "0Mi", "The amount of memory to add per node.")
	baseStorage    = flag.String("storage", noValue, "The base storage resource requirement.")
	storagePerNode = flag.String("extra-storage", "0Gi", "The amount of storage to add per node.")
	threshold      = flag.Int("threshold", 0, "A number between 0-100. The dependent's resources are rewritten when they deviate from expected by more than threshold.")
	// Flags to identify the container to nanny.
	podNamespace  = flag.String("namespace", os.Getenv("MY_POD_NAMESPACE"), "The namespace of the ward. This defaults to the nanny pod's own namespace.")
	deployment    = flag.String("deployment", "", "The name of the deployment being monitored. This is required.")
	podName       = flag.String("pod", os.Getenv("MY_POD_NAME"), "The name of the pod to watch. This defaults to the nanny's own pod.")
	containerName = flag.String("container", "pod-nanny", "The name of the container to watch. This defaults to the nanny itself.")
	// Flags to control runtime behavior.
	pollPeriodMillis = flag.Int("poll-period", 10000, "The time, in milliseconds, to poll the dependent container.")
	estimator  = flag.String("estimator", "linear", "The estimator to use. Currently supported: linear, exponential")
)

func main() {
	// First log our starting config, and then set up.
	log.Infof("Invoked by %v", os.Args)
	flag.Parse()

	// Perform further validation of flags.
	if *deployment == "" {
		log.Fatal("Must specify a deployment.")
	}

	if *threshold < 0 || *threshold > 100 {
		log.Fatalf("Threshold must be between 0 and 100 inclusively, was %d.", threshold)
	}

	pollPeriod := time.Millisecond * time.Duration(*pollPeriodMillis)
	log.Infof("Poll period: %+v", pollPeriod)
	log.Infof("Watching namespace: %s, pod: %s, container: %s.", *podNamespace, *podName, *containerName)
	log.Infof("cpu: %s, extra_cpu: %s, memory: %s, extra_memory: %s, storage: %s, extra_storage: %s", *baseCPU, *cpuPerNode, *baseMemory, *memoryPerNode, *baseStorage, *storagePerNode)

	// Set up work objects.
	config, err := restclient.InClusterConfig()
	if err != nil {
		log.Fatal(err)
	}

	clientset, err := client.NewForConfig(config)
	if err != nil {
		log.Fatal(err)
	}
	k8s := nanny.NewKubernetesClient(*podNamespace, *deployment, *podName, *containerName, clientset)

	var resources []nanny.Resource

	// Monitor only the resources specified.
	if *baseCPU != noValue {
		resources = append(resources, nanny.Resource{
			Base:         resource.MustParse(*baseCPU),
			ExtraPerNode: resource.MustParse(*cpuPerNode),
			Name:         "cpu",
		})
	}

	if *baseMemory != noValue {
		resources = append(resources, nanny.Resource{
			Base:         resource.MustParse(*baseMemory),
			ExtraPerNode: resource.MustParse(*memoryPerNode),
			Name:         "memory",
		})
	}

	if *baseStorage != noValue {
		resources = append(resources, nanny.Resource{
			Base:         resource.MustParse(*baseStorage),
			ExtraPerNode: resource.MustParse(*memoryPerNode),
			Name:         "storage",
		})
	}

	log.Infof("Resources: %+v", resources)

	var est nanny.ResourceEstimator
	if *estimator == "linear" {
		est = nanny.LinearEstimator{
			Resources: resources,
		}
	} else if *estimator == "exponential" {
		est = nanny.ExponentialEstimator{
			Resources:   resources,
			ScaleFactor: 1.5,
		}
	} else {
		log.Fatalf("Estimator %s not supported", *estimator)
	}

	// Begin nannying.
	nanny.PollAPIServer(k8s, est, *containerName, pollPeriod, uint64(*threshold))
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"time"
)

var (
	noTime     time.Time
	timeFormat = time.RFC3339
)

var usage = `tmux-pomodoro
github.com/justincampbell/tmux-pomodoro

  pomodoro start   Start a timer for 25 minutes
  pomodoro status  Show the remaining time, or an exclamation point if done
  pomodoro clear   Clear the timer
`

type State struct {
	endTime time.Time
	now     time.Time
}

type Output struct {
	text       string
	returnCode int
}

func init() {
	flag.Parse()
}

func main() {
	state := State{
		endTime: readExistingTime(),
		now:     time.Now(),
	}

	args := flag.Args()
	var command string
	if len(args) == 0 {
		command = ""
	} else {
		command = args[0]
	}

	newState, output := parseCommand(state, command)

	if newState.endTime != state.endTime {
		writeTime(newState.endTime)
	}

	fmt.Println(output.text)

	if output.returnCode != 0 {
		os.Exit(output.returnCode)
	}
}

func refreshTmux() {
	exec.Command("tmux", "refresh-client", "-S").Start()
}

func parseCommand(state State, command string) (newState State, output Output) {
	newState = state

	switch command {
	case "start":
		duration, _ := time.ParseDuration("25m")
		newState.endTime = state.now.Add(duration)
		output.text = "Timer started, 25 minutes remaining"
		refreshTmux()
	case "status":
		if state.endTime == noTime {
			return
		}
		output.text = formatRemainingTime(state.endTime, state.now) + " 🍅 "
	case "clear":
		newState.endTime = noTime
		output.text = "Pomodoro cleared!"
		refreshTmux()
	case "":
		output.text = usage
	default:
		output.text = usage
		output.returnCode = 1
	}

	return
}

func formatRemainingTime(existingTime time.Time, now time.Time) string {
	remaining := existingTime.Sub(now)
	remainingMinutes := remaining.Minutes()

	if remainingMinutes >= 0 {
		return strconv.FormatFloat(remainingMinutes, 'f', 0, 64)
	} else {
		return "❗"
	}
}

func writeTime(t time.Time) {
	var bytes []byte
	if t != noTime {
		bytes = []byte(t.Format(timeFormat))
	}
	err := ioutil.WriteFile(filePath(), bytes, 0644)
	if err != nil {
		panic(err)
	}
}

func readExistingTime() time.Time {
	bytes, err := ioutil.ReadFile(filePath())
	if err != nil {
		return noTime
	}

	contents := string(bytes[:])
	contents = strings.TrimSpace(contents)

	result, err := time.Parse(timeFormat, contents)
	if err != nil {
		return noTime
	}

	return result
}

func filePath() string {
	return homeDir() + "/.pomodoro"
}

func homeDir() string {
	return os.Getenv("HOME")
}
<commit_msg>Explicitly ignore err from tmux refresh-client<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"time"
)

var (
	noTime     time.Time
	timeFormat = time.RFC3339
)

var usage = `tmux-pomodoro
github.com/justincampbell/tmux-pomodoro

  pomodoro start   Start a timer for 25 minutes
  pomodoro status  Show the remaining time, or an exclamation point if done
  pomodoro clear   Clear the timer
`

type State struct {
	endTime time.Time
	now     time.Time
}

type Output struct {
	text       string
	returnCode int
}

func init() {
	flag.Parse()
}

func main() {
	state := State{
		endTime: readExistingTime(),
		now:     time.Now(),
	}

	args := flag.Args()
	var command string
	if len(args) == 0 {
		command = ""
	} else {
		command = args[0]
	}

	newState, output := parseCommand(state, command)

	if newState.endTime != state.endTime {
		writeTime(newState.endTime)
	}

	fmt.Println(output.text)

	if output.returnCode != 0 {
		os.Exit(output.returnCode)
	}
}

func refreshTmux() {
	_ = exec.Command("tmux", "refresh-client", "-S").Start()
}

func parseCommand(state State, command string) (newState State, output Output) {
	newState = state

	switch command {
	case "start":
		duration, _ := time.ParseDuration("25m")
		newState.endTime = state.now.Add(duration)
		output.text = "Timer started, 25 minutes remaining"
		refreshTmux()
	case "status":
		if state.endTime == noTime {
			return
		}
		output.text = formatRemainingTime(state.endTime, state.now) + " 🍅 "
	case "clear":
		newState.endTime = noTime
		output.text = "Pomodoro cleared!"
		refreshTmux()
	case "":
		output.text = usage
	default:
		output.text = usage
		output.returnCode = 1
	}

	return
}

func formatRemainingTime(existingTime time.Time, now time.Time) string {
	remaining := existingTime.Sub(now)
	remainingMinutes := remaining.Minutes()

	if remainingMinutes >= 0 {
		return strconv.FormatFloat(remainingMinutes, 'f', 0, 64)
	} else {
		return "❗"
	}
}

func writeTime(t time.Time) {
	var bytes []byte
	if t != noTime {
		bytes = []byte(t.Format(timeFormat))
	}
	err := ioutil.WriteFile(filePath(), bytes, 0644)
	if err != nil {
		panic(err)
	}
}

func readExistingTime() time.Time {
	bytes, err := ioutil.ReadFile(filePath())
	if err != nil {
		return noTime
	}

	contents := string(bytes[:])
	contents = strings.TrimSpace(contents)

	result, err := time.Parse(timeFormat, contents)
	if err != nil {
		return noTime
	}

	return result
}

func filePath() string {
	return homeDir() + "/.pomodoro"
}

func homeDir() string {
	return os.Getenv("HOME")
}
<|endoftext|>


<commit_before>//  Copyright (c) 2012 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package rest

import (
	"crypto/sha1"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
	"runtime"
	"runtime/pprof"
	"strconv"

	"github.com/couchbaselabs/sync_gateway/base"
	"github.com/couchbaselabs/sync_gateway/db"
)

const VersionString = "Couchbase Sync Gateway/0.91"

// HTTP handler for the root ("/")
func (h *handler) handleRoot() error {
	response := map[string]interface{}{
		"couchdb": "welcome",
		"version": VersionString,
	}
	if h.privs == adminPrivs {
		response["ADMIN"] = true
	}
	h.writeJSON(response)
	return nil
}

func (h *handler) handleAllDbs() error {
	h.writeJSON(h.server.AllDatabaseNames())
	return nil
}

func (h *handler) handleCompact() error {
	revsDeleted, err := h.db.Compact()
	if err != nil {
		return err
	}
	h.writeJSON(db.Body{"revs": revsDeleted})
	return nil
}

func (h *handler) handleVacuum() error {
	attsDeleted, err := db.VacuumAttachments(h.db.Bucket)
	if err != nil {
		return err
	}
	h.writeJSON(db.Body{"atts": attsDeleted})
	return nil
}

func (h *handler) instanceStartTime() json.Number {
	return json.Number(strconv.FormatInt(h.db.StartTime.UnixNano()/1000, 10))
}

func (h *handler) handleGetDB() error {
	if h.rq.Method == "HEAD" {
		return nil
	}
	lastSeq, err := h.db.LastSequence()
	if err != nil {
		return err
	}
	response := db.Body{
		"db_name":              h.db.Name,
		"doc_count":            h.db.DocCount(),
		"update_seq":           lastSeq,
		"committed_update_seq": lastSeq,
		"instance_start_time":  h.instanceStartTime(),
		"compact_running":      false, // TODO: Implement this
		"purge_seq":            0,     // TODO: Should track this value
		"disk_format_version":  0,     // Probably meaningless, but add for compatibility
	}
	h.writeJSON(response)
	return nil
}

func (h *handler) handleEFC() error { // Handles _ensure_full_commit.
	// no-op. CouchDB's replicator sends this, so don't barf. Status must be 201.
	h.writeJSONStatus(http.StatusCreated, db.Body{
		"ok": true,
		"instance_start_time": h.instanceStartTime(),
	})
	return nil
}

func (h *handler) handleDesign() error {
	// we serve this content here so that CouchDB 1.2 has something to
	// hash into the replication-id, to correspond to our filter.
	filter := "ok"
	if h.db.DatabaseContext.ChannelMapper != nil {
		hash := sha1.New()
		io.WriteString(hash, h.db.DatabaseContext.ChannelMapper.Function())
		filter = fmt.Sprint(hash.Sum(nil))
	}
	h.writeJSON(db.Body{"filters": db.Body{"bychannel": filter}})
	return nil
}

// ADMIN API to turn Go CPU profiling on/off
func (h *handler) handleProfiling() error {
	profileName := h.PathVar("name")
	var params struct {
		File string `json:"file"`
	}
	body, err := h.readBody()
	if err != nil {
		return err
	}
	if len(body) > 0 {
		if err = json.Unmarshal(body, &params); err != nil {
			return err
		}
	}

	if params.File != "" {
		f, err := os.Create(params.File)
		if err != nil {
			return err
		}
		if profileName != "" {
			defer f.Close()
			if profile := pprof.Lookup(profileName); profile != nil {
				profile.WriteTo(f, 0)
				base.Log("Wrote %s profile to %s", profileName, params.File)
			} else {
				return base.HTTPErrorf(http.StatusNotFound, "No such profile %q", profileName)
			}
		} else {
			base.Log("Starting CPU profile to %s ...", params.File)
			pprof.StartCPUProfile(f)
		}
	} else {
		if profileName != "" {
			return base.HTTPErrorf(http.StatusBadRequest, "Missing JSON 'file' parameter")
		} else {
			base.Log("...ending CPU profile.")
			pprof.StopCPUProfile()
		}
	}
	return nil
}

// ADMIN API to dump Go heap profile
func (h *handler) handleHeapProfiling() error {
	var params struct {
		File string `json:"file"`
	}
	body, err := h.readBody()
	if err != nil {
		return err
	}
	if err = json.Unmarshal(body, &params); err != nil {
		return err
	}

	base.Log("Dumping heap profile to %s ...", params.File)
	f, err := os.Create(params.File)
	if err != nil {
		return err
	}
	pprof.WriteHeapProfile(f)
	f.Close()
	return nil
}

type stats struct {
	MemStats runtime.MemStats
}

// ADMIN API to expose runtime and other stats
func (h *handler) handleStats() error {
	st := stats{}
	runtime.ReadMemStats(&st.MemStats)

	h.writeJSON(st)
	return nil
}
<commit_msg>Bump version string to 0.92<commit_after>//  Copyright (c) 2012 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package rest

import (
	"crypto/sha1"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
	"runtime"
	"runtime/pprof"
	"strconv"

	"github.com/couchbaselabs/sync_gateway/base"
	"github.com/couchbaselabs/sync_gateway/db"
)

const VersionString = "Couchbase Sync Gateway/0.92"

// HTTP handler for the root ("/")
func (h *handler) handleRoot() error {
	response := map[string]interface{}{
		"couchdb": "welcome",
		"version": VersionString,
	}
	if h.privs == adminPrivs {
		response["ADMIN"] = true
	}
	h.writeJSON(response)
	return nil
}

func (h *handler) handleAllDbs() error {
	h.writeJSON(h.server.AllDatabaseNames())
	return nil
}

func (h *handler) handleCompact() error {
	revsDeleted, err := h.db.Compact()
	if err != nil {
		return err
	}
	h.writeJSON(db.Body{"revs": revsDeleted})
	return nil
}

func (h *handler) handleVacuum() error {
	attsDeleted, err := db.VacuumAttachments(h.db.Bucket)
	if err != nil {
		return err
	}
	h.writeJSON(db.Body{"atts": attsDeleted})
	return nil
}

func (h *handler) instanceStartTime() json.Number {
	return json.Number(strconv.FormatInt(h.db.StartTime.UnixNano()/1000, 10))
}

func (h *handler) handleGetDB() error {
	if h.rq.Method == "HEAD" {
		return nil
	}
	lastSeq, err := h.db.LastSequence()
	if err != nil {
		return err
	}
	response := db.Body{
		"db_name":              h.db.Name,
		"doc_count":            h.db.DocCount(),
		"update_seq":           lastSeq,
		"committed_update_seq": lastSeq,
		"instance_start_time":  h.instanceStartTime(),
		"compact_running":      false, // TODO: Implement this
		"purge_seq":            0,     // TODO: Should track this value
		"disk_format_version":  0,     // Probably meaningless, but add for compatibility
	}
	h.writeJSON(response)
	return nil
}

func (h *handler) handleEFC() error { // Handles _ensure_full_commit.
	// no-op. CouchDB's replicator sends this, so don't barf. Status must be 201.
	h.writeJSONStatus(http.StatusCreated, db.Body{
		"ok": true,
		"instance_start_time": h.instanceStartTime(),
	})
	return nil
}

func (h *handler) handleDesign() error {
	// we serve this content here so that CouchDB 1.2 has something to
	// hash into the replication-id, to correspond to our filter.
	filter := "ok"
	if h.db.DatabaseContext.ChannelMapper != nil {
		hash := sha1.New()
		io.WriteString(hash, h.db.DatabaseContext.ChannelMapper.Function())
		filter = fmt.Sprint(hash.Sum(nil))
	}
	h.writeJSON(db.Body{"filters": db.Body{"bychannel": filter}})
	return nil
}

// ADMIN API to turn Go CPU profiling on/off
func (h *handler) handleProfiling() error {
	profileName := h.PathVar("name")
	var params struct {
		File string `json:"file"`
	}
	body, err := h.readBody()
	if err != nil {
		return err
	}
	if len(body) > 0 {
		if err = json.Unmarshal(body, &params); err != nil {
			return err
		}
	}

	if params.File != "" {
		f, err := os.Create(params.File)
		if err != nil {
			return err
		}
		if profileName != "" {
			defer f.Close()
			if profile := pprof.Lookup(profileName); profile != nil {
				profile.WriteTo(f, 0)
				base.Log("Wrote %s profile to %s", profileName, params.File)
			} else {
				return base.HTTPErrorf(http.StatusNotFound, "No such profile %q", profileName)
			}
		} else {
			base.Log("Starting CPU profile to %s ...", params.File)
			pprof.StartCPUProfile(f)
		}
	} else {
		if profileName != "" {
			return base.HTTPErrorf(http.StatusBadRequest, "Missing JSON 'file' parameter")
		} else {
			base.Log("...ending CPU profile.")
			pprof.StopCPUProfile()
		}
	}
	return nil
}

// ADMIN API to dump Go heap profile
func (h *handler) handleHeapProfiling() error {
	var params struct {
		File string `json:"file"`
	}
	body, err := h.readBody()
	if err != nil {
		return err
	}
	if err = json.Unmarshal(body, &params); err != nil {
		return err
	}

	base.Log("Dumping heap profile to %s ...", params.File)
	f, err := os.Create(params.File)
	if err != nil {
		return err
	}
	pprof.WriteHeapProfile(f)
	f.Close()
	return nil
}

type stats struct {
	MemStats runtime.MemStats
}

// ADMIN API to expose runtime and other stats
func (h *handler) handleStats() error {
	st := stats{}
	runtime.ReadMemStats(&st.MemStats)

	h.writeJSON(st)
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0

package krusty_test

import (
	"testing"

	kusttest_test "sigs.k8s.io/kustomize/api/testutils/kusttest"
)

func TestNullValues(t *testing.T) {
	th := kusttest_test.MakeHarness(t)
	th.WriteF("/app/deployment.yaml", `
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: example
  name: example
spec:
  selector:
    matchLabels:
      app: example
  template:
    metadata:
      labels:
        app: example
    spec:
      containers:
      - args: null
        image: image
        name: example
`)
	th.WriteF("/app/kustomization.yaml", `
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- deployment.yaml
`)
	m := th.Run("/app", th.MakeDefaultOptions())

	th.AssertActualEqualsExpected(m, `
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: example
  name: example
spec:
  selector:
    matchLabels:
      app: example
  template:
    metadata:
      labels:
        app: example
    spec:
      containers:
      - args: null
        image: image
        name: example
`)
}
<commit_msg>Add test representing 2960<commit_after>// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0

package krusty_test

import (
	"strings"
	"testing"

	kusttest_test "sigs.k8s.io/kustomize/api/testutils/kusttest"
)

func TestNullValues1(t *testing.T) {
	th := kusttest_test.MakeHarness(t)
	th.WriteF("/app/deployment.yaml", `
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: example
  name: example
spec:
  selector:
    matchLabels:
      app: example
  template:
    metadata:
      labels:
        app: example
    spec:
      containers:
      - args: null
        image: image
        name: example
`)
	th.WriteF("/app/kustomization.yaml", `
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- deployment.yaml
`)
	m := th.Run("/app", th.MakeDefaultOptions())

	th.AssertActualEqualsExpected(m, `
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: example
  name: example
spec:
  selector:
    matchLabels:
      app: example
  template:
    metadata:
      labels:
        app: example
    spec:
      containers:
      - args: null
        image: image
        name: example
`)
}

func TestNullValues2(t *testing.T) {
	th := kusttest_test.MakeHarness(t)
	th.WriteF("deploy.yaml", `
apiVersion: apps/v1
kind: Deployment
metadata:
  name: test
spec:
  template:
    spec:
      containers:
        - name: test
      volumes: null
`)
	th.WriteK(".", `
resources:
- deploy.yaml
`)
	err := th.RunWithErr(".", th.MakeDefaultOptions())
	if err == nil {
		t.Fatalf("expected trouble")
	}
	if !strings.Contains(
		err.Error(), "expected sequence or mapping node") {
		t.Fatalf("Unexpected err: %v", err)
	}
}
<|endoftext|>


<commit_before>// Package poly1305 provides a Go wrapper around floodyberry's optimized
// implementation of Poly1305.
// Poly1305 is a message authentication code (MAC) designed to meet the standard
// notion of unforgeability after a single message. After the sender
// authenticates one message, an attacker cannot find authenticators for any
// other messages.
// The sender MUST NOT use Poly1305 to authenticate more than one message under
// the same key. Authenticators for two messages under the same key should be
// expected to reveal enough information to allow forgeries of authenticators
// on other messages.
package poly1305

// BUG(codahale): Only supports AMD64.

// +build: amd64

import (
	// #cgo CFLAGS: -O3
	// #include "poly1305-donna.h"
	"C"
	"errors"
	"hash"
)

var (
	// ErrInvalidKey is returned when the provided key is not 256 bits long.
	ErrInvalidKey = errors.New("poly1305: invalid key length")
)

const (
	// KeySize is the length of Poly1305 keys, in bytes.
	KeySize = 32
	// BlockSize is the length of Poly1305 blocks, in bytes.
	BlockSize = 16
	// Size is the length of Poly1305 digests, in bytes.
	Size = 16
)

// A Poly1305 is an instance of Poly1305 using a particular key.
type Poly1305 struct {
	key   []byte
	state C.poly1305_state
}

// New creates and returns a keyed Hash implementation. The key argument must be
// 256 bits long, the value of which must only be used once.
func New(key []byte) (hash.Hash, error) {
	if len(key) != KeySize {
		return nil, ErrInvalidKey
	}

	h := new(Poly1305)
	h.key = make([]byte, KeySize)
	copy(h.key, key)
	h.Reset()

	return h, nil
}

// BlockSize returns the hash's underlying block size.
// The Write method must be able to accept any amount
// of data, but it may operate more efficiently if all writes
// are a multiple of the block size.
func (*Poly1305) BlockSize() int {
	return BlockSize
}

// Size returns the number of bytes Sum will return.
func (*Poly1305) Size() int {
	return Size
}

// Reset resets the Hash to its initial state.
func (s *Poly1305) Reset() {
	C.poly1305_init(&s.state, (*C.uchar)(&s.key[0]))
}

// Write (via the embedded io.Writer interface) adds more data to the running
// hash. It never returns an error.
func (s *Poly1305) Write(buf []byte) (int, error) {
	var p *C.uchar
	if len(buf) > 0 {
		p = (*C.uchar)(&buf[0])
	}

	C.poly1305_update(&s.state, p, (C.size_t)(len(buf)))

	return len(buf), nil
}

// Sum appends the current hash to b and returns the resulting slice.
// It does not change the underlying hash state.
func (s *Poly1305) Sum(buf []byte) []byte {
	if len(buf) < s.Size() {
		buf = make([]byte, s.Size())
	} else {
		buf = buf[0:s.Size()]
	}

	C.poly1305_finish(&s.state, (*C.uchar)(&buf[0]))

	return buf
}
<commit_msg>Append to buf, don't overwrite it.<commit_after>// Package poly1305 provides a Go wrapper around floodyberry's optimized
// implementation of Poly1305.
// Poly1305 is a message authentication code (MAC) designed to meet the standard
// notion of unforgeability after a single message. After the sender
// authenticates one message, an attacker cannot find authenticators for any
// other messages.
// The sender MUST NOT use Poly1305 to authenticate more than one message under
// the same key. Authenticators for two messages under the same key should be
// expected to reveal enough information to allow forgeries of authenticators
// on other messages.
package poly1305

// BUG(codahale): Only supports AMD64.

// +build: amd64

import (
	// #cgo CFLAGS: -O3
	// #include "poly1305-donna.h"
	"C"
	"errors"
	"hash"
)

var (
	// ErrInvalidKey is returned when the provided key is not 256 bits long.
	ErrInvalidKey = errors.New("poly1305: invalid key length")
)

const (
	// KeySize is the length of Poly1305 keys, in bytes.
	KeySize = 32
	// BlockSize is the length of Poly1305 blocks, in bytes.
	BlockSize = 16
	// Size is the length of Poly1305 digests, in bytes.
	Size = 16
)

// A Poly1305 is an instance of Poly1305 using a particular key.
type Poly1305 struct {
	key   []byte
	state C.poly1305_state
}

// New creates and returns a keyed Hash implementation. The key argument must be
// 256 bits long, the value of which must only be used once.
func New(key []byte) (hash.Hash, error) {
	if len(key) != KeySize {
		return nil, ErrInvalidKey
	}

	h := new(Poly1305)
	h.key = make([]byte, KeySize)
	copy(h.key, key)
	h.Reset()

	return h, nil
}

// BlockSize returns the hash's underlying block size.
// The Write method must be able to accept any amount
// of data, but it may operate more efficiently if all writes
// are a multiple of the block size.
func (*Poly1305) BlockSize() int {
	return BlockSize
}

// Size returns the number of bytes Sum will return.
func (*Poly1305) Size() int {
	return Size
}

// Reset resets the Hash to its initial state.
func (s *Poly1305) Reset() {
	C.poly1305_init(&s.state, (*C.uchar)(&s.key[0]))
}

// Write (via the embedded io.Writer interface) adds more data to the running
// hash. It never returns an error.
func (s *Poly1305) Write(buf []byte) (int, error) {
	var p *C.uchar
	if len(buf) > 0 {
		p = (*C.uchar)(&buf[0])
	}

	C.poly1305_update(&s.state, p, (C.size_t)(len(buf)))

	return len(buf), nil
}

// Sum appends the current hash to b and returns the resulting slice.
// It does not change the underlying hash state.
func (s *Poly1305) Sum(buf []byte) []byte {
	var mac [Size]byte
	C.poly1305_finish(&s.state, (*C.uchar)(&mac[0]))
	return append(buf, mac[0:]...)
}
<|endoftext|>