<commit_before>// Binding for GStreamer API
package gst

/*
#include <gst/gst.h>

char** _gst_init(int* argc, char** argv) {
	gst_init(argc, &argv);
	return argv;
}

#cgo pkg-config: gstreamer-0.10
*/
import "C"

import (
	"os"
	"unsafe"
	"fmt"
	"github.com/ziutek/glib"
)

func v2g(v *glib.Value) *C.GValue {
	return (*C.GValue)(unsafe.Pointer(v))
}

func g2v(v *C.GValue) *glib.Value {
	return (*glib.Value)(unsafe.Pointer(v))
}

type Fourcc C.guint32

func (f Fourcc) Type() glib.Type {
	return TYPE_FOURCC
}

func (f Fourcc) Value() *glib.Value {
	v := new(glib.Value)
	C.gst_value_set_fourcc(v2g(v), C.guint32(f))
	return v
}

func (f Fourcc) String() string {
	buf := make([]byte, 4)
	buf[0] = byte(f)
	buf[1] = byte(f>>8)
	buf[2] = byte(f>>16)
	buf[3] = byte(f>>32)
	return string(buf)
}

func MakeFourcc(a, b, c, d byte) Fourcc {
	return Fourcc(uint32(a) | uint32(b)<<8 | uint32(c)<<16 | uint32(d)<<24)
}

func StrFourcc(s string) Fourcc {
	if len(s) != 4 {
		panic("Fourcc string length != 4")
	}
	return MakeFourcc(s[0], s[1], s[2], s[3])
}

func ValueFourcc(v *glib.Value) Fourcc {
	return Fourcc(C.gst_value_get_fourcc(v2g(v)))
}

type IntRange struct {
	Start, End int
}

func (r *IntRange) Type() glib.Type {
	return TYPE_INT_RANGE
}

func (r *IntRange) Value() *glib.Value {
	v := new(glib.Value)
	C.gst_value_set_int_range(v2g(v), C.gint(r.Start), C.gint(r.End))
	return v
}

func (r *IntRange) String() string {
	return fmt.Sprintf("[%d,%d]", r.Start, r.End)
}

func ValueRange(v *glib.Value) *IntRange {
	return &IntRange{
		int(C.gst_value_get_int_range_min(v2g(v))),
		int(C.gst_value_get_int_range_max(v2g(v))),
	}
}

type Fraction struct {
	Numer, Denom int
}

func (f *Fraction) Type() glib.Type {
	return TYPE_FRACTION
}

func (f *Fraction) Value() *glib.Value {
	v := new(glib.Value)
	C.gst_value_set_fraction(v2g(v), C.gint(f.Numer), C.gint(f.Denom))
	return v
}

func (r *Fraction) String() string {
	return fmt.Sprintf("%d/%d", r.Numer, r.Denom)
}

func ValueFraction(v *glib.Value) *Fraction {
	return &Fraction{
		int(C.gst_value_get_fraction_numerator(v2g(v))),
		int(C.gst_value_get_fraction_denominator(v2g(v))),
	}
}

var TYPE_FOURCC, TYPE_INT_RANGE, TYPE_FRACTION glib.Type

func init() {
	alen := C.int(len(os.Args))
	argv := make([]*C.char, alen)
	for i, s := range os.Args {
		argv[i] = C.CString(s)
	}
	ret := C._gst_init(&alen, &argv[0])
	argv = (*[1<<16]*C.char)(unsafe.Pointer(ret))[:alen]
	os.Args = make([]string, alen)
	for i, s := range argv {
		os.Args[i] = C.GoString(s)
	}

	TYPE_FOURCC = glib.Type(C.gst_fourcc_get_type())
	TYPE_INT_RANGE = glib.Type(C.gst_int_range_get_type())
	TYPE_FRACTION = glib.Type(C.gst_fraction_get_type())
}
<commit_msg>Docstring modified<commit_after>// Bindings for GStreamer API
package gst

/*
#include <gst/gst.h>

char** _gst_init(int* argc, char** argv) {
	gst_init(argc, &argv);
	return argv;
}

#cgo pkg-config: gstreamer-0.10
*/
import "C"

import (
	"os"
	"unsafe"
	"fmt"
	"github.com/ziutek/glib"
)

func v2g(v *glib.Value) *C.GValue {
	return (*C.GValue)(unsafe.Pointer(v))
}

func g2v(v *C.GValue) *glib.Value {
	return (*glib.Value)(unsafe.Pointer(v))
}

type Fourcc C.guint32

func (f Fourcc) Type() glib.Type {
	return TYPE_FOURCC
}

func (f Fourcc) Value() *glib.Value {
	v := new(glib.Value)
	C.gst_value_set_fourcc(v2g(v), C.guint32(f))
	return v
}

func (f Fourcc) String() string {
	buf := make([]byte, 4)
	buf[0] = byte(f)
	buf[1] = byte(f>>8)
	buf[2] = byte(f>>16)
	buf[3] = byte(f>>32)
	return string(buf)
}

func MakeFourcc(a, b, c, d byte) Fourcc {
	return Fourcc(uint32(a) | uint32(b)<<8 | uint32(c)<<16 | uint32(d)<<24)
}

func StrFourcc(s string) Fourcc {
	if len(s) != 4 {
		panic("Fourcc string length != 4")
	}
	return MakeFourcc(s[0], s[1], s[2], s[3])
}

func ValueFourcc(v *glib.Value) Fourcc {
	return Fourcc(C.gst_value_get_fourcc(v2g(v)))
}

type IntRange struct {
	Start, End int
}

func (r *IntRange) Type() glib.Type {
	return TYPE_INT_RANGE
}

func (r *IntRange) Value() *glib.Value {
	v := new(glib.Value)
	C.gst_value_set_int_range(v2g(v), C.gint(r.Start), C.gint(r.End))
	return v
}

func (r *IntRange) String() string {
	return fmt.Sprintf("[%d,%d]", r.Start, r.End)
}

func ValueRange(v *glib.Value) *IntRange {
	return &IntRange{
		int(C.gst_value_get_int_range_min(v2g(v))),
		int(C.gst_value_get_int_range_max(v2g(v))),
	}
}

type Fraction struct {
	Numer, Denom int
}

func (f *Fraction) Type() glib.Type {
	return TYPE_FRACTION
}

func (f *Fraction) Value() *glib.Value {
	v := new(glib.Value)
	C.gst_value_set_fraction(v2g(v), C.gint(f.Numer), C.gint(f.Denom))
	return v
}

func (r *Fraction) String() string {
	return fmt.Sprintf("%d/%d", r.Numer, r.Denom)
}

func ValueFraction(v *glib.Value) *Fraction {
	return &Fraction{
		int(C.gst_value_get_fraction_numerator(v2g(v))),
		int(C.gst_value_get_fraction_denominator(v2g(v))),
	}
}

var TYPE_FOURCC, TYPE_INT_RANGE, TYPE_FRACTION glib.Type

func init() {
	alen := C.int(len(os.Args))
	argv := make([]*C.char, alen)
	for i, s := range os.Args {
		argv[i] = C.CString(s)
	}
	ret := C._gst_init(&alen, &argv[0])
	argv = (*[1<<16]*C.char)(unsafe.Pointer(ret))[:alen]
	os.Args = make([]string, alen)
	for i, s := range argv {
		os.Args[i] = C.GoString(s)
	}

	TYPE_FOURCC = glib.Type(C.gst_fourcc_get_type())
	TYPE_INT_RANGE = glib.Type(C.gst_int_range_get_type())
	TYPE_FRACTION = glib.Type(C.gst_fraction_get_type())
}
<|endoftext|>


<commit_before>package util

const (
	// UserRecordLastLoginAtKey is the key for the time when the user
	// last logged in.
	UserRecordLastLoginAtKey = "last_login_at"
)
<commit_msg>Remove not used file<commit_after><|endoftext|>


<commit_before>package roach

import (
	"context"
	"database/sql"
	"encoding/json"
	"fmt"
	"strings"
	"sync"

	"github.com/cockroachdb/cockroach-go/crdb"
	crdbH "github.com/tomogoma/crdb"
	"github.com/tomogoma/go-typed-errors"
	"github.com/tomogoma/seedms/pkg/config"
)

// Roach is a cockroach db store.
// Use NewRoach() to instantiate.
type Roach struct {
	errors.NotFoundErrCheck
	dsn              string
	dbName           string
	db               *sql.DB
	compatibilityErr error

	isDBInitMutex sync.Mutex
	isDBInit      bool
}

const (
	keyDBVersion = "db.version"
)

// NewRoach creates an instance of *Roach. A db connection is only established
// when InitDBIfNot() or one of the Execute/Query methods is called.
func NewRoach(opts ...Option) *Roach {
	r := &Roach{
		isDBInit:      false,
		isDBInitMutex: sync.Mutex{},
		dbName:        config.CanonicalName(),
	}
	for _, f := range opts {
		f(r)
	}
	return r
}

// InitDBIfNot connects to and sets up the DB; creating it and tables if necessary.
func (r *Roach) InitDBIfNot() error {
	var err error
	r.db, err = crdbH.TryConnect(r.dsn, r.db)
	if err != nil {
		return errors.Newf("connect to db: %v", err)
	}
	return r.instantiate()
}

// ExecuteTx prepares a transaction (with retries) for execution in fn.
// It commits the changes if fn returns nil, otherwise changes are rolled back.
func (r *Roach) ExecuteTx(fn func(*sql.Tx) error) error {
	if err := r.InitDBIfNot(); err != nil {
		return err
	}
	return crdb.ExecuteTx(context.Background(), r.db, nil, fn)
}

// ColDesc returns a string containing cols in the given order separated by ",".
func ColDesc(cols ...string) string {
	desc := ""
	for _, col := range cols {
		desc = desc + col + ","
	}
	return strings.TrimSuffix(desc, ",")
}

func (r *Roach) instantiate() error {
	r.isDBInitMutex.Lock()
	defer r.isDBInitMutex.Unlock()
	if r.compatibilityErr != nil {
		return r.compatibilityErr
	}
	if r.isDBInit {
		return nil
	}
	if err := crdbH.InstantiateDB(r.db, r.dbName, AllTableDescs...); err != nil {
		return errors.Newf("instantiating db: %v", err)
	}
	if err := r.validateRunningVersion(); err != nil {
		if !r.IsNotFoundError(err) {
			return fmt.Errorf("check db version: %v", err)
		}
		if err := r.setRunningVersionCurrent(); err != nil {
			return errors.Newf("set db version: %v", err)
		}
	}
	r.isDBInit = true
	return nil
}

func (r *Roach) validateRunningVersion() error {
	var runningVersion int
	q := `SELECT ` + ColValue + ` FROM ` + TblConfigurations + ` WHERE ` + ColKey + `=$1`
	var confB []byte
	if err := r.db.QueryRow(q, keyDBVersion).Scan(&confB); err != nil {
		if err == sql.ErrNoRows {
			return errors.NewNotFoundf("config not found")
		}
		return errors.Newf("get conf: %v", err)
	}
	if err := json.Unmarshal(confB, &runningVersion); err != nil {
		return errors.Newf("Unmarshalling config: %v", err)
	}
	if runningVersion != Version {
		r.compatibilityErr = errors.Newf("db incompatible: need db"+
			" version '%d', found '%d'", Version, runningVersion)
		return r.compatibilityErr
	}
	return nil
}

func (r *Roach) setRunningVersionCurrent() error {
	valB, err := json.Marshal(Version)
	if err != nil {
		return errors.Newf("marshal conf: %v", err)
	}
	cols := ColDesc(ColKey, ColValue, ColUpdateDate)
	updCols := ColDesc(ColValue, ColUpdateDate)
	q := `
		INSERT INTO ` + TblConfigurations + ` (` + cols + `)
			VALUES ($1, $2, CURRENT_TIMESTAMP)
			ON CONFLICT (` + ColKey + `)
			DO UPDATE SET (` + updCols + `) = ($2, CURRENT_TIMESTAMP)`
	res, err := r.db.Exec(q, keyDBVersion, valB)
	return checkRowsAffected(res, err, 1)
}

func checkRowsAffected(r sql.Result, err error, expAffected int64) error {
	if err != nil {
		if err == sql.ErrNoRows {
			return errors.NewNotFound("none found")
		}
		return err
	}
	c, err := r.RowsAffected()
	if err != nil {
		return err
	}
	if c == 0 {
		return errors.NewNotFound("none found for update")
	}
	if c != expAffected {
		return errors.Newf("expected %d affected rows but got %d",
			expAffected, c)
	}
	return nil
}
<commit_msg>ignore empty cols while generating column list<commit_after>package roach

import (
	"context"
	"database/sql"
	"encoding/json"
	"fmt"
	"strings"
	"sync"

	"github.com/cockroachdb/cockroach-go/crdb"
	crdbH "github.com/tomogoma/crdb"
	"github.com/tomogoma/go-typed-errors"
	"github.com/tomogoma/seedms/pkg/config"
)

// Roach is a cockroach db store.
// Use NewRoach() to instantiate.
type Roach struct {
	errors.NotFoundErrCheck
	dsn              string
	dbName           string
	db               *sql.DB
	compatibilityErr error

	isDBInitMutex sync.Mutex
	isDBInit      bool
}

const (
	keyDBVersion = "db.version"
)

// NewRoach creates an instance of *Roach. A db connection is only established
// when InitDBIfNot() or one of the Execute/Query methods is called.
func NewRoach(opts ...Option) *Roach {
	r := &Roach{
		isDBInit:      false,
		isDBInitMutex: sync.Mutex{},
		dbName:        config.CanonicalName(),
	}
	for _, f := range opts {
		f(r)
	}
	return r
}

// InitDBIfNot connects to and sets up the DB; creating it and tables if necessary.
func (r *Roach) InitDBIfNot() error {
	var err error
	r.db, err = crdbH.TryConnect(r.dsn, r.db)
	if err != nil {
		return errors.Newf("connect to db: %v", err)
	}
	return r.instantiate()
}

// ExecuteTx prepares a transaction (with retries) for execution in fn.
// It commits the changes if fn returns nil, otherwise changes are rolled back.
func (r *Roach) ExecuteTx(fn func(*sql.Tx) error) error {
	if err := r.InitDBIfNot(); err != nil {
		return err
	}
	return crdb.ExecuteTx(context.Background(), r.db, nil, fn)
}

// ColDesc returns a string containing cols in the given order separated by ",".
func ColDesc(cols ...string) string {
	desc := ""
	for _, col := range cols {
		if col == "" {
			continue
		}
		desc = desc + col + ", "
	}
	return strings.TrimSuffix(desc, ", ")
}

func (r *Roach) instantiate() error {
	r.isDBInitMutex.Lock()
	defer r.isDBInitMutex.Unlock()
	if r.compatibilityErr != nil {
		return r.compatibilityErr
	}
	if r.isDBInit {
		return nil
	}
	if err := crdbH.InstantiateDB(r.db, r.dbName, AllTableDescs...); err != nil {
		return errors.Newf("instantiating db: %v", err)
	}
	if err := r.validateRunningVersion(); err != nil {
		if !r.IsNotFoundError(err) {
			return fmt.Errorf("check db version: %v", err)
		}
		if err := r.setRunningVersionCurrent(); err != nil {
			return errors.Newf("set db version: %v", err)
		}
	}
	r.isDBInit = true
	return nil
}

func (r *Roach) validateRunningVersion() error {
	var runningVersion int
	q := `SELECT ` + ColValue + ` FROM ` + TblConfigurations + ` WHERE ` + ColKey + `=$1`
	var confB []byte
	if err := r.db.QueryRow(q, keyDBVersion).Scan(&confB); err != nil {
		if err == sql.ErrNoRows {
			return errors.NewNotFoundf("config not found")
		}
		return errors.Newf("get conf: %v", err)
	}
	if err := json.Unmarshal(confB, &runningVersion); err != nil {
		return errors.Newf("Unmarshalling config: %v", err)
	}
	if runningVersion != Version {
		r.compatibilityErr = errors.Newf("db incompatible: need db"+
			" version '%d', found '%d'", Version, runningVersion)
		return r.compatibilityErr
	}
	return nil
}

func (r *Roach) setRunningVersionCurrent() error {
	valB, err := json.Marshal(Version)
	if err != nil {
		return errors.Newf("marshal conf: %v", err)
	}
	cols := ColDesc(ColKey, ColValue, ColUpdateDate)
	updCols := ColDesc(ColValue, ColUpdateDate)
	q := `
		INSERT INTO ` + TblConfigurations + ` (` + cols + `)
			VALUES ($1, $2, CURRENT_TIMESTAMP)
			ON CONFLICT (` + ColKey + `)
			DO UPDATE SET (` + updCols + `) = ($2, CURRENT_TIMESTAMP)`
	res, err := r.db.Exec(q, keyDBVersion, valB)
	return checkRowsAffected(res, err, 1)
}

func checkRowsAffected(r sql.Result, err error, expAffected int64) error {
	if err != nil {
		if err == sql.ErrNoRows {
			return errors.NewNotFound("none found")
		}
		return err
	}
	c, err := r.RowsAffected()
	if err != nil {
		return err
	}
	if c == 0 {
		return errors.NewNotFound("none found for update")
	}
	if c != expAffected {
		return errors.Newf("expected %d affected rows but got %d",
			expAffected, c)
	}
	return nil
}
<|endoftext|>


<commit_before>package raft

import (
	"bytes"
	"context"
	"fmt"
	"time"

	gorums "github.com/relab/raft/pkg/raft/gorumspb"
	pb "github.com/relab/raft/pkg/raft/raftpb"
)

type appendEntriesHandler interface {
	sendRequest(*Replica)
}

type aeqrpc struct{}
type aenoqrpc struct{}

func (q *aeqrpc) sendRequest(r *Replica) {
	r.Lock()
	defer r.Unlock()

	if logLevel >= DEBUG {
		r.logger.log(fmt.Sprintf("Sending AppendEntries for term %d", r.persistent.currentTerm))
	}

	var buffer bytes.Buffer

LOOP:
	for i := r.maxAppendEntries; i > 0; i-- {
		select {
		case entry := <-r.queue:
			r.persistent.log = append(r.persistent.log, entry)

			buffer.WriteString(fmt.Sprintf(STORECOMMAND, r.persistent.currentTerm, entry.Data.ClientID, entry.Data.SequenceNumber, entry.Data.Command))
		default:
			break LOOP
		}
	}

	// Write to stable storage
	// TODO Assumes successful
	r.save(buffer.String())

	// #L1
	entries := []*pb.Entry{}

	nextIndex := 0

	// Always send entries for the most up-to-date client.
	for id := range r.nodes {
		next := r.nextIndex[id] - 1

		if next > nextIndex {
			nextIndex = next
		}
	}

	if len(r.persistent.log) > nextIndex {
		maxEntries := int(min(uint64(nextIndex+r.maxAppendEntries), uint64(len(r.persistent.log))))

		if !r.batch {
			maxEntries = nextIndex + 1
		}

		entries = r.persistent.log[nextIndex:maxEntries]
	}

	if logLevel >= DEBUG {
		r.logger.log(fmt.Sprintf("Sending %d entries", len(entries)))
	}

	req := &pb.AppendEntriesRequest{
		LeaderID:     r.id,
		Term:         r.persistent.currentTerm,
		PrevLogIndex: uint64(nextIndex),
		PrevLogTerm:  r.logTerm(nextIndex),
		CommitIndex:  r.commitIndex,
		Entries:      entries,
	}

	go func(req *pb.AppendEntriesRequest) {
		ctx, cancel := context.WithTimeout(context.Background(), TCPHeartbeat*time.Millisecond)
		defer cancel()

		resp, err := r.conf.AppendEntries(ctx, req)

		if err != nil {
			r.logger.log(fmt.Sprintf("AppendEntries failed = %v", err))

			// We can not return if there is a response, i.e., there is replicas that needs updating.
			if resp.AppendEntriesResponse == nil {
				return
			}
		}

		r.handleAppendEntriesResponse(resp.AppendEntriesResponse)
	}(req)

	r.heartbeat.Reset(r.heartbeatTimeout)
}

func (q *aenoqrpc) sendRequest(r *Replica) {
	r.Lock()
	defer r.Unlock()

	if logLevel >= DEBUG {
		r.logger.log(fmt.Sprintf("Sending AppendEntries for term %d", r.persistent.currentTerm))
	}

	var buffer bytes.Buffer

LOOP:
	for i := r.maxAppendEntries; i > 0; i-- {
		select {
		case entry := <-r.queue:
			r.persistent.log = append(r.persistent.log, entry)

			buffer.WriteString(fmt.Sprintf(STORECOMMAND, r.persistent.currentTerm, entry.Data.ClientID, entry.Data.SequenceNumber, entry.Data.Command))
		default:
			break LOOP
		}
	}

	// Write to stable storage
	// TODO Assumes successful
	r.save(buffer.String())

	// #L1
	for id, node := range r.nodes {
		entries := []*pb.Entry{}

		nextIndex := r.nextIndex[id] - 1

		if len(r.persistent.log) > nextIndex {
			maxEntries := int(min(uint64(nextIndex+r.maxAppendEntries), uint64(len(r.persistent.log))))

			if !r.batch {
				maxEntries = nextIndex + 1
			}

			entries = r.persistent.log[nextIndex:maxEntries]
		}

		if logLevel >= DEBUG {
			r.logger.to(id, fmt.Sprintf("Sending %d entries", len(entries)))
		}

		req := &pb.AppendEntriesRequest{
			LeaderID:     r.id,
			Term:         r.persistent.currentTerm,
			PrevLogIndex: uint64(nextIndex),
			PrevLogTerm:  r.logTerm(nextIndex),
			CommitIndex:  r.commitIndex,
			Entries:      entries,
		}

		go func(node *gorums.Node, req *pb.AppendEntriesRequest) {
			ctx, cancel := context.WithTimeout(context.Background(), TCPHeartbeat*time.Millisecond)
			defer cancel()

			resp, err := node.RaftClient.AppendEntries(ctx, req)

			if err != nil {
				r.logger.to(id, fmt.Sprintf("AppendEntries failed = %v", err))

				return
			}

			r.handleAppendEntriesResponse(resp)
		}(node, req)
	}

	r.heartbeat.Reset(r.heartbeatTimeout)
}
<commit_msg>Fix: Range variable id captured by func literal<commit_after>package raft

import (
	"bytes"
	"context"
	"fmt"
	"time"

	gorums "github.com/relab/raft/pkg/raft/gorumspb"
	pb "github.com/relab/raft/pkg/raft/raftpb"
)

type appendEntriesHandler interface {
	sendRequest(*Replica)
}

type aeqrpc struct{}
type aenoqrpc struct{}

func (q *aeqrpc) sendRequest(r *Replica) {
	r.Lock()
	defer r.Unlock()

	if logLevel >= DEBUG {
		r.logger.log(fmt.Sprintf("Sending AppendEntries for term %d", r.persistent.currentTerm))
	}

	var buffer bytes.Buffer

LOOP:
	for i := r.maxAppendEntries; i > 0; i-- {
		select {
		case entry := <-r.queue:
			r.persistent.log = append(r.persistent.log, entry)

			buffer.WriteString(fmt.Sprintf(STORECOMMAND, r.persistent.currentTerm, entry.Data.ClientID, entry.Data.SequenceNumber, entry.Data.Command))
		default:
			break LOOP
		}
	}

	// Write to stable storage
	// TODO Assumes successful
	r.save(buffer.String())

	// #L1
	entries := []*pb.Entry{}

	nextIndex := 0

	// Always send entries for the most up-to-date client.
	for id := range r.nodes {
		next := r.nextIndex[id] - 1

		if next > nextIndex {
			nextIndex = next
		}
	}

	if len(r.persistent.log) > nextIndex {
		maxEntries := int(min(uint64(nextIndex+r.maxAppendEntries), uint64(len(r.persistent.log))))

		if !r.batch {
			maxEntries = nextIndex + 1
		}

		entries = r.persistent.log[nextIndex:maxEntries]
	}

	if logLevel >= DEBUG {
		r.logger.log(fmt.Sprintf("Sending %d entries", len(entries)))
	}

	req := &pb.AppendEntriesRequest{
		LeaderID:     r.id,
		Term:         r.persistent.currentTerm,
		PrevLogIndex: uint64(nextIndex),
		PrevLogTerm:  r.logTerm(nextIndex),
		CommitIndex:  r.commitIndex,
		Entries:      entries,
	}

	go func(req *pb.AppendEntriesRequest) {
		ctx, cancel := context.WithTimeout(context.Background(), TCPHeartbeat*time.Millisecond)
		defer cancel()

		resp, err := r.conf.AppendEntries(ctx, req)

		if err != nil {
			r.logger.log(fmt.Sprintf("AppendEntries failed = %v", err))

			// We can not return if there is a response, i.e., there is replicas that needs updating.
			if resp.AppendEntriesResponse == nil {
				return
			}
		}

		r.handleAppendEntriesResponse(resp.AppendEntriesResponse)
	}(req)

	r.heartbeat.Reset(r.heartbeatTimeout)
}

func (q *aenoqrpc) sendRequest(r *Replica) {
	r.Lock()
	defer r.Unlock()

	if logLevel >= DEBUG {
		r.logger.log(fmt.Sprintf("Sending AppendEntries for term %d", r.persistent.currentTerm))
	}

	var buffer bytes.Buffer

LOOP:
	for i := r.maxAppendEntries; i > 0; i-- {
		select {
		case entry := <-r.queue:
			r.persistent.log = append(r.persistent.log, entry)

			buffer.WriteString(fmt.Sprintf(STORECOMMAND, r.persistent.currentTerm, entry.Data.ClientID, entry.Data.SequenceNumber, entry.Data.Command))
		default:
			break LOOP
		}
	}

	// Write to stable storage
	// TODO Assumes successful
	r.save(buffer.String())

	// #L1
	for id, node := range r.nodes {
		entries := []*pb.Entry{}

		nextIndex := r.nextIndex[id] - 1

		if len(r.persistent.log) > nextIndex {
			maxEntries := int(min(uint64(nextIndex+r.maxAppendEntries), uint64(len(r.persistent.log))))

			if !r.batch {
				maxEntries = nextIndex + 1
			}

			entries = r.persistent.log[nextIndex:maxEntries]
		}

		if logLevel >= DEBUG {
			r.logger.to(id, fmt.Sprintf("Sending %d entries", len(entries)))
		}

		req := &pb.AppendEntriesRequest{
			LeaderID:     r.id,
			Term:         r.persistent.currentTerm,
			PrevLogIndex: uint64(nextIndex),
			PrevLogTerm:  r.logTerm(nextIndex),
			CommitIndex:  r.commitIndex,
			Entries:      entries,
		}

		go func(node *gorums.Node, req *pb.AppendEntriesRequest) {
			ctx, cancel := context.WithTimeout(context.Background(), TCPHeartbeat*time.Millisecond)
			defer cancel()

			resp, err := node.RaftClient.AppendEntries(ctx, req)

			if err != nil {
				r.logger.to(req.LeaderID, fmt.Sprintf("AppendEntries failed = %v", err))

				return
			}

			r.handleAppendEntriesResponse(resp)
		}(node, req)
	}

	r.heartbeat.Reset(r.heartbeatTimeout)
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Camlistore Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package app helps with configuring and starting server applications
// from Camlistore.
package app

import (
	"fmt"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"camlistore.org/pkg/auth"
	camhttputil "camlistore.org/pkg/httputil"
	"camlistore.org/pkg/jsonconfig"
)

// Handler acts as a reverse proxy for a server application started by
// Camlistore. It can also serve some extra JSON configuration to the app.
type Handler struct {
	name    string            // Name of the app's program.
	envVars map[string]string // Variables set in the app's process environment. See pkg/app/vars.txt.

	auth      auth.AuthMode  // Used for basic HTTP authenticating against the app requests.
	appConfig jsonconfig.Obj // Additional parameters the app can request, or nil.

	proxy *httputil.ReverseProxy // For redirecting requests to the app.
}

func (a *Handler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	if camhttputil.PathSuffix(req) == "config.json" {
		if a.auth.AllowedAccess(req)&auth.OpGet == auth.OpGet {
			camhttputil.ReturnJSON(rw, a.appConfig)
		} else {
			auth.SendUnauthorized(rw, req)
		}
		return
	}
	if a.proxy == nil {
		http.Error(rw, "no proxy for the app", 500)
		return
	}
	a.proxy.ServeHTTP(rw, req)
}

// randPortBackendURL picks a random free port to listen on, and combines it
// with apiHost and appHandlerPrefix to create the appBackendURL that the app
// will listen on, and that the app handler will proxy to.
func randPortBackendURL(apiHost, appHandlerPrefix string) (string, error) {
	addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
	if err != nil {
		return "", err
	}
	listener, err := net.ListenTCP("tcp", addr)
	if err != nil {
		return "", fmt.Errorf("could not listen to find random port: %v", err)
	}
	randAddr := listener.Addr().(*net.TCPAddr)
	if err := listener.Close(); err != nil {
		return "", fmt.Errorf("could not close random listener: %v", err)
	}

	scheme := "https://"
	noScheme := strings.TrimPrefix(apiHost, scheme)
	if strings.HasPrefix(noScheme, "http://") {
		scheme = "http://"
		noScheme = strings.TrimPrefix(noScheme, scheme)
	}
	hostPortPrefix := strings.SplitN(noScheme, "/", 2)
	if len(hostPortPrefix) != 2 {
		return "", fmt.Errorf("invalid apiHost: %q (no trailing slash?)", apiHost)
	}
	var host string
	if strings.Contains(hostPortPrefix[0], "]") {
		// we've got some IPv6 probably
		hostPort := strings.Split(hostPortPrefix[0], "]")
		host = hostPort[0] + "]"
	} else {
		hostPort := strings.Split(hostPortPrefix[0], ":")
		host = hostPort[0]
	}
	return fmt.Sprintf("%s%s:%d%s", scheme, host, randAddr.Port, appHandlerPrefix), nil
}

// NewHandler returns a Handler that proxies requests to an app. Start() on the
// Handler starts the app.
// The apiHost must end in a slash and is the camlistored API server for the app
// process to hit.
// The appHandlerPrefix is the URL path prefix on apiHost where the app is mounted.
// It must end in a slash, and be at minimum "/".
// The conf object has the following members, related to the vars described in
// doc/app-environment.txt:
// "program", string, required. File name of the app's program executable. Either
// an absolute path, or the name of a file located in CAMLI_APP_BINDIR or in PATH.
// "backendURL", string, optional. Automatic if absent. It sets CAMLI_APP_BACKEND_URL.
// "appConfig", object, optional. Additional configuration that the app can request from Camlistore.
func NewHandler(conf jsonconfig.Obj, apiHost, appHandlerPrefix string) (*Handler, error) {
	// TODO: remove the appHandlerPrefix if/when we change where the app config JSON URL is made available.
	name := conf.RequiredString("program")
	backendURL := conf.OptionalString("backendURL", "")
	appConfig := conf.OptionalObject("appConfig")
	// TODO(mpl): add an auth token in the extra config of the dev server config,
	// that the hello app can use to setup a status handler than only responds
	// to requests with that token.
	if err := conf.Validate(); err != nil {
		return nil, err
	}

	if apiHost == "" {
		return nil, fmt.Errorf("app: could not initialize Handler for %q: Camlistore apiHost is unknown", name)
	}
	if appHandlerPrefix == "" {
		return nil, fmt.Errorf("app: could not initialize Handler for %q: empty appHandlerPrefix", name)
	}

	if backendURL == "" {
		var err error
		// If not specified in the conf, we're dynamically picking the port of the CAMLI_APP_BACKEND_URL
		// now (instead of letting the app itself do it), because we need to know it in advance in order
		// to set the app handler's proxy.
		backendURL, err = randPortBackendURL(apiHost, appHandlerPrefix)
		if err != nil {
			return nil, err
		}
	}

	username, password := auth.RandToken(20), auth.RandToken(20)
	camliAuth := username + ":" + password
	basicAuth := auth.NewBasicAuth(username, password)
	envVars := map[string]string{
		"CAMLI_API_HOST":        apiHost,
		"CAMLI_AUTH":            camliAuth,
		"CAMLI_APP_BACKEND_URL": backendURL,
	}
	if appConfig != nil {
		envVars["CAMLI_APP_CONFIG_URL"] = apiHost + strings.TrimPrefix(appHandlerPrefix, "/") + "config.json"
	}

	proxyURL, err := url.Parse(backendURL)
	if err != nil {
		return nil, fmt.Errorf("could not parse backendURL %q: %v", backendURL, err)
	}
	return &Handler{
		name:      name,
		envVars:   envVars,
		auth:      basicAuth,
		appConfig: appConfig,
		proxy:     httputil.NewSingleHostReverseProxy(proxyURL),
	}, nil
}

func (a *Handler) Start() error {
	name := a.name
	if name == "" {
		return fmt.Errorf("invalid app name: %q", name)
	}
	var binPath string
	var err error
	if e := os.Getenv("CAMLI_APP_BINDIR"); e != "" {
		binPath, err = exec.LookPath(filepath.Join(e, name))
		if err != nil {
			log.Printf("%q executable not found in %q", e)
		}
	}
	if err != nil {
		binPath, err = exec.LookPath(name)
		if err != nil {
			return fmt.Errorf("%q executable not found in PATH.", name)
		}
	}

	cmd := exec.Command(binPath)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	// TODO(mpl): extract Env methods from dev/devcam/env.go to a util pkg and use them here.
	newVars := make(map[string]string, len(a.envVars))
	for k, v := range a.envVars {
		newVars[k+"="] = v
	}
	env := os.Environ()
	for pos, oldkv := range env {
		for k, newVal := range newVars {
			if strings.HasPrefix(oldkv, k) {
				env[pos] = k + newVal
				delete(newVars, k)
				break
			}
		}
	}
	for k, v := range newVars {
		env = append(env, k+v)
	}
	cmd.Env = env
	if err := cmd.Start(); err != nil {
		return fmt.Errorf("could not start app %v: %v", name, err)
	}
	return nil
}

// ProgramName returns the name of the app's binary. It may be a file name in
// CAMLI_APP_BINDIR or PATH, or an absolute path.
func (a *Handler) ProgramName() string {
	return a.name
}

// AuthMode returns the app handler's auth mode, which is also the auth that the
// app's client will be configured with. This mode should be registered with
// the server's auth modes, for the app to have access to the server's resources.
func (a *Handler) AuthMode() auth.AuthMode {
	return a.auth
}

// AppConfig returns the optional configuration parameters object that the app
// can request from the app handler. It can be nil.
func (a *Handler) AppConfig() map[string]interface{} {
	return a.appConfig
}
<commit_msg>server/app: fix binPath logic bug<commit_after>/*
Copyright 2014 The Camlistore Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package app helps with configuring and starting server applications
// from Camlistore.
package app

import (
	"fmt"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"camlistore.org/pkg/auth"
	camhttputil "camlistore.org/pkg/httputil"
	"camlistore.org/pkg/jsonconfig"
)

// Handler acts as a reverse proxy for a server application started by
// Camlistore. It can also serve some extra JSON configuration to the app.
type Handler struct {
	name    string            // Name of the app's program.
	envVars map[string]string // Variables set in the app's process environment. See pkg/app/vars.txt.

	auth      auth.AuthMode  // Used for basic HTTP authenticating against the app requests.
	appConfig jsonconfig.Obj // Additional parameters the app can request, or nil.

	proxy *httputil.ReverseProxy // For redirecting requests to the app.
}

func (a *Handler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	if camhttputil.PathSuffix(req) == "config.json" {
		if a.auth.AllowedAccess(req)&auth.OpGet == auth.OpGet {
			camhttputil.ReturnJSON(rw, a.appConfig)
		} else {
			auth.SendUnauthorized(rw, req)
		}
		return
	}
	if a.proxy == nil {
		http.Error(rw, "no proxy for the app", 500)
		return
	}
	a.proxy.ServeHTTP(rw, req)
}

// randPortBackendURL picks a random free port to listen on, and combines it
// with apiHost and appHandlerPrefix to create the appBackendURL that the app
// will listen on, and that the app handler will proxy to.
func randPortBackendURL(apiHost, appHandlerPrefix string) (string, error) {
	addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
	if err != nil {
		return "", err
	}
	listener, err := net.ListenTCP("tcp", addr)
	if err != nil {
		return "", fmt.Errorf("could not listen to find random port: %v", err)
	}
	randAddr := listener.Addr().(*net.TCPAddr)
	if err := listener.Close(); err != nil {
		return "", fmt.Errorf("could not close random listener: %v", err)
	}

	scheme := "https://"
	noScheme := strings.TrimPrefix(apiHost, scheme)
	if strings.HasPrefix(noScheme, "http://") {
		scheme = "http://"
		noScheme = strings.TrimPrefix(noScheme, scheme)
	}
	hostPortPrefix := strings.SplitN(noScheme, "/", 2)
	if len(hostPortPrefix) != 2 {
		return "", fmt.Errorf("invalid apiHost: %q (no trailing slash?)", apiHost)
	}
	var host string
	if strings.Contains(hostPortPrefix[0], "]") {
		// we've got some IPv6 probably
		hostPort := strings.Split(hostPortPrefix[0], "]")
		host = hostPort[0] + "]"
	} else {
		hostPort := strings.Split(hostPortPrefix[0], ":")
		host = hostPort[0]
	}
	return fmt.Sprintf("%s%s:%d%s", scheme, host, randAddr.Port, appHandlerPrefix), nil
}

// NewHandler returns a Handler that proxies requests to an app. Start() on the
// Handler starts the app.
// The apiHost must end in a slash and is the camlistored API server for the app
// process to hit.
// The appHandlerPrefix is the URL path prefix on apiHost where the app is mounted.
// It must end in a slash, and be at minimum "/".
// The conf object has the following members, related to the vars described in
// doc/app-environment.txt:
// "program", string, required. File name of the app's program executable. Either
// an absolute path, or the name of a file located in CAMLI_APP_BINDIR or in PATH.
// "backendURL", string, optional. Automatic if absent. It sets CAMLI_APP_BACKEND_URL.
// "appConfig", object, optional. Additional configuration that the app can request from Camlistore.
func NewHandler(conf jsonconfig.Obj, apiHost, appHandlerPrefix string) (*Handler, error) {
	// TODO: remove the appHandlerPrefix if/when we change where the app config JSON URL is made available.
	name := conf.RequiredString("program")
	backendURL := conf.OptionalString("backendURL", "")
	appConfig := conf.OptionalObject("appConfig")
	// TODO(mpl): add an auth token in the extra config of the dev server config,
	// that the hello app can use to setup a status handler than only responds
	// to requests with that token.
	if err := conf.Validate(); err != nil {
		return nil, err
	}

	if apiHost == "" {
		return nil, fmt.Errorf("app: could not initialize Handler for %q: Camlistore apiHost is unknown", name)
	}
	if appHandlerPrefix == "" {
		return nil, fmt.Errorf("app: could not initialize Handler for %q: empty appHandlerPrefix", name)
	}

	if backendURL == "" {
		var err error
		// If not specified in the conf, we're dynamically picking the port of the CAMLI_APP_BACKEND_URL
		// now (instead of letting the app itself do it), because we need to know it in advance in order
		// to set the app handler's proxy.
		backendURL, err = randPortBackendURL(apiHost, appHandlerPrefix)
		if err != nil {
			return nil, err
		}
	}

	username, password := auth.RandToken(20), auth.RandToken(20)
	camliAuth := username + ":" + password
	basicAuth := auth.NewBasicAuth(username, password)
	envVars := map[string]string{
		"CAMLI_API_HOST":        apiHost,
		"CAMLI_AUTH":            camliAuth,
		"CAMLI_APP_BACKEND_URL": backendURL,
	}
	if appConfig != nil {
		envVars["CAMLI_APP_CONFIG_URL"] = apiHost + strings.TrimPrefix(appHandlerPrefix, "/") + "config.json"
	}

	proxyURL, err := url.Parse(backendURL)
	if err != nil {
		return nil, fmt.Errorf("could not parse backendURL %q: %v", backendURL, err)
	}
	return &Handler{
		name:      name,
		envVars:   envVars,
		auth:      basicAuth,
		appConfig: appConfig,
		proxy:     httputil.NewSingleHostReverseProxy(proxyURL),
	}, nil
}

func (a *Handler) Start() error {
	name := a.name
	if name == "" {
		return fmt.Errorf("invalid app name: %q", name)
	}
	var binPath string
	var err error
	if e := os.Getenv("CAMLI_APP_BINDIR"); e != "" {
		binPath, err = exec.LookPath(filepath.Join(e, name))
		if err != nil {
			log.Printf("%q executable not found in %q", e)
		}
	}
	if binPath == "" || err != nil {
		binPath, err = exec.LookPath(name)
		if err != nil {
			return fmt.Errorf("%q executable not found in PATH.", name)
		}
	}

	cmd := exec.Command(binPath)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	// TODO(mpl): extract Env methods from dev/devcam/env.go to a util pkg and use them here.
	newVars := make(map[string]string, len(a.envVars))
	for k, v := range a.envVars {
		newVars[k+"="] = v
	}
	env := os.Environ()
	for pos, oldkv := range env {
		for k, newVal := range newVars {
			if strings.HasPrefix(oldkv, k) {
				env[pos] = k + newVal
				delete(newVars, k)
				break
			}
		}
	}
	for k, v := range newVars {
		env = append(env, k+v)
	}
	cmd.Env = env
	if err := cmd.Start(); err != nil {
		return fmt.Errorf("could not start app %v: %v", name, err)
	}
	return nil
}

// ProgramName returns the name of the app's binary. It may be a file name in
// CAMLI_APP_BINDIR or PATH, or an absolute path.
func (a *Handler) ProgramName() string {
	return a.name
}

// AuthMode returns the app handler's auth mode, which is also the auth that the
// app's client will be configured with. This mode should be registered with
// the server's auth modes, for the app to have access to the server's resources.
func (a *Handler) AuthMode() auth.AuthMode {
	return a.auth
}

// AppConfig returns the optional configuration parameters object that the app
// can request from the app handler. It can be nil.
func (a *Handler) AppConfig() map[string]interface{} {
	return a.appConfig
}
<|endoftext|>


<commit_before>package storage

import (
	"errors"

	"github.com/deis/tiller/pkg/proto/hapi/release"
)

// Memory is an in-memory ReleaseStorage implementation.
type Memory struct {
	releases map[string]*release.Release
}

// NewMemory creates a new in-memory storage.
func NewMemory() *Memory {
	return &Memory{
		releases: map[string]*release.Release{},
	}
}

// ErrNotFound indicates that a release is not found.
var ErrNotFound = errors.New("release not found")

// Read returns the named Release.
//
// If the release is not found, an ErrNotFound error is returned.
func (m *Memory) Read(k string) (*release.Release, error) {
	v, ok := m.releases[k]
	if !ok {
		return v, ErrNotFound
	}
	return v, nil
}

// Create sets a release.
func (m *Memory) Create(rel *release.Release) error {
	m.releases[rel.Name] = rel
	return nil
}

// Update sets a release.
func (m *Memory) Update(rel *release.Release) error {
	if _, ok := m.releases[rel.Name]; !ok {
		return ErrNotFound
	}

	// FIXME: When Release is done, we need to do this right by marking the old
	// release as superseded, and creating a new release.
	m.releases[rel.Name] = rel
	return nil
}

// Delete removes a release.
func (m *Memory) Delete(name string) (*release.Release, error) {
	rel, ok := m.releases[name]
	if !ok {
		return nil, ErrNotFound
	}
	delete(m.releases, name)
	return rel, nil
}

// List returns all releases.
func (m *Memory) List() ([]*release.Release, error) {
	buf := make([]*release.Release, len(m.releases))
	i := 0
	for _, v := range m.releases {
		buf[i] = v
		i++
	}
	return buf, nil
}

// Query searches all releases for matches.
func (m *Memory) Query(labels map[string]string) ([]*release.Release, error) {
	return []*release.Release{}, errors.New("Cannot implement until release.Release is defined.")
}
<commit_msg>fix(storage): add mutex to in-memory<commit_after>package storage

import (
	"errors"
	"sync"

	"github.com/deis/tiller/pkg/proto/hapi/release"
)

// Memory is an in-memory ReleaseStorage implementation.
type Memory struct {
	sync.RWMutex
	releases map[string]*release.Release
}

// NewMemory creates a new in-memory storage.
func NewMemory() *Memory {
	return &Memory{
		releases: map[string]*release.Release{},
	}
}

// ErrNotFound indicates that a release is not found.
var ErrNotFound = errors.New("release not found")

// Read returns the named Release.
//
// If the release is not found, an ErrNotFound error is returned.
func (m *Memory) Read(k string) (*release.Release, error) {
	m.RLock()
	defer m.RUnlock()
	v, ok := m.releases[k]
	if !ok {
		return v, ErrNotFound
	}
	return v, nil
}

// Create sets a release.
func (m *Memory) Create(rel *release.Release) error {
	m.Lock()
	defer m.Unlock()
	m.releases[rel.Name] = rel
	return nil
}

// Update sets a release.
func (m *Memory) Update(rel *release.Release) error {
	m.Lock()
	defer m.Unlock()
	if _, ok := m.releases[rel.Name]; !ok {
		return ErrNotFound
	}

	// FIXME: When Release is done, we need to do this right by marking the old
	// release as superseded, and creating a new release.
	m.releases[rel.Name] = rel
	return nil
}

// Delete removes a release.
func (m *Memory) Delete(name string) (*release.Release, error) {
	m.Lock()
	defer m.Unlock()
	rel, ok := m.releases[name]
	if !ok {
		return nil, ErrNotFound
	}
	delete(m.releases, name)
	return rel, nil
}

// List returns all releases.
func (m *Memory) List() ([]*release.Release, error) {
	m.RLock()
	defer m.RUnlock()
	buf := make([]*release.Release, len(m.releases))
	i := 0
	for _, v := range m.releases {
		buf[i] = v
		i++
	}
	return buf, nil
}

// Query searches all releases for matches.
func (m *Memory) Query(labels map[string]string) ([]*release.Release, error) {
	m.RLock()
	defer m.RUnlock()
	return []*release.Release{}, errors.New("Cannot implement until release.Release is defined.")
}
<|endoftext|>


<commit_before>package system // import "github.com/docker/docker/pkg/system"

import (
	"os"
	"syscall"
	"time"
	"unsafe"
)

// Used by Chtimes
var maxTime time.Time

func init() {
	if unsafe.Sizeof(syscall.Timespec{}.Nsec) == 8 {
		// This is a 64 bit timespec
		// os.Chtimes limits time to the following
		maxTime = time.Unix(0, 1<<63-1)
	} else {
		// This is a 32 bit timespec
		maxTime = time.Unix(1<<31-1, 0)
	}
}

// Chtimes changes the access time and modified time of a file at the given path
func Chtimes(name string, atime time.Time, mtime time.Time) error {
	unixMinTime := time.Unix(0, 0)
	unixMaxTime := maxTime

	// If the modified time is prior to the Unix Epoch, or after the
	// end of Unix Time, os.Chtimes has undefined behavior
	// default to Unix Epoch in this case, just in case

	if atime.Before(unixMinTime) || atime.After(unixMaxTime) {
		atime = unixMinTime
	}

	if mtime.Before(unixMinTime) || mtime.After(unixMaxTime) {
		mtime = unixMinTime
	}

	if err := os.Chtimes(name, atime, mtime); err != nil {
		return err
	}

	// Take platform specific action for setting create time.
	return setCTime(name, mtime)
}
<commit_msg>pkg/system: add note about maxTime<commit_after>package system // import "github.com/docker/docker/pkg/system"

import (
	"os"
	"syscall"
	"time"
	"unsafe"
)

// Used by Chtimes
var maxTime time.Time

func init() {
	if unsafe.Sizeof(syscall.Timespec{}.Nsec) == 8 {
		// This is a 64 bit timespec
		// os.Chtimes limits time to the following
		//
		// Note that this intentionally sets nsec (not sec), which sets both sec
		// and nsec internally in time.Unix();
		// https://github.com/golang/go/blob/go1.19.2/src/time/time.go#L1364-L1380
		maxTime = time.Unix(0, 1<<63-1)
	} else {
		// This is a 32 bit timespec
		maxTime = time.Unix(1<<31-1, 0)
	}
}

// Chtimes changes the access time and modified time of a file at the given path
func Chtimes(name string, atime time.Time, mtime time.Time) error {
	unixMinTime := time.Unix(0, 0)
	unixMaxTime := maxTime

	// If the modified time is prior to the Unix Epoch, or after the
	// end of Unix Time, os.Chtimes has undefined behavior
	// default to Unix Epoch in this case, just in case

	if atime.Before(unixMinTime) || atime.After(unixMaxTime) {
		atime = unixMinTime
	}

	if mtime.Before(unixMinTime) || mtime.After(unixMaxTime) {
		mtime = unixMinTime
	}

	if err := os.Chtimes(name, atime, mtime); err != nil {
		return err
	}

	// Take platform specific action for setting create time.
	return setCTime(name, mtime)
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package lock

import (
	"crypto/sha1"
	"fmt"
	"io/ioutil"
	"os"
	"time"

	"github.com/golang/glog"
	"github.com/juju/clock"
	"github.com/juju/mutex"
	"github.com/pkg/errors"
)

var (
	// forceID is a user id for consistent testing
	forceID = ""
)

// WriteFile decorates ioutil.WriteFile with a file lock and retry
func WriteFile(filename string, data []byte, perm os.FileMode) error {
	spec := PathMutexSpec(filename)
	glog.Infof("WriteFile acquiring %s: %+v", filename, spec)
	releaser, err := mutex.Acquire(spec)
	if err != nil {
		return errors.Wrapf(err, "failed to acquire lock for %s: %+v", filename, spec)
	}

	defer releaser.Release()

	if err = ioutil.WriteFile(filename, data, perm); err != nil {
		return errors.Wrapf(err, "writefile failed for %s", filename)
	}
	return err
}

// PathMutexSpec returns a mutex spec for a path
func PathMutexSpec(path string) mutex.Spec {
	s := mutex.Spec{
		Name:  fmt.Sprintf("mk%x", sha1.Sum([]byte(path)))[0:40],
		Clock: clock.WallClock,
		// Poll the lock twice a second
		Delay: 500 * time.Millisecond,
		// panic after a minute instead of locking infinitely
		Timeout: 60 * time.Second,
	}
	return s
}
<commit_msg>Remove unused forceID var<commit_after>/*
Copyright 2019 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package lock

import (
	"crypto/sha1"
	"fmt"
	"io/ioutil"
	"os"
	"time"

	"github.com/golang/glog"
	"github.com/juju/clock"
	"github.com/juju/mutex"
	"github.com/pkg/errors"
)

// WriteFile decorates ioutil.WriteFile with a file lock and retry
func WriteFile(filename string, data []byte, perm os.FileMode) error {
	spec := PathMutexSpec(filename)
	glog.Infof("WriteFile acquiring %s: %+v", filename, spec)
	releaser, err := mutex.Acquire(spec)
	if err != nil {
		return errors.Wrapf(err, "failed to acquire lock for %s: %+v", filename, spec)
	}

	defer releaser.Release()

	if err = ioutil.WriteFile(filename, data, perm); err != nil {
		return errors.Wrapf(err, "writefile failed for %s", filename)
	}
	return err
}

// PathMutexSpec returns a mutex spec for a path
func PathMutexSpec(path string) mutex.Spec {
	s := mutex.Spec{
		Name:  fmt.Sprintf("mk%x", sha1.Sum([]byte(path)))[0:40],
		Clock: clock.WallClock,
		// Poll the lock twice a second
		Delay: 500 * time.Millisecond,
		// panic after a minute instead of locking infinitely
		Timeout: 60 * time.Second,
	}
	return s
}
<|endoftext|>


<commit_before>/*
Copyright 2017 Google, Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package util

import (
	"archive/tar"
	"io"
	"os"
	"path/filepath"
	"strings"
	"syscall"

	"github.com/sirupsen/logrus"
)

func unpackTar(tr *tar.Reader, path string) error {
	// drop the umask temporarily to 0, so we can create all files with correct permissions
	// otherwise the default (022) will cause file permission inconsistencies
	oldMask := syscall.Umask(0)
	defer syscall.Umask(oldMask)
	for {
		header, err := tr.Next()
		if err == io.EOF {
			// end of tar archive
			break
		}
		if err != nil {
			logrus.Error("Error getting next tar header")
			return err
		}

		if strings.Contains(header.Name, ".wh.") {
			rmPath := filepath.Join(path, header.Name)
			newName := strings.Replace(rmPath, ".wh.", "", 1)
			if err := os.Remove(rmPath); err != nil {
				logrus.Error(err)
			}
			if err = os.RemoveAll(newName); err != nil {
				logrus.Error(err)
			}
			continue
		}

		target := filepath.Join(path, header.Name)
		mode := header.FileInfo().Mode()
		switch header.Typeflag {

		// if its a dir and it doesn't exist create it
		case tar.TypeDir:
			if _, err := os.Stat(target); os.IsNotExist(err) {
				if err := os.MkdirAll(target, mode); err != nil {
					return err
				}
			} else {
				if err := os.Chmod(target, mode); err != nil {
					return err
				}
			}

		// if it's a file create it
		case tar.TypeReg:
			// It's possible for a file to be included before the directory it's in is created.
			baseDir := filepath.Dir(target)
			if _, err := os.Stat(baseDir); os.IsNotExist(err) {
				if err := os.MkdirAll(baseDir, 0755); err != nil {
					return err
				}
			}
			currFile, err := os.OpenFile(target, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode)
			if err != nil {
				logrus.Errorf("Error opening file %s", target)
				return err
			}
			_, err = io.Copy(currFile, tr)
			if err != nil {
				return err
			}
			currFile.Close()
		}

	}
	return nil
}

// UnTar takes in a path to a tar file and writes the untarred version to the provided target.
// Only untars one level, does not untar nested tars.
func UnTar(r io.Reader, target string) error {
	if _, ok := os.Stat(target); ok != nil {
		os.MkdirAll(target, 0775)
	}

	tr := tar.NewReader(r)
	if err := unpackTar(tr, target); err != nil {
		return err
	}
	return nil
}

func IsTar(path string) bool {
	return filepath.Ext(path) == ".tar" ||
		filepath.Ext(path) == ".tar.gz" ||
		filepath.Ext(path) == ".tgz"
}

func CheckTar(image string) bool {
	if strings.TrimSuffix(image, ".tar") == image {
		return false
	}
	if _, err := os.Stat(image); err != nil {
		logrus.Errorf("%s does not exist", image)
		return false
	}
	return true
}
<commit_msg>use os.Chmod over syscall.Umask<commit_after>/*
Copyright 2017 Google, Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package util

import (
	"archive/tar"
	"io"
	"os"
	"path/filepath"
	"strings"

	"github.com/sirupsen/logrus"
)

func unpackTar(tr *tar.Reader, path string) error {
	for {
		header, err := tr.Next()
		if err == io.EOF {
			// end of tar archive
			break
		}
		if err != nil {
			logrus.Error("Error getting next tar header")
			return err
		}

		if strings.Contains(header.Name, ".wh.") {
			rmPath := filepath.Join(path, header.Name)
			newName := strings.Replace(rmPath, ".wh.", "", 1)
			if err := os.Remove(rmPath); err != nil {
				logrus.Error(err)
			}
			if err = os.RemoveAll(newName); err != nil {
				logrus.Error(err)
			}
			continue
		}

		target := filepath.Join(path, header.Name)
		mode := header.FileInfo().Mode()
		switch header.Typeflag {

		// if its a dir and it doesn't exist create it
		case tar.TypeDir:
			if _, err := os.Stat(target); os.IsNotExist(err) {
				if err := os.MkdirAll(target, mode); err != nil {
					return err
				}
			} else {
				if err := os.Chmod(target, mode); err != nil {
					return err
				}
			}

		// if it's a file create it
		case tar.TypeReg:
			// It's possible for a file to be included before the directory it's in is created.
			baseDir := filepath.Dir(target)
			if _, err := os.Stat(baseDir); os.IsNotExist(err) {
				if err := os.MkdirAll(baseDir, 0755); err != nil {
					return err
				}
			}
			currFile, err := os.OpenFile(target, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode)
			if err != nil {
				logrus.Errorf("Error opening file %s", target)
				return err
			}
			// manually set permissions on file, since the default umask (022) will interfere
			err = os.Chmod(target, mode)
			if err != nil {
				logrus.Errorf("Error updating file permissions on %s", target)
				return err
			}
			_, err = io.Copy(currFile, tr)
			if err != nil {
				return err
			}
			currFile.Close()
		}

	}
	return nil
}

// UnTar takes in a path to a tar file and writes the untarred version to the provided target.
// Only untars one level, does not untar nested tars.
func UnTar(r io.Reader, target string) error {
	if _, ok := os.Stat(target); ok != nil {
		os.MkdirAll(target, 0775)
	}

	tr := tar.NewReader(r)
	if err := unpackTar(tr, target); err != nil {
		return err
	}
	return nil
}

func IsTar(path string) bool {
	return filepath.Ext(path) == ".tar" ||
		filepath.Ext(path) == ".tar.gz" ||
		filepath.Ext(path) == ".tgz"
}

func CheckTar(image string) bool {
	if strings.TrimSuffix(image, ".tar") == image {
		return false
	}
	if _, err := os.Stat(image); err != nil {
		logrus.Errorf("%s does not exist", image)
		return false
	}
	return true
}
<|endoftext|>


<commit_before>/*

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package webhook

import (
	"context"
	"errors"
	"flag"
	"fmt"
	"net/http"
	"strings"
	"time"

	opa "github.com/open-policy-agent/frameworks/constraint/pkg/client"
	"github.com/open-policy-agent/frameworks/constraint/pkg/core/templates"
	rtypes "github.com/open-policy-agent/frameworks/constraint/pkg/types"
	"github.com/open-policy-agent/gatekeeper/apis"
	"github.com/open-policy-agent/gatekeeper/apis/config/v1alpha1"
	"github.com/open-policy-agent/gatekeeper/pkg/controller/config/process"
	"github.com/open-policy-agent/gatekeeper/pkg/keys"
	"github.com/open-policy-agent/gatekeeper/pkg/target"
	"github.com/open-policy-agent/gatekeeper/pkg/util"
	admissionv1beta1 "k8s.io/api/admission/v1beta1"
	authenticationv1 "k8s.io/api/authentication/v1"
	corev1 "k8s.io/api/core/v1"
	k8serrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	k8sruntime "k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/serializer"
	"k8s.io/apimachinery/pkg/types"
	"sigs.k8s.io/controller-runtime/pkg/client"
	logf "sigs.k8s.io/controller-runtime/pkg/log"
	"sigs.k8s.io/controller-runtime/pkg/manager"
	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
)

func init() {
	AddToManagerFuncs = append(AddToManagerFuncs, AddPolicyWebhook)
	if err := apis.AddToScheme(runtimeScheme); err != nil {
		log.Error(err, "unable to add to scheme")
		panic(err)
	}
}

var log = logf.Log.WithName("webhook")

const (
	serviceAccountName = "gatekeeper-admin"
)

var (
	runtimeScheme                      = k8sruntime.NewScheme()
	codecs                             = serializer.NewCodecFactory(runtimeScheme)
	deserializer                       = codecs.UniversalDeserializer()
	disableEnforcementActionValidation = flag.Bool("disable-enforcementaction-validation", false, "disable validation of the enforcementAction field of a constraint")
	logDenies                          = flag.Bool("log-denies", false, "log detailed info on each deny")
	serviceaccount                     = fmt.Sprintf("system:serviceaccount:%s:%s", util.GetNamespace(), serviceAccountName)
	// webhookName is deprecated, set this on the manifest YAML if needed"
)

// +kubebuilder:webhook:verbs=create;update,path=/v1/admit,mutating=false,failurePolicy=ignore,groups=*,resources=*,versions=*,name=validation.gatekeeper.sh
// +kubebuilder:rbac:groups=*,resources=*,verbs=get;list;watch

// AddPolicyWebhook registers the policy webhook server with the manager
func AddPolicyWebhook(mgr manager.Manager, opa *opa.Client, processExcluder *process.Excluder) error {
	reporter, err := newStatsReporter()
	if err != nil {
		return err
	}
	wh := &admission.Webhook{
		Handler: &validationHandler{
			opa:             opa,
			client:          mgr.GetClient(),
			reader:          mgr.GetAPIReader(),
			reporter:        reporter,
			processExcluder: processExcluder,
		},
	}
	// TODO(https://github.com/open-policy-agent/gatekeeper/issues/661): remove log injection if the race condition in the cited bug is eliminated.
	// Otherwise we risk having unstable logger names for the webhook.
	if err := wh.InjectLogger(log); err != nil {
		return err
	}
	mgr.GetWebhookServer().Register("/v1/admit", wh)
	return nil
}

var _ admission.Handler = &validationHandler{}

type validationHandler struct {
	opa      *opa.Client
	client   client.Client
	reporter StatsReporter
	// reader that will be configured to use the API server
	// obtained from mgr.GetAPIReader()
	reader client.Reader
	// for testing
	injectedConfig  *v1alpha1.Config
	processExcluder *process.Excluder
}

type requestResponse string

const (
	errorResponse   requestResponse = "error"
	denyResponse    requestResponse = "deny"
	allowResponse   requestResponse = "allow"
	unknownResponse requestResponse = "unknown"
)

// Handle the validation request
func (h *validationHandler) Handle(ctx context.Context, req admission.Request) admission.Response {
	log := log.WithValues("hookType", "validation")

	var timeStart = time.Now()

	if isGkServiceAccount(req.AdmissionRequest.UserInfo) {
		return admission.ValidationResponse(true, "Gatekeeper does not self-manage")
	}

	if req.AdmissionRequest.Operation == admissionv1beta1.Delete {
		// oldObject is the existing object.
		// It is null for DELETE operations in API servers prior to v1.15.0.
		// https://github.com/kubernetes/website/pull/14671
		if req.AdmissionRequest.OldObject.Raw == nil {
			vResp := admission.ValidationResponse(false, "For admission webhooks registered for DELETE operations, please use Kubernetes v1.15.0+.")
			vResp.Result.Code = http.StatusInternalServerError
			return vResp
		}
		// For admission webhooks registered for DELETE operations on k8s built APIs or CRDs,
		// the apiserver now sends the existing object as admissionRequest.Request.OldObject to the webhook
		// object is the new object being admitted.
		// It is null for DELETE operations.
		// https://github.com/kubernetes/kubernetes/pull/76346
		req.AdmissionRequest.Object = req.AdmissionRequest.OldObject
	}

	if userErr, err := h.validateGatekeeperResources(ctx, req); err != nil {
		vResp := admission.ValidationResponse(false, err.Error())
		if vResp.Result == nil {
			vResp.Result = &metav1.Status{}
		}
		if userErr {
			vResp.Result.Code = http.StatusUnprocessableEntity
		} else {
			vResp.Result.Code = http.StatusInternalServerError
		}
		return vResp
	}

	requestResponse := unknownResponse
	defer func() {
		if h.reporter != nil {
			if err := h.reporter.ReportRequest(
				requestResponse, time.Since(timeStart)); err != nil {
				log.Error(err, "failed to report request")
			}
		}
	}()

	// namespace is excluded from webhook using config
	if h.skipExcludedNamespace(req.AdmissionRequest.Namespace) {
		requestResponse = allowResponse
		return admission.ValidationResponse(true, "Namespace is set to be ignored by Gatekeeper config")
	}

	resp, err := h.reviewRequest(ctx, req)
	if err != nil {
		log.Error(err, "error executing query")
		vResp := admission.ValidationResponse(false, err.Error())
		if vResp.Result == nil {
			vResp.Result = &metav1.Status{}
		}
		vResp.Result.Code = http.StatusInternalServerError
		requestResponse = errorResponse
		return vResp
	}

	res := resp.Results()
	msgs := h.getDenyMessages(res, req)
	if len(msgs) > 0 {
		vResp := admission.ValidationResponse(false, strings.Join(msgs, "\n"))
		if vResp.Result == nil {
			vResp.Result = &metav1.Status{}
		}
		vResp.Result.Code = http.StatusForbidden
		requestResponse = denyResponse
		return vResp
	}

	requestResponse = allowResponse
	return admission.ValidationResponse(true, "")
}

func (h *validationHandler) getDenyMessages(res []*rtypes.Result, req admission.Request) []string {
	var msgs []string
	var resourceName string
	if len(res) > 0 && *logDenies {
		resourceName = req.AdmissionRequest.Name
		if len(resourceName) == 0 && req.AdmissionRequest.Object.Raw != nil {
			// On a CREATE operation, the client may omit name and
			// rely on the server to generate the name.
			obj := &unstructured.Unstructured{}
			if _, _, err := deserializer.Decode(req.AdmissionRequest.Object.Raw, nil, obj); err == nil {
				resourceName = obj.GetName()
			}
		}
	}
	for _, r := range res {
		if r.EnforcementAction == "deny" || r.EnforcementAction == "dryrun" {
			if *logDenies {
				log.WithValues(
					"process", "admission",
					"event_type", "violation",
					"constraint_name", r.Constraint.GetName(),
					"constraint_kind", r.Constraint.GetKind(),
					"constraint_action", r.EnforcementAction,
					"resource_kind", req.AdmissionRequest.Kind.Kind,
					"resource_namespace", req.AdmissionRequest.Namespace,
					"resource_name", resourceName,
					"request_username", req.AdmissionRequest.UserInfo.Username,
				).Info("denied admission")
			}
		}
		// only deny enforcementAction should prompt deny admission response
		if r.EnforcementAction == "deny" {
			msgs = append(msgs, fmt.Sprintf("[denied by %s] %s", r.Constraint.GetName(), r.Msg))
		}
	}
	return msgs
}

func (h *validationHandler) getConfig(ctx context.Context) (*v1alpha1.Config, error) {
	if h.injectedConfig != nil {
		return h.injectedConfig, nil
	}
	if h.client == nil {
		return nil, errors.New("no client available to retrieve validation config")
	}
	cfg := &v1alpha1.Config{}
	return cfg, h.client.Get(ctx, keys.Config, cfg)
}

func isGkServiceAccount(user authenticationv1.UserInfo) bool {
	return user.Username == serviceaccount
}

// validateGatekeeperResources returns whether an issue is user error (vs internal) and any errors
// validating internal resources
func (h *validationHandler) validateGatekeeperResources(ctx context.Context, req admission.Request) (bool, error) {
	if req.AdmissionRequest.Kind.Group == "templates.gatekeeper.sh" && req.AdmissionRequest.Kind.Kind == "ConstraintTemplate" {
		return h.validateTemplate(ctx, req)
	}
	if req.AdmissionRequest.Kind.Group == "constraints.gatekeeper.sh" {
		return h.validateConstraint(ctx, req)
	}
	return false, nil
}

func (h *validationHandler) validateTemplate(ctx context.Context, req admission.Request) (bool, error) {
	templ, _, err := deserializer.Decode(req.AdmissionRequest.Object.Raw, nil, nil)
	if err != nil {
		return false, err
	}
	unversioned := &templates.ConstraintTemplate{}
	if err := runtimeScheme.Convert(templ, unversioned, nil); err != nil {
		return false, err
	}
	if _, err := h.opa.CreateCRD(ctx, unversioned); err != nil {
		return true, err
	}
	return false, nil
}

func (h *validationHandler) validateConstraint(ctx context.Context, req admission.Request) (bool, error) {
	obj := &unstructured.Unstructured{}
	if _, _, err := deserializer.Decode(req.AdmissionRequest.Object.Raw, nil, obj); err != nil {
		return false, err
	}
	if err := h.opa.ValidateConstraint(ctx, obj); err != nil {
		return true, err
	}

	enforcementActionString, found, err := unstructured.NestedString(obj.Object, "spec", "enforcementAction")
	if err != nil {
		return false, err
	}
	enforcementAction := util.EnforcementAction(enforcementActionString)
	if found && enforcementAction != "" {
		if !*disableEnforcementActionValidation {
			err = util.ValidateEnforcementAction(enforcementAction)
			if err != nil {
				return false, err
			}
		}
	} else {
		return true, nil
	}
	return false, nil
}

// traceSwitch returns true if a request should be traced
func (h *validationHandler) reviewRequest(ctx context.Context, req admission.Request) (*rtypes.Responses, error) {
	cfg, _ := h.getConfig(ctx)
	traceEnabled := false
	dump := false
	for _, trace := range cfg.Spec.Validation.Traces {
		if trace.User != req.AdmissionRequest.UserInfo.Username {
			continue
		}
		gvk := v1alpha1.GVK{
			Group:   req.AdmissionRequest.Kind.Group,
			Version: req.AdmissionRequest.Kind.Version,
			Kind:    req.AdmissionRequest.Kind.Kind,
		}
		if gvk == trace.Kind {
			traceEnabled = true
			if strings.EqualFold(trace.Dump, "All") {
				dump = true
			}
		}
	}
	review := &target.AugmentedReview{AdmissionRequest: &req.AdmissionRequest}
	if req.AdmissionRequest.Namespace != "" {
		ns := &corev1.Namespace{}
		if err := h.client.Get(ctx, types.NamespacedName{Name: req.AdmissionRequest.Namespace}, ns); err != nil {
			if !k8serrors.IsNotFound(err) {
				return nil, err
			}
			// bypass cached client and ask api-server directly
			err = h.reader.Get(ctx, types.NamespacedName{Name: req.AdmissionRequest.Namespace}, ns)
			if err != nil {
				return nil, err
			}
		}
		review.Namespace = ns
	}

	resp, err := h.opa.Review(ctx, review, opa.Tracing(traceEnabled))
	if traceEnabled {
		log.Info(resp.TraceDump())
	}
	if dump {
		dump, err := h.opa.Dump(ctx)
		if err != nil {
			log.Error(err, "dump error")
		} else {
			log.Info(dump)
		}
	}
	return resp, err
}

func (h *validationHandler) skipExcludedNamespace(namespace string) bool {
	return h.processExcluder.IsNamespaceExcluded(process.Webhook, namespace)
}
<commit_msg>extract tracing logic into a method (#726)<commit_after>/*

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package webhook

import (
	"context"
	"errors"
	"flag"
	"fmt"
	"net/http"
	"strings"
	"time"

	opa "github.com/open-policy-agent/frameworks/constraint/pkg/client"
	"github.com/open-policy-agent/frameworks/constraint/pkg/core/templates"
	rtypes "github.com/open-policy-agent/frameworks/constraint/pkg/types"
	"github.com/open-policy-agent/gatekeeper/apis"
	"github.com/open-policy-agent/gatekeeper/apis/config/v1alpha1"
	"github.com/open-policy-agent/gatekeeper/pkg/controller/config/process"
	"github.com/open-policy-agent/gatekeeper/pkg/keys"
	"github.com/open-policy-agent/gatekeeper/pkg/target"
	"github.com/open-policy-agent/gatekeeper/pkg/util"
	admissionv1beta1 "k8s.io/api/admission/v1beta1"
	authenticationv1 "k8s.io/api/authentication/v1"
	corev1 "k8s.io/api/core/v1"
	k8serrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	k8sruntime "k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/serializer"
	"k8s.io/apimachinery/pkg/types"
	"sigs.k8s.io/controller-runtime/pkg/client"
	logf "sigs.k8s.io/controller-runtime/pkg/log"
	"sigs.k8s.io/controller-runtime/pkg/manager"
	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
)

func init() {
	AddToManagerFuncs = append(AddToManagerFuncs, AddPolicyWebhook)
	if err := apis.AddToScheme(runtimeScheme); err != nil {
		log.Error(err, "unable to add to scheme")
		panic(err)
	}
}

var log = logf.Log.WithName("webhook")

const (
	serviceAccountName = "gatekeeper-admin"
)

var (
	runtimeScheme                      = k8sruntime.NewScheme()
	codecs                             = serializer.NewCodecFactory(runtimeScheme)
	deserializer                       = codecs.UniversalDeserializer()
	disableEnforcementActionValidation = flag.Bool("disable-enforcementaction-validation", false, "disable validation of the enforcementAction field of a constraint")
	logDenies                          = flag.Bool("log-denies", false, "log detailed info on each deny")
	serviceaccount                     = fmt.Sprintf("system:serviceaccount:%s:%s", util.GetNamespace(), serviceAccountName)
	// webhookName is deprecated, set this on the manifest YAML if needed"
)

// +kubebuilder:webhook:verbs=create;update,path=/v1/admit,mutating=false,failurePolicy=ignore,groups=*,resources=*,versions=*,name=validation.gatekeeper.sh
// +kubebuilder:rbac:groups=*,resources=*,verbs=get;list;watch

// AddPolicyWebhook registers the policy webhook server with the manager
func AddPolicyWebhook(mgr manager.Manager, opa *opa.Client, processExcluder *process.Excluder) error {
	reporter, err := newStatsReporter()
	if err != nil {
		return err
	}
	wh := &admission.Webhook{
		Handler: &validationHandler{
			opa:             opa,
			client:          mgr.GetClient(),
			reader:          mgr.GetAPIReader(),
			reporter:        reporter,
			processExcluder: processExcluder,
		},
	}
	// TODO(https://github.com/open-policy-agent/gatekeeper/issues/661): remove log injection if the race condition in the cited bug is eliminated.
	// Otherwise we risk having unstable logger names for the webhook.
	if err := wh.InjectLogger(log); err != nil {
		return err
	}
	mgr.GetWebhookServer().Register("/v1/admit", wh)
	return nil
}

var _ admission.Handler = &validationHandler{}

type validationHandler struct {
	opa      *opa.Client
	client   client.Client
	reporter StatsReporter
	// reader that will be configured to use the API server
	// obtained from mgr.GetAPIReader()
	reader client.Reader
	// for testing
	injectedConfig  *v1alpha1.Config
	processExcluder *process.Excluder
}

type requestResponse string

const (
	errorResponse   requestResponse = "error"
	denyResponse    requestResponse = "deny"
	allowResponse   requestResponse = "allow"
	unknownResponse requestResponse = "unknown"
)

// Handle the validation request
func (h *validationHandler) Handle(ctx context.Context, req admission.Request) admission.Response {
	log := log.WithValues("hookType", "validation")

	var timeStart = time.Now()

	if isGkServiceAccount(req.AdmissionRequest.UserInfo) {
		return admission.ValidationResponse(true, "Gatekeeper does not self-manage")
	}

	if req.AdmissionRequest.Operation == admissionv1beta1.Delete {
		// oldObject is the existing object.
		// It is null for DELETE operations in API servers prior to v1.15.0.
		// https://github.com/kubernetes/website/pull/14671
		if req.AdmissionRequest.OldObject.Raw == nil {
			vResp := admission.ValidationResponse(false, "For admission webhooks registered for DELETE operations, please use Kubernetes v1.15.0+.")
			vResp.Result.Code = http.StatusInternalServerError
			return vResp
		}
		// For admission webhooks registered for DELETE operations on k8s built APIs or CRDs,
		// the apiserver now sends the existing object as admissionRequest.Request.OldObject to the webhook
		// object is the new object being admitted.
		// It is null for DELETE operations.
		// https://github.com/kubernetes/kubernetes/pull/76346
		req.AdmissionRequest.Object = req.AdmissionRequest.OldObject
	}

	if userErr, err := h.validateGatekeeperResources(ctx, req); err != nil {
		vResp := admission.ValidationResponse(false, err.Error())
		if vResp.Result == nil {
			vResp.Result = &metav1.Status{}
		}
		if userErr {
			vResp.Result.Code = http.StatusUnprocessableEntity
		} else {
			vResp.Result.Code = http.StatusInternalServerError
		}
		return vResp
	}

	requestResponse := unknownResponse
	defer func() {
		if h.reporter != nil {
			if err := h.reporter.ReportRequest(
				requestResponse, time.Since(timeStart)); err != nil {
				log.Error(err, "failed to report request")
			}
		}
	}()

	// namespace is excluded from webhook using config
	if h.skipExcludedNamespace(req.AdmissionRequest.Namespace) {
		requestResponse = allowResponse
		return admission.ValidationResponse(true, "Namespace is set to be ignored by Gatekeeper config")
	}

	resp, err := h.reviewRequest(ctx, req)
	if err != nil {
		log.Error(err, "error executing query")
		vResp := admission.ValidationResponse(false, err.Error())
		if vResp.Result == nil {
			vResp.Result = &metav1.Status{}
		}
		vResp.Result.Code = http.StatusInternalServerError
		requestResponse = errorResponse
		return vResp
	}

	res := resp.Results()
	msgs := h.getDenyMessages(res, req)
	if len(msgs) > 0 {
		vResp := admission.ValidationResponse(false, strings.Join(msgs, "\n"))
		if vResp.Result == nil {
			vResp.Result = &metav1.Status{}
		}
		vResp.Result.Code = http.StatusForbidden
		requestResponse = denyResponse
		return vResp
	}

	requestResponse = allowResponse
	return admission.ValidationResponse(true, "")
}

func (h *validationHandler) getDenyMessages(res []*rtypes.Result, req admission.Request) []string {
	var msgs []string
	var resourceName string
	if len(res) > 0 && *logDenies {
		resourceName = req.AdmissionRequest.Name
		if len(resourceName) == 0 && req.AdmissionRequest.Object.Raw != nil {
			// On a CREATE operation, the client may omit name and
			// rely on the server to generate the name.
			obj := &unstructured.Unstructured{}
			if _, _, err := deserializer.Decode(req.AdmissionRequest.Object.Raw, nil, obj); err == nil {
				resourceName = obj.GetName()
			}
		}
	}
	for _, r := range res {
		if r.EnforcementAction == "deny" || r.EnforcementAction == "dryrun" {
			if *logDenies {
				log.WithValues(
					"process", "admission",
					"event_type", "violation",
					"constraint_name", r.Constraint.GetName(),
					"constraint_kind", r.Constraint.GetKind(),
					"constraint_action", r.EnforcementAction,
					"resource_kind", req.AdmissionRequest.Kind.Kind,
					"resource_namespace", req.AdmissionRequest.Namespace,
					"resource_name", resourceName,
					"request_username", req.AdmissionRequest.UserInfo.Username,
				).Info("denied admission")
			}
		}
		// only deny enforcementAction should prompt deny admission response
		if r.EnforcementAction == "deny" {
			msgs = append(msgs, fmt.Sprintf("[denied by %s] %s", r.Constraint.GetName(), r.Msg))
		}
	}
	return msgs
}

func (h *validationHandler) getConfig(ctx context.Context) (*v1alpha1.Config, error) {
	if h.injectedConfig != nil {
		return h.injectedConfig, nil
	}
	if h.client == nil {
		return nil, errors.New("no client available to retrieve validation config")
	}
	cfg := &v1alpha1.Config{}
	return cfg, h.client.Get(ctx, keys.Config, cfg)
}

func isGkServiceAccount(user authenticationv1.UserInfo) bool {
	return user.Username == serviceaccount
}

// validateGatekeeperResources returns whether an issue is user error (vs internal) and any errors
// validating internal resources
func (h *validationHandler) validateGatekeeperResources(ctx context.Context, req admission.Request) (bool, error) {
	if req.AdmissionRequest.Kind.Group == "templates.gatekeeper.sh" && req.AdmissionRequest.Kind.Kind == "ConstraintTemplate" {
		return h.validateTemplate(ctx, req)
	}
	if req.AdmissionRequest.Kind.Group == "constraints.gatekeeper.sh" {
		return h.validateConstraint(ctx, req)
	}
	return false, nil
}

func (h *validationHandler) validateTemplate(ctx context.Context, req admission.Request) (bool, error) {
	templ, _, err := deserializer.Decode(req.AdmissionRequest.Object.Raw, nil, nil)
	if err != nil {
		return false, err
	}
	unversioned := &templates.ConstraintTemplate{}
	if err := runtimeScheme.Convert(templ, unversioned, nil); err != nil {
		return false, err
	}
	if _, err := h.opa.CreateCRD(ctx, unversioned); err != nil {
		return true, err
	}
	return false, nil
}

func (h *validationHandler) validateConstraint(ctx context.Context, req admission.Request) (bool, error) {
	obj := &unstructured.Unstructured{}
	if _, _, err := deserializer.Decode(req.AdmissionRequest.Object.Raw, nil, obj); err != nil {
		return false, err
	}
	if err := h.opa.ValidateConstraint(ctx, obj); err != nil {
		return true, err
	}

	enforcementActionString, found, err := unstructured.NestedString(obj.Object, "spec", "enforcementAction")
	if err != nil {
		return false, err
	}
	enforcementAction := util.EnforcementAction(enforcementActionString)
	if found && enforcementAction != "" {
		if !*disableEnforcementActionValidation {
			err = util.ValidateEnforcementAction(enforcementAction)
			if err != nil {
				return false, err
			}
		}
	} else {
		return true, nil
	}
	return false, nil
}

// traceSwitch returns true if a request should be traced
func (h *validationHandler) reviewRequest(ctx context.Context, req admission.Request) (*rtypes.Responses, error) {
	trace, dump := h.tracingLevel(ctx, req)
	review := &target.AugmentedReview{AdmissionRequest: &req.AdmissionRequest}
	if req.AdmissionRequest.Namespace != "" {
		ns := &corev1.Namespace{}
		if err := h.client.Get(ctx, types.NamespacedName{Name: req.AdmissionRequest.Namespace}, ns); err != nil {
			if !k8serrors.IsNotFound(err) {
				return nil, err
			}
			// bypass cached client and ask api-server directly
			err = h.reader.Get(ctx, types.NamespacedName{Name: req.AdmissionRequest.Namespace}, ns)
			if err != nil {
				return nil, err
			}
		}
		review.Namespace = ns
	}

	resp, err := h.opa.Review(ctx, review, opa.Tracing(trace))
	if trace {
		log.Info(resp.TraceDump())
	}
	if dump {
		dump, err := h.opa.Dump(ctx)
		if err != nil {
			log.Error(err, "dump error")
		} else {
			log.Info(dump)
		}
	}
	return resp, err
}

func (h *validationHandler) tracingLevel(ctx context.Context, req admission.Request) (bool, bool) {
	cfg, _ := h.getConfig(ctx)
	traceEnabled := false
	dump := false
	for _, trace := range cfg.Spec.Validation.Traces {
		if trace.User != req.AdmissionRequest.UserInfo.Username {
			continue
		}
		gvk := v1alpha1.GVK{
			Group:   req.AdmissionRequest.Kind.Group,
			Version: req.AdmissionRequest.Kind.Version,
			Kind:    req.AdmissionRequest.Kind.Kind,
		}
		if gvk == trace.Kind {
			traceEnabled = true
			if strings.EqualFold(trace.Dump, "All") {
				dump = true
			}
		}
	}
	return traceEnabled, dump
}

func (h *validationHandler) skipExcludedNamespace(namespace string) bool {
	return h.processExcluder.IsNamespaceExcluded(process.Webhook, namespace)
}
<|endoftext|>


<commit_before>package registry

import (
	"net/http"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestBasicAuth(t *testing.T) {
	username := "foo"
	password := "bar"
	auth := BasicAuth{username, password}

	req, _ := http.NewRequest("GET", "http://foo.com/v1", nil)
	auth.ApplyAuthentication(req)

	u, p, _ := req.BasicAuth()
	assert.Equal(t, username, u)
	assert.Equal(t, password, p)
}

func TestTokenAuth(t *testing.T) {
	host := "bar.com"
	token := "token xyz"
	auth := TokenAuth{Host: host, Token: token}

	req, _ := http.NewRequest("GET", "http://foo.com/v1", nil)
	auth.ApplyAuthentication(req)

	assert.Equal(t, "Token "+token, req.Header.Get("Authorization"))
	assert.Equal(t, host, req.Host)
	assert.Equal(t, host, req.URL.Host)
}
<commit_msg>Update auth_test.go<commit_after>package registry

import (
	"net/http"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestBasicAuth(t *testing.T) {
	username := "foo"
	password := "bar"
	auth := BasicAuth{username, password}

	req, _ := http.NewRequest("GET", "http://foo.com/v1", nil)
	auth.ApplyAuthentication(req)

	u, p, _ := req.BasicAuth()
	assert.Equal(t, username, up)
	assert.Equal(t, password, p)
}

func TestTokenAuth(t *testing.T) {
	host := "bar.com"
	token := "token xyz"
	auth := TokenAuth{Host: host, Token: token}

	req, _ := http.NewRequest("GET", "http://foo.com/v1", nil)
	auth.ApplyAuthentication(req)

	assert.Equal(t, "Token "+token, req.Header.Get("Authorization"))
	assert.Equal(t, host, req.Host)
	assert.Equal(t, host, req.URL.Host)
}
<|endoftext|>


<commit_before>package harmonizer_test

import (
	"errors"
	"os"
	"time"

	"code.cloudfoundry.org/clock/fakeclock"
	mfakes "code.cloudfoundry.org/diego-logging-client/testhelpers"
	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/lager/lagertest"
	"code.cloudfoundry.org/operationq"
	"code.cloudfoundry.org/operationq/fake_operationq"
	"code.cloudfoundry.org/rep/evacuation/evacuation_context"
	"code.cloudfoundry.org/rep/generator/fake_generator"
	"code.cloudfoundry.org/rep/harmonizer"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/tedsuo/ifrit"
)

var _ = Describe("Bulker", func() {
	var (
		logger                 *lagertest.TestLogger
		pollInterval           time.Duration
		evacuationPollInterval time.Duration
		fakeClock              *fakeclock.FakeClock
		fakeGenerator          *fake_generator.FakeGenerator
		fakeQueue              *fake_operationq.FakeQueue
		evacuatable            evacuation_context.Evacuatable
		evacuationNotifier     evacuation_context.EvacuationNotifier
		fakeMetronClient       *mfakes.FakeIngressClient

		bulker  *harmonizer.Bulker
		process ifrit.Process
	)

	BeforeEach(func() {
		logger = lagertest.NewTestLogger("test")
		pollInterval = 30 * time.Second
		evacuationPollInterval = 10 * time.Second
		fakeClock = fakeclock.NewFakeClock(time.Now())
		fakeGenerator = new(fake_generator.FakeGenerator)
		fakeQueue = new(fake_operationq.FakeQueue)
		fakeMetronClient = new(mfakes.FakeIngressClient)

		evacuatable, _, evacuationNotifier = evacuation_context.New()

		bulker = harmonizer.NewBulker(
			logger,
			pollInterval,
			evacuationPollInterval,
			evacuationNotifier,
			fakeClock,
			fakeGenerator,
			fakeQueue,
			fakeMetronClient,
		)
	})

	JustBeforeEach(func() {
		process = ifrit.Invoke(bulker)
		Eventually(fakeClock.WatcherCount).Should(Equal(1))
	})

	AfterEach(func() {
		process.Signal(os.Interrupt)
		Eventually(process.Wait()).Should(Receive())
	})

	itPerformsBatchOperations := func(expectedQueueLength int) {
		Context("when generating the batch operations succeeds", func() {
			var (
				operation1 *fake_operationq.FakeOperation
				operation2 *fake_operationq.FakeOperation
			)

			BeforeEach(func() {
				operation1 = new(fake_operationq.FakeOperation)
				operation2 = new(fake_operationq.FakeOperation)

				fakeGenerator.BatchOperationsStub = func(lager.Logger) (map[string]operationq.Operation, error) {
					fakeClock.Increment(10 * time.Second)
					return map[string]operationq.Operation{"guid1": operation1, "guid2": operation2}, nil
				}
			})

			It("pushes them onto the queue", func() {
				Eventually(fakeQueue.PushCallCount).Should(Equal(expectedQueueLength))

				enqueuedOperations := make([]operationq.Operation, 0, 2)
				enqueuedOperations = append(enqueuedOperations, fakeQueue.PushArgsForCall(expectedQueueLength-2))
				enqueuedOperations = append(enqueuedOperations, fakeQueue.PushArgsForCall(expectedQueueLength-1))

				Expect(enqueuedOperations).To(ConsistOf(operation1, operation2))
			})

			It("emits the duration it took to generate the batch operations", func() {
				Eventually(fakeQueue.PushCallCount).Should(Equal(expectedQueueLength))

				name, value := fakeMetronClient.SendDurationArgsForCall(0)
				Expect(name).To(Equal("RepBulkSyncDuration"))
				Expect(value).To(BeNumerically("==", 10*time.Second))
			})
		})

		Context("when generating the batch operations fails", func() {
			disaster := errors.New("nope")

			BeforeEach(func() {
				fakeGenerator.BatchOperationsReturns(nil, disaster)
			})

			It("logs the error", func() {
				Eventually(logger).Should(gbytes.Say("failed-to-generate-operations"))
				Eventually(logger).Should(gbytes.Say("nope"))
			})
		})
	}

	Context("when the poll interval elapses", func() {
		JustBeforeEach(func() {
			fakeClock.WaitForWatcherAndIncrement(pollInterval)
		})

		itPerformsBatchOperations(2)

		Context("and elapses again", func() {
			JustBeforeEach(func() {
				fakeClock.WaitForWatcherAndIncrement(pollInterval)
			})

			itPerformsBatchOperations(4)
		})
	})

	Context("when the poll interval has not elapsed", func() {
		JustBeforeEach(func() {
			fakeClock.WaitForWatcherAndIncrement(pollInterval - 1)
		})

		It("does not fetch batch operations", func() {
			Consistently(fakeGenerator.BatchOperationsCallCount).Should(BeZero())
		})
	})

	Context("when evacuation starts", func() {
		BeforeEach(func() {
			evacuatable.Evacuate()
		})

		itPerformsBatchOperations(2)

		It("batches operations only once", func() {
			Eventually(fakeGenerator.BatchOperationsCallCount).Should(Equal(1))
			Consistently(fakeGenerator.BatchOperationsCallCount).Should(Equal(1))
		})

		Context("when the evacuation interval elapses", func() {
			It("batches operations again", func() {
				Eventually(fakeGenerator.BatchOperationsCallCount).Should(Equal(1))
				fakeClock.Increment(evacuationPollInterval + time.Second)
				Eventually(fakeGenerator.BatchOperationsCallCount).Should(Equal(2))
				Consistently(fakeGenerator.BatchOperationsCallCount).Should(Equal(2))
			})
		})
	})
})
<commit_msg>Update SendDuration signature to use EmitGaugeOption<commit_after>package harmonizer_test

import (
	"errors"
	"os"
	"time"

	"code.cloudfoundry.org/clock/fakeclock"
	mfakes "code.cloudfoundry.org/diego-logging-client/testhelpers"
	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/lager/lagertest"
	"code.cloudfoundry.org/operationq"
	"code.cloudfoundry.org/operationq/fake_operationq"
	"code.cloudfoundry.org/rep/evacuation/evacuation_context"
	"code.cloudfoundry.org/rep/generator/fake_generator"
	"code.cloudfoundry.org/rep/harmonizer"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/tedsuo/ifrit"
)

var _ = Describe("Bulker", func() {
	var (
		logger                 *lagertest.TestLogger
		pollInterval           time.Duration
		evacuationPollInterval time.Duration
		fakeClock              *fakeclock.FakeClock
		fakeGenerator          *fake_generator.FakeGenerator
		fakeQueue              *fake_operationq.FakeQueue
		evacuatable            evacuation_context.Evacuatable
		evacuationNotifier     evacuation_context.EvacuationNotifier
		fakeMetronClient       *mfakes.FakeIngressClient

		bulker  *harmonizer.Bulker
		process ifrit.Process
	)

	BeforeEach(func() {
		logger = lagertest.NewTestLogger("test")
		pollInterval = 30 * time.Second
		evacuationPollInterval = 10 * time.Second
		fakeClock = fakeclock.NewFakeClock(time.Now())
		fakeGenerator = new(fake_generator.FakeGenerator)
		fakeQueue = new(fake_operationq.FakeQueue)
		fakeMetronClient = new(mfakes.FakeIngressClient)

		evacuatable, _, evacuationNotifier = evacuation_context.New()

		bulker = harmonizer.NewBulker(
			logger,
			pollInterval,
			evacuationPollInterval,
			evacuationNotifier,
			fakeClock,
			fakeGenerator,
			fakeQueue,
			fakeMetronClient,
		)
	})

	JustBeforeEach(func() {
		process = ifrit.Invoke(bulker)
		Eventually(fakeClock.WatcherCount).Should(Equal(1))
	})

	AfterEach(func() {
		process.Signal(os.Interrupt)
		Eventually(process.Wait()).Should(Receive())
	})

	itPerformsBatchOperations := func(expectedQueueLength int) {
		Context("when generating the batch operations succeeds", func() {
			var (
				operation1 *fake_operationq.FakeOperation
				operation2 *fake_operationq.FakeOperation
			)

			BeforeEach(func() {
				operation1 = new(fake_operationq.FakeOperation)
				operation2 = new(fake_operationq.FakeOperation)

				fakeGenerator.BatchOperationsStub = func(lager.Logger) (map[string]operationq.Operation, error) {
					fakeClock.Increment(10 * time.Second)
					return map[string]operationq.Operation{"guid1": operation1, "guid2": operation2}, nil
				}
			})

			It("pushes them onto the queue", func() {
				Eventually(fakeQueue.PushCallCount).Should(Equal(expectedQueueLength))

				enqueuedOperations := make([]operationq.Operation, 0, 2)
				enqueuedOperations = append(enqueuedOperations, fakeQueue.PushArgsForCall(expectedQueueLength-2))
				enqueuedOperations = append(enqueuedOperations, fakeQueue.PushArgsForCall(expectedQueueLength-1))

				Expect(enqueuedOperations).To(ConsistOf(operation1, operation2))
			})

			It("emits the duration it took to generate the batch operations", func() {
				Eventually(fakeQueue.PushCallCount).Should(Equal(expectedQueueLength))

				name, value, _ := fakeMetronClient.SendDurationArgsForCall(0)
				Expect(name).To(Equal("RepBulkSyncDuration"))
				Expect(value).To(BeNumerically("==", 10*time.Second))
			})
		})

		Context("when generating the batch operations fails", func() {
			disaster := errors.New("nope")

			BeforeEach(func() {
				fakeGenerator.BatchOperationsReturns(nil, disaster)
			})

			It("logs the error", func() {
				Eventually(logger).Should(gbytes.Say("failed-to-generate-operations"))
				Eventually(logger).Should(gbytes.Say("nope"))
			})
		})
	}

	Context("when the poll interval elapses", func() {
		JustBeforeEach(func() {
			fakeClock.WaitForWatcherAndIncrement(pollInterval)
		})

		itPerformsBatchOperations(2)

		Context("and elapses again", func() {
			JustBeforeEach(func() {
				fakeClock.WaitForWatcherAndIncrement(pollInterval)
			})

			itPerformsBatchOperations(4)
		})
	})

	Context("when the poll interval has not elapsed", func() {
		JustBeforeEach(func() {
			fakeClock.WaitForWatcherAndIncrement(pollInterval - 1)
		})

		It("does not fetch batch operations", func() {
			Consistently(fakeGenerator.BatchOperationsCallCount).Should(BeZero())
		})
	})

	Context("when evacuation starts", func() {
		BeforeEach(func() {
			evacuatable.Evacuate()
		})

		itPerformsBatchOperations(2)

		It("batches operations only once", func() {
			Eventually(fakeGenerator.BatchOperationsCallCount).Should(Equal(1))
			Consistently(fakeGenerator.BatchOperationsCallCount).Should(Equal(1))
		})

		Context("when the evacuation interval elapses", func() {
			It("batches operations again", func() {
				Eventually(fakeGenerator.BatchOperationsCallCount).Should(Equal(1))
				fakeClock.Increment(evacuationPollInterval + time.Second)
				Eventually(fakeGenerator.BatchOperationsCallCount).Should(Equal(2))
				Consistently(fakeGenerator.BatchOperationsCallCount).Should(Equal(2))
			})
		})
	})
})
<|endoftext|>


<commit_before>package report

import(
	"fmt"
	"strings"
	fdb "github.com/skypies/flightdb2"
)

func init() {
	HandleReport(".list", ListReporter, "List flights meeting restrictions")
	TrackSpec(".list", []string{"fr24", "ADSB", "MLAT", "FA", "FOIA",})
}

var(
	ListReporterHeaders = []string{
		"ID", "FLIGHTNUMBER","ORIGIN","DESTINATION","TAGS",
		"YEAR(PST)", "MONTH(PST)","DAY(PST)","TIME(PST)",
		"ALTITUDE(FEET)","GROUNDSPEED(KNOTS)",
	}
)

func ListReporter(r *Report, f *fdb.Flight, intersections []fdb.TrackIntersection) (FlightReportOutcome, error) {
	s,e := f.Times()
	dur := e.Sub(s)

	waypoints := strings.Join(f.WaypointList(), ",")
	if len(waypoints) > 35 {
		waypoints = waypoints[:35] + "..."
	}

	htmlrow := []string{
		r.Links(f),
		"" + f.FullString() + "",
		fmt.Sprintf("[%s]", strings.Join(f.TagList(), ",")),
		fmt.Sprintf("{%s}", waypoints),
		fmt.Sprintf("+%s", dur),
	}

	textrow := []string{
		f.IdentString(), f.IataFlight(), f.Origin, f.Destination, strings.Join(f.TagList(), " "),
	}

	// Generate market distribution for matches
	if f.Origin == "SFO" || f.Destination == "SFO" {
		r.I[fmt.Sprintf("[F] %s:%s", f.Origin, f.Destination)]++
	}

	bucketsAdded := false
	addTrackpointIntersection := func(tp fdb.Trackpoint) {
		textrow = append(textrow, []string{
			tp.TimestampUTC.Format("2006"),
			tp.TimestampUTC.Format("01"),
			tp.TimestampUTC.Format("02"),
			tp.TimestampUTC.Format("15:04:05"),
			fmt.Sprintf("%.0f", tp.Altitude),
			fmt.Sprintf("%.0f", tp.GroundSpeed),
		}...)

		if !bucketsAdded {
			r.I[fmt.Sprintf("[D] %s", alt2bkt(tp.Altitude))]++
			r.I[fmt.Sprintf("[E] %s", speed2bkt(tp.GroundSpeed))]++
			bucketsAdded = true // Only once per flight
		}
	}

	if len(intersections) > 0 {
		addTrackpointIntersection(intersections[0].Start)
	} else if len(r.HackWaypoints) > 0 {
		for _,wpName := range r.HackWaypoints {
			if trackName,i := f.AtWaypoint(wpName); trackName != "" {
				track := f.Tracks[trackName]
				// for interpolation, see DistAlongLine - or is it busted ?
				addTrackpointIntersection((*track)[i])
			}
		}
	}

	for _,ti := range intersections {
		htmlrow = append(htmlrow, ti.RowHTML()...)
	}

	r.AddRow(&htmlrow, &textrow)
	r.SetHeaders(ListReporterHeaders)

	return Accepted, nil
}

// Helpers
func alt2bkt(f float64) string {
	g := float64(int((f+500)/1000.0))  // Round to nearest thousand: 11499 -> 11, 11501 -> 12
	return fmt.Sprintf("altband: %05.0f-%05.0f", g*1000-500, g*1000+499)
}

func speed2bkt(f float64) string {
	g := float64(int((f+10)/20.0))  // Round to nearest 20
	return fmt.Sprintf("speedband: %03.0f-%03.0f", g*20-10, g*20+9)
}
<commit_msg>Fix timezone bug<commit_after>package report

import(
	"fmt"
	"strings"

	"github.com/skypies/util/date"

	fdb "github.com/skypies/flightdb2"
)

func init() {
	HandleReport(".list", ListReporter, "List flights meeting restrictions")
	TrackSpec(".list", []string{"fr24", "ADSB", "MLAT", "FA", "FOIA",})
}

var(
	ListReporterHeaders = []string{
		"ID", "FLIGHTNUMBER","ORIGIN","DESTINATION","TAGS",
		"YEAR(PST)", "MONTH(PST)","DAY(PST)","TIME(PST)",
		"ALTITUDE(FEET)","GROUNDSPEED(KNOTS)",
	}
)

func ListReporter(r *Report, f *fdb.Flight, intersections []fdb.TrackIntersection) (FlightReportOutcome, error) {
	s,e := f.Times()
	dur := e.Sub(s)

	waypoints := strings.Join(f.WaypointList(), ",")
	if len(waypoints) > 35 {
		waypoints = waypoints[:35] + "..."
	}

	htmlrow := []string{
		r.Links(f),
		"" + f.FullString() + "",
		fmt.Sprintf("[%s]", strings.Join(f.TagList(), ",")),
		fmt.Sprintf("{%s}", waypoints),
		fmt.Sprintf("+%s", dur),
	}

	textrow := []string{
		f.IdentString(), f.IataFlight(), f.Origin, f.Destination, strings.Join(f.TagList(), " "),
	}

	// Generate market distribution for matches
	if f.Origin == "SFO" || f.Destination == "SFO" {
		r.I[fmt.Sprintf("[F] %s:%s", f.Origin, f.Destination)]++
	}

	bucketsAdded := false
	addTrackpointIntersection := func(tp fdb.Trackpoint) {
		tpInPT := date.InPdt(tp.TimestampUTC)
		textrow = append(textrow, []string{
			tpInPT.Format("2006"),
			tpInPT.Format("01"),
			tpInPT.Format("02"),
			tpInPT.Format("15:04:05"),
			fmt.Sprintf("%.0f", tp.Altitude),
			fmt.Sprintf("%.0f", tp.GroundSpeed),
		}...)

		if !bucketsAdded {
			r.I[fmt.Sprintf("[D] %s", alt2bkt(tp.Altitude))]++
			r.I[fmt.Sprintf("[E] %s", speed2bkt(tp.GroundSpeed))]++
			bucketsAdded = true // Only once per flight
		}
	}

	if len(intersections) > 0 {
		addTrackpointIntersection(intersections[0].Start)
	} else if len(r.HackWaypoints) > 0 {
		for _,wpName := range r.HackWaypoints {
			if trackName,i := f.AtWaypoint(wpName); trackName != "" {
				track := f.Tracks[trackName]
				// for interpolation, see DistAlongLine - or is it busted ?
				addTrackpointIntersection((*track)[i])
			}
		}
	}

	for _,ti := range intersections {
		htmlrow = append(htmlrow, ti.RowHTML()...)
	}

	r.AddRow(&htmlrow, &textrow)
	r.SetHeaders(ListReporterHeaders)

	return Accepted, nil
}

// Helpers
func alt2bkt(f float64) string {
	g := float64(int((f+500)/1000.0))  // Round to nearest thousand: 11499 -> 11, 11501 -> 12
	return fmt.Sprintf("altband: %05.0f-%05.0f", g*1000-500, g*1000+499)
}

func speed2bkt(f float64) string {
	g := float64(int((f+10)/20.0))  // Round to nearest 20
	return fmt.Sprintf("speedband: %03.0f-%03.0f", g*20-10, g*20+9)
}
<|endoftext|>


<commit_before>package sftp

import (
	"io"
	"os"
)

// Interfaces are differentiated based on required returned values.
// All input arguments are to be pulled from Request (the only arg).

// FileReader should return an io.Reader for the filepath
type FileReader interface {
	Fileread(*Request) (io.ReaderAt, error)
}

// FileWriter should return an io.Writer for the filepath
type FileWriter interface {
	Filewrite(*Request) (io.WriterAt, error)
}

// FileCmder should return an error (rename, remove, setstate, etc.)
type FileCmder interface {
	Filecmd(*Request) error
}

// FileLister should return file info interface and errors (readdir, stat)
type FileLister interface {
	Filelist(*Request) (ListerAt, error)
}

// ListerAt does for file lists what io.ReaderAt does for files.
// ListAt should return the number of entries copied and an io.EOF
// error if at end of list. This is testable by comparing how many you
// copied to how many could be copied (eg. n < len(ls) below).
// The copy() builtin is best for the copying.
type ListerAt interface {
	ListAt([]os.FileInfo, int64) (int, error)
}
<commit_msg>handler type documenation improvements<commit_after>package sftp

import (
	"io"
	"os"
)

// Interfaces are differentiated based on required returned values.
// All input arguments are to be pulled from Request (the only arg).

// FileReader should return an io.ReaderAt for the filepath
// Note in cases of an error, the error text will be sent to the client.
type FileReader interface {
	Fileread(*Request) (io.ReaderAt, error)
}

// FileWriter should return an io.WriterAt for the filepath.
//
// The request server code will call Close() on the returned io.WriterAt
// ojbect if an io.Closer type assertion succeeds.
// Note in cases of an error, the error text will be sent to the client.
type FileWriter interface {
	Filewrite(*Request) (io.WriterAt, error)
}

// FileCmder should return an error
// Note in cases of an error, the error text will be sent to the client.
type FileCmder interface {
	Filecmd(*Request) error
}

// FileLister should return an object that fulfils the ListerAt interface
// Note in cases of an error, the error text will be sent to the client.
type FileLister interface {
	Filelist(*Request) (ListerAt, error)
}

// ListerAt does for file lists what io.ReaderAt does for files.
// ListAt should return the number of entries copied and an io.EOF
// error if at end of list. This is testable by comparing how many you
// copied to how many could be copied (eg. n < len(ls) below).
// The copy() builtin is best for the copying.
// Note in cases of an error, the error text will be sent to the client.
type ListerAt interface {
	ListAt([]os.FileInfo, int64) (int, error)
}
<|endoftext|>