<commit_before>package redis

import (
	"fmt"

	redigo "github.com/garyburd/redigo/redis"
	"github.com/soveran/redisurl"
)

type Connection interface {
	Commands
	Transactions

	Send(command string, args ...interface{}) error
	Do(command string, args ...interface{}) (interface{}, error)

	Transaction(func(Transaction)) ([]interface{}, error)
	Pipelined(func(Pipeline)) ([]interface{}, error)
	PipelinedDiscarding(f func(Pipeline)) error

	Flush() error
	Receive() (interface{}, error)
}

type UnpooledConnection interface {
	Connection

	Close()
}

func NewConnection(url string) (UnpooledConnection, error) {
	c, err := redisurl.ConnectToURL(url)
	if err != nil {
		return nil, err
	}

	conn := &connection{c: c}

	return conn, nil
}

type connection struct {
	c    redigo.Conn
	pool Pool
}

// PooledConnection

func (s *connection) Release() {
	s.c.Close()
}

// UnpooledConnection

func (s *connection) Close() {
	if s.pool != nil {
		s.Release()
	} else {
		s.c.Close()
	}
}

// Connection

func (s *connection) Send(command string, args ...interface{}) error {
	return s.c.Send(command, args...)
}

func (s *connection) Do(command string, args ...interface{}) (interface{}, error) {
	return s.c.Do(command, args...)
}

func (s *connection) Transaction(f func(Transaction)) ([]interface{}, error) {
	if err := s.Multi(); err != nil {
		return nil, err
	}

	f(asTransaction(s))

	return s.Exec()
}

func (s *connection) Pipelined(f func(Pipeline)) ([]interface{}, error) {
	p := asPipeline(s)

	f(p)

	if err := s.Flush(); err != nil {
		return nil, err
	}

	return p.receiveAll()
}

func (s *connection) PipelinedDiscarding(f func(Pipeline)) error {
	f(asPipeline(s))

	return s.Flush()
}

func (s *connection) Flush() error {
	return s.c.Flush()
}

func (s *connection) Receive() (interface{}, error) {
	return s.c.Receive()
}

// Commands - Keys

func (s *connection) Del(keys ...string) (int, error) {
	return redigo.Int(s.Do("DEL", redigo.Args{}.AddFlat(keys)...))
}

func (s *connection) Exists(key string) (bool, error) {
	return redigo.Bool(s.Do("EXISTS", key))
}

func (s *connection) Rename(key, newKey string) error {
	_, err := s.Do("RENAME", key, newKey)
	return err
}

func (s *connection) RenameNX(key, newKey string) (bool, error) {
	return redigo.Bool(s.Do("RENAMENX", key, newKey))
}

// Commands - Strings

func (s *connection) Get(key string) (string, error) {
	return redigo.String(s.Do("GET", key))
}

func (s *connection) Set(key, value string) error {
	_, err := s.Do("SET", key, value)
	return err
}

func (s *connection) SetEx(key, value string, expire int) error {
	_, err := s.Do("SETEX", key, expire, value)
	return err
}

func (s *connection) Incr(key string) (int, error) {
	return redigo.Int(s.Do("INCR", key))
}

// Commands - Hashes

func (s *connection) HGet(key, field string) (string, error) {
	return redigo.String(s.Do("HGET", key, field))
}

func (s *connection) HIncrBy(key, field string, value int64) (int64, error) {
	return redigo.Int64(s.Do("HINCRBY", key, field, value))
}

func (s *connection) HSet(key string, field string, value string) (bool, error) {
	return redigo.Bool(s.Do("HSET", key, field, value))
}

func (s *connection) HDel(key string, field string) (bool, error) {
	return redigo.Bool(s.Do("HDEL", key, field))
}

// Commands - Lists

func (s *connection) BLPop(timeout int, keys ...string) (string, string, error) {
	reply, err := redigo.Values(s.Do("BLPOP", redigo.Args{}.AddFlat(keys).Add(timeout)...))
	if err != nil {
		return "", "", err
	}

	fmt.Printf("%v %T, %v %T\n", reply[0], reply[0], reply[1], reply[1])
	return reply[0].(string), reply[1].(string), nil
}

func (s *connection) BRPop(timeout int, keys ...string) (string, string, error) {
	reply, err := redigo.Values(s.Do("BRPOP", redigo.Args{}.AddFlat(keys).Add(timeout)...))
	if err != nil {
		return "", "", err
	}

	return string(reply[0].([]byte)), string(reply[1].([]byte)), nil
}

func (s *connection) LLen(key string) (int, error) {
	return redigo.Int(s.Do("LLEN", key))
}

func (s *connection) LPop(key string) (string, error) {
	return redigo.String(s.Do("LPOP", key))
}

func (s *connection) LPush(key string, values ...string) (int, error) {
	return redigo.Int(s.Do("LPUSH", redigo.Args{key}.AddFlat(values)...))
}

func (s *connection) RPop(key string) (string, error) {
	return redigo.String(s.Do("RPOP", key))
}

func (s *connection) RPush(key string, values ...string) (int, error) {
	return redigo.Int(s.Do("RPUSH", redigo.Args{key}.AddFlat(values)...))
}

// Commands - Sets

func (s *connection) SAdd(key string, member string, members ...string) (int, error) {
	return redigo.Int(s.Do("SADD", redigo.Args{key}.Add(member).AddFlat(members)...))
}

func (s *connection) SCard(key string) (int, error) {
	return redigo.Int(s.Do("SCARD", key))
}

func (s *connection) SRem(key string, member string, members ...string) (int, error) {
	return redigo.Int(s.Do("SREM", redigo.Args{key}.Add(member).AddFlat(members)...))
}

func (s *connection) SPop(key string) (string, error) {
	return redigo.String(s.Do("SPOP", key))
}

func (s *connection) SMembers(key string) ([]string, error) {
	return redigo.Strings(s.Do("SMEMBERS", key))
}

func (s *connection) SRandMember(key string, count int) ([]string, error) {
	return redigo.Strings(s.Do("SRANDMEMBER", key, count))
}

func (s *connection) SDiff(key string, keys ...string) ([]string, error) {
	return redigo.Strings(s.Do("SDIFF", redigo.Args{key}.AddFlat(keys)...))
}

// Commands - Sorted sets

func (s *connection) ZAdd(key string, score float64, value string) (int, error) {
	// Returns number of elements added, 0 if already exist
	return redigo.Int(s.Do("ZADD", key, score, value))
}

func (s *connection) ZCard(key string) (int, error) {
	return redigo.Int(s.Do("ZCARD", key))
}

func (s *connection) ZRangeByScore(key, start, stop string, options ...interface{}) ([]string, error) {
	return redigo.Strings(s.Do("ZRANGEBYSCORE", redigo.Args{key, start, stop}.AddFlat(options)...))
}

// KC: Deprecated. Please use ZRangeByScore(key, start, stop, "LIMIT", 0, 1)
func (s *connection) ZRangeByScoreWithLimit(key, start, stop string, offset, count int) ([]string, error) {
	return redigo.Strings(s.Do("ZRANGEBYSCORE", key, start, stop, "LIMIT", fmt.Sprint(offset), fmt.Sprint(count)))
}

func (s *connection) ZRem(key string, members ...string) (int, error) {
	if len(members) == 0 {
		return 0, nil
	}

	args := redigo.Args{}.Add(key).AddFlat(members)
	return redigo.Int(s.Do("ZREM", args...))
}

// Transactions

func (s *connection) Multi() error {
	return s.Send("MULTI")
}

func (s *connection) Exec() ([]interface{}, error) {
	return redigo.Values(s.Do("EXEC"))
}
<commit_msg>fix BLPOP<commit_after>package redis

import (
	"fmt"

	redigo "github.com/garyburd/redigo/redis"
	"github.com/soveran/redisurl"
)

type Connection interface {
	Commands
	Transactions

	Send(command string, args ...interface{}) error
	Do(command string, args ...interface{}) (interface{}, error)

	Transaction(func(Transaction)) ([]interface{}, error)
	Pipelined(func(Pipeline)) ([]interface{}, error)
	PipelinedDiscarding(f func(Pipeline)) error

	Flush() error
	Receive() (interface{}, error)
}

type UnpooledConnection interface {
	Connection

	Close()
}

func NewConnection(url string) (UnpooledConnection, error) {
	c, err := redisurl.ConnectToURL(url)
	if err != nil {
		return nil, err
	}

	conn := &connection{c: c}

	return conn, nil
}

type connection struct {
	c    redigo.Conn
	pool Pool
}

// PooledConnection

func (s *connection) Release() {
	s.c.Close()
}

// UnpooledConnection

func (s *connection) Close() {
	if s.pool != nil {
		s.Release()
	} else {
		s.c.Close()
	}
}

// Connection

func (s *connection) Send(command string, args ...interface{}) error {
	return s.c.Send(command, args...)
}

func (s *connection) Do(command string, args ...interface{}) (interface{}, error) {
	return s.c.Do(command, args...)
}

func (s *connection) Transaction(f func(Transaction)) ([]interface{}, error) {
	if err := s.Multi(); err != nil {
		return nil, err
	}

	f(asTransaction(s))

	return s.Exec()
}

func (s *connection) Pipelined(f func(Pipeline)) ([]interface{}, error) {
	p := asPipeline(s)

	f(p)

	if err := s.Flush(); err != nil {
		return nil, err
	}

	return p.receiveAll()
}

func (s *connection) PipelinedDiscarding(f func(Pipeline)) error {
	f(asPipeline(s))

	return s.Flush()
}

func (s *connection) Flush() error {
	return s.c.Flush()
}

func (s *connection) Receive() (interface{}, error) {
	return s.c.Receive()
}

// Commands - Keys

func (s *connection) Del(keys ...string) (int, error) {
	return redigo.Int(s.Do("DEL", redigo.Args{}.AddFlat(keys)...))
}

func (s *connection) Exists(key string) (bool, error) {
	return redigo.Bool(s.Do("EXISTS", key))
}

func (s *connection) Rename(key, newKey string) error {
	_, err := s.Do("RENAME", key, newKey)
	return err
}

func (s *connection) RenameNX(key, newKey string) (bool, error) {
	return redigo.Bool(s.Do("RENAMENX", key, newKey))
}

// Commands - Strings

func (s *connection) Get(key string) (string, error) {
	return redigo.String(s.Do("GET", key))
}

func (s *connection) Set(key, value string) error {
	_, err := s.Do("SET", key, value)
	return err
}

func (s *connection) SetEx(key, value string, expire int) error {
	_, err := s.Do("SETEX", key, expire, value)
	return err
}

func (s *connection) Incr(key string) (int, error) {
	return redigo.Int(s.Do("INCR", key))
}

// Commands - Hashes

func (s *connection) HGet(key, field string) (string, error) {
	return redigo.String(s.Do("HGET", key, field))
}

func (s *connection) HIncrBy(key, field string, value int64) (int64, error) {
	return redigo.Int64(s.Do("HINCRBY", key, field, value))
}

func (s *connection) HSet(key string, field string, value string) (bool, error) {
	return redigo.Bool(s.Do("HSET", key, field, value))
}

func (s *connection) HDel(key string, field string) (bool, error) {
	return redigo.Bool(s.Do("HDEL", key, field))
}

// Commands - Lists

func (s *connection) BLPop(timeout int, keys ...string) (string, string, error) {
	reply, err := redigo.Values(s.Do("BLPOP", redigo.Args{}.AddFlat(keys).Add(timeout)...))
	if err != nil {
		return "", "", err
	}

	return string(reply[0].([]byte)), string(reply[1].([]byte)), nil
}

func (s *connection) BRPop(timeout int, keys ...string) (string, string, error) {
	reply, err := redigo.Values(s.Do("BRPOP", redigo.Args{}.AddFlat(keys).Add(timeout)...))
	if err != nil {
		return "", "", err
	}

	return string(reply[0].([]byte)), string(reply[1].([]byte)), nil
}

func (s *connection) LLen(key string) (int, error) {
	return redigo.Int(s.Do("LLEN", key))
}

func (s *connection) LPop(key string) (string, error) {
	return redigo.String(s.Do("LPOP", key))
}

func (s *connection) LPush(key string, values ...string) (int, error) {
	return redigo.Int(s.Do("LPUSH", redigo.Args{key}.AddFlat(values)...))
}

func (s *connection) RPop(key string) (string, error) {
	return redigo.String(s.Do("RPOP", key))
}

func (s *connection) RPush(key string, values ...string) (int, error) {
	return redigo.Int(s.Do("RPUSH", redigo.Args{key}.AddFlat(values)...))
}

// Commands - Sets

func (s *connection) SAdd(key string, member string, members ...string) (int, error) {
	return redigo.Int(s.Do("SADD", redigo.Args{key}.Add(member).AddFlat(members)...))
}

func (s *connection) SCard(key string) (int, error) {
	return redigo.Int(s.Do("SCARD", key))
}

func (s *connection) SRem(key string, member string, members ...string) (int, error) {
	return redigo.Int(s.Do("SREM", redigo.Args{key}.Add(member).AddFlat(members)...))
}

func (s *connection) SPop(key string) (string, error) {
	return redigo.String(s.Do("SPOP", key))
}

func (s *connection) SMembers(key string) ([]string, error) {
	return redigo.Strings(s.Do("SMEMBERS", key))
}

func (s *connection) SRandMember(key string, count int) ([]string, error) {
	return redigo.Strings(s.Do("SRANDMEMBER", key, count))
}

func (s *connection) SDiff(key string, keys ...string) ([]string, error) {
	return redigo.Strings(s.Do("SDIFF", redigo.Args{key}.AddFlat(keys)...))
}

// Commands - Sorted sets

func (s *connection) ZAdd(key string, score float64, value string) (int, error) {
	// Returns number of elements added, 0 if already exist
	return redigo.Int(s.Do("ZADD", key, score, value))
}

func (s *connection) ZCard(key string) (int, error) {
	return redigo.Int(s.Do("ZCARD", key))
}

func (s *connection) ZRangeByScore(key, start, stop string, options ...interface{}) ([]string, error) {
	return redigo.Strings(s.Do("ZRANGEBYSCORE", redigo.Args{key, start, stop}.AddFlat(options)...))
}

// KC: Deprecated. Please use ZRangeByScore(key, start, stop, "LIMIT", 0, 1)
func (s *connection) ZRangeByScoreWithLimit(key, start, stop string, offset, count int) ([]string, error) {
	return redigo.Strings(s.Do("ZRANGEBYSCORE", key, start, stop, "LIMIT", fmt.Sprint(offset), fmt.Sprint(count)))
}

func (s *connection) ZRem(key string, members ...string) (int, error) {
	if len(members) == 0 {
		return 0, nil
	}

	args := redigo.Args{}.Add(key).AddFlat(members)
	return redigo.Int(s.Do("ZREM", args...))
}

// Transactions

func (s *connection) Multi() error {
	return s.Send("MULTI")
}

func (s *connection) Exec() ([]interface{}, error) {
	return redigo.Values(s.Do("EXEC"))
}
<|endoftext|>


<commit_before>package registry

import (
	"path"
	"time"

	"github.com/coreos/go-etcd/etcd"

	"github.com/coreos/fleet/event"
	"github.com/coreos/fleet/machine"
)

const (
	machinePrefix = "/machines/"
)

// Describe all active Machines
func (r *Registry) GetActiveMachines() []machine.Machine {
	key := path.Join(keyPrefix, machinePrefix)
	resp, err := r.etcd.Get(key, false, true)

	var machines []machine.Machine

	// Assume the error was KeyNotFound and return an empty data structure
	if err != nil {
		return machines
	}

	for _, kv := range resp.Node.Nodes {
		_, bootId := path.Split(kv.Key)
		mach := r.GetMachineState(bootId)
		if mach != nil {
			machines = append(machines, *mach)
		}
	}

	return machines
}

// Get Machine object from etcd
func (r *Registry) GetMachineState(bootid string) *machine.Machine {
	key := path.Join(keyPrefix, machinePrefix, bootid, "object")
	resp, err := r.etcd.Get(key, false, true)

	// Assume the error was KeyNotFound and return an empty data structure
	if err != nil {
		return nil
	}

	var mach machine.Machine
	if err := unmarshal(resp.Node.Value, &mach); err != nil {
		return nil
	}

	return &mach
}

// Push Machine object to etcd
func (r *Registry) SetMachineState(machine *machine.Machine, ttl time.Duration) {
	//TODO: Handle the error generated by marshal
	json, _ := marshal(machine)
	key := path.Join(keyPrefix, machinePrefix, machine.BootId, "object")
	r.etcd.Set(key, json, uint64(ttl.Seconds()))
}

// Remove Machine object from etcd
func (r *Registry) RemoveMachineState(machine *machine.Machine) error {
	key := path.Join(keyPrefix, machinePrefix, machine.BootId, "object")
	_, err := r.etcd.Delete(key, false)
	return err
}

func (self *EventStream) filterEventMachineUpdated(resp *etcd.Response) *event.Event {
	if base := path.Base(resp.Node.Key); base != "object" {
		return nil
	}

	if resp.Action != "set" {
		return nil
	}

	var m machine.Machine
	unmarshal(resp.Node.Value, &m)
	return &event.Event{"EventMachineUpdated", m, nil}
}

func (self *EventStream) filterEventMachineRemoved(resp *etcd.Response) *event.Event {
	if base := path.Base(resp.Node.Key); base != "object" {
		return nil
	}

	if resp.Action != "expire" && resp.Action != "delete" {
		return nil
	}

	machName := path.Base(path.Dir(resp.Node.Key))
	return &event.Event{"EventMachineRemoved", machName, nil}
}
<commit_msg>fix(machine): Explicitly create vs update machine state in registry<commit_after>package registry

import (
	"path"
	"time"

	"github.com/coreos/go-etcd/etcd"

	"github.com/coreos/fleet/event"
	"github.com/coreos/fleet/machine"
)

const (
	machinePrefix = "/machines/"
)

// Describe all active Machines
func (r *Registry) GetActiveMachines() []machine.Machine {
	key := path.Join(keyPrefix, machinePrefix)
	resp, err := r.etcd.Get(key, false, true)

	var machines []machine.Machine

	// Assume the error was KeyNotFound and return an empty data structure
	if err != nil {
		return machines
	}

	for _, kv := range resp.Node.Nodes {
		_, bootId := path.Split(kv.Key)
		mach := r.GetMachineState(bootId)
		if mach != nil {
			machines = append(machines, *mach)
		}
	}

	return machines
}

// Get Machine object from etcd
func (r *Registry) GetMachineState(bootid string) *machine.Machine {
	key := path.Join(keyPrefix, machinePrefix, bootid, "object")
	resp, err := r.etcd.Get(key, false, true)

	// Assume the error was KeyNotFound and return an empty data structure
	if err != nil {
		return nil
	}

	var mach machine.Machine
	if err := unmarshal(resp.Node.Value, &mach); err != nil {
		return nil
	}

	return &mach
}

// Push Machine object to etcd
func (r *Registry) SetMachineState(machine *machine.Machine, ttl time.Duration) {
	//TODO: Handle the error generated by marshal
	json, _ := marshal(machine)
	key := path.Join(keyPrefix, machinePrefix, machine.BootId, "object")

	// Assume state is already present, returning on success
	_, err := r.etcd.Update(key, json, uint64(ttl.Seconds()))
	if err == nil {
		return
	}

	// If state was not present, explicitly create it so the other members
	// in the cluster know this is a new member
	r.etcd.Create(key, json, uint64(ttl.Seconds()))
}

// Remove Machine object from etcd
func (r *Registry) RemoveMachineState(machine *machine.Machine) error {
	key := path.Join(keyPrefix, machinePrefix, machine.BootId, "object")
	_, err := r.etcd.Delete(key, false)
	return err
}

func (self *EventStream) filterEventMachineUpdated(resp *etcd.Response) *event.Event {
	if base := path.Base(resp.Node.Key); base != "object" {
		return nil
	}

	if resp.Action != "set" {
		return nil
	}

	var m machine.Machine
	unmarshal(resp.Node.Value, &m)
	return &event.Event{"EventMachineUpdated", m, nil}
}

func (self *EventStream) filterEventMachineRemoved(resp *etcd.Response) *event.Event {
	if base := path.Base(resp.Node.Key); base != "object" {
		return nil
	}

	if resp.Action != "expire" && resp.Action != "delete" {
		return nil
	}

	machName := path.Base(path.Dir(resp.Node.Key))
	return &event.Event{"EventMachineRemoved", machName, nil}
}
<|endoftext|>


<commit_before>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build aix darwin dragonfly freebsd linux netbsd openbsd

package test

import (
	"bytes"
	"io"
	"io/ioutil"
	"math/rand"
	"net"
	"testing"
	"time"
)

type closeWriter interface {
	CloseWrite() error
}

func testPortForward(t *testing.T, n, listenAddr string) {
	server := newServer(t)
	defer server.Shutdown()
	conn := server.Dial(clientConfig())
	defer conn.Close()

	sshListener, err := conn.Listen(n, listenAddr)
	if err != nil {
		t.Fatal(err)
	}

	go func() {
		sshConn, err := sshListener.Accept()
		if err != nil {
			t.Fatalf("listen.Accept failed: %v", err)
		}

		_, err = io.Copy(sshConn, sshConn)
		if err != nil && err != io.EOF {
			t.Fatalf("ssh client copy: %v", err)
		}
		sshConn.Close()
	}()

	forwardedAddr := sshListener.Addr().String()
	netConn, err := net.Dial(n, forwardedAddr)
	if err != nil {
		t.Fatalf("net dial failed: %v", err)
	}

	readChan := make(chan []byte)
	go func() {
		data, _ := ioutil.ReadAll(netConn)
		readChan <- data
	}()

	// Invent some data.
	data := make([]byte, 100*1000)
	for i := range data {
		data[i] = byte(i % 255)
	}

	var sent []byte
	for len(sent) < 1000*1000 {
		// Send random sized chunks
		m := rand.Intn(len(data))
		n, err := netConn.Write(data[:m])
		if err != nil {
			break
		}
		sent = append(sent, data[:n]...)
	}
	if err := netConn.(closeWriter).CloseWrite(); err != nil {
		t.Errorf("netConn.CloseWrite: %v", err)
	}

	read := <-readChan

	if len(sent) != len(read) {
		t.Fatalf("got %d bytes, want %d", len(read), len(sent))
	}
	if bytes.Compare(sent, read) != 0 {
		t.Fatalf("read back data does not match")
	}

	if err := sshListener.Close(); err != nil {
		t.Fatalf("sshListener.Close: %v", err)
	}

	// Check that the forward disappeared.
	netConn, err = net.Dial(n, forwardedAddr)
	if err == nil {
		netConn.Close()
		t.Errorf("still listening to %s after closing", forwardedAddr)
	}
}

func TestPortForwardTCP(t *testing.T) {
	testPortForward(t, "tcp", "localhost:0")
}

func TestPortForwardUnix(t *testing.T) {
	addr, cleanup := newTempSocket(t)
	defer cleanup()
	testPortForward(t, "unix", addr)
}

func testAcceptClose(t *testing.T, n, listenAddr string) {
	server := newServer(t)
	defer server.Shutdown()
	conn := server.Dial(clientConfig())

	sshListener, err := conn.Listen(n, listenAddr)
	if err != nil {
		t.Fatal(err)
	}

	quit := make(chan error, 1)
	go func() {
		for {
			c, err := sshListener.Accept()
			if err != nil {
				quit <- err
				break
			}
			c.Close()
		}
	}()
	sshListener.Close()

	select {
	case <-time.After(1 * time.Second):
		t.Errorf("timeout: listener did not close.")
	case err := <-quit:
		t.Logf("quit as expected (error %v)", err)
	}
}

func TestAcceptCloseTCP(t *testing.T) {
	testAcceptClose(t, "tcp", "localhost:0")
}

func TestAcceptCloseUnix(t *testing.T) {
	addr, cleanup := newTempSocket(t)
	defer cleanup()
	testAcceptClose(t, "unix", addr)
}

// Check that listeners exit if the underlying client transport dies.
func testPortForwardConnectionClose(t *testing.T, n, listenAddr string) {
	server := newServer(t)
	defer server.Shutdown()
	conn := server.Dial(clientConfig())

	sshListener, err := conn.Listen(n, listenAddr)
	if err != nil {
		t.Fatal(err)
	}

	quit := make(chan error, 1)
	go func() {
		for {
			c, err := sshListener.Accept()
			if err != nil {
				quit <- err
				break
			}
			c.Close()
		}
	}()

	// It would be even nicer if we closed the server side, but it
	// is more involved as the fd for that side is dup()ed.
	server.clientConn.Close()

	select {
	case <-time.After(1 * time.Second):
		t.Errorf("timeout: listener did not close.")
	case err := <-quit:
		t.Logf("quit as expected (error %v)", err)
	}
}

func TestPortForwardConnectionCloseTCP(t *testing.T) {
	testPortForwardConnectionClose(t, "tcp", "localhost:0")
}

func TestPortForwardConnectionCloseUnix(t *testing.T) {
	addr, cleanup := newTempSocket(t)
	defer cleanup()
	testPortForwardConnectionClose(t, "unix", addr)
}
<commit_msg>ssh/test: fix test goroutine error handling<commit_after>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build aix darwin dragonfly freebsd linux netbsd openbsd

package test

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net"
	"testing"
	"time"
)

type closeWriter interface {
	CloseWrite() error
}

func testPortForward(t *testing.T, n, listenAddr string) {
	server := newServer(t)
	defer server.Shutdown()
	conn := server.Dial(clientConfig())
	defer conn.Close()

	sshListener, err := conn.Listen(n, listenAddr)
	if err != nil {
		t.Fatal(err)
	}

	errCh := make(chan error, 1)

	go func() {
		defer close(errCh)
		sshConn, err := sshListener.Accept()
		if err != nil {
			errCh <- fmt.Errorf("listen.Accept failed: %v", err)
			return
		}
		defer sshConn.Close()

		_, err = io.Copy(sshConn, sshConn)
		if err != nil && err != io.EOF {
			errCh <- fmt.Errorf("ssh client copy: %v", err)
		}
	}()

	forwardedAddr := sshListener.Addr().String()
	netConn, err := net.Dial(n, forwardedAddr)
	if err != nil {
		t.Fatalf("net dial failed: %v", err)
	}

	readChan := make(chan []byte)
	go func() {
		data, _ := ioutil.ReadAll(netConn)
		readChan <- data
	}()

	// Invent some data.
	data := make([]byte, 100*1000)
	for i := range data {
		data[i] = byte(i % 255)
	}

	var sent []byte
	for len(sent) < 1000*1000 {
		// Send random sized chunks
		m := rand.Intn(len(data))
		n, err := netConn.Write(data[:m])
		if err != nil {
			break
		}
		sent = append(sent, data[:n]...)
	}
	if err := netConn.(closeWriter).CloseWrite(); err != nil {
		t.Errorf("netConn.CloseWrite: %v", err)
	}

	// Check for errors on server goroutine
	err = <-errCh
	if err != nil {
		t.Fatalf("server: %v", err)
	}

	read := <-readChan

	if len(sent) != len(read) {
		t.Fatalf("got %d bytes, want %d", len(read), len(sent))
	}
	if bytes.Compare(sent, read) != 0 {
		t.Fatalf("read back data does not match")
	}

	if err := sshListener.Close(); err != nil {
		t.Fatalf("sshListener.Close: %v", err)
	}

	// Check that the forward disappeared.
	netConn, err = net.Dial(n, forwardedAddr)
	if err == nil {
		netConn.Close()
		t.Errorf("still listening to %s after closing", forwardedAddr)
	}
}

func TestPortForwardTCP(t *testing.T) {
	testPortForward(t, "tcp", "localhost:0")
}

func TestPortForwardUnix(t *testing.T) {
	addr, cleanup := newTempSocket(t)
	defer cleanup()
	testPortForward(t, "unix", addr)
}

func testAcceptClose(t *testing.T, n, listenAddr string) {
	server := newServer(t)
	defer server.Shutdown()
	conn := server.Dial(clientConfig())

	sshListener, err := conn.Listen(n, listenAddr)
	if err != nil {
		t.Fatal(err)
	}

	quit := make(chan error, 1)
	go func() {
		for {
			c, err := sshListener.Accept()
			if err != nil {
				quit <- err
				break
			}
			c.Close()
		}
	}()
	sshListener.Close()

	select {
	case <-time.After(1 * time.Second):
		t.Errorf("timeout: listener did not close.")
	case err := <-quit:
		t.Logf("quit as expected (error %v)", err)
	}
}

func TestAcceptCloseTCP(t *testing.T) {
	testAcceptClose(t, "tcp", "localhost:0")
}

func TestAcceptCloseUnix(t *testing.T) {
	addr, cleanup := newTempSocket(t)
	defer cleanup()
	testAcceptClose(t, "unix", addr)
}

// Check that listeners exit if the underlying client transport dies.
func testPortForwardConnectionClose(t *testing.T, n, listenAddr string) {
	server := newServer(t)
	defer server.Shutdown()
	conn := server.Dial(clientConfig())

	sshListener, err := conn.Listen(n, listenAddr)
	if err != nil {
		t.Fatal(err)
	}

	quit := make(chan error, 1)
	go func() {
		for {
			c, err := sshListener.Accept()
			if err != nil {
				quit <- err
				break
			}
			c.Close()
		}
	}()

	// It would be even nicer if we closed the server side, but it
	// is more involved as the fd for that side is dup()ed.
	server.clientConn.Close()

	select {
	case <-time.After(1 * time.Second):
		t.Errorf("timeout: listener did not close.")
	case err := <-quit:
		t.Logf("quit as expected (error %v)", err)
	}
}

func TestPortForwardConnectionCloseTCP(t *testing.T) {
	testPortForwardConnectionClose(t, "tcp", "localhost:0")
}

func TestPortForwardConnectionCloseUnix(t *testing.T) {
	addr, cleanup := newTempSocket(t)
	defer cleanup()
	testPortForwardConnectionClose(t, "unix", addr)
}
<|endoftext|>


<commit_before>package models

import (
	"errors"
	"net/http"

	apiErrors "github.com/go-openapi/errors"
)

var (
	ErrRoutesCreate     = errors.New("Could not create route")
	ErrRoutesUpdate     = errors.New("Could not update route")
	ErrRoutesRemoving   = errors.New("Could not remove route from datastore")
	ErrRoutesGet        = errors.New("Could not get route from datastore")
	ErrRoutesList       = errors.New("Could not list routes from datastore")
	ErrRoutesNotFound   = errors.New("Route not found")
	ErrRoutesMissingNew = errors.New("Missing new route")
)

type Routes []*Route

type Route struct {
	Name    string      `json:"name"`
	AppName string      `json:"appname"`
	Path    string      `json:"path"`
	Image   string      `json:"image"`
	Headers http.Header `json:"headers,omitempty"`
}

var (
	ErrRoutesValidationName    = errors.New("Missing route Name")
	ErrRoutesValidationImage   = errors.New("Missing route Image")
	ErrRoutesValidationAppName = errors.New("Missing route AppName")
	ErrRoutesValidationPath    = errors.New("Missing route Path")
)

func (r *Route) Validate() error {
	var res []error

	if r.Name == "" {
		res = append(res, ErrRoutesValidationAppName)
	}

	if r.Image == "" {
		res = append(res, ErrRoutesValidationImage)
	}

	if r.AppName == "" {
		res = append(res, ErrRoutesValidationAppName)
	}

	if r.Path == "" {
		res = append(res, ErrRoutesValidationPath)
	}

	if len(res) > 0 {
		return apiErrors.CompositeValidationError(res...)
	}

	return nil
}

type RouteFilter struct {
	Path    string
	AppName string
}
<commit_msg>renamed route errors and added path validation<commit_after>package models

import (
	"errors"
	"net/http"
	"path"

	apiErrors "github.com/go-openapi/errors"
)

var (
	ErrRoutesCreate     = errors.New("Could not create route")
	ErrRoutesUpdate     = errors.New("Could not update route")
	ErrRoutesRemoving   = errors.New("Could not remove route from datastore")
	ErrRoutesGet        = errors.New("Could not get route from datastore")
	ErrRoutesList       = errors.New("Could not list routes from datastore")
	ErrRoutesNotFound   = errors.New("Route not found")
	ErrRoutesMissingNew = errors.New("Missing new route")
)

type Routes []*Route

type Route struct {
	Name    string      `json:"name"`
	AppName string      `json:"appname"`
	Path    string      `json:"path"`
	Image   string      `json:"image"`
	Headers http.Header `json:"headers,omitempty"`
}

var (
	ErrRoutesValidationMissingName    = errors.New("Missing route Name")
	ErrRoutesValidationMissingImage   = errors.New("Missing route Image")
	ErrRoutesValidationMissingAppName = errors.New("Missing route AppName")
	ErrRoutesValidationMissingPath    = errors.New("Missing route Path")
	ErrRoutesValidationInvalidPath    = errors.New("Invalid Path format")
)

func (r *Route) Validate() error {
	var res []error

	if r.Name == "" {
		res = append(res, ErrRoutesValidationMissingName)
	}

	if r.Image == "" {
		res = append(res, ErrRoutesValidationMissingImage)
	}

	if r.AppName == "" {
		res = append(res, ErrRoutesValidationMissingAppName)
	}

	if r.Path == "" {
		res = append(res, ErrRoutesValidationMissingPath)
	}

	if !path.IsAbs(r.Path) {
		res = append(res, ErrRoutesValidationInvalidPath)
	}

	if len(res) > 0 {
		return apiErrors.CompositeValidationError(res...)
	}

	return nil
}

type RouteFilter struct {
	Path    string
	AppName string
}
<|endoftext|>


<commit_before>/*
	NOTE YE WELL: this is a placeholder package,
	wherein we're mirroring many types declared in repeatr.

	We're evolving them freely and independently for the moment, but
	the time will come when we have to make both projects line up again!
*/
package rdef

import (
	"fmt"
	"strings"

	"github.com/polydawn/refmt/obj/atlas"
)

/*
	Ware IDs are content-addressable, cryptographic hashes which uniquely identify
	a "ware" -- a packed filesystem snapshot.

	Ware IDs are serialized as a string in two parts, separated by a colon --
	for example like "git:f23ae1829" or "tar:WJL8or32vD".
	The first part communicates which kind of packing system computed the hash,
	and the second part is the hash itself.
*/
type WareID struct {
	Type string
	Hash string
}

func ParseWareID(x string) (WareID, error) {
	ss := strings.Split(x, ":")
	if len(ss) != 2 {
		return WareID{}, fmt.Errorf("wareIDs always have a single colon (they are of form <type:hash>)")
	}
	return WareID{ss[0], ss[1]}, nil
}

var WareID_AtlasEntry = atlas.BuildEntry(WareID{}).Transform().
	TransformMarshal(atlas.MakeMarshalTransformFunc(
		func(x WareID) (string, error) {
			return string(x.Type) + ":" + string(x.Hash), nil
		})).
	TransformUnmarshal(atlas.MakeUnmarshalTransformFunc(
		func(x string) (WareID, error) {
			return ParseWareID(x)
		})).
	Complete()

type AbsPath string // Identifier for output slots.  Coincidentally, a path.

type (
	Formula struct {
		Inputs  FormulaInputs
		Action  FormulaAction
		Outputs FormulaOutputs
	}

	FormulaInputs map[AbsPath]WareID

	FormulaOutputs map[AbsPath]string // TODO probably need more there than the ware type name ... although we could put normalizers in the "action" section

	/*
		Defines the action to perform to evaluate the formula -- some commands
		or filesystem operations which will be run after the inputs have been
		assembled; the action is done, the outputs will be saved.
	*/
	FormulaAction struct {
		// An array of strings to hand as args to exec -- creates a single process.
		//
		// TODO we want to add a polymorphic option here, e.g.
		// one of 'Exec', 'Script', or 'Reshuffle' may be set.
		Exec []string
	}

	SetupHash string // HID of formula
)

var (
	Formula_AtlasEntry       = atlas.BuildEntry(Formula{}).StructMap().Autogenerate().Complete()
	FormulaAction_AtlasEntry = atlas.BuildEntry(FormulaAction{}).StructMap().Autogenerate().Complete()
)

type RunRecord struct {
	UID       string             // random number, presumed globally unique.
	Time      int64              // time at start of build.
	FormulaID SetupHash          // HID of formula ran.
	Results   map[AbsPath]WareID // wares produced by the run!

	// --- below: addntl optional metadata ---

	Hostname string            // hostname.  not a trusted field, but useful for debugging.
	Metadata map[string]string // escape valve.  you can attach freetext here.
}

var RunRecord_AtlasEntry = atlas.BuildEntry(RunRecord{}).StructMap().Autogenerate().Complete()

type RunRecordHash string // HID of RunRecord.  Includes UID, etc, so quite unique.  Prefer this to UID for primary key in storage (it's collision resistant).
<commit_msg>api: give WareID a stringer func.<commit_after>/*
	NOTE YE WELL: this is a placeholder package,
	wherein we're mirroring many types declared in repeatr.

	We're evolving them freely and independently for the moment, but
	the time will come when we have to make both projects line up again!
*/
package rdef

import (
	"fmt"
	"strings"

	"github.com/polydawn/refmt/obj/atlas"
)

/*
	Ware IDs are content-addressable, cryptographic hashes which uniquely identify
	a "ware" -- a packed filesystem snapshot.

	Ware IDs are serialized as a string in two parts, separated by a colon --
	for example like "git:f23ae1829" or "tar:WJL8or32vD".
	The first part communicates which kind of packing system computed the hash,
	and the second part is the hash itself.
*/
type WareID struct {
	Type string
	Hash string
}

func ParseWareID(x string) (WareID, error) {
	ss := strings.Split(x, ":")
	if len(ss) != 2 {
		return WareID{}, fmt.Errorf("wareIDs always have a single colon (they are of form <type:hash>)")
	}
	return WareID{ss[0], ss[1]}, nil
}

func (x WareID) String() string {
	return x.Type + ":" + x.Hash
}

var WareID_AtlasEntry = atlas.BuildEntry(WareID{}).Transform().
	TransformMarshal(atlas.MakeMarshalTransformFunc(
		func(x WareID) (string, error) {
			return x.String(), nil
		})).
	TransformUnmarshal(atlas.MakeUnmarshalTransformFunc(
		func(x string) (WareID, error) {
			return ParseWareID(x)
		})).
	Complete()

type AbsPath string // Identifier for output slots.  Coincidentally, a path.

type (
	Formula struct {
		Inputs  FormulaInputs
		Action  FormulaAction
		Outputs FormulaOutputs
	}

	FormulaInputs map[AbsPath]WareID

	FormulaOutputs map[AbsPath]string // TODO probably need more there than the ware type name ... although we could put normalizers in the "action" section

	/*
		Defines the action to perform to evaluate the formula -- some commands
		or filesystem operations which will be run after the inputs have been
		assembled; the action is done, the outputs will be saved.
	*/
	FormulaAction struct {
		// An array of strings to hand as args to exec -- creates a single process.
		//
		// TODO we want to add a polymorphic option here, e.g.
		// one of 'Exec', 'Script', or 'Reshuffle' may be set.
		Exec []string
	}

	SetupHash string // HID of formula
)

var (
	Formula_AtlasEntry       = atlas.BuildEntry(Formula{}).StructMap().Autogenerate().Complete()
	FormulaAction_AtlasEntry = atlas.BuildEntry(FormulaAction{}).StructMap().Autogenerate().Complete()
)

type RunRecord struct {
	UID       string             // random number, presumed globally unique.
	Time      int64              // time at start of build.
	FormulaID SetupHash          // HID of formula ran.
	Results   map[AbsPath]WareID // wares produced by the run!

	// --- below: addntl optional metadata ---

	Hostname string            // hostname.  not a trusted field, but useful for debugging.
	Metadata map[string]string // escape valve.  you can attach freetext here.
}

var RunRecord_AtlasEntry = atlas.BuildEntry(RunRecord{}).StructMap().Autogenerate().Complete()

type RunRecordHash string // HID of RunRecord.  Includes UID, etc, so quite unique.  Prefer this to UID for primary key in storage (it's collision resistant).
<|endoftext|>


<commit_before>// Copyright 2018 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package model

import (
	"fmt"
	"net/url"
	"strconv"

	"github.com/envoyproxy/go-control-plane/envoy/api/v2/auth"
	"github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
	"github.com/envoyproxy/go-control-plane/envoy/config/grpc_credential/v2alpha"
	"github.com/gogo/protobuf/types"

	authn "istio.io/api/authentication/v1alpha1"
)

const (
	// SDSStatPrefix is the human readable prefix to use when emitting statistics for the SDS service.
	SDSStatPrefix = "sdsstat"

	// SDSDefaultResourceName is the default name in sdsconfig, used for fetching normal key/cert.
	SDSDefaultResourceName = "default"

	// SDSRootResourceName is the sdsconfig name for root CA, used for fetching root cert.
	SDSRootResourceName = "ROOTCA"

	// K8sSATrustworthyJwtFileName is the token volume mount file name for k8s trustworthy jwt token.
	K8sSATrustworthyJwtFileName = "/var/run/secrets/tokens/istio-token"

	// K8sSAJwtFileName is the token volume mount file name for k8s jwt token.
	K8sSAJwtFileName = "/var/run/secrets/kubernetes.io/serviceaccount/token"

	// fileBasedMetadataPlugName is File Based Metadata credentials plugin name.
	fileBasedMetadataPlugName = "envoy.grpc_credentials.file_based_metadata"

	// k8sSAJwtTokenHeaderKey is the request header key for k8s jwt token.
	// Binary header name must has suffix "-bin", according to https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md.
	k8sSAJwtTokenHeaderKey = "istio_sds_credentail_header-bin"

	// IngressGatewaySdsUdsPath is the UDS path for ingress gateway to get credentials via SDS.
	IngressGatewaySdsUdsPath = "unix:/var/run/ingress_gateway/sds"

	// IngressGatewaySdsCaSuffix is the suffix of the sds resource name for root CA.
	IngressGatewaySdsCaSuffix = "-cacert"
)

// JwtKeyResolver resolves JWT public key and JwksURI.
var JwtKeyResolver = newJwksResolver(JwtPubKeyExpireDuration, JwtPubKeyEvictionDuration, JwtPubKeyRefreshInterval)

// GetConsolidateAuthenticationPolicy returns the authentication policy for
// service specified by hostname and port, if defined. It also tries to resolve JWKS URI if necessary.
func GetConsolidateAuthenticationPolicy(store IstioConfigStore, service *Service, port *Port) *authn.Policy {
	config := store.AuthenticationPolicyByDestination(service, port)
	if config != nil {
		policy := config.Spec.(*authn.Policy)
		if err := JwtKeyResolver.SetAuthenticationPolicyJwksURIs(policy); err == nil {
			return policy
		}
	}

	return nil
}

// ConstructSdsSecretConfig constructs SDS secret configuration for ingress gateway.
func ConstructSdsSecretConfigForGatewayListener(name, sdsUdsPath string) *auth.SdsSecretConfig {
	if name == "" || sdsUdsPath == "" {
		return nil
	}

	gRPCConfig := &core.GrpcService_GoogleGrpc{
		TargetUri:  sdsUdsPath,
		StatPrefix: SDSStatPrefix,
	}

	return &auth.SdsSecretConfig{
		Name: name,
		SdsConfig: &core.ConfigSource{
			ConfigSourceSpecifier: &core.ConfigSource_ApiConfigSource{
				ApiConfigSource: &core.ApiConfigSource{
					ApiType: core.ApiConfigSource_GRPC,
					GrpcServices: []*core.GrpcService{
						{
							TargetSpecifier: &core.GrpcService_GoogleGrpc_{
								GoogleGrpc: gRPCConfig,
							},
						},
					},
				},
			},
		},
	}
}

// ConstructSdsSecretConfig constructs SDS Sececret Configuration for workload proxy.
func ConstructSdsSecretConfig(name, sdsUdsPath string, useK8sSATrustworthyJwt, useK8sSANormalJwt bool, metadata map[string]string) *auth.SdsSecretConfig {
	if name == "" || sdsUdsPath == "" {
		return nil
	}

	gRPCConfig := &core.GrpcService_GoogleGrpc{
		TargetUri:  sdsUdsPath,
		StatPrefix: SDSStatPrefix,
		ChannelCredentials: &core.GrpcService_GoogleGrpc_ChannelCredentials{
			CredentialSpecifier: &core.GrpcService_GoogleGrpc_ChannelCredentials_LocalCredentials{
				LocalCredentials: &core.GrpcService_GoogleGrpc_GoogleLocalCredentials{},
			},
		},
	}

	// If metadata[NodeMetadataSdsTokenPath] is non-empty, envoy will fetch tokens from metadata[NodeMetadataSdsTokenPath].
	// Otherwise, if useK8sSATrustworthyJwt is set, envoy will fetch and pass k8s sa trustworthy jwt(which is available for k8s 1.10 or higher),
	// pass it to SDS server to request key/cert; if trustworthy jwt isn't available, envoy will fetch and pass normal k8s sa jwt to
	// request key/cert.
	if sdsTokenPath, found := metadata[NodeMetadataSdsTokenPath]; found && len(sdsTokenPath) > 0 {
		log.Debugf("SDS token path is (%v)", sdsTokenPath)
		gRPCConfig.CredentialsFactoryName = fileBasedMetadataPlugName
		gRPCConfig.CallCredentials = constructgRPCCallCredentials(sdsTokenPath, k8sSAJwtTokenHeaderKey)
	} else if useK8sSATrustworthyJwt {
		gRPCConfig.CredentialsFactoryName = fileBasedMetadataPlugName
		gRPCConfig.CallCredentials = constructgRPCCallCredentials(K8sSATrustworthyJwtFileName, k8sSAJwtTokenHeaderKey)
	} else if useK8sSANormalJwt {
		gRPCConfig.CredentialsFactoryName = fileBasedMetadataPlugName
		gRPCConfig.CallCredentials = constructgRPCCallCredentials(K8sSAJwtFileName, k8sSAJwtTokenHeaderKey)
	} else {
		gRPCConfig.CallCredentials = []*core.GrpcService_GoogleGrpc_CallCredentials{
			&core.GrpcService_GoogleGrpc_CallCredentials{
				CredentialSpecifier: &core.GrpcService_GoogleGrpc_CallCredentials_GoogleComputeEngine{
					GoogleComputeEngine: &types.Empty{},
				},
			},
		}
	}

	return &auth.SdsSecretConfig{
		Name: name,
		SdsConfig: &core.ConfigSource{
			ConfigSourceSpecifier: &core.ConfigSource_ApiConfigSource{
				ApiConfigSource: &core.ApiConfigSource{
					ApiType: core.ApiConfigSource_GRPC,
					GrpcServices: []*core.GrpcService{
						{
							TargetSpecifier: &core.GrpcService_GoogleGrpc_{
								GoogleGrpc: gRPCConfig,
							},
						},
					},
				},
			},
		},
	}
}

// ConstructValidationContext constructs ValidationContext in CommonTlsContext.
func ConstructValidationContext(rootCAFilePath string, subjectAltNames []string) *auth.CommonTlsContext_ValidationContext {
	ret := &auth.CommonTlsContext_ValidationContext{
		ValidationContext: &auth.CertificateValidationContext{
			TrustedCa: &core.DataSource{
				Specifier: &core.DataSource_Filename{
					Filename: rootCAFilePath,
				},
			},
		},
	}

	if len(subjectAltNames) > 0 {
		ret.ValidationContext.VerifySubjectAltName = subjectAltNames
	}

	return ret
}

// ParseJwksURI parses the input URI and returns the corresponding hostname, port, and whether SSL is used.
// URI must start with "http://" or "https://", which corresponding to "http" or "https" scheme.
// Port number is extracted from URI if available (i.e from postfix :<port>, eg. ":80"), or assigned
// to a default value based on URI scheme (80 for http and 443 for https).
// Port name is set to URI scheme value.
// Note: this is to replace [buildJWKSURIClusterNameAndAddress]
// (https://github.com/istio/istio/blob/master/pilot/pkg/proxy/envoy/v1/mixer.go#L401),
// which is used for the old EUC policy.
func ParseJwksURI(jwksURI string) (string, *Port, bool, error) {
	u, err := url.Parse(jwksURI)
	if err != nil {
		return "", nil, false, err
	}
	var useSSL bool
	var portNumber int
	switch u.Scheme {
	case "http":
		useSSL = false
		portNumber = 80
	case "https":
		useSSL = true
		portNumber = 443
	default:
		return "", nil, false, fmt.Errorf("URI scheme %q is not supported", u.Scheme)
	}

	if u.Port() != "" {
		portNumber, err = strconv.Atoi(u.Port())
		if err != nil {
			return "", nil, useSSL, err
		}
	}

	return u.Hostname(), &Port{
		Name: u.Scheme,
		Port: portNumber,
	}, useSSL, nil
}

// this function is used to construct SDS config which is only available from 1.1
func constructgRPCCallCredentials(tokenFileName, headerKey string) []*core.GrpcService_GoogleGrpc_CallCredentials {
	// If k8s sa jwt token file exists, envoy only handles plugin credentials.
	config := &v2alpha.FileBasedMetadataConfig{
		SecretData: &core.DataSource{
			Specifier: &core.DataSource_Filename{
				Filename: tokenFileName,
			},
		},
		HeaderKey: headerKey,
	}

	any := findOrMarshalFileBasedMetadataConfig(tokenFileName, headerKey, config)

	return []*core.GrpcService_GoogleGrpc_CallCredentials{
		&core.GrpcService_GoogleGrpc_CallCredentials{
			CredentialSpecifier: &core.GrpcService_GoogleGrpc_CallCredentials_FromPlugin{
				FromPlugin: &core.GrpcService_GoogleGrpc_CallCredentials_MetadataCredentialsFromPlugin{
					Name: fileBasedMetadataPlugName,
					ConfigType: &core.GrpcService_GoogleGrpc_CallCredentials_MetadataCredentialsFromPlugin_TypedConfig{
						TypedConfig: any},
				},
			},
		},
	}
}

type fbMetadataAnyKey struct {
	tokenFileName string
	headerKey     string
}

var fileBasedMetadataConfigAnyMap = map[fbMetadataAnyKey]*types.Any{}

// findOrMarshalFileBasedMetadataConfig searches google.protobuf.Any in fileBasedMetadataConfigAnyMap
// by tokenFileName and headerKey, and returns google.protobuf.Any proto if found. If not found,
// it takes the fbMetadata and marshals it into google.protobuf.Any, and stores this new
// google.protobuf.Any into fileBasedMetadataConfigAnyMap.
// FileBasedMetadataConfig only supports non-deterministic marshaling. As each SDS config contains
// marshaled FileBasedMetadataConfig, the SDS config would differ if marshaling FileBasedMetadataConfig
// returns different result. Once SDS config differs, Envoy will create multiple SDS clients to fetch
// same SDS resource. To solve this problem, we use findOrMarshalFileBasedMetadataConfig so that
// FileBasedMetadataConfig is marshaled once, and is reused in all SDS configs.
func findOrMarshalFileBasedMetadataConfig(tokenFileName, headerKey string, fbMetadata *v2alpha.FileBasedMetadataConfig) *types.Any {
	key := fbMetadataAnyKey{
		tokenFileName: tokenFileName,
		headerKey:     headerKey,
	}
	if marshalAny, found := fileBasedMetadataConfigAnyMap[key]; found {
		return marshalAny
	}
	any, _ := types.MarshalAny(fbMetadata)
	fileBasedMetadataConfigAnyMap[key] = any
	return any
}
<commit_msg>use syncmap to avoid race conditions (#12433)<commit_after>// Copyright 2018 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package model

import (
	"fmt"
	"net/url"
	"strconv"
	"sync"

	"github.com/envoyproxy/go-control-plane/envoy/api/v2/auth"
	"github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
	"github.com/envoyproxy/go-control-plane/envoy/config/grpc_credential/v2alpha"
	"github.com/gogo/protobuf/types"

	authn "istio.io/api/authentication/v1alpha1"
)

const (
	// SDSStatPrefix is the human readable prefix to use when emitting statistics for the SDS service.
	SDSStatPrefix = "sdsstat"

	// SDSDefaultResourceName is the default name in sdsconfig, used for fetching normal key/cert.
	SDSDefaultResourceName = "default"

	// SDSRootResourceName is the sdsconfig name for root CA, used for fetching root cert.
	SDSRootResourceName = "ROOTCA"

	// K8sSATrustworthyJwtFileName is the token volume mount file name for k8s trustworthy jwt token.
	K8sSATrustworthyJwtFileName = "/var/run/secrets/tokens/istio-token"

	// K8sSAJwtFileName is the token volume mount file name for k8s jwt token.
	K8sSAJwtFileName = "/var/run/secrets/kubernetes.io/serviceaccount/token"

	// fileBasedMetadataPlugName is File Based Metadata credentials plugin name.
	fileBasedMetadataPlugName = "envoy.grpc_credentials.file_based_metadata"

	// k8sSAJwtTokenHeaderKey is the request header key for k8s jwt token.
	// Binary header name must has suffix "-bin", according to https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md.
	k8sSAJwtTokenHeaderKey = "istio_sds_credentail_header-bin"

	// IngressGatewaySdsUdsPath is the UDS path for ingress gateway to get credentials via SDS.
	IngressGatewaySdsUdsPath = "unix:/var/run/ingress_gateway/sds"

	// IngressGatewaySdsCaSuffix is the suffix of the sds resource name for root CA.
	IngressGatewaySdsCaSuffix = "-cacert"
)

// JwtKeyResolver resolves JWT public key and JwksURI.
var JwtKeyResolver = newJwksResolver(JwtPubKeyExpireDuration, JwtPubKeyEvictionDuration, JwtPubKeyRefreshInterval)

// GetConsolidateAuthenticationPolicy returns the authentication policy for
// service specified by hostname and port, if defined. It also tries to resolve JWKS URI if necessary.
func GetConsolidateAuthenticationPolicy(store IstioConfigStore, service *Service, port *Port) *authn.Policy {
	config := store.AuthenticationPolicyByDestination(service, port)
	if config != nil {
		policy := config.Spec.(*authn.Policy)
		if err := JwtKeyResolver.SetAuthenticationPolicyJwksURIs(policy); err == nil {
			return policy
		}
	}

	return nil
}

// ConstructSdsSecretConfig constructs SDS secret configuration for ingress gateway.
func ConstructSdsSecretConfigForGatewayListener(name, sdsUdsPath string) *auth.SdsSecretConfig {
	if name == "" || sdsUdsPath == "" {
		return nil
	}

	gRPCConfig := &core.GrpcService_GoogleGrpc{
		TargetUri:  sdsUdsPath,
		StatPrefix: SDSStatPrefix,
	}

	return &auth.SdsSecretConfig{
		Name: name,
		SdsConfig: &core.ConfigSource{
			ConfigSourceSpecifier: &core.ConfigSource_ApiConfigSource{
				ApiConfigSource: &core.ApiConfigSource{
					ApiType: core.ApiConfigSource_GRPC,
					GrpcServices: []*core.GrpcService{
						{
							TargetSpecifier: &core.GrpcService_GoogleGrpc_{
								GoogleGrpc: gRPCConfig,
							},
						},
					},
				},
			},
		},
	}
}

// ConstructSdsSecretConfig constructs SDS Sececret Configuration for workload proxy.
func ConstructSdsSecretConfig(name, sdsUdsPath string, useK8sSATrustworthyJwt, useK8sSANormalJwt bool, metadata map[string]string) *auth.SdsSecretConfig {
	if name == "" || sdsUdsPath == "" {
		return nil
	}

	gRPCConfig := &core.GrpcService_GoogleGrpc{
		TargetUri:  sdsUdsPath,
		StatPrefix: SDSStatPrefix,
		ChannelCredentials: &core.GrpcService_GoogleGrpc_ChannelCredentials{
			CredentialSpecifier: &core.GrpcService_GoogleGrpc_ChannelCredentials_LocalCredentials{
				LocalCredentials: &core.GrpcService_GoogleGrpc_GoogleLocalCredentials{},
			},
		},
	}

	// If metadata[NodeMetadataSdsTokenPath] is non-empty, envoy will fetch tokens from metadata[NodeMetadataSdsTokenPath].
	// Otherwise, if useK8sSATrustworthyJwt is set, envoy will fetch and pass k8s sa trustworthy jwt(which is available for k8s 1.10 or higher),
	// pass it to SDS server to request key/cert; if trustworthy jwt isn't available, envoy will fetch and pass normal k8s sa jwt to
	// request key/cert.
	if sdsTokenPath, found := metadata[NodeMetadataSdsTokenPath]; found && len(sdsTokenPath) > 0 {
		log.Debugf("SDS token path is (%v)", sdsTokenPath)
		gRPCConfig.CredentialsFactoryName = fileBasedMetadataPlugName
		gRPCConfig.CallCredentials = constructgRPCCallCredentials(sdsTokenPath, k8sSAJwtTokenHeaderKey)
	} else if useK8sSATrustworthyJwt {
		gRPCConfig.CredentialsFactoryName = fileBasedMetadataPlugName
		gRPCConfig.CallCredentials = constructgRPCCallCredentials(K8sSATrustworthyJwtFileName, k8sSAJwtTokenHeaderKey)
	} else if useK8sSANormalJwt {
		gRPCConfig.CredentialsFactoryName = fileBasedMetadataPlugName
		gRPCConfig.CallCredentials = constructgRPCCallCredentials(K8sSAJwtFileName, k8sSAJwtTokenHeaderKey)
	} else {
		gRPCConfig.CallCredentials = []*core.GrpcService_GoogleGrpc_CallCredentials{
			&core.GrpcService_GoogleGrpc_CallCredentials{
				CredentialSpecifier: &core.GrpcService_GoogleGrpc_CallCredentials_GoogleComputeEngine{
					GoogleComputeEngine: &types.Empty{},
				},
			},
		}
	}

	return &auth.SdsSecretConfig{
		Name: name,
		SdsConfig: &core.ConfigSource{
			ConfigSourceSpecifier: &core.ConfigSource_ApiConfigSource{
				ApiConfigSource: &core.ApiConfigSource{
					ApiType: core.ApiConfigSource_GRPC,
					GrpcServices: []*core.GrpcService{
						{
							TargetSpecifier: &core.GrpcService_GoogleGrpc_{
								GoogleGrpc: gRPCConfig,
							},
						},
					},
				},
			},
		},
	}
}

// ConstructValidationContext constructs ValidationContext in CommonTlsContext.
func ConstructValidationContext(rootCAFilePath string, subjectAltNames []string) *auth.CommonTlsContext_ValidationContext {
	ret := &auth.CommonTlsContext_ValidationContext{
		ValidationContext: &auth.CertificateValidationContext{
			TrustedCa: &core.DataSource{
				Specifier: &core.DataSource_Filename{
					Filename: rootCAFilePath,
				},
			},
		},
	}

	if len(subjectAltNames) > 0 {
		ret.ValidationContext.VerifySubjectAltName = subjectAltNames
	}

	return ret
}

// ParseJwksURI parses the input URI and returns the corresponding hostname, port, and whether SSL is used.
// URI must start with "http://" or "https://", which corresponding to "http" or "https" scheme.
// Port number is extracted from URI if available (i.e from postfix :<port>, eg. ":80"), or assigned
// to a default value based on URI scheme (80 for http and 443 for https).
// Port name is set to URI scheme value.
// Note: this is to replace [buildJWKSURIClusterNameAndAddress]
// (https://github.com/istio/istio/blob/master/pilot/pkg/proxy/envoy/v1/mixer.go#L401),
// which is used for the old EUC policy.
func ParseJwksURI(jwksURI string) (string, *Port, bool, error) {
	u, err := url.Parse(jwksURI)
	if err != nil {
		return "", nil, false, err
	}
	var useSSL bool
	var portNumber int
	switch u.Scheme {
	case "http":
		useSSL = false
		portNumber = 80
	case "https":
		useSSL = true
		portNumber = 443
	default:
		return "", nil, false, fmt.Errorf("URI scheme %q is not supported", u.Scheme)
	}

	if u.Port() != "" {
		portNumber, err = strconv.Atoi(u.Port())
		if err != nil {
			return "", nil, useSSL, err
		}
	}

	return u.Hostname(), &Port{
		Name: u.Scheme,
		Port: portNumber,
	}, useSSL, nil
}

// this function is used to construct SDS config which is only available from 1.1
func constructgRPCCallCredentials(tokenFileName, headerKey string) []*core.GrpcService_GoogleGrpc_CallCredentials {
	// If k8s sa jwt token file exists, envoy only handles plugin credentials.
	config := &v2alpha.FileBasedMetadataConfig{
		SecretData: &core.DataSource{
			Specifier: &core.DataSource_Filename{
				Filename: tokenFileName,
			},
		},
		HeaderKey: headerKey,
	}

	any := findOrMarshalFileBasedMetadataConfig(tokenFileName, headerKey, config)

	return []*core.GrpcService_GoogleGrpc_CallCredentials{
		&core.GrpcService_GoogleGrpc_CallCredentials{
			CredentialSpecifier: &core.GrpcService_GoogleGrpc_CallCredentials_FromPlugin{
				FromPlugin: &core.GrpcService_GoogleGrpc_CallCredentials_MetadataCredentialsFromPlugin{
					Name: fileBasedMetadataPlugName,
					ConfigType: &core.GrpcService_GoogleGrpc_CallCredentials_MetadataCredentialsFromPlugin_TypedConfig{
						TypedConfig: any},
				},
			},
		},
	}
}

type fbMetadataAnyKey struct {
	tokenFileName string
	headerKey     string
}

var fileBasedMetadataConfigAnyMap sync.Map

// findOrMarshalFileBasedMetadataConfig searches google.protobuf.Any in fileBasedMetadataConfigAnyMap
// by tokenFileName and headerKey, and returns google.protobuf.Any proto if found. If not found,
// it takes the fbMetadata and marshals it into google.protobuf.Any, and stores this new
// google.protobuf.Any into fileBasedMetadataConfigAnyMap.
// FileBasedMetadataConfig only supports non-deterministic marshaling. As each SDS config contains
// marshaled FileBasedMetadataConfig, the SDS config would differ if marshaling FileBasedMetadataConfig
// returns different result. Once SDS config differs, Envoy will create multiple SDS clients to fetch
// same SDS resource. To solve this problem, we use findOrMarshalFileBasedMetadataConfig so that
// FileBasedMetadataConfig is marshaled once, and is reused in all SDS configs.
func findOrMarshalFileBasedMetadataConfig(tokenFileName, headerKey string, fbMetadata *v2alpha.FileBasedMetadataConfig) *types.Any {
	key := fbMetadataAnyKey{
		tokenFileName: tokenFileName,
		headerKey:     headerKey,
	}
	if v, found := fileBasedMetadataConfigAnyMap.Load(key); found {
		marshalAny := v.(types.Any)
		return &marshalAny
	}
	any, _ := types.MarshalAny(fbMetadata)
	fileBasedMetadataConfigAnyMap.Store(key, *any)
	return any
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1

import (
	"encoding/json"
	"fmt"
	"strings"

	"k8s.io/apimachinery/pkg/runtime/schema"
)

// GroupResource specifies a Group and a Resource, but does not force a version.  This is useful for identifying
// concepts during lookup stages without having partially valid types
//
// +protobuf.options.(gogoproto.goproto_stringer)=false
type GroupResource struct {
	Group    string `protobuf:"bytes,1,opt,name=group"`
	Resource string `protobuf:"bytes,2,opt,name=resource"`
}

func (gr *GroupResource) String() string {
	if len(gr.Group) == 0 {
		return gr.Resource
	}
	return gr.Resource + "." + gr.Group
}

// GroupVersionResource unambiguously identifies a resource.  It doesn't anonymously include GroupVersion
// to avoid automatic coersion.  It doesn't use a GroupVersion to avoid custom marshalling
//
// +protobuf.options.(gogoproto.goproto_stringer)=false
type GroupVersionResource struct {
	Group    string `protobuf:"bytes,1,opt,name=group"`
	Version  string `protobuf:"bytes,2,opt,name=version"`
	Resource string `protobuf:"bytes,3,opt,name=resource"`
}

func (gvr *GroupVersionResource) String() string {
	return strings.Join([]string{gvr.Group, "/", gvr.Version, ", Resource=", gvr.Resource}, "")
}

// GroupKind specifies a Group and a Kind, but does not force a version.  This is useful for identifying
// concepts during lookup stages without having partially valid types
//
// +protobuf.options.(gogoproto.goproto_stringer)=false
type GroupKind struct {
	Group string `protobuf:"bytes,1,opt,name=group"`
	Kind  string `protobuf:"bytes,2,opt,name=kind"`
}

func (gk *GroupKind) String() string {
	if len(gk.Group) == 0 {
		return gk.Kind
	}
	return gk.Kind + "." + gk.Group
}

// GroupVersionKind unambiguously identifies a kind.  It doesn't anonymously include GroupVersion
// to avoid automatic coersion.  It doesn't use a GroupVersion to avoid custom marshalling
//
// +protobuf.options.(gogoproto.goproto_stringer)=false
type GroupVersionKind struct {
	Group   string `json:"group" protobuf:"bytes,1,opt,name=group"`
	Version string `json:"version" protobuf:"bytes,2,opt,name=version"`
	Kind    string `json:"kind" protobuf:"bytes,3,opt,name=kind"`
}

func (gvk GroupVersionKind) String() string {
	return gvk.Group + "/" + gvk.Version + ", Kind=" + gvk.Kind
}

// GroupVersion contains the "group" and the "version", which uniquely identifies the API.
//
// +protobuf.options.(gogoproto.goproto_stringer)=false
type GroupVersion struct {
	Group   string `protobuf:"bytes,1,opt,name=group"`
	Version string `protobuf:"bytes,2,opt,name=version"`
}

// Empty returns true if group and version are empty
func (gv GroupVersion) Empty() bool {
	return len(gv.Group) == 0 && len(gv.Version) == 0
}

// String puts "group" and "version" into a single "group/version" string. For the legacy v1
// it returns "v1".
func (gv GroupVersion) String() string {
	// special case the internal apiVersion for the legacy kube types
	if gv.Empty() {
		return ""
	}

	// special case of "v1" for backward compatibility
	if len(gv.Group) == 0 && gv.Version == "v1" {
		return gv.Version
	}
	if len(gv.Group) > 0 {
		return gv.Group + "/" + gv.Version
	}
	return gv.Version
}

// MarshalJSON implements the json.Marshaller interface.
func (gv GroupVersion) MarshalJSON() ([]byte, error) {
	s := gv.String()
	if strings.Count(s, "/") > 1 {
		return []byte{}, fmt.Errorf("illegal GroupVersion %v: contains more than one /", s)
	}
	return json.Marshal(s)
}

func (gv *GroupVersion) unmarshal(value []byte) error {
	var s string
	if err := json.Unmarshal(value, &s); err != nil {
		return err
	}
	parsed, err := schema.ParseGroupVersion(s)
	if err != nil {
		return err
	}
	gv.Group, gv.Version = parsed.Group, parsed.Version
	return nil
}

// UnmarshalJSON implements the json.Unmarshaller interface.
func (gv *GroupVersion) UnmarshalJSON(value []byte) error {
	return gv.unmarshal(value)
}

// UnmarshalTEXT implements the Ugorji's encoding.TextUnmarshaler interface.
func (gv *GroupVersion) UnmarshalText(value []byte) error {
	return gv.unmarshal(value)
}
<commit_msg>Implement dynamic admission webhooks<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1

import (
	"encoding/json"
	"fmt"
	"strings"

	"k8s.io/apimachinery/pkg/runtime/schema"
)

// GroupResource specifies a Group and a Resource, but does not force a version.  This is useful for identifying
// concepts during lookup stages without having partially valid types
//
// +protobuf.options.(gogoproto.goproto_stringer)=false
type GroupResource struct {
	Group    string `json:"group" protobuf:"bytes,1,opt,name=group"`
	Resource string `json:"resource" protobuf:"bytes,2,opt,name=resource"`
}

func (gr *GroupResource) String() string {
	if len(gr.Group) == 0 {
		return gr.Resource
	}
	return gr.Resource + "." + gr.Group
}

// GroupVersionResource unambiguously identifies a resource.  It doesn't anonymously include GroupVersion
// to avoid automatic coersion.  It doesn't use a GroupVersion to avoid custom marshalling
//
// +protobuf.options.(gogoproto.goproto_stringer)=false
type GroupVersionResource struct {
	Group    string `json:"group" protobuf:"bytes,1,opt,name=group"`
	Version  string `json:"version" protobuf:"bytes,2,opt,name=version"`
	Resource string `json:"resource" protobuf:"bytes,3,opt,name=resource"`
}

func (gvr *GroupVersionResource) String() string {
	return strings.Join([]string{gvr.Group, "/", gvr.Version, ", Resource=", gvr.Resource}, "")
}

// GroupKind specifies a Group and a Kind, but does not force a version.  This is useful for identifying
// concepts during lookup stages without having partially valid types
//
// +protobuf.options.(gogoproto.goproto_stringer)=false
type GroupKind struct {
	Group string `json:"group" protobuf:"bytes,1,opt,name=group"`
	Kind  string `json:"kind" protobuf:"bytes,2,opt,name=kind"`
}

func (gk *GroupKind) String() string {
	if len(gk.Group) == 0 {
		return gk.Kind
	}
	return gk.Kind + "." + gk.Group
}

// GroupVersionKind unambiguously identifies a kind.  It doesn't anonymously include GroupVersion
// to avoid automatic coersion.  It doesn't use a GroupVersion to avoid custom marshalling
//
// +protobuf.options.(gogoproto.goproto_stringer)=false
type GroupVersionKind struct {
	Group   string `json:"group" protobuf:"bytes,1,opt,name=group"`
	Version string `json:"version" protobuf:"bytes,2,opt,name=version"`
	Kind    string `json:"kind" protobuf:"bytes,3,opt,name=kind"`
}

func (gvk GroupVersionKind) String() string {
	return gvk.Group + "/" + gvk.Version + ", Kind=" + gvk.Kind
}

// GroupVersion contains the "group" and the "version", which uniquely identifies the API.
//
// +protobuf.options.(gogoproto.goproto_stringer)=false
type GroupVersion struct {
	Group   string `json:"group" protobuf:"bytes,1,opt,name=group"`
	Version string `json:"version" protobuf:"bytes,2,opt,name=version"`
}

// Empty returns true if group and version are empty
func (gv GroupVersion) Empty() bool {
	return len(gv.Group) == 0 && len(gv.Version) == 0
}

// String puts "group" and "version" into a single "group/version" string. For the legacy v1
// it returns "v1".
func (gv GroupVersion) String() string {
	// special case the internal apiVersion for the legacy kube types
	if gv.Empty() {
		return ""
	}

	// special case of "v1" for backward compatibility
	if len(gv.Group) == 0 && gv.Version == "v1" {
		return gv.Version
	}
	if len(gv.Group) > 0 {
		return gv.Group + "/" + gv.Version
	}
	return gv.Version
}

// MarshalJSON implements the json.Marshaller interface.
func (gv GroupVersion) MarshalJSON() ([]byte, error) {
	s := gv.String()
	if strings.Count(s, "/") > 1 {
		return []byte{}, fmt.Errorf("illegal GroupVersion %v: contains more than one /", s)
	}
	return json.Marshal(s)
}

func (gv *GroupVersion) unmarshal(value []byte) error {
	var s string
	if err := json.Unmarshal(value, &s); err != nil {
		return err
	}
	parsed, err := schema.ParseGroupVersion(s)
	if err != nil {
		return err
	}
	gv.Group, gv.Version = parsed.Group, parsed.Version
	return nil
}

// UnmarshalJSON implements the json.Unmarshaller interface.
func (gv *GroupVersion) UnmarshalJSON(value []byte) error {
	return gv.unmarshal(value)
}

// UnmarshalTEXT implements the Ugorji's encoding.TextUnmarshaler interface.
func (gv *GroupVersion) UnmarshalText(value []byte) error {
	return gv.unmarshal(value)
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Tekton Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package reconciler

import (
	"math"
	"math/rand"
	"sync"

	"time"

	"github.com/tektoncd/pipeline/pkg/apis/config"
	"github.com/tektoncd/pipeline/pkg/apis/pipeline/v1alpha1"
	clientset "github.com/tektoncd/pipeline/pkg/client/clientset/versioned"
	"go.uber.org/zap"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
)

const (
	maxBackoffSeconds = 120
)

var (
	defaultFunc        = func(i interface{}) {}
	maxBackoffExponent = math.Ceil(math.Log2(maxBackoffSeconds))
)

// StatusKey interface to be implemented by Taskrun Pipelinerun types
type StatusKey interface {
	GetRunKey() string
}

// backoff contains state of exponential backoff for a given StatusKey
type backoff struct {
	// NumAttempts reflects the number of times a given StatusKey has been delayed
	NumAttempts uint
	// NextAttempt is the point in time at which this backoff expires
	NextAttempt time.Time
}

// jitterFunc is a func applied to a computed backoff duration to remove uniformity
// from its results. A jitterFunc receives the number of seconds calculated by a
// backoff algorithm and returns the "jittered" result.
type jitterFunc func(numSeconds int) (jitteredSeconds int)

// TimeoutSet contains required k8s interfaces to handle build timeouts
type TimeoutSet struct {
	logger                  *zap.SugaredLogger
	taskRunCallbackFunc     func(interface{})
	pipelineRunCallbackFunc func(interface{})
	stopCh                  <-chan struct{}
	done                    map[string]chan bool
	doneMut                 sync.Mutex
	backoffs                map[string]backoff
	backoffsMut             sync.Mutex
}

// NewTimeoutHandler returns TimeoutSet filled structure
func NewTimeoutHandler(
	stopCh <-chan struct{},
	logger *zap.SugaredLogger,
) *TimeoutSet {
	return &TimeoutSet{
		stopCh:   stopCh,
		done:     make(map[string]chan bool),
		backoffs: make(map[string]backoff),
		logger:   logger,
	}
}

// SetTaskRunCallbackFunc sets the callback function when timeout occurs for taskrun objects
func (t *TimeoutSet) SetTaskRunCallbackFunc(f func(interface{})) {
	t.taskRunCallbackFunc = f
}

// SetPipelineRunCallbackFunc sets the callback function when timeout occurs for pipelinerun objects
func (t *TimeoutSet) SetPipelineRunCallbackFunc(f func(interface{})) {
	t.pipelineRunCallbackFunc = f
}

// Release deletes channels and data that are specific to a StatusKey object.
func (t *TimeoutSet) Release(runObj StatusKey) {
	key := runObj.GetRunKey()
	t.doneMut.Lock()
	defer t.doneMut.Unlock()

	t.backoffsMut.Lock()
	defer t.backoffsMut.Unlock()

	if finished, ok := t.done[key]; ok {
		delete(t.done, key)
		close(finished)
	}
	delete(t.backoffs, key)
}

func (t *TimeoutSet) getOrCreateFinishedChan(runObj StatusKey) chan bool {
	var finished chan bool
	key := runObj.GetRunKey()
	t.doneMut.Lock()
	defer t.doneMut.Unlock()
	if existingfinishedChan, ok := t.done[key]; ok {
		finished = existingfinishedChan
	} else {
		finished = make(chan bool)
	}
	t.done[key] = finished
	return finished
}

// GetBackoff records the number of times it has seen a TaskRun and calculates an
// appropriate backoff deadline based on that count. Only one backoff per TaskRun
// may be active at any moment. Requests for a new backoff in the face of an
// existing one will be ignored and details of the existing backoff will be returned
// instead. Further, if a calculated backoff time is after the timeout of the TaskRun
// then the time of the timeout will be returned instead.
//
// Returned values are a backoff struct containing a NumAttempts field with the
// number of attempts performed for this TaskRun and a NextAttempt field
// describing the time at which the next attempt should be performed.
// Additionally a boolean is returned indicating whether a backoff for the
// TaskRun is already in progress.
func (t *TimeoutSet) GetBackoff(tr *v1alpha1.TaskRun) (backoff, bool) {
	t.backoffsMut.Lock()
	defer t.backoffsMut.Unlock()
	b := t.backoffs[tr.GetRunKey()]
	if time.Now().Before(b.NextAttempt) {
		return b, true
	}
	b.NumAttempts += 1
	b.NextAttempt = time.Now().Add(backoffDuration(b.NumAttempts, rand.Intn))
	timeoutDeadline := tr.Status.StartTime.Time.Add(tr.Spec.Timeout.Duration)
	if timeoutDeadline.Before(b.NextAttempt) {
		b.NextAttempt = timeoutDeadline
	}
	t.backoffs[tr.GetRunKey()] = b
	return b, false
}

func backoffDuration(count uint, jf jitterFunc) time.Duration {
	exp := float64(count)
	if exp > maxBackoffExponent {
		exp = maxBackoffExponent
	}
	seconds := int(math.Exp2(exp))
	jittered := 1 + jf(seconds)
	if jittered > maxBackoffSeconds {
		jittered = maxBackoffSeconds
	}
	return time.Duration(jittered) * time.Second
}

// checkPipelineRunTimeouts function creates goroutines to wait for pipelinerun to
// finish/timeout in a given namespace
func (t *TimeoutSet) checkPipelineRunTimeouts(namespace string, pipelineclientset clientset.Interface) {
	pipelineRuns, err := pipelineclientset.TektonV1alpha1().PipelineRuns(namespace).List(metav1.ListOptions{})
	if err != nil {
		t.logger.Errorf("Can't get pipelinerun list in namespace %s: %s", namespace, err)
		return
	}
	for _, pipelineRun := range pipelineRuns.Items {
		pipelineRun := pipelineRun
		if pipelineRun.IsDone() || pipelineRun.IsCancelled() {
			continue
		}
		if pipelineRun.HasStarted() {
			go t.WaitPipelineRun(&pipelineRun, pipelineRun.Status.StartTime)
		}
	}
}

// CheckTimeouts function iterates through all namespaces and calls corresponding
// taskrun/pipelinerun timeout functions
func (t *TimeoutSet) CheckTimeouts(kubeclientset kubernetes.Interface, pipelineclientset clientset.Interface) {
	namespaces, err := kubeclientset.CoreV1().Namespaces().List(metav1.ListOptions{})
	if err != nil {
		t.logger.Errorf("Can't get namespaces list: %s", err)
		return
	}
	for _, namespace := range namespaces.Items {
		t.checkTaskRunTimeouts(namespace.GetName(), pipelineclientset)
		t.checkPipelineRunTimeouts(namespace.GetName(), pipelineclientset)
	}
}

// checkTaskRunTimeouts function creates goroutines to wait for pipelinerun to
// finish/timeout in a given namespace
func (t *TimeoutSet) checkTaskRunTimeouts(namespace string, pipelineclientset clientset.Interface) {
	taskruns, err := pipelineclientset.TektonV1alpha1().TaskRuns(namespace).List(metav1.ListOptions{})
	if err != nil {
		t.logger.Errorf("Can't get taskrun list in namespace %s: %s", namespace, err)
		return
	}
	for _, taskrun := range taskruns.Items {
		taskrun := taskrun
		if taskrun.IsDone() || taskrun.IsCancelled() {
			continue
		}
		if taskrun.HasStarted() {
			go t.WaitTaskRun(&taskrun, taskrun.Status.StartTime)
		}
	}
}

// WaitTaskRun function creates a blocking function for taskrun to wait for
// 1. Stop signal, 2. TaskRun to complete or 3. Taskrun to time out, which is
// determined by checking if the tr's timeout has occurred since the startTime
func (t *TimeoutSet) WaitTaskRun(tr *v1alpha1.TaskRun, startTime *metav1.Time) {
	var timeout time.Duration
	if tr.Spec.Timeout == nil {
		timeout = config.DefaultTimeoutMinutes * time.Minute
	} else {
		timeout = tr.Spec.Timeout.Duration
	}
	t.waitRun(tr, timeout, startTime, t.taskRunCallbackFunc)
}

// WaitPipelineRun function creates a blocking function for pipelinerun to wait for
// 1. Stop signal, 2. pipelinerun to complete or 3. pipelinerun to time out which is
// determined by checking if the tr's timeout has occurred since the startTime
func (t *TimeoutSet) WaitPipelineRun(pr *v1alpha1.PipelineRun, startTime *metav1.Time) {
	var timeout time.Duration
	if pr.Spec.Timeout == nil {
		timeout = config.DefaultTimeoutMinutes * time.Minute
	} else {
		timeout = pr.Spec.Timeout.Duration
	}
	t.waitRun(pr, timeout, startTime, t.pipelineRunCallbackFunc)
}

func (t *TimeoutSet) waitRun(runObj StatusKey, timeout time.Duration, startTime *metav1.Time, callback func(interface{})) {
	if startTime == nil {
		t.logger.Errorf("startTime must be specified in order for a timeout to be calculated accurately for %s", runObj.GetRunKey())
		return
	}
	if callback == nil {
		callback = defaultFunc
	}
	runtime := time.Since(startTime.Time)
	t.logger.Infof("About to start timeout timer for %s. started at %s, timeout is %s, running for %s", runObj.GetRunKey(), startTime.Time, timeout, runtime)
	defer t.Release(runObj)
	t.setTimer(runObj, timeout-runtime, callback)
}

// SetTaskRunTimer creates a blocking function for taskrun to wait for
// 1. Stop signal, 2. TaskRun to complete or 3. a given Duration to elapse.
//
// Since the timer's duration is a parameter rather than being tied to
// the lifetime of the TaskRun no resources are released after the timer
// fires. It is the caller's responsibility to Release() the TaskRun when
// work with it has completed.
func (t *TimeoutSet) SetTaskRunTimer(tr *v1alpha1.TaskRun, d time.Duration) {
	callback := t.taskRunCallbackFunc
	if callback == nil {
		t.logger.Errorf("attempted to set a timer for %q but no task run callback has been assigned", tr.Name)
		return
	}
	t.setTimer(tr, d, callback)
}

func (t *TimeoutSet) setTimer(runObj StatusKey, timeout time.Duration, callback func(interface{})) {
	finished := t.getOrCreateFinishedChan(runObj)
	started := time.Now()
	select {
	case <-t.stopCh:
		t.logger.Infof("stopping timer for %q", runObj.GetRunKey())
		return
	case <-finished:
		t.logger.Infof("%q finished, stopping timer", runObj.GetRunKey())
		return
	case <-time.After(timeout):
		t.logger.Infof("timer for %q has activated after %s", runObj.GetRunKey(), time.Since(started).String())
		callback(runObj)
	}
}
<commit_msg>Remove logging from timeout handler ✏<commit_after>/*
Copyright 2019 The Tekton Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package reconciler

import (
	"math"
	"math/rand"
	"sync"

	"time"

	"github.com/tektoncd/pipeline/pkg/apis/config"
	"github.com/tektoncd/pipeline/pkg/apis/pipeline/v1alpha1"
	clientset "github.com/tektoncd/pipeline/pkg/client/clientset/versioned"
	"go.uber.org/zap"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
)

const (
	maxBackoffSeconds = 120
)

var (
	defaultFunc        = func(i interface{}) {}
	maxBackoffExponent = math.Ceil(math.Log2(maxBackoffSeconds))
)

// StatusKey interface to be implemented by Taskrun Pipelinerun types
type StatusKey interface {
	GetRunKey() string
}

// backoff contains state of exponential backoff for a given StatusKey
type backoff struct {
	// NumAttempts reflects the number of times a given StatusKey has been delayed
	NumAttempts uint
	// NextAttempt is the point in time at which this backoff expires
	NextAttempt time.Time
}

// jitterFunc is a func applied to a computed backoff duration to remove uniformity
// from its results. A jitterFunc receives the number of seconds calculated by a
// backoff algorithm and returns the "jittered" result.
type jitterFunc func(numSeconds int) (jitteredSeconds int)

// TimeoutSet contains required k8s interfaces to handle build timeouts
type TimeoutSet struct {
	logger                  *zap.SugaredLogger
	taskRunCallbackFunc     func(interface{})
	pipelineRunCallbackFunc func(interface{})
	stopCh                  <-chan struct{}
	done                    map[string]chan bool
	doneMut                 sync.Mutex
	backoffs                map[string]backoff
	backoffsMut             sync.Mutex
}

// NewTimeoutHandler returns TimeoutSet filled structure
func NewTimeoutHandler(
	stopCh <-chan struct{},
	logger *zap.SugaredLogger,
) *TimeoutSet {
	return &TimeoutSet{
		stopCh:   stopCh,
		done:     make(map[string]chan bool),
		backoffs: make(map[string]backoff),
		logger:   logger,
	}
}

// SetTaskRunCallbackFunc sets the callback function when timeout occurs for taskrun objects
func (t *TimeoutSet) SetTaskRunCallbackFunc(f func(interface{})) {
	t.taskRunCallbackFunc = f
}

// SetPipelineRunCallbackFunc sets the callback function when timeout occurs for pipelinerun objects
func (t *TimeoutSet) SetPipelineRunCallbackFunc(f func(interface{})) {
	t.pipelineRunCallbackFunc = f
}

// Release deletes channels and data that are specific to a StatusKey object.
func (t *TimeoutSet) Release(runObj StatusKey) {
	key := runObj.GetRunKey()
	t.doneMut.Lock()
	defer t.doneMut.Unlock()

	t.backoffsMut.Lock()
	defer t.backoffsMut.Unlock()

	if finished, ok := t.done[key]; ok {
		delete(t.done, key)
		close(finished)
	}
	delete(t.backoffs, key)
}

func (t *TimeoutSet) getOrCreateFinishedChan(runObj StatusKey) chan bool {
	var finished chan bool
	key := runObj.GetRunKey()
	t.doneMut.Lock()
	defer t.doneMut.Unlock()
	if existingfinishedChan, ok := t.done[key]; ok {
		finished = existingfinishedChan
	} else {
		finished = make(chan bool)
	}
	t.done[key] = finished
	return finished
}

// GetBackoff records the number of times it has seen a TaskRun and calculates an
// appropriate backoff deadline based on that count. Only one backoff per TaskRun
// may be active at any moment. Requests for a new backoff in the face of an
// existing one will be ignored and details of the existing backoff will be returned
// instead. Further, if a calculated backoff time is after the timeout of the TaskRun
// then the time of the timeout will be returned instead.
//
// Returned values are a backoff struct containing a NumAttempts field with the
// number of attempts performed for this TaskRun and a NextAttempt field
// describing the time at which the next attempt should be performed.
// Additionally a boolean is returned indicating whether a backoff for the
// TaskRun is already in progress.
func (t *TimeoutSet) GetBackoff(tr *v1alpha1.TaskRun) (backoff, bool) {
	t.backoffsMut.Lock()
	defer t.backoffsMut.Unlock()
	b := t.backoffs[tr.GetRunKey()]
	if time.Now().Before(b.NextAttempt) {
		return b, true
	}
	b.NumAttempts += 1
	b.NextAttempt = time.Now().Add(backoffDuration(b.NumAttempts, rand.Intn))
	timeoutDeadline := tr.Status.StartTime.Time.Add(tr.Spec.Timeout.Duration)
	if timeoutDeadline.Before(b.NextAttempt) {
		b.NextAttempt = timeoutDeadline
	}
	t.backoffs[tr.GetRunKey()] = b
	return b, false
}

func backoffDuration(count uint, jf jitterFunc) time.Duration {
	exp := float64(count)
	if exp > maxBackoffExponent {
		exp = maxBackoffExponent
	}
	seconds := int(math.Exp2(exp))
	jittered := 1 + jf(seconds)
	if jittered > maxBackoffSeconds {
		jittered = maxBackoffSeconds
	}
	return time.Duration(jittered) * time.Second
}

// checkPipelineRunTimeouts function creates goroutines to wait for pipelinerun to
// finish/timeout in a given namespace
func (t *TimeoutSet) checkPipelineRunTimeouts(namespace string, pipelineclientset clientset.Interface) {
	pipelineRuns, err := pipelineclientset.TektonV1alpha1().PipelineRuns(namespace).List(metav1.ListOptions{})
	if err != nil {
		return
	}
	for _, pipelineRun := range pipelineRuns.Items {
		pipelineRun := pipelineRun
		if pipelineRun.IsDone() || pipelineRun.IsCancelled() {
			continue
		}
		if pipelineRun.HasStarted() {
			go t.WaitPipelineRun(&pipelineRun, pipelineRun.Status.StartTime)
		}
	}
}

// CheckTimeouts function iterates through all namespaces and calls corresponding
// taskrun/pipelinerun timeout functions
func (t *TimeoutSet) CheckTimeouts(kubeclientset kubernetes.Interface, pipelineclientset clientset.Interface) {
	namespaces, err := kubeclientset.CoreV1().Namespaces().List(metav1.ListOptions{})
	if err != nil {
		return
	}
	for _, namespace := range namespaces.Items {
		t.checkTaskRunTimeouts(namespace.GetName(), pipelineclientset)
		t.checkPipelineRunTimeouts(namespace.GetName(), pipelineclientset)
	}
}

// checkTaskRunTimeouts function creates goroutines to wait for pipelinerun to
// finish/timeout in a given namespace
func (t *TimeoutSet) checkTaskRunTimeouts(namespace string, pipelineclientset clientset.Interface) {
	taskruns, err := pipelineclientset.TektonV1alpha1().TaskRuns(namespace).List(metav1.ListOptions{})
	if err != nil {
		return
	}
	for _, taskrun := range taskruns.Items {
		taskrun := taskrun
		if taskrun.IsDone() || taskrun.IsCancelled() {
			continue
		}
		if taskrun.HasStarted() {
			go t.WaitTaskRun(&taskrun, taskrun.Status.StartTime)
		}
	}
}

// WaitTaskRun function creates a blocking function for taskrun to wait for
// 1. Stop signal, 2. TaskRun to complete or 3. Taskrun to time out, which is
// determined by checking if the tr's timeout has occurred since the startTime
func (t *TimeoutSet) WaitTaskRun(tr *v1alpha1.TaskRun, startTime *metav1.Time) {
	var timeout time.Duration
	if tr.Spec.Timeout == nil {
		timeout = config.DefaultTimeoutMinutes * time.Minute
	} else {
		timeout = tr.Spec.Timeout.Duration
	}
	t.waitRun(tr, timeout, startTime, t.taskRunCallbackFunc)
}

// WaitPipelineRun function creates a blocking function for pipelinerun to wait for
// 1. Stop signal, 2. pipelinerun to complete or 3. pipelinerun to time out which is
// determined by checking if the tr's timeout has occurred since the startTime
func (t *TimeoutSet) WaitPipelineRun(pr *v1alpha1.PipelineRun, startTime *metav1.Time) {
	var timeout time.Duration
	if pr.Spec.Timeout == nil {
		timeout = config.DefaultTimeoutMinutes * time.Minute
	} else {
		timeout = pr.Spec.Timeout.Duration
	}
	t.waitRun(pr, timeout, startTime, t.pipelineRunCallbackFunc)
}

func (t *TimeoutSet) waitRun(runObj StatusKey, timeout time.Duration, startTime *metav1.Time, callback func(interface{})) {
	if startTime == nil {
		return
	}
	if callback == nil {
		callback = defaultFunc
	}
	runtime := time.Since(startTime.Time)
	defer t.Release(runObj)
	t.setTimer(runObj, timeout-runtime, callback)
}

// SetTaskRunTimer creates a blocking function for taskrun to wait for
// 1. Stop signal, 2. TaskRun to complete or 3. a given Duration to elapse.
//
// Since the timer's duration is a parameter rather than being tied to
// the lifetime of the TaskRun no resources are released after the timer
// fires. It is the caller's responsibility to Release() the TaskRun when
// work with it has completed.
func (t *TimeoutSet) SetTaskRunTimer(tr *v1alpha1.TaskRun, d time.Duration) {
	callback := t.taskRunCallbackFunc
	if callback == nil {
		return
	}
	t.setTimer(tr, d, callback)
}

func (t *TimeoutSet) setTimer(runObj StatusKey, timeout time.Duration, callback func(interface{})) {
	finished := t.getOrCreateFinishedChan(runObj)
	select {
	case <-t.stopCh:
		return
	case <-finished:
		return
	case <-time.After(timeout):
		callback(runObj)
	}
}
<|endoftext|>


<commit_before>package zk

import (
	"strings"
	"time"
)

type Config struct {
	Name           string
	ZkAddrs        string
	SessionTimeout time.Duration
	PanicOnError   bool
}

func DefaultConfig(name, addrs string) *Config {
	return &Config{
		Name:           name,
		ZkAddrs:        addrs,
		SessionTimeout: DefaultZkSessionTimeout(),
		PanicOnError:   false,
	}
}

func (this *Config) ZkServers() []string {
	return strings.Split(this.ZkAddrs, ",")
}

func DefaultZkSessionTimeout() time.Duration {
	// online zk tickTime=2000, valid timeout: 4s ~ 40s
	return time.Second * 20
}
<commit_msg>zk ioTimeout=sessionTimeout*2/3, ping interval=ioTimeout/2<commit_after>package zk

import (
	"strings"
	"time"
)

type Config struct {
	Name           string
	ZkAddrs        string
	SessionTimeout time.Duration
	PanicOnError   bool
}

func DefaultConfig(name, addrs string) *Config {
	return &Config{
		Name:           name,
		ZkAddrs:        addrs,
		SessionTimeout: DefaultZkSessionTimeout(),
		PanicOnError:   false,
	}
}

func (this *Config) ZkServers() []string {
	return strings.Split(this.ZkAddrs, ",")
}

func DefaultZkSessionTimeout() time.Duration {
	// online zk tickTime=2000, valid timeout: 4s ~ 40s
	// io timeout: 13s  ping interval: 6.5s
	return time.Second * 20
}
<|endoftext|>


<commit_before>package todolist

import (
	"fmt"
	"strconv"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

func TestParseSubject(t *testing.T) {
	parser := &Parser{}
	todo := parser.ParseNewTodo("do this thing")
	if todo.Subject != "do this thing" {
		t.Error("Expected todo.Subject to equal 'do this thing'")
	}
}

func TestParseSubjectWithDue(t *testing.T) {
	parser := &Parser{}
	todo := parser.ParseNewTodo("do this thing due tomorrow")
	if todo.Subject != "do this thing" {
		t.Error("Expected todo.Subject to equal 'do this thing', got ", todo.Subject)
	}
}

func TestParseExpandProjects(t *testing.T) {
	assert := assert.New(t)
	parser := &Parser{}
	correctFormat := parser.ExpandProject("ex 113 +meeting: figures, slides, coffee, suger")
	assert.Equal("+meeting", correctFormat)
	wrongFormat1 := parser.ExpandProject("ex 114 +meeting figures, slides, coffee, suger")
	assert.Equal("", wrongFormat1)
	wrongFormat2 := parser.ExpandProject("ex 115 meeting: figures, slides, coffee, suger")
	assert.Equal("", wrongFormat2)
	wrongFormat3 := parser.ExpandProject("ex 116 meeting figures, slides, coffee, suger")
	assert.Equal("", wrongFormat3)
	wrongFormat4 := parser.ExpandProject("ex 117 +重要な會議: 図, コーヒー, 砂糖")
	assert.Equal("+重要な會議", wrongFormat4)
}

func TestParseProjects(t *testing.T) {
	parser := &Parser{}
	todo := parser.ParseNewTodo("do this thing +proj1 +proj2 +專案3 +proj-name due tomorrow")
	if len(todo.Projects) != 4 {
		t.Error("Expected Projects length to be 3")
	}
	if todo.Projects[0] != "proj1" {
		t.Error("todo.Projects[0] should equal 'proj1' but got", todo.Projects[0])
	}
	if todo.Projects[1] != "proj2" {
		t.Error("todo.Projects[1] should equal 'proj2' but got", todo.Projects[1])
	}
	if todo.Projects[2] != "專案3" {
		t.Error("todo.Projects[2] should equal '專案3' but got", todo.Projects[2])
	}
	if todo.Projects[3] != "proj-name" {
		t.Error("todo.Projects[3] should equal 'proj-name' but got", todo.Projects[3])
	}
}

func TestParseContexts(t *testing.T) {
	parser := &Parser{}
	todo := parser.ParseNewTodo("do this thing with @bob and @mary due tomorrow")
	if len(todo.Contexts) != 2 {
		t.Error("Expected Projects length to be 2")
	}
	if todo.Contexts[0] != "bob" {
		t.Error("todo.Contexts[0] should equal 'mary' but got", todo.Contexts[0])
	}
	if todo.Contexts[1] != "mary" {
		t.Error("todo.Contexts[1] should equal 'mary' but got", todo.Contexts[1])
	}
}

func TestDueToday(t *testing.T) {
	parser := &Parser{}
	todo := parser.ParseNewTodo("do this thing with @bob and @mary due today")
	if todo.Due != bod(time.Now()).Format("2006-01-02") {
		fmt.Println("Date is different", todo.Due, time.Now())
	}
	todo = parser.ParseNewTodo("do this thing with @bob and @mary due tod")
	if todo.Due != bod(time.Now()).Format("2006-01-02") {
		fmt.Println("Date is different", todo.Due, time.Now())
	}
}

func TestDueTomorrow(t *testing.T) {
	parser := &Parser{}
	todo := parser.ParseNewTodo("do this thing with @bob and @mary due tomorrow")
	if todo.Due != bod(time.Now()).AddDate(0, 0, 1).Format("2006-01-02") {
		fmt.Println("Date is different", todo.Due, time.Now())
	}
	todo = parser.ParseNewTodo("do this thing with @bob and @mary due tom")
	if todo.Due != bod(time.Now()).AddDate(0, 0, 1).Format("2006-01-02") {
		fmt.Println("Date is different", todo.Due, time.Now())
	}
}

func TestDueSpecific(t *testing.T) {
	assert := assert.New(t)
	parser := &Parser{}
	todo := parser.ParseNewTodo("do this thing with @bob and @mary due jun 1")
	year := strconv.Itoa(time.Now().Year())
	assert.Equal(fmt.Sprintf("%s-06-01", year), todo.Due)
}

func TestMondayOnSunday(t *testing.T) {
	assert := assert.New(t)
	parser := &Parser{}
	now, _ := time.Parse("2006-01-02", "2016-04-24")
	assert.Equal("2016-04-25", parser.monday(now))
}

func TestMondayOnMonday(t *testing.T) {
	assert := assert.New(t)
	parser := &Parser{}
	now, _ := time.Parse("2006-01-02", "2016-04-25")
	assert.Equal("2016-04-25", parser.monday(now))
}

func TestMondayOnTuesday(t *testing.T) {
	assert := assert.New(t)
	parser := &Parser{}
	now, _ := time.Parse("2006-01-02", "2016-04-26")
	assert.Equal("2016-05-02", parser.monday(now))
}

func TestTuesdayOnMonday(t *testing.T) {
	assert := assert.New(t)
	parser := &Parser{}
	now, _ := time.Parse("2006-01-02", "2016-04-25")
	assert.Equal("2016-04-26", parser.tuesday(now))
}

func TestTuesdayOnWednesday(t *testing.T) {
	assert := assert.New(t)
	parser := &Parser{}
	now, _ := time.Parse("2006-01-02", "2016-04-27")
	assert.Equal("2016-05-03", parser.tuesday(now))
}

func TestDueOnSpecificDate(t *testing.T) {
	assert := assert.New(t)
	parser := &Parser{}
	year := strconv.Itoa(time.Now().Year())
	assert.Equal(fmt.Sprintf("%s-05-02", year), parser.Due("due may 2", time.Now()))
	assert.Equal(fmt.Sprintf("%s-06-01", year), parser.Due("due jun 1", time.Now()))
}

func TestDueOnSpecificDateEuropean(t *testing.T) {
	assert := assert.New(t)
	parser := &Parser{}
	year := strconv.Itoa(time.Now().Year())
	assert.Equal(fmt.Sprintf("%s-05-02", year), parser.Due("due 2 may", time.Now()))
}

func TestDueIntelligentlyChoosesCorrectYear(t *testing.T) {
	assert := assert.New(t)
	parser := &Parser{}
	marchTime, _ := time.Parse("2006-01-02", "2016-03-25")
	januaryTime, _ := time.Parse("2006-01-02", "2016-01-05")
	septemberTime, _ := time.Parse("2006-01-02", "2016-09-25")
	decemberTime, _ := time.Parse("2006-01-02", "2016-12-25")

	assert.Equal("2016-01-10", parser.parseArbitraryDate("jan 10", januaryTime))
	assert.Equal("2016-01-10", parser.parseArbitraryDate("jan 10", marchTime))
	assert.Equal("2017-01-10", parser.parseArbitraryDate("jan 10", septemberTime))
	assert.Equal("2017-01-10", parser.parseArbitraryDate("jan 10", decemberTime))
}
<commit_msg>add test for func (p Parser) Parse()<commit_after>package todolist

import (
	"fmt"
	"strconv"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

func TestParseSubject(t *testing.T) {
	parser := &Parser{}
	todo := parser.ParseNewTodo("do this thing")
	if todo.Subject != "do this thing" {
		t.Error("Expected todo.Subject to equal 'do this thing'")
	}
}

func TestParseSubjectWithDue(t *testing.T) {
	parser := &Parser{}
	todo := parser.ParseNewTodo("do this thing due tomorrow")
	if todo.Subject != "do this thing" {
		t.Error("Expected todo.Subject to equal 'do this thing', got ", todo.Subject)
	}
}

func TestParseExpandProjects(t *testing.T) {
	assert := assert.New(t)
	parser := &Parser{}
	correctFormat := parser.ExpandProject("ex 113 +meeting: figures, slides, coffee, suger")
	assert.Equal("+meeting", correctFormat)
	wrongFormat1 := parser.ExpandProject("ex 114 +meeting figures, slides, coffee, suger")
	assert.Equal("", wrongFormat1)
	wrongFormat2 := parser.ExpandProject("ex 115 meeting: figures, slides, coffee, suger")
	assert.Equal("", wrongFormat2)
	wrongFormat3 := parser.ExpandProject("ex 116 meeting figures, slides, coffee, suger")
	assert.Equal("", wrongFormat3)
	wrongFormat4 := parser.ExpandProject("ex 117 +重要な會議: 図, コーヒー, 砂糖")
	assert.Equal("+重要な會議", wrongFormat4)
}

func TestParseProjects(t *testing.T) {
	parser := &Parser{}
	todo := parser.ParseNewTodo("do this thing +proj1 +proj2 +專案3 +proj-name due tomorrow")
	if len(todo.Projects) != 4 {
		t.Error("Expected Projects length to be 3")
	}
	if todo.Projects[0] != "proj1" {
		t.Error("todo.Projects[0] should equal 'proj1' but got", todo.Projects[0])
	}
	if todo.Projects[1] != "proj2" {
		t.Error("todo.Projects[1] should equal 'proj2' but got", todo.Projects[1])
	}
	if todo.Projects[2] != "專案3" {
		t.Error("todo.Projects[2] should equal '專案3' but got", todo.Projects[2])
	}
	if todo.Projects[3] != "proj-name" {
		t.Error("todo.Projects[3] should equal 'proj-name' but got", todo.Projects[3])
	}
}

func TestParseContexts(t *testing.T) {
	parser := &Parser{}
	todo := parser.ParseNewTodo("do this thing with @bob and @mary due tomorrow")
	if len(todo.Contexts) != 2 {
		t.Error("Expected Projects length to be 2")
	}
	if todo.Contexts[0] != "bob" {
		t.Error("todo.Contexts[0] should equal 'mary' but got", todo.Contexts[0])
	}
	if todo.Contexts[1] != "mary" {
		t.Error("todo.Contexts[1] should equal 'mary' but got", todo.Contexts[1])
	}
}

func TestDueToday(t *testing.T) {
	parser := &Parser{}
	todo := parser.ParseNewTodo("do this thing with @bob and @mary due today")
	if todo.Due != bod(time.Now()).Format("2006-01-02") {
		fmt.Println("Date is different", todo.Due, time.Now())
	}
	todo = parser.ParseNewTodo("do this thing with @bob and @mary due tod")
	if todo.Due != bod(time.Now()).Format("2006-01-02") {
		fmt.Println("Date is different", todo.Due, time.Now())
	}
}

func TestDueTomorrow(t *testing.T) {
	parser := &Parser{}
	todo := parser.ParseNewTodo("do this thing with @bob and @mary due tomorrow")
	if todo.Due != bod(time.Now()).AddDate(0, 0, 1).Format("2006-01-02") {
		fmt.Println("Date is different", todo.Due, time.Now())
	}
	todo = parser.ParseNewTodo("do this thing with @bob and @mary due tom")
	if todo.Due != bod(time.Now()).AddDate(0, 0, 1).Format("2006-01-02") {
		fmt.Println("Date is different", todo.Due, time.Now())
	}
}

func TestDueSpecific(t *testing.T) {
	assert := assert.New(t)
	parser := &Parser{}
	todo := parser.ParseNewTodo("do this thing with @bob and @mary due jun 1")
	year := strconv.Itoa(time.Now().Year())
	assert.Equal(fmt.Sprintf("%s-06-01", year), todo.Due)
}

func TestMondayOnSunday(t *testing.T) {
	assert := assert.New(t)
	parser := &Parser{}
	now, _ := time.Parse("2006-01-02", "2016-04-24")
	assert.Equal("2016-04-25", parser.monday(now))
}

func TestMondayOnMonday(t *testing.T) {
	assert := assert.New(t)
	parser := &Parser{}
	now, _ := time.Parse("2006-01-02", "2016-04-25")
	assert.Equal("2016-04-25", parser.monday(now))
}

func TestMondayOnTuesday(t *testing.T) {
	assert := assert.New(t)
	parser := &Parser{}
	now, _ := time.Parse("2006-01-02", "2016-04-26")
	assert.Equal("2016-05-02", parser.monday(now))
}

func TestTuesdayOnMonday(t *testing.T) {
	assert := assert.New(t)
	parser := &Parser{}
	now, _ := time.Parse("2006-01-02", "2016-04-25")
	assert.Equal("2016-04-26", parser.tuesday(now))
}

func TestTuesdayOnWednesday(t *testing.T) {
	assert := assert.New(t)
	parser := &Parser{}
	now, _ := time.Parse("2006-01-02", "2016-04-27")
	assert.Equal("2016-05-03", parser.tuesday(now))
}

func TestDueOnSpecificDate(t *testing.T) {
	assert := assert.New(t)
	parser := &Parser{}
	year := strconv.Itoa(time.Now().Year())
	assert.Equal(fmt.Sprintf("%s-05-02", year), parser.Due("due may 2", time.Now()))
	assert.Equal(fmt.Sprintf("%s-06-01", year), parser.Due("due jun 1", time.Now()))
}

func TestDueOnSpecificDateEuropean(t *testing.T) {
	assert := assert.New(t)
	parser := &Parser{}
	year := strconv.Itoa(time.Now().Year())
	assert.Equal(fmt.Sprintf("%s-05-02", year), parser.Due("due 2 may", time.Now()))
}

func TestDueIntelligentlyChoosesCorrectYear(t *testing.T) {
	assert := assert.New(t)
	parser := &Parser{}
	marchTime, _ := time.Parse("2006-01-02", "2016-03-25")
	januaryTime, _ := time.Parse("2006-01-02", "2016-01-05")
	septemberTime, _ := time.Parse("2006-01-02", "2016-09-25")
	decemberTime, _ := time.Parse("2006-01-02", "2016-12-25")

	assert.Equal("2016-01-10", parser.parseArbitraryDate("jan 10", januaryTime))
	assert.Equal("2016-01-10", parser.parseArbitraryDate("jan 10", marchTime))
	assert.Equal("2017-01-10", parser.parseArbitraryDate("jan 10", septemberTime))
	assert.Equal("2017-01-10", parser.parseArbitraryDate("jan 10", decemberTime))
}

func TestParseCommandIdSubject(t *testing.T) {
	assert := assert.New(t)
	parser := Parser{"es 24 a new subject"}
	command, id, subject := parser.Parse()

	assert.Equal("es", command)
	assert.Equal(24, id)
	assert.Equal("a new subject", subject)
}

func TestParseInvalidCommandIdSubject(t *testing.T) {
	assert := assert.New(t)
	input := "es a new project"
	parser := Parser{input}
	command, id, subject := parser.Parse()

	assert.Equal("", command)
	assert.Equal(-1, id)
	assert.Equal(input, subject)
}
<|endoftext|>


<commit_before><commit_msg>add transformation flag.<commit_after><|endoftext|>