<commit_before>package cloudprovider

import (
	"fmt"
	"sync"
	"time"

	"github.com/jtblin/gostatsd/types"

	log "github.com/Sirupsen/logrus"
	"github.com/koding/cache"
	"github.com/spf13/viper"
)

// All registered cloud providers.
var providersMutex sync.Mutex
var providers = make(map[string]Factory)

// Factory is a function that returns a cloud provider Interface.
type Factory func(v *viper.Viper) (Interface, error)

// Interface represents a cloud provider.
type Interface interface {
	// ProviderName returns the name of the cloud provider.
	ProviderName() string
	// SampleConfig returns the sample config for the cloud provider.
	SampleConfig() string
	// Instance returns the instance details from the cloud provider.
	Instance(IP string) (*types.Instance, error)
}

// RegisterCloudProvider registers a cloud provider.
func RegisterCloudProvider(name string, provider Factory) {
	providersMutex.Lock()
	defer providersMutex.Unlock()
	if _, found := providers[name]; found {
		log.Fatalf("Backend %q was registered twice", name)
	}
	log.Infof("Registered cloud provider %q", name)
	providers[name] = provider
}

// GetCloudProvider creates an instance of the named provider, or nil if
// the name is not known.  The error return is only used if the named provider
// was known but failed to initialize.
func GetCloudProvider(name string, v *viper.Viper) (Interface, error) {
	providersMutex.Lock()
	defer providersMutex.Unlock()
	f, found := providers[name]
	if !found {
		return nil, nil
	}
	return f(v)
}

// InitCloudProvider creates an instance of the named cloud provider.
func InitCloudProvider(name string, v *viper.Viper) (Interface, error) {
	if name == "" {
		log.Info("No cloud provider specified.")
		return nil, nil
	}

	provider, err := GetCloudProvider(name, v)
	if err != nil {
		return nil, fmt.Errorf("could not init cloud provider %q: %v", name, err)
	}
	if provider == nil {
		return nil, fmt.Errorf("unknown cloud provider %q", name)
	}
	log.Infof("Initialised cloud provider %q", name)

	return provider, nil
}

// TODO: review mutex e.g. RWMutex
var runningMutex sync.Mutex
var running = make(map[string]time.Time)
var instances = cache.NewMemoryWithTTL(1 * time.Hour)

// GetInstance returns an instance from the cache or from the cloud provider.
func GetInstance(cloud Interface, IP string) (instance *types.Instance, err error) {
	iface, err := instances.Get(IP)
	if err == nil {
		instance = iface.(*types.Instance)
		return instance, nil
	}

	if err != cache.ErrNotFound {
		// Better returning an error than hitting the cloud provider thousands of times per second
		return nil, err
	}

	runningMutex.Lock()
	last, ok := running[IP]
	runningMutex.Unlock()
	if ok {
		if last.Add(60 * time.Second).After(time.Now()) {
			time.Sleep(100 * time.Microsecond)
			return GetInstance(cloud, IP)
		}
	}
	runningMutex.Lock()
	running[IP] = time.Now()
	runningMutex.Unlock()
	defer func() {
		runningMutex.Lock()
		defer runningMutex.Unlock()
		delete(running, IP)
	}()

	if instance, err = cloud.Instance(IP); err != nil {
		return nil, err
	}
	instances.Set(IP, instance)

	return instance, nil
}

func init() {
	instances.StartGC(1 * time.Minute)
}
<commit_msg>Implement negative lookup cache (#8)<commit_after>package cloudprovider

import (
	"fmt"
	"sync"
	"time"

	"github.com/jtblin/gostatsd/types"

	log "github.com/Sirupsen/logrus"
	"github.com/koding/cache"
	"github.com/spf13/viper"
)

// All registered cloud providers.
var providersMutex sync.Mutex
var providers = make(map[string]Factory)

// Factory is a function that returns a cloud provider Interface.
type Factory func(v *viper.Viper) (Interface, error)

// Interface represents a cloud provider.
type Interface interface {
	// ProviderName returns the name of the cloud provider.
	ProviderName() string
	// SampleConfig returns the sample config for the cloud provider.
	SampleConfig() string
	// Instance returns the instance details from the cloud provider.
	Instance(IP string) (*types.Instance, error)
}

// RegisterCloudProvider registers a cloud provider.
func RegisterCloudProvider(name string, provider Factory) {
	providersMutex.Lock()
	defer providersMutex.Unlock()
	if _, found := providers[name]; found {
		log.Fatalf("Backend %q was registered twice", name)
	}
	log.Infof("Registered cloud provider %q", name)
	providers[name] = provider
}

// GetCloudProvider creates an instance of the named provider, or nil if
// the name is not known.  The error return is only used if the named provider
// was known but failed to initialize.
func GetCloudProvider(name string, v *viper.Viper) (Interface, error) {
	providersMutex.Lock()
	defer providersMutex.Unlock()
	f, found := providers[name]
	if !found {
		return nil, nil
	}
	return f(v)
}

// InitCloudProvider creates an instance of the named cloud provider.
func InitCloudProvider(name string, v *viper.Viper) (Interface, error) {
	if name == "" {
		log.Info("No cloud provider specified.")
		return nil, nil
	}

	provider, err := GetCloudProvider(name, v)
	if err != nil {
		return nil, fmt.Errorf("could not init cloud provider %q: %v", name, err)
	}
	if provider == nil {
		return nil, fmt.Errorf("unknown cloud provider %q", name)
	}
	log.Infof("Initialised cloud provider %q", name)

	return provider, nil
}

// TODO: review mutex e.g. RWMutex
var runningMutex sync.Mutex
var running = make(map[string]time.Time)
var instances = cache.NewMemoryWithTTL(1 * time.Hour)
var failed = cache.NewMemoryWithTTL(1 * time.Minute)

// GetInstance returns an instance from the cache or from the cloud provider.
func GetInstance(cloud Interface, IP string) (instance *types.Instance, err error) {
	iface, err := instances.Get(IP)
	if err == nil {
		instance = iface.(*types.Instance)
		return instance, nil
	}

	if err != cache.ErrNotFound {
		// Better returning an error than hitting the cloud provider thousands of times per second
		return nil, err
	}

	cachedErr, err := failed.Get(IP)
	if err == nil {
		// We have a cached failure
		return nil, cachedErr.(error)
	}

	if err != cache.ErrNotFound {
		// Some error getting it from cache?
		return nil, err
	}

	runningMutex.Lock()
	last, ok := running[IP]
	runningMutex.Unlock()
	if ok {
		if last.Add(60 * time.Second).After(time.Now()) {
			time.Sleep(100 * time.Microsecond)
			return GetInstance(cloud, IP)
		}
	}
	runningMutex.Lock()
	running[IP] = time.Now()
	runningMutex.Unlock()
	defer func() {
		runningMutex.Lock()
		defer runningMutex.Unlock()
		delete(running, IP)
	}()

	if instance, err = cloud.Instance(IP); err != nil {
		failed.Set(IP, fmt.Errorf("Cached failure: %v for %s", err, IP))
		return nil, err
	}
	instances.Set(IP, instance)

	return instance, nil
}

func init() {
	instances.StartGC(1 * time.Minute)
}
<|endoftext|>


<commit_before>package DUP

import (
	"github.com/stretchr/testify/assert"
	"net"
	"sync"
	"testing"
)

// Mock messenger for client tests. Collects all sent messages into a channel
// Does no resolving. Has a capacity for 50 sent messages
type MockMessenger struct {
	sentMessages chan Message
}

func NewMockMessenger() (Messenger, chan Message) {
	msgChan := make(chan Message, 50)
	return MockMessenger{
		sentMessages: msgChan,
	}, msgChan
}

func (messenger MockMessenger) Send(msg Message) error {
	messenger.sentMessages <- msg
	return nil
}

func (messenger MockMessenger) Recv(channel chan Message) error {
	return nil
}

// Does nothing
func (messenger MockMessenger) resolve(addr string) (interface{}, error) {
	return nil, nil
}

var nodeNumLock sync.Mutex
var nodeNum int = 0

func GetNode(t *testing.T) Node {
	nodeNumLock.Lock()
	addr := net.ParseIP("0.0.0.0")

	node := Node{
		Name: string(nodeNum),
		Port: nodeNum,
		Addr: addr,
	}
	nodeNum++
	nodeNumLock.Unlock()
	return node
}

func GetClient_DataOnly(t *testing.T) *client {
	c := new(client)
	f := ClientFactory{}
	f.initializeData(c)

	c.ActiveMembers[c.node.Name] = c.node
	return c
}

func GetActivateMessage(t *testing.T, nodes []Node) Message {
	msg, err := createActivateMsg(nodes)
	if err != nil {
		t.Errorf("Failed to create activate message")
	}

	return msg
}

func GetBarrierMessage(t *testing.T, source string) Message {
	msg := createBarrierMsg(source)
	return msg
}

func TestClient_IsActive(t *testing.T) {
	assert := assert.New(t)
	c := GetClient_DataOnly(t)

	assert.True(c.IsActive())

	c.updateActiveMemberList([]Node{})
	assert.False(c.IsActive())

}

func TestClient_Barrier_Blocking(t *testing.T) {
	t.Errorf("Not Implemented")
}

func TestClient_HandleActivate(t *testing.T) {
	assert := assert.New(t)
	c := GetClient_DataOnly(t)
	emptyActivate := GetActivateMessage(t, []Node{})
	c.handleActivateMessage(emptyActivate)
	assert.Equal(0, c.NumActiveMembers())

	node := GetNode(t)
	sameActivate := GetActivateMessage(t, []Node{node, node})
	c.handleActivateMessage(sameActivate)
	assert.Equal(1, c.NumActiveMembers())

	twoActivate := GetActivateMessage(t, []Node{GetNode(t), GetNode(t)})
	c.handleActivateMessage(twoActivate)
	assert.Equal(2, c.NumActiveMembers())

}

func TestClient_Close(t *testing.T) {
	t.Errorf("Not Implemented")
}
<commit_msg>Implemented test for Barrier.<commit_after>package DUP

import (
	"github.com/stretchr/testify/assert"
	"net"
	"sync"
	"testing"
	"time"
)

// Mock messenger for client tests. Collects all sent messages into a channel
// Does no resolving. Has a capacity for 50 sent messages
type MockMessenger struct {
	sentMessages chan Message
}

func NewMockMessenger() (Messenger, chan Message) {
	msgChan := make(chan Message, 50)
	return MockMessenger{
		sentMessages: msgChan,
	}, msgChan
}

func (messenger MockMessenger) Send(msg Message) error {
	messenger.sentMessages <- msg
	return nil
}

func (messenger MockMessenger) Recv(channel chan Message) error {
	return nil
}

// Does nothing
func (messenger MockMessenger) resolve(addr string) (interface{}, error) {
	return nil, nil
}

var nodeNumLock sync.Mutex
var nodeNum int = 0

func GetNode(t *testing.T) Node {
	nodeNumLock.Lock()
	addr := net.ParseIP("0.0.0.0")

	node := Node{
		Name: string(nodeNum),
		Port: nodeNum,
		Addr: addr,
	}
	nodeNum++
	nodeNumLock.Unlock()
	return node
}

func GetClient_DataOnly(t *testing.T) *client {
	c := new(client)
	f := ClientFactory{}
	f.initializeData(c)

	c.ActiveMembers[c.node.Name] = c.node
	return c
}

func GetActivateMessage(t *testing.T, nodes []Node) Message {
	msg, err := createActivateMsg(nodes)
	if err != nil {
		t.Errorf("Failed to create activate message")
	}

	return msg
}

func GetBarrierMessage(t *testing.T, source string) Message {
	msg := createBarrierMsg(source)
	return msg
}

func TestClient_IsActive(t *testing.T) {
	assert := assert.New(t)
	c := GetClient_DataOnly(t)

	assert.True(c.IsActive())

	c.updateActiveMemberList([]Node{})
	assert.False(c.IsActive())

}

func TestClient_Barrier(t *testing.T) {
	assert := assert.New(t)
	timer := time.AfterFunc(500*time.Millisecond, func() {
		panic("Hung during barrier test!")
	})
	defer timer.Stop()

	c := GetClient_DataOnly(t)

	// Test single client case, only active node, should return immediately
	c.Barrier()

	// Test with multiple active nodes
	messenger, sent := NewMockMessenger()
	c.messenger = messenger
	activeNodes := []Node{GetNode(t), GetNode(t), c.node}
	c.updateActiveMemberList(activeNodes)

	blocked := false
	go func() {
		c.Barrier()
		if !blocked {
			t.Error("Barrier didn't block")
		}

	}()
	// Should send 3 messages
	for i := 0; i < len(activeNodes); i++ {
		msg := <-sent
		assert.Equal(barrierMsg, msg.Type)
	}

	c.HandleMessage(GetBarrierMessage(t, activeNodes[0].Name))
	c.HandleMessage(GetBarrierMessage(t, activeNodes[1].Name))
	blocked = true
	c.HandleMessage(GetBarrierMessage(t, activeNodes[2].Name))

}

func TestClient_HandleActivate(t *testing.T) {
	assert := assert.New(t)
	c := GetClient_DataOnly(t)
	emptyActivate := GetActivateMessage(t, []Node{})
	c.handleActivateMessage(emptyActivate)
	assert.Equal(0, c.NumActiveMembers())

	node := GetNode(t)
	sameActivate := GetActivateMessage(t, []Node{node, node})
	c.handleActivateMessage(sameActivate)
	assert.Equal(1, c.NumActiveMembers())

	twoActivate := GetActivateMessage(t, []Node{GetNode(t), GetNode(t)})
	c.handleActivateMessage(twoActivate)
	assert.Equal(2, c.NumActiveMembers())

}

func TestClient_Close(t *testing.T) {
	t.Errorf("Not Implemented")
}

func TestClient_Broadcast(t *testing.T) {
	t.Errorf("Not Implemented")
}
<|endoftext|>


<commit_before>package app

import (
	"bytes"
	"net/http"
	"fmt"
	"encoding/xml"

	"appengine"
	"appengine/datastore"
	"appengine/mail"
	"appengine/urlfetch"
	//"appengine/user"
)

func updateFeed(c appengine.Context, cl *http.Client, fk *datastore.Key) error {
	resp, err := cl.Get(fk.StringID())
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	decoder := xml.NewDecoder(resp.Body)
	var rfeed RSS
	err = datastore.Get(c, fk, &rfeed)
	if err != nil {
		return err
	}
	if rfeed.IsAtom {
		var afeed Atom
		err = decoder.Decode(&afeed)
		if err != nil {
			return err
		}
		return afeed.update(c, fk)
	} else {
		err = decoder.Decode(&rfeed)
		if err != nil {
			return err
		}
		return rfeed.update(c, fk)
	}
	panic("unreachable")
}

func updater(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	if !user.IsAdmin(c) {
		http.Error(w, "Access denied.", http.StatusUnauthorized)
		return
	}
	cl := urlfetch.Client(c)
	feedRoot := datastore.NewKey(c, "feedRoot", "feedRoot", 0, nil)
	q := datastore.NewQuery("feed").Ancestor(feedRoot).KeysOnly()
	iter := q.Run(c)
	ch := make(chan error)
	count := 0
	for {
		fk, err := iter.Next(c)
		if err == datastore.Done {
			break
		}
		if err != nil {
			handleError(w, err)
			return
		}
		go func(fk *datastore.Key) {
			err = updateFeed(c, cl, fk)
			ch <- err
		}(fk)
		count++
	}
	buf := new(bytes.Buffer)
	for count != 0 {
		fmt.Fprintln(buf, <-ch)
		count--
	}
	err := mail.Send(c, &mail.Message{
		Sender: "updates@simplecta.appspot.com",
		To: []string{"anschelsc@gmail.com"},
		Subject: "Errors from simplecta update",
		Body: buf.String(),
	})
	if err != nil {
		handleError(w, err)
		return
	}
	fmt.Fprintln(w, "Done.")
}

<commit_msg>go fmt<commit_after>package app

import (
	"bytes"
	"encoding/xml"
	"fmt"
	"net/http"

	"appengine"
	"appengine/datastore"
	"appengine/mail"
	"appengine/urlfetch"
	"appengine/user"
)

func updateFeed(c appengine.Context, cl *http.Client, fk *datastore.Key) error {
	resp, err := cl.Get(fk.StringID())
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	decoder := xml.NewDecoder(resp.Body)
	var rfeed RSS
	err = datastore.Get(c, fk, &rfeed)
	if err != nil {
		return err
	}
	if rfeed.IsAtom {
		var afeed Atom
		err = decoder.Decode(&afeed)
		if err != nil {
			return err
		}
		return afeed.update(c, fk)
	} else {
		err = decoder.Decode(&rfeed)
		if err != nil {
			return err
		}
		return rfeed.update(c, fk)
	}
	panic("unreachable")
}

func updater(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	if !user.IsAdmin(c) {
		http.Error(w, "Access denied.", http.StatusUnauthorized)
		return
	}
	cl := urlfetch.Client(c)
	feedRoot := datastore.NewKey(c, "feedRoot", "feedRoot", 0, nil)
	q := datastore.NewQuery("feed").Ancestor(feedRoot).KeysOnly()
	iter := q.Run(c)
	ch := make(chan error)
	count := 0
	for {
		fk, err := iter.Next(c)
		if err == datastore.Done {
			break
		}
		if err != nil {
			handleError(w, err)
			return
		}
		go func(fk *datastore.Key) {
			err = updateFeed(c, cl, fk)
			ch <- err
		}(fk)
		count++
	}
	buf := new(bytes.Buffer)
	for count != 0 {
		fmt.Fprintln(buf, <-ch)
		count--
	}
	err := mail.Send(c, &mail.Message{
		Sender:  "updates@simplecta.appspot.com",
		To:      []string{"anschelsc@gmail.com"},
		Subject: "Errors from simplecta update",
		Body:    buf.String(),
	})
	if err != nil {
		handleError(w, err)
		return
	}
	fmt.Fprintln(w, "Done.")
}
<|endoftext|>


<commit_before>package work

import (
	"fmt"
	// "github.com/garyburd/redigo/redis"
	"github.com/stretchr/testify/assert"
	// "sort"
	"testing"
	"time"
)

type TestContext struct{}

func TestClientWorkerPoolHeartbeats(t *testing.T) {
	pool := newTestPool(":6379")
	ns := "work"
	cleanKeyspace(ns, pool)

	wp := NewWorkerPool(TestContext{}, 10, ns, pool)
	wp.Job("wat", func(job *Job) error { return nil })
	wp.Job("bob", func(job *Job) error { return nil })
	wp.Start()

	wp2 := NewWorkerPool(TestContext{}, 11, ns, pool)
	wp2.Job("foo", func(job *Job) error { return nil })
	wp2.Job("bar", func(job *Job) error { return nil })
	wp2.Start()

	time.Sleep(20 * time.Millisecond)

	client := NewClient(ns, pool)

	hbs, err := client.WorkerPoolHeartbeats()
	assert.NoError(t, err)
	assert.Equal(t, 2, len(hbs))
	if len(hbs) == 2 {
		var hbwp, hbwp2 *WorkerPoolHeartbeat

		if wp.workerPoolID == hbs[0].WorkerPoolID {
			hbwp = hbs[0]
			hbwp2 = hbs[1]
		} else {
			hbwp = hbs[1]
			hbwp2 = hbs[0]
		}

		assert.Equal(t, wp.workerPoolID, hbwp.WorkerPoolID)
		assert.Equal(t, uint(10), hbwp.Concurrency)
		assert.Equal(t, []string{"bob", "wat"}, hbwp.JobNames)
		assert.Equal(t, wp.workerIDs(), hbwp.WorkerIDs)

		assert.Equal(t, wp2.workerPoolID, hbwp2.WorkerPoolID)
		assert.Equal(t, uint(11), hbwp2.Concurrency)
		assert.Equal(t, []string{"bar", "foo"}, hbwp2.JobNames)
		assert.Equal(t, wp2.workerIDs(), hbwp2.WorkerIDs)
	}

	wp.Stop()
	wp2.Stop()

	hbs, err = client.WorkerPoolHeartbeats()
	assert.NoError(t, err)
	assert.Equal(t, 0, len(hbs))
}

func TestClientWorkerObservations(t *testing.T) {
	pool := newTestPool(":6379")
	ns := "work"
	cleanKeyspace(ns, pool)

	enqueuer := NewEnqueuer(ns, pool)
	err := enqueuer.Enqueue("wat", 1, 2)
	assert.Nil(t, err)
	err = enqueuer.Enqueue("foo", 3, 4)
	assert.Nil(t, err)

	wp := NewWorkerPool(TestContext{}, 10, ns, pool)
	wp.Job("wat", func(job *Job) error {
		time.Sleep(50 * time.Millisecond)
		return nil
	})
	wp.Job("foo", func(job *Job) error {
		time.Sleep(50 * time.Millisecond)
		return nil
	})
	wp.Start()

	time.Sleep(10 * time.Millisecond)

	client := NewClient(ns, pool)
	observations, err := client.WorkerObservations()
	assert.NoError(t, err)
	assert.Equal(t, 10, len(observations))

	watCount := 0
	fooCount := 0
	for _, ob := range observations {
		if ob.JobName == "foo" {
			fooCount++
			assert.True(t, ob.IsBusy)
			assert.Equal(t, "[3,4]", ob.ArgsJSON)
			assert.True(t, (nowEpochSeconds()-ob.StartedAt) <= 3)
			assert.True(t, ob.JobID != "")
		} else if ob.JobName == "wat" {
			watCount++
			assert.True(t, ob.IsBusy)
			assert.Equal(t, "[1,2]", ob.ArgsJSON)
			assert.True(t, (nowEpochSeconds()-ob.StartedAt) <= 3)
			assert.True(t, ob.JobID != "")
		} else {
			assert.False(t, ob.IsBusy)
		}
		assert.True(t, ob.WorkerID != "")
	}
	assert.Equal(t, 1, watCount)
	assert.Equal(t, 1, fooCount)

	// time.Sleep(2000 * time.Millisecond)
	//
	// observations, err = client.WorkerObservations()
	// assert.NoError(t, err)
	// assert.Equal(t, 10, len(observations))
	// for _, ob := range observations {
	// 	assert.False(t, ob.IsBusy)
	// 	assert.True(t, ob.WorkerID != "")
	// }

	wp.Stop()

	observations, err = client.WorkerObservations()
	assert.NoError(t, err)
	assert.Equal(t, 0, len(observations))
}

func TestClientQueues(t *testing.T) {
	pool := newTestPool(":6379")
	ns := "work"
	cleanKeyspace(ns, pool)

	enqueuer := NewEnqueuer(ns, pool)
	err := enqueuer.Enqueue("wat", 1, 2)
	err = enqueuer.Enqueue("foo", 3, 4)
	err = enqueuer.Enqueue("zaz", 3, 4)

	// Start a pool to work on it. It's going to work on the queues
	// side effect of that is knowing which jobs are avail
	wp := NewWorkerPool(TestContext{}, 10, ns, pool)
	wp.Job("wat", func(job *Job) error {
		return nil
	})
	wp.Job("foo", func(job *Job) error {
		return nil
	})
	wp.Job("zaz", func(job *Job) error {
		return nil
	})
	wp.Start()
	time.Sleep(20 * time.Millisecond)
	wp.Stop()

	setNowEpochSecondsMock(1425263409)
	defer resetNowEpochSecondsMock()
	err = enqueuer.Enqueue("foo", 3, 4)
	setNowEpochSecondsMock(1425263509)
	err = enqueuer.Enqueue("foo", 3, 4)
	setNowEpochSecondsMock(1425263609)
	err = enqueuer.Enqueue("wat", 3, 4)

	setNowEpochSecondsMock(1425263709)
	client := NewClient(ns, pool)
	queues, err := client.Queues()
	assert.NoError(t, err)

	assert.Equal(t, 3, len(queues))
	assert.Equal(t, "foo", queues[0].JobName)
	assert.Equal(t, 2, queues[0].Count)
	assert.Equal(t, 300, queues[0].Latency)
	assert.Equal(t, "wat", queues[1].JobName)
	assert.Equal(t, 1, queues[1].Count)
	assert.Equal(t, 100, queues[1].Latency)
	assert.Equal(t, "zaz", queues[2].JobName)
	assert.Equal(t, 0, queues[2].Count)
	assert.Equal(t, 0, queues[2].Latency)
}

func TestClientScheduledJobs(t *testing.T) {
	pool := newTestPool(":6379")
	ns := "work"
	cleanKeyspace(ns, pool)

	enqueuer := NewEnqueuer(ns, pool)

	setNowEpochSecondsMock(1425263409)
	defer resetNowEpochSecondsMock()
	err := enqueuer.EnqueueIn("wat", 0, 1, 2)
	err = enqueuer.EnqueueIn("zaz", 4, 3, 4)
	err = enqueuer.EnqueueIn("foo", 2, 3, 4)

	client := NewClient(ns, pool)
	jobs, err := client.ScheduledJobs(1)
	assert.NoError(t, err)
	assert.Equal(t, 3, len(jobs))
	if len(jobs) == 3 {
		assert.Equal(t, 1425263409, jobs[0].RunAt)
		assert.Equal(t, 1425263411, jobs[1].RunAt)
		assert.Equal(t, 1425263413, jobs[2].RunAt)

		assert.Equal(t, "wat", jobs[0].Name)
		assert.Equal(t, "foo", jobs[1].Name)
		assert.Equal(t, "zaz", jobs[2].Name)

		assert.Equal(t, 1425263409, jobs[0].EnqueuedAt)
		assert.Equal(t, 1425263409, jobs[1].EnqueuedAt)
		assert.Equal(t, 1425263409, jobs[2].EnqueuedAt)

		assert.Equal(t, interface{}(1), jobs[0].Args[0])
		assert.Equal(t, interface{}(2), jobs[0].Args[1])

		assert.Equal(t, 0, jobs[0].Fails)
		assert.Equal(t, 0, jobs[1].Fails)
		assert.Equal(t, 0, jobs[2].Fails)

		assert.Equal(t, 0, jobs[0].FailedAt)
		assert.Equal(t, 0, jobs[1].FailedAt)
		assert.Equal(t, 0, jobs[2].FailedAt)

		assert.Equal(t, "", jobs[0].LastErr)
		assert.Equal(t, "", jobs[1].LastErr)
		assert.Equal(t, "", jobs[2].LastErr)
	}
}

func TestClientRetryJobs(t *testing.T) {
	pool := newTestPool(":6379")
	ns := "work"
	cleanKeyspace(ns, pool)

	setNowEpochSecondsMock(1425263409)
	defer resetNowEpochSecondsMock()

	enqueuer := NewEnqueuer(ns, pool)
	err := enqueuer.Enqueue("wat", 1, 2)
	assert.Nil(t, err)

	setNowEpochSecondsMock(1425263429)

	wp := NewWorkerPool(TestContext{}, 10, ns, pool)
	wp.Job("wat", func(job *Job) error {
		return fmt.Errorf("ohno")
	})
	wp.Start()
	wp.Join()
	wp.Stop()

	client := NewClient(ns, pool)
	jobs, err := client.RetryJobs(1)
	assert.NoError(t, err)
	assert.Equal(t, 1, len(jobs))

	if len(jobs) == 1 {
		assert.Equal(t, 1425263429, jobs[0].FailedAt)
		assert.Equal(t, "wat", jobs[0].Name)
		assert.Equal(t, 1425263409, jobs[0].EnqueuedAt)
		assert.Equal(t, interface{}(1), jobs[0].Args[0])
		assert.Equal(t, 1, jobs[0].Fails)
		assert.Equal(t, 1425263429, jobs[0].Job.FailedAt)
		assert.Equal(t, "ohno", jobs[0].LastErr)
	}
}
<commit_msg>test dead jobs<commit_after>package work

import (
	"fmt"
	// "github.com/garyburd/redigo/redis"
	"github.com/stretchr/testify/assert"
	// "sort"
	"testing"
	"time"
)

type TestContext struct{}

func TestClientWorkerPoolHeartbeats(t *testing.T) {
	pool := newTestPool(":6379")
	ns := "work"
	cleanKeyspace(ns, pool)

	wp := NewWorkerPool(TestContext{}, 10, ns, pool)
	wp.Job("wat", func(job *Job) error { return nil })
	wp.Job("bob", func(job *Job) error { return nil })
	wp.Start()

	wp2 := NewWorkerPool(TestContext{}, 11, ns, pool)
	wp2.Job("foo", func(job *Job) error { return nil })
	wp2.Job("bar", func(job *Job) error { return nil })
	wp2.Start()

	time.Sleep(20 * time.Millisecond)

	client := NewClient(ns, pool)

	hbs, err := client.WorkerPoolHeartbeats()
	assert.NoError(t, err)
	assert.Equal(t, 2, len(hbs))
	if len(hbs) == 2 {
		var hbwp, hbwp2 *WorkerPoolHeartbeat

		if wp.workerPoolID == hbs[0].WorkerPoolID {
			hbwp = hbs[0]
			hbwp2 = hbs[1]
		} else {
			hbwp = hbs[1]
			hbwp2 = hbs[0]
		}

		assert.Equal(t, wp.workerPoolID, hbwp.WorkerPoolID)
		assert.Equal(t, uint(10), hbwp.Concurrency)
		assert.Equal(t, []string{"bob", "wat"}, hbwp.JobNames)
		assert.Equal(t, wp.workerIDs(), hbwp.WorkerIDs)

		assert.Equal(t, wp2.workerPoolID, hbwp2.WorkerPoolID)
		assert.Equal(t, uint(11), hbwp2.Concurrency)
		assert.Equal(t, []string{"bar", "foo"}, hbwp2.JobNames)
		assert.Equal(t, wp2.workerIDs(), hbwp2.WorkerIDs)
	}

	wp.Stop()
	wp2.Stop()

	hbs, err = client.WorkerPoolHeartbeats()
	assert.NoError(t, err)
	assert.Equal(t, 0, len(hbs))
}

func TestClientWorkerObservations(t *testing.T) {
	pool := newTestPool(":6379")
	ns := "work"
	cleanKeyspace(ns, pool)

	enqueuer := NewEnqueuer(ns, pool)
	err := enqueuer.Enqueue("wat", 1, 2)
	assert.Nil(t, err)
	err = enqueuer.Enqueue("foo", 3, 4)
	assert.Nil(t, err)

	wp := NewWorkerPool(TestContext{}, 10, ns, pool)
	wp.Job("wat", func(job *Job) error {
		time.Sleep(50 * time.Millisecond)
		return nil
	})
	wp.Job("foo", func(job *Job) error {
		time.Sleep(50 * time.Millisecond)
		return nil
	})
	wp.Start()

	time.Sleep(10 * time.Millisecond)

	client := NewClient(ns, pool)
	observations, err := client.WorkerObservations()
	assert.NoError(t, err)
	assert.Equal(t, 10, len(observations))

	watCount := 0
	fooCount := 0
	for _, ob := range observations {
		if ob.JobName == "foo" {
			fooCount++
			assert.True(t, ob.IsBusy)
			assert.Equal(t, "[3,4]", ob.ArgsJSON)
			assert.True(t, (nowEpochSeconds()-ob.StartedAt) <= 3)
			assert.True(t, ob.JobID != "")
		} else if ob.JobName == "wat" {
			watCount++
			assert.True(t, ob.IsBusy)
			assert.Equal(t, "[1,2]", ob.ArgsJSON)
			assert.True(t, (nowEpochSeconds()-ob.StartedAt) <= 3)
			assert.True(t, ob.JobID != "")
		} else {
			assert.False(t, ob.IsBusy)
		}
		assert.True(t, ob.WorkerID != "")
	}
	assert.Equal(t, 1, watCount)
	assert.Equal(t, 1, fooCount)

	// time.Sleep(2000 * time.Millisecond)
	//
	// observations, err = client.WorkerObservations()
	// assert.NoError(t, err)
	// assert.Equal(t, 10, len(observations))
	// for _, ob := range observations {
	// 	assert.False(t, ob.IsBusy)
	// 	assert.True(t, ob.WorkerID != "")
	// }

	wp.Stop()

	observations, err = client.WorkerObservations()
	assert.NoError(t, err)
	assert.Equal(t, 0, len(observations))
}

func TestClientQueues(t *testing.T) {
	pool := newTestPool(":6379")
	ns := "work"
	cleanKeyspace(ns, pool)

	enqueuer := NewEnqueuer(ns, pool)
	err := enqueuer.Enqueue("wat", 1, 2)
	err = enqueuer.Enqueue("foo", 3, 4)
	err = enqueuer.Enqueue("zaz", 3, 4)

	// Start a pool to work on it. It's going to work on the queues
	// side effect of that is knowing which jobs are avail
	wp := NewWorkerPool(TestContext{}, 10, ns, pool)
	wp.Job("wat", func(job *Job) error {
		return nil
	})
	wp.Job("foo", func(job *Job) error {
		return nil
	})
	wp.Job("zaz", func(job *Job) error {
		return nil
	})
	wp.Start()
	time.Sleep(20 * time.Millisecond)
	wp.Stop()

	setNowEpochSecondsMock(1425263409)
	defer resetNowEpochSecondsMock()
	err = enqueuer.Enqueue("foo", 3, 4)
	setNowEpochSecondsMock(1425263509)
	err = enqueuer.Enqueue("foo", 3, 4)
	setNowEpochSecondsMock(1425263609)
	err = enqueuer.Enqueue("wat", 3, 4)

	setNowEpochSecondsMock(1425263709)
	client := NewClient(ns, pool)
	queues, err := client.Queues()
	assert.NoError(t, err)

	assert.Equal(t, 3, len(queues))
	assert.Equal(t, "foo", queues[0].JobName)
	assert.Equal(t, 2, queues[0].Count)
	assert.Equal(t, 300, queues[0].Latency)
	assert.Equal(t, "wat", queues[1].JobName)
	assert.Equal(t, 1, queues[1].Count)
	assert.Equal(t, 100, queues[1].Latency)
	assert.Equal(t, "zaz", queues[2].JobName)
	assert.Equal(t, 0, queues[2].Count)
	assert.Equal(t, 0, queues[2].Latency)
}

func TestClientScheduledJobs(t *testing.T) {
	pool := newTestPool(":6379")
	ns := "work"
	cleanKeyspace(ns, pool)

	enqueuer := NewEnqueuer(ns, pool)

	setNowEpochSecondsMock(1425263409)
	defer resetNowEpochSecondsMock()
	err := enqueuer.EnqueueIn("wat", 0, 1, 2)
	err = enqueuer.EnqueueIn("zaz", 4, 3, 4)
	err = enqueuer.EnqueueIn("foo", 2, 3, 4)

	client := NewClient(ns, pool)
	jobs, err := client.ScheduledJobs(1)
	assert.NoError(t, err)
	assert.Equal(t, 3, len(jobs))
	if len(jobs) == 3 {
		assert.Equal(t, 1425263409, jobs[0].RunAt)
		assert.Equal(t, 1425263411, jobs[1].RunAt)
		assert.Equal(t, 1425263413, jobs[2].RunAt)

		assert.Equal(t, "wat", jobs[0].Name)
		assert.Equal(t, "foo", jobs[1].Name)
		assert.Equal(t, "zaz", jobs[2].Name)

		assert.Equal(t, 1425263409, jobs[0].EnqueuedAt)
		assert.Equal(t, 1425263409, jobs[1].EnqueuedAt)
		assert.Equal(t, 1425263409, jobs[2].EnqueuedAt)

		assert.Equal(t, interface{}(1), jobs[0].Args[0])
		assert.Equal(t, interface{}(2), jobs[0].Args[1])

		assert.Equal(t, 0, jobs[0].Fails)
		assert.Equal(t, 0, jobs[1].Fails)
		assert.Equal(t, 0, jobs[2].Fails)

		assert.Equal(t, 0, jobs[0].FailedAt)
		assert.Equal(t, 0, jobs[1].FailedAt)
		assert.Equal(t, 0, jobs[2].FailedAt)

		assert.Equal(t, "", jobs[0].LastErr)
		assert.Equal(t, "", jobs[1].LastErr)
		assert.Equal(t, "", jobs[2].LastErr)
	}
}

func TestClientRetryJobs(t *testing.T) {
	pool := newTestPool(":6379")
	ns := "work"
	cleanKeyspace(ns, pool)

	setNowEpochSecondsMock(1425263409)
	defer resetNowEpochSecondsMock()

	enqueuer := NewEnqueuer(ns, pool)
	err := enqueuer.Enqueue("wat", 1, 2)
	assert.Nil(t, err)

	setNowEpochSecondsMock(1425263429)

	wp := NewWorkerPool(TestContext{}, 10, ns, pool)
	wp.Job("wat", func(job *Job) error {
		return fmt.Errorf("ohno")
	})
	wp.Start()
	wp.Join()
	wp.Stop()

	client := NewClient(ns, pool)
	jobs, err := client.RetryJobs(1)
	assert.NoError(t, err)
	assert.Equal(t, 1, len(jobs))

	if len(jobs) == 1 {
		assert.Equal(t, 1425263429, jobs[0].FailedAt)
		assert.Equal(t, "wat", jobs[0].Name)
		assert.Equal(t, 1425263409, jobs[0].EnqueuedAt)
		assert.Equal(t, interface{}(1), jobs[0].Args[0])
		assert.Equal(t, 1, jobs[0].Fails)
		assert.Equal(t, 1425263429, jobs[0].Job.FailedAt)
		assert.Equal(t, "ohno", jobs[0].LastErr)
	}
}

func TestClientDeadJobs(t *testing.T) {
	pool := newTestPool(":6379")
	ns := "work"
	cleanKeyspace(ns, pool)

	setNowEpochSecondsMock(1425263409)
	defer resetNowEpochSecondsMock()

	enqueuer := NewEnqueuer(ns, pool)
	err := enqueuer.Enqueue("wat", 1, 2)
	assert.Nil(t, err)

	setNowEpochSecondsMock(1425263429)

	wp := NewWorkerPool(TestContext{}, 10, ns, pool)
	wp.JobWithOptions("wat", JobOptions{Priority: 1, MaxFails: 0}, func(job *Job) error {
		return fmt.Errorf("ohno")
	})
	wp.Start()
	wp.Join()
	wp.Stop()

	client := NewClient(ns, pool)
	jobs, err := client.DeadJobs(1)
	assert.NoError(t, err)
	assert.Equal(t, 1, len(jobs))

	if len(jobs) == 1 {
		assert.Equal(t, 1425263429, jobs[0].FailedAt)
		assert.Equal(t, "wat", jobs[0].Name)
		assert.Equal(t, 1425263409, jobs[0].EnqueuedAt)
		assert.Equal(t, interface{}(1), jobs[0].Args[0])
		assert.Equal(t, 1, jobs[0].Fails)
		assert.Equal(t, 1425263429, jobs[0].Job.FailedAt)
		assert.Equal(t, "ohno", jobs[0].LastErr)
	}

	// Test pagination a bit
	jobs, err = client.DeadJobs(2)
	assert.NoError(t, err)
	assert.Equal(t, 0, len(jobs))
}
<|endoftext|>


<commit_before>package command

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/funkygao/gocli"
)

type Deploy struct {
	Ui  cli.Ui
	Cmd string

	root    string
	rsyslog bool
}

func (this *Deploy) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("deploy", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&this.root, "p", defaultPrefix, "")
	cmdFlags.BoolVar(&this.rsyslog, "rsyslog", true, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	err := os.MkdirAll(this.root, 0755)
	swalllow(err)
	err = os.MkdirAll(fmt.Sprintf("%s/bin", this.root), 0755)
	swalllow(err)
	err = os.MkdirAll(fmt.Sprintf("%s/sbin", this.root), 0755)
	swalllow(err)
	err = os.MkdirAll(fmt.Sprintf("%s/logs", this.root), 0755)
	swalllow(err)
	err = os.MkdirAll(fmt.Sprintf("%s/src", this.root), 0755)
	swalllow(err)

	// install files
	b, _ := Asset("templates/haproxy-1.6.3.tar.gz")
	srcPath := fmt.Sprintf("%s/src/haproxy-1.6.3.tar.gz", this.root)
	err = ioutil.WriteFile(srcPath, b, 0644)
	swalllow(err)
	b, _ = Asset("templates/hatop-0.7.7.tar.gz")
	hatop := fmt.Sprintf("%s/src/hatop-0.7.7.tar.gz", this.root)
	err = ioutil.WriteFile(hatop, b, 0644)
	swalllow(err)
	b, _ = Asset("templates/init.ehaproxy")
	initPath := fmt.Sprintf("%s/src/init.ehaproxy", this.root)
	err = ioutil.WriteFile(initPath, b, 0755)
	swalllow(err)

	this.Ui.Info("useradd haproxy")
	this.Ui.Info(fmt.Sprintf("compile haproxy to %s/sbin: make TARGET=xxx USE_ZLIB=yes", this.root))
	this.Ui.Info(fmt.Sprintf("cp %s to /etc/init.d/ehaproxy", initPath))
	this.Ui.Info(fmt.Sprintf("chkconfig --add ehaproxy"))

	this.configKernal()

	if this.rsyslog {
		this.configRsyslog()
	}

	return
}

func (this *Deploy) configKernal() {
	this.Ui.Warn("net.core.somaxconn = 16384")
	this.Ui.Warn("net.core.netdev_max_backlog = 2500")
}

func (this *Deploy) configRsyslog() {
	this.Ui.Output("install and setup rsyslog for haproxy")
	this.Ui.Output(fmt.Sprintf(`
vim /etc/rsyslog.conf
$ModLoad imudp
$UDPServerAddress 127.0.0.1
$UDPServerRun 514

vim  /etc/rsyslog.d/haproxy.conf
local3.*     /var/log/haproxy.log

vim /etc/sysconfig/rsyslog
SYSLOGD_OPTIONS=”-c 2 -r -m 0″
#-c 2 使用兼容模式，默认是 -c 5
#-r 开启远程日志
#-m 0 标记时间戳。单位是分钟，为0时，表示禁用该功能
		`))
}

func (this *Deploy) Synopsis() string {
	return fmt.Sprintf("Deploy %s system on localhost", this.Cmd)
}

func (this *Deploy) Help() string {
	help := fmt.Sprintf(`
Usage: %s deploy [options]

    Deploy %s system on localhost

Options:

    -p prefix dir
      Defaults %s

    -rsyslog
      Display rsyslog integration with haproxy

`, this.Cmd, this.Cmd, defaultPrefix)
	return strings.TrimSpace(help)
}
<commit_msg>check user 'haproxy' exists before deploy ehaproxy<commit_after>package command

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/user"
	"strings"

	"github.com/funkygao/gocli"
)

type Deploy struct {
	Ui  cli.Ui
	Cmd string

	root    string
	rsyslog bool
}

func (this *Deploy) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("deploy", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&this.root, "p", defaultPrefix, "")
	cmdFlags.BoolVar(&this.rsyslog, "rsyslog", true, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	// must useradd haproxy before deploy
	if _, err := user.Lookup("haproxy"); err != nil {
		panic(err)
	}

	err := os.MkdirAll(this.root, 0755)
	swalllow(err)
	err = os.MkdirAll(fmt.Sprintf("%s/bin", this.root), 0755)
	swalllow(err)
	err = os.MkdirAll(fmt.Sprintf("%s/sbin", this.root), 0755)
	swalllow(err)
	err = os.MkdirAll(fmt.Sprintf("%s/logs", this.root), 0755)
	swalllow(err)
	err = os.MkdirAll(fmt.Sprintf("%s/src", this.root), 0755)
	swalllow(err)

	// install files
	b, _ := Asset("templates/haproxy-1.6.3.tar.gz")
	srcPath := fmt.Sprintf("%s/src/haproxy-1.6.3.tar.gz", this.root)
	err = ioutil.WriteFile(srcPath, b, 0644)
	swalllow(err)
	b, _ = Asset("templates/hatop-0.7.7.tar.gz")
	hatop := fmt.Sprintf("%s/src/hatop-0.7.7.tar.gz", this.root)
	err = ioutil.WriteFile(hatop, b, 0644)
	swalllow(err)
	b, _ = Asset("templates/init.ehaproxy")
	initPath := fmt.Sprintf("%s/src/init.ehaproxy", this.root)
	err = ioutil.WriteFile(initPath, b, 0755)
	swalllow(err)

	this.Ui.Info(fmt.Sprintf("compile haproxy to %s/sbin: make TARGET=xxx USE_ZLIB=yes", this.root))
	this.Ui.Info(fmt.Sprintf("cp %s to /etc/init.d/ehaproxy", initPath))
	this.Ui.Info(fmt.Sprintf("chkconfig --add ehaproxy"))

	this.configKernal()

	if this.rsyslog {
		this.configRsyslog()
	}

	return
}

func (this *Deploy) configKernal() {
	this.Ui.Warn("net.core.somaxconn = 16384")
	this.Ui.Warn("net.core.netdev_max_backlog = 2500")
}

func (this *Deploy) configRsyslog() {
	this.Ui.Output("install and setup rsyslog for haproxy")
	this.Ui.Output(fmt.Sprintf(`
vim /etc/rsyslog.conf
$ModLoad imudp
$UDPServerAddress 127.0.0.1
$UDPServerRun 514

vim  /etc/rsyslog.d/haproxy.conf
local3.*     /var/log/haproxy.log

vim /etc/sysconfig/rsyslog
SYSLOGD_OPTIONS=”-c 2 -r -m 0″
#-c 2 使用兼容模式，默认是 -c 5
#-r 开启远程日志
#-m 0 标记时间戳。单位是分钟，为0时，表示禁用该功能
		`))
}

func (this *Deploy) Synopsis() string {
	return fmt.Sprintf("Deploy %s system on localhost", this.Cmd)
}

func (this *Deploy) Help() string {
	help := fmt.Sprintf(`
Usage: %s deploy [options]

    Deploy %s system on localhost

Options:

    -p prefix dir
      Defaults %s

    -rsyslog
      Display rsyslog integration with haproxy

`, this.Cmd, this.Cmd, defaultPrefix)
	return strings.TrimSpace(help)
}
<|endoftext|>


<commit_before>package gateway

import (
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"runtime"
	"strings"
	"sync"
	"syscall"
	"time"

	_ "expvar" // register /debug/vars HTTP handler

	"github.com/funkygao/fae/config"
	"github.com/funkygao/gafka"
	"github.com/funkygao/gafka/cmd/kateway/hh"
	hhdisk "github.com/funkygao/gafka/cmd/kateway/hh/disk"
	"github.com/funkygao/gafka/cmd/kateway/job"
	jobdummy "github.com/funkygao/gafka/cmd/kateway/job/dummy"
	jobmysql "github.com/funkygao/gafka/cmd/kateway/job/mysql"
	"github.com/funkygao/gafka/cmd/kateway/manager"
	mandummy "github.com/funkygao/gafka/cmd/kateway/manager/dummy"
	mandb "github.com/funkygao/gafka/cmd/kateway/manager/mysql"
	manopen "github.com/funkygao/gafka/cmd/kateway/manager/open"
	"github.com/funkygao/gafka/cmd/kateway/meta"
	"github.com/funkygao/gafka/cmd/kateway/meta/zkmeta"
	"github.com/funkygao/gafka/cmd/kateway/store"
	storedummy "github.com/funkygao/gafka/cmd/kateway/store/dummy"
	storekfk "github.com/funkygao/gafka/cmd/kateway/store/kafka"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/registry"
	"github.com/funkygao/gafka/registry/zk"
	"github.com/funkygao/gafka/telemetry"
	"github.com/funkygao/gafka/telemetry/influxdb"
	gzk "github.com/funkygao/gafka/zk"
	"github.com/funkygao/go-metrics"
	"github.com/funkygao/golib/signal"
	"github.com/funkygao/golib/timewheel"
	log "github.com/funkygao/log4go"
	zklib "github.com/samuel/go-zookeeper/zk"
)

// Gateway is a distributed Pub/Sub HTTP endpoint.
//
// Working with ehaproxy, it can form a Pub/Sub cluster system.
type Gateway struct {
	id string // must be unique across the zone

	zkzone       *gzk.ZkZone // load/resume/flush counter metrics to zk
	svrMetrics   *serverMetrics
	accessLogger *AccessLogger
	timer        *timewheel.TimeWheel

	shutdownOnce        sync.Once
	shutdownCh, quiting chan struct{}
	wg                  sync.WaitGroup

	certFile string
	keyFile  string

	pubServer *pubServer
	subServer *subServer
	manServer *manServer
	debugMux  *http.ServeMux
}

func New(id string) *Gateway {
	this := &Gateway{
		id:         id,
		shutdownCh: make(chan struct{}),
		quiting:    make(chan struct{}),
		certFile:   Options.CertFile,
		keyFile:    Options.KeyFile,
	}

	this.zkzone = gzk.NewZkZone(gzk.DefaultConfig(Options.Zone, ctx.ZoneZkAddrs(Options.Zone)))
	if err := this.zkzone.Ping(); err != nil {
		panic(err)
	}

	if Options.EnableRegistry {
		registry.Default = zk.New(this.zkzone, this.id, this.InstanceInfo())
	}
	metaConf := zkmeta.DefaultConfig()
	metaConf.Refresh = Options.MetaRefresh
	meta.Default = zkmeta.New(metaConf, this.zkzone)
	this.timer = timewheel.NewTimeWheel(time.Second, 120)
	this.accessLogger = NewAccessLogger("access_log", 100)
	this.svrMetrics = NewServerMetrics(Options.ReporterInterval, this)
	rc, err := influxdb.NewConfig(Options.InfluxServer, Options.InfluxDbName, "", "", Options.ReporterInterval)
	if err != nil {
		log.Error("telemetry: %v", err)
	} else {
		telemetry.Default = influxdb.New(metrics.DefaultRegistry, rc)
	}
	switch Options.HintedHandoffType {
	case "disk":
		cfg := hhdisk.DefaultConfig()
		cfg.Dir = Options.HintedHandoffDir
		hh.Default = hhdisk.New(cfg)
	}

	// initialize the manager store
	switch Options.ManagerStore {
	case "mysql":
		cf := mandb.DefaultConfig(Options.Zone)
		cf.Refresh = Options.ManagerRefresh
		manager.Default = mandb.New(cf)
		manager.Default.AllowSubWithUnregisteredGroup(Options.PermitUnregisteredGroup)

	case "dummy":
		manager.Default = mandummy.New(Options.DummyCluster)

	case "open":
		cf := manopen.DefaultConfig(Options.Zone)
		cf.Refresh = Options.ManagerRefresh
		manager.Default = manopen.New(cf)
		manager.Default.AllowSubWithUnregisteredGroup(Options.PermitUnregisteredGroup)
		HttpHeaderAppid = "devid"
		HttpHeaderPubkey = "devsecret"
		HttpHeaderSubkey = "devsecret"

	default:
		panic("invalid manager store:" + Options.ManagerStore)
	}

	// initialize the servers on demand
	if Options.DebugHttpAddr != "" {
		this.debugMux = http.NewServeMux()
	}
	if Options.ManHttpAddr != "" || Options.ManHttpsAddr != "" {
		this.manServer = newManServer(Options.ManHttpAddr, Options.ManHttpsAddr,
			Options.MaxClients, this)
	} else {
		panic("manager server must be present")
	}
	if Options.PubHttpAddr != "" || Options.PubHttpsAddr != "" {
		this.pubServer = newPubServer(Options.PubHttpAddr, Options.PubHttpsAddr,
			Options.MaxClients, this)

		switch Options.Store {
		case "kafka":
			store.DefaultPubStore = storekfk.NewPubStore(Options.PubPoolCapcity, Options.PubPoolIdleTimeout,
				Options.UseCompress, &this.wg, Options.Debug, Options.DryRun)

		case "dummy":
			store.DefaultPubStore = storedummy.NewPubStore(&this.wg, Options.Debug)

		default:
			panic("invalid message store")
		}

		switch Options.JobStore {
		case "mysql":
			var mcc = &config.ConfigMysql{}
			b, err := this.zkzone.KatewayJobClusterConfig()
			if err != nil {
				panic(err)
			}
			if err = mcc.From(b); err != nil {
				panic(err)
			}
			jm, err := jobmysql.New(id, mcc)
			if err != nil {
				panic(err)
			}

			job.Default = jm

		case "dummy":
			job.Default = jobdummy.New()

		default:
			panic("invalid job store")
		}
	}
	if Options.SubHttpAddr != "" || Options.SubHttpsAddr != "" {
		this.subServer = newSubServer(Options.SubHttpAddr, Options.SubHttpsAddr,
			Options.MaxClients, this)

		switch Options.Store {
		case "kafka":
			store.DefaultSubStore = storekfk.NewSubStore(&this.wg,
				this.subServer.closedConnCh, Options.Debug)

		case "dummy":
			store.DefaultSubStore = storedummy.NewSubStore(&this.wg,
				this.subServer.closedConnCh, Options.Debug)

		default:
			panic("invalid store")

		}
	}

	return this
}

func (this *Gateway) InstanceInfo() []byte {
	ip, err := ctx.LocalIP()
	if err != nil {
		panic(err)
	}
	info := gzk.KatewayMeta{
		Id:        this.id,
		Zone:      Options.Zone,
		Ver:       gafka.Version,
		Build:     gafka.BuildId,
		BuiltAt:   gafka.BuiltAt,
		Host:      ctx.Hostname(),
		Ip:        ip.String(),
		Cpu:       ctx.NumCPUStr(),
		Arch:      fmt.Sprintf("%s:%s-%s/%s", runtime.Compiler, runtime.Version(), runtime.GOOS, runtime.GOARCH),
		PubAddr:   Options.PubHttpAddr,
		SPubAddr:  Options.PubHttpsAddr,
		SubAddr:   Options.SubHttpAddr,
		SSubAddr:  Options.SubHttpsAddr,
		ManAddr:   Options.ManHttpAddr,
		SManAddr:  Options.ManHttpsAddr,
		DebugAddr: Options.DebugHttpAddr,
	}
	d, _ := json.Marshal(info)
	return d
}

func (this *Gateway) Start() (err error) {
	log.Info("starting gateway[%s@%s]...", gafka.BuildId, gafka.BuiltAt)

	signal.RegisterSignalsHandler(func(sig os.Signal) {
		this.shutdownOnce.Do(func() {
			log.Info("gateway[%s@%s] received signal: %s", gafka.BuildId, gafka.BuiltAt, strings.ToUpper(sig.String()))

			close(this.quiting)
		})
	}, syscall.SIGINT, syscall.SIGTERM) // yes we ignore HUP

	// keep watch on zk connection jitter
	go func() {
		evtCh, ok := this.zkzone.SessionEvents()
		if !ok {
			log.Error("someone else is stealing my zk events?")
			return
		}

		// during connecting phase, the following events are fired:
		// StateConnecting -> StateConnected -> StateHasSession
		firstHandShaked := false
		for {
			select {
			case <-this.shutdownCh:
				return

			case evt, ok := <-evtCh:
				if !ok {
					return
				}

				if !firstHandShaked {
					if evt.State == zklib.StateHasSession {
						firstHandShaked = true
					}

					continue
				}

				log.Warn("zk jitter: %+v", evt)

				if evt.State == zklib.StateHasSession {
					log.Warn("zk reconnected after session lost, watcher/ephemeral lost")

					this.zkzone.CallSOS(fmt.Sprintf("kateway[%s]", this.id), "zk session expired")
				}
			}
		}
	}()

	if err = hh.Default.Start(); err != nil {
		return
	}
	log.Trace("hh[%s] started", hh.Default.Name())

	meta.Default.Start()
	log.Trace("meta store[%s] started", meta.Default.Name())

	if err = manager.Default.Start(); err != nil {
		return
	}
	log.Trace("manager store[%s] started", manager.Default.Name())

	if telemetry.Default != nil {
		go func() {
			log.Trace("telemetry[%s] started", telemetry.Default.Name())

			if err = telemetry.Default.Start(); err != nil {
				log.Error("telemetry[%s]: %v", telemetry.Default.Name(), err)
			}
		}()
	}

	if Options.EnableAccessLog {
		if err = this.accessLogger.Start(); err != nil {
			log.Error("access logger: %s", err)
		}
	}

	this.buildRouting()

	this.svrMetrics.Load()
	go startRuntimeMetrics(Options.ReporterInterval)

	// start up the servers
	this.manServer.Start() // man server is always present
	if this.pubServer != nil {
		if err = store.DefaultPubStore.Start(); err != nil {
			panic(err)
		}
		log.Trace("pub store[%s] started", store.DefaultPubStore.Name())

		if err = job.Default.Start(); err != nil {
			panic(err)
		}
		log.Trace("job store[%s] started", job.Default.Name())

		this.pubServer.Start()
	}
	if this.subServer != nil {
		if err = store.DefaultSubStore.Start(); err != nil {
			panic(err)
		}
		log.Trace("sub store[%s] started", store.DefaultSubStore.Name())

		this.subServer.Start()
	}

	// the last thing is to register: notify others: come on baby!
	if registry.Default != nil {
		if err = registry.Default.Register(); err != nil {
			panic(err)
		}

		log.Info("gateway[%s:%s] ready, registered in %s :-)", ctx.Hostname(), this.id,
			registry.Default.Name())
	} else {
		log.Info("gateway[%s:%s] ready, unregistered", ctx.Hostname(), this.id)
	}

	return nil
}

func (this *Gateway) ServeForever() {
	select {
	case <-this.quiting:
		// the 1st thing is to deregister
		if registry.Default != nil {
			if err := registry.Default.Deregister(); err != nil {
				log.Error("de-register: %v", err)
			} else {
				log.Info("de-registered from %s", registry.Default.Name())
			}
		}

		close(this.shutdownCh)

		// store can only be closed after web server closed
		if this.pubServer != nil {
			log.Trace("awaiting pub server stop...")
			<-this.pubServer.Closed()
		}
		if this.subServer != nil {
			log.Trace("awaiting sub server stop...")
			<-this.subServer.Closed()
		}
		<-this.manServer.Closed()

		if hh.Default != nil {
			log.Trace("hh[%s] stop...", hh.Default.Name())
			hh.Default.Stop()
			log.Trace("hh[%s] flush inflights...", hh.Default.Name())
			hh.Default.FlushInflights()
		}

		//log.Trace("stopping access logger")
		//this.accessLogger.Stop() FIXME it will hang on linux

		if store.DefaultPubStore != nil {
			log.Trace("pub store[%s] stop...", store.DefaultPubStore.Name())
			go store.DefaultPubStore.Stop()
		}
		if store.DefaultSubStore != nil {
			log.Trace("sub store[%s] stop...", store.DefaultSubStore.Name())
			go store.DefaultSubStore.Stop()
		}
		if job.Default != nil {
			job.Default.Stop()
			log.Trace("job store[%s] stopped", job.Default.Name())
		}

		log.Info("...waiting for services shutdown...")
		this.wg.Wait()
		log.Info("<----- all services shutdown ----->")

		this.svrMetrics.Flush()
		log.Trace("svr metrics flushed")

		if telemetry.Default != nil {
			telemetry.Default.Stop()
			log.Trace("telemetry[%s] stopped", telemetry.Default.Name())
		}

		meta.Default.Stop()
		log.Trace("meta store[%s] stopped", meta.Default.Name())

		manager.Default.Stop()
		log.Trace("manager store[%s] stopped", manager.Default.Name())

		if this.zkzone != nil {
			this.zkzone.Close()
			log.Trace("zkzone stopped")
		}

		this.timer.Stop()
	}

}
<commit_msg>hh service must start after pub store<commit_after>package gateway

import (
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"runtime"
	"strings"
	"sync"
	"syscall"
	"time"

	_ "expvar" // register /debug/vars HTTP handler

	"github.com/funkygao/fae/config"
	"github.com/funkygao/gafka"
	"github.com/funkygao/gafka/cmd/kateway/hh"
	hhdisk "github.com/funkygao/gafka/cmd/kateway/hh/disk"
	"github.com/funkygao/gafka/cmd/kateway/job"
	jobdummy "github.com/funkygao/gafka/cmd/kateway/job/dummy"
	jobmysql "github.com/funkygao/gafka/cmd/kateway/job/mysql"
	"github.com/funkygao/gafka/cmd/kateway/manager"
	mandummy "github.com/funkygao/gafka/cmd/kateway/manager/dummy"
	mandb "github.com/funkygao/gafka/cmd/kateway/manager/mysql"
	manopen "github.com/funkygao/gafka/cmd/kateway/manager/open"
	"github.com/funkygao/gafka/cmd/kateway/meta"
	"github.com/funkygao/gafka/cmd/kateway/meta/zkmeta"
	"github.com/funkygao/gafka/cmd/kateway/store"
	storedummy "github.com/funkygao/gafka/cmd/kateway/store/dummy"
	storekfk "github.com/funkygao/gafka/cmd/kateway/store/kafka"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/registry"
	"github.com/funkygao/gafka/registry/zk"
	"github.com/funkygao/gafka/telemetry"
	"github.com/funkygao/gafka/telemetry/influxdb"
	gzk "github.com/funkygao/gafka/zk"
	"github.com/funkygao/go-metrics"
	"github.com/funkygao/golib/signal"
	"github.com/funkygao/golib/timewheel"
	log "github.com/funkygao/log4go"
	zklib "github.com/samuel/go-zookeeper/zk"
)

// Gateway is a distributed Pub/Sub HTTP endpoint.
//
// Working with ehaproxy, it can form a Pub/Sub cluster system.
type Gateway struct {
	id string // must be unique across the zone

	zkzone       *gzk.ZkZone // load/resume/flush counter metrics to zk
	svrMetrics   *serverMetrics
	accessLogger *AccessLogger
	timer        *timewheel.TimeWheel

	shutdownOnce        sync.Once
	shutdownCh, quiting chan struct{}
	wg                  sync.WaitGroup

	certFile string
	keyFile  string

	pubServer *pubServer
	subServer *subServer
	manServer *manServer
	debugMux  *http.ServeMux
}

func New(id string) *Gateway {
	this := &Gateway{
		id:         id,
		shutdownCh: make(chan struct{}),
		quiting:    make(chan struct{}),
		certFile:   Options.CertFile,
		keyFile:    Options.KeyFile,
	}

	this.zkzone = gzk.NewZkZone(gzk.DefaultConfig(Options.Zone, ctx.ZoneZkAddrs(Options.Zone)))
	if err := this.zkzone.Ping(); err != nil {
		panic(err)
	}

	if Options.EnableRegistry {
		registry.Default = zk.New(this.zkzone, this.id, this.InstanceInfo())
	}
	metaConf := zkmeta.DefaultConfig()
	metaConf.Refresh = Options.MetaRefresh
	meta.Default = zkmeta.New(metaConf, this.zkzone)
	this.timer = timewheel.NewTimeWheel(time.Second, 120)
	this.accessLogger = NewAccessLogger("access_log", 100)
	this.svrMetrics = NewServerMetrics(Options.ReporterInterval, this)
	rc, err := influxdb.NewConfig(Options.InfluxServer, Options.InfluxDbName, "", "", Options.ReporterInterval)
	if err != nil {
		log.Error("telemetry: %v", err)
	} else {
		telemetry.Default = influxdb.New(metrics.DefaultRegistry, rc)
	}
	switch Options.HintedHandoffType {
	case "disk":
		cfg := hhdisk.DefaultConfig()
		cfg.Dir = Options.HintedHandoffDir
		hh.Default = hhdisk.New(cfg)
	}

	// initialize the manager store
	switch Options.ManagerStore {
	case "mysql":
		cf := mandb.DefaultConfig(Options.Zone)
		cf.Refresh = Options.ManagerRefresh
		manager.Default = mandb.New(cf)
		manager.Default.AllowSubWithUnregisteredGroup(Options.PermitUnregisteredGroup)

	case "dummy":
		manager.Default = mandummy.New(Options.DummyCluster)

	case "open":
		cf := manopen.DefaultConfig(Options.Zone)
		cf.Refresh = Options.ManagerRefresh
		manager.Default = manopen.New(cf)
		manager.Default.AllowSubWithUnregisteredGroup(Options.PermitUnregisteredGroup)
		HttpHeaderAppid = "devid"
		HttpHeaderPubkey = "devsecret"
		HttpHeaderSubkey = "devsecret"

	default:
		panic("invalid manager store:" + Options.ManagerStore)
	}

	// initialize the servers on demand
	if Options.DebugHttpAddr != "" {
		this.debugMux = http.NewServeMux()
	}
	if Options.ManHttpAddr != "" || Options.ManHttpsAddr != "" {
		this.manServer = newManServer(Options.ManHttpAddr, Options.ManHttpsAddr,
			Options.MaxClients, this)
	} else {
		panic("manager server must be present")
	}
	if Options.PubHttpAddr != "" || Options.PubHttpsAddr != "" {
		this.pubServer = newPubServer(Options.PubHttpAddr, Options.PubHttpsAddr,
			Options.MaxClients, this)

		switch Options.Store {
		case "kafka":
			store.DefaultPubStore = storekfk.NewPubStore(Options.PubPoolCapcity, Options.PubPoolIdleTimeout,
				Options.UseCompress, &this.wg, Options.Debug, Options.DryRun)

		case "dummy":
			store.DefaultPubStore = storedummy.NewPubStore(&this.wg, Options.Debug)

		default:
			panic("invalid message store")
		}

		switch Options.JobStore {
		case "mysql":
			var mcc = &config.ConfigMysql{}
			b, err := this.zkzone.KatewayJobClusterConfig()
			if err != nil {
				panic(err)
			}
			if err = mcc.From(b); err != nil {
				panic(err)
			}
			jm, err := jobmysql.New(id, mcc)
			if err != nil {
				panic(err)
			}

			job.Default = jm

		case "dummy":
			job.Default = jobdummy.New()

		default:
			panic("invalid job store")
		}
	}
	if Options.SubHttpAddr != "" || Options.SubHttpsAddr != "" {
		this.subServer = newSubServer(Options.SubHttpAddr, Options.SubHttpsAddr,
			Options.MaxClients, this)

		switch Options.Store {
		case "kafka":
			store.DefaultSubStore = storekfk.NewSubStore(&this.wg,
				this.subServer.closedConnCh, Options.Debug)

		case "dummy":
			store.DefaultSubStore = storedummy.NewSubStore(&this.wg,
				this.subServer.closedConnCh, Options.Debug)

		default:
			panic("invalid store")

		}
	}

	return this
}

func (this *Gateway) InstanceInfo() []byte {
	ip, err := ctx.LocalIP()
	if err != nil {
		panic(err)
	}
	info := gzk.KatewayMeta{
		Id:        this.id,
		Zone:      Options.Zone,
		Ver:       gafka.Version,
		Build:     gafka.BuildId,
		BuiltAt:   gafka.BuiltAt,
		Host:      ctx.Hostname(),
		Ip:        ip.String(),
		Cpu:       ctx.NumCPUStr(),
		Arch:      fmt.Sprintf("%s:%s-%s/%s", runtime.Compiler, runtime.Version(), runtime.GOOS, runtime.GOARCH),
		PubAddr:   Options.PubHttpAddr,
		SPubAddr:  Options.PubHttpsAddr,
		SubAddr:   Options.SubHttpAddr,
		SSubAddr:  Options.SubHttpsAddr,
		ManAddr:   Options.ManHttpAddr,
		SManAddr:  Options.ManHttpsAddr,
		DebugAddr: Options.DebugHttpAddr,
	}
	d, _ := json.Marshal(info)
	return d
}

func (this *Gateway) Start() (err error) {
	log.Info("starting gateway[%s@%s]...", gafka.BuildId, gafka.BuiltAt)

	signal.RegisterSignalsHandler(func(sig os.Signal) {
		this.shutdownOnce.Do(func() {
			log.Info("gateway[%s@%s] received signal: %s", gafka.BuildId, gafka.BuiltAt, strings.ToUpper(sig.String()))

			close(this.quiting)
		})
	}, syscall.SIGINT, syscall.SIGTERM) // yes we ignore HUP

	// keep watch on zk connection jitter
	go func() {
		evtCh, ok := this.zkzone.SessionEvents()
		if !ok {
			log.Error("someone else is stealing my zk events?")
			return
		}

		// during connecting phase, the following events are fired:
		// StateConnecting -> StateConnected -> StateHasSession
		firstHandShaked := false
		for {
			select {
			case <-this.shutdownCh:
				return

			case evt, ok := <-evtCh:
				if !ok {
					return
				}

				if !firstHandShaked {
					if evt.State == zklib.StateHasSession {
						firstHandShaked = true
					}

					continue
				}

				log.Warn("zk jitter: %+v", evt)

				if evt.State == zklib.StateHasSession {
					log.Warn("zk reconnected after session lost, watcher/ephemeral lost")

					this.zkzone.CallSOS(fmt.Sprintf("kateway[%s]", this.id), "zk session expired")
				}
			}
		}
	}()

	meta.Default.Start()
	log.Trace("meta store[%s] started", meta.Default.Name())

	if err = manager.Default.Start(); err != nil {
		return
	}
	log.Trace("manager store[%s] started", manager.Default.Name())

	if telemetry.Default != nil {
		go func() {
			log.Trace("telemetry[%s] started", telemetry.Default.Name())

			if err = telemetry.Default.Start(); err != nil {
				log.Error("telemetry[%s]: %v", telemetry.Default.Name(), err)
			}
		}()
	}

	if Options.EnableAccessLog {
		if err = this.accessLogger.Start(); err != nil {
			log.Error("access logger: %s", err)
		}
	}

	this.buildRouting()

	this.svrMetrics.Load()
	go startRuntimeMetrics(Options.ReporterInterval)

	// start up the servers
	this.manServer.Start() // man server is always present
	if this.pubServer != nil {
		if err = store.DefaultPubStore.Start(); err != nil {
			panic(err)
		}
		log.Trace("pub store[%s] started", store.DefaultPubStore.Name())

		if err = hh.Default.Start(); err != nil {
			return
		}
		log.Trace("hh[%s] started", hh.Default.Name())

		if err = job.Default.Start(); err != nil {
			panic(err)
		}
		log.Trace("job store[%s] started", job.Default.Name())

		this.pubServer.Start()
	}
	if this.subServer != nil {
		if err = store.DefaultSubStore.Start(); err != nil {
			panic(err)
		}
		log.Trace("sub store[%s] started", store.DefaultSubStore.Name())

		this.subServer.Start()
	}

	// the last thing is to register: notify others: come on baby!
	if registry.Default != nil {
		if err = registry.Default.Register(); err != nil {
			panic(err)
		}

		log.Info("gateway[%s:%s] ready, registered in %s :-)", ctx.Hostname(), this.id,
			registry.Default.Name())
	} else {
		log.Info("gateway[%s:%s] ready, unregistered", ctx.Hostname(), this.id)
	}

	return nil
}

func (this *Gateway) ServeForever() {
	select {
	case <-this.quiting:
		// the 1st thing is to deregister
		if registry.Default != nil {
			if err := registry.Default.Deregister(); err != nil {
				log.Error("de-register: %v", err)
			} else {
				log.Info("de-registered from %s", registry.Default.Name())
			}
		}

		close(this.shutdownCh)

		// store can only be closed after web server closed
		if this.pubServer != nil {
			log.Trace("awaiting pub server stop...")
			<-this.pubServer.Closed()
		}
		if this.subServer != nil {
			log.Trace("awaiting sub server stop...")
			<-this.subServer.Closed()
		}
		<-this.manServer.Closed()

		if hh.Default != nil {
			log.Trace("hh[%s] stop...", hh.Default.Name())
			hh.Default.Stop()
			log.Trace("hh[%s] flush inflights...", hh.Default.Name())
			hh.Default.FlushInflights()
		}

		//log.Trace("stopping access logger")
		//this.accessLogger.Stop() FIXME it will hang on linux

		if store.DefaultPubStore != nil {
			log.Trace("pub store[%s] stop...", store.DefaultPubStore.Name())
			go store.DefaultPubStore.Stop()
		}
		if store.DefaultSubStore != nil {
			log.Trace("sub store[%s] stop...", store.DefaultSubStore.Name())
			go store.DefaultSubStore.Stop()
		}
		if job.Default != nil {
			job.Default.Stop()
			log.Trace("job store[%s] stopped", job.Default.Name())
		}

		log.Info("...waiting for services shutdown...")
		this.wg.Wait()
		log.Info("<----- all services shutdown ----->")

		this.svrMetrics.Flush()
		log.Trace("svr metrics flushed")

		if telemetry.Default != nil {
			telemetry.Default.Stop()
			log.Trace("telemetry[%s] stopped", telemetry.Default.Name())
		}

		meta.Default.Stop()
		log.Trace("meta store[%s] stopped", meta.Default.Name())

		manager.Default.Stop()
		log.Trace("manager store[%s] stopped", manager.Default.Name())

		if this.zkzone != nil {
			this.zkzone.Close()
			log.Trace("zkzone stopped")
		}

		this.timer.Stop()
	}

}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package namer

import (
	"path/filepath"
	"strings"

	"k8s.io/kubernetes/cmd/libs/go2idl/types"
)

// Returns whether a name is a private Go name.
func IsPrivateGoName(name string) bool {
	return len(name) == 0 || strings.ToLower(name[:1]) == name[:1]
}

// NewPublicNamer is a helper function that returns a namer that makes
// CamelCase names. See the NameStrategy struct for an explanation of the
// arguments to this constructor.
func NewPublicNamer(prependPackageNames int, ignoreWords ...string) *NameStrategy {
	n := &NameStrategy{
		Join:                Joiner(IC, IC),
		IgnoreWords:         map[string]bool{},
		PrependPackageNames: prependPackageNames,
	}
	for _, w := range ignoreWords {
		n.IgnoreWords[w] = true
	}
	return n
}

// NewPrivateNamer is a helper function that returns a namer that makes
// camelCase names. See the NameStrategy struct for an explanation of the
// arguments to this constructor.
func NewPrivateNamer(prependPackageNames int, ignoreWords ...string) *NameStrategy {
	n := &NameStrategy{
		Join:                Joiner(IL, IC),
		IgnoreWords:         map[string]bool{},
		PrependPackageNames: prependPackageNames,
	}
	for _, w := range ignoreWords {
		n.IgnoreWords[w] = true
	}
	return n
}

// NewRawNamer will return a Namer that makes a name by which you would
// directly refer to a type, optionally keeping track of the import paths
// necessary to reference the names it provides. Tracker may be nil.
// The 'pkg' is the full package name, in which the Namer is used - all
// types from that package will be referenced by just type name without
// referencing the package.
//
// For example, if the type is map[string]int, a raw namer will literally
// return "map[string]int".
//
// Or if the type, in package foo, is "type Bar struct { ... }", then the raw
// namer will return "foo.Bar" as the name of the type, and if 'tracker' was
// not nil, will record that package foo needs to be imported.
func NewRawNamer(pkg string, tracker ImportTracker) *rawNamer {
	return &rawNamer{pkg: pkg, tracker: tracker}
}

// Names is a map from Type to name, as defined by some Namer.
type Names map[*types.Type]string

// Namer takes a type, and assigns a name.
//
// The purpose of this complexity is so that you can assign coherent
// side-by-side systems of names for the types. For example, you might want a
// public interface, a private implementation struct, and also to reference
// literally the type name.
//
// Note that it is safe to call your own Name() function recursively to find
// the names of keys, elements, etc. This is because anonymous types can't have
// cycles in their names, and named types don't require the sort of recursion
// that would be problematic.
type Namer interface {
	Name(*types.Type) string
}

// NameSystems is a map of a system name to a namer for that system.
type NameSystems map[string]Namer

// NameStrategy is a general Namer. The easiest way to use it is to copy the
// Public/PrivateNamer variables, and modify the members you wish to change.
//
// The Name method produces a name for the given type, of the forms:
// Anonymous types: <Prefix><Type description><Suffix>
// Named types: <Prefix><Optional Prepended Package name(s)><Original name><Suffix>
//
// In all cases, every part of the name is run through the capitalization
// functions.
//
// The IgnoreWords map can be set if you have directory names that are
// semantically meaningless for naming purposes, e.g. "proto".
//
// Prefix and Suffix can be used to disambiguate parallel systems of type
// names. For example, if you want to generate an interface and an
// implementation, you might want to suffix one with "Interface" and the other
// with "Implementation". Another common use-- if you want to generate private
// types, and one of your source types could be "string", you can't use the
// default lowercase private namer. You'll have to add a suffix or prefix.
type NameStrategy struct {
	Prefix, Suffix string
	Join           func(pre string, parts []string, post string) string

	// Add non-meaningful package directory names here (e.g. "proto") and
	// they will be ignored.
	IgnoreWords map[string]bool

	// If > 0, prepend exactly that many package directory names (or as
	// many as there are).  Package names listed in "IgnoreWords" will be
	// ignored.
	//
	// For example, if Ignore words lists "proto" and type Foo is in
	// pkg/server/frobbing/proto, then a value of 1 will give a type name
	// of FrobbingFoo, 2 gives ServerFrobbingFoo, etc.
	PrependPackageNames int

	// A cache of names thus far assigned by this namer.
	Names
}

// IC ensures the first character is uppercase.
func IC(in string) string {
	if in == "" {
		return in
	}
	return strings.ToUpper(in[:1]) + in[1:]
}

// IL ensures the first character is lowercase.
func IL(in string) string {
	if in == "" {
		return in
	}
	return strings.ToLower(in[:1]) + in[1:]
}

// Joiner lets you specify functions that preprocess the various components of
// a name before joining them. You can construct e.g. camelCase or CamelCase or
// any other way of joining words. (See the IC and IL convenience functions.)
func Joiner(first, others func(string) string) func(pre string, in []string, post string) string {
	return func(pre string, in []string, post string) string {
		tmp := []string{others(pre)}
		for i := range in {
			tmp = append(tmp, others(in[i]))
		}
		tmp = append(tmp, others(post))
		return first(strings.Join(tmp, ""))
	}
}

func (ns *NameStrategy) removePrefixAndSuffix(s string) string {
	// The join function may have changed capitalization.
	lowerIn := strings.ToLower(s)
	lowerP := strings.ToLower(ns.Prefix)
	lowerS := strings.ToLower(ns.Suffix)
	b, e := 0, len(s)
	if strings.HasPrefix(lowerIn, lowerP) {
		b = len(ns.Prefix)
	}
	if strings.HasSuffix(lowerIn, lowerS) {
		e -= len(ns.Suffix)
	}
	return s[b:e]
}

var (
	importPathNameSanitizer = strings.NewReplacer("-", "_", ".", "")
)

// filters out unwanted directory names and sanitizes remaining names.
func (ns *NameStrategy) filterDirs(path string) []string {
	allDirs := strings.Split(path, string(filepath.Separator))
	dirs := make([]string, 0, len(allDirs))
	for _, p := range allDirs {
		if ns.IgnoreWords == nil || !ns.IgnoreWords[p] {
			dirs = append(dirs, importPathNameSanitizer.Replace(p))
		}
	}
	return dirs
}

// See the comment on NameStrategy.
func (ns *NameStrategy) Name(t *types.Type) string {
	if ns.Names == nil {
		ns.Names = Names{}
	}
	if s, ok := ns.Names[t]; ok {
		return s
	}

	if t.Name.Package != "" {
		dirs := append(ns.filterDirs(t.Name.Package), t.Name.Name)
		i := ns.PrependPackageNames + 1
		dn := len(dirs)
		if i > dn {
			i = dn
		}
		name := ns.Join(ns.Prefix, dirs[dn-i:], ns.Suffix)
		ns.Names[t] = name
		return name
	}

	// Only anonymous types remain.
	var name string
	switch t.Kind {
	case types.Builtin:
		name = ns.Join(ns.Prefix, []string{t.Name.Name}, ns.Suffix)
	case types.Map:
		name = ns.Join(ns.Prefix, []string{
			"Map",
			ns.removePrefixAndSuffix(ns.Name(t.Key)),
			"To",
			ns.removePrefixAndSuffix(ns.Name(t.Elem)),
		}, ns.Suffix)
	case types.Slice:
		name = ns.Join(ns.Prefix, []string{
			"Slice",
			ns.removePrefixAndSuffix(ns.Name(t.Elem)),
		}, ns.Suffix)
	case types.Pointer:
		name = ns.Join(ns.Prefix, []string{
			"Pointer",
			ns.removePrefixAndSuffix(ns.Name(t.Elem)),
		}, ns.Suffix)
	case types.Struct:
		names := []string{"Struct"}
		for _, m := range t.Members {
			names = append(names, ns.removePrefixAndSuffix(ns.Name(m.Type)))
		}
		name = ns.Join(ns.Prefix, names, ns.Suffix)
	// TODO: add types.Chan
	case types.Interface:
		// TODO: add to name test
		names := []string{"Interface"}
		for _, m := range t.Methods {
			// TODO: include function signature
			names = append(names, m.Name.Name)
		}
		name = ns.Join(ns.Prefix, names, ns.Suffix)
	case types.Func:
		// TODO: add to name test
		parts := []string{"Func"}
		for _, pt := range t.Signature.Parameters {
			parts = append(parts, ns.removePrefixAndSuffix(ns.Name(pt)))
		}
		parts = append(parts, "Returns")
		for _, rt := range t.Signature.Results {
			parts = append(parts, ns.removePrefixAndSuffix(ns.Name(rt)))
		}
		name = ns.Join(ns.Prefix, parts, ns.Suffix)
	default:
		name = "unnameable_" + string(t.Kind)
	}
	ns.Names[t] = name
	return name
}

// ImportTracker allows a raw namer to keep track of the packages needed for
// import. You can implement yourself or use the one in the generation package.
type ImportTracker interface {
	AddType(*types.Type)
	LocalNameOf(packagePath string) string
	PathOf(localName string) (string, bool)
	ImportLines() []string
}

type rawNamer struct {
	pkg     string
	tracker ImportTracker
	Names
}

// Name makes a name the way you'd write it to literally refer to type t,
// making ordinary assumptions about how you've imported t's package (or using
// r.tracker to specifically track the package imports).
func (r *rawNamer) Name(t *types.Type) string {
	if r.Names == nil {
		r.Names = Names{}
	}
	if name, ok := r.Names[t]; ok {
		return name
	}
	if t.Name.Package != "" {
		var name string
		if r.tracker != nil {
			r.tracker.AddType(t)
			if t.Name.Package == r.pkg {
				name = t.Name.Name
			} else {
				name = r.tracker.LocalNameOf(t.Name.Package) + "." + t.Name.Name
			}
		} else {
			if t.Name.Package == r.pkg {
				name = t.Name.Name
			} else {
				name = filepath.Base(t.Name.Package) + "." + t.Name.Name
			}
		}
		r.Names[t] = name
		return name
	}
	var name string
	switch t.Kind {
	case types.Builtin:
		name = t.Name.Name
	case types.Map:
		name = "map[" + r.Name(t.Key) + "]" + r.Name(t.Elem)
	case types.Slice:
		name = "[]" + r.Name(t.Elem)
	case types.Pointer:
		name = "*" + r.Name(t.Elem)
	case types.Struct:
		elems := []string{}
		for _, m := range t.Members {
			elems = append(elems, m.Name+" "+r.Name(m.Type))
		}
		name = "struct{" + strings.Join(elems, "; ") + "}"
	// TODO: add types.Chan
	case types.Interface:
		// TODO: add to name test
		elems := []string{}
		for _, m := range t.Methods {
			// TODO: include function signature
			elems = append(elems, m.Name.Name)
		}
		name = "interface{" + strings.Join(elems, "; ") + "}"
	case types.Func:
		// TODO: add to name test
		params := []string{}
		for _, pt := range t.Signature.Parameters {
			params = append(params, r.Name(pt))
		}
		results := []string{}
		for _, rt := range t.Signature.Results {
			results = append(results, r.Name(rt))
		}
		name = "func(" + strings.Join(params, ",") + ")"
		if len(results) == 1 {
			name += " " + results[0]
		} else if len(results) > 1 {
			name += " (" + strings.Join(results, ",") + ")"
		}
	default:
		name = "unnameable_" + string(t.Kind)
	}
	r.Names[t] = name
	return name
}
<commit_msg>Only add to tracker if it's non-local<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package namer

import (
	"path/filepath"
	"strings"

	"k8s.io/kubernetes/cmd/libs/go2idl/types"
)

// Returns whether a name is a private Go name.
func IsPrivateGoName(name string) bool {
	return len(name) == 0 || strings.ToLower(name[:1]) == name[:1]
}

// NewPublicNamer is a helper function that returns a namer that makes
// CamelCase names. See the NameStrategy struct for an explanation of the
// arguments to this constructor.
func NewPublicNamer(prependPackageNames int, ignoreWords ...string) *NameStrategy {
	n := &NameStrategy{
		Join:                Joiner(IC, IC),
		IgnoreWords:         map[string]bool{},
		PrependPackageNames: prependPackageNames,
	}
	for _, w := range ignoreWords {
		n.IgnoreWords[w] = true
	}
	return n
}

// NewPrivateNamer is a helper function that returns a namer that makes
// camelCase names. See the NameStrategy struct for an explanation of the
// arguments to this constructor.
func NewPrivateNamer(prependPackageNames int, ignoreWords ...string) *NameStrategy {
	n := &NameStrategy{
		Join:                Joiner(IL, IC),
		IgnoreWords:         map[string]bool{},
		PrependPackageNames: prependPackageNames,
	}
	for _, w := range ignoreWords {
		n.IgnoreWords[w] = true
	}
	return n
}

// NewRawNamer will return a Namer that makes a name by which you would
// directly refer to a type, optionally keeping track of the import paths
// necessary to reference the names it provides. Tracker may be nil.
// The 'pkg' is the full package name, in which the Namer is used - all
// types from that package will be referenced by just type name without
// referencing the package.
//
// For example, if the type is map[string]int, a raw namer will literally
// return "map[string]int".
//
// Or if the type, in package foo, is "type Bar struct { ... }", then the raw
// namer will return "foo.Bar" as the name of the type, and if 'tracker' was
// not nil, will record that package foo needs to be imported.
func NewRawNamer(pkg string, tracker ImportTracker) *rawNamer {
	return &rawNamer{pkg: pkg, tracker: tracker}
}

// Names is a map from Type to name, as defined by some Namer.
type Names map[*types.Type]string

// Namer takes a type, and assigns a name.
//
// The purpose of this complexity is so that you can assign coherent
// side-by-side systems of names for the types. For example, you might want a
// public interface, a private implementation struct, and also to reference
// literally the type name.
//
// Note that it is safe to call your own Name() function recursively to find
// the names of keys, elements, etc. This is because anonymous types can't have
// cycles in their names, and named types don't require the sort of recursion
// that would be problematic.
type Namer interface {
	Name(*types.Type) string
}

// NameSystems is a map of a system name to a namer for that system.
type NameSystems map[string]Namer

// NameStrategy is a general Namer. The easiest way to use it is to copy the
// Public/PrivateNamer variables, and modify the members you wish to change.
//
// The Name method produces a name for the given type, of the forms:
// Anonymous types: <Prefix><Type description><Suffix>
// Named types: <Prefix><Optional Prepended Package name(s)><Original name><Suffix>
//
// In all cases, every part of the name is run through the capitalization
// functions.
//
// The IgnoreWords map can be set if you have directory names that are
// semantically meaningless for naming purposes, e.g. "proto".
//
// Prefix and Suffix can be used to disambiguate parallel systems of type
// names. For example, if you want to generate an interface and an
// implementation, you might want to suffix one with "Interface" and the other
// with "Implementation". Another common use-- if you want to generate private
// types, and one of your source types could be "string", you can't use the
// default lowercase private namer. You'll have to add a suffix or prefix.
type NameStrategy struct {
	Prefix, Suffix string
	Join           func(pre string, parts []string, post string) string

	// Add non-meaningful package directory names here (e.g. "proto") and
	// they will be ignored.
	IgnoreWords map[string]bool

	// If > 0, prepend exactly that many package directory names (or as
	// many as there are).  Package names listed in "IgnoreWords" will be
	// ignored.
	//
	// For example, if Ignore words lists "proto" and type Foo is in
	// pkg/server/frobbing/proto, then a value of 1 will give a type name
	// of FrobbingFoo, 2 gives ServerFrobbingFoo, etc.
	PrependPackageNames int

	// A cache of names thus far assigned by this namer.
	Names
}

// IC ensures the first character is uppercase.
func IC(in string) string {
	if in == "" {
		return in
	}
	return strings.ToUpper(in[:1]) + in[1:]
}

// IL ensures the first character is lowercase.
func IL(in string) string {
	if in == "" {
		return in
	}
	return strings.ToLower(in[:1]) + in[1:]
}

// Joiner lets you specify functions that preprocess the various components of
// a name before joining them. You can construct e.g. camelCase or CamelCase or
// any other way of joining words. (See the IC and IL convenience functions.)
func Joiner(first, others func(string) string) func(pre string, in []string, post string) string {
	return func(pre string, in []string, post string) string {
		tmp := []string{others(pre)}
		for i := range in {
			tmp = append(tmp, others(in[i]))
		}
		tmp = append(tmp, others(post))
		return first(strings.Join(tmp, ""))
	}
}

func (ns *NameStrategy) removePrefixAndSuffix(s string) string {
	// The join function may have changed capitalization.
	lowerIn := strings.ToLower(s)
	lowerP := strings.ToLower(ns.Prefix)
	lowerS := strings.ToLower(ns.Suffix)
	b, e := 0, len(s)
	if strings.HasPrefix(lowerIn, lowerP) {
		b = len(ns.Prefix)
	}
	if strings.HasSuffix(lowerIn, lowerS) {
		e -= len(ns.Suffix)
	}
	return s[b:e]
}

var (
	importPathNameSanitizer = strings.NewReplacer("-", "_", ".", "")
)

// filters out unwanted directory names and sanitizes remaining names.
func (ns *NameStrategy) filterDirs(path string) []string {
	allDirs := strings.Split(path, string(filepath.Separator))
	dirs := make([]string, 0, len(allDirs))
	for _, p := range allDirs {
		if ns.IgnoreWords == nil || !ns.IgnoreWords[p] {
			dirs = append(dirs, importPathNameSanitizer.Replace(p))
		}
	}
	return dirs
}

// See the comment on NameStrategy.
func (ns *NameStrategy) Name(t *types.Type) string {
	if ns.Names == nil {
		ns.Names = Names{}
	}
	if s, ok := ns.Names[t]; ok {
		return s
	}

	if t.Name.Package != "" {
		dirs := append(ns.filterDirs(t.Name.Package), t.Name.Name)
		i := ns.PrependPackageNames + 1
		dn := len(dirs)
		if i > dn {
			i = dn
		}
		name := ns.Join(ns.Prefix, dirs[dn-i:], ns.Suffix)
		ns.Names[t] = name
		return name
	}

	// Only anonymous types remain.
	var name string
	switch t.Kind {
	case types.Builtin:
		name = ns.Join(ns.Prefix, []string{t.Name.Name}, ns.Suffix)
	case types.Map:
		name = ns.Join(ns.Prefix, []string{
			"Map",
			ns.removePrefixAndSuffix(ns.Name(t.Key)),
			"To",
			ns.removePrefixAndSuffix(ns.Name(t.Elem)),
		}, ns.Suffix)
	case types.Slice:
		name = ns.Join(ns.Prefix, []string{
			"Slice",
			ns.removePrefixAndSuffix(ns.Name(t.Elem)),
		}, ns.Suffix)
	case types.Pointer:
		name = ns.Join(ns.Prefix, []string{
			"Pointer",
			ns.removePrefixAndSuffix(ns.Name(t.Elem)),
		}, ns.Suffix)
	case types.Struct:
		names := []string{"Struct"}
		for _, m := range t.Members {
			names = append(names, ns.removePrefixAndSuffix(ns.Name(m.Type)))
		}
		name = ns.Join(ns.Prefix, names, ns.Suffix)
	// TODO: add types.Chan
	case types.Interface:
		// TODO: add to name test
		names := []string{"Interface"}
		for _, m := range t.Methods {
			// TODO: include function signature
			names = append(names, m.Name.Name)
		}
		name = ns.Join(ns.Prefix, names, ns.Suffix)
	case types.Func:
		// TODO: add to name test
		parts := []string{"Func"}
		for _, pt := range t.Signature.Parameters {
			parts = append(parts, ns.removePrefixAndSuffix(ns.Name(pt)))
		}
		parts = append(parts, "Returns")
		for _, rt := range t.Signature.Results {
			parts = append(parts, ns.removePrefixAndSuffix(ns.Name(rt)))
		}
		name = ns.Join(ns.Prefix, parts, ns.Suffix)
	default:
		name = "unnameable_" + string(t.Kind)
	}
	ns.Names[t] = name
	return name
}

// ImportTracker allows a raw namer to keep track of the packages needed for
// import. You can implement yourself or use the one in the generation package.
type ImportTracker interface {
	AddType(*types.Type)
	LocalNameOf(packagePath string) string
	PathOf(localName string) (string, bool)
	ImportLines() []string
}

type rawNamer struct {
	pkg     string
	tracker ImportTracker
	Names
}

// Name makes a name the way you'd write it to literally refer to type t,
// making ordinary assumptions about how you've imported t's package (or using
// r.tracker to specifically track the package imports).
func (r *rawNamer) Name(t *types.Type) string {
	if r.Names == nil {
		r.Names = Names{}
	}
	if name, ok := r.Names[t]; ok {
		return name
	}
	if t.Name.Package != "" {
		var name string
		if r.tracker != nil {
			if t.Name.Package == r.pkg {
				name = t.Name.Name
			} else {
				r.tracker.AddType(t)
				name = r.tracker.LocalNameOf(t.Name.Package) + "." + t.Name.Name
			}
		} else {
			if t.Name.Package == r.pkg {
				name = t.Name.Name
			} else {
				name = filepath.Base(t.Name.Package) + "." + t.Name.Name
			}
		}
		r.Names[t] = name
		return name
	}
	var name string
	switch t.Kind {
	case types.Builtin:
		name = t.Name.Name
	case types.Map:
		name = "map[" + r.Name(t.Key) + "]" + r.Name(t.Elem)
	case types.Slice:
		name = "[]" + r.Name(t.Elem)
	case types.Pointer:
		name = "*" + r.Name(t.Elem)
	case types.Struct:
		elems := []string{}
		for _, m := range t.Members {
			elems = append(elems, m.Name+" "+r.Name(m.Type))
		}
		name = "struct{" + strings.Join(elems, "; ") + "}"
	// TODO: add types.Chan
	case types.Interface:
		// TODO: add to name test
		elems := []string{}
		for _, m := range t.Methods {
			// TODO: include function signature
			elems = append(elems, m.Name.Name)
		}
		name = "interface{" + strings.Join(elems, "; ") + "}"
	case types.Func:
		// TODO: add to name test
		params := []string{}
		for _, pt := range t.Signature.Parameters {
			params = append(params, r.Name(pt))
		}
		results := []string{}
		for _, rt := range t.Signature.Results {
			results = append(results, r.Name(rt))
		}
		name = "func(" + strings.Join(params, ",") + ")"
		if len(results) == 1 {
			name += " " + results[0]
		} else if len(results) > 1 {
			name += " (" + strings.Join(results, ",") + ")"
		}
	default:
		name = "unnameable_" + string(t.Kind)
	}
	r.Names[t] = name
	return name
}
<|endoftext|>


<commit_before>package handlers

import (
	"encoding/json"
	"net/http"
	"strings"

	"github.com/cloudfoundry-incubator/routing-api/authentication"
	"github.com/cloudfoundry-incubator/routing-api/db"
	"github.com/pivotal-golang/lager"
)

const (
	AdminRouteScope     = "route.admin"
	AdvertiseRouteScope = "route.advertise"
)

type RoutesHandler struct {
	token     authentication.Token
	maxTTL    int
	validator RouteValidator
	db        db.DB
	logger    lager.Logger
}

func NewRoutesHandler(token authentication.Token, maxTTL int, validator RouteValidator, database db.DB, logger lager.Logger) *RoutesHandler {
	return &RoutesHandler{
		token:     token,
		maxTTL:    maxTTL,
		validator: validator,
		db:        database,
		logger:    logger,
	}
}

func (h *RoutesHandler) List(w http.ResponseWriter, req *http.Request) {
	log := h.logger.Session("list-routes")

	err := h.token.DecodeToken(req.Header.Get("Authorization"), AdminRouteScope)
	if err != nil {
		handleUnauthorizedError(w, err, log)
		return
	}
	routes, err := h.db.ReadRoutes()
	if err != nil {
		handleDBError(w, err, log)
		return
	}
	encoder := json.NewEncoder(w)
	encoder.Encode(routes)
}

func (h *RoutesHandler) Upsert(w http.ResponseWriter, req *http.Request) {
	log := h.logger.Session("create-route")
	decoder := json.NewDecoder(req.Body)

	var routes []db.Route
	err := decoder.Decode(&routes)
	if err != nil {
		handleProcessRequestError(w, err, log)
		return
	}

	log.Info("request", lager.Data{"route_creation": routes})

	err = h.token.DecodeToken(req.Header.Get("Authorization"), AdvertiseRouteScope, AdminRouteScope)
	if err != nil {
		handleUnauthorizedError(w, err, log)
		return
	}

	apiErr := h.validator.ValidateCreate(routes, h.maxTTL)
	if apiErr != nil {
		handleApiError(w, apiErr, log)
		return
	}

	for _, route := range routes {
		err = h.db.SaveRoute(route)
		if err != nil {
			handleDBError(w, err, log)
			return
		}
	}

	w.WriteHeader(http.StatusCreated)
}

func (h *RoutesHandler) Delete(w http.ResponseWriter, req *http.Request) {
	log := h.logger.Session("delete-route")
	decoder := json.NewDecoder(req.Body)

	var routes []db.Route
	err := decoder.Decode(&routes)
	if err != nil {
		handleProcessRequestError(w, err, log)
		return
	}

	log.Info("request", lager.Data{"route_deletion": routes})

	err = h.token.DecodeToken(req.Header.Get("Authorization"), AdvertiseRouteScope, AdminRouteScope)
	if err != nil {
		handleUnauthorizedError(w, err, log)
		return
	}

	apiErr := h.validator.ValidateDelete(routes)
	if apiErr != nil {
		handleApiError(w, apiErr, log)
		return
	}

	for _, route := range routes {
		err = h.db.DeleteRoute(route)
		if err != nil && !strings.Contains(err.Error(), "Key not found") {
			handleDBError(w, err, log)
			return
		}
	}

	w.WriteHeader(http.StatusNoContent)
}
<commit_msg>remove unused dependency<commit_after>package handlers

import (
	"encoding/json"
	"net/http"

	"github.com/cloudfoundry-incubator/routing-api/authentication"
	"github.com/cloudfoundry-incubator/routing-api/db"
	"github.com/pivotal-golang/lager"
)

const (
	AdminRouteScope     = "route.admin"
	AdvertiseRouteScope = "route.advertise"
)

type RoutesHandler struct {
	token     authentication.Token
	maxTTL    int
	validator RouteValidator
	db        db.DB
	logger    lager.Logger
}

func NewRoutesHandler(token authentication.Token, maxTTL int, validator RouteValidator, database db.DB, logger lager.Logger) *RoutesHandler {
	return &RoutesHandler{
		token:     token,
		maxTTL:    maxTTL,
		validator: validator,
		db:        database,
		logger:    logger,
	}
}

func (h *RoutesHandler) List(w http.ResponseWriter, req *http.Request) {
	log := h.logger.Session("list-routes")

	err := h.token.DecodeToken(req.Header.Get("Authorization"), AdminRouteScope)
	if err != nil {
		handleUnauthorizedError(w, err, log)
		return
	}
	routes, err := h.db.ReadRoutes()
	if err != nil {
		handleDBError(w, err, log)
		return
	}
	encoder := json.NewEncoder(w)
	encoder.Encode(routes)
}

func (h *RoutesHandler) Upsert(w http.ResponseWriter, req *http.Request) {
	log := h.logger.Session("create-route")
	decoder := json.NewDecoder(req.Body)

	var routes []db.Route
	err := decoder.Decode(&routes)
	if err != nil {
		handleProcessRequestError(w, err, log)
		return
	}

	log.Info("request", lager.Data{"route_creation": routes})

	err = h.token.DecodeToken(req.Header.Get("Authorization"), AdvertiseRouteScope, AdminRouteScope)
	if err != nil {
		handleUnauthorizedError(w, err, log)
		return
	}

	apiErr := h.validator.ValidateCreate(routes, h.maxTTL)
	if apiErr != nil {
		handleApiError(w, apiErr, log)
		return
	}

	for _, route := range routes {
		err = h.db.SaveRoute(route)
		if err != nil {
			handleDBError(w, err, log)
			return
		}
	}

	w.WriteHeader(http.StatusCreated)
}

func (h *RoutesHandler) Delete(w http.ResponseWriter, req *http.Request) {
	log := h.logger.Session("delete-route")
	decoder := json.NewDecoder(req.Body)

	var routes []db.Route
	err := decoder.Decode(&routes)
	if err != nil {
		handleProcessRequestError(w, err, log)
		return
	}

	log.Info("request", lager.Data{"route_deletion": routes})

	err = h.token.DecodeToken(req.Header.Get("Authorization"), AdvertiseRouteScope, AdminRouteScope)
	if err != nil {
		handleUnauthorizedError(w, err, log)
		return
	}

	apiErr := h.validator.ValidateDelete(routes)
	if apiErr != nil {
		handleApiError(w, apiErr, log)
		return
	}

	for _, route := range routes {
		err = h.db.DeleteRoute(route)
		if err != nil && !strings.Contains(err.Error(), "Key not found") {
			handleDBError(w, err, log)
			return
		}
	}

	w.WriteHeader(http.StatusNoContent)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"runtime/pprof"
	"strings"

	"github.com/nelhage/taktician/ai"
	"github.com/nelhage/taktician/ptn"
	"github.com/nelhage/taktician/tak"
)

var (
	size    = flag.Int("size", 5, "board size")
	zero    = flag.Bool("zero", false, "start with zero weights, not defaults")
	w1      = flag.String("w1", "", "first set of weights")
	w2      = flag.String("w2", "", "second set of weights")
	c1      = flag.String("c1", "", "custom config 1")
	c2      = flag.String("c2", "", "custom config 2")
	perturb = flag.Float64("perturb", 0.0, "perturb weights")
	seed    = flag.Int64("seed", 1, "starting random seed")
	games   = flag.Int("games", 10, "number of games to play")
	cutoff  = flag.Int("cutoff", 80, "cut games off after how many plies")
	swap    = flag.Bool("swap", true, "swap colors each game")

	prefix = flag.String("prefix", "", "ptn file to start games at the end of")
	seeds  = flag.String("seeds", "", "directory of seed positions")

	depth = flag.Int("depth", 3, "depth to search each move")
	limit = flag.Duration("limit", 0, "amount of time to search each move")

	threads = flag.Int("threads", 4, "number of parallel threads")

	out = flag.String("out", "", "directory to write ptns to")

	search = flag.Bool("search", false, "search for a good set of weights")

	memProfile = flag.String("mem-profile", "", "write memory profile")
)

func addSeeds(g *ptn.PTN, ps []*tak.Position) ([]*tak.Position, error) {
	const (
		minPly = 5
		maxPly = 10
	)
	ply := 0
	p, e := g.InitialPosition()
	if e != nil {
		return nil, e
	}
	for _, op := range g.Ops {
		mo, ok := op.(*ptn.Move)
		if !ok {
			continue
		}
		ply++
		next, e := p.Move(&mo.Move)
		if e != nil {
			return nil, fmt.Errorf("ply %d: %v", ply, e)
		}
		if ok, _ := next.GameOver(); ok {
			break
		}
		if ply >= minPly {
			ps = append(ps, next)
		}
		p = next
		if ply >= maxPly {
			break
		}
	}
	return ps, nil
}

func readSeeds(d string) ([]*tak.Position, error) {
	ents, e := ioutil.ReadDir(d)
	if e != nil {
		return nil, e
	}
	var ps []*tak.Position
	for _, de := range ents {
		if !strings.HasSuffix(de.Name(), ".ptn") {
			continue
		}
		f, e := os.Open(path.Join(d, de.Name()))
		if e != nil {
			return nil, fmt.Errorf("%s/%s: %v", d, de.Name(), e)
		}
		g, e := ptn.ParsePTN(f)
		if e != nil {
			return nil, fmt.Errorf("%s/%s: %v", d, de.Name(), e)
		}
		f.Close()
		ps, e = addSeeds(g, ps)
		if e != nil {
			return nil, fmt.Errorf("%s/%s: %v", d, de.Name(), e)
		}
	}
	return ps, nil
}

func main() {
	flag.Parse()
	if *memProfile != "" {
		defer func() {
			f, e := os.OpenFile(*memProfile,
				os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0644)
			if e != nil {
				log.Printf("open memory profile: %v", e)
				return
			}
			pprof.Lookup("heap").WriteTo(f, 0)
		}()
	}

	var starts []*tak.Position
	if *prefix != "" {
		bs, e := ioutil.ReadFile(*prefix)
		if e != nil {
			log.Fatalf("Read %s: %v", *prefix, e)
		}
		pt, e := ptn.ParsePTN(bytes.NewBuffer(bs))
		if e != nil {
			log.Fatalf("Parse PTN: %v", e)
		}
		p, e := pt.PositionAtMove(0, tak.NoColor)
		if e != nil {
			log.Fatalf("PTN: %v", e)
		}
		starts = []*tak.Position{p}
	}
	if *seeds != "" {
		var e error
		starts, e = readSeeds(*seeds)
		if e != nil {
			log.Fatalf("-seeds: %v", e)
		}
	}

	weights1 := ai.DefaultWeights[*size]
	weights2 := ai.DefaultWeights[*size]
	if *zero {
		weights1 = ai.Weights{}
		weights2 = ai.Weights{}
	}
	if *w1 != "" {
		if err := json.Unmarshal([]byte(*w1), &weights1); err != nil {
			log.Fatal("w1:", err)
		}
	}
	if *w2 != "" {
		if err := json.Unmarshal([]byte(*w2), &weights2); err != nil {
			log.Fatal("w2:", err)
		}
	}

	cfg1 := ai.MinimaxConfig{
		Depth: *depth,
		Size:  *size,
	}
	cfg2 := ai.MinimaxConfig{
		Depth: *depth,
		Size:  *size,
	}
	if *c1 != "" {
		if err := json.Unmarshal([]byte(*c1), &cfg1); err != nil {
			log.Fatal("c1:", err)
		}
	}
	if *c2 != "" {
		if err := json.Unmarshal([]byte(*c2), &cfg2); err != nil {
			log.Fatal("c2:", err)
		}
	}

	if *search {
		doSearch(cfg1, weights1)
		return
	}

	st := Simulate(&Config{
		Cfg1:    cfg1,
		Cfg2:    cfg2,
		W1:      weights1,
		W2:      weights2,
		Swap:    *swap,
		Games:   *games,
		Threads: *threads,
		Seed:    *seed,
		Cutoff:  *cutoff,
		Limit:   *limit,
		Perturb: *perturb,
		Initial: starts,
	})

	if *out != "" {
		for _, r := range st.Games {
			writeGame(*out, &r)
		}
	}

	var j []byte
	j, _ = json.Marshal(&weights1)
	log.Printf("p1w=%s", j)
	if *c1 != "" {
		log.Printf("p1c=%s", *c1)
	}
	j, _ = json.Marshal(&weights2)
	log.Printf("p2w=%s", j)
	if *c2 != "" {
		log.Printf("p2c=%s", *c2)
	}
	log.Printf("done games=%d seed=%d ties=%d cutoff=%d white=%d black=%d",
		*games, *seed, st.Ties, st.Cutoff, st.White, st.Black)
	log.Printf("p1.wins=%d (%d road/%d flat) p2.wins=%d (%d road/%d flat)",
		st.Players[0].Wins, st.Players[0].RoadWins, st.Players[0].FlatWins,
		st.Players[1].Wins, st.Players[1].RoadWins, st.Players[1].FlatWins)
	a, b := int64(st.Players[0].Wins), int64(st.Players[1].Wins)
	if a < b {
		a, b = b, a
	}
	log.Printf("p[one-sided]=%f", binomTest(a, b, 0.5))
}

func writeGame(d string, r *Result) {
	os.MkdirAll(d, 0755)
	p := &ptn.PTN{}
	p.Tags = []ptn.Tag{
		{"Size", fmt.Sprintf("%d", r.Position.Size())},
		{"Player1", r.spec.p1color.String()},
	}
	if r.Initial != nil {
		p.Tags = append(p.Tags, ptn.Tag{
			Name: "TPS", Value: ptn.FormatTPS(r.Initial)})
	}
	for i, m := range r.Moves {
		if i%2 == 0 {
			p.Ops = append(p.Ops, &ptn.MoveNumber{Number: i/2 + 1})
		}
		p.Ops = append(p.Ops, &ptn.Move{Move: m})
	}
	ptnPath := path.Join(d, fmt.Sprintf("%d.ptn", r.spec.i))
	ioutil.WriteFile(ptnPath, []byte(p.Render()), 0644)
}
<commit_msg>default to time-based seeding<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"runtime/pprof"
	"strings"
	"time"

	"github.com/nelhage/taktician/ai"
	"github.com/nelhage/taktician/ptn"
	"github.com/nelhage/taktician/tak"
)

var (
	size    = flag.Int("size", 5, "board size")
	zero    = flag.Bool("zero", false, "start with zero weights, not defaults")
	w1      = flag.String("w1", "", "first set of weights")
	w2      = flag.String("w2", "", "second set of weights")
	c1      = flag.String("c1", "", "custom config 1")
	c2      = flag.String("c2", "", "custom config 2")
	perturb = flag.Float64("perturb", 0.0, "perturb weights")
	seed    = flag.Int64("seed", 0, "starting random seed")
	games   = flag.Int("games", 10, "number of games to play")
	cutoff  = flag.Int("cutoff", 80, "cut games off after how many plies")
	swap    = flag.Bool("swap", true, "swap colors each game")

	prefix = flag.String("prefix", "", "ptn file to start games at the end of")
	seeds  = flag.String("seeds", "", "directory of seed positions")

	depth = flag.Int("depth", 3, "depth to search each move")
	limit = flag.Duration("limit", 0, "amount of time to search each move")

	threads = flag.Int("threads", 4, "number of parallel threads")

	out = flag.String("out", "", "directory to write ptns to")

	search = flag.Bool("search", false, "search for a good set of weights")

	memProfile = flag.String("mem-profile", "", "write memory profile")
)

func addSeeds(g *ptn.PTN, ps []*tak.Position) ([]*tak.Position, error) {
	const (
		minPly = 5
		maxPly = 10
	)
	ply := 0
	p, e := g.InitialPosition()
	if e != nil {
		return nil, e
	}
	for _, op := range g.Ops {
		mo, ok := op.(*ptn.Move)
		if !ok {
			continue
		}
		ply++
		next, e := p.Move(&mo.Move)
		if e != nil {
			return nil, fmt.Errorf("ply %d: %v", ply, e)
		}
		if ok, _ := next.GameOver(); ok {
			break
		}
		if ply >= minPly {
			ps = append(ps, next)
		}
		p = next
		if ply >= maxPly {
			break
		}
	}
	return ps, nil
}

func readSeeds(d string) ([]*tak.Position, error) {
	ents, e := ioutil.ReadDir(d)
	if e != nil {
		return nil, e
	}
	var ps []*tak.Position
	for _, de := range ents {
		if !strings.HasSuffix(de.Name(), ".ptn") {
			continue
		}
		f, e := os.Open(path.Join(d, de.Name()))
		if e != nil {
			return nil, fmt.Errorf("%s/%s: %v", d, de.Name(), e)
		}
		g, e := ptn.ParsePTN(f)
		if e != nil {
			return nil, fmt.Errorf("%s/%s: %v", d, de.Name(), e)
		}
		f.Close()
		ps, e = addSeeds(g, ps)
		if e != nil {
			return nil, fmt.Errorf("%s/%s: %v", d, de.Name(), e)
		}
	}
	return ps, nil
}

func main() {
	flag.Parse()
	if *memProfile != "" {
		defer func() {
			f, e := os.OpenFile(*memProfile,
				os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0644)
			if e != nil {
				log.Printf("open memory profile: %v", e)
				return
			}
			pprof.Lookup("heap").WriteTo(f, 0)
		}()
	}

	if *seed == 0 {
		*seed = time.Now().Unix()
	}

	var starts []*tak.Position
	if *prefix != "" {
		bs, e := ioutil.ReadFile(*prefix)
		if e != nil {
			log.Fatalf("Read %s: %v", *prefix, e)
		}
		pt, e := ptn.ParsePTN(bytes.NewBuffer(bs))
		if e != nil {
			log.Fatalf("Parse PTN: %v", e)
		}
		p, e := pt.PositionAtMove(0, tak.NoColor)
		if e != nil {
			log.Fatalf("PTN: %v", e)
		}
		starts = []*tak.Position{p}
	}
	if *seeds != "" {
		var e error
		starts, e = readSeeds(*seeds)
		if e != nil {
			log.Fatalf("-seeds: %v", e)
		}
	}

	weights1 := ai.DefaultWeights[*size]
	weights2 := ai.DefaultWeights[*size]
	if *zero {
		weights1 = ai.Weights{}
		weights2 = ai.Weights{}
	}
	if *w1 != "" {
		if err := json.Unmarshal([]byte(*w1), &weights1); err != nil {
			log.Fatal("w1:", err)
		}
	}
	if *w2 != "" {
		if err := json.Unmarshal([]byte(*w2), &weights2); err != nil {
			log.Fatal("w2:", err)
		}
	}

	cfg1 := ai.MinimaxConfig{
		Depth: *depth,
		Size:  *size,
	}
	cfg2 := ai.MinimaxConfig{
		Depth: *depth,
		Size:  *size,
	}
	if *c1 != "" {
		if err := json.Unmarshal([]byte(*c1), &cfg1); err != nil {
			log.Fatal("c1:", err)
		}
	}
	if *c2 != "" {
		if err := json.Unmarshal([]byte(*c2), &cfg2); err != nil {
			log.Fatal("c2:", err)
		}
	}

	if *search {
		doSearch(cfg1, weights1)
		return
	}

	st := Simulate(&Config{
		Cfg1:    cfg1,
		Cfg2:    cfg2,
		W1:      weights1,
		W2:      weights2,
		Swap:    *swap,
		Games:   *games,
		Threads: *threads,
		Seed:    *seed,
		Cutoff:  *cutoff,
		Limit:   *limit,
		Perturb: *perturb,
		Initial: starts,
	})

	if *out != "" {
		for _, r := range st.Games {
			writeGame(*out, &r)
		}
	}

	var j []byte
	j, _ = json.Marshal(&weights1)
	log.Printf("p1w=%s", j)
	if *c1 != "" {
		log.Printf("p1c=%s", *c1)
	}
	j, _ = json.Marshal(&weights2)
	log.Printf("p2w=%s", j)
	if *c2 != "" {
		log.Printf("p2c=%s", *c2)
	}
	log.Printf("done games=%d seed=%d ties=%d cutoff=%d white=%d black=%d",
		*games, *seed, st.Ties, st.Cutoff, st.White, st.Black)
	log.Printf("p1.wins=%d (%d road/%d flat) p2.wins=%d (%d road/%d flat)",
		st.Players[0].Wins, st.Players[0].RoadWins, st.Players[0].FlatWins,
		st.Players[1].Wins, st.Players[1].RoadWins, st.Players[1].FlatWins)
	a, b := int64(st.Players[0].Wins), int64(st.Players[1].Wins)
	if a < b {
		a, b = b, a
	}
	log.Printf("p[one-sided]=%f", binomTest(a, b, 0.5))
}

func writeGame(d string, r *Result) {
	os.MkdirAll(d, 0755)
	p := &ptn.PTN{}
	p.Tags = []ptn.Tag{
		{"Size", fmt.Sprintf("%d", r.Position.Size())},
		{"Player1", r.spec.p1color.String()},
	}
	if r.Initial != nil {
		p.Tags = append(p.Tags, ptn.Tag{
			Name: "TPS", Value: ptn.FormatTPS(r.Initial)})
	}
	for i, m := range r.Moves {
		if i%2 == 0 {
			p.Ops = append(p.Ops, &ptn.MoveNumber{Number: i/2 + 1})
		}
		p.Ops = append(p.Ops, &ptn.Move{Move: m})
	}
	ptnPath := path.Join(d, fmt.Sprintf("%d.ptn", r.spec.i))
	ioutil.WriteFile(ptnPath, []byte(p.Render()), 0644)
}
<|endoftext|>