<commit_before>package app

import (
	"fmt"
	"github.com/Tapjoy/dynamiq/app/stats"
	"github.com/hashicorp/memberlist"
	"github.com/tpjg/goriakpbc"
	"log"
	"math"
	"math/rand"
	"strconv"
	"time"
)

// Define statistics keys suffixes

const QUEUE_SENT_STATS_SUFFIX = "sent.count"
const QUEUE_RECEIVED_STATS_SUFFIX = "received.count"
const QUEUE_DELETED_STATS_SUFFIX = "deleted.count"
const QUEUE_DEPTH_STATS_SUFFIX = "depth.count"
const QUEUE_INFLIGHT_STATS_SUFFIX = "inflight.count"
const QUEUE_AVAILABLE_STATS_SUFFIX = "available.count"

type Queues struct {
	// a container for all queues
	QueueMap map[string]Queue
	// Settings for Queues in general, ie queue list
	Config *riak.RDtMap
}

type Queue struct {
	// the definition of a queue
	// name of the queue
	Name string
	// the partitions of the queue
	Parts Partitions
	// Individual settings for the queue
	Config *riak.RDtMap
}

func incrementMessageCount(c stats.StatsClient, queueName string, numberOfMessages int64) error {
	// Increment # Sent
	key := fmt.Sprintf("%s.%s", queueName, QUEUE_SENT_STATS_SUFFIX)
	err := c.Incr(key, numberOfMessages)
	// Increment Depth count
	key = fmt.Sprintf("%s.%s", queueName, QUEUE_DEPTH_STATS_SUFFIX)
	err = c.IncrGauge(key, numberOfMessages)
	return err
}

func decrementMessageCount(c stats.StatsClient, queueName string, numberOfMessages int64) error {
	// Increment # Deleted
	key := fmt.Sprintf("%s.%s", queueName, QUEUE_DELETED_STATS_SUFFIX)
	err := c.Incr(key, numberOfMessages)
	// Decrement Inflight count
	key = fmt.Sprintf("%s.%s", queueName, QUEUE_INFLIGHT_STATS_SUFFIX)
	err = c.DecrGauge(key, numberOfMessages)
	// Decrement Depth count
	key = fmt.Sprintf("%s.%s", queueName, QUEUE_DEPTH_STATS_SUFFIX)
	err = c.DecrGauge(key, numberOfMessages)
	return err
}

func incrementReceiveCount(c stats.StatsClient, queueName string, numberOfMessages int64) error {
	// Increment # Received
	key := fmt.Sprintf("%s.%s", queueName, QUEUE_RECEIVED_STATS_SUFFIX)
	err := c.Incr(key, numberOfMessages)
	// Increment Inflight count
	key = fmt.Sprintf("%s.%s", queueName, QUEUE_INFLIGHT_STATS_SUFFIX)
	err = c.IncrGauge(key, numberOfMessages)
	return err
}
func (queue Queue) setQueueAvailable(c stats.StatsClient, list *memberlist.Memberlist, queueName string, ids []string) error {
	// set  depth
	key := fmt.Sprintf("%s.%s", queueName, QUEUE_AVAILABLE_STATS_SUFFIX)
	// find the difference between the first messages id and the last messages id

	first, _ := strconv.ParseInt(ids[0], 10, 64)
	last, _ := strconv.ParseInt(ids[len(ids)-1], 10, 64)
	difference := last - first
	//find the density of messages
	density := float64(len(ids)) / float64(difference)
	// find the total count of messages
	count := density * math.MaxInt64

	// for small queues where we only return 1 message guesstimate ( or should we return 0? )
	multiplier := queue.Parts.PartitionCount() * len(list.Members())
	if len(ids) == 1 {
		return c.SetGauge(key, int64(len(ids)*multiplier))
	} else {
		return c.SetGauge(key, int64(count))
	}
}

func (queues Queues) Exists(cfg Config, queueName string) bool {
	// For now, lets go right to Riak for this
	// Because of the config delay, we don't wanna check the memory values
	client := cfg.RiakConnection()
	defer cfg.ReleaseRiakConnection(client)

	bucket, _ := client.NewBucketType("maps", CONFIGURATION_BUCKET)
	m, _ := bucket.FetchMap(QUEUE_CONFIG_NAME)
	set := m.AddSet(QUEUE_SET_NAME)

	for _, value := range set.GetValue() {
		log.Printf("Looking for %s, found %s", queueName, string(value[:]))
		if string(value[:]) == queueName {
			return true
		}
	}
	return false
}

// get a message from the queue
func (queue Queue) Get(cfg Config, list *memberlist.Memberlist, batchsize uint32) ([]riak.RObject, error) {
	// get the top and bottom partitions
	partBottom, partTop, err := queue.Parts.GetPartition(cfg, queue.Name, list)

	if err != nil {
		return nil, err
	}
	// grab a riak client
	client := cfg.RiakConnection()
	defer cfg.ReleaseRiakConnection(client)

	//set the bucket
	bucket, err := client.NewBucket(queue.Name)
	if err != nil {
		log.Printf("Error%v", err)
	}
	//get a list of batchsize message ids
	messageIds, _, err := bucket.IndexQueryRangePage("id_int", strconv.Itoa(partBottom), strconv.Itoa(partTop), batchsize, "")
	defer queue.setQueueAvailable(cfg.Stats.Client, list, queue.Name, messageIds)

	if err != nil {
		log.Printf("Error%v", err)
	}
	messageCount := len(messageIds)
	defer incrementReceiveCount(cfg.Stats.Client, queue.Name, int64(messageCount))
	log.Println("Message retrieved ", messageCount)
	return queue.RetrieveMessages(messageIds, cfg), err
}

// Put a Message onto the queue
func (queue Queue) Put(cfg Config, message string) string {
	//Grab our bucket
	client := cfg.RiakConnection()
	defer cfg.ReleaseRiakConnection(client)
	bucket, err := client.NewBucket(queue.Name)
	if err == nil {
		//Retrieve a UUID
		rand.Seed(time.Now().UnixNano())
		randInt := rand.Int63n(math.MaxInt64)
		uuid := strconv.FormatInt(randInt, 10)

		messageObj := bucket.NewObject(uuid)
		messageObj.Indexes["id_int"] = []string{uuid}
		messageObj.Data = []byte(message)
		messageObj.Store()

		defer incrementMessageCount(cfg.Stats.Client, queue.Name, 1)
		return uuid
	} else {
		//Actually want to handle this in some other way
		return ""
	}
}

// Delete a Message from the queue
func (queue Queue) Delete(cfg Config, id string) bool {
	client := cfg.RiakConnection()
	defer cfg.ReleaseRiakConnection(client)
	bucket, err := client.NewBucket(queue.Name)
	if err == nil {
		log.Println("Deleting: ", id)
		err = bucket.Delete(id)
		if err == nil {
			defer decrementMessageCount(cfg.Stats.Client, queue.Name, 1)
			return true
		}
	}
	if err != nil {
		log.Println(err)
	}
	// if we got here we're borked
	// TODO stats cleanup? Possibility that this gets us out of sync
	return false
}

// helpers
func (queue Queue) RetrieveMessages(ids []string, cfg Config) []riak.RObject {
	var rObjectArrayChan = make(chan []riak.RObject, len(ids))
	var rKeys = make(chan string, len(ids))

	start := time.Now()
	//fmt.Println("In get multi")
	for i := 0; i < len(ids); i++ {
		go func() {
			var riakKey string
			client := cfg.RiakConnection()
			defer cfg.ReleaseRiakConnection(client)
			//fmt.Println("Getting bucket")
			bucket, _ := client.NewBucket(queue.Name)
			riakKey = <-rKeys
			//fmt.Println("Getting value")
			rObject, _ := bucket.Get(riakKey)

			rObjectArrayChan <- []riak.RObject{*rObject}

			//fmt.Println("Returning value")
		}()
		rKeys <- ids[i]
	}
	returnVals := make([]riak.RObject, 0)
	for i := 0; i < len(ids); i++ {
		var rObjectArray = <-rObjectArrayChan
		//If the key isn't blank, we've got a meaningful object to deal with
		if len(rObjectArray) == 1 {
			returnVals = append(returnVals, rObjectArray[0])
		}
	}
	elapsed := time.Since(start)
	log.Printf("Get Multi Took %s\n", elapsed)
	return returnVals
}

func (queues Queues) syncConfig(cfg Config) {
	for {
		log.Println("syncing Queue config with Riak")
		client := cfg.RiakConnection()
		bucket, err := client.NewBucketType("maps", CONFIGURATION_BUCKET)
		if err != nil {
			log.Println(err)
		}
		queues.Config, err = bucket.FetchMap(QUEUE_CONFIG_NAME)
		if err != nil {
			log.Println(err)
		}

		//iterate the map and add or remove topics that need to be destroyed
		queueSet := queues.Config.AddSet(QUEUE_SET_NAME)

		if queueSet == nil {
			//bail if there aren't any topics
			//but not before sleeping
			cfg.ReleaseRiakConnection(client)
			time.Sleep(cfg.Core.SyncConfigInterval * time.Second)
			continue
		}
		queueSlice := queueSet.GetValue()
		if queueSlice == nil {
			//bail if there aren't any topics
			//but not before sleeping
			cfg.ReleaseRiakConnection(client)
			time.Sleep(cfg.Core.SyncConfigInterval * time.Second)
			continue
		}

		//Is there a better way to do this?
		//iterate over the queues in riak and add the missing ones
		queuesToKeep := make(map[string]bool)
		for _, queue := range queueSlice {
			queueName := string(queue)
			var present bool
			_, present = queues.QueueMap[queueName]
			if present != true {
				initQueueFromRiak(cfg, queueName)
			}
			queuesToKeep[queueName] = true
		}

		//iterate over the topics in topics.TopicMap and delete the ones no longer used
		for queue, _ := range queues.QueueMap {
			var present bool
			_, present = queuesToKeep[queue]
			if present != true {
				delete(queues.QueueMap, queue)
			}
		}

		//sync all topics with riak

		for _, queue := range queues.QueueMap {
			queue.syncConfig(cfg)
		}
		//sleep for the configured interval
		cfg.ReleaseRiakConnection(client)
		time.Sleep(cfg.Core.SyncConfigInterval * time.Millisecond)
	}
}

func initQueueFromRiak(cfg Config, queueName string) {
	client := cfg.RiakConnection()
	defer cfg.ReleaseRiakConnection(client)

	bucket, _ := client.NewBucketType("maps", CONFIGURATION_BUCKET)
	config, _ := bucket.FetchMap(queueConfigRecordName(queueName))

	queue := Queue{
		Name:   queueName,
		Parts:  InitPartitions(cfg, queueName),
		Config: config,
	}

	cfg.Queues.QueueMap[queueName] = queue
}

func (queue Queue) syncConfig(cfg Config) {
	//refresh the queue RDtMap
	client := cfg.RiakConnection()
	defer cfg.ReleaseRiakConnection(client)
	bucket, _ := client.NewBucketType("maps", CONFIGURATION_BUCKET)
	recordName := queueConfigRecordName(queue.Name)
	queue.Config, _ = bucket.FetchMap(recordName)
	cfg.Queues.QueueMap[queue.Name] = queue
	queue.Parts.syncPartitions(cfg, queue.Name)
}
<commit_msg>changing the names<commit_after>package app

import (
	"fmt"
	"github.com/Tapjoy/dynamiq/app/stats"
	"github.com/hashicorp/memberlist"
	"github.com/tpjg/goriakpbc"
	"log"
	"math"
	"math/rand"
	"strconv"
	"time"
)

// Define statistics keys suffixes

const QUEUE_SENT_STATS_SUFFIX = "sent.count"
const QUEUE_RECEIVED_STATS_SUFFIX = "received.count"
const QUEUE_DELETED_STATS_SUFFIX = "deleted.count"
const QUEUE_DEPTH_STATS_SUFFIX = "depth.count"
const QUEUE_INFLIGHT_STATS_SUFFIX = "inflight.count"
const QUEUE_DEPTHAPR_STATS_SUFFIX = "approximate_depth.count"

type Queues struct {
	// a container for all queues
	QueueMap map[string]Queue
	// Settings for Queues in general, ie queue list
	Config *riak.RDtMap
}

type Queue struct {
	// the definition of a queue
	// name of the queue
	Name string
	// the partitions of the queue
	Parts Partitions
	// Individual settings for the queue
	Config *riak.RDtMap
}

func incrementMessageCount(c stats.StatsClient, queueName string, numberOfMessages int64) error {
	// Increment # Sent
	key := fmt.Sprintf("%s.%s", queueName, QUEUE_SENT_STATS_SUFFIX)
	err := c.Incr(key, numberOfMessages)
	// Increment Depth count
	key = fmt.Sprintf("%s.%s", queueName, QUEUE_DEPTH_STATS_SUFFIX)
	err = c.IncrGauge(key, numberOfMessages)
	return err
}

func decrementMessageCount(c stats.StatsClient, queueName string, numberOfMessages int64) error {
	// Increment # Deleted
	key := fmt.Sprintf("%s.%s", queueName, QUEUE_DELETED_STATS_SUFFIX)
	err := c.Incr(key, numberOfMessages)
	// Decrement Inflight count
	key = fmt.Sprintf("%s.%s", queueName, QUEUE_INFLIGHT_STATS_SUFFIX)
	err = c.DecrGauge(key, numberOfMessages)
	// Decrement Depth count
	key = fmt.Sprintf("%s.%s", queueName, QUEUE_DEPTH_STATS_SUFFIX)
	err = c.DecrGauge(key, numberOfMessages)
	return err
}

func incrementReceiveCount(c stats.StatsClient, queueName string, numberOfMessages int64) error {
	// Increment # Received
	key := fmt.Sprintf("%s.%s", queueName, QUEUE_RECEIVED_STATS_SUFFIX)
	err := c.Incr(key, numberOfMessages)
	// Increment Inflight count
	key = fmt.Sprintf("%s.%s", queueName, QUEUE_INFLIGHT_STATS_SUFFIX)
	err = c.IncrGauge(key, numberOfMessages)
	return err
}
func (queue Queue) setQueueDepthApr(c stats.StatsClient, list *memberlist.Memberlist, queueName string, ids []string) error {
	// set  depth
	key := fmt.Sprintf("%s.%s", queueName, QUEUE_DEPTHAPR_STATS_SUFFIX)
	// find the difference between the first messages id and the last messages id

	first, _ := strconv.ParseInt(ids[0], 10, 64)
	last, _ := strconv.ParseInt(ids[len(ids)-1], 10, 64)
	difference := last - first
	//find the density of messages
	density := float64(len(ids)) / float64(difference)
	// find the total count of messages
	count := density * math.MaxInt64

	// for small queues where we only return 1 message guesstimate ( or should we return 0? )
	multiplier := queue.Parts.PartitionCount() * len(list.Members())
	if len(ids) == 1 {
		return c.SetGauge(key, int64(len(ids)*multiplier))
	} else {
		return c.SetGauge(key, int64(count))
	}
}

func (queues Queues) Exists(cfg Config, queueName string) bool {
	// For now, lets go right to Riak for this
	// Because of the config delay, we don't wanna check the memory values
	client := cfg.RiakConnection()
	defer cfg.ReleaseRiakConnection(client)

	bucket, _ := client.NewBucketType("maps", CONFIGURATION_BUCKET)
	m, _ := bucket.FetchMap(QUEUE_CONFIG_NAME)
	set := m.AddSet(QUEUE_SET_NAME)

	for _, value := range set.GetValue() {
		log.Printf("Looking for %s, found %s", queueName, string(value[:]))
		if string(value[:]) == queueName {
			return true
		}
	}
	return false
}

// get a message from the queue
func (queue Queue) Get(cfg Config, list *memberlist.Memberlist, batchsize uint32) ([]riak.RObject, error) {
	// get the top and bottom partitions
	partBottom, partTop, err := queue.Parts.GetPartition(cfg, queue.Name, list)

	if err != nil {
		return nil, err
	}
	// grab a riak client
	client := cfg.RiakConnection()
	defer cfg.ReleaseRiakConnection(client)

	//set the bucket
	bucket, err := client.NewBucket(queue.Name)
	if err != nil {
		log.Printf("Error%v", err)
	}
	//get a list of batchsize message ids
	messageIds, _, err := bucket.IndexQueryRangePage("id_int", strconv.Itoa(partBottom), strconv.Itoa(partTop), batchsize, "")
	defer queue.setQueueDepthApr(cfg.Stats.Client, list, queue.Name, messageIds)

	if err != nil {
		log.Printf("Error%v", err)
	}
	messageCount := len(messageIds)
	defer incrementReceiveCount(cfg.Stats.Client, queue.Name, int64(messageCount))
	log.Println("Message retrieved ", messageCount)
	return queue.RetrieveMessages(messageIds, cfg), err
}

// Put a Message onto the queue
func (queue Queue) Put(cfg Config, message string) string {
	//Grab our bucket
	client := cfg.RiakConnection()
	defer cfg.ReleaseRiakConnection(client)
	bucket, err := client.NewBucket(queue.Name)
	if err == nil {
		//Retrieve a UUID
		rand.Seed(time.Now().UnixNano())
		randInt := rand.Int63n(math.MaxInt64)
		uuid := strconv.FormatInt(randInt, 10)

		messageObj := bucket.NewObject(uuid)
		messageObj.Indexes["id_int"] = []string{uuid}
		messageObj.Data = []byte(message)
		messageObj.Store()

		defer incrementMessageCount(cfg.Stats.Client, queue.Name, 1)
		return uuid
	} else {
		//Actually want to handle this in some other way
		return ""
	}
}

// Delete a Message from the queue
func (queue Queue) Delete(cfg Config, id string) bool {
	client := cfg.RiakConnection()
	defer cfg.ReleaseRiakConnection(client)
	bucket, err := client.NewBucket(queue.Name)
	if err == nil {
		log.Println("Deleting: ", id)
		err = bucket.Delete(id)
		if err == nil {
			defer decrementMessageCount(cfg.Stats.Client, queue.Name, 1)
			return true
		}
	}
	if err != nil {
		log.Println(err)
	}
	// if we got here we're borked
	// TODO stats cleanup? Possibility that this gets us out of sync
	return false
}

// helpers
func (queue Queue) RetrieveMessages(ids []string, cfg Config) []riak.RObject {
	var rObjectArrayChan = make(chan []riak.RObject, len(ids))
	var rKeys = make(chan string, len(ids))

	start := time.Now()
	//fmt.Println("In get multi")
	for i := 0; i < len(ids); i++ {
		go func() {
			var riakKey string
			client := cfg.RiakConnection()
			defer cfg.ReleaseRiakConnection(client)
			//fmt.Println("Getting bucket")
			bucket, _ := client.NewBucket(queue.Name)
			riakKey = <-rKeys
			//fmt.Println("Getting value")
			rObject, _ := bucket.Get(riakKey)

			rObjectArrayChan <- []riak.RObject{*rObject}

			//fmt.Println("Returning value")
		}()
		rKeys <- ids[i]
	}
	returnVals := make([]riak.RObject, 0)
	for i := 0; i < len(ids); i++ {
		var rObjectArray = <-rObjectArrayChan
		//If the key isn't blank, we've got a meaningful object to deal with
		if len(rObjectArray) == 1 {
			returnVals = append(returnVals, rObjectArray[0])
		}
	}
	elapsed := time.Since(start)
	log.Printf("Get Multi Took %s\n", elapsed)
	return returnVals
}

func (queues Queues) syncConfig(cfg Config) {
	for {
		log.Println("syncing Queue config with Riak")
		client := cfg.RiakConnection()
		bucket, err := client.NewBucketType("maps", CONFIGURATION_BUCKET)
		if err != nil {
			log.Println(err)
		}
		queues.Config, err = bucket.FetchMap(QUEUE_CONFIG_NAME)
		if err != nil {
			log.Println(err)
		}

		//iterate the map and add or remove topics that need to be destroyed
		queueSet := queues.Config.AddSet(QUEUE_SET_NAME)

		if queueSet == nil {
			//bail if there aren't any topics
			//but not before sleeping
			cfg.ReleaseRiakConnection(client)
			time.Sleep(cfg.Core.SyncConfigInterval * time.Second)
			continue
		}
		queueSlice := queueSet.GetValue()
		if queueSlice == nil {
			//bail if there aren't any topics
			//but not before sleeping
			cfg.ReleaseRiakConnection(client)
			time.Sleep(cfg.Core.SyncConfigInterval * time.Second)
			continue
		}

		//Is there a better way to do this?
		//iterate over the queues in riak and add the missing ones
		queuesToKeep := make(map[string]bool)
		for _, queue := range queueSlice {
			queueName := string(queue)
			var present bool
			_, present = queues.QueueMap[queueName]
			if present != true {
				initQueueFromRiak(cfg, queueName)
			}
			queuesToKeep[queueName] = true
		}

		//iterate over the topics in topics.TopicMap and delete the ones no longer used
		for queue, _ := range queues.QueueMap {
			var present bool
			_, present = queuesToKeep[queue]
			if present != true {
				delete(queues.QueueMap, queue)
			}
		}

		//sync all topics with riak

		for _, queue := range queues.QueueMap {
			queue.syncConfig(cfg)
		}
		//sleep for the configured interval
		cfg.ReleaseRiakConnection(client)
		time.Sleep(cfg.Core.SyncConfigInterval * time.Millisecond)
	}
}

func initQueueFromRiak(cfg Config, queueName string) {
	client := cfg.RiakConnection()
	defer cfg.ReleaseRiakConnection(client)

	bucket, _ := client.NewBucketType("maps", CONFIGURATION_BUCKET)
	config, _ := bucket.FetchMap(queueConfigRecordName(queueName))

	queue := Queue{
		Name:   queueName,
		Parts:  InitPartitions(cfg, queueName),
		Config: config,
	}

	cfg.Queues.QueueMap[queueName] = queue
}

func (queue Queue) syncConfig(cfg Config) {
	//refresh the queue RDtMap
	client := cfg.RiakConnection()
	defer cfg.ReleaseRiakConnection(client)
	bucket, _ := client.NewBucketType("maps", CONFIGURATION_BUCKET)
	recordName := queueConfigRecordName(queue.Name)
	queue.Config, _ = bucket.FetchMap(recordName)
	cfg.Queues.QueueMap[queue.Name] = queue
	queue.Parts.syncPartitions(cfg, queue.Name)
}
<|endoftext|>


<commit_before>package mastodon

import (
	"context"
	"fmt"
	"net/http"
	"net/http/httptest"
	"testing"
	"time"
)

func TestRegisterApp(t *testing.T) {
	isNotJSON := true
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Method != http.MethodPost {
			http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
			return
		} else if r.URL.Path != "/api/v1/apps" {
			http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
			return
		} else if r.FormValue("redirect_uris") != "urn:ietf:wg:oauth:2.0:oob" {
			http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
			return
		} else if isNotJSON {
			isNotJSON = false
			fmt.Fprintln(w, `<html><head><title>Apps</title></head></html>`)
			return
		}
		fmt.Fprintln(w, `{"client_id": "foo", "client_secret": "bar"}`)
		return
	}))
	defer ts.Close()

	// Status not ok.
	_, err := RegisterApp(context.Background(), &AppConfig{
		Server:       ts.URL,
		RedirectURIs: "/",
	})
	if err == nil {
		t.Fatalf("should be fail: %v", err)
	}

	// Error in url.Parse
	_, err = RegisterApp(context.Background(), &AppConfig{
		Server: ":",
	})
	if err == nil {
		t.Fatalf("should be fail: %v", err)
	}

	// Error in json.NewDecoder
	_, err = RegisterApp(context.Background(), &AppConfig{
		Server: ts.URL,
	})
	if err == nil {
		t.Fatalf("should be fail: %v", err)
	}

	// Success.
	app, err := RegisterApp(context.Background(), &AppConfig{
		Server: ts.URL,
		Scopes: "read write follow",
	})
	if err != nil {
		t.Fatalf("should not be fail: %v", err)
	}
	if app.ClientID != "foo" {
		t.Fatalf("want %q but %q", "foo", app.ClientID)
	}
	if app.ClientSecret != "bar" {
		t.Fatalf("want %q but %q", "bar", app.ClientSecret)
	}
}

func TestRegisterAppWithCancel(t *testing.T) {
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		time.Sleep(3 * time.Second)
		fmt.Fprintln(w, `{"client_id": "foo", "client_secret": "bar"}`)
		return
	}))
	defer ts.Close()

	ctx, cancel := context.WithCancel(context.Background())
	cancel()
	_, err := RegisterApp(ctx, &AppConfig{
		Server: ts.URL,
		Scopes: "read write follow",
	})
	if err == nil {
		t.Fatalf("should be fail: %v", err)
	}
	if want := "Post " + ts.URL + "/api/v1/apps: context canceled"; want != err.Error() {
		t.Fatalf("want %q but %q", want, err.Error())
	}
}
<commit_msg>add test<commit_after>package mastodon

import (
	"context"
	"fmt"
	"net/http"
	"net/http/httptest"
	"testing"
	"time"
)

func TestRegisterApp(t *testing.T) {
	isNotJSON := true
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Method != http.MethodPost {
			http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
			return
		} else if r.URL.Path != "/api/v1/apps" {
			http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
			return
		} else if r.FormValue("redirect_uris") != "urn:ietf:wg:oauth:2.0:oob" {
			http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
			return
		} else if isNotJSON {
			isNotJSON = false
			fmt.Fprintln(w, `<html><head><title>Apps</title></head></html>`)
			return
		}
		fmt.Fprintln(w, `{"id": 123, "client_id": "foo", "client_secret": "bar"}`)
		return
	}))
	defer ts.Close()

	// Status not ok.
	_, err := RegisterApp(context.Background(), &AppConfig{
		Server:       ts.URL,
		RedirectURIs: "/",
	})
	if err == nil {
		t.Fatalf("should be fail: %v", err)
	}

	// Error in url.Parse
	_, err = RegisterApp(context.Background(), &AppConfig{
		Server: ":",
	})
	if err == nil {
		t.Fatalf("should be fail: %v", err)
	}

	// Error in json.NewDecoder
	_, err = RegisterApp(context.Background(), &AppConfig{
		Server: ts.URL,
	})
	if err == nil {
		t.Fatalf("should be fail: %v", err)
	}

	// Success.
	app, err := RegisterApp(context.Background(), &AppConfig{
		Server: ts.URL,
		Scopes: "read write follow",
	})
	if err != nil {
		t.Fatalf("should not be fail: %v", err)
	}
	if string(app.ID) != "123" {
		t.Fatalf("want %q but %q", "bar", app.ClientSecret)
	}
	if app.ClientID != "foo" {
		t.Fatalf("want %q but %q", "foo", app.ClientID)
	}
	if app.ClientSecret != "bar" {
		t.Fatalf("want %q but %q", "bar", app.ClientSecret)
	}
}

func TestRegisterAppWithCancel(t *testing.T) {
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		time.Sleep(3 * time.Second)
		fmt.Fprintln(w, `{"client_id": "foo", "client_secret": "bar"}`)
		return
	}))
	defer ts.Close()

	ctx, cancel := context.WithCancel(context.Background())
	cancel()
	_, err := RegisterApp(ctx, &AppConfig{
		Server: ts.URL,
		Scopes: "read write follow",
	})
	if err == nil {
		t.Fatalf("should be fail: %v", err)
	}
	if want := "Post " + ts.URL + "/api/v1/apps: context canceled"; want != err.Error() {
		t.Fatalf("want %q but %q", want, err.Error())
	}
}
<|endoftext|>


<commit_before>package delta_test

import (
	"testing"

	"github.com/GeoNet/delta/meta"
)

var testViews = map[string]func([]meta.View) func(t *testing.T){

	"check for duplicated views": func(views []meta.View) func(t *testing.T) {
		return func(t *testing.T) {
			for i := 0; i < len(views); i++ {
				for j := i + 1; j < len(views); j++ {
					if views[i].Mount != views[j].Mount {
						continue
					}
					if views[i].Code == views[j].Code {
						t.Errorf("view duplication: %s/%s", views[i].Mount, views[i].Code)
					}
				}
			}
		}
	},
}

func TestViews(t *testing.T) {

	var views meta.ViewList
	loadListFile(t, "../network/views.csv", &views)

	for k, fn := range testViews {
		t.Run(k, fn(views))
	}
}
<commit_msg>update view test to allow for multiple installs<commit_after>package delta_test

import (
	"testing"

	"github.com/GeoNet/delta/meta"
)

var testViews = map[string]func([]meta.View) func(t *testing.T){

	"check for duplicated views": func(views []meta.View) func(t *testing.T) {
		return func(t *testing.T) {
			for i := 0; i < len(views); i++ {
				for j := i + 1; j < len(views); j++ {
					if views[i].Mount != views[j].Mount {
						continue
					}
					if views[i].Code != views[j].Code {
						continue
					}
					if views[i].Start.After(views[j].End) {
						continue
					}
					if views[i].End.Before(views[j].Start) {
						continue
					}
					t.Errorf("view duplication: %s/%s", views[i].Mount, views[i].Code)
				}
			}
		}
	},
}

func TestViews(t *testing.T) {

	var views meta.ViewList
	loadListFile(t, "../network/views.csv", &views)

	for k, fn := range testViews {
		t.Run(k, fn(views))
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 Uber Technologies, Inc.

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package testutils

import (
	"strings"
	"testing"

	"github.com/uber/tchannel-go"
)

type errorLoggerState struct {
	matchCount int
}

type errorLogger struct {
	tchannel.Logger
	t testing.TB
	v *LogVerification
	s *errorLoggerState
}

func (l errorLogger) checkErr(msg string, args ...interface{}) {
	allowedCount := l.v.AllowedCount

	if filter := l.v.AllowedFilter; filter != "" {
		if !strings.Contains(msg, filter) {
			l.t.Errorf(msg, args...)
		}
	}

	l.s.matchCount++
	if l.s.matchCount <= allowedCount {
		return
	}

	l.t.Errorf(msg, args...)
}

func (l errorLogger) Fatalf(msg string, args ...interface{}) {
	l.checkErr("[Fatal] "+msg, args...)
	l.Logger.Fatalf(msg, args...)
}

func (l errorLogger) Errorf(msg string, args ...interface{}) {
	l.checkErr("[Error] "+msg, args...)
	l.Logger.Errorf(msg, args...)
}

func (l errorLogger) Warnf(msg string, args ...interface{}) {
	l.checkErr("[Warn] "+msg, args...)
	l.Logger.Warnf(msg, args...)
}

func (l errorLogger) WithFields(fields ...tchannel.LogField) tchannel.Logger {
	return errorLogger{l.Logger.WithFields(fields...), l.t, l.v, l.s}
}
<commit_msg>Fix race in testutils.errorLogger<commit_after>// Copyright (c) 2015 Uber Technologies, Inc.

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package testutils

import (
	"strings"
	"sync/atomic"
	"testing"

	"github.com/uber/tchannel-go"
)

type errorLoggerState struct {
	matchCount uint32
}

type errorLogger struct {
	tchannel.Logger
	t testing.TB
	v *LogVerification
	s *errorLoggerState
}

func (l errorLogger) checkErr(msg string, args ...interface{}) {
	allowedCount := l.v.AllowedCount

	if filter := l.v.AllowedFilter; filter != "" {
		if !strings.Contains(msg, filter) {
			l.t.Errorf(msg, args...)
		}
	}

	matchCount := atomic.AddUint32(&l.s.matchCount, 1)
	if int(matchCount) <= allowedCount {
		return
	}

	l.t.Errorf(msg, args...)
}

func (l errorLogger) Fatalf(msg string, args ...interface{}) {
	l.checkErr("[Fatal] "+msg, args...)
	l.Logger.Fatalf(msg, args...)
}

func (l errorLogger) Errorf(msg string, args ...interface{}) {
	l.checkErr("[Error] "+msg, args...)
	l.Logger.Errorf(msg, args...)
}

func (l errorLogger) Warnf(msg string, args ...interface{}) {
	l.checkErr("[Warn] "+msg, args...)
	l.Logger.Warnf(msg, args...)
}

func (l errorLogger) WithFields(fields ...tchannel.LogField) tchannel.Logger {
	return errorLogger{l.Logger.WithFields(fields...), l.t, l.v, l.s}
}
<|endoftext|>


<commit_before>package dispatch

import "errors"

// ErrorManagerNotReady ErrorManagerNotReady
var ErrorManagerNotReady = errors.New("Manager is not ready")

// Manager 管理员，接受任务和派发任务
type Manager struct {
	workPool    chan *Worker  // the pool of workers
	jobQueue    chan Job      // accept incoming job
	stopChannel chan struct{} // 停止
	isReady     bool          // 是处于可用状态
}

// NewManager create a new *Manager
func NewManager(maxWorkerCount int) *Manager {
	//
	pool := make(chan *Worker, maxWorkerCount)
	return &Manager{
		workPool:    pool,
		jobQueue:    make(chan Job),
		stopChannel: make(chan struct{}),
		isReady:     false}
}

// Setup 创建workers
func (manager *Manager) Setup() {
	count := cap(manager.workPool)
	for i := 0; i < count; i++ {
		worker := NewWorker(manager)
		worker.Start()
	}
}

// Start 开始接受任务
func (manager *Manager) Start() {
	go manager.dispatch()
	manager.isReady = true
}

// Accept 接收到新的任务
func (manager *Manager) Accept(job Job) error {
	if manager.IsReady() {
		manager.jobQueue <- job
		return nil
	}
	return ErrorManagerNotReady
}

// Stop 停止接受任务
func (manager *Manager) Stop() {
	manager.isReady = false
	manager.stopChannel <- struct{}{}
}

// IsReady 是否可用
func (manager *Manager) IsReady() bool {
	return manager.isReady
}

// dispatch 派发任务
func (manager *Manager) dispatch() {
	for {
		select {
		case job := <-manager.jobQueue:
			go func(job Job) {
				// try to obtain a worker job channel that is available.
				// this will block until a worker is idle
				worker := <-manager.workPool
				// dispatch the job to the worker's job channel
				worker.Receive(job)
			}(job)
		case <-manager.stopChannel:
			return
		}
	}
}

// register 注册worker，表示worker现在空闲可用
func (manager *Manager) register(worker *Worker) {
	manager.workPool <- worker
}
<commit_msg>format error name<commit_after>package dispatch

import "errors"

// ErrManagerNotReady ErrManagerNotReady
var ErrManagerNotReady = errors.New("Manager is not ready")

// Manager 管理员，接受任务和派发任务
type Manager struct {
	workPool    chan *Worker  // the pool of workers
	jobQueue    chan Job      // accept incoming job
	stopChannel chan struct{} // 停止
	isReady     bool          // 是处于可用状态
}

// NewManager create a new *Manager
func NewManager(maxWorkerCount int) *Manager {
	//
	pool := make(chan *Worker, maxWorkerCount)
	return &Manager{
		workPool:    pool,
		jobQueue:    make(chan Job),
		stopChannel: make(chan struct{}),
		isReady:     false}
}

// Setup 创建workers
func (manager *Manager) Setup() {
	count := cap(manager.workPool)
	for i := 0; i < count; i++ {
		worker := NewWorker(manager)
		worker.Start()
	}
}

// Start 开始接受任务
func (manager *Manager) Start() {
	go manager.dispatch()
	manager.isReady = true
}

// Accept 接收到新的任务
func (manager *Manager) Accept(job Job) error {
	if manager.IsReady() {
		manager.jobQueue <- job
		return nil
	}
	return ErrManagerNotReady
}

// Stop 停止接受任务
func (manager *Manager) Stop() {
	manager.isReady = false
	manager.stopChannel <- struct{}{}
}

// IsReady 是否可用
func (manager *Manager) IsReady() bool {
	return manager.isReady
}

// dispatch 派发任务
func (manager *Manager) dispatch() {
	for {
		select {
		case job := <-manager.jobQueue:
			go func(job Job) {
				// try to obtain a worker job channel that is available.
				// this will block until a worker is idle
				worker := <-manager.workPool
				// dispatch the job to the worker's job channel
				worker.Receive(job)
			}(job)
		case <-manager.stopChannel:
			return
		}
	}
}

// register 注册worker，表示worker现在空闲可用
func (manager *Manager) register(worker *Worker) {
	manager.workPool <- worker
}
<|endoftext|>


<commit_before>/**
 * (C) Copyright 2013, Deft Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package dlshared

import (
	"net/url"
	"net/http"
	"io/ioutil"
	"time"
	"bytes"
	"strings"
	"crypto/tls"
	"labix.org/v2/mgo/bson"
	"encoding/json"
	"github.com/mreiferson/go-httpclient"
)

const (
	DefaultSocketTimeout = 40
	HttpPostMethod = "POST"
	HttpGetMethod = "GET"

	ContentTypeHeader = "Content-Type"

	ContentTypeTextPlain = "text/plain; charset=utf-8"
	ContentTypePostForm = "application/x-www-form-urlencoded"
	ContentTypeJson = "application/json"
)

// The http request client allows you to configure global values for timeout/connect etc. It also
// handles closing the transport and provides convenience methods for posting some types of data.
type HttpRequestClient struct {
	DisableKeepAlives bool
	DisableCompression bool
	SkipSslVerify bool
	MaxIdleConnsPerHost int
	ConnectTimeout time.Duration
	ResponseHeaderTimeout time.Duration
	RequestTimeout time.Duration
	ReadWriteTimeout time.Duration
}

func NewDefaultHttpRequestClient() *HttpRequestClient {
	return NewHttpRequestClient(true, false, false, 0, DefaultSocketTimeout, DefaultSocketTimeout, DefaultSocketTimeout, DefaultSocketTimeout)
}

func NewHttpRequestClient(	disableKeepAlives,
							disableCompression,
							skipSslVerify bool,
							maxIdleConnsPerHost,
							connectTimeoutInMs,
							responseHeaderTimeoutInMs,
							requestTimeoutInMs,
							readWriteTimeoutInMs int) *HttpRequestClient {

	return &HttpRequestClient{ 	DisableKeepAlives: disableKeepAlives,
								DisableCompression: disableCompression,
								SkipSslVerify: skipSslVerify,
								MaxIdleConnsPerHost: maxIdleConnsPerHost,
								ConnectTimeout: time.Duration(connectTimeoutInMs) * time.Millisecond,
								ResponseHeaderTimeout: time.Duration(responseHeaderTimeoutInMs) * time.Millisecond,
								RequestTimeout: time.Duration(requestTimeoutInMs) * time.Millisecond,
								ReadWriteTimeout: time.Duration(readWriteTimeoutInMs) * time.Millisecond,
	}
}

// Post the values to the url.
func (self *HttpRequestClient) Post(url string, values url.Values, headers map[string]string) ([]byte, error) {

	client, transport := self.getClientAndTransport()
	defer transport.Close()

	request, err := http.NewRequest(HttpPostMethod, url,  strings.NewReader(values.Encode()))
	if err != nil { return nil, err }

	request.Header.Set(ContentTypeHeader, ContentTypePostForm)
	for key, value := range headers { request.Header.Set(key, value) }

	response, err := client.Do(request)
	if err != nil { return nil, err }

	defer response.Body.Close()

	if data, err := ioutil.ReadAll(response.Body); err != nil { return nil, err
	} else { return data, nil }
}

// Post the raw string to the url.
func (self *HttpRequestClient) PostStr(url string, value string, headers map[string]string) ([]byte, error) {

	client, transport := self.getClientAndTransport()
	defer transport.Close()

	request, err := http.NewRequest(HttpPostMethod, url, bytes.NewReader([]byte(value)))
	if err != nil { return nil, err }

	request.Header.Set(ContentTypeHeader, ContentTypePostForm)
	for key, value := range headers { request.Header.Set(key, value) }

	response, err := client.Do(request)
	if err != nil { return nil, err }

	defer response.Body.Close()

	if data, err := ioutil.ReadAll(response.Body); err != nil { return nil, err
	} else { return data, nil }
}

// Post the json struct to the url.
func (self *HttpRequestClient) PostJson(url string, value interface{}, headers map[string]string) ([]byte, error) {

	rawJson, err := json.Marshal(value)
	if err != nil { return nil, err }

	client, transport := self.getClientAndTransport()
	defer transport.Close()

	request, err := http.NewRequest(HttpPostMethod, url, bytes.NewReader(rawJson))
	if err != nil { return nil, err }

	request.Header.Set(ContentTypeHeader, ContentTypeJson)
	for key, value := range headers { request.Header.Set(key, value) }

	response, err := client.Do(request)
	if err != nil { return nil, err }

	defer response.Body.Close()

	if data, err := ioutil.ReadAll(response.Body); err != nil { return nil, err
	} else { return data, nil }
}

// Post the bson doc to the url.
func (self *HttpRequestClient) PostBson(url string, bsonDoc interface{}, headers map[string]string) ([]byte, error) {

	rawBson, err := bson.Marshal(bsonDoc)
	if err != nil { return nil, err }

	client, transport := self.getClientAndTransport()
	defer transport.Close()

	request, err := http.NewRequest(HttpPostMethod, url, bytes.NewReader(rawBson))
	if err != nil { return nil, err }

	request.Header.Set(ContentTypeHeader, ContentTypePostForm)
	for key, value := range headers { request.Header.Set(key, value) }

	response, err := client.Do(request)
	if err != nil { return nil, err }

	defer response.Body.Close()

	// We do not return the response so don't report if there is an error.
	if data, err := ioutil.ReadAll(response.Body); err != nil { return nil, err
	} else { return data, nil }
}

// Issue a GET to retrieve a bson doc.
func (self *HttpRequestClient) GetBson(url string, headers map[string]string) (bson.M, error) {

	client, transport := self.getClientAndTransport()
	defer transport.Close()

	request, requestErr := http.NewRequest(HttpGetMethod, url, nil)
	if requestErr != nil { return nil, requestErr }

	for key, value := range headers { request.Header.Set(key, value) }

	response, err := client.Do(request)

	if err != nil { return nil, err }

	defer response.Body.Close()

	rawBson, err := ioutil.ReadAll(response.Body)
	if err != nil { return nil, err }

	var bsonDoc bson.M
	if err := bson.Unmarshal(rawBson, &bsonDoc); err != nil { return nil, err }

	return bsonDoc, nil
}

// Use the clone method if you need to override some/all of the configured values.
func (self *HttpRequestClient) Clone() *HttpRequestClient {
	val := *self
	return &val
}

func (self *HttpRequestClient) getClientAndTransport() (*http.Client, *httpclient.Transport) {
	transport := self.getTransport()
	return &http.Client{ Transport: transport }, transport
}

func (self *HttpRequestClient) getTransport() *httpclient.Transport {
	return &httpclient.Transport {

		DisableKeepAlives: self.DisableKeepAlives,
		DisableCompression: self.DisableCompression,

		MaxIdleConnsPerHost: self.MaxIdleConnsPerHost,

		ConnectTimeout: self.ConnectTimeout,
		ResponseHeaderTimeout: self.ResponseHeaderTimeout,
		RequestTimeout: self.RequestTimeout,
		ReadWriteTimeout: self.ReadWriteTimeout,

		TLSClientConfig: &tls.Config{ InsecureSkipVerify: self.SkipSslVerify },
	}
}

func HttpPost(url string, values url.Values, headers map[string]string) ([]byte, error) { return NewDefaultHttpRequestClient().Post(url, values, headers) }

func HttpPostStr(url string, value string, headers map[string]string) ([]byte, error) { return NewDefaultHttpRequestClient().PostStr(url, value, headers) }

func HttpPostJson(url string, value interface{}, headers map[string]string) ([]byte, error) { return NewDefaultHttpRequestClient().PostJson(url, value, headers) }

func HttpPostBson(url string, bsonDoc interface{}, headers map[string]string) ([]byte, error) { return NewDefaultHttpRequestClient().PostBson(url, bsonDoc, headers) }

func HttpGetBson(url string, headers map[string]string) (bson.M, error) { return NewDefaultHttpRequestClient().GetBson(url, headers) }

<commit_msg>fixed timeout - was in seconds, moved to ms<commit_after>/**
 * (C) Copyright 2013, Deft Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package dlshared

import (
	"net/url"
	"net/http"
	"io/ioutil"
	"time"
	"bytes"
	"strings"
	"crypto/tls"
	"labix.org/v2/mgo/bson"
	"encoding/json"
	"github.com/mreiferson/go-httpclient"
)

const (
	DefaultSocketTimeout = 4000
	HttpPostMethod = "POST"
	HttpGetMethod = "GET"

	ContentTypeHeader = "Content-Type"

	NoStatusCode = -1 // This is the value used if an error is generated before the status code is available

	ContentTypeTextPlain = "text/plain; charset=utf-8"
	ContentTypePostForm = "application/x-www-form-urlencoded"
	ContentTypeJson = "application/json"
)

// The http request client allows you to configure global values for timeout/connect etc. It also
// handles closing the transport and provides convenience methods for posting some types of data.
type HttpRequestClient struct {
	DisableKeepAlives bool
	DisableCompression bool
	SkipSslVerify bool
	MaxIdleConnsPerHost int
	ConnectTimeout time.Duration
	ResponseHeaderTimeout time.Duration
	RequestTimeout time.Duration
	ReadWriteTimeout time.Duration
}

func NewDefaultHttpRequestClient() *HttpRequestClient {
	return NewHttpRequestClient(true, false, false, 0, DefaultSocketTimeout, DefaultSocketTimeout, DefaultSocketTimeout, DefaultSocketTimeout)
}

func NewHttpRequestClient(	disableKeepAlives,
							disableCompression,
							skipSslVerify bool,
							maxIdleConnsPerHost,
							connectTimeoutInMs,
							responseHeaderTimeoutInMs,
							requestTimeoutInMs,
							readWriteTimeoutInMs int) *HttpRequestClient {

	return &HttpRequestClient{ 	DisableKeepAlives: disableKeepAlives,
								DisableCompression: disableCompression,
								SkipSslVerify: skipSslVerify,
								MaxIdleConnsPerHost: maxIdleConnsPerHost,
								ConnectTimeout: time.Duration(connectTimeoutInMs) * time.Millisecond,
								ResponseHeaderTimeout: time.Duration(responseHeaderTimeoutInMs) * time.Millisecond,
								RequestTimeout: time.Duration(requestTimeoutInMs) * time.Millisecond,
								ReadWriteTimeout: time.Duration(readWriteTimeoutInMs) * time.Millisecond,
	}
}

// Post the values to the url.
func (self *HttpRequestClient) Post(url string, values url.Values, headers map[string]string) (int, []byte, error) {

	client, transport := self.getClientAndTransport()
	defer transport.Close()

	request, err := http.NewRequest(HttpPostMethod, url,  strings.NewReader(values.Encode()))
	if err != nil { return NoStatusCode, nil, err }

	request.Header.Set(ContentTypeHeader, ContentTypePostForm)
	for key, value := range headers { request.Header.Set(key, value) }

	response, err := client.Do(request)
	if err != nil { return NoStatusCode, nil, err }

	statusCode := response.StatusCode

	defer response.Body.Close()

	if data, err := ioutil.ReadAll(response.Body); err != nil { return statusCode, nil, err
	} else { return statusCode, data, nil }
}

// Post the raw string to the url.
func (self *HttpRequestClient) PostStr(url string, value string, headers map[string]string) (int, []byte, error) {

	client, transport := self.getClientAndTransport()
	defer transport.Close()

	request, err := http.NewRequest(HttpPostMethod, url, bytes.NewReader([]byte(value)))
	if err != nil { return NoStatusCode, nil, err }

	request.Header.Set(ContentTypeHeader, ContentTypePostForm)
	for key, value := range headers { request.Header.Set(key, value) }

	response, err := client.Do(request)
	if err != nil { return NoStatusCode, nil, err }

	statusCode := response.StatusCode

	defer response.Body.Close()

	if data, err := ioutil.ReadAll(response.Body); err != nil { return statusCode, nil, err
	} else { return statusCode, data, nil }
}

// Post the json struct to the url.
func (self *HttpRequestClient) PostJson(url string, value interface{}, headers map[string]string) (int, []byte, error) {

	rawJson, err := json.Marshal(value)
	if err != nil { return NoStatusCode, nil, err }

	client, transport := self.getClientAndTransport()
	defer transport.Close()

	request, err := http.NewRequest(HttpPostMethod, url, bytes.NewReader(rawJson))
	if err != nil { return NoStatusCode, nil, err }

	request.Header.Set(ContentTypeHeader, ContentTypeJson)
	for key, value := range headers { request.Header.Set(key, value) }

	response, err := client.Do(request)
	if err != nil { return NoStatusCode, nil, err }

	statusCode := response.StatusCode

	defer response.Body.Close()

	if data, err := ioutil.ReadAll(response.Body); err != nil { return statusCode, nil, err
	} else { return statusCode, data, nil }
}

// Post the bson doc to the url.
func (self *HttpRequestClient) PostBson(url string, bsonDoc interface{}, headers map[string]string) (int, []byte, error) {

	rawBson, err := bson.Marshal(bsonDoc)
	if err != nil { return NoStatusCode, nil, err }

	client, transport := self.getClientAndTransport()
	defer transport.Close()

	request, err := http.NewRequest(HttpPostMethod, url, bytes.NewReader(rawBson))
	if err != nil { return NoStatusCode, nil, err }

	request.Header.Set(ContentTypeHeader, ContentTypePostForm)
	for key, value := range headers { request.Header.Set(key, value) }

	response, err := client.Do(request)
	if err != nil { return NoStatusCode, nil, err }

	statusCode := response.StatusCode

	defer response.Body.Close()

	// We do not return the response so don't report if there is an error.
	if data, err := ioutil.ReadAll(response.Body); err != nil { return statusCode, nil, err
	} else { return statusCode, data, nil }
}

// Issue a GET to retrieve a bson doc.
func (self *HttpRequestClient) GetBson(url string, headers map[string]string) (int, bson.M, error) {

	client, transport := self.getClientAndTransport()
	defer transport.Close()

	request, requestErr := http.NewRequest(HttpGetMethod, url, nil)
	if requestErr != nil { return NoStatusCode, nil, requestErr }

	for key, value := range headers { request.Header.Set(key, value) }

	response, err := client.Do(request)
	if err != nil { return NoStatusCode, nil, err }

	statusCode := response.StatusCode

	defer response.Body.Close()

	rawBson, err := ioutil.ReadAll(response.Body)
	if err != nil { return statusCode, nil, err }

	var bsonDoc bson.M
	if err := bson.Unmarshal(rawBson, &bsonDoc); err != nil { return statusCode, nil, err }

	return statusCode, bsonDoc, nil
}

// Use the clone method if you need to override some/all of the configured values.
func (self *HttpRequestClient) Clone() *HttpRequestClient {
	val := *self
	return &val
}

func (self *HttpRequestClient) getClientAndTransport() (*http.Client, *httpclient.Transport) {
	transport := self.getTransport()
	return &http.Client{ Transport: transport }, transport
}

func (self *HttpRequestClient) getTransport() *httpclient.Transport {
	return &httpclient.Transport {

		DisableKeepAlives: self.DisableKeepAlives,
		DisableCompression: self.DisableCompression,

		MaxIdleConnsPerHost: self.MaxIdleConnsPerHost,

		ConnectTimeout: self.ConnectTimeout,
		ResponseHeaderTimeout: self.ResponseHeaderTimeout,
		RequestTimeout: self.RequestTimeout,
		ReadWriteTimeout: self.ReadWriteTimeout,

		TLSClientConfig: &tls.Config{ InsecureSkipVerify: self.SkipSslVerify },
	}
}

func HttpPost(url string, values url.Values, headers map[string]string) (int, []byte, error) { return NewDefaultHttpRequestClient().Post(url, values, headers) }

func HttpPostStr(url string, value string, headers map[string]string) (int, []byte, error) { return NewDefaultHttpRequestClient().PostStr(url, value, headers) }

func HttpPostJson(url string, value interface{}, headers map[string]string) (int, []byte, error) { return NewDefaultHttpRequestClient().PostJson(url, value, headers) }

func HttpPostBson(url string, bsonDoc interface{}, headers map[string]string) (int, []byte, error) { return NewDefaultHttpRequestClient().PostBson(url, bsonDoc, headers) }

func HttpGetBson(url string, headers map[string]string) (int, bson.M, error) { return NewDefaultHttpRequestClient().GetBson(url, headers) }

<|endoftext|>


<commit_before>package main

import (
	"crypto/md5"
	"database/sql"
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net/http"
	"strconv"

	_ "github.com/mattn/go-sqlite3"
)

const listenAddr = "127.0.0.1:9292"

var (
	db         *sql.DB
	getSetStmt *sql.Stmt
)

type Set struct {
	Id             int
	Name           string
	PhotosCount    int
	TakenAt        sql.NullString
	ThumbPhotoId   int
	ThumbPhotoPath string
}

func (s *Set) ThumbURL() string {
	identifier := fmt.Sprintf("%x", md5.Sum([]byte(s.ThumbPhotoPath)))
	return fmt.Sprintf("/thumbs/%s_small.jpg", identifier)
}

func (s *Set) MarshalJSON() ([]byte, error) { // implements Marshaler
	setMap := map[string]interface{}{
		"id":             s.Id,
		"name":           s.Name,
		"photos_count":   s.PhotosCount,
		"thumb_photo_id": s.ThumbPhotoId,
		"thumb_url":      s.ThumbURL(),
	}
	setMap["taken_at"], _ = s.TakenAt.Value()
	return json.Marshal(setMap)
}

func badRequest(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(http.StatusBadRequest)
	fmt.Fprintln(w, http.StatusBadRequest, "Bad Request")
}

func requireParam(param string, w http.ResponseWriter, r *http.Request) error {
	if len(r.URL.Query()[param]) == 0 {
		badRequest(w, r)
		return errors.New(fmt.Sprintf("missing %s parameter", param))
	}
	return nil
}

func getSetById(setId int) (*Set, error) {
	set := Set{}
	row := getSetStmt.QueryRow(setId)
	err := row.Scan(
		&set.Id,
		&set.Name,
		&set.PhotosCount,
		&set.TakenAt,
		&set.ThumbPhotoId,
		&set.ThumbPhotoPath,
	)
	return &set, err
}

func getSetHandler(w http.ResponseWriter, r *http.Request) {
	if requireParam("id", w, r) != nil {
		return
	}

	setId, err := strconv.Atoi(r.URL.Query()["id"][0])
	if err != nil {
		log.Fatalln("Failed to convert id to integer:", err)
	}

	set, err := getSetById(setId)
	if err == sql.ErrNoRows { // set does not exist
		http.NotFound(w, r)
		return
	}

	w.Header().Set("Content-Type", "application/json")
	json.NewEncoder(w).Encode(set)
}

func main() {
	db, err := sql.Open("sqlite3", "thyme.db")
	if err != nil {
		log.Fatalln("Failed to open database:", err)
	}
	defer db.Close()

	getSetStmt, err = db.Prepare(`
	SELECT
	sets.id, name, photos_count, sets.taken_at, thumb_photo_id, photos.path
	FROM sets
	JOIN photos ON sets.thumb_photo_id = photos.id
	WHERE sets.id = ?
	`)
	if err != nil {
		log.Fatalln("Failed to access table:", err)
	}
	defer getSetStmt.Close()

	http.Handle("/", http.FileServer(http.Dir("public"))) // static
	http.HandleFunc("/set", getSetHandler)

	fmt.Printf("Listening on http://%s\n", listenAddr)
	fmt.Println("Press Ctrl-C to exit")

	log.Fatal(http.ListenAndServe(listenAddr, nil))
}
<commit_msg>Implement JSON API response for photo<commit_after>package main

import (
	"crypto/md5"
	"database/sql"
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"math"
	"math/big"
	"net/http"
	"os"
	"path"
	"strconv"

	_ "github.com/mattn/go-sqlite3"
)

const (
	listenAddr     = "127.0.0.1:9292"
	bigThumbSize   = 1000
	smallThumbSize = 200
)

var (
	db           *sql.DB
	getSetStmt   *sql.Stmt
	getPhotoStmt *sql.Stmt
)

type Set struct {
	Id             int
	Name           string
	PhotosCount    int
	TakenAt        sql.NullString
	ThumbPhotoId   int
	ThumbPhotoPath string
}

type Photo struct {
	Aperture      sql.NullFloat64
	Camera        sql.NullString
	ExposureComp  sql.NullInt64
	ExposureTime  sql.NullFloat64
	Flash         sql.NullString
	FocalLength   sql.NullFloat64
	FocalLength35 sql.NullInt64
	Height        int64
	ISO           sql.NullInt64
	Id            int
	Lat           sql.NullFloat64
	Lens          sql.NullString
	Lng           sql.NullFloat64
	NextPhotoId   sql.NullInt64
	Path          string
	PrevPhotoId   sql.NullInt64
	SetId         int
	Size          int
	TakenAt       sql.NullString
	Width         int64
}

func (s *Set) ThumbURL() string {
	identifier := fmt.Sprintf("%x", md5.Sum([]byte(s.ThumbPhotoPath)))
	return fmt.Sprintf("/thumbs/%s_small.jpg", identifier)
}

func (s *Set) MarshalJSON() ([]byte, error) { // implements Marshaler
	setMap := map[string]interface{}{
		"id":             s.Id,
		"name":           s.Name,
		"photos_count":   s.PhotosCount,
		"thumb_photo_id": s.ThumbPhotoId,
		"thumb_url":      s.ThumbURL(),
	}
	setMap["taken_at"], _ = s.TakenAt.Value()
	return json.Marshal(setMap)
}

func (p *Photo) AspectRatio() [2]int64 {
	gcd := new(big.Int).GCD(nil, nil, big.NewInt(p.Width), big.NewInt(p.Height)).Int64()
	return [2]int64{p.Width / gcd, p.Height / gcd}
}

func (p *Photo) BigThumbHeight() int64 {
	if p.Orientation() == "portrait" {
		if p.Height < bigThumbSize {
			return p.Height
		}

		return bigThumbSize
	}

	aspectRatio := p.AspectRatio()
	return int64(math.Floor((float64(aspectRatio[1])/float64(aspectRatio[0]))*float64(p.BigThumbWidth()) + .5))
}

func (p *Photo) BigThumbWidth() int64 {
	if p.Orientation() == "portrait" {
		aspectRatio := p.AspectRatio()
		return int64(math.Floor((float64(aspectRatio[0])/float64(aspectRatio[1]))*float64(p.BigThumbHeight()) + .5))
	}

	if p.Width < bigThumbSize {
		return p.Width
	}

	return bigThumbSize
}

func (p *Photo) Filename() string {
	return path.Base(p.Path)
}

func (p *Photo) Identifier() string {
	return fmt.Sprintf("%x", md5.Sum([]byte(p.Path)))
}

func (p *Photo) Orientation() string {
	if p.Height > p.Width {
		return "portrait"
	}
	return "landscape"
}

func (p *Photo) ThumbFilename(suffix string) string {
	return fmt.Sprintf("%s_%s.jpg", p.Identifier(), suffix)
}

func (p *Photo) ThumbURL(suffix string) string {
	return "/thumbs/" + p.ThumbFilename(suffix)
}

func (p *Photo) MarshalJSON() ([]byte, error) { // implements Marshaler
	photoMap := map[string]interface{}{
		"aspect_ratio":     p.AspectRatio(),
		"big_thumb_height": p.BigThumbHeight(),
		"big_thumb_url":    p.ThumbURL("big"),
		"big_thumb_width":  p.BigThumbWidth(),
		"filename":         p.Filename(),
		"height":           p.Height,
		"id":               p.Id,
		"orientation":      p.Orientation(),
		"path":             p.Path,
		"set_id":           p.SetId,
		"size":             p.Size,
		"small_thumb_url":  p.ThumbURL("small"),
		"width":            p.Width,
	}

	photoMap["aperture"], _ = p.Aperture.Value()
	photoMap["camera"], _ = p.Camera.Value()
	photoMap["exposure_comp"], _ = p.ExposureComp.Value()
	photoMap["exposure_time"], _ = p.ExposureTime.Value()
	photoMap["flash"], _ = p.Flash.Value()
	photoMap["focal_length"], _ = p.FocalLength.Value()
	photoMap["focal_length_35"], _ = p.FocalLength35.Value()
	photoMap["iso"], _ = p.ISO.Value()
	photoMap["lat"], _ = p.Lat.Value()
	photoMap["lens"], _ = p.Lens.Value()
	photoMap["lng"], _ = p.Lng.Value()
	photoMap["next_photo_id"], _ = p.NextPhotoId.Value()
	photoMap["prev_photo_id"], _ = p.PrevPhotoId.Value()
	photoMap["taken_at"], _ = p.TakenAt.Value()

	return json.Marshal(photoMap)
}

func badRequest(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(http.StatusBadRequest)
	fmt.Fprintln(w, http.StatusBadRequest, "Bad Request")
}

func requireParam(param string, w http.ResponseWriter, r *http.Request) error {
	if len(r.URL.Query()[param]) == 0 {
		badRequest(w, r)
		return errors.New(fmt.Sprintf("missing %s parameter", param))
	}
	return nil
}

func getSetById(setId int) (*Set, error) {
	set := Set{}
	row := getSetStmt.QueryRow(setId)
	err := row.Scan(
		&set.Id,
		&set.Name,
		&set.PhotosCount,
		&set.TakenAt,
		&set.ThumbPhotoId,
		&set.ThumbPhotoPath,
	)
	return &set, err
}

func getPhotoById(photoId int) (*Photo, error) {
	photo := Photo{}
	row := getPhotoStmt.QueryRow(photoId)
	err := row.Scan(
		&photo.Aperture,
		&photo.Camera,
		&photo.ExposureComp,
		&photo.ExposureTime,
		&photo.Flash,
		&photo.FocalLength,
		&photo.FocalLength35,
		&photo.Height,
		&photo.Id,
		&photo.ISO,
		&photo.Lat,
		&photo.Lens,
		&photo.Lng,
		&photo.NextPhotoId,
		&photo.Path,
		&photo.PrevPhotoId,
		&photo.SetId,
		&photo.Size,
		&photo.TakenAt,
		&photo.Width,
	)
	return &photo, err
}

func getSetHandler(w http.ResponseWriter, r *http.Request) {
	if requireParam("id", w, r) != nil {
		return
	}

	setId, err := strconv.Atoi(r.URL.Query()["id"][0])
	if err != nil {
		log.Fatalln("Failed to convert id to integer:", err)
	}

	set, err := getSetById(setId)
	if err == sql.ErrNoRows { // set does not exist
		http.NotFound(w, r)
		return
	}

	w.Header().Set("Content-Type", "application/json")
	json.NewEncoder(w).Encode(set)
}

func getPhotoHandler(w http.ResponseWriter, r *http.Request) {
	if requireParam("id", w, r) != nil {
		return
	}

	photoId, err := strconv.Atoi(r.URL.Query()["id"][0])
	if err != nil {
		log.Fatalln("Failed to convert id to integer:", err)
	}

	photo, err := getPhotoById(photoId)
	if err == sql.ErrNoRows { // photo does not exist
		http.NotFound(w, r)
		return
	}

	w.Header().Set("Content-Type", "application/json")
	json.NewEncoder(w).Encode(photo)
}

func main() {
	db, err := sql.Open("sqlite3", "thyme.db")
	if err != nil {
		log.Fatalln("Failed to open database:", err)
	}
	defer db.Close()

	getSetStmt, err = db.Prepare(`
	SELECT
	sets.id, name, photos_count, sets.taken_at, thumb_photo_id, photos.path
	FROM sets
	JOIN photos ON sets.thumb_photo_id = photos.id
	WHERE sets.id = ?
	`)
	if err != nil {
		log.Fatalln("Failed to access table:", err)
	}
	defer getSetStmt.Close()

	getPhotoStmt, err = db.Prepare(`
	SELECT
	aperture, camera, exposure_comp, exposure_time, flash, focal_length,
	focal_length_35, height, id, iso, lat, lens, lng, next_photo_id, path,
	prev_photo_id, set_id, size, taken_at, width
	FROM photos
	WHERE id = ?
	`)
	if err != nil {
		log.Fatalln("Failed to access table:", err)
	}
	defer getPhotoStmt.Close()

	http.Handle("/", http.FileServer(http.Dir("public"))) // static
	http.HandleFunc("/set", getSetHandler)
	http.HandleFunc("/photo", getPhotoHandler)

	fmt.Printf("Listening on http://%s\n", listenAddr)
	fmt.Println("Press Ctrl-C to exit")

	log.Fatal(http.ListenAndServe(listenAddr, nil))
}
<|endoftext|>


<commit_before>package audit

import (
	"encoding/json"
	"net"

	"github.com/af83/edwig/logger"
)

type LogStashEvent map[string]string

type LogStash interface {
	WriteEvent(event LogStashEvent) error
}

type NullLogStash struct {
}

func (logStash *NullLogStash) WriteEvent(event LogStashEvent) error {
	return nil
}

func NewNullLogStash() LogStash {
	return &NullLogStash{}
}

var currentLogStash LogStash = NewNullLogStash()

func CurrentLogStash() LogStash {
	return currentLogStash
}

func SetCurrentLogstash(logStash LogStash) {
	currentLogStash = logStash
}

type TCPLogStash struct {
	address string
}

func NewTCPLogStash(address string) *TCPLogStash {
	return &TCPLogStash{address: address}
}

type FakeLogStash struct {
	events []LogStashEvent
}

func NewFakeLogStash() *FakeLogStash {
	return &FakeLogStash{}
}

func (logStash *FakeLogStash) WriteEvent(event LogStashEvent) error {
	logStash.events = append(logStash.events, event)
	return nil
}

func (logStash *FakeLogStash) Events() []LogStashEvent {
	return logStash.events
}

func (logStash *TCPLogStash) WriteEvent(datas LogStashEvent) error {
	conn, err := net.Dial("tcp", logStash.address)
	if err != nil {
		return err
	}
	defer conn.Close()
	jsonBytes, err := json.Marshal(datas)
	if err != nil {
		return err
	}
	jsonBytes = append(jsonBytes, '\n')

	logger.Log.Debugf("Sending data to logstash: %s", string(jsonBytes))

	_, err = conn.Write(jsonBytes)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>Disable debug message 'Sending data to logstash'<commit_after>package audit

import (
	"encoding/json"
	"net"
)

type LogStashEvent map[string]string

type LogStash interface {
	WriteEvent(event LogStashEvent) error
}

type NullLogStash struct {
}

func (logStash *NullLogStash) WriteEvent(event LogStashEvent) error {
	return nil
}

func NewNullLogStash() LogStash {
	return &NullLogStash{}
}

var currentLogStash LogStash = NewNullLogStash()

func CurrentLogStash() LogStash {
	return currentLogStash
}

func SetCurrentLogstash(logStash LogStash) {
	currentLogStash = logStash
}

type TCPLogStash struct {
	address string
}

func NewTCPLogStash(address string) *TCPLogStash {
	return &TCPLogStash{address: address}
}

type FakeLogStash struct {
	events []LogStashEvent
}

func NewFakeLogStash() *FakeLogStash {
	return &FakeLogStash{}
}

func (logStash *FakeLogStash) WriteEvent(event LogStashEvent) error {
	logStash.events = append(logStash.events, event)
	return nil
}

func (logStash *FakeLogStash) Events() []LogStashEvent {
	return logStash.events
}

func (logStash *TCPLogStash) WriteEvent(datas LogStashEvent) error {
	conn, err := net.Dial("tcp", logStash.address)
	if err != nil {
		return err
	}
	defer conn.Close()
	jsonBytes, err := json.Marshal(datas)
	if err != nil {
		return err
	}
	jsonBytes = append(jsonBytes, '\n')

	// too verbose
	// logger.Log.Debugf("Sending data to logstash: %s", string(jsonBytes))

	_, err = conn.Write(jsonBytes)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/TimothyJones/Go-HTTP-JSON-RPC/httpjsonrpc"
	"log"
	"strings"
)

func word2vec(str string) string {
	args := make([]interface{}, 1)
	args[0] = str

	r, err := httpjsonrpc.Call(config.Word2VecServer, "expand", 0, args)
	if err != nil {
		log.Printf("Unable to get word2vec expansion for '%s'\n", str)
		return ""
	}

	if _, ok := r["result"]; !ok {
		return ""
	}

	x, ok := r["result"].([]string)
	if ok {
		return strings.Join(x, " ")
	} else {
		return ""
	}

	return str
}
<commit_msg>Added word2vec code<commit_after>package main

import (
	"github.com/TimothyJones/Go-HTTP-JSON-RPC/httpjsonrpc"
	"log"
	"strings"
)

func word2vec(word string) string {
	args := make([]interface{}, 1)
	args[0] = word

	r, err := httpjsonrpc.Call(config.Word2VecServer, "expand", 0, args)
	if err != nil {
		log.Printf("verbose error info: %#v", err)
	}

	if _, ok := r["result"]; !ok {
		log.Println("Word2Vec: not getting any result")
		return ""
	}

	x, ok := r["result"].([]interface{})
	if !ok {
		log.Println("Word2Vec: cannot parse the result")
		return ""
	} else {
		strs := make([]string, len(x))
		for i, item := range x {
			strs[i] = item.(string)
		}
		result := strings.Join(strs, " ")
		log.Printf("Word2Vec expanded '%s' to '%s'\n", word, result)
		return result
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package printer

import (
	"bytes"
	"encoding/json"
	"fmt"
	_ "runtime" // import link package
	_ "unsafe"  // required by go:linkname

	"github.com/pingcap/tidb/config"
	"github.com/pingcap/tidb/parser/mysql"
	"github.com/pingcap/tidb/util/israce"
	"github.com/pingcap/tidb/util/logutil"
	"github.com/pingcap/tidb/util/versioninfo"
	"go.uber.org/zap"
)

// PrintTiDBInfo prints the TiDB version information.
func PrintTiDBInfo() {
	logutil.BgLogger().Info("Welcome to TiDB.",
		zap.String("Release Version", mysql.TiDBReleaseVersion),
		zap.String("Edition", versioninfo.TiDBEdition),
		zap.String("Git Commit Hash", versioninfo.TiDBGitHash),
		zap.String("Git Branch", versioninfo.TiDBGitBranch),
		zap.String("UTC Build Time", versioninfo.TiDBBuildTS),
		zap.String("GoVersion", buildVersion),
		zap.Bool("Race Enabled", israce.RaceEnabled),
		zap.Bool("Check Table Before Drop", config.CheckTableBeforeDrop),
		zap.String("TiKV Min Version", versioninfo.TiKVMinVersion))
	configJSON, err := json.Marshal(config.GetGlobalConfig())
	if err != nil {
		panic(err)
	}
	logutil.BgLogger().Info("loaded config", zap.ByteString("config", configJSON))
}

// GetTiDBInfo returns the git hash and build time of this tidb-server binary.
func GetTiDBInfo() string {
	return fmt.Sprintf("Release Version: %s\n"+
		"Edition: %s\n"+
		"Git Commit Hash: %s\n"+
		"Git Branch: %s\n"+
		"UTC Build Time: %s\n"+
		"GoVersion: %s\n"+
		"Race Enabled: %v\n"+
		"TiKV Min Version: %s\n"+
		"Check Table Before Drop: %v",
		mysql.TiDBReleaseVersion,
		versioninfo.TiDBEdition,
		versioninfo.TiDBGitHash,
		versioninfo.TiDBGitBranch,
		versioninfo.TiDBBuildTS,
		buildVersion,
		israce.RaceEnabled,
		versioninfo.TiKVMinVersion,
		config.CheckTableBeforeDrop)
}

// checkValidity checks whether cols and every data have the same length.
func checkValidity(cols []string, datas [][]string) bool {
	colLen := len(cols)
	if len(datas) == 0 || colLen == 0 {
		return false
	}

	for _, data := range datas {
		if colLen != len(data) {
			return false
		}
	}

	return true
}

func getMaxColLen(cols []string, datas [][]string) []int {
	maxColLen := make([]int, len(cols))
	for i, col := range cols {
		maxColLen[i] = len(col)
	}

	for _, data := range datas {
		for i, v := range data {
			if len(v) > maxColLen[i] {
				maxColLen[i] = len(v)
			}
		}
	}

	return maxColLen
}

func getPrintDivLine(maxColLen []int) []byte {
	var value = make([]byte, 0)
	for _, v := range maxColLen {
		value = append(value, '+')
		value = append(value, bytes.Repeat([]byte{'-'}, v+2)...)
	}
	value = append(value, '+')
	value = append(value, '\n')
	return value
}

func getPrintCol(cols []string, maxColLen []int) []byte {
	var value = make([]byte, 0)
	for i, v := range cols {
		value = append(value, '|')
		value = append(value, ' ')
		value = append(value, []byte(v)...)
		value = append(value, bytes.Repeat([]byte{' '}, maxColLen[i]+1-len(v))...)
	}
	value = append(value, '|')
	value = append(value, '\n')
	return value
}

func getPrintRow(data []string, maxColLen []int) []byte {
	var value = make([]byte, 0)
	for i, v := range data {
		value = append(value, '|')
		value = append(value, ' ')
		value = append(value, []byte(v)...)
		value = append(value, bytes.Repeat([]byte{' '}, maxColLen[i]+1-len(v))...)
	}
	value = append(value, '|')
	value = append(value, '\n')
	return value
}

func getPrintRows(datas [][]string, maxColLen []int) []byte {
	var value = make([]byte, 0)
	for _, data := range datas {
		value = append(value, getPrintRow(data, maxColLen)...)
	}
	return value
}

// GetPrintResult gets a result with a formatted string.
func GetPrintResult(cols []string, datas [][]string) (string, bool) {
	if !checkValidity(cols, datas) {
		return "", false
	}

	var value = make([]byte, 0)
	maxColLen := getMaxColLen(cols, datas)

	value = append(value, getPrintDivLine(maxColLen)...)
	value = append(value, getPrintCol(cols, maxColLen)...)
	value = append(value, getPrintDivLine(maxColLen)...)
	value = append(value, getPrintRows(datas, maxColLen)...)
	value = append(value, getPrintDivLine(maxColLen)...)
	return string(value), true
}

//go:linkname buildVersion runtime.buildVersion
var buildVersion string
<commit_msg>util/printer: add store to tidb_version() info (#35055)<commit_after>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package printer

import (
	"bytes"
	"encoding/json"
	"fmt"
	_ "runtime" // import link package
	_ "unsafe"  // required by go:linkname

	"github.com/pingcap/tidb/config"
	"github.com/pingcap/tidb/parser/mysql"
	"github.com/pingcap/tidb/util/israce"
	"github.com/pingcap/tidb/util/logutil"
	"github.com/pingcap/tidb/util/versioninfo"
	"go.uber.org/zap"
)

// PrintTiDBInfo prints the TiDB version information.
func PrintTiDBInfo() {
	logutil.BgLogger().Info("Welcome to TiDB.",
		zap.String("Release Version", mysql.TiDBReleaseVersion),
		zap.String("Edition", versioninfo.TiDBEdition),
		zap.String("Git Commit Hash", versioninfo.TiDBGitHash),
		zap.String("Git Branch", versioninfo.TiDBGitBranch),
		zap.String("UTC Build Time", versioninfo.TiDBBuildTS),
		zap.String("GoVersion", buildVersion),
		zap.Bool("Race Enabled", israce.RaceEnabled),
		zap.Bool("Check Table Before Drop", config.CheckTableBeforeDrop),
		zap.String("TiKV Min Version", versioninfo.TiKVMinVersion))
	configJSON, err := json.Marshal(config.GetGlobalConfig())
	if err != nil {
		panic(err)
	}
	logutil.BgLogger().Info("loaded config", zap.ByteString("config", configJSON))
}

// GetTiDBInfo returns the git hash and build time of this tidb-server binary.
func GetTiDBInfo() string {
	return fmt.Sprintf("Release Version: %s\n"+
		"Edition: %s\n"+
		"Git Commit Hash: %s\n"+
		"Git Branch: %s\n"+
		"UTC Build Time: %s\n"+
		"GoVersion: %s\n"+
		"Race Enabled: %v\n"+
		"TiKV Min Version: %s\n"+
		"Check Table Before Drop: %v\n"+
		"Store: %s",
		mysql.TiDBReleaseVersion,
		versioninfo.TiDBEdition,
		versioninfo.TiDBGitHash,
		versioninfo.TiDBGitBranch,
		versioninfo.TiDBBuildTS,
		buildVersion,
		israce.RaceEnabled,
		versioninfo.TiKVMinVersion,
		config.CheckTableBeforeDrop,
		config.GetGlobalConfig().Store,
	)
}

// checkValidity checks whether cols and every data have the same length.
func checkValidity(cols []string, datas [][]string) bool {
	colLen := len(cols)
	if len(datas) == 0 || colLen == 0 {
		return false
	}

	for _, data := range datas {
		if colLen != len(data) {
			return false
		}
	}

	return true
}

func getMaxColLen(cols []string, datas [][]string) []int {
	maxColLen := make([]int, len(cols))
	for i, col := range cols {
		maxColLen[i] = len(col)
	}

	for _, data := range datas {
		for i, v := range data {
			if len(v) > maxColLen[i] {
				maxColLen[i] = len(v)
			}
		}
	}

	return maxColLen
}

func getPrintDivLine(maxColLen []int) []byte {
	var value = make([]byte, 0)
	for _, v := range maxColLen {
		value = append(value, '+')
		value = append(value, bytes.Repeat([]byte{'-'}, v+2)...)
	}
	value = append(value, '+')
	value = append(value, '\n')
	return value
}

func getPrintCol(cols []string, maxColLen []int) []byte {
	var value = make([]byte, 0)
	for i, v := range cols {
		value = append(value, '|')
		value = append(value, ' ')
		value = append(value, []byte(v)...)
		value = append(value, bytes.Repeat([]byte{' '}, maxColLen[i]+1-len(v))...)
	}
	value = append(value, '|')
	value = append(value, '\n')
	return value
}

func getPrintRow(data []string, maxColLen []int) []byte {
	var value = make([]byte, 0)
	for i, v := range data {
		value = append(value, '|')
		value = append(value, ' ')
		value = append(value, []byte(v)...)
		value = append(value, bytes.Repeat([]byte{' '}, maxColLen[i]+1-len(v))...)
	}
	value = append(value, '|')
	value = append(value, '\n')
	return value
}

func getPrintRows(datas [][]string, maxColLen []int) []byte {
	var value = make([]byte, 0)
	for _, data := range datas {
		value = append(value, getPrintRow(data, maxColLen)...)
	}
	return value
}

// GetPrintResult gets a result with a formatted string.
func GetPrintResult(cols []string, datas [][]string) (string, bool) {
	if !checkValidity(cols, datas) {
		return "", false
	}

	var value = make([]byte, 0)
	maxColLen := getMaxColLen(cols, datas)

	value = append(value, getPrintDivLine(maxColLen)...)
	value = append(value, getPrintCol(cols, maxColLen)...)
	value = append(value, getPrintDivLine(maxColLen)...)
	value = append(value, getPrintRows(datas, maxColLen)...)
	value = append(value, getPrintDivLine(maxColLen)...)
	return string(value), true
}

//go:linkname buildVersion runtime.buildVersion
var buildVersion string
<|endoftext|>


<commit_before>// Copyright (C) 2019-2021, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package cpu

import (
	"math"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

func TestGetSampleWeights(t *testing.T) {
	tests := []struct {
		name      string
		frequency time.Duration
		halflife  time.Duration
		oldWeight float64
	}{
		{
			name:      "simple equal values",
			frequency: 2 * time.Second,
			halflife:  2 * time.Second,
			oldWeight: .5,
		},
		{
			name:      "two periods values",
			frequency: 2 * time.Second,
			halflife:  4 * time.Second,
			oldWeight: math.Sqrt(.5),
		},
	}
	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			assert := assert.New(t)

			newWeight, oldWeight := getSampleWeights(test.frequency, test.halflife)
			assert.Equal(1-test.oldWeight, newWeight)
			assert.Equal(test.oldWeight, oldWeight)
		})
	}
}
<commit_msg>Fix flaky float equality test (#1475)<commit_after>// Copyright (C) 2019-2021, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package cpu

import (
	"math"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

const epsilon = 1e-9

func TestGetSampleWeights(t *testing.T) {
	tests := []struct {
		name      string
		frequency time.Duration
		halflife  time.Duration
		oldWeight float64
	}{
		{
			name:      "simple equal values",
			frequency: 2 * time.Second,
			halflife:  2 * time.Second,
			oldWeight: .5,
		},
		{
			name:      "two periods values",
			frequency: 2 * time.Second,
			halflife:  4 * time.Second,
			oldWeight: math.Sqrt(.5),
		},
	}
	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			assert := assert.New(t)

			newWeight, oldWeight := getSampleWeights(test.frequency, test.halflife)
			assert.InDelta(1-test.oldWeight, newWeight, epsilon)
			assert.InDelta(test.oldWeight, oldWeight, epsilon)
		})
	}
}
<|endoftext|>


<commit_before>package builder

import (
	"errors"
	"github.com/appc/spec/schema"
	"github.com/appc/spec/schema/types"
	"github.com/blablacar/cnt/log"
	"github.com/blablacar/cnt/spec"
	"github.com/blablacar/cnt/utils"
	"os"
	"strconv"
	"text/template"
)

func (p *Pod) Build() {
	log.Get().Info("Building POD : ", p.manifest.Name)

	os.RemoveAll(p.target)
	os.MkdirAll(p.target, 0777)

	apps := p.processAci()

	p.writeSystemdUnit(apps)
}

func (p *Pod) processAci() []schema.RuntimeApp {
	apps := []schema.RuntimeApp{}
	for _, e := range p.manifest.Pod.Apps {

		aciName := p.buildAciIfNeeded(e)
		// TODO: support not FS override by only storing info pod manifest
		//		if aciName == nil {
		//			aciName = &e.Image
		//		}

		name, _ := types.NewACName(e.Name)

		sum, err := utils.Sha512sum(p.path + "/" + e.Name + "/target/image.aci")
		if err != nil {
			log.Get().Panic(err)
		}

		tmp, _ := types.NewHash("sha512-" + sum)

		labels := types.Labels{}
		labels = append(labels, types.Label{Name: "version", Value: aciName.Version()})
		identifier, _ := types.NewACIdentifier(aciName.Name())
		ttmp := schema.RuntimeImage{Name: identifier, ID: *tmp, Labels: labels}

		if e.App.User == "" {
			e.App.User = "0"
		}
		if e.App.Group == "" {
			e.App.Group = "0"
		}

		apps = append(apps, schema.RuntimeApp{
			Name:  *name,
			Image: ttmp,
			App: &types.App{
				Exec:             e.App.Exec,
				User:             e.App.User,
				Group:            e.App.Group,
				WorkingDirectory: e.App.WorkingDirectory,
				Environment:      e.App.Environment,
				MountPoints:      e.App.MountPoints,
				Ports:            e.App.Ports,
				Isolators:        e.App.Isolators,
			},
			Mounts:      e.Mounts,
			Annotations: e.Annotations})

	}

	return apps
}

func (p *Pod) buildAciIfNeeded(e spec.RuntimeApp) *spec.ACFullname {
	if dir, err := os.Stat(p.path + "/" + e.Name); err == nil && dir.IsDir() {
		aci, err := NewAciWithManifest(p.path+"/"+e.Name, p.args, p.toAciManifest(e))
		if err != nil {
			log.Get().Panic(err)
		}
		aci.PodName = &p.manifest.Name
		aci.Build()
		return &aci.manifest.NameAndVersion
	}
	return nil
}

//func (p *Pod) writePodManifest(apps []schema.RuntimeApp) {
//	m := p.manifest.Pod
//	ver, _ := types.NewSemVer("0.6.1")
//	manifest := schema.PodManifest{
//		ACKind:      "PodManifest",
//		ACVersion:   *ver,
//		Apps:        apps,
//		Volumes:     m.Volumes,
//		Isolators:   m.Isolators,
//		Annotations: m.Annotations,
//		Ports:       m.Ports}
//	utils.WritePodManifest(&manifest, p.target+POD_TARGET_MANIFEST)
//}

const SYSTEMD_TEMPLATE = `[Unit]
Description={{ .Shortname }} %i

[Service]
ExecStartPre=/opt/bin/rkt gc --grace-period=0s --expire-prepared=0s
ExecStart=/opt/bin/rkt --insecure-skip-verify run \
{{range $i, $e := .Commands}}  {{$e}} \
{{end}}{{range $i, $e := .Acilist}}{{if $i}} \
{{end}}  {{$e}}{{end}}

[Install]
WantedBy=multi-user.target
`

type SystemdUnit struct {
	Shortname string
	Commands  []string
	Acilist   []string
}

func (p *Pod) writeSystemdUnit(apps []schema.RuntimeApp) {
	volumes := []string{}
	envs := []string{}
	acilist := []string{}

	for _, env := range p.manifest.Envs {
		envs = append(envs, "--set-env="+env.Name+"='"+env.Value+"'")
	}
	for _, app := range apps {
		for _, mount := range app.App.MountPoints {
			mountPoint, err := p.getVolumeMountValue(mount.Name)
			if err != nil {
				log.Get().Panic(err)
			}
			volumes = append(volumes,
				"--volume="+mount.Name.String()+
					",kind="+mountPoint.Kind+
					",source="+mountPoint.Source+
					",readOnly="+strconv.FormatBool(*mountPoint.ReadOnly))
		}
		version, _ := app.Image.Labels.Get("version")
		acilist = append(acilist, app.Image.Name.String()+":"+version)
	}

	commands := []string{}
	if p.manifest.PrivateNet != "" {
		commands = append(commands, "--private-net="+p.manifest.PrivateNet)
	}

	commands = append(commands, volumes...)
	commands = append(commands, envs...)

	info := SystemdUnit{Shortname: p.manifest.Name.ShortName(), Commands: commands, Acilist: acilist}
	tmpl, _ := template.New("test").Parse(SYSTEMD_TEMPLATE)
	w, _ := os.Create(p.target + "/" + p.manifest.Name.ShortName() + "@.service")
	tmpl.Execute(w, info)
}

func (p *Pod) getVolumeMountValue(mountName types.ACName) (*types.Volume, error) {
	for _, volume := range p.manifest.Pod.Volumes {
		if volume.Name.Equals(mountName) {
			return &volume, nil
		}
	}
	return nil, errors.New("Volume mount point not set :" + mountName.String())
}
<commit_msg>fail when pod's aci directory not found<commit_after>package builder

import (
	"errors"
	"github.com/appc/spec/schema"
	"github.com/appc/spec/schema/types"
	"github.com/blablacar/cnt/log"
	"github.com/blablacar/cnt/spec"
	"github.com/blablacar/cnt/utils"
	"os"
	"strconv"
	"text/template"
)

func (p *Pod) Build() {
	log.Get().Info("Building POD : ", p.manifest.Name)

	os.RemoveAll(p.target)
	os.MkdirAll(p.target, 0777)

	apps := p.processAci()

	p.writeSystemdUnit(apps)
}

func (p *Pod) processAci() []schema.RuntimeApp {
	apps := []schema.RuntimeApp{}
	for _, e := range p.manifest.Pod.Apps {

		aciName := p.buildAciIfNeeded(e)

		name, _ := types.NewACName(e.Name)

		sum, err := utils.Sha512sum(p.path + "/" + e.Name + "/target/image.aci")
		if err != nil {
			log.Get().Panic(err)
		}

		tmp, _ := types.NewHash("sha512-" + sum)

		labels := types.Labels{}
		labels = append(labels, types.Label{Name: "version", Value: aciName.Version()})
		identifier, _ := types.NewACIdentifier(aciName.Name())
		ttmp := schema.RuntimeImage{Name: identifier, ID: *tmp, Labels: labels}

		if e.App.User == "" {
			e.App.User = "0"
		}
		if e.App.Group == "" {
			e.App.Group = "0"
		}

		apps = append(apps, schema.RuntimeApp{
			Name:  *name,
			Image: ttmp,
			App: &types.App{
				Exec:             e.App.Exec,
				User:             e.App.User,
				Group:            e.App.Group,
				WorkingDirectory: e.App.WorkingDirectory,
				Environment:      e.App.Environment,
				MountPoints:      e.App.MountPoints,
				Ports:            e.App.Ports,
				Isolators:        e.App.Isolators,
			},
			Mounts:      e.Mounts,
			Annotations: e.Annotations})
	}

	return apps
}

func (p *Pod) buildAciIfNeeded(e spec.RuntimeApp) *spec.ACFullname {
	if dir, err := os.Stat(p.path + "/" + e.Name); err == nil && dir.IsDir() {
		aci, err := NewAciWithManifest(p.path+"/"+e.Name, p.args, p.toAciManifest(e))
		if err != nil {
			log.Get().Panic(err)
		}
		aci.PodName = &p.manifest.Name
		aci.Build()
		return &aci.manifest.NameAndVersion
	}
	log.Get().Panic("Cannot found Pod's aci directory :" + p.path + "/" + e.Name)
	return nil
}

//func (p *Pod) writePodManifest(apps []schema.RuntimeApp) {
//	m := p.manifest.Pod
//	ver, _ := types.NewSemVer("0.6.1")
//	manifest := schema.PodManifest{
//		ACKind:      "PodManifest",
//		ACVersion:   *ver,
//		Apps:        apps,
//		Volumes:     m.Volumes,
//		Isolators:   m.Isolators,
//		Annotations: m.Annotations,
//		Ports:       m.Ports}
//	utils.WritePodManifest(&manifest, p.target+POD_TARGET_MANIFEST)
//}

const SYSTEMD_TEMPLATE = `[Unit]
Description={{ .Shortname }} %i

[Service]
ExecStartPre=/opt/bin/rkt gc --grace-period=0s --expire-prepared=0s
ExecStart=/opt/bin/rkt --insecure-skip-verify run \
{{range $i, $e := .Commands}}  {{$e}} \
{{end}}{{range $i, $e := .Acilist}}{{if $i}} \
{{end}}  {{$e}}{{end}}

[Install]
WantedBy=multi-user.target
`

type SystemdUnit struct {
	Shortname string
	Commands  []string
	Acilist   []string
}

func (p *Pod) writeSystemdUnit(apps []schema.RuntimeApp) {
	volumes := []string{}
	envs := []string{}
	acilist := []string{}

	for _, env := range p.manifest.Envs {
		envs = append(envs, "--set-env="+env.Name+"='"+env.Value+"'")
	}
	for _, app := range apps {
		for _, mount := range app.App.MountPoints {
			mountPoint, err := p.getVolumeMountValue(mount.Name)
			if err != nil {
				log.Get().Panic(err)
			}
			volumes = append(volumes,
				"--volume="+mount.Name.String()+
					",kind="+mountPoint.Kind+
					",source="+mountPoint.Source+
					",readOnly="+strconv.FormatBool(*mountPoint.ReadOnly))
		}
		version, _ := app.Image.Labels.Get("version")
		acilist = append(acilist, app.Image.Name.String()+":"+version)
	}

	commands := []string{}
	if p.manifest.PrivateNet != "" {
		commands = append(commands, "--private-net="+p.manifest.PrivateNet)
	}

	commands = append(commands, volumes...)
	commands = append(commands, envs...)

	info := SystemdUnit{Shortname: p.manifest.Name.ShortName(), Commands: commands, Acilist: acilist}
	tmpl, _ := template.New("test").Parse(SYSTEMD_TEMPLATE)
	w, _ := os.Create(p.target + "/" + p.manifest.Name.ShortName() + "@.service")
	tmpl.Execute(w, info)
}

func (p *Pod) getVolumeMountValue(mountName types.ACName) (*types.Volume, error) {
	for _, volume := range p.manifest.Pod.Volumes {
		if volume.Name.Equals(mountName) {
			return &volume, nil
		}
	}
	return nil, errors.New("Volume mount point not set :" + mountName.String())
}
<|endoftext|>


<commit_before>package main

func main() {
}
<commit_msg>Added no-op in implementation<commit_after>package main

import "fmt"

func main() {
	fmt.Println("[]")
}
<|endoftext|>