<commit_before>package digest

import (
	"crypto"
	"hash"
	"io"
)

// Algorithm identifies and implementation of a digester by an identifier.
// Note the that this defines both the hash algorithm used and the string
// encoding.
type Algorithm string

// supported digest types
const (
	SHA256 Algorithm = "sha256" // sha256 with hex encoding
	SHA384 Algorithm = "sha384" // sha384 with hex encoding
	SHA512 Algorithm = "sha512" // sha512 with hex encoding

	// Canonical is the primary digest algorithm used with the distribution
	// project. Other digests may be used but this one is the primary storage
	// digest.
	Canonical = SHA256
)

var (
	// TODO(stevvooe): Follow the pattern of the standard crypto package for
	// registration of digests. Effectively, we are a registerable set and
	// common symbol access.

	// algorithms maps values to hash.Hash implementations. Other algorithms
	// may be available but they cannot be calculated by the digest package.
	algorithms = map[Algorithm]crypto.Hash{
		SHA256: crypto.SHA256,
		SHA384: crypto.SHA384,
		SHA512: crypto.SHA512,
	}
)

// Available returns true if the digest type is available for use. If this
// returns false, New and Hash will return nil.
func (a Algorithm) Available() bool {
	h, ok := algorithms[a]
	if !ok {
		return false
	}

	// check availability of the hash, as well
	return h.Available()
}

func (a Algorithm) String() string {
	return string(a)
}

// Size returns number of bytes returned by the hash.
func (a Algorithm) Size() int {
	h, ok := algorithms[a]
	if !ok {
		return 0
	}
	return h.Size()
}

// Set implemented to allow use of Algorithm as a command line flag.
func (a *Algorithm) Set(value string) error {
	if value == "" {
		*a = Canonical
	} else {
		// just do a type conversion, support is queried with Available.
		*a = Algorithm(value)
	}

	return nil
}

// New returns a new digester for the specified algorithm. If the algorithm
// does not have a digester implementation, nil will be returned. This can be
// checked by calling Available before calling New.
func (a Algorithm) New() Digester {
	return &digester{
		alg:  a,
		hash: a.Hash(),
	}
}

// Hash returns a new hash as used by the algorithm. If not available, nil is
// returned. Make sure to check Available before calling.
func (a Algorithm) Hash() hash.Hash {
	if !a.Available() {
		return nil
	}

	return algorithms[a].New()
}

// FromReader returns the digest of the reader using the algorithm.
func (a Algorithm) FromReader(rd io.Reader) (Digest, error) {
	digester := a.New()

	if _, err := io.Copy(digester.Hash(), rd); err != nil {
		return "", err
	}

	return digester.Digest(), nil
}

// TODO(stevvooe): Allow resolution of verifiers using the digest type and
// this registration system.

// Digester calculates the digest of written data. Writes should go directly
// to the return value of Hash, while calling Digest will return the current
// value of the digest.
type Digester interface {
	Hash() hash.Hash // provides direct access to underlying hash instance.
	Digest() Digest
}

// digester provides a simple digester definition that embeds a hasher.
type digester struct {
	alg  Algorithm
	hash hash.Hash
}

func (d *digester) Hash() hash.Hash {
	return d.hash
}

func (d *digester) Digest() Digest {
	return NewDigest(d.alg, d.hash)
}
<commit_msg>digest: panic on unavailable hash algorithm<commit_after>package digest

import (
	"crypto"
	"fmt"
	"hash"
	"io"
)

// Algorithm identifies and implementation of a digester by an identifier.
// Note the that this defines both the hash algorithm used and the string
// encoding.
type Algorithm string

// supported digest types
const (
	SHA256 Algorithm = "sha256" // sha256 with hex encoding
	SHA384 Algorithm = "sha384" // sha384 with hex encoding
	SHA512 Algorithm = "sha512" // sha512 with hex encoding

	// Canonical is the primary digest algorithm used with the distribution
	// project. Other digests may be used but this one is the primary storage
	// digest.
	Canonical = SHA256
)

var (
	// TODO(stevvooe): Follow the pattern of the standard crypto package for
	// registration of digests. Effectively, we are a registerable set and
	// common symbol access.

	// algorithms maps values to hash.Hash implementations. Other algorithms
	// may be available but they cannot be calculated by the digest package.
	algorithms = map[Algorithm]crypto.Hash{
		SHA256: crypto.SHA256,
		SHA384: crypto.SHA384,
		SHA512: crypto.SHA512,
	}
)

// Available returns true if the digest type is available for use. If this
// returns false, New and Hash will return nil.
func (a Algorithm) Available() bool {
	h, ok := algorithms[a]
	if !ok {
		return false
	}

	// check availability of the hash, as well
	return h.Available()
}

func (a Algorithm) String() string {
	return string(a)
}

// Size returns number of bytes returned by the hash.
func (a Algorithm) Size() int {
	h, ok := algorithms[a]
	if !ok {
		return 0
	}
	return h.Size()
}

// Set implemented to allow use of Algorithm as a command line flag.
func (a *Algorithm) Set(value string) error {
	if value == "" {
		*a = Canonical
	} else {
		// just do a type conversion, support is queried with Available.
		*a = Algorithm(value)
	}

	return nil
}

// New returns a new digester for the specified algorithm. If the algorithm
// does not have a digester implementation, nil will be returned. This can be
// checked by calling Available before calling New.
func (a Algorithm) New() Digester {
	return &digester{
		alg:  a,
		hash: a.Hash(),
	}
}

// Hash returns a new hash as used by the algorithm. If not available, the
// method will panic. Check Algorithm.Available() before calling.
func (a Algorithm) Hash() hash.Hash {
	if !a.Available() {
		// NOTE(stevvooe): A missing hash is usually a programming error that
		// must be resolved at compile time. We don't import in the digest
		// package to allow users to choose their hash implementation (such as
		// when using stevvooe/resumable or a hardware accelerated package).
		//
		// Applications that may want to resolve the hash at runtime should
		// call Algorithm.Available before call Algorithm.Hash().
		panic(fmt.Sprintf("%v not available (make sure it is imported)", a))
	}

	return algorithms[a].New()
}

// FromReader returns the digest of the reader using the algorithm.
func (a Algorithm) FromReader(rd io.Reader) (Digest, error) {
	digester := a.New()

	if _, err := io.Copy(digester.Hash(), rd); err != nil {
		return "", err
	}

	return digester.Digest(), nil
}

// TODO(stevvooe): Allow resolution of verifiers using the digest type and
// this registration system.

// Digester calculates the digest of written data. Writes should go directly
// to the return value of Hash, while calling Digest will return the current
// value of the digest.
type Digester interface {
	Hash() hash.Hash // provides direct access to underlying hash instance.
	Digest() Digest
}

// digester provides a simple digester definition that embeds a hasher.
type digester struct {
	alg  Algorithm
	hash hash.Hash
}

func (d *digester) Hash() hash.Hash {
	return d.hash
}

func (d *digester) Digest() Digest {
	return NewDigest(d.alg, d.hash)
}
<|endoftext|>


<commit_before>package main

import (
	".."
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/BurntSushi/toml"
	"github.com/bitly/go-nsq"
	log "github.com/Sirupsen/logrus"
	"net/http"
	"os"
	"os/signal"
	"sync"
	"syscall"
	"time"
)

var (
	ip = flag.String("ip", "4.3.2.1", "Node ip address")
	configFile = flag.String("config", "sidekick.conf", "Configuration file")
	version = flag.Bool("version", false, "Print current version")
	verbose = flag.Bool("verbose", false, "Log in verbose mode")
	config = nsq.NewConfig()
	properties *sidekick.Config
	daemon      *sidekick.Daemon
	producer    *nsq.Producer
	syslog        *sidekick.Syslog
	reloadChan = make(chan sidekick.ReloadEvent)
	lastSyslogReload = time.Now()
)

func main() {
	log.SetFormatter(&log.TextFormatter{})
	flag.Parse()

	if *version {
		println(sidekick.VERSION)
		os.Exit(0)
	}

	if *verbose {
		log.SetLevel(log.DebugLevel)
		log.WithField("loglevel", "debug").Info("Change loglevel")
	}

	loadProperties()

	daemon = sidekick.NewDaemon(properties)
	syslog = sidekick.NewSyslog(properties)
	syslog.Init()
	log.WithFields(log.Fields{
		"status": properties.Status,
		"id":    properties.NodeId(),
	}).Info("Starting sidekick")

	producer, _ = nsq.NewProducer(properties.ProducerAddr, config)

	createTopicsAndChannels()
	time.Sleep(1 * time.Second)

	var wg sync.WaitGroup
	// Start http API
	restApi := sidekick.NewRestApi(properties)
	go func() {
		defer wg.Done()
		wg.Add(1)
		err := restApi.Start()
		if err != nil {
			log.Fatal("Cannot start api")
		}
	}()

	// Start slave consumer
	go func() {
		defer wg.Done()
		wg.Add(1)
		consumer, _ := nsq.NewConsumer(fmt.Sprintf("commit_requested_%s", properties.ClusterId), properties.NodeId(), config)
		consumer.AddHandler(nsq.HandlerFunc(onCommitRequested))
		err := consumer.ConnectToNSQLookupd(properties.LookupdAddr)
		if err != nil {
			log.Panic("Could not connect")
		}
	}()

	// Start master consumer
	go func() {
		defer wg.Done()
		wg.Add(1)
		consumer, _ := nsq.NewConsumer(fmt.Sprintf("commit_slave_completed_%s", properties.ClusterId), properties.NodeId(), config)
		consumer.AddHandler(nsq.HandlerFunc(onCommitSlaveRequested))
		err := consumer.ConnectToNSQLookupd(properties.LookupdAddr)
		if err != nil {
			log.Panic("Could not connect")
		}
	}()

	// Start complete consumer
	go func() {
		defer wg.Done()
		wg.Add(1)
		consumer, _ := nsq.NewConsumer(fmt.Sprintf("commit_completed_%s", properties.ClusterId), properties.NodeId(), config)
		consumer.AddHandler(nsq.HandlerFunc(onCommitCompleted))
		err := consumer.ConnectToNSQLookupd(properties.LookupdAddr)
		if err != nil {
			log.Panic("Could not connect")
		}
	}()

	// Start reload pipeline
	stopChan := make(chan interface{}, 1)
	go func() {
		defer wg.Done()
		wg.Add(1)
		for {
			select {
			case reload := <-reloadChan:
				reload.Execute()
			case <-stopChan:
				return
			}
		}
	}()

	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT)
	select {
	case signal := <-sigChan:
		log.Printf("Got signal: %v\n", signal)
	}
	stopChan <- true
	restApi.Stop()

	log.Printf("Waiting on server to stop\n")
	wg.Wait()
}

func createTopicsAndChannels() {
	// Create required topics
	topics := []string{"commit_slave_completed", "commit_completed", "commit_failed"}
	channels := []string{"slave", "master"}
	topicChan := make(chan string, len(topics))

	// fill the channel
	for i := range topics {
		topicChan <- topics[i]
	}

	for len(topicChan) > 0 {
		// create the topic
		topic := <-topicChan
		log.WithField("topic", topic).Info("Creating topic")
		url := fmt.Sprintf("%s/topic/create?topic=%s_%s", properties.ProducerRestAddr, topic, properties.ClusterId)
		resp, err := http.PostForm(url, nil)
		if err != nil || resp.StatusCode != 200 {
			// retry to create this topic
			topicChan <- topic
			continue
		}

		// create the channels of the topics
		for _, channel := range channels {
			log.WithField("channel", channel).Info("Creating channel")
			url := fmt.Sprintf("%s/channel/create?topic=%s_%s&channel=%s-%s", properties.ProducerRestAddr, topic, properties.ClusterId, properties.ClusterId, channel)
			resp, err := http.PostForm(url, nil)
			if err != nil || resp.StatusCode != 200 {
				// retry all for this topic if channel creation failed
				topicChan <- topic
				continue
			}
		}

		log.WithField("topic", topic).Info("Topic created")
	}
}

// loadProperties load properties file
func loadProperties() {
	properties = sidekick.DefaultConfig()
	if _, err := toml.DecodeFile(*configFile, properties); err != nil {
		log.Fatal(err)
		os.Exit(1)
	}
	properties.IpAddr = *ip
	len := len(properties.HapHome)
	if properties.HapHome[len - 1] == '/' {
		properties.HapHome = properties.HapHome[:len - 1]
	}
}

func filteredHandler(event string, message *nsq.Message, target string, f sidekick.HandlerFunc) error {
	defer message.Finish()
	match, err := daemon.Is(target)
	if err != nil {
		return err
	}

	if match {
		log.WithField("event", event).Debug("Handle event")
		data, err := bodyToData(message.Body)
		if err != nil {
			log.WithError(err).Error("Unable to read data")
			return err
		}

		switch event {
		case "commit_requested":
			reloadChan <- sidekick.ReloadEvent{F: f, Message: data}
		case "commit_slave_completed":
			reloadChan <- sidekick.ReloadEvent{F: f, Message: data}
		case "commit_completed":
			logAndForget(data)
		}

	} else {
		log.WithField("event", event).Debug("Ignore event")
	}

	return nil
}

func onCommitRequested(message *nsq.Message) error {
	return filteredHandler("commit_requested", message, "slave", reloadSlave)
}
func onCommitSlaveRequested(message *nsq.Message) error {
	return filteredHandler("commit_slave_completed", message, "master", reloadMaster)
}
func onCommitCompleted(message *nsq.Message) error {
	return filteredHandler("commit_completed", message, "slave", logAndForget)
}

// logAndForget is a generic function to just log event
func logAndForget(data *sidekick.EventMessage) error {
	log.WithFields(data.Context().Fields()).Debug("Commit completed")
	return nil
}

func reloadSlave(data *sidekick.EventMessage) error {
	context := data.Context()
	hap := sidekick.NewHaproxy("slave", properties,data.HapVersion, context)

	status, err := hap.ApplyConfiguration(data)
	if err == nil {
		if status != sidekick.UNCHANGED {
			elapsed := time.Now().Sub(lastSyslogReload)
			if elapsed.Seconds() > 10 {
				syslog.Restart()
				lastSyslogReload = time.Now()
			} else {
				log.WithField("elapsed time in second", elapsed.Seconds()).Debug("skip syslog reload")
			}
		}
		publishMessage("commit_slave_completed_", data, context)
	} else {
		log.WithFields(context.Fields()).WithError(err).Error("Commit failed")
		publishContextMessage("commit_failed_", context)
	}
	return nil
}

func reloadMaster(data *sidekick.EventMessage) error {
	context := data.Context()

	hap := sidekick.NewHaproxy("master", properties, data.HapVersion, context)
	status, err := hap.ApplyConfiguration(data)
	if err == nil {
		if status != sidekick.UNCHANGED {
			syslog.Restart()
		}
		publishContextMessage("commit_completed_", context)
	} else {
		log.WithFields(context.Fields()).WithError(err).Error("Commit failed")
		publishContextMessage("commit_failed_", context)
	}
	return nil
}

// Unmarshal json to EventMessage
func bodyToData(jsonStream []byte) (*sidekick.EventMessage, error) {
	dec := json.NewDecoder(bytes.NewReader(jsonStream))
	var message sidekick.EventMessage
	err := dec.Decode(&message)
	return &message, err
}

func publishContextMessage(topic_prefix string, context sidekick.Context) error {
	return publishMessage(topic_prefix, context, context.UpdateTimestamp())
}
func publishMessage(topic_prefix string, data interface{}, context sidekick.Context) error {
	jsonMsg, _ := json.Marshal(data)
	topic := topic_prefix + properties.ClusterId
	log.WithFields(context.Fields()).WithField("topic", topic).WithField("payload", string(jsonMsg)).Debug("Publish")
	return producer.Publish(topic, []byte(jsonMsg))
}
<commit_msg>Update dep<commit_after>package main

import (
	"github.com/voyages-sncf-technologies/strowgr/sidekick"
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/BurntSushi/toml"
	"github.com/bitly/go-nsq"
	log "github.com/Sirupsen/logrus"
	"net/http"
	"os"
	"os/signal"
	"sync"
	"syscall"
	"time"
)

var (
	ip = flag.String("ip", "4.3.2.1", "Node ip address")
	configFile = flag.String("config", "sidekick.conf", "Configuration file")
	version = flag.Bool("version", false, "Print current version")
	verbose = flag.Bool("verbose", false, "Log in verbose mode")
	config = nsq.NewConfig()
	properties *sidekick.Config
	daemon      *sidekick.Daemon
	producer    *nsq.Producer
	syslog        *sidekick.Syslog
	reloadChan = make(chan sidekick.ReloadEvent)
	lastSyslogReload = time.Now()
)

func main() {
	log.SetFormatter(&log.TextFormatter{})
	flag.Parse()

	if *version {
		println(sidekick.VERSION)
		os.Exit(0)
	}

	if *verbose {
		log.SetLevel(log.DebugLevel)
		log.WithField("loglevel", "debug").Info("Change loglevel")
	}

	loadProperties()

	daemon = sidekick.NewDaemon(properties)
	syslog = sidekick.NewSyslog(properties)
	syslog.Init()
	log.WithFields(log.Fields{
		"status": properties.Status,
		"id":    properties.NodeId(),
	}).Info("Starting sidekick")

	producer, _ = nsq.NewProducer(properties.ProducerAddr, config)

	createTopicsAndChannels()
	time.Sleep(1 * time.Second)

	var wg sync.WaitGroup
	// Start http API
	restApi := sidekick.NewRestApi(properties)
	go func() {
		defer wg.Done()
		wg.Add(1)
		err := restApi.Start()
		if err != nil {
			log.Fatal("Cannot start api")
		}
	}()

	// Start slave consumer
	go func() {
		defer wg.Done()
		wg.Add(1)
		consumer, _ := nsq.NewConsumer(fmt.Sprintf("commit_requested_%s", properties.ClusterId), properties.NodeId(), config)
		consumer.AddHandler(nsq.HandlerFunc(onCommitRequested))
		err := consumer.ConnectToNSQLookupd(properties.LookupdAddr)
		if err != nil {
			log.Panic("Could not connect")
		}
	}()

	// Start master consumer
	go func() {
		defer wg.Done()
		wg.Add(1)
		consumer, _ := nsq.NewConsumer(fmt.Sprintf("commit_slave_completed_%s", properties.ClusterId), properties.NodeId(), config)
		consumer.AddHandler(nsq.HandlerFunc(onCommitSlaveRequested))
		err := consumer.ConnectToNSQLookupd(properties.LookupdAddr)
		if err != nil {
			log.Panic("Could not connect")
		}
	}()

	// Start complete consumer
	go func() {
		defer wg.Done()
		wg.Add(1)
		consumer, _ := nsq.NewConsumer(fmt.Sprintf("commit_completed_%s", properties.ClusterId), properties.NodeId(), config)
		consumer.AddHandler(nsq.HandlerFunc(onCommitCompleted))
		err := consumer.ConnectToNSQLookupd(properties.LookupdAddr)
		if err != nil {
			log.Panic("Could not connect")
		}
	}()

	// Start reload pipeline
	stopChan := make(chan interface{}, 1)
	go func() {
		defer wg.Done()
		wg.Add(1)
		for {
			select {
			case reload := <-reloadChan:
				reload.Execute()
			case <-stopChan:
				return
			}
		}
	}()

	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT)
	select {
	case signal := <-sigChan:
		log.Printf("Got signal: %v\n", signal)
	}
	stopChan <- true
	restApi.Stop()

	log.Printf("Waiting on server to stop\n")
	wg.Wait()
}

func createTopicsAndChannels() {
	// Create required topics
	topics := []string{"commit_slave_completed", "commit_completed", "commit_failed"}
	channels := []string{"slave", "master"}
	topicChan := make(chan string, len(topics))

	// fill the channel
	for i := range topics {
		topicChan <- topics[i]
	}

	for len(topicChan) > 0 {
		// create the topic
		topic := <-topicChan
		log.WithField("topic", topic).Info("Creating topic")
		url := fmt.Sprintf("%s/topic/create?topic=%s_%s", properties.ProducerRestAddr, topic, properties.ClusterId)
		resp, err := http.PostForm(url, nil)
		if err != nil || resp.StatusCode != 200 {
			// retry to create this topic
			topicChan <- topic
			continue
		}

		// create the channels of the topics
		for _, channel := range channels {
			log.WithField("channel", channel).Info("Creating channel")
			url := fmt.Sprintf("%s/channel/create?topic=%s_%s&channel=%s-%s", properties.ProducerRestAddr, topic, properties.ClusterId, properties.ClusterId, channel)
			resp, err := http.PostForm(url, nil)
			if err != nil || resp.StatusCode != 200 {
				// retry all for this topic if channel creation failed
				topicChan <- topic
				continue
			}
		}

		log.WithField("topic", topic).Info("Topic created")
	}
}

// loadProperties load properties file
func loadProperties() {
	properties = sidekick.DefaultConfig()
	if _, err := toml.DecodeFile(*configFile, properties); err != nil {
		log.Fatal(err)
		os.Exit(1)
	}
	properties.IpAddr = *ip
	len := len(properties.HapHome)
	if properties.HapHome[len - 1] == '/' {
		properties.HapHome = properties.HapHome[:len - 1]
	}
}

func filteredHandler(event string, message *nsq.Message, target string, f sidekick.HandlerFunc) error {
	defer message.Finish()
	match, err := daemon.Is(target)
	if err != nil {
		return err
	}

	if match {
		log.WithField("event", event).Debug("Handle event")
		data, err := bodyToData(message.Body)
		if err != nil {
			log.WithError(err).Error("Unable to read data")
			return err
		}

		switch event {
		case "commit_requested":
			reloadChan <- sidekick.ReloadEvent{F: f, Message: data}
		case "commit_slave_completed":
			reloadChan <- sidekick.ReloadEvent{F: f, Message: data}
		case "commit_completed":
			logAndForget(data)
		}

	} else {
		log.WithField("event", event).Debug("Ignore event")
	}

	return nil
}

func onCommitRequested(message *nsq.Message) error {
	return filteredHandler("commit_requested", message, "slave", reloadSlave)
}
func onCommitSlaveRequested(message *nsq.Message) error {
	return filteredHandler("commit_slave_completed", message, "master", reloadMaster)
}
func onCommitCompleted(message *nsq.Message) error {
	return filteredHandler("commit_completed", message, "slave", logAndForget)
}

// logAndForget is a generic function to just log event
func logAndForget(data *sidekick.EventMessage) error {
	log.WithFields(data.Context().Fields()).Debug("Commit completed")
	return nil
}

func reloadSlave(data *sidekick.EventMessage) error {
	context := data.Context()
	hap := sidekick.NewHaproxy("slave", properties,data.HapVersion, context)

	status, err := hap.ApplyConfiguration(data)
	if err == nil {
		if status != sidekick.UNCHANGED {
			elapsed := time.Now().Sub(lastSyslogReload)
			if elapsed.Seconds() > 10 {
				syslog.Restart()
				lastSyslogReload = time.Now()
			} else {
				log.WithField("elapsed time in second", elapsed.Seconds()).Debug("skip syslog reload")
			}
		}
		publishMessage("commit_slave_completed_", data, context)
	} else {
		log.WithFields(context.Fields()).WithError(err).Error("Commit failed")
		publishContextMessage("commit_failed_", context)
	}
	return nil
}

func reloadMaster(data *sidekick.EventMessage) error {
	context := data.Context()

	hap := sidekick.NewHaproxy("master", properties, data.HapVersion, context)
	status, err := hap.ApplyConfiguration(data)
	if err == nil {
		if status != sidekick.UNCHANGED {
			syslog.Restart()
		}
		publishContextMessage("commit_completed_", context)
	} else {
		log.WithFields(context.Fields()).WithError(err).Error("Commit failed")
		publishContextMessage("commit_failed_", context)
	}
	return nil
}

// Unmarshal json to EventMessage
func bodyToData(jsonStream []byte) (*sidekick.EventMessage, error) {
	dec := json.NewDecoder(bytes.NewReader(jsonStream))
	var message sidekick.EventMessage
	err := dec.Decode(&message)
	return &message, err
}

func publishContextMessage(topic_prefix string, context sidekick.Context) error {
	return publishMessage(topic_prefix, context, context.UpdateTimestamp())
}
func publishMessage(topic_prefix string, data interface{}, context sidekick.Context) error {
	jsonMsg, _ := json.Marshal(data)
	topic := topic_prefix + properties.ClusterId
	log.WithFields(context.Fields()).WithField("topic", topic).WithField("payload", string(jsonMsg)).Debug("Publish")
	return producer.Publish(topic, []byte(jsonMsg))
}
<|endoftext|>


<commit_before>package write

import (
	"image"
	"net/http"
	"net/http/httptest"
	"testing"
)

func TestImageWithTemplate(t *testing.T) {

	recorder := httptest.NewRecorder()
	var img image.Image = image.NewRGBA(image.Rect(0, 0, 0, 0))

	ImageWithTemplate(recorder, &img)
	if recorder.Code != http.StatusOK {
		t.Errorf("returned %v. Expected %v.", recorder.Code, http.StatusOK)
	}
}

func TestImageJPEG(t *testing.T) {
}

func TestImageSVG(t *testing.T) {

}
<commit_msg>test WriteJPEG<commit_after>package write

import (
	"image"
	"net/http"
	"net/http/httptest"
	"testing"
)

func TestImageWithTemplate(t *testing.T) {

	recorder := httptest.NewRecorder()
	var img image.Image = image.NewRGBA(image.Rect(0, 0, 0, 0))

	ImageWithTemplate(recorder, &img)
	if recorder.Code != http.StatusOK {
		t.Errorf("returned %v. Expected %v.", recorder.Code, http.StatusOK)
	}
}

func TestImageJPEG(t *testing.T) {
	recorder := httptest.NewRecorder()
	var img image.Image = image.NewRGBA(image.Rect(0, 0, 0, 0))

	ImageJPEG(recorder, &img)
	if recorder.Code != http.StatusOK {
		t.Errorf("returned %v. Expected %v.", recorder.Code, http.StatusOK)
	}
}
<|endoftext|>


<commit_before>package controllers

import (
	"encoding/json"
	"fmt"
	"github.com/gojp/kana"
	"github.com/gojp/nihongo/app/models"
	"github.com/gojp/nihongo/app/routes"
	"github.com/jgraham909/revmgo"
	"github.com/mattbaird/elastigo/api"
	"github.com/mattbaird/elastigo/core"
	"github.com/robfig/revel"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
	"log"
	"regexp"
	"strings"
)

type App struct {
	*revel.Controller
	revmgo.MongoController
}

type Gloss struct {
	English string
	Tags    []string
	Related []string
	Common  bool
}

type Highlight struct {
	Furigana string
	Japanese string
	Romaji   string
	English  []string
}

type Word struct {
	Romaji    string
	Common    bool
	Dialects  []string
	Fields    []string
	Glosses   []Gloss
	English   []string
	Furigana  string
	Japanese  string
	MainEntry string
	Tags      []string
	Pos       []string
}

// wrap a string in strong tags
func makeStrong(query string) string {
	return "" + query + ""
}

// Wrap the query in  tags so that we can highlight it in the results
func (w *Word) highlightQuery(query string) {
	re := regexp.MustCompile(`\b` + query + `\b`)
	// convert the query to hiragana and katakana. if it's already in
	// hiragana or katakana, it will just be the same.
	kana := kana.NewKana()
	h := kana.Romaji_to_hiragana(query)
	k := kana.Romaji_to_katakana(query)
	// make regular expressions that match the hiragana and katakana
	hiraganaRe := regexp.MustCompile(h)
	katakanaRe := regexp.MustCompile(k)
	// wrap the query in strong tags
	queryHighlighted := makeStrong(query)
	katakanaHighlighted := makeStrong(k)
	hiraganaHighlighted := makeStrong(h)
	// if the query is originally in Japanese, highlight it
	w.Japanese = re.ReplaceAllString(w.Japanese, queryHighlighted)
	// highlight the katakana or hiragana that has been converted from romaji
	w.Japanese = hiraganaRe.ReplaceAllString(w.Japanese, hiraganaHighlighted)
	w.Japanese = katakanaRe.ReplaceAllString(w.Japanese, katakanaHighlighted)
	// highlight the furigana too, same as above
	w.Furigana = hiraganaRe.ReplaceAllString(w.Furigana, hiraganaHighlighted)
	w.Furigana = katakanaRe.ReplaceAllString(w.Furigana, katakanaHighlighted)
	// highlight the query inside the list of English definitions
	for i, e := range w.English {
		e = re.ReplaceAllString(e, queryHighlighted)
		w.English[i] = e
	}
}

func search(query string) []Word {
	fmt.Println("Searching for... ", query)
	api.Domain = "localhost"
	searchJson := fmt.Sprintf(`{"query": {"multi_match": {"query": "%s", "fields": ["japanese", "furigana", "romaji", "english"]}}, "highlight": {"fields": {"furigana": {}, "japanese": {}, "romaji": {}, "english": {}}}}`, query)
	out, err := core.SearchRequest(true, "edict", "entry", searchJson, "", 0)
	if err != nil {
		log.Println(err)
	}

	hits := [][]byte{}
	for _, hit := range out.Hits.Hits {
		hits = append(hits, hit.Source)
	}

	wordList := []Word{}
	for _, hit := range hits {
		w := Word{}
		err := json.Unmarshal(hit, &w)
		if err != nil {
			log.Println(err)
		}
		w.MainEntry = w.Japanese
		w.highlightQuery(query)
		wordList = append(wordList, w)
	}
	return wordList
}

func (a App) Search(query string) revel.Result {
	if len(query) == 0 {
		return a.Redirect(routes.App.Index())
	}
	wordList := search(query)
	return a.Render(wordList)
}

func (c App) Details(query string) revel.Result {
	if len(query) == 0 {
		return c.Redirect(routes.App.Index())
	}
	if strings.Contains(query, " ") {
		return c.Redirect(routes.App.Details(strings.Replace(query, " ", "-", -1)))
	}
	query = strings.Replace(query, "-", " ", -1)
	wordList := search(query)
	pageTitle := query + " in Japanese"

	// log this call in mongo
	collection := c.MongoSession.DB("greenbook").C("hits")
	_, err := collection.Upsert(bson.M{"term": query}, bson.M{"$inc": bson.M{"count": 1}})
	if err != nil {
		// mongo failed to log, but who cares
	}

	index := mgo.Index{
		Key:        []string{"count"},
		Unique:     false,
		DropDups:   false,
		Background: true,
		Sparse:     true,
	}
	collection.EnsureIndex(index)

	return c.Render(wordList, query, pageTitle)
}

func (c App) SearchGet() revel.Result {
	if query, ok := c.Params.Values["q"]; ok && len(query) > 0 {
		return c.Redirect(routes.App.Details(query[0]))
	}
	return c.Redirect(routes.App.Index())
}

func (c App) About() revel.Result {
	return c.Render()
}

func (c App) Index() revel.Result {

	// get the popular searches
	collection := c.MongoSession.DB("greenbook").C("hits")
	q := collection.Find(nil).Sort("-count")

	termList := []models.SearchTerm{}
	iter := q.Limit(10).Iter()
	iter.All(&termList)

	return c.Render(termList)
}
<commit_msg>add function to convert query to hiragana and katakana<commit_after>package controllers

import (
	"encoding/json"
	"fmt"
	"github.com/gojp/kana"
	"github.com/gojp/nihongo/app/models"
	"github.com/gojp/nihongo/app/routes"
	"github.com/jgraham909/revmgo"
	"github.com/mattbaird/elastigo/api"
	"github.com/mattbaird/elastigo/core"
	"github.com/robfig/revel"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
	"log"
	"regexp"
	"strings"
)

type App struct {
	*revel.Controller
	revmgo.MongoController
}

type Gloss struct {
	English string
	Tags    []string
	Related []string
	Common  bool
}

type Highlight struct {
	Furigana string
	Japanese string
	Romaji   string
	English  []string
}

type Word struct {
	Romaji    string
	Common    bool
	Dialects  []string
	Fields    []string
	Glosses   []Gloss
	English   []string
	Furigana  string
	Japanese  string
	MainEntry string
	Tags      []string
	Pos       []string
}

// wrap a string in strong tags
func makeStrong(query string) string {
	return "" + query + ""
}

// convert the query to hiragana and katakana. if it's already in
// hiragana or katakana, it will just be the same.
func convertQueryToKana(query string) (hiragana, katakana string) {
	kana := kana.NewKana()
	h := kana.Romaji_to_hiragana(query)
	k := kana.Romaji_to_katakana(query)
	return h, k
}

// Wrap the query in  tags so that we can highlight it in the results
func (w *Word) highlightQuery(query string) {
	// make regular expression that matches the original query
	re := regexp.MustCompile(`\b` + query + `\b`)
	// convert original query to kana
	h, k := convertQueryToKana(query)
	// make regular expressions that match the hiragana and katakana
	hiraganaRe := regexp.MustCompile(h)
	katakanaRe := regexp.MustCompile(k)
	// wrap the query in strong tags
	queryHighlighted := makeStrong(query)
	katakanaHighlighted := makeStrong(k)
	hiraganaHighlighted := makeStrong(h)
	// if the query is originally in Japanese, highlight it
	w.Japanese = re.ReplaceAllString(w.Japanese, queryHighlighted)
	// highlight the katakana or hiragana that has been converted from romaji
	w.Japanese = hiraganaRe.ReplaceAllString(w.Japanese, hiraganaHighlighted)
	w.Japanese = katakanaRe.ReplaceAllString(w.Japanese, katakanaHighlighted)
	// highlight the furigana too, same as above
	w.Furigana = hiraganaRe.ReplaceAllString(w.Furigana, hiraganaHighlighted)
	w.Furigana = katakanaRe.ReplaceAllString(w.Furigana, katakanaHighlighted)
	// highlight the query inside the list of English definitions
	for i, e := range w.English {
		e = re.ReplaceAllString(e, queryHighlighted)
		w.English[i] = e
	}
}

func search(query string) []Word {
	fmt.Println("Searching for... ", query)
	api.Domain = "localhost"
	searchJson := fmt.Sprintf(`{"query": {"multi_match": {"query": "%s", "fields": ["japanese", "furigana", "romaji", "english"]}}, "highlight": {"fields": {"furigana": {}, "japanese": {}, "romaji": {}, "english": {}}}}`, query)
	out, err := core.SearchRequest(true, "edict", "entry", searchJson, "", 0)
	if err != nil {
		log.Println(err)
	}

	hits := [][]byte{}
	for _, hit := range out.Hits.Hits {
		hits = append(hits, hit.Source)
	}

	wordList := []Word{}
	for _, hit := range hits {
		w := Word{}
		err := json.Unmarshal(hit, &w)
		if err != nil {
			log.Println(err)
		}
		w.MainEntry = w.Japanese
		w.highlightQuery(query)
		wordList = append(wordList, w)
	}
	return wordList
}

func (a App) Search(query string) revel.Result {
	if len(query) == 0 {
		return a.Redirect(routes.App.Index())
	}
	wordList := search(query)
	return a.Render(wordList)
}

func (c App) Details(query string) revel.Result {
	if len(query) == 0 {
		return c.Redirect(routes.App.Index())
	}
	if strings.Contains(query, " ") {
		return c.Redirect(routes.App.Details(strings.Replace(query, " ", "-", -1)))
	}
	query = strings.Replace(query, "-", " ", -1)
	wordList := search(query)
	pageTitle := query + " in Japanese"

	// log this call in mongo
	collection := c.MongoSession.DB("greenbook").C("hits")
	_, err := collection.Upsert(bson.M{"term": query}, bson.M{"$inc": bson.M{"count": 1}})
	if err != nil {
		// mongo failed to log, but who cares
	}

	index := mgo.Index{
		Key:        []string{"count"},
		Unique:     false,
		DropDups:   false,
		Background: true,
		Sparse:     true,
	}
	collection.EnsureIndex(index)

	return c.Render(wordList, query, pageTitle)
}

func (c App) SearchGet() revel.Result {
	if query, ok := c.Params.Values["q"]; ok && len(query) > 0 {
		return c.Redirect(routes.App.Details(query[0]))
	}
	return c.Redirect(routes.App.Index())
}

func (c App) About() revel.Result {
	return c.Render()
}

func (c App) Index() revel.Result {

	// get the popular searches
	collection := c.MongoSession.DB("greenbook").C("hits")
	q := collection.Find(nil).Sort("-count")

	termList := []models.SearchTerm{}
	iter := q.Limit(10).Iter()
	iter.All(&termList)

	return c.Render(termList)
}
<|endoftext|>


<commit_before>package controllers

import (
	"github.com/robfig/revel"
	"github.com/PacketFire/goqdb/app/models"
	"github.com/PacketFire/goqdb/app/routes"
	"time"
)

type App struct {
	Base
}

var appDefaults = models.Args{
	To: models.UnixEpoch(time.Now().Unix()),
	Sort: models.SORT_DATE,
	Page: 1,
	Size: models.SIZE_DEFAULT,
}

func (c *App) Index (arg models.Args) revel.Result {

	var entries []models.Quote

	c.RenderArgs["hasNext"] = false
	c.RenderArgs["hasPrev"] = false

	if arg.Id > 0 {
		entries = c.getEntry(arg.Id)
	} else {

		argMerged := arg.Merge(appDefaults)

		count := c.getTotal(argMerged)

		if count == 0 {
			return c.Render(arg)
		}

		entries = c.getEntries(argMerged)

		offset  := argMerged.Size * (argMerged.Page - 1)
		hasNext := int64(offset + argMerged.Size) < count
		hasPrev := offset > 0

		c.RenderArgs["hasNext"] = hasNext
		c.RenderArgs["hasPrev"] = hasPrev
	}

	if author, ok := c.Session["author"]; ok {
		c.Flash.Data["quote.Author"] = author
	}

	return c.Render(arg, entries)
}

func (c *App) Quote (id int) revel.Result {
	e := c.getEntry(id)
	return Utf8Result(e[0].Quote)
}

func (c *App) AdvSearch () revel.Result {
	return c.Render()
}

func (c *App) Random () revel.Result {
	arg := models.Args{Sort: models.SORT_RANDOM, Size:1}
	id := c.getEntries(arg)[0].QuoteId
	return c.Redirect(routes.App.Index(models.Args{Id: id}))
}

func (c *App) Insert (quote models.Quote, arg models.Args) revel.Result {
	c.Session["author"] = quote.Author
	c.insertQuote(&quote)
	return c.Redirect(routes.App.Index(arg))
}

func (c *App) Vote (voteId int, voteType string, arg models.Args) revel.Result {
	c.vote(voteId, voteType)
	return c.Redirect(routes.App.Index(arg))
}
<commit_msg>fixed 'index out of range' error<commit_after>package controllers

import (
	"github.com/robfig/revel"
	"github.com/PacketFire/goqdb/app/models"
	"github.com/PacketFire/goqdb/app/routes"
	"time"
)

type App struct {
	Base
}

var appDefaults = models.Args{
	To: models.UnixEpoch(time.Now().Unix()),
	Sort: models.SORT_DATE,
	Page: 1,
	Size: models.SIZE_DEFAULT,
}

func (c *App) Index (arg models.Args) revel.Result {

	var entries []models.Quote

	c.RenderArgs["hasNext"] = false
	c.RenderArgs["hasPrev"] = false

	if arg.Id > 0 {
		entries = c.getEntry(arg.Id)
	} else {

		argMerged := arg.Merge(appDefaults)

		count := c.getTotal(argMerged)

		if count == 0 {
			return c.Render(arg)
		}

		entries = c.getEntries(argMerged)

		offset  := argMerged.Size * (argMerged.Page - 1)
		hasNext := int64(offset + argMerged.Size) < count
		hasPrev := offset > 0

		c.RenderArgs["hasNext"] = hasNext
		c.RenderArgs["hasPrev"] = hasPrev
	}

	if author, ok := c.Session["author"]; ok {
		c.Flash.Data["quote.Author"] = author
	}

	return c.Render(arg, entries)
}

func (c *App) Quote (id int) revel.Result {
	e := c.getEntry(id)
	return Utf8Result(e[0].Quote)
}

func (c *App) AdvSearch () revel.Result {
	return c.Render()
}

func (c *App) Random () revel.Result {
	arg := models.Args{Sort: models.SORT_RANDOM, Size:1}

	var id int
	if e := c.getEntries(arg); len(e) != 0 {
		id = e[0].QuoteId
	}
	return c.Redirect(routes.App.Index(models.Args{Id: id}))
}

func (c *App) Insert (quote models.Quote, arg models.Args) revel.Result {
	c.Session["author"] = quote.Author
	c.insertQuote(&quote)
	return c.Redirect(routes.App.Index(arg))
}

func (c *App) Vote (voteId int, voteType string, arg models.Args) revel.Result {
	c.vote(voteId, voteType)
	return c.Redirect(routes.App.Index(arg))
}
<|endoftext|>


<commit_before>package bsw

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/http"
	"net/url"
)

type bingMessage struct {
	D bingResults
}

type bingResults struct {
	Results []bingResult
}

type bingResult struct {
	__metadata  bingMetadata
	ID          string
	Title       string
	Description string
	DisplayUrl  string
	Url         string
}

type bingMetadata struct {
	Uri  string
	Type string
}

const host = "https://api.datamarket.azure.com"

// Attempts an authenticated search request to two different Bing API paths. If and when a
// search is successfull, that path will be returned. If no path is valid this function
// returns an error.
func FindBingSearchPath(key string) (string, error) {
	var paths = []string{"/Data.ashx/Bing/Search/v1/Web", "/Data.ashx/Bing/SearchWeb/v1/Web"}
	var query = "?Query=%27I<3BSW%27"
	for _, path := range paths {
		var fullUrl = host + path + query
		client := &http.Client{}
		req, err := http.NewRequest("GET", fullUrl, nil)
		if err != nil {
			return "", err
		}
		req.SetBasicAuth(key, key)
		resp, err := client.Do(req)
		if err != nil {
			return "", err
		}
		if resp.StatusCode == 200 {
			return path, nil
		}
	}
	return "", errors.New("Invalid Bing API key")
}

// Uses the Bing search API and 'ip' search operator to find alternate hostnames for
// a single IP.
func BingAPI(ip, key, path string) (Results, error) {
	results := Results{}
	var query = "?Query=%27ip:" + ip + "%27&$top=50&Adult=%27off%27&$format=json"
	var fullUrl = host + path + query
	client := &http.Client{}
	req, err := http.NewRequest("GET", fullUrl, nil)
	if err != nil {
		return results, err
	}
	req.SetBasicAuth(key, key)
	resp, err := client.Do(req)
	if err != nil {
		return results, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return results, err
	}
	var m bingMessage
	err = json.Unmarshal(body, &m)
	if err != nil {
		return results, err
	}
	for _, res := range m.D.Results {
		u, err := url.Parse(res.Url)
		if err == nil {
			results = append(results, Result{Source: "Bing API", IP: ip, Hostname: u.Host})
		}
	}
	return results, nil
}
<commit_msg>Rename host variable to prevent possible collision later on<commit_after>package bsw

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/http"
	"net/url"
)

type bingMessage struct {
	D bingResults
}

type bingResults struct {
	Results []bingResult
}

type bingResult struct {
	__metadata  bingMetadata
	ID          string
	Title       string
	Description string
	DisplayUrl  string
	Url         string
}

type bingMetadata struct {
	Uri  string
	Type string
}

const azureUrl = "https://api.datamarket.azure.com"

// Attempts an authenticated search request to two different Bing API paths. If and when a
// search is successfull, that path will be returned. If no path is valid this function
// returns an error.
func FindBingSearchPath(key string) (string, error) {
	var paths = []string{"/Data.ashx/Bing/Search/v1/Web", "/Data.ashx/Bing/SearchWeb/v1/Web"}
	var query = "?Query=%27I<3BSW%27"
	for _, path := range paths {
		var fullUrl = azureUrl + path + query
		client := &http.Client{}
		req, err := http.NewRequest("GET", fullUrl, nil)
		if err != nil {
			return "", err
		}
		req.SetBasicAuth(key, key)
		resp, err := client.Do(req)
		if err != nil {
			return "", err
		}
		if resp.StatusCode == 200 {
			return path, nil
		}
	}
	return "", errors.New("Invalid Bing API key")
}

// Uses the Bing search API and 'ip' search operator to find alternate hostnames for
// a single IP.
func BingAPI(ip, key, path string) (Results, error) {
	results := Results{}
	var query = "?Query=%27ip:" + ip + "%27&$top=50&Adult=%27off%27&$format=json"
	var fullUrl = azureUrl + path + query
	client := &http.Client{}
	req, err := http.NewRequest("GET", fullUrl, nil)
	if err != nil {
		return results, err
	}
	req.SetBasicAuth(key, key)
	resp, err := client.Do(req)
	if err != nil {
		return results, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return results, err
	}
	var m bingMessage
	err = json.Unmarshal(body, &m)
	if err != nil {
		return results, err
	}
	for _, res := range m.D.Results {
		u, err := url.Parse(res.Url)
		if err == nil {
			results = append(results, Result{Source: "Bing API", IP: ip, Hostname: u.Host})
		}
	}
	return results, nil
}
<|endoftext|>


<commit_before>package Kinsokujiko

import (
	"bufio"
	"errors"
	"fmt"
	"os"
	"strings"
)

// Item is dictionary Item
type Item struct {
	Surf  string
	Slice string
	Kana  string
	Pos   string
}

// Dictionary is Slice any Item
type Dictionary []Item

var dicPath = "_dic/dic.txt"

func Update(dic Dictionary) (string, error) {
	_, err := _add_item(dic)
	if err != nil {
		fmt.Println(err)
	}
	for index, item := range dic {
		if !_check_item(item) {
			return "Error", errors.New(fmt.Sprintf("%d 行目に問題があるようです", index))
		}
	}
	return _add_item(dic)
}

func _check_item(it Item) bool {
	var res = false
	if strings.Count(it.Surf, "") > 1 {
		res = true
	}
	if strings.Count(it.Slice, "") > 1 {
		res = true
	}
	if strings.Count(it.Kana, "") > 1 {
		res = true
	}
	if strings.Count(it.Pos, "") > 1 {
		res = true
	}
	return res
}

func _add_item(dic Dictionary) (string, error) {
	dir, _ := os.Getwd()
	dicPath := fmt.Sprintf("%s/%s", dir, dicPath)
	file, err := os.OpenFile(dicPath, os.O_RDWR|os.O_CREATE, 0755)
	defer file.Close()
	read := bufio.NewReader(file)
	res := fmt.Sprintf("ファイルの詳細：%+v", read)
	return res, err
}

func Show() (Dictionary, error) {
	return Dictionary{}, nil
}

func _get_dic() {}
<commit_msg>use ioutil for debug.<commit_after>package Kinsokujiko

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
)

// Item is dictionary Item
type Item struct {
	Surf  string
	Slice string
	Kana  string
	Pos   string
}

// Dictionary is Slice any Item
type Dictionary []Item

var dicPath = "_dic/dic.txt"

func Update(dic Dictionary) (string, error) {
	_, err := _add_item(dic)
	if err != nil {
		fmt.Println(err)
	}
	for index, item := range dic {
		if !_check_item(item) {
			return "Error", errors.New(fmt.Sprintf("%d 行目に問題があるようです", index))
		}
	}
	return _add_item(dic)
}

func _check_item(it Item) bool {
	var res = false
	if strings.Count(it.Surf, "") > 1 {
		res = true
	}
	if strings.Count(it.Slice, "") > 1 {
		res = true
	}
	if strings.Count(it.Kana, "") > 1 {
		res = true
	}
	if strings.Count(it.Pos, "") > 1 {
		res = true
	}
	return res
}

func _add_item(dic Dictionary) (string, error) {
	dir, _ := os.Getwd()
	dicPath := fmt.Sprintf("%s/%s", dir, dicPath)
	file, err := os.OpenFile(dicPath, os.O_RDWR|os.O_CREATE, 0755)
	defer file.Close()
	read, _ := ioutil.ReadAll(file)
	res := fmt.Sprintf("ファイルの詳細：%+v", read)
	return res, err
}

func Show() (Dictionary, error) {
	return Dictionary{}, nil
}

func _get_dic() {}
<|endoftext|>


<commit_before>package dingo

import (
	"testing"

	"github.com/mission-liao/dingo/backend"
	"github.com/mission-liao/dingo/broker"
	"github.com/mission-liao/dingo/meta"
	"github.com/stretchr/testify/suite"
)

type DingoTestSuite struct {
	suite.Suite

	_apps []App
}

func TestDingoSuite(t *testing.T) {
	suite.Run(t, &DingoTestSuite{})
}

func (me *DingoTestSuite) SetupSuite() {
	// local + local
	{
		cfg := Default()
		cfg.Broker().Local_().Bypass(false)
		cfg.Backend().Local_().Bypass(false)
		app, err := NewApp(*cfg)
		me.Nil(err)

		// broker
		{
			v, err := broker.New("local", cfg.Broker())
			me.Nil(err)
			_, used, err := app.Use(v, InstT.DEFAULT)
			me.Nil(err)
			me.Equal(InstT.PRODUCER|InstT.CONSUMER, used)
		}

		// backend
		{
			v, err := backend.New("local", cfg.Backend())
			me.Nil(err)
			_, used, err := app.Use(v, InstT.DEFAULT)
			me.Nil(err)
			me.Equal(InstT.REPORTER|InstT.STORE, used)
		}

		me._apps = append(me._apps, app)
	}

	// TODO: amqp + amqp
	// TODO: amqp + redis
}

func (me *DingoTestSuite) TearDownSuite() {
	for _, v := range me._apps {
		me.Nil(v.Close())
	}
}

//
// test cases
//

func (me *DingoTestSuite) TestBasic() {
	for _, v := range me._apps {
		// register a set of workers
		called := 0
		_, remain, err := v.Register(&StrMatcher{"test_basic"}, func(n int) int {
			called = n
			return n + 1
		}, 1)
		me.Equal(0, remain)
		me.Nil(err)

		// call that function
		reports, err := v.Call("test_basic", nil, 5)
		me.Nil(err)
		me.NotNil(reports)
		if reports == nil {
			continue
		}

		// await for reports
		status := []int{
			meta.Status.Sent,
			meta.Status.Progress,
			meta.Status.Done,
		}
		for {
			done := false
			select {
			case v, ok := <-reports:
				me.True(ok)
				if !ok {
					break
				}

				// make sure the order of status is right
				me.True(len(status) > 0)
				if len(status) > 0 {
					me.Equal(v.GetStatus(), status[0])
					status = status[1:]
				}

				if v.Done() {
					me.Equal(5, called)
					me.Len(v.GetReturn(), 1)
					if len(v.GetReturn()) > 0 {
						ret, ok := v.GetReturn()[0].(int)
						me.True(ok)
						me.Equal(called+1, ret)
					}
					done = true
				}
			}

			if done {
				break
			}
		}
	}
}
<commit_msg>config is reworked.<commit_after>package dingo

import (
	"testing"

	"github.com/mission-liao/dingo/backend"
	"github.com/mission-liao/dingo/broker"
	"github.com/mission-liao/dingo/meta"
	"github.com/stretchr/testify/suite"
)

type DingoTestSuite struct {
	suite.Suite

	_apps []App
}

func TestDingoSuite(t *testing.T) {
	suite.Run(t, &DingoTestSuite{})
}

func (me *DingoTestSuite) SetupSuite() {
	// local + local
	{
		cfg := Default()
		cfg.Broker().Local.Bypass(false)
		cfg.Backend().Local.Bypass(false)
		app, err := NewApp(*cfg)
		me.Nil(err)

		// broker
		{
			v, err := broker.New("local", cfg.Broker())
			me.Nil(err)
			_, used, err := app.Use(v, InstT.DEFAULT)
			me.Nil(err)
			me.Equal(InstT.PRODUCER|InstT.CONSUMER, used)
		}

		// backend
		{
			v, err := backend.New("local", cfg.Backend())
			me.Nil(err)
			_, used, err := app.Use(v, InstT.DEFAULT)
			me.Nil(err)
			me.Equal(InstT.REPORTER|InstT.STORE, used)
		}

		me._apps = append(me._apps, app)
	}

	// TODO: amqp + amqp
	// TODO: amqp + redis
}

func (me *DingoTestSuite) TearDownSuite() {
	for _, v := range me._apps {
		me.Nil(v.Close())
	}
}

//
// test cases
//

func (me *DingoTestSuite) TestBasic() {
	for _, v := range me._apps {
		// register a set of workers
		called := 0
		_, remain, err := v.Register(&StrMatcher{"test_basic"}, func(n int) int {
			called = n
			return n + 1
		}, 1)
		me.Equal(0, remain)
		me.Nil(err)

		// call that function
		reports, err := v.Call("test_basic", nil, 5)
		me.Nil(err)
		me.NotNil(reports)
		if reports == nil {
			continue
		}

		// await for reports
		status := []int{
			meta.Status.Sent,
			meta.Status.Progress,
			meta.Status.Done,
		}
		for {
			done := false
			select {
			case v, ok := <-reports:
				me.True(ok)
				if !ok {
					break
				}

				// make sure the order of status is right
				me.True(len(status) > 0)
				if len(status) > 0 {
					me.Equal(v.GetStatus(), status[0])
					status = status[1:]
				}

				if v.Done() {
					me.Equal(5, called)
					me.Len(v.GetReturn(), 1)
					if len(v.GetReturn()) > 0 {
						ret, ok := v.GetReturn()[0].(int)
						me.True(ok)
						me.Equal(called+1, ret)
					}
					done = true
				}
			}

			if done {
				break
			}
		}
	}
}
<|endoftext|>


<commit_before>/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied.  See the License for the
specific language governing permissions and limitations
under the License.
*/

package obcpbft

import (
	"fmt"
	"strconv"
	"strings"
	"os"
	"path/filepath"

	"github.com/hyperledger/fabric/consensus"
	pb "github.com/hyperledger/fabric/protos"

	"github.com/spf13/viper"
)

const configPrefix = "CORE_PBFT"

var pluginInstance consensus.Consenter // singleton service
var config *viper.Viper

func init() {
	config = loadConfig()
}

// GetPlugin returns the handle to the Consenter singleton
func GetPlugin(c consensus.Stack) consensus.Consenter {
	if pluginInstance == nil {
		pluginInstance = New(c)
	}
	return pluginInstance
}

// New creates a new Obc* instance that provides the Consenter interface.
// Internally, it uses an opaque pbft-core instance.
func New(stack consensus.Stack) consensus.Consenter {
	handle, _, _ := stack.GetNetworkHandles()
	id, _ := getValidatorID(handle)

	switch strings.ToLower(config.GetString("general.mode")) {
	case "classic":
		return newObcClassic(id, config, stack)
	case "batch":
		return newObcBatch(id, config, stack)
	case "sieve":
		return newObcSieve(id, config, stack)
	default:
		panic(fmt.Errorf("Invalid PBFT mode: %s", config.GetString("general.mode")))
	}
}

func loadConfig() (config *viper.Viper) {
	config = viper.New()

	// for environment variables
	config.SetEnvPrefix(configPrefix)
	config.AutomaticEnv()
	replacer := strings.NewReplacer(".", "_")
	config.SetEnvKeyReplacer(replacer)

	config.SetConfigName("config")
	config.AddConfigPath("./")
	config.AddConfigPath("../consensus/obcpbft/")
	config.AddConfigPath("../../consensus/obcpbft")
	gopath := os.Getenv("GOPATH")
	obcpbftpath := filepath.Join(gopath, "src/github.com/hyperledger/fabric/consensus/obcpbft")
	config.AddConfigPath(obcpbftpath)    // Path to look for the config file in
	err := config.ReadInConfig()
	if err != nil {
		panic(fmt.Errorf("Error reading %s plugin config: %s", configPrefix, err))
	}
	return
}

// Returns the uint64 ID corresponding to a peer handle
func getValidatorID(handle *pb.PeerID) (id uint64, err error) {
	// as requested here: https://github.com/hyperledger/fabric/issues/462#issuecomment-170785410
	if startsWith := strings.HasPrefix(handle.Name, "vp"); startsWith {
		id, err = strconv.ParseUint(handle.Name[2:], 10, 64)
		if err != nil {
			return id, fmt.Errorf("Error extracting ID from \"%s\" handle: %v", handle.Name, err)
		}
		return
	}

	err = fmt.Errorf(`For MVP, set the VP's peer.id to vpX,
		where X is a unique integer between 0 and N-1
		(N being the maximum number of VPs in the network`)
	return
}

// Returns the peer handle that corresponds to a validator ID (uint64 assigned to it for PBFT)
func getValidatorHandle(id uint64) (handle *pb.PeerID, err error) {
	// as requested here: https://github.com/hyperledger/fabric/issues/462#issuecomment-170785410
	name := "vp" + strconv.FormatUint(id, 10)
	return &pb.PeerID{Name: name}, nil
}

// Returns the peer handles corresponding to a list of replica ids
func getValidatorHandles(ids []uint64) (handles []*pb.PeerID) {
	handles = make([]*pb.PeerID, len(ids))
	for i, id := range ids {
		handles[i], _ = getValidatorHandle(id)
	}
	return
}
<commit_msg>revert file mode change<commit_after>/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied.  See the License for the
specific language governing permissions and limitations
under the License.
*/

package obcpbft

import (
	"fmt"
	"strconv"
	"strings"
	"os"
	"path/filepath"

	"github.com/hyperledger/fabric/consensus"
	pb "github.com/hyperledger/fabric/protos"

	"github.com/spf13/viper"
)

const configPrefix = "CORE_PBFT"

var pluginInstance consensus.Consenter // singleton service
var config *viper.Viper

func init() {
	config = loadConfig()
}

// GetPlugin returns the handle to the Consenter singleton
func GetPlugin(c consensus.Stack) consensus.Consenter {
	if pluginInstance == nil {
		pluginInstance = New(c)
	}
	return pluginInstance
}

// New creates a new Obc* instance that provides the Consenter interface.
// Internally, it uses an opaque pbft-core instance.
func New(stack consensus.Stack) consensus.Consenter {
	handle, _, _ := stack.GetNetworkHandles()
	id, _ := getValidatorID(handle)

	switch strings.ToLower(config.GetString("general.mode")) {
	case "classic":
		return newObcClassic(id, config, stack)
	case "batch":
		return newObcBatch(id, config, stack)
	case "sieve":
		return newObcSieve(id, config, stack)
	default:
		panic(fmt.Errorf("Invalid PBFT mode: %s", config.GetString("general.mode")))
	}
}

func loadConfig() (config *viper.Viper) {
	config = viper.New()

	// for environment variables
	config.SetEnvPrefix(configPrefix)
	config.AutomaticEnv()
	replacer := strings.NewReplacer(".", "_")
	config.SetEnvKeyReplacer(replacer)

	config.SetConfigName("config")
	config.AddConfigPath("./")
	config.AddConfigPath("../consensus/obcpbft/")
	config.AddConfigPath("../../consensus/obcpbft")
	gopath := os.Getenv("GOPATH")
	obcpbftpath := filepath.Join(gopath, "src/github.com/hyperledger/fabric/consensus/obcpbft")
	config.AddConfigPath(obcpbftpath)    // Path to look for the config file in
	err := config.ReadInConfig()
	if err != nil {
		panic(fmt.Errorf("Error reading %s plugin config: %s", configPrefix, err))
	}
	return
}

// Returns the uint64 ID corresponding to a peer handle
func getValidatorID(handle *pb.PeerID) (id uint64, err error) {
	// as requested here: https://github.com/hyperledger/fabric/issues/462#issuecomment-170785410
	if startsWith := strings.HasPrefix(handle.Name, "vp"); startsWith {
		id, err = strconv.ParseUint(handle.Name[2:], 10, 64)
		if err != nil {
			return id, fmt.Errorf("Error extracting ID from \"%s\" handle: %v", handle.Name, err)
		}
		return
	}

	err = fmt.Errorf(`For MVP, set the VP's peer.id to vpX,
		where X is a unique integer between 0 and N-1
		(N being the maximum number of VPs in the network`)
	return
}

// Returns the peer handle that corresponds to a validator ID (uint64 assigned to it for PBFT)
func getValidatorHandle(id uint64) (handle *pb.PeerID, err error) {
	// as requested here: https://github.com/hyperledger/fabric/issues/462#issuecomment-170785410
	name := "vp" + strconv.FormatUint(id, 10)
	return &pb.PeerID{Name: name}, nil
}

// Returns the peer handles corresponding to a list of replica ids
func getValidatorHandles(ids []uint64) (handles []*pb.PeerID) {
	handles = make([]*pb.PeerID, len(ids))
	for i, id := range ids {
		handles[i], _ = getValidatorHandle(id)
	}
	return
}
<|endoftext|>


<commit_before>package mocks

import "github.com/stretchr/testify/mock"

import "github.com/mendersoftware/deployments/resources/deployments"

// DeploymentsModel is an autogenerated mock type for the DeploymentsModel type
type DeploymentsModel struct {
	mock.Mock
}

// CreateDeployment provides a mock function with given fields: constructor
func (_m *DeploymentsModel) CreateDeployment(constructor *deployments.DeploymentConstructor) (string, error) {
	ret := _m.Called(constructor)

	var r0 string
	if rf, ok := ret.Get(0).(func(*deployments.DeploymentConstructor) string); ok {
		r0 = rf(constructor)
	} else {
		r0 = ret.Get(0).(string)
	}

	var r1 error
	if rf, ok := ret.Get(1).(func(*deployments.DeploymentConstructor) error); ok {
		r1 = rf(constructor)
	} else {
		r1 = ret.Error(1)
	}

	return r0, r1
}

// GetDeployment provides a mock function with given fields: deploymentID
func (_m *DeploymentsModel) GetDeployment(deploymentID string) (*deployments.Deployment, error) {
	ret := _m.Called(deploymentID)

	var r0 *deployments.Deployment
	if rf, ok := ret.Get(0).(func(string) *deployments.Deployment); ok {
		r0 = rf(deploymentID)
	} else {
		if ret.Get(0) != nil {
			r0 = ret.Get(0).(*deployments.Deployment)
		}
	}

	var r1 error
	if rf, ok := ret.Get(1).(func(string) error); ok {
		r1 = rf(deploymentID)
	} else {
		r1 = ret.Error(1)
	}

	return r0, r1
}

// GetDeploymentForDevice provides a mock function with given fields: deviceID
func (_m *DeploymentsModel) GetDeploymentForDevice(deviceID string) (*deployments.DeploymentInstructions, error) {
	ret := _m.Called(deviceID)

	var r0 *deployments.DeploymentInstructions
	if rf, ok := ret.Get(0).(func(string) *deployments.DeploymentInstructions); ok {
		r0 = rf(deviceID)
	} else {
		if ret.Get(0) != nil {
			r0 = ret.Get(0).(*deployments.DeploymentInstructions)
		}
	}

	var r1 error
	if rf, ok := ret.Get(1).(func(string) error); ok {
		r1 = rf(deviceID)
	} else {
		r1 = ret.Error(1)
	}

	return r0, r1
}
<commit_msg>Add missing license from DeploymentsModel auto-generated mock. Cleanup imports.<commit_after>// Copyright 2016 Mender Software AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

package mocks

import (
	"github.com/mendersoftware/deployments/resources/deployments"
	"github.com/stretchr/testify/mock"
)

// DeploymentsModel is an autogenerated mock type for the DeploymentsModel type
type DeploymentsModel struct {
	mock.Mock
}

// CreateDeployment provides a mock function with given fields: constructor
func (_m *DeploymentsModel) CreateDeployment(constructor *deployments.DeploymentConstructor) (string, error) {
	ret := _m.Called(constructor)

	var r0 string
	if rf, ok := ret.Get(0).(func(*deployments.DeploymentConstructor) string); ok {
		r0 = rf(constructor)
	} else {
		r0 = ret.Get(0).(string)
	}

	var r1 error
	if rf, ok := ret.Get(1).(func(*deployments.DeploymentConstructor) error); ok {
		r1 = rf(constructor)
	} else {
		r1 = ret.Error(1)
	}

	return r0, r1
}

// GetDeployment provides a mock function with given fields: deploymentID
func (_m *DeploymentsModel) GetDeployment(deploymentID string) (*deployments.Deployment, error) {
	ret := _m.Called(deploymentID)

	var r0 *deployments.Deployment
	if rf, ok := ret.Get(0).(func(string) *deployments.Deployment); ok {
		r0 = rf(deploymentID)
	} else {
		if ret.Get(0) != nil {
			r0 = ret.Get(0).(*deployments.Deployment)
		}
	}

	var r1 error
	if rf, ok := ret.Get(1).(func(string) error); ok {
		r1 = rf(deploymentID)
	} else {
		r1 = ret.Error(1)
	}

	return r0, r1
}

// GetDeploymentForDevice provides a mock function with given fields: deviceID
func (_m *DeploymentsModel) GetDeploymentForDevice(deviceID string) (*deployments.DeploymentInstructions, error) {
	ret := _m.Called(deviceID)

	var r0 *deployments.DeploymentInstructions
	if rf, ok := ret.Get(0).(func(string) *deployments.DeploymentInstructions); ok {
		r0 = rf(deviceID)
	} else {
		if ret.Get(0) != nil {
			r0 = ret.Get(0).(*deployments.DeploymentInstructions)
		}
	}

	var r1 error
	if rf, ok := ret.Get(1).(func(string) error); ok {
		r1 = rf(deviceID)
	} else {
		r1 = ret.Error(1)
	}

	return r0, r1
}
<|endoftext|>


<commit_before>package tagsvc

import (
	"fmt"
	"github.com/golang/protobuf/proto"
	"github.com/pgm/pliant/v2"
	. "gopkg.in/check.v1"
	"testing"
)

type TagSvcSuite struct{}

var _ = Suite(&TagSvcSuite{})
var _ = fmt.Sprintf("hello!")

func Test(t *testing.T) { TestingT(t) }

func (s *TagSvcSuite) TestLeases(c *C) {
	key1 := v2.Key{1}
	key2 := v2.Key{2}
	key3 := v2.Key{3}

	root := NewRoots()
	c.Assert(len(root.GetRoots()), Equals, 0)
	root.AddLease(100, &key1)
	c.Assert(len(root.GetRoots()), Equals, 1)
	root.AddLease(101, &key2)
	c.Assert(len(root.GetRoots()), Equals, 2)
	root.AddLease(102, &key3)

	c.Assert(len(root.GetRoots()), Equals, 3)

	root.Expire(100)
	c.Assert(len(root.GetRoots()), Equals, 3)
	root.Expire(101)
	c.Assert(len(root.GetRoots()), Equals, 2)
	root.Expire(103)
	c.Assert(len(root.GetRoots()), Equals, 0)
}

func (s *TagSvcSuite) TestSetRoot(c *C) {
	key1 := v2.Key{1}
	key2 := v2.Key{2}
	key3 := v2.Key{3}

	root := NewRoots()
	root.Set("1", &key1)
	root.Set("2", &key2)
	c.Assert(len(root.GetRoots()), Equals, 2)

	root.Set("2", &key3)
	c.Assert(len(root.GetRoots()), Equals, 2)

	root.Set("1", nil)
	c.Assert(len(root.GetRoots()), Equals, 1)
}

func (s *TagSvcSuite) TestSimpleGC(c *C) {
	fileKey1 := v2.Key{10}
	fileKey2 := v2.Key{11}
	fileKey3 := v2.Key{12}

	root := NewRoots()
	count := 0
	countPtr := &count
	chunks := v2.NewMemChunkService()
	chunks.Put(&fileKey1, v2.NewMemResource(make([]byte, 1)))
	chunks.Put(&fileKey2, v2.NewMemResource(make([]byte, 1)))
	chunks.Put(&fileKey3, v2.NewMemResource(make([]byte, 1)))
	dirService := v2.NewLeafDirService(chunks)
	dir := dirService.GetDirectory(v2.EMPTY_DIR_KEY)
	dirKey := dir.Put("a", &v2.FileMetadata{Length: proto.Int64(1), Key: fileKey1.AsBytes(), IsDir: proto.Bool(false)})
	root.Set("1", dirKey)

	fmt.Printf("GC\n")
	root.GC(dirService, chunks, func(key *v2.Key) {
		fmt.Printf("free %s\n", key.String())
		*countPtr += 1
	})

	c.Assert(*countPtr, Equals, 2)
}

func (s *TagSvcSuite) TestClientServer (c *C) {

	config := &Config{
		AccessKeyId     :"access",
		SecretAccessKey :"secret",
		Endpoint        :"http://endpoint",
		Bucket          :"bucket",
		MasterPort      :0,
		Prefix          :"prefix"}

	l, err := StartServer(config)
	c.Assert(err, IsNil)

	client := NewClient(l.Addr().String())
	vconfig, err := client.GetConfig()
	c.Assert(err, IsNil)
	c.Assert(vconfig, DeepEquals, config)

	key1 := v2.Key{10}
	err2 := client.AddLease(uint64(100), &key1)
	c.Assert(err2, IsNil)

	key2 := v2.Key{10}
	tagSvc := NewTagService(client)
	tagSvc.Put("label", &key2)
	vkey := tagSvc.Get("label")

	c.Assert(vkey, DeepEquals, &key2)
	l.Close()
}
<commit_msg>update test to verify get of an invalid tag yields nil<commit_after>package tagsvc

import (
	"fmt"
	"github.com/golang/protobuf/proto"
	"github.com/pgm/pliant/v2"
	. "gopkg.in/check.v1"
	"testing"
)

type TagSvcSuite struct{}

var _ = Suite(&TagSvcSuite{})
var _ = fmt.Sprintf("hello!")

func Test(t *testing.T) { TestingT(t) }

func (s *TagSvcSuite) TestLeases(c *C) {
	key1 := v2.Key{1}
	key2 := v2.Key{2}
	key3 := v2.Key{3}

	root := NewRoots()
	c.Assert(len(root.GetRoots()), Equals, 0)
	root.AddLease(100, &key1)
	c.Assert(len(root.GetRoots()), Equals, 1)
	root.AddLease(101, &key2)
	c.Assert(len(root.GetRoots()), Equals, 2)
	root.AddLease(102, &key3)

	c.Assert(len(root.GetRoots()), Equals, 3)

	root.Expire(100)
	c.Assert(len(root.GetRoots()), Equals, 3)
	root.Expire(101)
	c.Assert(len(root.GetRoots()), Equals, 2)
	root.Expire(103)
	c.Assert(len(root.GetRoots()), Equals, 0)
}

func (s *TagSvcSuite) TestSetRoot(c *C) {
	key1 := v2.Key{1}
	key2 := v2.Key{2}
	key3 := v2.Key{3}

	root := NewRoots()
	root.Set("1", &key1)
	root.Set("2", &key2)
	c.Assert(len(root.GetRoots()), Equals, 2)

	root.Set("2", &key3)
	c.Assert(len(root.GetRoots()), Equals, 2)

	root.Set("1", nil)
	c.Assert(len(root.GetRoots()), Equals, 1)
}

func (s *TagSvcSuite) TestSimpleGC(c *C) {
	fileKey1 := v2.Key{10}
	fileKey2 := v2.Key{11}
	fileKey3 := v2.Key{12}

	root := NewRoots()
	count := 0
	countPtr := &count
	chunks := v2.NewMemChunkService()
	chunks.Put(&fileKey1, v2.NewMemResource(make([]byte, 1)))
	chunks.Put(&fileKey2, v2.NewMemResource(make([]byte, 1)))
	chunks.Put(&fileKey3, v2.NewMemResource(make([]byte, 1)))
	dirService := v2.NewLeafDirService(chunks)
	dir := dirService.GetDirectory(v2.EMPTY_DIR_KEY)
	dirKey := dir.Put("a", &v2.FileMetadata{Length: proto.Int64(1), Key: fileKey1.AsBytes(), IsDir: proto.Bool(false)})
	root.Set("1", dirKey)

	fmt.Printf("GC\n")
	root.GC(dirService, chunks, func(key *v2.Key) {
		fmt.Printf("free %s\n", key.String())
		*countPtr += 1
	})

	c.Assert(*countPtr, Equals, 2)
}

func (s *TagSvcSuite) TestClientServer (c *C) {
	config := &Config{
		AccessKeyId     :"access",
		SecretAccessKey :"secret",
		Endpoint        :"http://endpoint",
		Bucket          :"bucket",
		MasterPort      :0,
		Prefix          :"prefix"}

	l, err := StartServer(config)
	c.Assert(err, IsNil)

	client := NewClient(l.Addr().String())
	vconfig, err := client.GetConfig()
	c.Assert(err, IsNil)
	c.Assert(vconfig, DeepEquals, config)

	key1 := v2.Key{10}
	err2 := client.AddLease(uint64(100), &key1)
	c.Assert(err2, IsNil)

	key2 := v2.Key{10}
	tagSvc := NewTagService(client)
	tagSvc.Put("label", &key2)
	vkey := tagSvc.Get("label")

	c.Assert(vkey, DeepEquals, &key2)

	c.Assert(tagSvc.Get("label2"), IsNil)

	l.Close()
}
<|endoftext|>


<commit_before>package capsicum

import (
	"os"
	"path"
	"runtime"
	"syscall"

	"golang.org/x/sys/unix"
)

const supportsCloseOnExec bool = true // True for any system that supports Capsicum (so far)

// FIXME: unexported function from os
// syscallMode returns the syscall-specific mode bits from Go's portable mode bits.
func syscallMode(i os.FileMode) (o uint32) {
	o |= uint32(i.Perm())
	if i&os.ModeSetuid != 0 {
		o |= syscall.S_ISUID
	}
	if i&os.ModeSetgid != 0 {
		o |= syscall.S_ISGID
	}
	if i&os.ModeSticky != 0 {
		o |= syscall.S_ISVTX
	}
	// No mapping for Go's ModeTemporary (plan9 only).
	return
}

// OpenFileAt is the generalized open call; most users will use Open
// or Create instead. It opens the named file with specified flag
// (O_RDONLY etc.) and perm, (0666 etc.) if applicable. If successful,
// methods on the returned File can be used for I/O.
// If there is an error, it will be of type *PathError.
// FIXME: this is mostly just os.OpenFile - probably that should call this
// FIXME: this isn't just a FreeBSD call, Linux has it, too
func OpenFileAt(f *os.File, name string, flag int, perm os.FileMode) (*os.File, error) {
	chmod := false
	if !supportsCreateWithStickyBit && flag&os.O_CREATE != 0 && perm&os.ModeSticky != 0 {
		if _, err := os.Stat(name); os.IsNotExist(err) {
			chmod = true
		}
	}

	var r int
	for {
		var e error
		r, e = unix.Openat(int(f.Fd()), name, flag|unix.O_CLOEXEC, syscallMode(perm))
		if e == nil {
			break
		}

		// On OS X, sigaction(2) doesn't guarantee that SA_RESTART will cause
		// open(2) to be restarted for regular files. This is easy to reproduce on
		// fuse file systems (see http://golang.org/issue/11180).
		if runtime.GOOS == "darwin" && e == syscall.EINTR {
			continue
		}

		return nil, &os.PathError{"open", name, e}
	}

	// open(2) itself won't handle the sticky bit on *BSD and Solaris
	if chmod {
		// FIXME: change to FchmodAt()
		os.Chmod(name, perm)
	}

	// There's a race here with fork/exec, which we are
	// content to live with. See ../syscall/exec_unix.go.
	if !supportsCloseOnExec {
		syscall.CloseOnExec(r)
	}

	return os.NewFile(uintptr(r), path.Join(f.Name(), name)), nil
}
<commit_msg>Fix comment.<commit_after>package capsicum

import (
	"os"
	"path"
	"runtime"
	"syscall"

	"golang.org/x/sys/unix"
)

const supportsCloseOnExec bool = true // True for any system that supports Capsicum (so far)

// FIXME: unexported function from os
// syscallMode returns the syscall-specific mode bits from Go's portable mode bits.
func syscallMode(i os.FileMode) (o uint32) {
	o |= uint32(i.Perm())
	if i&os.ModeSetuid != 0 {
		o |= syscall.S_ISUID
	}
	if i&os.ModeSetgid != 0 {
		o |= syscall.S_ISGID
	}
	if i&os.ModeSticky != 0 {
		o |= syscall.S_ISVTX
	}
	// No mapping for Go's ModeTemporary (plan9 only).
	return
}

// OpenFileAt is the generalized open call; most users will use Open
// or Create instead. It opens the named file with specified flag
// (O_RDONLY etc.) and perm, (0666 etc.) if applicable. If successful,
// methods on the returned File can be used for I/O.
// If there is an error, it will be of type *PathError.
// FIXME: this is mostly just os.OpenFile - probably that should call this
func OpenFileAt(f *os.File, name string, flag int, perm os.FileMode) (*os.File, error) {
	chmod := false
	if !supportsCreateWithStickyBit && flag&os.O_CREATE != 0 && perm&os.ModeSticky != 0 {
		if _, err := os.Stat(name); os.IsNotExist(err) {
			chmod = true
		}
	}

	var r int
	for {
		var e error
		r, e = unix.Openat(int(f.Fd()), name, flag|unix.O_CLOEXEC, syscallMode(perm))
		if e == nil {
			break
		}

		// On OS X, sigaction(2) doesn't guarantee that SA_RESTART will cause
		// open(2) to be restarted for regular files. This is easy to reproduce on
		// fuse file systems (see http://golang.org/issue/11180).
		if runtime.GOOS == "darwin" && e == syscall.EINTR {
			continue
		}

		return nil, &os.PathError{"open", name, e}
	}

	// open(2) itself won't handle the sticky bit on *BSD and Solaris
	if chmod {
		// FIXME: change to FchmodAt()
		os.Chmod(name, perm)
	}

	// There's a race here with fork/exec, which we are
	// content to live with. See ../syscall/exec_unix.go.
	if !supportsCloseOnExec {
		syscall.CloseOnExec(r)
	}

	return os.NewFile(uintptr(r), path.Join(f.Name(), name)), nil
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package utils

import (
	"sync"
	"time"

	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider"

	"k8s.io/klog"
)

const (
	// CloudProviderNodeInstancesCacheRefreshInterval is the interval between nodegroup instances cache refreshes.
	CloudProviderNodeInstancesCacheRefreshInterval = 2 * time.Minute
	// CloudProviderNodeInstancesCacheEntryFreshnessThreshold is the duration when cache entry is fresh.
	CloudProviderNodeInstancesCacheEntryFreshnessThreshold = 5 * time.Minute
)

type cloudProviderNodeInstancesCacheEntry struct {
	instances   []cloudprovider.Instance
	refreshTime time.Time
}

func (cacheEntry *cloudProviderNodeInstancesCacheEntry) isStale() bool {
	return cacheEntry.refreshTime.Add(CloudProviderNodeInstancesCacheEntryFreshnessThreshold).Before(time.Now())
}

// CloudProviderNodeInstancesCache caches cloud provider node instances.
type CloudProviderNodeInstancesCache struct {
	sync.Mutex
	cloudProviderNodeInstances map[string]*cloudProviderNodeInstancesCacheEntry
	cloudProvider              cloudprovider.CloudProvider
}

// NewCloudProviderNodeInstancesCache creates new cache instance.
func NewCloudProviderNodeInstancesCache(cloudProvider cloudprovider.CloudProvider) *CloudProviderNodeInstancesCache {
	return &CloudProviderNodeInstancesCache{
		cloudProviderNodeInstances: map[string]*cloudProviderNodeInstancesCacheEntry{},
		cloudProvider:              cloudProvider,
	}
}

func (cache *CloudProviderNodeInstancesCache) updateCacheEntryLocked(nodeGroup cloudprovider.NodeGroup, cacheEntry *cloudProviderNodeInstancesCacheEntry) {
	cache.Lock()
	defer cache.Unlock()
	cache.cloudProviderNodeInstances[nodeGroup.Id()] = cacheEntry
}

func (cache *CloudProviderNodeInstancesCache) removeCacheEntryLocked(nodeGroup cloudprovider.NodeGroup) {
	cache.Lock()
	defer cache.Unlock()
	delete(cache.cloudProviderNodeInstances, nodeGroup.Id())
}

func (cache *CloudProviderNodeInstancesCache) getCacheEntryLocked(nodeGroup cloudprovider.NodeGroup) (*cloudProviderNodeInstancesCacheEntry, bool) {
	cache.Lock()
	defer cache.Unlock()
	cacheEntry, found := cache.cloudProviderNodeInstances[nodeGroup.Id()]
	return cacheEntry, found
}

func (cache *CloudProviderNodeInstancesCache) removeEntriesForNonExistingNodeGroupsLocked(nodeGroups []cloudprovider.NodeGroup) {
	cache.Lock()
	defer cache.Unlock()
	nodeGroupExists := map[string]bool{}
	for _, nodeGroup := range nodeGroups {
		nodeGroupExists[nodeGroup.Id()] = true
	}
	for nodeGroupId := range cache.cloudProviderNodeInstances {
		if !nodeGroupExists[nodeGroupId] {
			delete(cache.cloudProviderNodeInstances, nodeGroupId)
		}
	}
}

// GetCloudProviderNodeInstances returns cloud provider node instances for all node groups returned by cloud provider.
func (cache *CloudProviderNodeInstancesCache) GetCloudProviderNodeInstances() (map[string][]cloudprovider.Instance, error) {
	results := map[string][]cloudprovider.Instance{}
	for _, nodeGroup := range cache.cloudProvider.NodeGroups() {
		nodeGroupInstances, err := cache.GetCloudProviderNodeInstancesForNodeGroup(nodeGroup)
		if err != nil {
			return nil, err
		}
		results[nodeGroup.Id()] = nodeGroupInstances
	}
	return results, nil
}

// GetCloudProviderNodeInstancesForNodeGroup returns cloud provider node instances for the given node group.
func (cache *CloudProviderNodeInstancesCache) GetCloudProviderNodeInstancesForNodeGroup(nodeGroup cloudprovider.NodeGroup) ([]cloudprovider.Instance, error) {
	cacheEntry, found := cache.getCacheEntryLocked(nodeGroup)
	if found {
		if cacheEntry.isStale() {
			klog.Warningf("Entry cloudProviderNodeInstances is stale, refresh time is %v", cacheEntry.refreshTime)
		}
		klog.V(5).Infof("Get cached cloud provider node instances for %v", nodeGroup.Id())
		return cacheEntry.instances, nil
	}
	klog.V(5).Infof("Cloud provider node instances for %v hasn't been found in cache, fetch from cloud provider", nodeGroup.Id())
	nodeGroupInstances, err := nodeGroup.Nodes()
	if err != nil {
		return nil, err
	}
	cache.updateCacheEntryLocked(nodeGroup, &cloudProviderNodeInstancesCacheEntry{nodeGroupInstances, time.Now()})
	return nodeGroupInstances, nil

}

// InvalidateCacheEntry removes entry for the given node group from cache.
func (cache *CloudProviderNodeInstancesCache) InvalidateCacheEntry(nodeGroup cloudprovider.NodeGroup) {
	klog.V(5).Infof("Invalidate entry in cloud provider node instances cache %v", nodeGroup.Id())
	cache.removeCacheEntryLocked(nodeGroup)
}

// Refresh refreshes cache.
func (cache *CloudProviderNodeInstancesCache) Refresh() {
	klog.Infof("Start refreshing cloud provider node instances cache")
	refreshStart := time.Now()

	nodeGroups := cache.cloudProvider.NodeGroups()
	cache.removeEntriesForNonExistingNodeGroupsLocked(nodeGroups)
	for _, nodeGroup := range nodeGroups {
		nodeGroupInstances, err := nodeGroup.Nodes()
		if err != nil {
			klog.Errorf("Failed to get cloud provider node instance for node group %v, error %v", nodeGroup.Id(), err)
		}
		cache.updateCacheEntryLocked(nodeGroup, &cloudProviderNodeInstancesCacheEntry{nodeGroupInstances, time.Now()})
	}
	klog.Infof("Refresh cloud provider node instances cache finished, refresh took %v", time.Now().Sub(refreshStart))
}

// Start starts components running in background.
func (cache *CloudProviderNodeInstancesCache) Start(interrupt chan struct{}) {
	go wait.Until(func() {
		cache.Refresh()
	}, CloudProviderNodeInstancesCacheRefreshInterval, interrupt)
}
<commit_msg>Fetch node instances for scaling up node groups in parallel.<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package utils

import (
	"sync"
	"time"

	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider"

	"k8s.io/klog"
)

const (
	// CloudProviderNodeInstancesCacheRefreshInterval is the interval between nodegroup instances cache refreshes.
	CloudProviderNodeInstancesCacheRefreshInterval = 2 * time.Minute
	// CloudProviderNodeInstancesCacheEntryFreshnessThreshold is the duration when cache entry is fresh.
	CloudProviderNodeInstancesCacheEntryFreshnessThreshold = 5 * time.Minute
)

type cloudProviderNodeInstancesCacheEntry struct {
	instances   []cloudprovider.Instance
	refreshTime time.Time
}

func (cacheEntry *cloudProviderNodeInstancesCacheEntry) isStale() bool {
	return cacheEntry.refreshTime.Add(CloudProviderNodeInstancesCacheEntryFreshnessThreshold).Before(time.Now())
}

// CloudProviderNodeInstancesCache caches cloud provider node instances.
type CloudProviderNodeInstancesCache struct {
	sync.Mutex
	cloudProviderNodeInstances map[string]*cloudProviderNodeInstancesCacheEntry
	cloudProvider              cloudprovider.CloudProvider
}

// NewCloudProviderNodeInstancesCache creates new cache instance.
func NewCloudProviderNodeInstancesCache(cloudProvider cloudprovider.CloudProvider) *CloudProviderNodeInstancesCache {
	return &CloudProviderNodeInstancesCache{
		cloudProviderNodeInstances: map[string]*cloudProviderNodeInstancesCacheEntry{},
		cloudProvider:              cloudProvider,
	}
}

func (cache *CloudProviderNodeInstancesCache) updateCacheEntryLocked(nodeGroup cloudprovider.NodeGroup, cacheEntry *cloudProviderNodeInstancesCacheEntry) {
	cache.Lock()
	defer cache.Unlock()
	cache.cloudProviderNodeInstances[nodeGroup.Id()] = cacheEntry
}

func (cache *CloudProviderNodeInstancesCache) removeCacheEntryLocked(nodeGroup cloudprovider.NodeGroup) {
	cache.Lock()
	defer cache.Unlock()
	delete(cache.cloudProviderNodeInstances, nodeGroup.Id())
}

func (cache *CloudProviderNodeInstancesCache) getCacheEntryLocked(nodeGroup cloudprovider.NodeGroup) (*cloudProviderNodeInstancesCacheEntry, bool) {
	cache.Lock()
	defer cache.Unlock()
	cacheEntry, found := cache.cloudProviderNodeInstances[nodeGroup.Id()]
	return cacheEntry, found
}

func (cache *CloudProviderNodeInstancesCache) removeEntriesForNonExistingNodeGroupsLocked(nodeGroups []cloudprovider.NodeGroup) {
	cache.Lock()
	defer cache.Unlock()
	nodeGroupExists := map[string]bool{}
	for _, nodeGroup := range nodeGroups {
		nodeGroupExists[nodeGroup.Id()] = true
	}
	for nodeGroupId := range cache.cloudProviderNodeInstances {
		if !nodeGroupExists[nodeGroupId] {
			delete(cache.cloudProviderNodeInstances, nodeGroupId)
		}
	}
}

// GetCloudProviderNodeInstances returns cloud provider node instances for all node groups returned by cloud provider.
func (cache *CloudProviderNodeInstancesCache) GetCloudProviderNodeInstances() (map[string][]cloudprovider.Instance, error) {
	nodeGroups := cache.cloudProvider.NodeGroups()

	// Fetch missing node instances.
	var wg sync.WaitGroup
	for _, nodeGroup := range nodeGroups {
		nodeGroup := nodeGroup
		if _, found := cache.getCacheEntryLocked(nodeGroup); !found {
			wg.Add(1)
			go func() {
				defer wg.Done()
				_, err := cache.fetchCloudProviderNodeInstancesForNodeGroup(nodeGroup)
				klog.Errorf("Failed to fetch cloud provider node instances for %v, error %v", nodeGroup.Id(), err)
			}()
		}
	}
	wg.Wait()

	// Get data from cache.
	results := map[string][]cloudprovider.Instance{}
	for _, nodeGroup := range nodeGroups {
		nodeGroupInstances, err := cache.GetCloudProviderNodeInstancesForNodeGroup(nodeGroup)
		if err != nil {
			return nil, err
		}
		results[nodeGroup.Id()] = nodeGroupInstances
	}
	return results, nil
}

// GetCloudProviderNodeInstancesForNodeGroup returns cloud provider node instances for the given node group.
func (cache *CloudProviderNodeInstancesCache) GetCloudProviderNodeInstancesForNodeGroup(nodeGroup cloudprovider.NodeGroup) ([]cloudprovider.Instance, error) {
	cacheEntry, found := cache.getCacheEntryLocked(nodeGroup)
	if found {
		if cacheEntry.isStale() {
			klog.Warningf("Entry cloudProviderNodeInstances is stale, refresh time is %v", cacheEntry.refreshTime)
		}
		klog.V(5).Infof("Get cached cloud provider node instances for %v", nodeGroup.Id())
		return cacheEntry.instances, nil
	}
	klog.V(5).Infof("Cloud provider node instances for %v hasn't been found in cache, fetch from cloud provider", nodeGroup.Id())
	return cache.fetchCloudProviderNodeInstancesForNodeGroup(nodeGroup)
}

func (cache *CloudProviderNodeInstancesCache) fetchCloudProviderNodeInstancesForNodeGroup(nodeGroup cloudprovider.NodeGroup) ([]cloudprovider.Instance, error) {
	nodeGroupInstances, err := nodeGroup.Nodes()
	if err != nil {
		return nil, err
	}
	cache.updateCacheEntryLocked(nodeGroup, &cloudProviderNodeInstancesCacheEntry{nodeGroupInstances, time.Now()})
	return nodeGroupInstances, nil
}

// InvalidateCacheEntry removes entry for the given node group from cache.
func (cache *CloudProviderNodeInstancesCache) InvalidateCacheEntry(nodeGroup cloudprovider.NodeGroup) {
	klog.V(5).Infof("Invalidate entry in cloud provider node instances cache %v", nodeGroup.Id())
	cache.removeCacheEntryLocked(nodeGroup)
}

// Refresh refreshes cache.
func (cache *CloudProviderNodeInstancesCache) Refresh() {
	klog.Infof("Start refreshing cloud provider node instances cache")
	refreshStart := time.Now()

	nodeGroups := cache.cloudProvider.NodeGroups()
	cache.removeEntriesForNonExistingNodeGroupsLocked(nodeGroups)
	for _, nodeGroup := range nodeGroups {
		nodeGroupInstances, err := nodeGroup.Nodes()
		if err != nil {
			klog.Errorf("Failed to get cloud provider node instance for node group %v, error %v", nodeGroup.Id(), err)
		}
		cache.updateCacheEntryLocked(nodeGroup, &cloudProviderNodeInstancesCacheEntry{nodeGroupInstances, time.Now()})
	}
	klog.Infof("Refresh cloud provider node instances cache finished, refresh took %v", time.Now().Sub(refreshStart))
}

// Start starts components running in background.
func (cache *CloudProviderNodeInstancesCache) Start(interrupt chan struct{}) {
	go wait.Until(func() {
		cache.Refresh()
	}, CloudProviderNodeInstancesCacheRefreshInterval, interrupt)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/ziutek/mymysql/mysql"
	_ "github.com/ziutek/mymysql/native"
	"log"
	"os"
)

const _DB_CONFIG_FILENAME = "db_config.SECRET.json"

type dbConfig struct {
	Url, Username, Password, DbName, SolvesTable, SolvesID, SolvesTotalTime, SolvesUser string
}

type solve struct {
	puzzleID  int
	totalTime int
}

type userSolvesCollection struct {
	solves []*solve
	max    int
	min    int
}

type puzzle struct {
	id                     int
	userRelativeDifficulty float32
}

func (self *userSolvesCollection) addSolve(solve *solve) {
	self.solves = append(self.solves, solve)
	if len(self.solves) == 1 {
		self.max = solve.totalTime
		self.min = solve.totalTime
	} else {
		if self.max < solve.totalTime {
			self.max = solve.totalTime
		}
		if self.min > solve.totalTime {
			self.min = solve.totalTime
		}
	}
}

func (self *userSolvesCollection) relativeDifficulties() map[int]float32 {
	//Returns a map of puzzle id to relative difficulty, normalized by our max and min.
	avgSolveTimes := make(map[int]float32)
	//Keep track of how many times we've seen each puzzle solved by this user so we can do correct averaging.
	avgSolveTimesCount := make(map[int]int)

	//First, collect the average solve time (in case the same user has solved more than once the same puzzle)

	for _, solve := range self.solves {
		currentAvgSolveTime := avgSolveTimes[solve.puzzleID]

		avgSolveTimes[solve.puzzleID] = (currentAvgSolveTime*float32(avgSolveTimesCount[solve.puzzleID]) + float32(solve.totalTime)) / float32(avgSolveTimesCount[solve.puzzleID]+1)

		avgSolveTimesCount[solve.puzzleID]++
	}

	//Now, relativize all of the scores.

	result := make(map[int]float32)

	for puzzleID, avgSolveTime := range avgSolveTimes {
		result[puzzleID] = (avgSolveTime - float32(self.min)) / float32(self.max-self.min)
	}

	return result
}

func main() {
	file, err := os.Open(_DB_CONFIG_FILENAME)
	if err != nil {
		log.Fatal("Could not find the config file at ", _DB_CONFIG_FILENAME, ". You should copy the SAMPLE one to that filename and configure.")
		os.Exit(1)
	}
	defer file.Close()
	decoder := json.NewDecoder(file)
	var config dbConfig
	if err := decoder.Decode(&config); err != nil {
		log.Fatal("There was an error parsing JSON from the config file: ", err)
		os.Exit(1)
	}

	db := mysql.New("tcp", "", config.Url, config.Username, config.Password, config.DbName)

	if err := db.Connect(); err != nil {
		log.Fatal(err)
		os.Exit(1)
	}

	res, err := db.Start("select %s, %s, %s from %s limit 100", config.SolvesUser, config.SolvesID, config.SolvesTotalTime, config.SolvesTable)

	if err != nil {
		log.Fatal(err)
		os.Exit(1)
	}

	solvesByUser := make(map[string]*userSolvesCollection)

	var userSolves *userSolvesCollection
	var ok bool
	var i int

	//First, process all user records in the DB to collect all solves by userName.
	for {

		row, _ := res.GetRow()

		if row == nil {
			break
		}

		userSolves, ok = solvesByUser[row.Str(0)]

		if !ok {
			userSolves = new(userSolvesCollection)
			solvesByUser[row.Str(0)] = userSolves
		}

		userSolves.addSolve(&solve{row.Int(1), row.Int(2)})
		i++
	}

	fmt.Println("Processed ", i, " solves by ", len(solvesByUser), " users.")

	//Now get the relative difficulty for each user's puzzles, and collect them.

	relativeDifficultiesByPuzzle := make(map[int][]float32)

	var skippedUsers int

	for _, collection := range solvesByUser {

		if len(collection.solves) < 2 {
			skippedUsers++
			continue
		}

		for puzzleID, relativeDifficulty := range collection.relativeDifficulties() {
			relativeDifficultiesByPuzzle[puzzleID] = append(relativeDifficultiesByPuzzle[puzzleID], relativeDifficulty)
		}

	}

	fmt.Println("Skipped ", skippedUsers, " users because they had only solved one unique puzzle.")

	puzzles := make([]puzzle, len(relativeDifficultiesByPuzzle))

	var index int

	for puzzleID, difficulties := range relativeDifficultiesByPuzzle {
		var sum float32
		for _, difficulty := range difficulties {
			sum += difficulty
		}
		puzzles[index] = puzzle{puzzleID, sum / float32(len(difficulties))}
		index++
	}

}
<commit_msg>Actually sort the puzzles by userRelativeDifficulty.<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/ziutek/mymysql/mysql"
	_ "github.com/ziutek/mymysql/native"
	"log"
	"os"
	"sort"
)

const _DB_CONFIG_FILENAME = "db_config.SECRET.json"

type dbConfig struct {
	Url, Username, Password, DbName, SolvesTable, SolvesID, SolvesTotalTime, SolvesUser string
}

type solve struct {
	puzzleID  int
	totalTime int
}

type userSolvesCollection struct {
	solves []*solve
	max    int
	min    int
}

type puzzle struct {
	id                     int
	userRelativeDifficulty float32
}

type puzzles []puzzle

type byUserRelativeDifficulty struct {
	puzzles
}

func (self puzzles) Len() int {
	return len(self)
}

func (self puzzles) Swap(i, j int) {
	self[i], self[j] = self[j], self[i]
}

func (self byUserRelativeDifficulty) Less(i, j int) bool {
	return self.puzzles[i].userRelativeDifficulty < self.puzzles[j].userRelativeDifficulty
}

func (self *userSolvesCollection) addSolve(solve *solve) {
	self.solves = append(self.solves, solve)
	if len(self.solves) == 1 {
		self.max = solve.totalTime
		self.min = solve.totalTime
	} else {
		if self.max < solve.totalTime {
			self.max = solve.totalTime
		}
		if self.min > solve.totalTime {
			self.min = solve.totalTime
		}
	}
}

func (self *userSolvesCollection) relativeDifficulties() map[int]float32 {
	//Returns a map of puzzle id to relative difficulty, normalized by our max and min.
	avgSolveTimes := make(map[int]float32)
	//Keep track of how many times we've seen each puzzle solved by this user so we can do correct averaging.
	avgSolveTimesCount := make(map[int]int)

	//First, collect the average solve time (in case the same user has solved more than once the same puzzle)

	for _, solve := range self.solves {
		currentAvgSolveTime := avgSolveTimes[solve.puzzleID]

		avgSolveTimes[solve.puzzleID] = (currentAvgSolveTime*float32(avgSolveTimesCount[solve.puzzleID]) + float32(solve.totalTime)) / float32(avgSolveTimesCount[solve.puzzleID]+1)

		avgSolveTimesCount[solve.puzzleID]++
	}

	//Now, relativize all of the scores.

	result := make(map[int]float32)

	for puzzleID, avgSolveTime := range avgSolveTimes {
		result[puzzleID] = (avgSolveTime - float32(self.min)) / float32(self.max-self.min)
	}

	return result
}

func main() {
	file, err := os.Open(_DB_CONFIG_FILENAME)
	if err != nil {
		log.Fatal("Could not find the config file at ", _DB_CONFIG_FILENAME, ". You should copy the SAMPLE one to that filename and configure.")
		os.Exit(1)
	}
	defer file.Close()
	decoder := json.NewDecoder(file)
	var config dbConfig
	if err := decoder.Decode(&config); err != nil {
		log.Fatal("There was an error parsing JSON from the config file: ", err)
		os.Exit(1)
	}

	db := mysql.New("tcp", "", config.Url, config.Username, config.Password, config.DbName)

	if err := db.Connect(); err != nil {
		log.Fatal(err)
		os.Exit(1)
	}

	res, err := db.Start("select %s, %s, %s from %s limit 100", config.SolvesUser, config.SolvesID, config.SolvesTotalTime, config.SolvesTable)

	if err != nil {
		log.Fatal(err)
		os.Exit(1)
	}

	solvesByUser := make(map[string]*userSolvesCollection)

	var userSolves *userSolvesCollection
	var ok bool
	var i int

	//First, process all user records in the DB to collect all solves by userName.
	for {

		row, _ := res.GetRow()

		if row == nil {
			break
		}

		userSolves, ok = solvesByUser[row.Str(0)]

		if !ok {
			userSolves = new(userSolvesCollection)
			solvesByUser[row.Str(0)] = userSolves
		}

		userSolves.addSolve(&solve{row.Int(1), row.Int(2)})
		i++
	}

	fmt.Println("Processed ", i, " solves by ", len(solvesByUser), " users.")

	//Now get the relative difficulty for each user's puzzles, and collect them.

	relativeDifficultiesByPuzzle := make(map[int][]float32)

	var skippedUsers int

	for _, collection := range solvesByUser {

		if len(collection.solves) < 2 {
			skippedUsers++
			continue
		}

		for puzzleID, relativeDifficulty := range collection.relativeDifficulties() {
			relativeDifficultiesByPuzzle[puzzleID] = append(relativeDifficultiesByPuzzle[puzzleID], relativeDifficulty)
		}

	}

	fmt.Println("Skipped ", skippedUsers, " users because they had only solved one unique puzzle.")

	puzzles := make([]puzzle, len(relativeDifficultiesByPuzzle))

	var index int

	for puzzleID, difficulties := range relativeDifficultiesByPuzzle {
		var sum float32
		for _, difficulty := range difficulties {
			sum += difficulty
		}
		puzzles[index] = puzzle{puzzleID, sum / float32(len(difficulties))}
		index++
	}

	//Sort the puzzles by relative user difficulty
	//We actually don't need the wrapper, since it will modify the underlying slice.
	sort.Sort(byUserRelativeDifficulty{puzzles})
}
<|endoftext|>