<commit_before>package handler

import (
	"fullerite/metric"

	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"time"

	"github.com/Sirupsen/logrus"
)

// Datadog handler
type Datadog struct {
	BaseHandler
	endpoint string
	apiKey   string
}

type datadogPayload struct {
	Series []datadogMetric `json:"series"`
}

type datadogMetric struct {
	Metric     string         `json:"metric"`
	Points     []datadogPoint `json:"points"`
	MetricType string         `json:"type"`
	Host       string         `json:"host"`
	Tags       []string       `json:"tags"`
}

type datadogPoint [2]float64

// NewDatadog returns a new Datadog handler
func NewDatadog() *Datadog {
	d := new(Datadog)
	d.name = "Datadog"
	d.maxBufferSize = DefaultBufferSize
	d.timeout = time.Duration(DefaultHandlerTimeoutSec * time.Second)
	d.log = logrus.WithFields(logrus.Fields{"app": "fullerite", "pkg": "handler", "handler": "Datadog"})
	d.channel = make(chan metric.Metric)
	return d
}

// Configure the Datadog handler
func (d *Datadog) Configure(config map[string]interface{}) {
	if apiKey, exists := config["apiKey"]; exists == true {
		d.apiKey = apiKey.(string)
	} else {
		d.log.Error("There was no API key specified for the Datadog handler, there won't be any emissions")
	}
	if endpoint, exists := config["endpoint"]; exists == true {
		d.endpoint = endpoint.(string)
	} else {
		d.log.Error("There was no endpoint specified for the Datadog Handler, there won't be any emissions")
	}
	if timeout, exists := config["timeout"]; exists == true {
		d.timeout = time.Duration(timeout.(float64)) * time.Second
	}
	d.log.Info("timeout is ", d.timeout)
}

// Run runs the Datadog handler
func (d *Datadog) Run() {
	datapoints := make([]datadogMetric, 0, d.maxBufferSize)

	lastEmission := time.Now()
	for incomingMetric := range d.Channel() {
		datapoint := d.convertToDatadog(incomingMetric)
		d.log.Debug("Datadog datapoint: ", datapoint)
		datapoints = append(datapoints, datapoint)
		if time.Since(lastEmission).Seconds() >= float64(d.interval) || len(datapoints) >= d.maxBufferSize {
			d.emitMetrics(datapoints)
			lastEmission = time.Now()
			datapoints = make([]datadogMetric, 0, d.maxBufferSize)
		}
	}
}

func (d *Datadog) convertToDatadog(incomingMetric metric.Metric) (datapoint datadogMetric) {
	dog := new(datadogMetric)
	dog.Metric = incomingMetric.Name
	dog.Points = makeDatadogPoints(incomingMetric)
	dog.MetricType = incomingMetric.MetricType
	if host, ok := incomingMetric.GetDimensionValue("host", d.DefaultDimensions()); ok {
		dog.Host = host
	} else {
		dog.Host = "unknown"
	}
	dog.Tags = d.serializedDimensions(incomingMetric)
	return *dog
}

func (d *Datadog) emitMetrics(series []datadogMetric) {
	d.log.Info("Starting to emit ", len(series), " datapoints")

	if len(series) == 0 {
		d.log.Warn("Skipping send because of an empty payload")
		return
	}

	p := datadogPayload{Series: series}
	payload, err := json.Marshal(p)
	if err != nil {
		d.log.Error("Failed marshaling datapoints to Datadog format")
		d.log.Error("Dropping Datadog datapoints ", series)
		return
	}

	apiURL := fmt.Sprintf("%s/series?api_key=%s", d.endpoint, d.apiKey)
	req, err := http.NewRequest("POST", apiURL, bytes.NewBuffer(payload))
	if err != nil {
		d.log.Error("Failed to create a request to endpoint ", d.endpoint)
		return
	}
	req.Header.Set("Content-Type", "application/json")

	transport := http.Transport{
		Dial: d.dialTimeout,
	}
	client := &http.Client{
		Transport: &transport,
	}
	rsp, err := client.Do(req)
	if err != nil {
		d.log.Error("Failed to complete POST ", err)
		return
	}

	defer rsp.Body.Close()
	if (rsp.StatusCode == http.StatusOK) || (rsp.StatusCode == http.StatusAccepted) {
		d.log.Info("Successfully sent ", len(series), " datapoints to Datadog")
	} else {
		body, _ := ioutil.ReadAll(rsp.Body)
		d.log.Error("Failed to post to Datadog @", d.endpoint,
			" status was ", rsp.Status,
			" rsp body was ", string(body),
			" payload was ", payload)
		return
	}

}

func (d *Datadog) dialTimeout(network, addr string) (net.Conn, error) {
	return net.DialTimeout(network, addr, d.timeout)
}

func (d *Datadog) serializedDimensions(m metric.Metric) (dimensions []string) {
	for name, value := range m.GetDimensions(d.DefaultDimensions()) {
		dimensions = append(dimensions, name+":"+value)
	}
	return dimensions
}

func makeDatadogPoints(m metric.Metric) []datadogPoint {
	point := datadogPoint{float64(time.Now().Unix()), m.Value}
	return []datadogPoint{point}
}
<commit_msg>print payload as string on error<commit_after>package handler

import (
	"fullerite/metric"

	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"time"

	"github.com/Sirupsen/logrus"
)

// Datadog handler
type Datadog struct {
	BaseHandler
	endpoint string
	apiKey   string
}

type datadogPayload struct {
	Series []datadogMetric `json:"series"`
}

type datadogMetric struct {
	Metric     string         `json:"metric"`
	Points     []datadogPoint `json:"points"`
	MetricType string         `json:"type"`
	Host       string         `json:"host"`
	Tags       []string       `json:"tags"`
}

type datadogPoint [2]float64

// NewDatadog returns a new Datadog handler
func NewDatadog() *Datadog {
	d := new(Datadog)
	d.name = "Datadog"
	d.maxBufferSize = DefaultBufferSize
	d.timeout = time.Duration(DefaultHandlerTimeoutSec * time.Second)
	d.log = logrus.WithFields(logrus.Fields{"app": "fullerite", "pkg": "handler", "handler": "Datadog"})
	d.channel = make(chan metric.Metric)
	return d
}

// Configure the Datadog handler
func (d *Datadog) Configure(config map[string]interface{}) {
	if apiKey, exists := config["apiKey"]; exists == true {
		d.apiKey = apiKey.(string)
	} else {
		d.log.Error("There was no API key specified for the Datadog handler, there won't be any emissions")
	}
	if endpoint, exists := config["endpoint"]; exists == true {
		d.endpoint = endpoint.(string)
	} else {
		d.log.Error("There was no endpoint specified for the Datadog Handler, there won't be any emissions")
	}
	if timeout, exists := config["timeout"]; exists == true {
		d.timeout = time.Duration(timeout.(float64)) * time.Second
	}
	d.log.Info("timeout is ", d.timeout)
}

// Run runs the Datadog handler
func (d *Datadog) Run() {
	datapoints := make([]datadogMetric, 0, d.maxBufferSize)

	lastEmission := time.Now()
	for incomingMetric := range d.Channel() {
		datapoint := d.convertToDatadog(incomingMetric)
		d.log.Debug("Datadog datapoint: ", datapoint)
		datapoints = append(datapoints, datapoint)
		if time.Since(lastEmission).Seconds() >= float64(d.interval) || len(datapoints) >= d.maxBufferSize {
			d.emitMetrics(datapoints)
			lastEmission = time.Now()
			datapoints = make([]datadogMetric, 0, d.maxBufferSize)
		}
	}
}

func (d *Datadog) convertToDatadog(incomingMetric metric.Metric) (datapoint datadogMetric) {
	dog := new(datadogMetric)
	dog.Metric = incomingMetric.Name
	dog.Points = makeDatadogPoints(incomingMetric)
	dog.MetricType = incomingMetric.MetricType
	if host, ok := incomingMetric.GetDimensionValue("host", d.DefaultDimensions()); ok {
		dog.Host = host
	} else {
		dog.Host = "unknown"
	}
	dog.Tags = d.serializedDimensions(incomingMetric)
	return *dog
}

func (d *Datadog) emitMetrics(series []datadogMetric) {
	d.log.Info("Starting to emit ", len(series), " datapoints")

	if len(series) == 0 {
		d.log.Warn("Skipping send because of an empty payload")
		return
	}

	p := datadogPayload{Series: series}
	payload, err := json.Marshal(p)
	if err != nil {
		d.log.Error("Failed marshaling datapoints to Datadog format")
		d.log.Error("Dropping Datadog datapoints ", series)
		return
	}

	apiURL := fmt.Sprintf("%s/series?api_key=%s", d.endpoint, d.apiKey)
	req, err := http.NewRequest("POST", apiURL, bytes.NewBuffer(payload))
	if err != nil {
		d.log.Error("Failed to create a request to endpoint ", d.endpoint)
		return
	}
	req.Header.Set("Content-Type", "application/json")

	transport := http.Transport{
		Dial: d.dialTimeout,
	}
	client := &http.Client{
		Transport: &transport,
	}
	rsp, err := client.Do(req)
	if err != nil {
		d.log.Error("Failed to complete POST ", err)
		return
	}

	defer rsp.Body.Close()
	if (rsp.StatusCode == http.StatusOK) || (rsp.StatusCode == http.StatusAccepted) {
		d.log.Info("Successfully sent ", len(series), " datapoints to Datadog")
	} else {
		body, _ := ioutil.ReadAll(rsp.Body)
		d.log.Error("Failed to post to Datadog @", d.endpoint,
			" status was ", rsp.Status,
			" rsp body was ", string(body),
			" payload was ", string(payload))
		return
	}

}

func (d *Datadog) dialTimeout(network, addr string) (net.Conn, error) {
	return net.DialTimeout(network, addr, d.timeout)
}

func (d *Datadog) serializedDimensions(m metric.Metric) (dimensions []string) {
	for name, value := range m.GetDimensions(d.DefaultDimensions()) {
		dimensions = append(dimensions, name+":"+value)
	}
	return dimensions
}

func makeDatadogPoints(m metric.Metric) []datadogPoint {
	point := datadogPoint{float64(time.Now().Unix()), m.Value}
	return []datadogPoint{point}
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package inclusterclient

import (
	"crypto/sha256"
	"encoding/base64"
	"flag"
	"fmt"
	"log"
	"net/http"
	"time"

	"github.com/spf13/cobra"

	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/component-base/logs"
	"k8s.io/klog"
)

var pollInterval int

// CmdInClusterClient is used by agnhost Cobra.
var CmdInClusterClient = &cobra.Command{
	Use:   "inclusterclient",
	Short: "Periodically poll the Kubernetes \"/healthz\" endpoint",
	Long: `Periodically polls the Kubernetes "/healthz" endpoint using the in-cluster config. Because of this, this subcommand is meant to be run inside of a Kubernetes pod.

This subcommand can also be used to validate token rotation.`,
	Args: cobra.MaximumNArgs(0),
	Run:  main,
}

func init() {
	CmdInClusterClient.Flags().IntVar(&pollInterval, "poll-interval", 30,
		"poll interval of call to /healhtz in seconds")
}

func main(cmd *cobra.Command, args []string) {
	logs.InitLogs()
	defer logs.FlushLogs()

	flag.Set("logtostderr", "true")

	klog.Infof("started")

	cfg, err := rest.InClusterConfig()
	if err != nil {
		log.Fatalf("err: %v", err)
	}

	cfg.Wrap(func(rt http.RoundTripper) http.RoundTripper {
		return &debugRt{
			rt: rt,
		}
	})

	c := kubernetes.NewForConfigOrDie(cfg).RESTClient()

	t := time.Tick(time.Duration(pollInterval) * time.Second)
	for {
		<-t
		klog.Infof("calling /healthz")
		b, err := c.Get().AbsPath("/healthz").Do().Raw()
		if err != nil {
			klog.Errorf("status=failed")
			klog.Errorf("error checking /healthz: %v\n%s\n", err, string(b))
		}
	}
}

type debugRt struct {
	rt http.RoundTripper
}

func (rt *debugRt) RoundTrip(req *http.Request) (*http.Response, error) {
	authHeader := req.Header.Get("Authorization")
	if len(authHeader) != 0 {
		authHash := sha256.Sum256([]byte(fmt.Sprintf("%s|%s", "salt", authHeader)))
		klog.Infof("authz_header=%s", base64.RawURLEncoding.EncodeToString(authHash[:]))
	} else {
		klog.Errorf("authz_header=<empty>")
	}
	return rt.rt.RoundTrip(req)
}

func (rt *debugRt) WrappedRoundTripper() http.RoundTripper { return rt.rt }
<commit_msg>Cleanup: fix typo in flag help message<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package inclusterclient

import (
	"crypto/sha256"
	"encoding/base64"
	"flag"
	"fmt"
	"log"
	"net/http"
	"time"

	"github.com/spf13/cobra"

	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/component-base/logs"
	"k8s.io/klog"
)

var pollInterval int

// CmdInClusterClient is used by agnhost Cobra.
var CmdInClusterClient = &cobra.Command{
	Use:   "inclusterclient",
	Short: "Periodically poll the Kubernetes \"/healthz\" endpoint",
	Long: `Periodically polls the Kubernetes "/healthz" endpoint using the in-cluster config. Because of this, this subcommand is meant to be run inside of a Kubernetes pod.

This subcommand can also be used to validate token rotation.`,
	Args: cobra.MaximumNArgs(0),
	Run:  main,
}

func init() {
	CmdInClusterClient.Flags().IntVar(&pollInterval, "poll-interval", 30,
		"poll interval of call to /healthz in seconds")
}

func main(cmd *cobra.Command, args []string) {
	logs.InitLogs()
	defer logs.FlushLogs()

	flag.Set("logtostderr", "true")

	klog.Infof("started")

	cfg, err := rest.InClusterConfig()
	if err != nil {
		log.Fatalf("err: %v", err)
	}

	cfg.Wrap(func(rt http.RoundTripper) http.RoundTripper {
		return &debugRt{
			rt: rt,
		}
	})

	c := kubernetes.NewForConfigOrDie(cfg).RESTClient()

	t := time.Tick(time.Duration(pollInterval) * time.Second)
	for {
		<-t
		klog.Infof("calling /healthz")
		b, err := c.Get().AbsPath("/healthz").Do().Raw()
		if err != nil {
			klog.Errorf("status=failed")
			klog.Errorf("error checking /healthz: %v\n%s\n", err, string(b))
		}
	}
}

type debugRt struct {
	rt http.RoundTripper
}

func (rt *debugRt) RoundTrip(req *http.Request) (*http.Response, error) {
	authHeader := req.Header.Get("Authorization")
	if len(authHeader) != 0 {
		authHash := sha256.Sum256([]byte(fmt.Sprintf("%s|%s", "salt", authHeader)))
		klog.Infof("authz_header=%s", base64.RawURLEncoding.EncodeToString(authHash[:]))
	} else {
		klog.Errorf("authz_header=<empty>")
	}
	return rt.rt.RoundTrip(req)
}

func (rt *debugRt) WrappedRoundTripper() http.RoundTripper { return rt.rt }
<|endoftext|>


<commit_before>package app

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/koding/klient/Godeps/_workspace/src/github.com/boltdb/bolt"
	"github.com/koding/klient/Godeps/_workspace/src/github.com/koding/kite"
	"github.com/koding/klient/Godeps/_workspace/src/github.com/koding/kite/config"
	"github.com/koding/klient/collaboration"
	"github.com/koding/klient/command"
	"github.com/koding/klient/fs"
	"github.com/koding/klient/sshkeys"
	"github.com/koding/klient/storage"
	"github.com/koding/klient/terminal"
	"github.com/koding/klient/usage"
)

var (
	// we also could use an atomic boolean this is simple for now.
	updating   = false
	updatingMu sync.Mutex // protects updating
)

// Klient is the central app which provides all available methods.
type Klient struct {
	// kite defines the server running and powering Klient.
	kite *kite.Kite

	// collab is used to store authorized third party users and allows them to
	// use the available methods. It provides methods to add or remove users
	// from the storage
	collab *collaboration.Collaboration

	storage *storage.Storage

	// terminal provides wmethods
	terminal *terminal.Terminal

	// docker provides the docker related methods.
	// docker *docker.Docker

	// usage counts and tracks all called metrics. It also provides a method
	// that return those informations
	usage *usage.Usage

	log kite.Logger

	// disconnectTimer is used track disconnected users and eventually remove
	// them from the collaboration storage.
	disconnectTimer *time.Timer

	// config stores all necessary configuration needed for Klient to work.
	// It's supplied with the NewKlient() function.
	config *KlientConfig
}

// KlientConfig defines a Klient's config
type KlientConfig struct {
	Name    string
	Version string

	IP          string
	Port        int
	Environment string
	Region      string
	RegisterURL string
	KontrolURL  string
	Debug       bool

	ScreenrcPath string
	DBPath       string

	UpdateInterval time.Duration
	UpdateURL      string
}

// NewKlient returns a new Klient instance
func NewKlient(conf *KlientConfig) *Klient {
	// this is our main reference to count and measure metrics for the klient
	// we count only those methods, please add/remove methods here that will
	// reset the timer of a klient.
	usg := usage.NewUsage(map[string]bool{
		"fs.readDirectory":     true,
		"fs.glob":              true,
		"fs.readFile":          true,
		"fs.writeFile":         true,
		"fs.uniquePath":        true,
		"fs.getInfo":           true,
		"fs.setPermissions":    true,
		"fs.remove":            true,
		"fs.rename":            true,
		"fs.createDirectory":   true,
		"fs.move":              true,
		"fs.copy":              true,
		"webterm.getSessions":  true,
		"webterm.connect":      true,
		"webterm.killSession":  true,
		"webterm.killSessions": true,
		"webterm.rename":       true,
		"exec":                 true,
		"klient.share":         true,
		"klient.unshare":       true,
		"klient.shared":        true,
		"sshkeys.List":         true,
		"sshkeys.Add":          true,
		"sshkeys.Delete":       true,
		"storage.Get":          true,
		"storage.Set":          true,
		"storage.Delete":       true,
		// "docker.create":       true,
		// "docker.connect":      true,
		// "docker.stop":         true,
		// "docker.start":        true,
		// "docker.remove":       true,
		// "docker.list":         true,
	})

	k := newKite(conf)
	term := terminal.New(k.Log, conf.ScreenrcPath)
	term.InputHook = usg.Reset

	db, err := openBoltDb(conf.DBPath)
	if err != nil {
		k.Log.Warning("Couldn't open BoltDB: %s", err)
	}

	kl := &Klient{
		kite:    k,
		collab:  collaboration.New(db), // nil is ok, fallbacks to in memory storage
		storage: storage.New(db),       // nil is ok, fallbacks to in memory storage
		// docker:   docker.New("unix://var/run/docker.sock", k.Log),
		terminal: term,
		usage:    usg,
		log:      k.Log,
		config:   conf,
	}

	// This is important, don't forget it
	kl.RegisterMethods()

	return kl
}

// RegisterMethods registers all public available methods
func (k *Klient) RegisterMethods() {
	// don't allow anyone to call a method if we are during an update.
	k.kite.PreHandleFunc(func(r *kite.Request) (interface{}, error) {
		// Koding (kloud) connects to much, don't display it.
		if r.Username != "koding" {
			k.log.Info("Kite '%s/%s/%s' called method: '%s'",
				r.Username, r.Client.Environment, r.Client.Name, r.Method)
		}

		updatingMu.Lock()
		defer updatingMu.Unlock()

		if updating {
			return nil, errors.New("Updating klient. Can't accept any method.")
		}

		return true, nil
	})

	k.kite.PreHandleFunc(k.checkAuth)

	// Metrics, is used by Kloud to get usage so Kloud can stop free VMs
	k.kite.PreHandleFunc(k.usage.Counter) // we measure every incoming request
	k.kite.HandleFunc("klient.usage", k.usage.Current)

	// Collaboration, is used by our Koding.com browser client.
	k.kite.HandleFunc("klient.share", k.collab.Share)
	k.kite.HandleFunc("klient.unshare", k.collab.Unshare)
	k.kite.HandleFunc("klient.shared", k.collab.Shared)

	// SSH keys
	k.kite.HandleFunc("sshkeys.List", sshkeys.List)
	k.kite.HandleFunc("sshkeys.Add", sshkeys.Add)
	k.kite.HandleFunc("sshkeys.Delete", sshkeys.Delete)

	// Storage
	k.kite.HandleFunc("storage.Set", k.storage.SetValue)
	k.kite.HandleFunc("storage.Get", k.storage.GetValue)
	k.kite.HandleFunc("storage.Delete", k.storage.DeleteValue)

	// Filesystem
	k.kite.HandleFunc("fs.readDirectory", fs.ReadDirectory)
	k.kite.HandleFunc("fs.glob", fs.Glob)
	k.kite.HandleFunc("fs.readFile", fs.ReadFile)
	k.kite.HandleFunc("fs.writeFile", fs.WriteFile)
	k.kite.HandleFunc("fs.uniquePath", fs.UniquePath)
	k.kite.HandleFunc("fs.getInfo", fs.GetInfo)
	k.kite.HandleFunc("fs.setPermissions", fs.SetPermissions)
	k.kite.HandleFunc("fs.remove", fs.Remove)
	k.kite.HandleFunc("fs.rename", fs.Rename)
	k.kite.HandleFunc("fs.createDirectory", fs.CreateDirectory)
	k.kite.HandleFunc("fs.move", fs.Move)
	k.kite.HandleFunc("fs.copy", fs.Copy)

	// Docker
	// k.kite.HandleFunc("docker.create", k.docker.Create)
	// k.kite.HandleFunc("docker.connect", k.docker.Connect)
	// k.kite.HandleFunc("docker.stop", k.docker.Stop)
	// k.kite.HandleFunc("docker.start", k.docker.Start)
	// k.kite.HandleFunc("docker.remove", k.docker.RemoveContainer)
	// k.kite.HandleFunc("docker.list", k.docker.List)

	// Execution
	k.kite.HandleFunc("exec", command.Exec)

	// Terminal
	k.kite.HandleFunc("webterm.getSessions", k.terminal.GetSessions)
	k.kite.HandleFunc("webterm.connect", k.terminal.Connect)
	k.kite.HandleFunc("webterm.killSession", k.terminal.KillSession)
	k.kite.HandleFunc("webterm.killSessions", k.terminal.KillSessions)
	k.kite.HandleFunc("webterm.rename", k.terminal.RenameSession)

	k.kite.OnFirstRequest(func(c *kite.Client) {
		// Koding (kloud) connects to much, don't display it.
		if c.Username != "koding" {
			k.log.Info("Kite '%s/%s/%s' is connected", c.Username, c.Environment, c.Name)
		}

		if c.Username != k.kite.Config.Username {
			return // we don't care for others
		}

		// it's still not initialized, so don't do anything
		if k.disconnectTimer != nil {
			// stop previously started disconnect timer.
			k.log.Info("Disconnection timer is cancelled.")
			k.disconnectTimer.Stop()
		}

	})

	// Unshare collab users if the klient owner disconnects
	k.kite.OnDisconnect(func(c *kite.Client) {
		// Koding (kloud) connects to much, don't display it.
		if c.Username != "koding" {
			k.log.Info("Kite '%s/%s/%s' is disconnected", c.Username, c.Environment, c.Name)
		}

		if c.Username != k.kite.Config.Username {
			return // we don't care for others
		}

		// if there is any previously created timers stop them so we don't leak
		// goroutines
		if k.disconnectTimer != nil {
			k.disconnectTimer.Stop()
		}

		k.log.Info("Disconnection timer of 1 minutes is fired.")
		k.disconnectTimer = time.NewTimer(time.Minute * 1)

		// Close all active sessions of the current. Do not close it
		// immediately, instead of give some time so users can safely exit. If
		// the user reconnects again the timer will be stopped so we don't
		// unshare for network hiccups accidentally.
		go func() {
			select {
			case <-k.disconnectTimer.C:
				sharedUsers, err := k.collab.GetAll()
				if err != nil {
					k.log.Warning("Couldn't unshare users: '%s'", err)
					return
				}

				if len(sharedUsers) == 0 {
					return // nothing to do ...
				}

				k.log.Info("Unsharing users '%s'", sharedUsers)
				for user, option := range sharedUsers {
					// dont touch permanent users
					if option.Permanent {
						k.log.Info("User is permanent, avoiding it: '%s'", user)
						continue
					}

					if err := k.collab.Delete(user); err != nil {
						k.log.Warning("Couldn't delete user from storage: '%s'", err)
					}
					k.terminal.CloseSessions(user)
				}
			}
		}()
	})
}

// Run registers klient to Kontrol and starts the kite server. It also runs any
// necessary workers in the background.
func (k *Klient) Run() {
	k.startUpdater()

	if err := k.register(); err != nil {
		panic(err)
	}

	k.kite.Run()
}

func (k *Klient) startUpdater() {
	if k.config.UpdateInterval < time.Minute {
		k.log.Warning("Update interval can't be less than one minute. Setting to one minute.")
		k.config.UpdateInterval = time.Minute
	}

	// start our updater in the background
	updater := &Updater{
		Endpoint:       k.config.UpdateURL,
		Interval:       k.config.UpdateInterval,
		CurrentVersion: k.config.Version,
		Log:            k.log,
	}
	go updater.Run()
}

func (k *Klient) Close() {
	k.collab.Close()
	k.kite.Close()
}

func newKite(kconf *KlientConfig) *kite.Kite {
	k := kite.New(kconf.Name, kconf.Version)

	if kconf.Debug {
		k.SetLogLevel(kite.DEBUG)
	}

	conf := config.MustGet()
	k.Config = conf
	k.Config.Port = kconf.Port
	k.Config.Environment = kconf.Environment
	k.Config.Region = kconf.Region
	k.Id = conf.Id // always boot up with the same id in the kite.key
	return k
}

// checkAuth checks whether the given incoming request is authenticated or not.
// It don't pass any request if the caller is outside of our scope.
func (k *Klient) checkAuth(r *kite.Request) (interface{}, error) {
	// only authenticated methods have correct username. For example
	// kite.ping has authentication disabled so username can be empty.
	if r.Auth == nil {
		return true, nil
	}

	// lazy return for those, no need to fetch from the DB
	if userIn(r.Username, []string{k.kite.Config.Username, "koding"}...) {
		return true, nil
	}

	// Allow collaboration users as well
	sharedUsers, err := k.collab.GetAll()
	if err != nil {
		return nil, fmt.Errorf("Can't read shared users from the storage. Err: %v", err)
	}

	sharedUsernames := make([]string, 0)
	for username := range sharedUsers {
		sharedUsernames = append(sharedUsernames, username)
	}

	if !userIn(r.Username, sharedUsernames...) {
		return nil, fmt.Errorf("User '%s' is not allowed to make a call to us.", r.Username)
	}

	return true, nil
}

func openBoltDb(dbpath string) (*bolt.DB, error) {
	if dbpath == "" {
		return nil, errors.New("DB path is empty")
	}

	// create if it doesn't exists
	if err := os.MkdirAll(filepath.Dir(dbpath), 0755); err != nil {
		return nil, err
	}

	return bolt.Open(dbpath, 0644, &bolt.Options{Timeout: 5 * time.Second})
}

// userIn checks whether the given user exists in the users list or not. It
// returns true if the user exists.
func userIn(user string, users ...string) bool {
	for _, u := range users {
		if u == user {
			return true
		}
	}
	return false
}
<commit_msg>app/klient: make methods lowercase<commit_after>package app

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/koding/klient/Godeps/_workspace/src/github.com/boltdb/bolt"
	"github.com/koding/klient/Godeps/_workspace/src/github.com/koding/kite"
	"github.com/koding/klient/Godeps/_workspace/src/github.com/koding/kite/config"
	"github.com/koding/klient/collaboration"
	"github.com/koding/klient/command"
	"github.com/koding/klient/fs"
	"github.com/koding/klient/sshkeys"
	"github.com/koding/klient/storage"
	"github.com/koding/klient/terminal"
	"github.com/koding/klient/usage"
)

var (
	// we also could use an atomic boolean this is simple for now.
	updating   = false
	updatingMu sync.Mutex // protects updating
)

// Klient is the central app which provides all available methods.
type Klient struct {
	// kite defines the server running and powering Klient.
	kite *kite.Kite

	// collab is used to store authorized third party users and allows them to
	// use the available methods. It provides methods to add or remove users
	// from the storage
	collab *collaboration.Collaboration

	storage *storage.Storage

	// terminal provides wmethods
	terminal *terminal.Terminal

	// docker provides the docker related methods.
	// docker *docker.Docker

	// usage counts and tracks all called metrics. It also provides a method
	// that return those informations
	usage *usage.Usage

	log kite.Logger

	// disconnectTimer is used track disconnected users and eventually remove
	// them from the collaboration storage.
	disconnectTimer *time.Timer

	// config stores all necessary configuration needed for Klient to work.
	// It's supplied with the NewKlient() function.
	config *KlientConfig
}

// KlientConfig defines a Klient's config
type KlientConfig struct {
	Name    string
	Version string

	IP          string
	Port        int
	Environment string
	Region      string
	RegisterURL string
	KontrolURL  string
	Debug       bool

	ScreenrcPath string
	DBPath       string

	UpdateInterval time.Duration
	UpdateURL      string
}

// NewKlient returns a new Klient instance
func NewKlient(conf *KlientConfig) *Klient {
	// this is our main reference to count and measure metrics for the klient
	// we count only those methods, please add/remove methods here that will
	// reset the timer of a klient.
	usg := usage.NewUsage(map[string]bool{
		"fs.readDirectory":     true,
		"fs.glob":              true,
		"fs.readFile":          true,
		"fs.writeFile":         true,
		"fs.uniquePath":        true,
		"fs.getInfo":           true,
		"fs.setPermissions":    true,
		"fs.remove":            true,
		"fs.rename":            true,
		"fs.createDirectory":   true,
		"fs.move":              true,
		"fs.copy":              true,
		"webterm.getSessions":  true,
		"webterm.connect":      true,
		"webterm.killSession":  true,
		"webterm.killSessions": true,
		"webterm.rename":       true,
		"exec":                 true,
		"klient.share":         true,
		"klient.unshare":       true,
		"klient.shared":        true,
		"sshkeys.List":         true,
		"sshkeys.Add":          true,
		"sshkeys.Delete":       true,
		"storage.Get":          true,
		"storage.Set":          true,
		"storage.Delete":       true,
		// "docker.create":       true,
		// "docker.connect":      true,
		// "docker.stop":         true,
		// "docker.start":        true,
		// "docker.remove":       true,
		// "docker.list":         true,
	})

	k := newKite(conf)
	term := terminal.New(k.Log, conf.ScreenrcPath)
	term.InputHook = usg.Reset

	db, err := openBoltDb(conf.DBPath)
	if err != nil {
		k.Log.Warning("Couldn't open BoltDB: %s", err)
	}

	kl := &Klient{
		kite:    k,
		collab:  collaboration.New(db), // nil is ok, fallbacks to in memory storage
		storage: storage.New(db),       // nil is ok, fallbacks to in memory storage
		// docker:   docker.New("unix://var/run/docker.sock", k.Log),
		terminal: term,
		usage:    usg,
		log:      k.Log,
		config:   conf,
	}

	// This is important, don't forget it
	kl.RegisterMethods()

	return kl
}

// RegisterMethods registers all public available methods
func (k *Klient) RegisterMethods() {
	// don't allow anyone to call a method if we are during an update.
	k.kite.PreHandleFunc(func(r *kite.Request) (interface{}, error) {
		// Koding (kloud) connects to much, don't display it.
		if r.Username != "koding" {
			k.log.Info("Kite '%s/%s/%s' called method: '%s'",
				r.Username, r.Client.Environment, r.Client.Name, r.Method)
		}

		updatingMu.Lock()
		defer updatingMu.Unlock()

		if updating {
			return nil, errors.New("Updating klient. Can't accept any method.")
		}

		return true, nil
	})

	k.kite.PreHandleFunc(k.checkAuth)

	// Metrics, is used by Kloud to get usage so Kloud can stop free VMs
	k.kite.PreHandleFunc(k.usage.Counter) // we measure every incoming request
	k.kite.HandleFunc("klient.usage", k.usage.Current)

	// Collaboration, is used by our Koding.com browser client.
	k.kite.HandleFunc("klient.share", k.collab.Share)
	k.kite.HandleFunc("klient.unshare", k.collab.Unshare)
	k.kite.HandleFunc("klient.shared", k.collab.Shared)

	// SSH keys
	k.kite.HandleFunc("sshkeys.list", sshkeys.List)
	k.kite.HandleFunc("sshkeys.add", sshkeys.Add)
	k.kite.HandleFunc("sshkeys.delete", sshkeys.Delete)

	// Storage
	k.kite.HandleFunc("storage.set", k.storage.SetValue)
	k.kite.HandleFunc("storage.get", k.storage.GetValue)
	k.kite.HandleFunc("storage.delete", k.storage.DeleteValue)

	// Filesystem
	k.kite.HandleFunc("fs.readDirectory", fs.ReadDirectory)
	k.kite.HandleFunc("fs.glob", fs.Glob)
	k.kite.HandleFunc("fs.readFile", fs.ReadFile)
	k.kite.HandleFunc("fs.writeFile", fs.WriteFile)
	k.kite.HandleFunc("fs.uniquePath", fs.UniquePath)
	k.kite.HandleFunc("fs.getInfo", fs.GetInfo)
	k.kite.HandleFunc("fs.setPermissions", fs.SetPermissions)
	k.kite.HandleFunc("fs.remove", fs.Remove)
	k.kite.HandleFunc("fs.rename", fs.Rename)
	k.kite.HandleFunc("fs.createDirectory", fs.CreateDirectory)
	k.kite.HandleFunc("fs.move", fs.Move)
	k.kite.HandleFunc("fs.copy", fs.Copy)

	// Docker
	// k.kite.HandleFunc("docker.create", k.docker.Create)
	// k.kite.HandleFunc("docker.connect", k.docker.Connect)
	// k.kite.HandleFunc("docker.stop", k.docker.Stop)
	// k.kite.HandleFunc("docker.start", k.docker.Start)
	// k.kite.HandleFunc("docker.remove", k.docker.RemoveContainer)
	// k.kite.HandleFunc("docker.list", k.docker.List)

	// Execution
	k.kite.HandleFunc("exec", command.Exec)

	// Terminal
	k.kite.HandleFunc("webterm.getSessions", k.terminal.GetSessions)
	k.kite.HandleFunc("webterm.connect", k.terminal.Connect)
	k.kite.HandleFunc("webterm.killSession", k.terminal.KillSession)
	k.kite.HandleFunc("webterm.killSessions", k.terminal.KillSessions)
	k.kite.HandleFunc("webterm.rename", k.terminal.RenameSession)

	k.kite.OnFirstRequest(func(c *kite.Client) {
		// Koding (kloud) connects to much, don't display it.
		if c.Username != "koding" {
			k.log.Info("Kite '%s/%s/%s' is connected", c.Username, c.Environment, c.Name)
		}

		if c.Username != k.kite.Config.Username {
			return // we don't care for others
		}

		// it's still not initialized, so don't do anything
		if k.disconnectTimer != nil {
			// stop previously started disconnect timer.
			k.log.Info("Disconnection timer is cancelled.")
			k.disconnectTimer.Stop()
		}

	})

	// Unshare collab users if the klient owner disconnects
	k.kite.OnDisconnect(func(c *kite.Client) {
		// Koding (kloud) connects to much, don't display it.
		if c.Username != "koding" {
			k.log.Info("Kite '%s/%s/%s' is disconnected", c.Username, c.Environment, c.Name)
		}

		if c.Username != k.kite.Config.Username {
			return // we don't care for others
		}

		// if there is any previously created timers stop them so we don't leak
		// goroutines
		if k.disconnectTimer != nil {
			k.disconnectTimer.Stop()
		}

		k.log.Info("Disconnection timer of 1 minutes is fired.")
		k.disconnectTimer = time.NewTimer(time.Minute * 1)

		// Close all active sessions of the current. Do not close it
		// immediately, instead of give some time so users can safely exit. If
		// the user reconnects again the timer will be stopped so we don't
		// unshare for network hiccups accidentally.
		go func() {
			select {
			case <-k.disconnectTimer.C:
				sharedUsers, err := k.collab.GetAll()
				if err != nil {
					k.log.Warning("Couldn't unshare users: '%s'", err)
					return
				}

				if len(sharedUsers) == 0 {
					return // nothing to do ...
				}

				k.log.Info("Unsharing users '%s'", sharedUsers)
				for user, option := range sharedUsers {
					// dont touch permanent users
					if option.Permanent {
						k.log.Info("User is permanent, avoiding it: '%s'", user)
						continue
					}

					if err := k.collab.Delete(user); err != nil {
						k.log.Warning("Couldn't delete user from storage: '%s'", err)
					}
					k.terminal.CloseSessions(user)
				}
			}
		}()
	})
}

// Run registers klient to Kontrol and starts the kite server. It also runs any
// necessary workers in the background.
func (k *Klient) Run() {
	k.startUpdater()

	if err := k.register(); err != nil {
		panic(err)
	}

	k.kite.Run()
}

func (k *Klient) startUpdater() {
	if k.config.UpdateInterval < time.Minute {
		k.log.Warning("Update interval can't be less than one minute. Setting to one minute.")
		k.config.UpdateInterval = time.Minute
	}

	// start our updater in the background
	updater := &Updater{
		Endpoint:       k.config.UpdateURL,
		Interval:       k.config.UpdateInterval,
		CurrentVersion: k.config.Version,
		Log:            k.log,
	}
	go updater.Run()
}

func (k *Klient) Close() {
	k.collab.Close()
	k.kite.Close()
}

func newKite(kconf *KlientConfig) *kite.Kite {
	k := kite.New(kconf.Name, kconf.Version)

	if kconf.Debug {
		k.SetLogLevel(kite.DEBUG)
	}

	conf := config.MustGet()
	k.Config = conf
	k.Config.Port = kconf.Port
	k.Config.Environment = kconf.Environment
	k.Config.Region = kconf.Region
	k.Id = conf.Id // always boot up with the same id in the kite.key
	return k
}

// checkAuth checks whether the given incoming request is authenticated or not.
// It don't pass any request if the caller is outside of our scope.
func (k *Klient) checkAuth(r *kite.Request) (interface{}, error) {
	// only authenticated methods have correct username. For example
	// kite.ping has authentication disabled so username can be empty.
	if r.Auth == nil {
		return true, nil
	}

	// lazy return for those, no need to fetch from the DB
	if userIn(r.Username, []string{k.kite.Config.Username, "koding"}...) {
		return true, nil
	}

	// Allow collaboration users as well
	sharedUsers, err := k.collab.GetAll()
	if err != nil {
		return nil, fmt.Errorf("Can't read shared users from the storage. Err: %v", err)
	}

	sharedUsernames := make([]string, 0)
	for username := range sharedUsers {
		sharedUsernames = append(sharedUsernames, username)
	}

	if !userIn(r.Username, sharedUsernames...) {
		return nil, fmt.Errorf("User '%s' is not allowed to make a call to us.", r.Username)
	}

	return true, nil
}

func openBoltDb(dbpath string) (*bolt.DB, error) {
	if dbpath == "" {
		return nil, errors.New("DB path is empty")
	}

	// create if it doesn't exists
	if err := os.MkdirAll(filepath.Dir(dbpath), 0755); err != nil {
		return nil, err
	}

	return bolt.Open(dbpath, 0644, &bolt.Options{Timeout: 5 * time.Second})
}

// userIn checks whether the given user exists in the users list or not. It
// returns true if the user exists.
func userIn(user string, users ...string) bool {
	for _, u := range users {
		if u == user {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/camd67/moebot/moebot_bot/bot/permissions"
	"github.com/camd67/moebot/moebot_bot/util"
	"github.com/camd67/moebot/moebot_bot/util/db"
)

type TimerCommand struct {
	chTimers util.SyncChannelTimerMap
	Checker  permissions.PermissionChecker
}

func NewTimerCommand() *TimerCommand {
	tc := &TimerCommand{}
	tc.chTimers = util.SyncChannelTimerMap{
		RWMutex: sync.RWMutex{},
		M:       make(map[string]time.Time),
	}
	return tc
}

func (tc *TimerCommand) Execute(pack *CommPackage) {
	channelID := pack.message.ChannelID
	if len(pack.params) > 0 && strings.EqualFold(pack.params[0], "start") {
		if tc.Checker.HasPermission(pack.message.Author.ID, pack.member.Roles, pack.guild, db.PermMod) {
			tc.chTimers.Lock()
			tc.chTimers.M[channelID] = time.Now()
			tc.chTimers.Unlock()
			pack.session.ChannelMessageSend(pack.message.ChannelID, "Timer started!")
		} else {
			pack.session.ChannelMessageSend(pack.message.ChannelID, pack.message.Author.Mention()+", you... you don't have permission to do that!")
		}
	} else {
		tc.chTimers.RLock()
		if v, ok := tc.chTimers.M[channelID]; ok {
			pack.session.ChannelMessageSend(pack.message.ChannelID, fmtDuration(time.Since(v)))
		} else {
			pack.session.ChannelMessageSend(pack.message.ChannelID, "No timer started for this channel...")
		}
		tc.chTimers.RUnlock()
	}
}

// fmtDuration formats a duration into a hh:mm:ss format
func fmtDuration(dur time.Duration) string {
	remainingDur := dur.Round(time.Second)
	hours := remainingDur / time.Hour
	remainingDur -= hours * time.Hour
	minutes := remainingDur / time.Minute
	remainingDur -= minutes * time.Minute
	seconds := remainingDur / time.Second
	return fmt.Sprintf("%02d:%02d:%02d", hours, minutes, seconds)
}

func (tc *TimerCommand) GetPermLevel() db.Permission {
	return db.PermAll
}

func (tc *TimerCommand) GetCommandKeys() []string {
	return []string{"TIMER"}
}

func (tc *TimerCommand) GetCommandHelp(commPrefix string) string {
	return fmt.Sprintf("`%[1]s timer` - Checks the timestamp. Moderators may provide the `--start` option to begin start or restart the timer.", commPrefix)
}
<commit_msg>Add a basic extended timer that will write forever<commit_after>package commands

import (
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/camd67/moebot/moebot_bot/bot/permissions"
	"github.com/camd67/moebot/moebot_bot/util"
	"github.com/camd67/moebot/moebot_bot/util/db"
)

type TimerCommand struct {
	chTimers util.SyncChannelTimerMap
	Checker  permissions.PermissionChecker
}

func NewTimerCommand() *TimerCommand {
	tc := &TimerCommand{}
	tc.chTimers = util.SyncChannelTimerMap{
		RWMutex: sync.RWMutex{},
		M:       make(map[string]time.Time),
	}
	return tc
}

func (tc *TimerCommand) Execute(pack *CommPackage) {
	channelID := pack.message.ChannelID
	if len(pack.params) > 0 && strings.EqualFold(pack.params[0], "start") {
		if tc.Checker.HasPermission(pack.message.Author.ID, pack.member.Roles, pack.guild, db.PermMod) {
			tc.chTimers.Lock()
			tc.chTimers.M[channelID] = time.Now()
			tc.chTimers.Unlock()
			pack.session.ChannelMessageSend(pack.message.ChannelID, "Timer started!")
		} else {
			pack.session.ChannelMessageSend(pack.message.ChannelID, pack.message.Author.Mention()+", you... you don't have permission to do that!")
		}
	} else {
		tc.chTimers.RLock()
		if v, ok := tc.chTimers.M[channelID]; ok {
			go tc.writeTimesToChannel(pack, v)
		} else {
			pack.session.ChannelMessageSend(pack.message.ChannelID, "No timer started for this channel...")
		}
		tc.chTimers.RUnlock()
	}
}

func (tc *TimerCommand) writeTimesToChannel(pack *CommPackage, startTime time.Time) {
	//Write the time once right away
	pack.session.ChannelMessageSend(pack.message.ChannelID, fmtDuration(time.Since(startTime)))

	addedSeconds := 0
	for {
		select {
		case <-time.After(time.Second * 1):
			// Increment the duration by one second, post the time
			addedSeconds++
			// pack.session.ChannelMessageSend(pack.message.ChannelID, fmtDuration(time.Since(startTime.Add(time.Second*time.Duration(addedSeconds)))))
			pack.session.ChannelMessageSend(pack.message.ChannelID, fmtDuration(time.Since(startTime)+time.Duration(addedSeconds)))
		}
	}
}

// fmtDuration formats a duration into a hh:mm:ss format
func fmtDuration(dur time.Duration) string {
	remainingDur := dur.Round(time.Second)
	hours := remainingDur / time.Hour
	remainingDur -= hours * time.Hour
	minutes := remainingDur / time.Minute
	remainingDur -= minutes * time.Minute
	seconds := remainingDur / time.Second
	return fmt.Sprintf("%02d:%02d:%02d", hours, minutes, seconds)
}

func (tc *TimerCommand) GetPermLevel() db.Permission {
	return db.PermAll
}

func (tc *TimerCommand) GetCommandKeys() []string {
	return []string{"TIMER"}
}

func (tc *TimerCommand) GetCommandHelp(commPrefix string) string {
	return fmt.Sprintf("`%[1]s timer` - Checks the timestamp. Moderators may provide the `--start` option to begin start or restart the timer.", commPrefix)
}
<|endoftext|>


<commit_before>package flying_test

import (
	"net/http"
	"os"

	"github.com/concourse/testflight/bosh"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"

	"testing"
	"time"
)

var flyBin string

var _ = BeforeSuite(func() {
	Ω(os.Getenv("BOSH_LITE_IP")).ShouldNot(BeEmpty(), "must specify $BOSH_LITE_IP")

	var err error

	flyBin, err = gexec.Build("github.com/concourse/fly", "-race")
	Ω(err).ShouldNot(HaveOccurred())

	bosh.DeleteDeployment("concourse")

	bosh.Deploy("noop.yml")

	atcURL := "http://" + os.Getenv("BOSH_LITE_IP") + ":8080"

	os.Setenv("ATC_URL", atcURL)

	Eventually(func() error {
		resp, err := http.Get(atcURL)
		if err == nil {
			resp.Body.Close()
		}

		return err
	}, 1*time.Minute).ShouldNot(HaveOccurred())
})

func TestFlying(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "Flying Suite")
}
<commit_msg>warm testflight-helper cache<commit_after>package flying_test

import (
	"net/http"
	"os"

	gapi "github.com/cloudfoundry-incubator/garden/api"
	"github.com/cloudfoundry-incubator/garden/client"
	"github.com/cloudfoundry-incubator/garden/client/connection"
	"github.com/concourse/testflight/bosh"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"

	"testing"
	"time"
)

var flyBin string

var _ = BeforeSuite(func() {
	Ω(os.Getenv("BOSH_LITE_IP")).ShouldNot(BeEmpty(), "must specify $BOSH_LITE_IP")

	var err error

	flyBin, err = gexec.Build("github.com/concourse/fly", "-race")
	Ω(err).ShouldNot(HaveOccurred())

	bosh.DeleteDeployment("concourse")

	bosh.Deploy("noop.yml")

	atcURL := "http://" + os.Getenv("BOSH_LITE_IP") + ":8080"

	os.Setenv("ATC_URL", atcURL)

	Eventually(func() error {
		resp, err := http.Get(atcURL)
		if err == nil {
			resp.Body.Close()
		}

		return err
	}, 1*time.Minute).ShouldNot(HaveOccurred())

	gardenClient := client.New(connection.New("tcp", os.Getenv("BOSH_LITE_IP")+":7777"))
	Eventually(gardenClient.Ping, 10*time.Second).ShouldNot(HaveOccurred())

	// warm cache with testflight-helper image so flying doesn't take forever
	container, err := gardenClient.Create(gapi.ContainerSpec{
		RootFSPath: "docker:///concourse/testflight-helper",
	})
	Ω(err).ShouldNot(HaveOccurred())

	err = gardenClient.Destroy(container.Handle())
	Ω(err).ShouldNot(HaveOccurred())
})

func TestFlying(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "Flying Suite")
}
<|endoftext|>


<commit_before>/**
 * @file main.go
 * @author Mikhail Klementyev jollheef<AT>riseup.net
 * @license GNU AGPLv3
 * @date December, 2015
 * @brief contest checking system CLI
 *
 * Entry point for contest checking system CLI
 */

package main

import (
	"database/sql"
	"encoding/xml"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"sort"

	"github.com/jollheef/henhouse/config"
	"github.com/jollheef/henhouse/db"
	"github.com/olekukonko/tablewriter"
	kingpin "gopkg.in/alecthomas/kingpin.v2"
)

var (
	configPath = kingpin.Flag("config", "Path to configuration file.").String()

	// Task
	task = kingpin.Command("task", "Work with tasks.")

	taskList = task.Command("list", "List tasks.")

	taskUpdate    = task.Command("update", "Update task.")
	taskUpdateID  = taskUpdate.Arg("id", "ID of task.").Required().Int()
	taskUpdateXML = taskUpdate.Arg("xml", "Path to xml.").Required().String()

	taskOpen   = task.Command("open", "Open task.")
	taskOpenID = taskOpen.Arg("id", "ID of task").Required().Int()

	taskClose   = task.Command("close", "Close task.")
	taskCloseID = taskClose.Arg("id", "ID of task").Required().Int()

	taskDump   = task.Command("dump", "Dump task to xml.")
	taskDumpID = taskDump.Arg("id", "ID of task").Required().Int()

	// Category
	category = kingpin.Command("category", "Work with categories.")

	categoryList = category.Command("list", "List categories.")

	categoryAdd  = category.Command("add", "Add category.")
	categoryName = categoryAdd.Arg("name", "Name.").Required().String()

	// Team
	team = kingpin.Command("team", "Work with teams.")

	teamList = team.Command("list", "List teams.")
)

var (
	// CommitID fill in ./build.sh
	CommitID string
	// BuildDate fill in ./build.sh
	BuildDate string
	// BuildTime fill in ./build.sh
	BuildTime string
)

func getCategoryByID(categoryID int, categories []db.Category) string {
	for _, cat := range categories {
		if cat.ID == categoryID {
			return cat.Name
		}
	}
	return "Unknown"
}

func getCategoryByName(name string, categories []db.Category) (id int, err error) {
	for _, cat := range categories {
		if cat.Name == name {
			return cat.ID, nil
		}
	}

	return 0, errors.New("Category " + name + " not found")
}

func taskRow(task db.Task, categories []db.Category) (row []string) {
	row = append(row, fmt.Sprintf("%d", task.ID))
	row = append(row, task.Name)
	row = append(row, getCategoryByID(task.CategoryID, categories))
	row = append(row, task.Flag)
	row = append(row, fmt.Sprintf("%v", task.Opened))
	return
}

type byID []db.Task

func (t byID) Len() int           { return len(t) }
func (t byID) Swap(i, j int)      { t[i], t[j] = t[j], t[i] }
func (t byID) Less(i, j int) bool { return t[i].ID < t[j].ID }

func parseTask(path string, categories []db.Category) (t db.Task, err error) {

	content, err := ioutil.ReadFile(path)
	if err != nil {
		return
	}

	task, err := config.ParseXMLTask(content)
	if err != nil {
		return
	}

	t.Name = task.Name
	t.Desc = task.Description
	t.NameEn = task.NameEn
	t.DescEn = task.DescriptionEn
	t.CategoryID, err = getCategoryByName(task.Category, categories)
	if err != nil {
		return
	}

	t.Level = task.Level
	t.Flag = task.Flag
	t.Price = 500         // TODO support non-shared task
	t.Shared = true       // TODO support non-shared task
	t.MaxSharePrice = 500 // TODO support value from xml
	t.MinSharePrice = 100 // TODO support value from xml
	t.Opened = false      // by default task is closed
	t.Author = task.Author
	t.ForceClosed = task.ForceClosed

	return
}

var cfgFiles = []string{"/etc/henhouse/cli.toml", "/etc/henhouse.toml",
	"cli.toml", "henhouse.toml"}

func taskUpdateCmd(database *sql.DB, categories []db.Category) (err error) {
	task, err := db.GetTask(database, *taskUpdateID)
	if err != nil {
		return
	}

	id := task.ID

	task, err = parseTask(*taskUpdateXML, categories)
	if err != nil {
		return
	}

	task.ID = id

	err = db.UpdateTask(database, &task)
	if err != nil {
		return
	}

	return
}

func taskListCmd(database *sql.DB, categories []db.Category) (err error) {
	tasks, err := db.GetTasks(database)
	if err != nil {
		return
	}

	sort.Sort(byID(tasks))

	table := tablewriter.NewWriter(os.Stdout)
	header := []string{"ID", "Name", "Category", "Flag", "Opened"}
	table.SetHeader(header)

	for _, task := range tasks {
		table.Append(taskRow(task, categories))
	}

	table.Render()

	return
}

func taskDumpCmd(database *sql.DB, categories []db.Category) (err error) {
	task, err := db.GetTask(database, *taskDumpID)
	if err != nil {
		return
	}

	xmlTask := config.Task{
		Name:          task.Name,
		NameEn:        task.NameEn,
		Description:   task.Desc,
		DescriptionEn: task.DescEn,
		Category:      getCategoryByID(task.CategoryID, categories),
		Level:         task.Level,
		Flag:          task.Flag,
		Author:        task.Author,
		ForceClosed:   task.ForceClosed,
	}

	output, err := xml.MarshalIndent(xmlTask, "", "	")
	if err != nil {
		return

	}

	fmt.Fprintln(os.Stdout, string(output))

	return
}

func categoryListCmd(database *sql.DB) (err error) {
	categories, err := db.GetCategories(database)
	if err != nil {
		return
	}

	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{"ID", "Name"})

	for _, cat := range categories {
		row := []string{fmt.Sprintf("%d", cat.ID), cat.Name}
		table.Append(row)
	}

	table.Render()

	return
}

func teamListCmd(database *sql.DB, categories []db.Category) (err error) {
	teams, err := db.GetTeams(database)
	if err != nil {
		return
	}

	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{"ID", "Name"})

	for _, t := range teams {
		row := []string{fmt.Sprintf("%d", t.ID), t.Name}
		table.Append(row)
	}

	table.Render()

	return
}

func runCommandLine(database *sql.DB, categories []db.Category) (err error) {
	switch kingpin.Parse() {
	case "task update":
		err = taskUpdateCmd(database, categories)
	case "task list":
		err = taskListCmd(database, categories)
	case "task open":
		err = db.SetOpened(database, *taskOpenID, true)
	case "task close":
		err = db.SetOpened(database, *taskCloseID, false)
	case "task dump":
		err = taskDumpCmd(database, categories)
	case "category add":
		err = db.AddCategory(database, &db.Category{Name: *categoryName})
	case "category list":
		err = categoryListCmd(database)
	case "team list":
		err = teamListCmd(database, categories)
	}

	return
}

func main() {

	if len(CommitID) > 7 {
		CommitID = CommitID[:7] // abbreviated commit hash
	}

	kingpin.Version(BuildDate + " " + CommitID +
		" (Mikhail Klementyev <jollheef@riseup.net>)")

	kingpin.Parse()

	var cfgPath string

	if *configPath != "" {
		cfgPath = *configPath
	} else {

		for _, cfgFile := range cfgFiles {
			_, err := os.Stat(cfgFile)
			if err == nil {
				cfgPath = cfgFile
				break
			}
		}
	}

	if cfgPath == "" {
		log.Fatalln("Config not found")
	}

	cfg, err := config.ReadConfig(cfgPath)
	if err != nil {
		log.Fatalln("Cannot open config:", err)
	}

	database, err := db.OpenDatabase(cfg.Database.Connection)
	if err != nil {
		log.Fatalln("Error:", err)
	}

	defer database.Close()

	database.SetMaxOpenConns(cfg.Database.MaxConnections)

	categories, err := db.GetCategories(database)
	if err != nil {
		log.Fatalln("Error:", err)
	}

	err = runCommandLine(database, categories)
	if err != nil {
		log.Fatalln("Error:", err)
	}
}
<commit_msg>Dump tags too<commit_after>/**
 * @file main.go
 * @author Mikhail Klementyev jollheef<AT>riseup.net
 * @license GNU AGPLv3
 * @date December, 2015
 * @brief contest checking system CLI
 *
 * Entry point for contest checking system CLI
 */

package main

import (
	"database/sql"
	"encoding/xml"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"sort"

	"github.com/jollheef/henhouse/config"
	"github.com/jollheef/henhouse/db"
	"github.com/olekukonko/tablewriter"
	kingpin "gopkg.in/alecthomas/kingpin.v2"
)

var (
	configPath = kingpin.Flag("config", "Path to configuration file.").String()

	// Task
	task = kingpin.Command("task", "Work with tasks.")

	taskList = task.Command("list", "List tasks.")

	taskUpdate    = task.Command("update", "Update task.")
	taskUpdateID  = taskUpdate.Arg("id", "ID of task.").Required().Int()
	taskUpdateXML = taskUpdate.Arg("xml", "Path to xml.").Required().String()

	taskOpen   = task.Command("open", "Open task.")
	taskOpenID = taskOpen.Arg("id", "ID of task").Required().Int()

	taskClose   = task.Command("close", "Close task.")
	taskCloseID = taskClose.Arg("id", "ID of task").Required().Int()

	taskDump   = task.Command("dump", "Dump task to xml.")
	taskDumpID = taskDump.Arg("id", "ID of task").Required().Int()

	// Category
	category = kingpin.Command("category", "Work with categories.")

	categoryList = category.Command("list", "List categories.")

	categoryAdd  = category.Command("add", "Add category.")
	categoryName = categoryAdd.Arg("name", "Name.").Required().String()

	// Team
	team = kingpin.Command("team", "Work with teams.")

	teamList = team.Command("list", "List teams.")
)

var (
	// CommitID fill in ./build.sh
	CommitID string
	// BuildDate fill in ./build.sh
	BuildDate string
	// BuildTime fill in ./build.sh
	BuildTime string
)

func getCategoryByID(categoryID int, categories []db.Category) string {
	for _, cat := range categories {
		if cat.ID == categoryID {
			return cat.Name
		}
	}
	return "Unknown"
}

func getCategoryByName(name string, categories []db.Category) (id int, err error) {
	for _, cat := range categories {
		if cat.Name == name {
			return cat.ID, nil
		}
	}

	return 0, errors.New("Category " + name + " not found")
}

func taskRow(task db.Task, categories []db.Category) (row []string) {
	row = append(row, fmt.Sprintf("%d", task.ID))
	row = append(row, task.Name)
	row = append(row, getCategoryByID(task.CategoryID, categories))
	row = append(row, task.Flag)
	row = append(row, fmt.Sprintf("%v", task.Opened))
	return
}

type byID []db.Task

func (t byID) Len() int           { return len(t) }
func (t byID) Swap(i, j int)      { t[i], t[j] = t[j], t[i] }
func (t byID) Less(i, j int) bool { return t[i].ID < t[j].ID }

func parseTask(path string, categories []db.Category) (t db.Task, err error) {

	content, err := ioutil.ReadFile(path)
	if err != nil {
		return
	}

	task, err := config.ParseXMLTask(content)
	if err != nil {
		return
	}

	t.Name = task.Name
	t.Desc = task.Description
	t.NameEn = task.NameEn
	t.DescEn = task.DescriptionEn
	t.CategoryID, err = getCategoryByName(task.Category, categories)
	if err != nil {
		return
	}

	t.Level = task.Level
	t.Flag = task.Flag
	t.Price = 500         // TODO support non-shared task
	t.Shared = true       // TODO support non-shared task
	t.MaxSharePrice = 500 // TODO support value from xml
	t.MinSharePrice = 100 // TODO support value from xml
	t.Opened = false      // by default task is closed
	t.Author = task.Author
	t.Tags = task.Tags
	t.ForceClosed = task.ForceClosed

	return
}

var cfgFiles = []string{"/etc/henhouse/cli.toml", "/etc/henhouse.toml",
	"cli.toml", "henhouse.toml"}

func taskUpdateCmd(database *sql.DB, categories []db.Category) (err error) {
	task, err := db.GetTask(database, *taskUpdateID)
	if err != nil {
		return
	}

	id := task.ID

	task, err = parseTask(*taskUpdateXML, categories)
	if err != nil {
		return
	}

	task.ID = id

	err = db.UpdateTask(database, &task)
	if err != nil {
		return
	}

	return
}

func taskListCmd(database *sql.DB, categories []db.Category) (err error) {
	tasks, err := db.GetTasks(database)
	if err != nil {
		return
	}

	sort.Sort(byID(tasks))

	table := tablewriter.NewWriter(os.Stdout)
	header := []string{"ID", "Name", "Category", "Flag", "Opened"}
	table.SetHeader(header)

	for _, task := range tasks {
		table.Append(taskRow(task, categories))
	}

	table.Render()

	return
}

func taskDumpCmd(database *sql.DB, categories []db.Category) (err error) {
	task, err := db.GetTask(database, *taskDumpID)
	if err != nil {
		return
	}

	xmlTask := config.Task{
		Name:          task.Name,
		NameEn:        task.NameEn,
		Description:   task.Desc,
		DescriptionEn: task.DescEn,
		Category:      getCategoryByID(task.CategoryID, categories),
		Level:         task.Level,
		Flag:          task.Flag,
		Author:        task.Author,
		ForceClosed:   task.ForceClosed,
	}

	output, err := xml.MarshalIndent(xmlTask, "", "	")
	if err != nil {
		return

	}

	fmt.Fprintln(os.Stdout, string(output))

	return
}

func categoryListCmd(database *sql.DB) (err error) {
	categories, err := db.GetCategories(database)
	if err != nil {
		return
	}

	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{"ID", "Name"})

	for _, cat := range categories {
		row := []string{fmt.Sprintf("%d", cat.ID), cat.Name}
		table.Append(row)
	}

	table.Render()

	return
}

func teamListCmd(database *sql.DB, categories []db.Category) (err error) {
	teams, err := db.GetTeams(database)
	if err != nil {
		return
	}

	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{"ID", "Name"})

	for _, t := range teams {
		row := []string{fmt.Sprintf("%d", t.ID), t.Name}
		table.Append(row)
	}

	table.Render()

	return
}

func runCommandLine(database *sql.DB, categories []db.Category) (err error) {
	switch kingpin.Parse() {
	case "task update":
		err = taskUpdateCmd(database, categories)
	case "task list":
		err = taskListCmd(database, categories)
	case "task open":
		err = db.SetOpened(database, *taskOpenID, true)
	case "task close":
		err = db.SetOpened(database, *taskCloseID, false)
	case "task dump":
		err = taskDumpCmd(database, categories)
	case "category add":
		err = db.AddCategory(database, &db.Category{Name: *categoryName})
	case "category list":
		err = categoryListCmd(database)
	case "team list":
		err = teamListCmd(database, categories)
	}

	return
}

func main() {

	if len(CommitID) > 7 {
		CommitID = CommitID[:7] // abbreviated commit hash
	}

	kingpin.Version(BuildDate + " " + CommitID +
		" (Mikhail Klementyev <jollheef@riseup.net>)")

	kingpin.Parse()

	var cfgPath string

	if *configPath != "" {
		cfgPath = *configPath
	} else {

		for _, cfgFile := range cfgFiles {
			_, err := os.Stat(cfgFile)
			if err == nil {
				cfgPath = cfgFile
				break
			}
		}
	}

	if cfgPath == "" {
		log.Fatalln("Config not found")
	}

	cfg, err := config.ReadConfig(cfgPath)
	if err != nil {
		log.Fatalln("Cannot open config:", err)
	}

	database, err := db.OpenDatabase(cfg.Database.Connection)
	if err != nil {
		log.Fatalln("Error:", err)
	}

	defer database.Close()

	database.SetMaxOpenConns(cfg.Database.MaxConnections)

	categories, err := db.GetCategories(database)
	if err != nil {
		log.Fatalln("Error:", err)
	}

	err = runCommandLine(database, categories)
	if err != nil {
		log.Fatalln("Error:", err)
	}
}
<|endoftext|>


<commit_before>/*
 * Copyright 2013 Nan Deng
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package proto

import (
	"io"
	"hash"
	"crypto/aes"
	"crypto/hmac"
	"crypto/sha256"
	"crypto/cipher"
	"code.google.com/p/snappy-go/snappy"
	"labix.org/v2/mgo/bson"
	"encoding/binary"
	"net"
)

type commandIO struct {
	writeAuth hash.Hash
	cryptWriter io.Writer

	readAuth hash.Hash
	cryptReader io.Reader

	conn net.Conn
}

func (self *commandIO) writeThenHmac(data []byte) (mac []byte, err error) {
	self.writeAuth.Reset()
	err = writen(self.cryptWriter, data)
	if err != nil {
		return
	}
	mac = self.writeAuth.Sum(nil)
	return
}

func (self *commandIO) readThenHmac(data []byte) (mac []byte, err error) {
	self.readAuth.Reset()
	n, err := io.ReadFull(self.cryptReader, data)
	if err != nil {
		return
	}
	if n != len(data) {
		err = io.EOF
		return
	}
	mac = self.readAuth.Sum(nil)
	return
}

func cmpHmac(a, b []byte) bool {
	if len(a) != len(b) {
		return false
	}
	for i, c := range a {
		if c != b[i] {
			return false
		}
	}
	return true
}

func (self *commandIO) writeHmac(mac []byte) error {
	return writen(self.conn, mac)
}

func (self *commandIO) readAndCmpHmac(mac []byte) error {
	macRecved := make([]byte, self.readAuth.BlockSize())
	n, err := io.ReadFull(self.conn, macRecved)
	if err != nil {
		return err
	}
	if n != len(macRecved) {
		return ErrCorruptedData
	}
	if !cmpHmac(mac, macRecved) {
		return ErrCorruptedData
	}
	return nil
}

func (self *commandIO) readEncodedMessage(data []byte) (cmd *command, err error) {
	decoded, err := snappy.Decode(nil, data)
	if err != nil {
		return
	}
	cmd = new(command)
	err = bson.Unmarshal(decoded, cmd)
	if err != nil {
		return
	}
	return
}

func (self *commandIO) encodeCommand(cmd *command)(data []byte, err error) {
	bsonEncoded, err := bson.Marshal(cmd)
	if err != nil {
		return
	}
	data, err = snappy.Encode(nil, bsonEncoded)
	if err != nil {
		return
	}
	return
}

func (self *commandIO) WriteCommand(cmd *command) error {
	data, err := self.encodeCommand(cmd)
	if err != nil {
		return err
	}
	var cmdLen uint16
	cmdLen = uint16(len(data))
	err = binary.Write(self.conn, binary.LittleEndian, cmdLen)
	mac, err := self.writeThenHmac(data)
	if err != nil {
		return err
	}
	err = self.writeHmac(mac)
	if err != nil {
		return err
	}
	return nil
}

func (self *commandIO) ReadCommand() (cmd *command, err error) {
	var cmdLen uint16
	err = binary.Read(self.conn, binary.LittleEndian, &cmdLen)
	if err != nil {
		return
	}
	data := make([]byte, int(cmdLen))
	mac, err := self.readThenHmac(data)
	if err != nil {
		return
	}
	err = self.readAndCmpHmac(mac)
	if err != nil {
		return
	}
	cmd, err = self.readEncodedMessage(data)
	return
}

func newCommandIO(writeKey, writeAuthKey, readKey, readAuthKey []byte, conn net.Conn) *commandIO {
	ret := new(commandIO)
	ret.writeAuth = hmac.New(sha256.New, writeAuthKey)
	ret.readAuth = hmac.New(sha256.New, readAuthKey)
	ret.conn = conn

	writeBlkCipher, _ := aes.NewCipher(writeKey)
	readBlkCipher, _ := aes.NewCipher(readKey)

	// IV: 0 for all. Since we change keys for each connection, letting IV=0 won't hurt.
	writeIV := make([]byte, writeBlkCipher.BlockSize())
	readIV := make([]byte, readBlkCipher.BlockSize())

	writeStream := cipher.NewCTR(writeBlkCipher, writeIV)
	readStream := cipher.NewCTR(readBlkCipher, readIV)

	// Then for each encrypted bit,
	// it will be written to both the connection and the hmac
	// We use encrypt-then-hmac scheme.
	mwriter := io.MultiWriter(conn, ret.writeAuth)
	swriter := new(cipher.StreamWriter)
	swriter.S = writeStream
	swriter.W = mwriter
	ret.cryptWriter = swriter

	// Similarly, for each bit read from the connection,
	// it will be written to the hmac as well.
	tee := io.TeeReader(conn, ret.readAuth)
	sreader := new(cipher.StreamReader)
	sreader.S = readStream
	sreader.R = tee
	ret.cryptReader = sreader
	return ret
}

<commit_msg>[Add] Swith for compression and encryption.<commit_after>/*
 * Copyright 2013 Nan Deng
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package proto

import (
	"io"
	"hash"
	"crypto/aes"
	"crypto/hmac"
	"crypto/sha256"
	"crypto/cipher"
	"code.google.com/p/snappy-go/snappy"
	"labix.org/v2/mgo/bson"
	"encoding/binary"
)

type commandIO struct {
	writeAuth hash.Hash
	cryptWriter io.Writer

	// we want to make
	// compress-then-encrypt
	// be the default configuration
	noWriteEncrypt bool
	noWriteCompress bool

	readAuth hash.Hash
	cryptReader io.Reader
	noReadEncrypt bool
	noReadCompress bool

	conn io.ReadWriter
}

func (self *commandIO) ReadCompressOn() {
	self.noReadCompress = false
}

func (self *commandIO) ReadEncryptOn() {
	self.noReadEncrypt= false
}

func (self *commandIO) ReadCompressOff() {
	self.noReadCompress = true
}

func (self *commandIO) ReadEncryptOff() {
	self.noReadEncrypt= true
}

func (self *commandIO) WriteCompressOn() {
	self.noWriteCompress = false
}
func (self *commandIO) WriteEncryptOn() {
	self.noWriteEncrypt= false
}

func (self *commandIO) WriteCompressOff() {
	self.noWriteCompress = true
}

func (self *commandIO) WriteEncryptOff() {
	self.noWriteEncrypt= true
}

func (self *commandIO) writeThenHmac(data []byte) (mac []byte, err error) {
	writer := self.cryptWriter
	if self.noWriteEncrypt {
		writer = self.conn
	}
	err = writen(writer, data)
	if err != nil {
		return
	}
	if self.noWriteEncrypt {
		return
	}
	self.writeAuth.Reset()
	mac = self.writeAuth.Sum(nil)
	return
}

func (self *commandIO) readThenHmac(data []byte) (mac []byte, err error) {
	reader := self.cryptReader
	if self.noReadEncrypt {
		reader = self.conn
	}
	n, err := io.ReadFull(reader, data)
	if err != nil {
		return
	}
	if n != len(data) {
		err = io.EOF
		return
	}
	if self.noReadEncrypt {
		return
	}
	self.readAuth.Reset()
	mac = self.readAuth.Sum(nil)
	return
}

func cmpHmac(a, b []byte) bool {
	if len(a) != len(b) {
		return false
	}
	for i, c := range a {
		if c != b[i] {
			return false
		}
	}
	return true
}

func (self *commandIO) writeHmac(mac []byte) error {
	if self.noWriteEncrypt {
		return nil
	}
	return writen(self.conn, mac)
}

func (self *commandIO) readAndCmpHmac(mac []byte) error {
	if self.noReadEncrypt {
		return nil
	}
	macRecved := make([]byte, self.readAuth.BlockSize())
	n, err := io.ReadFull(self.conn, macRecved)
	if err != nil {
		return err
	}
	if n != len(macRecved) {
		return ErrCorruptedData
	}
	if !cmpHmac(mac, macRecved) {
		return ErrCorruptedData
	}
	return nil
}

func (self *commandIO) readEncodedMessage(data []byte) (cmd *command, err error) {
	decoded := data
	if !self.noReadCompress {
		decoded, err = snappy.Decode(nil, data)
		if err != nil {
			return
		}
	}
	cmd = new(command)
	err = bson.Unmarshal(decoded, cmd)
	if err != nil {
		return
	}
	return
}

func (self *commandIO) encodeCommand(cmd *command)(data []byte, err error) {
	bsonEncoded, err := bson.Marshal(cmd)
	if err != nil {
		return
	}

	data = bsonEncoded
	if !self.noWriteCompress {
		data, err = snappy.Encode(nil, bsonEncoded)
		if err != nil {
			return
		}
	}
	return
}

func (self *commandIO) WriteCommand(cmd *command) error {
	data, err := self.encodeCommand(cmd)
	if err != nil {
		return err
	}
	var cmdLen uint16
	cmdLen = uint16(len(data))
	err = binary.Write(self.conn, binary.LittleEndian, cmdLen)
	mac, err := self.writeThenHmac(data)
	if err != nil {
		return err
	}
	err = self.writeHmac(mac)
	if err != nil {
		return err
	}
	return nil
}

func (self *commandIO) ReadCommand() (cmd *command, err error) {
	var cmdLen uint16
	err = binary.Read(self.conn, binary.LittleEndian, &cmdLen)
	if err != nil {
		return
	}
	data := make([]byte, int(cmdLen))
	mac, err := self.readThenHmac(data)
	if err != nil {
		return
	}
	err = self.readAndCmpHmac(mac)
	if err != nil {
		return
	}
	cmd, err = self.readEncodedMessage(data)
	return
}

func newCommandIO(writeKey, writeAuthKey, readKey, readAuthKey []byte, conn io.ReadWriter) *commandIO {
	ret := new(commandIO)
	ret.writeAuth = hmac.New(sha256.New, writeAuthKey)
	ret.readAuth = hmac.New(sha256.New, readAuthKey)
	ret.conn = conn

	writeBlkCipher, _ := aes.NewCipher(writeKey)
	readBlkCipher, _ := aes.NewCipher(readKey)

	// IV: 0 for all. Since we change keys for each connection, letting IV=0 won't hurt.
	writeIV := make([]byte, writeBlkCipher.BlockSize())
	readIV := make([]byte, readBlkCipher.BlockSize())

	writeStream := cipher.NewCTR(writeBlkCipher, writeIV)
	readStream := cipher.NewCTR(readBlkCipher, readIV)

	// Then for each encrypted bit,
	// it will be written to both the connection and the hmac
	// We use encrypt-then-hmac scheme.
	mwriter := io.MultiWriter(conn, ret.writeAuth)
	swriter := new(cipher.StreamWriter)
	swriter.S = writeStream
	swriter.W = mwriter
	ret.cryptWriter = swriter

	// Similarly, for each bit read from the connection,
	// it will be written to the hmac as well.
	tee := io.TeeReader(conn, ret.readAuth)
	sreader := new(cipher.StreamReader)
	sreader.S = readStream
	sreader.R = tee
	ret.cryptReader = sreader
	return ret
}

<|endoftext|>


<commit_before>package semrel

import (
	"context"
	"errors"
	"fmt"
	"github.com/Masterminds/semver"
	"github.com/google/go-github/github"
	"golang.org/x/oauth2"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"
)

var commitPattern = regexp.MustCompile("^(\\w*)(?:\\((.*)\\))?\\: (.*)$")
var breakingPattern = regexp.MustCompile("BREAKING CHANGES?")

type Change struct {
	Major, Minor, Patch bool
}

type Commit struct {
	SHA     string
	Raw     []string
	Type    string
	Scope   string
	Message string
	Change  Change
}

type Release struct {
	SHA     string
	Version *semver.Version
}

type Releases []*Release

func (r Releases) Len() int {
	return len(r)
}

func (r Releases) Less(i, j int) bool {
	return r[j].Version.LessThan(r[i].Version)
}

func (r Releases) Swap(i, j int) {
	r[i], r[j] = r[j], r[i]
}

type Repository struct {
	Owner  string
	Repo   string
	Ctx    context.Context
	Client *github.Client
}

func NewRepository(ctx context.Context, slug, token string) (*Repository, error) {
	if !strings.Contains(slug, "/") {
		return nil, errors.New("invalid slug")
	}
	repo := new(Repository)
	splited := strings.Split(slug, "/")
	repo.Owner = splited[0]
	repo.Repo = splited[1]
	repo.Ctx = ctx
	repo.Client = github.NewClient(oauth2.NewClient(ctx, oauth2.StaticTokenSource(
		&oauth2.Token{AccessToken: token},
	)))
	return repo, nil
}

func (repo *Repository) GetInfo() (string, bool, error) {
	r, _, err := repo.Client.Repositories.Get(repo.Ctx, repo.Owner, repo.Repo)
	if err != nil {
		return "", false, err
	}
	return r.GetDefaultBranch(), r.GetPrivate(), nil
}

func parseCommit(commit *github.RepositoryCommit) *Commit {
	c := new(Commit)
	c.SHA = commit.GetSHA()
	c.Raw = strings.Split(commit.Commit.GetMessage(), "\n")
	found := commitPattern.FindAllStringSubmatch(c.Raw[0], -1)
	if len(found) < 1 {
		return c
	}
	c.Type = strings.ToLower(found[0][1])
	c.Scope = found[0][2]
	c.Message = found[0][3]
	c.Change = Change{
		Major: breakingPattern.MatchString(commit.Commit.GetMessage()),
		Minor: c.Type == "feat",
		Patch: c.Type == "fix",
	}
	return c
}

func (repo *Repository) GetCommits(branch string) ([]*Commit, error) {
	opts := &github.CommitsListOptions{
		SHA:         branch,
		ListOptions: github.ListOptions{PerPage: 100},
	}
	commits, _, err := repo.Client.Repositories.ListCommits(repo.Ctx, repo.Owner, repo.Repo, opts)
	if err != nil {
		return nil, err
	}
	ret := make([]*Commit, len(commits))
	for i, commit := range commits {
		ret[i] = parseCommit(commit)
	}
	return ret, nil
}

func (repo *Repository) GetLatestRelease(vrange string) (*Release, error) {
	allReleases := make(Releases, 0)
	opts := &github.ReferenceListOptions{"tags", github.ListOptions{PerPage: 100}}
	for {
		refs, resp, err := repo.Client.Git.ListRefs(repo.Ctx, repo.Owner, repo.Repo, opts)
		if resp != nil && resp.StatusCode == 404 {
			return &Release{"", &semver.Version{}}, nil
		}
		if err != nil {
			return nil, err
		}
		for _, r := range refs {
			version, err := semver.NewVersion(strings.TrimPrefix(r.GetRef(), "refs/tags/"))
			if err != nil {
				continue
			}
			allReleases = append(allReleases, &Release{r.Object.GetSHA(), version})
		}
		if resp.NextPage == 0 {
			break
		}
		opts.Page = resp.NextPage
	}
	sort.Sort(allReleases)
	if vrange == "" {
		for _, r := range allReleases {
			if r.Version.Prerelease() == "" {
				return r, nil
			}
		}
		return &Release{"", &semver.Version{}}, nil
	}

	constraint, err := semver.NewConstraint(vrange)
	if err != nil {
		return nil, err
	}
	for _, r := range allReleases {
		if constraint.Check(r.Version) {
			return r, nil
		}
	}

	nver, err := semver.NewVersion(vrange)
	if err != nil {
		return nil, err
	}

	splitPre := strings.SplitN(vrange, "-", 2)
	if len(splitPre) == 1 {
		return &Release{allReleases[0].SHA, nver}, nil
	}

	npver, err := nver.SetPrerelease(splitPre[1])
	if err != nil {
		return nil, err
	}
	return &Release{allReleases[0].SHA, &npver}, nil
}

func (repo *Repository) CreateRelease(commits []*Commit, latestRelease *Release, newVersion *semver.Version, branch string) error {
	tag := fmt.Sprintf("v%s", newVersion.String())
	changelog := GetChangelog(commits, latestRelease, newVersion)
	hasPrerelease := newVersion.Prerelease() != ""
	opts := &github.RepositoryRelease{
		TagName:         &tag,
		TargetCommitish: &branch,
		Body:            &changelog,
		Prerelease:      &hasPrerelease,
	}
	_, _, err := repo.Client.Repositories.CreateRelease(repo.Ctx, repo.Owner, repo.Repo, opts)
	if err != nil {
		return err
	}
	return nil
}

func CaluclateChange(commits []*Commit, latestRelease *Release) Change {
	var change Change
	for _, commit := range commits {
		if latestRelease.SHA == commit.SHA {
			break
		}
		change.Major = change.Major || commit.Change.Major
		change.Minor = change.Minor || commit.Change.Minor
		change.Patch = change.Patch || commit.Change.Patch
	}
	return change
}

func ApplyChange(version *semver.Version, change Change) *semver.Version {
	if version.Major() == 0 {
		change.Major = true
	}
	if !change.Major && !change.Minor && !change.Patch {
		return nil
	}
	var newVersion semver.Version
	preRel := version.Prerelease()
	if preRel == "" {
		switch {
		case change.Major:
			newVersion = version.IncMajor()
			break
		case change.Minor:
			newVersion = version.IncMinor()
			break
		case change.Patch:
			newVersion = version.IncPatch()
			break
		}
		return &newVersion
	}
	preRelVer := strings.Split(preRel, ".")
	if len(preRelVer) > 1 {
		idx, err := strconv.ParseInt(preRelVer[1], 10, 32)
		if err != nil {
			idx = 0
		}
		preRel = fmt.Sprintf("%s.%d", preRelVer[0], idx+1)
	} else {
		preRel += ".1"
	}
	newVersion, _ = version.SetPrerelease(preRel)
	return &newVersion
}

func GetNewVersion(commits []*Commit, latestRelease *Release) *semver.Version {
	return ApplyChange(latestRelease.Version, CaluclateChange(commits, latestRelease))
}

func trimSHA(sha string) string {
	if len(sha) < 9 {
		return sha
	}
	return sha[:8]
}

func formatCommit(c *Commit) string {
	ret := "* "
	if c.Scope != "" {
		ret += fmt.Sprintf("**%s:** ", c.Scope)
	}
	ret += fmt.Sprintf("%s (%s)\n", c.Message, trimSHA(c.SHA))
	return ret
}

var typeToText = map[string]string{
	"feat":     "Feature",
	"fix":      "Bug Fixes",
	"perf":     "Performance Improvements",
	"revert":   "Reverts",
	"docs":     "Documentation",
	"style":    "Styles",
	"refactor": "Code Refactoring",
	"test":     "Tests",
	"chore":    "Chores",
	"%%bc%%":   "Breaking Changes",
}

func getSortedKeys(m *map[string]string) []string {
	keys := make([]string, len(*m))
	i := 0
	for k := range *m {
		keys[i] = k
		i++
	}
	sort.Strings(keys)
	return keys
}

func GetChangelog(commits []*Commit, latestRelease *Release, newVersion *semver.Version) string {
	ret := fmt.Sprintf("## %s (%s)\n\n", newVersion.String(), time.Now().UTC().Format("2006-01-02"))
	typeScopeMap := make(map[string]string)
	for _, commit := range commits {
		if latestRelease.SHA == commit.SHA {
			break
		}
		if commit.Change.Major {
			typeScopeMap["%%bc%%"] += fmt.Sprintf("%s\n```%s\n```\n", formatCommit(commit), strings.Join(commit.Raw[1:], "\n"))
			continue
		}
		if commit.Type == "" {
			continue
		}
		typeScopeMap[commit.Type] += formatCommit(commit)
	}
	for _, t := range getSortedKeys(&typeScopeMap) {
		msg := typeScopeMap[t]
		typeName, found := typeToText[t]
		if !found {
			typeName = t
		}
		ret += fmt.Sprintf("#### %s\n\n%s\n", typeName, msg)
	}
	return ret
}
<commit_msg>fix: use correct SHA for changelogs<commit_after>package semrel

import (
	"context"
	"errors"
	"fmt"
	"github.com/Masterminds/semver"
	"github.com/google/go-github/github"
	"golang.org/x/oauth2"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"
)

var commitPattern = regexp.MustCompile("^(\\w*)(?:\\((.*)\\))?\\: (.*)$")
var breakingPattern = regexp.MustCompile("BREAKING CHANGES?")

type Change struct {
	Major, Minor, Patch bool
}

type Commit struct {
	SHA     string
	Raw     []string
	Type    string
	Scope   string
	Message string
	Change  Change
}

type Release struct {
	SHA     string
	Version *semver.Version
}

type Releases []*Release

func (r Releases) Len() int {
	return len(r)
}

func (r Releases) Less(i, j int) bool {
	return r[j].Version.LessThan(r[i].Version)
}

func (r Releases) Swap(i, j int) {
	r[i], r[j] = r[j], r[i]
}

type Repository struct {
	Owner  string
	Repo   string
	Ctx    context.Context
	Client *github.Client
}

func NewRepository(ctx context.Context, slug, token string) (*Repository, error) {
	if !strings.Contains(slug, "/") {
		return nil, errors.New("invalid slug")
	}
	repo := new(Repository)
	splited := strings.Split(slug, "/")
	repo.Owner = splited[0]
	repo.Repo = splited[1]
	repo.Ctx = ctx
	repo.Client = github.NewClient(oauth2.NewClient(ctx, oauth2.StaticTokenSource(
		&oauth2.Token{AccessToken: token},
	)))
	return repo, nil
}

func (repo *Repository) GetInfo() (string, bool, error) {
	r, _, err := repo.Client.Repositories.Get(repo.Ctx, repo.Owner, repo.Repo)
	if err != nil {
		return "", false, err
	}
	return r.GetDefaultBranch(), r.GetPrivate(), nil
}

func parseCommit(commit *github.RepositoryCommit) *Commit {
	c := new(Commit)
	c.SHA = commit.GetSHA()
	c.Raw = strings.Split(commit.Commit.GetMessage(), "\n")
	found := commitPattern.FindAllStringSubmatch(c.Raw[0], -1)
	if len(found) < 1 {
		return c
	}
	c.Type = strings.ToLower(found[0][1])
	c.Scope = found[0][2]
	c.Message = found[0][3]
	c.Change = Change{
		Major: breakingPattern.MatchString(commit.Commit.GetMessage()),
		Minor: c.Type == "feat",
		Patch: c.Type == "fix",
	}
	return c
}

func (repo *Repository) GetCommits(branch string) ([]*Commit, error) {
	opts := &github.CommitsListOptions{
		SHA:         branch,
		ListOptions: github.ListOptions{PerPage: 100},
	}
	commits, _, err := repo.Client.Repositories.ListCommits(repo.Ctx, repo.Owner, repo.Repo, opts)
	if err != nil {
		return nil, err
	}
	ret := make([]*Commit, len(commits))
	for i, commit := range commits {
		ret[i] = parseCommit(commit)
	}
	return ret, nil
}

func (repo *Repository) GetLatestRelease(vrange string) (*Release, error) {
	allReleases := make(Releases, 0)
	opts := &github.ReferenceListOptions{"tags", github.ListOptions{PerPage: 100}}
	for {
		refs, resp, err := repo.Client.Git.ListRefs(repo.Ctx, repo.Owner, repo.Repo, opts)
		if resp != nil && resp.StatusCode == 404 {
			return &Release{"", &semver.Version{}}, nil
		}
		if err != nil {
			return nil, err
		}
		for _, r := range refs {
			version, err := semver.NewVersion(strings.TrimPrefix(r.GetRef(), "refs/tags/"))
			if err != nil {
				continue
			}
			allReleases = append(allReleases, &Release{r.Object.GetSHA(), version})
		}
		if resp.NextPage == 0 {
			break
		}
		opts.Page = resp.NextPage
	}
	sort.Sort(allReleases)

	var lastRelease *Release
	for _, r := range allReleases {
		if r.Version.Prerelease() == "" {
			lastRelease = r
			break
		}
	}

	if vrange == "" {
		if lastRelease != nil {
			return lastRelease, nil
		}
		return &Release{"", &semver.Version{}}, nil
	}

	constraint, err := semver.NewConstraint(vrange)
	if err != nil {
		return nil, err
	}
	for _, r := range allReleases {
		if constraint.Check(r.Version) {
			return r, nil
		}
	}

	nver, err := semver.NewVersion(vrange)
	if err != nil {
		return nil, err
	}

	splitPre := strings.SplitN(vrange, "-", 2)
	if len(splitPre) == 1 {
		return &Release{lastRelease.SHA, nver}, nil
	}

	npver, err := nver.SetPrerelease(splitPre[1])
	if err != nil {
		return nil, err
	}
	return &Release{lastRelease.SHA, &npver}, nil
}

func (repo *Repository) CreateRelease(commits []*Commit, latestRelease *Release, newVersion *semver.Version, branch string) error {
	tag := fmt.Sprintf("v%s", newVersion.String())
	changelog := GetChangelog(commits, latestRelease, newVersion)
	hasPrerelease := newVersion.Prerelease() != ""
	opts := &github.RepositoryRelease{
		TagName:         &tag,
		TargetCommitish: &branch,
		Body:            &changelog,
		Prerelease:      &hasPrerelease,
	}
	_, _, err := repo.Client.Repositories.CreateRelease(repo.Ctx, repo.Owner, repo.Repo, opts)
	if err != nil {
		return err
	}
	return nil
}

func CaluclateChange(commits []*Commit, latestRelease *Release) Change {
	var change Change
	for _, commit := range commits {
		if latestRelease.SHA == commit.SHA {
			break
		}
		change.Major = change.Major || commit.Change.Major
		change.Minor = change.Minor || commit.Change.Minor
		change.Patch = change.Patch || commit.Change.Patch
	}
	return change
}

func ApplyChange(version *semver.Version, change Change) *semver.Version {
	if version.Major() == 0 {
		change.Major = true
	}
	if !change.Major && !change.Minor && !change.Patch {
		return nil
	}
	var newVersion semver.Version
	preRel := version.Prerelease()
	if preRel == "" {
		switch {
		case change.Major:
			newVersion = version.IncMajor()
			break
		case change.Minor:
			newVersion = version.IncMinor()
			break
		case change.Patch:
			newVersion = version.IncPatch()
			break
		}
		return &newVersion
	}
	preRelVer := strings.Split(preRel, ".")
	if len(preRelVer) > 1 {
		idx, err := strconv.ParseInt(preRelVer[1], 10, 32)
		if err != nil {
			idx = 0
		}
		preRel = fmt.Sprintf("%s.%d", preRelVer[0], idx+1)
	} else {
		preRel += ".1"
	}
	newVersion, _ = version.SetPrerelease(preRel)
	return &newVersion
}

func GetNewVersion(commits []*Commit, latestRelease *Release) *semver.Version {
	return ApplyChange(latestRelease.Version, CaluclateChange(commits, latestRelease))
}

func trimSHA(sha string) string {
	if len(sha) < 9 {
		return sha
	}
	return sha[:8]
}

func formatCommit(c *Commit) string {
	ret := "* "
	if c.Scope != "" {
		ret += fmt.Sprintf("**%s:** ", c.Scope)
	}
	ret += fmt.Sprintf("%s (%s)\n", c.Message, trimSHA(c.SHA))
	return ret
}

var typeToText = map[string]string{
	"feat":     "Feature",
	"fix":      "Bug Fixes",
	"perf":     "Performance Improvements",
	"revert":   "Reverts",
	"docs":     "Documentation",
	"style":    "Styles",
	"refactor": "Code Refactoring",
	"test":     "Tests",
	"chore":    "Chores",
	"%%bc%%":   "Breaking Changes",
}

func getSortedKeys(m *map[string]string) []string {
	keys := make([]string, len(*m))
	i := 0
	for k := range *m {
		keys[i] = k
		i++
	}
	sort.Strings(keys)
	return keys
}

func GetChangelog(commits []*Commit, latestRelease *Release, newVersion *semver.Version) string {
	ret := fmt.Sprintf("## %s (%s)\n\n", newVersion.String(), time.Now().UTC().Format("2006-01-02"))
	typeScopeMap := make(map[string]string)
	for _, commit := range commits {
		if latestRelease.SHA == commit.SHA {
			break
		}
		if commit.Change.Major {
			typeScopeMap["%%bc%%"] += fmt.Sprintf("%s\n```%s\n```\n", formatCommit(commit), strings.Join(commit.Raw[1:], "\n"))
			continue
		}
		if commit.Type == "" {
			continue
		}
		typeScopeMap[commit.Type] += formatCommit(commit)
	}
	for _, t := range getSortedKeys(&typeScopeMap) {
		msg := typeScopeMap[t]
		typeName, found := typeToText[t]
		if !found {
			typeName = t
		}
		ret += fmt.Sprintf("#### %s\n\n%s\n", typeName, msg)
	}
	return ret
}
<|endoftext|>


<commit_before>package formatter

import (
  "github.com/phemmer/sawmill/event"
  "golang.org/x/crypto/ssh/terminal"
  "fmt"
  "reflect"
  "sort"
)

func IsTerminal(stream interface{Fd() uintptr}) bool {
  return terminal.IsTerminal(int(stream.Fd()))
}
var colors = terminal.EscapeCodes{
	Black:   []byte{27, '[', '3', '0', 'm'},
	Red:     []byte{27, '[', '3', '1', 'm'},
	Green:   []byte{27, '[', '3', '2', 'm'},
	Yellow:  []byte{27, '[', '3', '3', 'm'},
	Blue:    []byte{27, '[', '3', '4', 'm'},
	Magenta: []byte{27, '[', '3', '5', 'm'},
	Cyan:    []byte{27, '[', '3', '6', 'm'},
	White:   []byte{27, '[', '3', '7', 'm'},

	Reset:   []byte{27, '[', '0', 'm'},
}

type TextFormatter struct {
  color bool
}
func NewTextFormatter(color bool) *TextFormatter {
  return &TextFormatter{color: color}
}
func (formatter *TextFormatter) Format(logEvent *event.Event) ([]byte) {
  fields := flatten(logEvent.Fields)

  timestamp := logEvent.Timestamp.Format("2006-01-02_15:04:05.00")
	var levelName string
	if formatter.color {
		var levelColor []byte
		if logEvent.Level <= event.Error {
			levelColor = colors.Red
		} else if logEvent.Level == event.Warning {
			levelColor = colors.Yellow
		} else {
			levelColor = colors.Blue
		}
		levelName = fmt.Sprintf("%s%s%s", levelColor, logEvent.LevelName(), colors.Reset)
	} else {
		levelName = logEvent.LevelName()
	}
  buf := []byte(fmt.Sprintf("%s %s> %s ", timestamp, levelName, logEvent.Message))

  flatFields := flatten(fields)
  keys := make([]string, len(flatFields))
  i := 0
  for k := range flatFields {
    keys[i] = k
    i += 1
  }
  sort.Strings(keys)
  for _, k := range keys {
    v := flatFields[k]
    buf = append(buf, []byte(fmt.Sprintf("%s=%v ", k, v))...)
  }
  return buf
}

func flatten(fields interface{}) (map[string]interface{}) {
  flat := make(map[string]interface{})

  value := reflect.ValueOf(fields)
  for value.Kind() == reflect.Ptr { // shouldn't ever happen since deStruct() also does this
    value = value.Elem()
  }

  //fmt.Printf("flattening: %#v\n", fields)
  if value.Kind() == reflect.Map {
    for _, kV := range value.MapKeys() {
      vV := value.MapIndex(kV)
      k := kV.Interface()
      flattenValue(flat, k, vV)
    }
  } else if value.Kind() == reflect.Array || value.Kind() == reflect.Slice {
    for k, v := range value.Interface().([]interface{}) {
      flattenValue(flat, k, reflect.ValueOf(v))
    }
  } else {
    if value.IsValid() {
      flat[""] = value.Interface()
    } else {
      flat[""] = nil
    }
  }

  //fmt.Printf("Flatten result: %#v\n", flat)
  return flat
}
func flattenValue(flattened map[string]interface{}, parentKey interface{}, value reflect.Value) {
  kind := value.Kind()
  for kind == reflect.Ptr || kind == reflect.Interface {
    kind = value.Elem().Kind()
  }

  if kind == reflect.Struct || kind == reflect.Map || kind == reflect.Array || kind == reflect.Slice {
    for vk,vv := range flatten(value.Interface()) {
      flat_k := fmt.Sprintf("%s.%s", parentKey, vk)
      flattened[flat_k] = vv
    }
  } else {
    flat_k := fmt.Sprintf("%s", parentKey)
    flattened[flat_k] = value.Interface()
  }
}
<commit_msg>split time & level formatters to their own functions<commit_after>package formatter

import (
  "github.com/phemmer/sawmill/event"
  "golang.org/x/crypto/ssh/terminal"
  "fmt"
  "reflect"
  "sort"
)

func IsTerminal(stream interface{Fd() uintptr}) bool {
  return terminal.IsTerminal(int(stream.Fd()))
}
var colors = terminal.EscapeCodes{
	Black:   []byte{27, '[', '3', '0', 'm'},
	Red:     []byte{27, '[', '3', '1', 'm'},
	Green:   []byte{27, '[', '3', '2', 'm'},
	Yellow:  []byte{27, '[', '3', '3', 'm'},
	Blue:    []byte{27, '[', '3', '4', 'm'},
	Magenta: []byte{27, '[', '3', '5', 'm'},
	Cyan:    []byte{27, '[', '3', '6', 'm'},
	White:   []byte{27, '[', '3', '7', 'm'},

	Reset:   []byte{27, '[', '0', 'm'},
}

type TextFormatter struct {
  UseColor bool
	TimeFormat string
}
func NewTextFormatter(color bool) *TextFormatter {
  return &TextFormatter{
		UseColor: color,
		TimeFormat: "2006-01-02_15:04:05.000",
	}
}
func (formatter *TextFormatter) Format(logEvent *event.Event) ([]byte) {
  fields := flatten(logEvent.Fields)

	timestamp := formatter.FormatTimestamp(logEvent)
	levelName := formatter.FormatLevel(logEvent)
  buf := []byte(fmt.Sprintf("%s %s> %s ", timestamp, levelName, logEvent.Message))

  flatFields := flatten(fields)
  keys := make([]string, len(flatFields))
  i := 0
  for k := range flatFields {
    keys[i] = k
    i += 1
  }
  sort.Strings(keys)
  for _, k := range keys {
    v := flatFields[k]
    buf = append(buf, []byte(fmt.Sprintf("%s=%v ", k, v))...)
  }
  return buf
}

func (formatter *TextFormatter) FormatTimestamp(logEvent *event.Event) string {
	return logEvent.Timestamp.Format(formatter.TimeFormat)
}
func (formatter *TextFormatter) FormatLevel(logEvent *event.Event) string {
	var levelName string
	if formatter.UseColor {
		var levelColor []byte
		if logEvent.Level <= event.Error {
			levelColor = colors.Red
		} else if logEvent.Level == event.Warning {
			levelColor = colors.Yellow
		} else {
			levelColor = colors.Blue
		}
		levelName = fmt.Sprintf("%s%s%s", levelColor, logEvent.LevelName(), colors.Reset)
	} else {
		levelName = logEvent.LevelName()
	}

	return levelName
}

func flatten(fields interface{}) (map[string]interface{}) {
  flat := make(map[string]interface{})

  value := reflect.ValueOf(fields)
  for value.Kind() == reflect.Ptr { // shouldn't ever happen since deStruct() also does this
    value = value.Elem()
  }

  //fmt.Printf("flattening: %#v\n", fields)
  if value.Kind() == reflect.Map {
    for _, kV := range value.MapKeys() {
      vV := value.MapIndex(kV)
      k := kV.Interface()
      flattenValue(flat, k, vV)
    }
  } else if value.Kind() == reflect.Array || value.Kind() == reflect.Slice {
    for k, v := range value.Interface().([]interface{}) {
      flattenValue(flat, k, reflect.ValueOf(v))
    }
  } else {
    if value.IsValid() {
      flat[""] = value.Interface()
    } else {
      flat[""] = nil
    }
  }

  //fmt.Printf("Flatten result: %#v\n", flat)
  return flat
}
func flattenValue(flattened map[string]interface{}, parentKey interface{}, value reflect.Value) {
  kind := value.Kind()
  for kind == reflect.Ptr || kind == reflect.Interface {
    kind = value.Elem().Kind()
  }

  if kind == reflect.Struct || kind == reflect.Map || kind == reflect.Array || kind == reflect.Slice {
    for vk,vv := range flatten(value.Interface()) {
      flat_k := fmt.Sprintf("%s.%s", parentKey, vk)
      flattened[flat_k] = vv
    }
  } else {
    flat_k := fmt.Sprintf("%s", parentKey)
    flattened[flat_k] = value.Interface()
  }
}
<|endoftext|>


<commit_before>package main

import (
    "github.com/pboehm/series/renamer"
    "github.com/pboehm/series/util"
    "fmt"
    "io/ioutil"
    "path"
    "os"
    "regexp"
    "flag"
)

func main() {
    // parse command flags/args
    FlagRenameFiles := flag.Bool("rename", true, "should the files be renamed")

    flag.Parse()
    argv := flag.Args()

    // change to the series directory
    dir := path.Join(util.HomeDirectory(), "Downloads")
    if len(argv) > 0 {
        dir = argv[0]
    }

    err := os.Chdir(dir)
    if err != nil { panic(err) }

    content, err := ioutil.ReadDir(".")
    if err != nil { panic(err) }

    // handle all files that are interesting and not already in the right format
    valid_regex := regexp.MustCompile("^S\\d+E\\d+.-.\\w+.*\\.\\w+$")

    for _, entry := range content {
        entry_path := entry.Name()

        if ! renamer.IsInterestingDirEntry(entry_path) { continue }
        if valid_regex.Match([]byte(entry_path))       { continue }

        episode, err := renamer.CreateEpisodeFromPath(entry_path)
        if err != nil {
            fmt.Printf("!!! '%s' - %s\n\n", entry_path, err)
            continue
        }

        episode.RemoveTrashwords()
        if ! episode.HasValidEpisodeName() {
            episode.SetDefaultEpisodeName()
        }

        fmt.Printf("<<< %s\n", entry_path)
        fmt.Printf(">>> %s\n", episode.CleanedFileName())

        if ! episode.CanBeRenamed() {
            fmt.Printf("!!! %s couldn't be renamed\n", entry_path)
            continue
        }

        if *FlagRenameFiles {
            rename_err := episode.Rename(".")
            if rename_err != nil { panic(rename_err) }

            fmt.Println("--> episode has been renamed succesfully")
        }

        fmt.Println("")
    }
}
<commit_msg>Added first working version of the series binary<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/pboehm/series/config"
	"github.com/pboehm/series/index"
	"github.com/pboehm/series/renamer"
	"github.com/pboehm/series/util"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"regexp"
)

var CONFIG_DIR, CONFIG_FILE string
var DEFAULT_CONFIG, APP_CONFIG config.Config

func setup() {
	CONFIG_DIR = path.Join(util.HomeDirectory(), ".series")
	CONFIG_FILE = path.Join(CONFIG_DIR, "config.json")

	DEFAULT_CONFIG = config.Config{
		EpisodeDirectory: path.Join(util.HomeDirectory(), "Downloads"),
		IndexFile:        path.Join(CONFIG_DIR, "index.xml"),
	}
}

func GetInterestingDirEntries() []string {
	content, err := ioutil.ReadDir(".")
	if err != nil {
		panic(err)
	}

	valid_regex := regexp.MustCompile("^S\\d+E\\d+.-.\\w+.*\\.\\w+$")

	interesting := []string{}
	for _, entry := range content {
		entry_path := entry.Name()

		if !renamer.IsInterestingDirEntry(entry_path) {
			continue
		}
		if valid_regex.Match([]byte(entry_path)) {
			continue
		}

		interesting = append(interesting, entry_path)
	}

	return interesting
}

func HandleInterestingEpisodes(index *index.SeriesIndex, entries []string) []*renamer.Episode {
	renameable_episodes := []*renamer.Episode{}

	for _, entry_path := range entries {

		episode, err := renamer.CreateEpisodeFromPath(entry_path)
		if err != nil {
			fmt.Printf("!!! '%s' - %s\n\n", entry_path, err)
			continue
		}

		episode.RemoveTrashwords()
		if !episode.HasValidEpisodeName() {
			episode.SetDefaultEpisodeName()
		}

		fmt.Printf("<<< %s\n", entry_path)
		fmt.Printf(">>> %s\n", episode.CleanedFileName())

		if !episode.CanBeRenamed() {
			fmt.Printf("!!! '%s' is currently not renameable\n\n", entry_path)
			continue
		}

		added, added_err := index.AddEpisode(episode)
		if !added {
			fmt.Printf("!!! couldn't be added to the index: %s\n\n", added_err)
			continue
		}
		fmt.Println("---> succesfully added to series index\n")

		renameable_episodes = append(renameable_episodes, episode)
	}

	return renameable_episodes
}

func main() {
	setup()
	APP_CONFIG = config.GetConfig(CONFIG_FILE, DEFAULT_CONFIG)

	// parse command flags/args
	FlagRenameFiles := flag.Bool("rename", true, "should the files be renamed")

	flag.Parse()
	argv := flag.Args()

	// change to the series directory
	dir := path.Join(APP_CONFIG.EpisodeDirectory)
	if len(argv) > 0 {
		dir = argv[0]
	}

	err := os.Chdir(dir)
	if err != nil {
		panic(err)
	}

	// get all interesting episodes and stop if there aren't any
	interesting_entries := GetInterestingDirEntries()
	if len(interesting_entries) == 0 {
		os.Exit(0)
	}

	// Call PreProcessingHook
	if APP_CONFIG.PreProcessingHook != "" {
		fmt.Println("### Calling PreProcessingHook ...")

		cmd := exec.Command("bash", "-c", APP_CONFIG.PreProcessingHook)
		out, err := cmd.Output()
		if err != nil {
			fmt.Printf("PreProcessingHook ended with an error: %s\n", err)
		}
		fmt.Println(string(out))
	}

	fmt.Println("### Parsing series index ...")
	index, index_err := index.ParseSeriesIndex(APP_CONFIG.IndexFile)
	if index_err != nil {
		panic(index_err)
	}

	fmt.Println("### Process all interesting files ...")
	renameable_episodes := HandleInterestingEpisodes(index, interesting_entries)

	if len(renameable_episodes) > 0 && *FlagRenameFiles {
		fmt.Println("### Writing new index version ...")
		index.WriteToFile(APP_CONFIG.IndexFile)

		fmt.Println("### Renaming episodes ...")

		for _, episode := range renameable_episodes {
			fmt.Printf("> %s: %s", episode.Series, episode.CleanedFileName())

			// Rename episode file
			rename_err := episode.Rename(".")
			if rename_err != nil {
				panic(rename_err)
			}

			fmt.Printf("  [OK]\n")

			// Call EpisodeHook
			if APP_CONFIG.EpisodeHook != "" {
				fmt.Println("# Calling EpisodeHook ...")

				hook_cmd := fmt.Sprintf("%s \"%s\" \"%s\"",
					APP_CONFIG.EpisodeHook,
					episode.CleanedFileName(), episode.Series)

				cmd := exec.Command("bash", "-c", hook_cmd)
				out, err := cmd.Output()
				if err != nil {
					fmt.Printf("EpisodeHook ended with an error: %s\n", err)
				}
				fmt.Println(string(out))
			}
		}

		// Call PostProcessingHook
		if APP_CONFIG.PostProcessingHook != "" {
			fmt.Println("\n### Calling PostProcessingHook ...")

			cmd := exec.Command("bash", "-c", APP_CONFIG.PostProcessingHook)
			out, err := cmd.Output()
			if err != nil {
				fmt.Printf("PostProcessingHook ended with an error: %s\n", err)
			}
			fmt.Println(string(out))
		}
	}
}
<|endoftext|>