<commit_before>// +build linux darwin freebsd

package commands

import (
	"fmt"
	consul "github.com/hashicorp/consul/api"
	"strings"
	"time"
)

const (
	consulTries = 5
)

// Connect sets up a connection to Consul.
func Connect(server string, token string) (*consul.Client, error) {
	consul, err := consulConnect(server, token)
	if err != nil {
		Log("Consul connection is bad.", "info")
		return nil, err
	}
	return consul, err
}

// consulConnect to the Consul server and hand back a client object.
func consulConnect(server, token string) (*consul.Client, error) {
	config := consul.DefaultConfig()
	config.Address = server
	// Let's clean up the token so it doesn't appear in the logs.
	if token != "" {
		config.Token = token
	}
	consul, err := consul.NewClient(config)
	if err != nil {
		return nil, err
	}
	Log(fmt.Sprintf("server='%s' token='%s'", server, cleanupToken(token)), "debug")
	return consul, nil
}

// Standard Consul tokens have lots of dashes in them - let's split on the dash
// so that we can see part of the token in the logs - helps with debugging.
// We don't want the full token in any logs - that's bad.
func cleanupToken(token string) string {
	first := strings.SplitN(token, "-", 2)[0]
	return first
}

// Get the value from a key in the Consul KV store.
func Get(c *consul.Client, key string) string {
	var str string
	Retry(func() error {
		var err error
		str, err = consulGet(c, key)
		return err
	}, consulTries)
	return str
}

// Retry loops through the callback func and tries several times to do the thing.
func Retry(callback func() error, tries int) {
	var err error
	for i := 1; i <= tries; i++ {
		err = callback()
		if err == nil {
			return
		}
		waitTime := time.Duration(tries) * time.Second
		Log(fmt.Sprintf("Consul Failure (%d) - trying again. Max: %d", i, tries), "info")
		StatsdReconnect(tries)
		if i < tries {
			time.Sleep(waitTime)
		}
	}
	LogFatal("Panic: Giving up on Consul.", "nokey", "no_more_retries")
}

// consulGet the value from a key in the Consul KV store.
func consulGet(c *consul.Client, key string) (string, error) {
	var value string
	kv := c.KV()
	key = strings.TrimPrefix(key, "/")
	pair, _, err := kv.Get(key, nil)
	if err != nil {
		return "", err
	}
	if pair != nil {
		value = string(pair.Value[:])
	} else {
		value = ""
	}
	Log(fmt.Sprintf("action='consulGet' key='%s'", key), "debug")
	return value, err
}

// Set the value for a key in the Consul KV store.
func Set(c *consul.Client, key string, value string) bool {
	var success bool
	Retry(func() error {
		var err error
		success, err = consulSet(c, key, value)
		if success != true {
			StatsdConsul(key, "set")
		}
		return err
	}, consulTries)
	return true
}

// consulSet a value for a key in the Consul KV store.
func consulSet(c *consul.Client, key string, value string) (bool, error) {
	key = strings.TrimPrefix(key, "/")
	p := &consul.KVPair{Key: key, Value: []byte(value)}
	kv := c.KV()
	_, err := kv.Put(p, nil)
	if err != nil {
		return false, err
	}
	Log(fmt.Sprintf("action='consulSet' key='%s'", key), "debug")
	return true, err
}

// Del removes a key from the Consul KV store.
func Del(c *consul.Client, key string) bool {
	var success bool
	Retry(func() error {
		var err error
		success, err = consulDel(c, key)
		if success != true {
			StatsdConsul(key, "delete")
		}
		return err
	}, consulTries)
	return true
}

// consulDel removes a key from the Consul KV store.
func consulDel(c *consul.Client, key string) (bool, error) {
	kv := c.KV()
	key = strings.TrimPrefix(key, "/")
	_, err := kv.Delete(key, nil)
	if err != nil {
		return false, err
	}
	Log(fmt.Sprintf("action='consulDel' key='%s'", key), "info")
	return true, err
}
<commit_msg>Log error returned by consul while retrying<commit_after>// +build linux darwin freebsd

package commands

import (
	"fmt"
	consul "github.com/hashicorp/consul/api"
	"strings"
	"time"
)

const (
	consulTries = 5
)

// Connect sets up a connection to Consul.
func Connect(server string, token string) (*consul.Client, error) {
	consul, err := consulConnect(server, token)
	if err != nil {
		Log("Consul connection is bad.", "info")
		return nil, err
	}
	return consul, err
}

// consulConnect to the Consul server and hand back a client object.
func consulConnect(server, token string) (*consul.Client, error) {
	config := consul.DefaultConfig()
	config.Address = server
	// Let's clean up the token so it doesn't appear in the logs.
	if token != "" {
		config.Token = token
	}
	consul, err := consul.NewClient(config)
	if err != nil {
		return nil, err
	}
	Log(fmt.Sprintf("server='%s' token='%s'", server, cleanupToken(token)), "debug")
	return consul, nil
}

// Standard Consul tokens have lots of dashes in them - let's split on the dash
// so that we can see part of the token in the logs - helps with debugging.
// We don't want the full token in any logs - that's bad.
func cleanupToken(token string) string {
	first := strings.SplitN(token, "-", 2)[0]
	return first
}

// Get the value from a key in the Consul KV store.
func Get(c *consul.Client, key string) string {
	var str string
	Retry(func() error {
		var err error
		str, err = consulGet(c, key)
		return err
	}, consulTries)
	return str
}

// Retry loops through the callback func and tries several times to do the thing.
func Retry(callback func() error, tries int) {
	var err error
	for i := 1; i <= tries; i++ {
		err = callback()
		if err == nil {
			return
		}
		waitTime := time.Duration(tries) * time.Second
		Log(fmt.Sprintf("Consul Failure (%d) - trying again. Max: %d. Error: %s", i, tries, err), "info")
		StatsdReconnect(tries)
		if i < tries {
			time.Sleep(waitTime)
		}
	}
	LogFatal("Panic: Giving up on Consul.", "nokey", "no_more_retries")
}

// consulGet the value from a key in the Consul KV store.
func consulGet(c *consul.Client, key string) (string, error) {
	var value string
	kv := c.KV()
	key = strings.TrimPrefix(key, "/")
	pair, _, err := kv.Get(key, nil)
	if err != nil {
		return "", err
	}
	if pair != nil {
		value = string(pair.Value[:])
	} else {
		value = ""
	}
	Log(fmt.Sprintf("action='consulGet' key='%s'", key), "debug")
	return value, err
}

// Set the value for a key in the Consul KV store.
func Set(c *consul.Client, key string, value string) bool {
	var success bool
	Retry(func() error {
		var err error
		success, err = consulSet(c, key, value)
		if success != true {
			StatsdConsul(key, "set")
		}
		return err
	}, consulTries)
	return true
}

// consulSet a value for a key in the Consul KV store.
func consulSet(c *consul.Client, key string, value string) (bool, error) {
	key = strings.TrimPrefix(key, "/")
	p := &consul.KVPair{Key: key, Value: []byte(value)}
	kv := c.KV()
	_, err := kv.Put(p, nil)
	if err != nil {
		return false, err
	}
	Log(fmt.Sprintf("action='consulSet' key='%s'", key), "debug")
	return true, err
}

// Del removes a key from the Consul KV store.
func Del(c *consul.Client, key string) bool {
	var success bool
	Retry(func() error {
		var err error
		success, err = consulDel(c, key)
		if success != true {
			StatsdConsul(key, "delete")
		}
		return err
	}, consulTries)
	return true
}

// consulDel removes a key from the Consul KV store.
func consulDel(c *consul.Client, key string) (bool, error) {
	kv := c.KV()
	key = strings.TrimPrefix(key, "/")
	_, err := kv.Delete(key, nil)
	if err != nil {
		return false, err
	}
	Log(fmt.Sprintf("action='consulDel' key='%s'", key), "info")
	return true, err
}
<|endoftext|>


<commit_before>package commands

import (
	"bufio"
	"fmt"
	"os"
	"regexp"
	"strings"

	"github.com/github/hub/github"
	"github.com/github/hub/ui"
	"github.com/github/hub/utils"
)

var cmdDelete = &Command{
	Run:   delete,
	Usage: "delete [-y] [<ORGANIZATION>/]<NAME>",
	Long: `Delete an existing repository on GitHub.

## Options:

	-y, --yes
		Skip the confirmation prompt and immediately delete the repository.

	[<ORGANIZATION>/]<NAME>
		The name for the repository on GitHub.

## Examples:
		$ hub delete recipes
		[ personal repo deleted on GitHub ]

		$ hub delete sinatra/recipes
		[ repo deleted in GitHub organization ]

## See also:

hub-init(1), hub(1)
`,
}

var (
	flagDeleteAssumeYes bool
)

func init() {
	cmdDelete.Flag.BoolVarP(&flagDeleteAssumeYes, "--yes", "y", false, "YES")

	CmdRunner.Use(cmdDelete)
}

func delete(command *Command, args *Args) {
	var repoName string
	if args.IsParamsEmpty() {
		ui.Errorln("Expecting name of repository to delete")
		ui.Errorln(command.Synopsis())
		os.Exit( 1 );
	} else {
		reg := regexp.MustCompile("^[^-]")
		if !reg.MatchString(args.FirstParam()) {
			err := fmt.Errorf("invalid argument: %s", args.FirstParam())
			utils.Check(err)
		}
		repoName = args.FirstParam()
	}

	config := github.CurrentConfig()
	host, err := config.DefaultHost()
	if err != nil {
		utils.Check(github.FormatError("deleting repository", err))
	}

	owner := host.User
	if strings.Contains(repoName, "/") {
		split := strings.SplitN(repoName, "/", 2)
		owner = split[0]
		repoName = split[1]
	}

	project := github.NewProject(owner, repoName, host.Host)
	gh := github.NewClient(project.Host)

	if !flagDeleteAssumeYes {
		ui.Printf("Really delete repository '%s' (yes/N)? ", project)
		answer := ""
		scanner := bufio.NewScanner(os.Stdin)
		if scanner.Scan() {
			answer = strings.TrimSpace(scanner.Text())
		}
		utils.Check(scanner.Err())
		if answer != "yes" {
			utils.Check(fmt.Errorf("Please type 'yes' for confirmation."))
		}
	}

	err = gh.DeleteRepository(project)
	if err != nil && strings.Contains(err.Error(), "HTTP 403") {
		fmt.Println("Please edit the token used for hub at https://github.com/settings/tokens\nand verify that the `delete_repo` scope is enabled.\n")
	}
	utils.Check(err)

	fmt.Printf("Deleted repository %v\n", repoName)

	args.NoForward()
}
<commit_msg>Simplify `hub delete` arguments check<commit_after>package commands

import (
	"bufio"
	"fmt"
	"os"
	"regexp"
	"strings"

	"github.com/github/hub/github"
	"github.com/github/hub/ui"
	"github.com/github/hub/utils"
)

var cmdDelete = &Command{
	Run:   delete,
	Usage: "delete [-y] [<ORGANIZATION>/]<NAME>",
	Long: `Delete an existing repository on GitHub.

## Options:

	-y, --yes
		Skip the confirmation prompt and immediately delete the repository.

	[<ORGANIZATION>/]<NAME>
		The name for the repository on GitHub.

## Examples:
		$ hub delete recipes
		[ personal repo deleted on GitHub ]

		$ hub delete sinatra/recipes
		[ repo deleted in GitHub organization ]

## See also:

hub-init(1), hub(1)
`,
}

var (
	flagDeleteAssumeYes bool
)

func init() {
	cmdDelete.Flag.BoolVarP(&flagDeleteAssumeYes, "--yes", "y", false, "YES")

	CmdRunner.Use(cmdDelete)
}

func delete(command *Command, args *Args) {
	var repoName string
	if !args.IsParamsEmpty() {
		repoName = args.FirstParam()
	}

	re := regexp.MustCompile(NameWithOwnerRe)
	if !re.MatchString(repoName) {
		utils.Check(fmt.Errorf(command.Synopsis()))
	}

	config := github.CurrentConfig()
	host, err := config.DefaultHost()
	if err != nil {
		utils.Check(github.FormatError("deleting repository", err))
	}

	owner := host.User
	if strings.Contains(repoName, "/") {
		split := strings.SplitN(repoName, "/", 2)
		owner = split[0]
		repoName = split[1]
	}

	project := github.NewProject(owner, repoName, host.Host)
	gh := github.NewClient(project.Host)

	if !flagDeleteAssumeYes {
		ui.Printf("Really delete repository '%s' (yes/N)? ", project)
		answer := ""
		scanner := bufio.NewScanner(os.Stdin)
		if scanner.Scan() {
			answer = strings.TrimSpace(scanner.Text())
		}
		utils.Check(scanner.Err())
		if answer != "yes" {
			utils.Check(fmt.Errorf("Please type 'yes' for confirmation."))
		}
	}

	err = gh.DeleteRepository(project)
	if err != nil && strings.Contains(err.Error(), "HTTP 403") {
		fmt.Println("Please edit the token used for hub at https://github.com/settings/tokens\nand verify that the `delete_repo` scope is enabled.\n")
	}
	utils.Check(err)

	fmt.Printf("Deleted repository %v\n", repoName)

	args.NoForward()
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 Pagoda Box Inc
//
// This Source Code Form is subject to the terms of the Mozilla Public License, v.
// 2.0. If a copy of the MPL was not distributed with this file, You can obtain one
// at http://mozilla.org/MPL/2.0/.
//

package commands

import (
	"encoding/json"
	"flag"
	"fmt"
	// "os"

	"github.com/pagodabox/golang-mist"
	api "github.com/pagodabox/nanobox-api-client"
	"github.com/pagodabox/nanobox-cli/config"
	"github.com/pagodabox/nanobox-cli/ui"
	"github.com/pagodabox/nanobox-golang-stylish"
)

type (

	// DeployCommand satisfies the Command interface for deploying to nanobox
	DeployCommand struct{}

	// Sync
	Sync struct {
		ID     string `json:"id"`
		Status string `json:"status"`
	}

	// Entry
	Entry struct {
		Action   string `json:"action"`
		Document string `json:"document"`
		Log      string `json:"log"`
		Model    string `json:"model"`
		Time     string `json:"time"`
	}
)

// Help prints detailed help text for the app list command
func (c *DeployCommand) Help() {
	ui.CPrint(`
Description:
  Issues a deploy to your nanobox

Usage:
  nanobox deploy
  nanobox deploy -v
  nanobox deploy -r

Options:
  -v, --verbose
    Increase the level of log output from 'info' to 'debug'

  -r, --reset
    Clears cached libraries the project might use
  `)
}

// Run issues a deploy to the running nanobox VM
func (c *DeployCommand) Run(opts []string) {

	// flags
	flags := flag.NewFlagSet("flags", flag.ContinueOnError)
	flags.Usage = func() { c.Help() }

	//
	var fReset bool
	flags.BoolVar(&fReset, "r", false, "")
	flags.BoolVar(&fReset, "reset", false, "")

	// the verbose flag allows a user to request verbose output during a deploy. The
	// default is false
	var fVerbose bool
	flags.BoolVar(&fVerbose, "v", false, "")
	flags.BoolVar(&fVerbose, "verbose", false, "")

	//
	if err := flags.Parse(opts); err != nil {
		ui.LogFatal("[commands.destroy] flags.Parse() failed", err)
	}

	// logLevel determines the amount of output that is displayed during a deploy,
	// the default is 'info'. This can be changed to 'debug' by passing the
	// -v or --verbose flag when running this command
	logLevel := "info"

	// if the verbose flag is included, upgrade the logLevel to 'debug'
	if fVerbose {
		logLevel = "debug"
	}

	// start the vm if it's not already running
	// resume := ResumeCommand{}
	// resume.Run(opts)

	// create a 'mist' client to communicate with the mist server running on the
	// guest machine
	client := mist.Client{Host: config.Boxfile.IP, Port: "1445"}

	//
	// connect the 'mist' client to the 'mist' server
	if err := client.Connect(); err != nil {
		ui.LogFatal("[commands deploy] client.Connect() failed ", err)
	}
	defer client.Close()

	// subscribe to 'sync' updates
	fmt.Printf(stylish.Bullet("Subscribing to app logs..."))
	syncSub, err := client.Subscribe([]string{"sync"})
	if err != nil {
		fmt.Printf("   [!] FAILED\n")
	}
	defer client.Unsubscribe(syncSub)

	fmt.Printf("   [√] SUCCESS\n")

	// subscribe to the deploy logs, at the desired logLevel
	fmt.Printf(stylish.Bullet("Subscribing to deploy logs..."))
	logSub, err := client.Subscribe([]string{"deploy", logLevel})
	if err != nil {
		fmt.Printf("   [!] FAILED\n")
	}
	defer client.Unsubscribe(logSub)

	fmt.Printf("   [√] SUCCESS\n")

	//
	// issue a deploy

	path := fmt.Sprintf("http://%v:1757/deploys", config.Boxfile.IP)

	// if the reset flag is passed append a "reset=true" query string param
	if fReset {
		path += "?reset=true"
	}

	//
	fmt.Print(stylish.Bullet("Deploying to app..."))
	if err := api.DoRawRequest(nil, "POST", path, nil, nil); err != nil {
		ui.LogFatal("[commands deploy] api.DoRawRequest() failed ", err)
	}

	//
	entry := &Entry{}

	// listen for messages coming from mist
stream:
	for msg := range client.Data {

		// check for any error message
		if msg.Tags[0] == "err" {
			fmt.Printf(stylish.Error("deploy stream disconnected", "An unexpected error caused the deploy stream to disconnect. Your deploy should continue as normal, and you can see the log output from your dashboard."))
		}

		//
		if err := json.Unmarshal([]byte(msg.Data), &entry); err != nil {
			ui.LogFatal("[commands deploy] json.Unmarshal() failed", err)
		}

		// depending on what fields the data has, determines what needs to happen...
		switch {

		// if the message contains the log field, the log is printed. The message is
		// then checked to see if it contains a model field...
		// example entry: {Time: "time", Log: "content"}
		case entry.Log != "":
			fmt.Println(fmt.Sprintf("%v", entry.Log))
			fallthrough

		// if the message contains the model field...
		case entry.Model != "":

			// depending on the type of model, different things may happen...
			switch entry.Model {

			// in the case of a sync model, listen for a complete to close the stream
			case "Sync", "sync":

				sync := &Sync{}

				if err := json.Unmarshal([]byte(entry.Document), sync); err != nil {
					ui.LogFatal("[commands deploy] json.Unmarshal() failed ", err)
				}

				// once the sync is 'complete' unsubscribe from mist
				if sync.Status == "complete" {
					break stream
				}

				if sync.Status == "errored" {
					fmt.Printf(stylish.Error("deploy failed", "Your deploy failed to... well... deploy..."))
					break stream
				}

			// report any unhandled models, incase cases need to be added to handle them
			case "default":
				config.Console.Debug("Nanobox has encountered an unknown model (%v), and doesn't know what to do with it...", entry.Model)
				break stream
			}

		// report any unhandled entries, incase cases need to be added to handle them
		default:
			config.Console.Debug("Nanobox has encountered an Entry with neither a 'log' nor 'model' field and doesnt know what to do...")
			break stream
		}
	}
}
<commit_msg>fixing deploy log output (removing new lines)<commit_after>// Copyright (c) 2015 Pagoda Box Inc
//
// This Source Code Form is subject to the terms of the Mozilla Public License, v.
// 2.0. If a copy of the MPL was not distributed with this file, You can obtain one
// at http://mozilla.org/MPL/2.0/.
//

package commands

import (
	"encoding/json"
	"flag"
	"fmt"
	// "os"

	"github.com/pagodabox/golang-mist"
	api "github.com/pagodabox/nanobox-api-client"
	"github.com/pagodabox/nanobox-cli/config"
	"github.com/pagodabox/nanobox-cli/ui"
	"github.com/pagodabox/nanobox-golang-stylish"
)

type (

	// DeployCommand satisfies the Command interface for deploying to nanobox
	DeployCommand struct{}

	// Sync
	Sync struct {
		ID     string `json:"id"`
		Status string `json:"status"`
	}

	// Entry
	Entry struct {
		Action   string `json:"action"`
		Document string `json:"document"`
		Log      string `json:"log"`
		Model    string `json:"model"`
		Time     string `json:"time"`
	}
)

// Help prints detailed help text for the app list command
func (c *DeployCommand) Help() {
	ui.CPrint(`
Description:
  Issues a deploy to your nanobox

Usage:
  nanobox deploy
  nanobox deploy -v
  nanobox deploy -r

Options:
  -v, --verbose
    Increase the level of log output from 'info' to 'debug'

  -r, --reset
    Clears cached libraries the project might use
  `)
}

// Run issues a deploy to the running nanobox VM
func (c *DeployCommand) Run(opts []string) {

	// flags
	flags := flag.NewFlagSet("flags", flag.ContinueOnError)
	flags.Usage = func() { c.Help() }

	//
	var fReset bool
	flags.BoolVar(&fReset, "r", false, "")
	flags.BoolVar(&fReset, "reset", false, "")

	// the verbose flag allows a user to request verbose output during a deploy. The
	// default is false
	var fVerbose bool
	flags.BoolVar(&fVerbose, "v", false, "")
	flags.BoolVar(&fVerbose, "verbose", false, "")

	//
	if err := flags.Parse(opts); err != nil {
		ui.LogFatal("[commands.destroy] flags.Parse() failed", err)
	}

	// logLevel determines the amount of output that is displayed during a deploy,
	// the default is 'info'. This can be changed to 'debug' by passing the
	// -v or --verbose flag when running this command
	logLevel := "info"

	// if the verbose flag is included, upgrade the logLevel to 'debug'
	if fVerbose {
		logLevel = "debug"
	}

	// start the vm if it's not already running
	// resume := ResumeCommand{}
	// resume.Run(opts)

	// create a 'mist' client to communicate with the mist server running on the
	// guest machine
	client := mist.Client{Host: config.Boxfile.IP, Port: "1445"}

	//
	// connect the 'mist' client to the 'mist' server
	if err := client.Connect(); err != nil {
		ui.LogFatal("[commands deploy] client.Connect() failed ", err)
	}
	defer client.Close()

	// subscribe to 'sync' updates
	fmt.Printf(stylish.Bullet("Subscribing to app logs..."))
	syncSub, err := client.Subscribe([]string{"sync"})
	if err != nil {
		fmt.Printf("   [!] FAILED\n")
	}
	defer client.Unsubscribe(syncSub)

	fmt.Printf("   [√] SUCCESS\n")

	// subscribe to the deploy logs, at the desired logLevel
	fmt.Printf(stylish.Bullet("Subscribing to deploy logs..."))
	logSub, err := client.Subscribe([]string{"deploy", logLevel})
	if err != nil {
		fmt.Printf("   [!] FAILED\n")
	}
	defer client.Unsubscribe(logSub)

	fmt.Printf("   [√] SUCCESS\n")

	//
	// issue a deploy

	path := fmt.Sprintf("http://%v:1757/deploys", config.Boxfile.IP)

	// if the reset flag is passed append a "reset=true" query string param
	if fReset {
		path += "?reset=true"
	}

	//
	fmt.Print(stylish.Bullet("Deploying to app..."))
	if err := api.DoRawRequest(nil, "POST", path, nil, nil); err != nil {
		ui.LogFatal("[commands deploy] api.DoRawRequest() failed ", err)
	}

	//
	entry := &Entry{}

	// listen for messages coming from mist
stream:
	for msg := range client.Data {

		// check for any error message
		if msg.Tags[0] == "err" {
			fmt.Printf(stylish.Error("deploy stream disconnected", "An unexpected error caused the deploy stream to disconnect. Your deploy should continue as normal, and you can see the log output from your dashboard."))
		}

		//
		if err := json.Unmarshal([]byte(msg.Data), &entry); err != nil {
			ui.LogFatal("[commands deploy] json.Unmarshal() failed", err)
		}

		// depending on what fields the data has, determines what needs to happen...
		switch {

		// if the message contains the log field, the log is printed. The message is
		// then checked to see if it contains a model field...
		// example entry: {Time: "time", Log: "content"}
		case entry.Log != "":
			fmt.Printf(entry.Log)
			fallthrough

		// if the message contains the model field...
		case entry.Model != "":

			// depending on the type of model, different things may happen...
			switch entry.Model {

			// in the case of a sync model, listen for a complete to close the stream
			case "Sync", "sync":

				sync := &Sync{}

				if err := json.Unmarshal([]byte(entry.Document), sync); err != nil {
					ui.LogFatal("[commands deploy] json.Unmarshal() failed ", err)
				}

				// once the sync is 'complete' unsubscribe from mist
				if sync.Status == "complete" {
					break stream
				}

				if sync.Status == "errored" {
					fmt.Printf(stylish.Error("deploy failed", "Your deploy failed to... well... deploy..."))
					break stream
				}

			// report any unhandled models, incase cases need to be added to handle them
			case "default":
				config.Console.Debug("Nanobox has encountered an unknown model (%v), and doesn't know what to do with it...", entry.Model)
				break stream
			}

		// report any unhandled entries, incase cases need to be added to handle them
		default:
			config.Console.Debug("Nanobox has encountered an Entry with neither a 'log' nor 'model' field and doesnt know what to do...")
			break stream
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/eaigner/hood"
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"
	"time"
)

func ClearDeployments() (*hood.Hood) {
	hd := Hd()
	hd.Exec("DELETE FROM deployments")
	hd.Exec("DELETE FROM projects")
	return hd
}

func CreateTestProject(hd *hood.Hood) (Projects) {
	var project Projects = Projects{ Name: "Test", ApiToken: "test" }
	hd.Save(&project)
	return project
}

func TestCreateRevisionReturnsCreatedRevision(t *testing.T) {
	hd := ClearDeployments()
	defer hd.Db.Close()

	project := CreateTestProject(hd)

	request, _ := http.NewRequest("POST", "/revisions", strings.NewReader("{\"sha\":\"asd\"}"))
	request.Header.Set("API-TOKEN", project.ApiToken)
	response := httptest.NewRecorder()

	CreateRevision(response, request)

	if response.Code != http.StatusOK {
		t.Fatalf("Non-expected status code%v:\n\tbody: %v", "200", response.Code)
	}

	var deployments []Deployments
	err := hd.OrderBy("deployed_at").Find(&deployments)
	if err != nil {
		t.Fatalf("Unable to read from PostgreSQL: %v", err)
	}
	if len(deployments) != 1 {
		t.Fatalf("More than 1 entry created: %d", len(deployments))
	}

	var newDeploy Deployments = deployments[0]
	if newDeploy.Sha != "asd" {
		t.Fatalf("Did not read proper SHA: %v", newDeploy.Sha)
	}
}

func TestListRevisionsReturnsWithStatusOK(t *testing.T) {
	hd := ClearDeployments()
	defer hd.Db.Close()
	project := CreateTestProject(hd)

	request, _ := http.NewRequest("GET", "/revisions", nil)
	request.Header.Set("API-TOKEN", project.ApiToken)
	response := httptest.NewRecorder()

	ListRevisions(response, request)

	if response.Code != http.StatusOK {
		t.Fatalf("Non-expected status code%v:\n\tbody: %v", "200", response.Code)
	}
}

func TestListRevisionsReturnsValidJSON(t *testing.T) {
	hd := ClearDeployments()
	defer hd.Db.Close()
	project := CreateTestProject(hd)

	var deployedAt time.Time = time.Now()
	var deploy Deployments = Deployments{Sha: "a", DeployedAt: deployedAt, ProjectId: int(project.Id)}
	_, _ = hd.Save(&deploy)

	request, _ := http.NewRequest("GET", "/revisions", nil)
	request.Header.Set("API-TOKEN", project.ApiToken)
	response := httptest.NewRecorder()

	ListRevisions(response, request)

	decoder := json.NewDecoder(response.Body)

	var deployments []Deployments
	err := decoder.Decode(&deployments)

	if err != nil {
		t.Fatalf("Decoding should pass: %v", err)
	}
	if len(deployments) != 1 || deploy.DeployedAt != deployedAt || deploy.Sha != "a" {
		t.Fatalf("Decoding failed: %v", deployments)
	}
}
<commit_msg>add tests for deployments scoping<commit_after>package main

import (
	"github.com/eaigner/hood"
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"
	"time"
)

func ClearDeployments() (*hood.Hood) {
	hd := Hd()
	hd.Exec("DELETE FROM deployments")
	hd.Exec("DELETE FROM projects")
	return hd
}

func CreateTestProject(hd *hood.Hood, apiToken string) (Projects) {
	if apiToken == "" {
		apiToken = "test"
	}

	var project Projects = Projects{ Name: "Test", ApiToken: apiToken }
	hd.Save(&project)
	return project
}

func CreateTestRevision(hd *hood.Hood, project Projects, sha string) (Deployments) {
	var deployedAt time.Time = time.Now()
	var deploy Deployments = Deployments{ Sha: sha, DeployedAt: deployedAt, ProjectId: int(project.Id) }
	_, _ = hd.Save(&deploy)
	return deploy
}

func TestCreateRevisionReturnsCreatedRevision(t *testing.T) {
	hd := ClearDeployments()
	defer hd.Db.Close()

	project := CreateTestProject(hd, "")

	request, _ := http.NewRequest("POST", "/revisions", strings.NewReader("{\"sha\":\"asd\"}"))
	request.Header.Set("API-TOKEN", project.ApiToken)
	response := httptest.NewRecorder()

	CreateRevision(response, request)

	if response.Code != http.StatusOK {
		t.Fatalf("Non-expected status code%v:\n\tbody: %v", "200", response.Code)
	}

	var deployments []Deployments
	err := hd.OrderBy("deployed_at").Find(&deployments)
	if err != nil {
		t.Fatalf("Unable to read from PostgreSQL: %v", err)
	}
	if len(deployments) != 1 {
		t.Fatalf("More than 1 entry created: %d", len(deployments))
	}

	var newDeploy Deployments = deployments[0]
	if newDeploy.Sha != "asd" {
		t.Fatalf("Did not read proper SHA: %v", newDeploy.Sha)
	}
}

func TestListRevisionsReturnsWithStatusOK(t *testing.T) {
	hd := ClearDeployments()
	defer hd.Db.Close()
	project := CreateTestProject(hd, "")

	request, _ := http.NewRequest("GET", "/revisions", nil)
	request.Header.Set("API-TOKEN", project.ApiToken)
	response := httptest.NewRecorder()

	ListRevisions(response, request)

	if response.Code != http.StatusOK {
		t.Fatalf("Non-expected status code%v:\n\tbody: %v", "200", response.Code)
	}
}

func TestRevisionsAreScopedByApiToken(t *testing.T) {
	hd := ClearDeployments()
	defer hd.Db.Close()

	projectA := CreateTestProject(hd, "testA")
	projectB := CreateTestProject(hd, "testB")

	revA := CreateTestRevision(hd, projectA, "a")
	revB := CreateTestRevision(hd, projectB, "b")

	request, _ := http.NewRequest("GET", "/revisions", nil)
	request.Header.Set("API-TOKEN", projectA.ApiToken)
	response := httptest.NewRecorder()

	ListRevisions(response, request)

	decoder := json.NewDecoder(response.Body)

	var deploymentsA []Deployments
	_ = decoder.Decode(&deploymentsA)
	if deploymentsA[0].Sha != revA.Sha || len(deploymentsA) > 1 {
		t.Fatalf("Received foreign deployment: %v", deploymentsA);
	}

	request, _ = http.NewRequest("GET", "/revisions", nil)
	request.Header.Set("API-TOKEN", projectB.ApiToken)
	response = httptest.NewRecorder()

	ListRevisions(response, request)

	decoder = json.NewDecoder(response.Body)

	var deploymentsB []Deployments
	_ = decoder.Decode(&deploymentsB)
	if deploymentsB[0].Sha != revB.Sha || len(deploymentsB) > 1 {
		t.Fatalf("Received foreign deployment: %v", deploymentsB);
	}
}

func TestListRevisionsReturnsValidJSON(t *testing.T) {
	hd := ClearDeployments()
	defer hd.Db.Close()
	project := CreateTestProject(hd, "")

	var deploy Deployments = CreateTestRevision(hd, project, "test")

	request, _ := http.NewRequest("GET", "/revisions", nil)
	request.Header.Set("API-TOKEN", project.ApiToken)
	response := httptest.NewRecorder()

	ListRevisions(response, request)

	decoder := json.NewDecoder(response.Body)

	var deployments []Deployments
	err := decoder.Decode(&deployments)

	if err != nil {
		t.Fatalf("Decoding should pass: %v", err)
	}
	if len(deployments) != 1 || deploy.Sha != "test" {
		t.Fatalf("Decoding failed: %v", deployments)
	}
}
<|endoftext|>


<commit_before>package lfs

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/github/git-lfs/git"
	"github.com/github/git-lfs/vendor/_nuts/github.com/rubyist/tracerx"
)

const Version = "0.5.1"

var (
	LargeSizeThreshold = 5 * 1024 * 1024
	TempDir            = filepath.Join(os.TempDir(), "git-lfs")
	UserAgent          string
	LocalWorkingDir    string
	LocalGitDir        string
	LocalMediaDir      string
	LocalLogDir        string
	checkedTempDir     string
)

func TempFile(prefix string) (*os.File, error) {
	if checkedTempDir != TempDir {
		if err := os.MkdirAll(TempDir, 0755); err != nil {
			return nil, err
		}
		checkedTempDir = TempDir
	}

	return ioutil.TempFile(TempDir, prefix)
}

func ResetTempDir() error {
	checkedTempDir = ""
	return os.RemoveAll(TempDir)
}

func LocalMediaPath(sha string) (string, error) {
	path := filepath.Join(LocalMediaDir, sha[0:2], sha[2:4])
	if err := os.MkdirAll(path, 0755); err != nil {
		return "", fmt.Errorf("Error trying to create local media directory in '%s': %s", path, err)
	}

	return filepath.Join(path, sha), nil
}

func Environ() []string {
	osEnviron := os.Environ()
	env := make([]string, 6, len(osEnviron)+6)
	env[0] = fmt.Sprintf("LocalWorkingDir=%s", LocalWorkingDir)
	env[1] = fmt.Sprintf("LocalGitDir=%s", LocalGitDir)
	env[2] = fmt.Sprintf("LocalMediaDir=%s", LocalMediaDir)
	env[3] = fmt.Sprintf("TempDir=%s", TempDir)
	env[4] = fmt.Sprintf("ConcurrentTransfers=%d", Config.ConcurrentTransfers())
	env[5] = fmt.Sprintf("BatchTransfer=%v", Config.BatchTransfer())

	for _, e := range osEnviron {
		if !strings.Contains(e, "GIT_") {
			continue
		}
		env = append(env, e)
	}

	return env
}

func InRepo() bool {
	return LocalWorkingDir != ""
}

func init() {
	var err error

	tracerx.DefaultKey = "GIT"
	tracerx.Prefix = "trace git-lfs: "

	LocalWorkingDir, LocalGitDir, err = resolveGitDir()
	if err == nil {
		LocalMediaDir = filepath.Join(LocalGitDir, "lfs", "objects")
		LocalLogDir = filepath.Join(LocalMediaDir, "logs")
		TempDir = filepath.Join(LocalGitDir, "lfs", "tmp")

		if err := os.MkdirAll(LocalMediaDir, 0755); err != nil {
			panic(fmt.Errorf("Error trying to create objects directory in '%s': %s", LocalMediaDir, err))
		}

		if err := os.MkdirAll(LocalLogDir, 0755); err != nil {
			panic(fmt.Errorf("Error trying to create log directory in '%s': %s", LocalLogDir, err))
		}

		if err := os.MkdirAll(TempDir, 0755); err != nil {
			panic(fmt.Errorf("Error trying to create temp directory in '%s': %s", TempDir, err))
		}

	}

	gitVersion, err := git.Config.Version()
	if err != nil {
		gitVersion = "unknown"
	}

	UserAgent = fmt.Sprintf("git-lfs/%s (GitHub; %s %s; git %s; go %s)", Version,
		runtime.GOOS,
		runtime.GOARCH,
		strings.Replace(gitVersion, "git version ", "", 1),
		strings.Replace(runtime.Version(), "go", "", 1))
}

func resolveGitDir() (string, string, error) {
	gitDir := Config.Getenv("GIT_DIR")
	workTree := Config.Getenv("GIT_WORK_TREE")

	if gitDir != "" {
		return processGitDirVar(gitDir, workTree)
	}

	wd, err := os.Getwd()
	if err != nil {
		return "", "", err
	}

	return recursiveResolveGitDir(wd)
}

func processGitDirVar(gitDir, workTree string) (string, string, error) {
	if workTree != "" {
		return workTree, gitDir, nil
	}

	// See `core.worktree` in `man git-config`:
	// “If --git-dir or GIT_DIR is specified but none of --work-tree, GIT_WORK_TREE and
	// core.worktree is specified, the current working directory is regarded as the top
	// level of your working tree.”

	wd, err := os.Getwd()
	if err != nil {
		return "", "", err
	}

	return wd, gitDir, nil
}

func recursiveResolveGitDir(dir string) (string, string, error) {
	var cleanDir = filepath.Clean(dir)
	if cleanDir[len(cleanDir)-1] == os.PathSeparator {
		return "", "", fmt.Errorf("Git repository not found")
	}

	if filepath.Base(dir) == gitExt {
		// We're in the `.git` directory.  Make no assumptions about the working directory.
		return "", dir, nil
	}

	gitDir := filepath.Join(dir, gitExt)
	info, err := os.Stat(gitDir)
	if err != nil {
		// Found neither a directory nor a file named `.git`.
		// Move one directory up.
		return recursiveResolveGitDir(filepath.Dir(dir))
	}

	if !info.IsDir() {
		// Found a file named `.git` (we're in a submodule).
		return resolveDotGitFile(gitDir)
	}

	// Found the `.git` directory.
	return dir, gitDir, nil
}

func resolveDotGitFile(file string) (string, string, error) {
	// The local working directory is the directory the `.git` file is located in.
	wd := filepath.Dir(file)

	// The `.git` file tells us where the submodules `.git` directory is.
	gitDir, err := processDotGitFile(file)
	if err != nil {
		return "", "", err
	}

	return wd, gitDir, nil
}

func processDotGitFile(file string) (string, error) {
	f, err := os.Open(file)
	if err != nil {
		return "", err
	}
	defer f.Close()

	data := make([]byte, 512)
	n, err := f.Read(data)
	if err != nil {
		return "", err
	}

	contents := string(data[0:n])

	if !strings.HasPrefix(contents, gitPtrPrefix) {
		// The `.git` file has no entry telling us about gitdir.
		return "", nil
	}

	dir := strings.TrimSpace(strings.Split(contents, gitPtrPrefix)[1])

	if filepath.IsAbs(dir) {
		// The .git file contains an absolute path.
		return dir, nil
	}

	// The .git file contains a relative path.
	// Create an absolute path based on the directory the .git file is located in.
	absDir := filepath.Join(filepath.Dir(file), dir)

	return absDir, nil
}

const (
	gitExt       = ".git"
	gitPtrPrefix = "gitdir: "
)
<commit_msg>ララララー ラララー ララ<commit_after>package lfs

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/github/git-lfs/git"
	"github.com/github/git-lfs/vendor/_nuts/github.com/rubyist/tracerx"
)

const Version = "0.5.1"

var (
	LargeSizeThreshold = 5 * 1024 * 1024
	TempDir            = filepath.Join(os.TempDir(), "git-lfs")
	UserAgent          string
	LocalWorkingDir    string
	LocalGitDir        string
	LocalMediaDir      string
	LocalLogDir        string
	checkedTempDir     string
)

func TempFile(prefix string) (*os.File, error) {
	if checkedTempDir != TempDir {
		if err := os.MkdirAll(TempDir, 0755); err != nil {
			return nil, err
		}
		checkedTempDir = TempDir
	}

	return ioutil.TempFile(TempDir, prefix)
}

func ResetTempDir() error {
	checkedTempDir = ""
	return os.RemoveAll(TempDir)
}

func LocalMediaPath(sha string) (string, error) {
	path := filepath.Join(LocalMediaDir, sha[0:2], sha[2:4])
	if err := os.MkdirAll(path, 0755); err != nil {
		return "", fmt.Errorf("Error trying to create local media directory in '%s': %s", path, err)
	}

	return filepath.Join(path, sha), nil
}

func Environ() []string {
	osEnviron := os.Environ()
	env := make([]string, 6, len(osEnviron)+6)
	env[0] = fmt.Sprintf("LocalWorkingDir=%s", LocalWorkingDir)
	env[1] = fmt.Sprintf("LocalGitDir=%s", LocalGitDir)
	env[2] = fmt.Sprintf("LocalMediaDir=%s", LocalMediaDir)
	env[3] = fmt.Sprintf("TempDir=%s", TempDir)
	env[4] = fmt.Sprintf("ConcurrentTransfers=%d", Config.ConcurrentTransfers())
	env[5] = fmt.Sprintf("BatchTransfer=%v", Config.BatchTransfer())

	for _, e := range osEnviron {
		if !strings.Contains(e, "GIT_") {
			continue
		}
		env = append(env, e)
	}

	return env
}

func InRepo() bool {
	return LocalWorkingDir != ""
}

func init() {
	var err error

	tracerx.DefaultKey = "GIT"
	tracerx.Prefix = "trace git-lfs: "

	LocalWorkingDir, LocalGitDir, err = resolveGitDir()
	if err == nil {
		LocalMediaDir = filepath.Join(LocalGitDir, "lfs", "objects")
		LocalLogDir = filepath.Join(LocalMediaDir, "logs")
		TempDir = filepath.Join(LocalGitDir, "lfs", "tmp")

		if err := os.MkdirAll(LocalMediaDir, 0755); err != nil {
			panic(fmt.Errorf("Error trying to create objects directory in '%s': %s", LocalMediaDir, err))
		}

		if err := os.MkdirAll(LocalLogDir, 0755); err != nil {
			panic(fmt.Errorf("Error trying to create log directory in '%s': %s", LocalLogDir, err))
		}

		if err := os.MkdirAll(TempDir, 0755); err != nil {
			panic(fmt.Errorf("Error trying to create temp directory in '%s': %s", TempDir, err))
		}

	}

	gitVersion, err := git.Config.Version()
	if err != nil {
		gitVersion = "unknown"
	}

	UserAgent = fmt.Sprintf("git-lfs/%s (GitHub; %s %s; git %s; go %s)", Version,
		runtime.GOOS,
		runtime.GOARCH,
		strings.Replace(gitVersion, "git version ", "", 1),
		strings.Replace(runtime.Version(), "go", "", 1))
}

func resolveGitDir() (string, string, error) {
	gitDir := Config.Getenv("GIT_DIR")
	workTree := Config.Getenv("GIT_WORK_TREE")

	if gitDir != "" {
		return processGitDirVar(gitDir, workTree)
	}

	wd, err := os.Getwd()
	if err != nil {
		return "", "", err
	}

	workTreeR, gitDirR, err := recursiveResolveGitDir(wd)
	if err != nil {
		return "", "", err
	}

	if workTree != "" {
		return processWorkTree(gitDirR, workTree)
	}

	return workTreeR, gitDirR, nil
}

func processGitDirVar(gitDir, workTree string) (string, string, error) {
	if workTree != "" {
		return processWorkTree(gitDir, workTree)
	}

	// See `core.worktree` in `man git-config`:
	// “If --git-dir or GIT_DIR is specified but none of --work-tree, GIT_WORK_TREE and
	// core.worktree is specified, the current working directory is regarded as the top
	// level of your working tree.”

	wd, err := os.Getwd()
	if err != nil {
		return "", "", err
	}

	return wd, gitDir, nil
}

func processWorkTree(gitDir, workTree string) (string, string, error) {
	// See `core.worktree` in `man git-config`:
	// “The value [of core.worktree, GIT_WORK_TREE, or --work-tree] can be an absolute path
	// or relative to the path to the .git directory, which is either specified
	// by --git-dir or GIT_DIR, or automatically discovered.”

	if filepath.IsAbs(workTree) {
		return workTree, gitDir, nil
	}

	base := filepath.Dir(filepath.Clean(gitDir))
	absWorkTree := filepath.Join(base, workTree)
	return absWorkTree, gitDir, nil
}

func recursiveResolveGitDir(dir string) (string, string, error) {
	var cleanDir = filepath.Clean(dir)
	if cleanDir[len(cleanDir)-1] == os.PathSeparator {
		return "", "", fmt.Errorf("Git repository not found")
	}

	if filepath.Base(dir) == gitExt {
		// We're in the `.git` directory.  Make no assumptions about the working directory.
		return "", dir, nil
	}

	gitDir := filepath.Join(dir, gitExt)
	info, err := os.Stat(gitDir)
	if err != nil {
		// Found neither a directory nor a file named `.git`.
		// Move one directory up.
		return recursiveResolveGitDir(filepath.Dir(dir))
	}

	if !info.IsDir() {
		// Found a file named `.git` (we're in a submodule).
		return resolveDotGitFile(gitDir)
	}

	// Found the `.git` directory.
	return dir, gitDir, nil
}

func resolveDotGitFile(file string) (string, string, error) {
	// The local working directory is the directory the `.git` file is located in.
	wd := filepath.Dir(file)

	// The `.git` file tells us where the submodules `.git` directory is.
	gitDir, err := processDotGitFile(file)
	if err != nil {
		return "", "", err
	}

	return wd, gitDir, nil
}

func processDotGitFile(file string) (string, error) {
	f, err := os.Open(file)
	if err != nil {
		return "", err
	}
	defer f.Close()

	data := make([]byte, 512)
	n, err := f.Read(data)
	if err != nil {
		return "", err
	}

	contents := string(data[0:n])

	if !strings.HasPrefix(contents, gitPtrPrefix) {
		// The `.git` file has no entry telling us about gitdir.
		return "", nil
	}

	dir := strings.TrimSpace(strings.Split(contents, gitPtrPrefix)[1])

	if filepath.IsAbs(dir) {
		// The .git file contains an absolute path.
		return dir, nil
	}

	// The .git file contains a relative path.
	// Create an absolute path based on the directory the .git file is located in.
	absDir := filepath.Join(filepath.Dir(file), dir)

	return absDir, nil
}

const (
	gitExt       = ".git"
	gitPtrPrefix = "gitdir: "
)
<|endoftext|>


<commit_before>// Copyright 2014 Jarmo Puttonen <jarmo.puttonen@gmail.com>. All rights reserved.
// Use of this source code is governed by a MIT-style
// licence that can be found in the LICENCE file.

/*Package paparazzogo implements a caching proxy for
serving MJPEG-stream as JPG-images.
*/
package paparazzogo

import (
	"bytes"
	"errors"
	"io"
	"log"
	"mime"
	"mime/multipart"
	"net"
	"net/http"
	"strings"
	"sync"
	"time"
)

// A Mjpegproxy implements http.Handler	interface and generates
// JPG-images from a MJPEG-stream.
type Mjpegproxy struct {
	partbufsize      int64
	waittime         time.Duration
	responseduration time.Duration

	mjpegStream  string
	curImg       bytes.Buffer
	curImgLock   sync.RWMutex
	conChan      chan time.Time
	lastConn     time.Time
	lastConnLock sync.RWMutex
	running      bool
	runningLock  sync.RWMutex
	l            net.Listener
	writer       io.Writer
	handler      http.Handler
}

// NewMjpegproxy returns a new Mjpegproxy with default values.
func NewMjpegproxy() *Mjpegproxy {
	p := &Mjpegproxy{
		// Max MJPEG-frame size 5Mb.
		partbufsize: 625000,
		// Sleep time between error and reconnecting to stream.
		waittime: time.Second * 1,
		// How long to use one stream response before reconnecting.
		responseduration: time.Hour,
	}
	return p
}

// ServeHTTP uses w to serve current last MJPEG-frame
// as JPG. It also reopens MJPEG-stream
// if it was closed by idle timeout.
func (m *Mjpegproxy) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	buf := bytes.Buffer{}

	m.curImgLock.RLock()
	buf.Write(m.curImg.Bytes())
	m.curImgLock.RUnlock()

	w.Write(buf.Bytes())
	select {
	case m.conChan <- time.Now():
	default:
		m.lastConnLock.Lock()
		m.lastConn = time.Now()
		m.lastConnLock.Unlock()
	}
}

// CloseStream stops and closes MJPEG-stream.
func (m *Mjpegproxy) CloseStream() {
	m.setRunning(false)
}

// OpenStream creates a go-routine of openstream.
func (m *Mjpegproxy) OpenStream(mjpegStream, user, pass string, timeout time.Duration) {
	go m.openstream(mjpegStream, user, pass, timeout)
}

// GetRunning returns state of openstream.
func (m *Mjpegproxy) GetRunning() bool {
	m.runningLock.RLock()
	defer m.runningLock.RUnlock()
	return m.running
}

func (m *Mjpegproxy) setRunning(r bool) {
	m.runningLock.Lock()
	defer m.runningLock.Unlock()
	m.running = r
}

func (m *Mjpegproxy) getresponse(request *http.Request) (*http.Response, error) {
	tr := &http.Transport{DisableKeepAlives: true}
	client := &http.Client{Transport: tr}
	response, err := client.Do(request)
	if err != nil {
		return nil, err
	}
	if response.StatusCode != 200 {
		response.Body.Close()
		errs := "Got invalid response status: " + response.Status
		return nil, errors.New(errs)
	}
	return response, nil
}

func (m *Mjpegproxy) getboundary(response *http.Response) (string, error) {
	header := response.Header.Get("Content-Type")
	if header == "" {
		return "", errors.New("Content-Type isn't specified!")
	}
	ct, params, err := mime.ParseMediaType(header)
	if err != nil {
		return "", err
	}
	if ct != "multipart/x-mixed-replace" {
		errs := "Wrong Content-Type: expected multipart/x-mixed-replace, got " + ct
		return "", errors.New(errs)
	}
	boundary, ok := params["boundary"]
	if !ok {
		return "", errors.New("No multipart boundary param in Content-Type!")
	}
	// Some IP-cameras screw up boundary strings so we
	// have to remove excessive "--" characters manually.
	boundary = strings.Replace(boundary, "--", "", -1)
	return boundary, nil
}

// OpenStream sends request to target and handles
// response. It opens MJPEG-stream and copies received
// frame to m.curImg. It closes stream if m.CloseStream()
// is called or if difference between current time and
// time of last request to ServeHTTP is bigger than timeout.
func (m *Mjpegproxy) openstream(mjpegStream, user, pass string, timeout time.Duration) {
	m.setRunning(true)
	m.conChan = make(chan time.Time)
	m.mjpegStream = mjpegStream
	var lastconn time.Time
	var img *multipart.Part

	request, err := http.NewRequest("GET", mjpegStream, nil)
	if err != nil {
		log.Fatal(m.mjpegStream, err)
	}
	if user != "" && pass != "" {
		request.SetBasicAuth(user, pass)
	}
	var response *http.Response
	var boundary string
	var mpread *multipart.Reader
	var starttime time.Time

	log.Println("Starting streaming from", mjpegStream)

	for m.GetRunning() {
		lastconn = <-m.conChan
		m.lastConnLock.Lock()
		m.lastConn = lastconn
		m.lastConnLock.Unlock()
		if !m.GetRunning() {
			continue
		}

		response, err = m.getresponse(request)
		if err != nil {
			log.Println(m.mjpegStream, err)
			time.Sleep(m.waittime)
			continue
		}
		starttime = time.Now()
		boundary, err = m.getboundary(response)

		if err != nil {
			log.Println(m.mjpegStream, err)
			response.Body.Close()
			time.Sleep(m.waittime)
			continue
		}
		mpread = multipart.NewReader(response.Body, boundary)
		for m.GetRunning() && (time.Since(lastconn) < timeout) && err == nil {
			if time.Since(starttime) > m.responseduration {
				break
			}
			m.lastConnLock.RLock()
			lastconn = m.lastConn
			m.lastConnLock.RUnlock()
			img, err = mpread.NextPart()
			if err != nil {
				log.Println(m.mjpegStream, err)
				break
			}
			m.curImgLock.Lock()
			m.curImg.Reset()
			_, err = m.curImg.ReadFrom(img)
			m.curImgLock.Unlock()
			img.Close()
			if err != nil {
				log.Println(m.mjpegStream, err)
				break
			}
		}
		response.Body.Close()
		time.Sleep(m.waittime)
	}
	log.Println("Stopped streaming from", mjpegStream)
}
<commit_msg>Modify openstream() to read m.lastcomm less often.<commit_after>// Copyright 2014 Jarmo Puttonen <jarmo.puttonen@gmail.com>. All rights reserved.
// Use of this source code is governed by a MIT-style
// licence that can be found in the LICENCE file.

/*Package paparazzogo implements a caching proxy for
serving MJPEG-stream as JPG-images.
*/
package paparazzogo

import (
	"bytes"
	"errors"
	"io"
	"log"
	"mime"
	"mime/multipart"
	"net"
	"net/http"
	"strings"
	"sync"
	"time"
)

// A Mjpegproxy implements http.Handler	interface and generates
// JPG-images from a MJPEG-stream.
type Mjpegproxy struct {
	partbufsize      int64
	waittime         time.Duration
	responseduration time.Duration

	mjpegStream  string
	curImg       bytes.Buffer
	curImgLock   sync.RWMutex
	conChan      chan time.Time
	lastConn     time.Time
	lastConnLock sync.RWMutex
	running      bool
	runningLock  sync.RWMutex
	l            net.Listener
	writer       io.Writer
	handler      http.Handler
}

// NewMjpegproxy returns a new Mjpegproxy with default values.
func NewMjpegproxy() *Mjpegproxy {
	p := &Mjpegproxy{
		// Max MJPEG-frame size 5Mb.
		partbufsize: 625000,
		// Sleep time between error and reconnecting to stream.
		waittime: time.Second * 1,
		// How long to use one stream response before reconnecting.
		responseduration: time.Hour,
	}
	return p
}

// ServeHTTP uses w to serve current last MJPEG-frame
// as JPG. It also reopens MJPEG-stream
// if it was closed by idle timeout.
func (m *Mjpegproxy) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	buf := bytes.Buffer{}

	m.curImgLock.RLock()
	buf.Write(m.curImg.Bytes())
	m.curImgLock.RUnlock()

	w.Write(buf.Bytes())
	select {
	case m.conChan <- time.Now():
	default:
		m.lastConnLock.Lock()
		m.lastConn = time.Now()
		m.lastConnLock.Unlock()
	}
}

// CloseStream stops and closes MJPEG-stream.
func (m *Mjpegproxy) CloseStream() {
	m.setRunning(false)
}

// OpenStream creates a go-routine of openstream.
func (m *Mjpegproxy) OpenStream(mjpegStream, user, pass string, timeout time.Duration) {
	go m.openstream(mjpegStream, user, pass, timeout)
}

// GetRunning returns state of openstream.
func (m *Mjpegproxy) GetRunning() bool {
	m.runningLock.RLock()
	defer m.runningLock.RUnlock()
	return m.running
}

func (m *Mjpegproxy) setRunning(r bool) {
	m.runningLock.Lock()
	defer m.runningLock.Unlock()
	m.running = r
}

func (m *Mjpegproxy) getresponse(request *http.Request) (*http.Response, error) {
	tr := &http.Transport{DisableKeepAlives: true}
	client := &http.Client{Transport: tr}
	response, err := client.Do(request)
	if err != nil {
		return nil, err
	}
	if response.StatusCode != 200 {
		response.Body.Close()
		errs := "Got invalid response status: " + response.Status
		return nil, errors.New(errs)
	}
	return response, nil
}

func (m *Mjpegproxy) getboundary(response *http.Response) (string, error) {
	header := response.Header.Get("Content-Type")
	if header == "" {
		return "", errors.New("Content-Type isn't specified!")
	}
	ct, params, err := mime.ParseMediaType(header)
	if err != nil {
		return "", err
	}
	if ct != "multipart/x-mixed-replace" {
		errs := "Wrong Content-Type: expected multipart/x-mixed-replace, got " + ct
		return "", errors.New(errs)
	}
	boundary, ok := params["boundary"]
	if !ok {
		return "", errors.New("No multipart boundary param in Content-Type!")
	}
	// Some IP-cameras screw up boundary strings so we
	// have to remove excessive "--" characters manually.
	boundary = strings.Replace(boundary, "--", "", -1)
	return boundary, nil
}

// OpenStream sends request to target and handles
// response. It opens MJPEG-stream and copies received
// frame to m.curImg. It closes stream if m.CloseStream()
// is called or if difference between current time and
// time of last request to ServeHTTP is bigger than timeout.
func (m *Mjpegproxy) openstream(mjpegStream, user, pass string, timeout time.Duration) {
	m.setRunning(true)
	m.conChan = make(chan time.Time)
	m.mjpegStream = mjpegStream
	var lastconn time.Time
	var img *multipart.Part

	request, err := http.NewRequest("GET", mjpegStream, nil)
	if err != nil {
		log.Fatal(m.mjpegStream, err)
	}
	if user != "" && pass != "" {
		request.SetBasicAuth(user, pass)
	}
	var response *http.Response
	var boundary string
	var mpread *multipart.Reader
	var starttime time.Time

	log.Println("Starting streaming from", mjpegStream)

	for m.GetRunning() {
		lastconn = <-m.conChan
		m.lastConnLock.Lock()
		m.lastConn = lastconn
		m.lastConnLock.Unlock()
		if !m.GetRunning() {
			continue
		}

		response, err = m.getresponse(request)
		if err != nil {
			log.Println(m.mjpegStream, err)
			time.Sleep(m.waittime)
			continue
		}
		starttime = time.Now()
		boundary, err = m.getboundary(response)

		if err != nil {
			log.Println(m.mjpegStream, err)
			response.Body.Close()
			time.Sleep(m.waittime)
			continue
		}
		mpread = multipart.NewReader(response.Body, boundary)
		for m.GetRunning() && (time.Since(lastconn) < timeout) && err == nil {
			if time.Since(starttime) > m.responseduration {
				break
			}
			if time.Since(lastconn) > timeout/2 {
				m.lastConnLock.RLock()
				lastconn = m.lastConn
				m.lastConnLock.RUnlock()
			}
			img, err = mpread.NextPart()
			if err != nil {
				log.Println(m.mjpegStream, err)
				break
			}
			m.curImgLock.Lock()
			m.curImg.Reset()
			_, err = m.curImg.ReadFrom(img)
			m.curImgLock.Unlock()
			img.Close()
			if err != nil {
				log.Println(m.mjpegStream, err)
				break
			}
		}
		response.Body.Close()
		time.Sleep(m.waittime)
	}
	log.Println("Stopped streaming from", mjpegStream)
}
<|endoftext|>


<commit_before>// Copyright 2014 Jarmo Puttonen <jarmo.puttonen@gmail.com>. All rights reserved.
// Use of this source code is governed by a MIT-style
// licence that can be found in the LICENCE file.

/* Package paparazzogo implements a caching proxy for
serving MJPEG-stream as JPG-images.
*/
package paparazzogo

import (
	"bytes"
	"io"
	"log"
	"mime/multipart"
	"net"
	"net/http"
	"strings"
	"sync"
	"time"
)

// A Mjpegproxy implements http.Handler	interface and generates
// JPG-images from a MJPEG-stream.
type Mjpegproxy struct {
	partbufsize int
	imgbufsize  int

	curImg     bytes.Buffer
	curImgLock sync.RWMutex
	conChan    chan time.Time
	running    bool
	l          net.Listener
	writer     io.Writer
	handler    http.Handler
}

// NewMjpegproxy returns a new Mjpegproxy with default buffer
// sizes.
func NewMjpegproxy() *Mjpegproxy {
	p := &Mjpegproxy{
		conChan: make(chan time.Time),
		// Max MJPEG-frame part size 1Mb.
		partbufsize: 125000,
		// Max MJPEG-frame size 5Mb.
		imgbufsize: 625000,
	}
	return p
}

// ServeHTTP uses w to serve current last MJPEG-frame
// as JPG. It also reopens MJPEG-stream
// if it was closed by idle timeout.
func (m *Mjpegproxy) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	m.curImgLock.RLock()
	w.Write(m.curImg.Bytes())
	m.curImgLock.RUnlock()

	select {
	case m.conChan <- time.Now():
	default:
	}
}

// CloseStream stops and closes MJPEG-stream.
func (m *Mjpegproxy) CloseStream() {
	m.running = false
}

// OpenStream creates a go-routine of openstream.
func (m *Mjpegproxy) OpenStream(mjpegStream, user, pass string, timeout time.Duration) {
	go m.openstream(mjpegStream, user, pass, timeout)
}

// OpenStream sends request to target and handles
// response. It opens MJPEG-stream and received frame
// to m.curImg. It closes stream if m.running is set
// to false or if difference between current time and
// lastconn (time of last request to ServeHTTP)
// is bigger than timeout.
func (m *Mjpegproxy) openstream(mjpegStream, user, pass string, timeout time.Duration) {
	m.running = true
	client := new(http.Client)
	request, err := http.NewRequest("GET", mjpegStream, nil)
	if user != "" && pass != "" {
		request.SetBasicAuth(user, pass)
	}
	if err != nil {
		log.Fatal(err)
	}

	buffer := make([]byte, m.partbufsize)
	img := bytes.Buffer{}

	var part *multipart.Part
	var lastconn time.Time
	var response *http.Response

	for m.running == true {
		lastconn = <-m.conChan
		if m.running && (time.Since(lastconn) < timeout || timeout == 0) {
			response, err = client.Do(request)
			if err != nil {
				log.Println(err.Error())
				continue
			}
			if response.StatusCode == 503 {
				response.Close()
				log.Println(response.Status)
				continue
			}
			if response.StatusCode != 200 {
				response.Close()
				log.Fatalln("Got invalid response status: ", response.Status)
			}
			defer response.Body.Close()
			// Get boundary string from response and clean it up
			split := strings.Split(response.Header.Get("Content-Type"), "boundary=")
			boundary := split[1]
			// TODO: Find out what happens when boundarystring ends in --
			boundary = strings.Replace(boundary, "--", "", 1)

			reader := io.ReadCloser(response.Body)
			mpread := multipart.NewReader(reader, boundary)
			for m.running && (time.Since(lastconn) < timeout || timeout == 0) {
				part, err = mpread.NextPart()
				if err != nil {
					log.Fatal(err)
				}
				// Get frame parts until err is EOF or running is false
				for err == nil && m.running {
					amnt := 0
					amnt, err = part.Read(buffer)
					if err != nil && err.Error() != "EOF" {
						if part != nil {
							part.Close()
						}
						log.Fatal(err)
					}
					part.Close()
					img.Write(buffer[0:amnt])
				}
				err = nil

				if img.Len() > m.imgbufsize {
					img.Truncate(m.imgbufsize)
				}
				m.curImgLock.Lock()
				m.curImg.Reset()
				_, err = m.curImg.Write(img.Bytes())
				if err != nil {
					m.curImgLock.Unlock()
					log.Fatal(err)
				}
				m.curImgLock.Unlock()
				img.Reset()
			}
		} else if response != nil {
			{
				response.Body.Close()
			}
		}
	}
}
<commit_msg>Fix leaks<commit_after>// Copyright 2014 Jarmo Puttonen <jarmo.puttonen@gmail.com>. All rights reserved.
// Use of this source code is governed by a MIT-style
// licence that can be found in the LICENCE file.

/* Package paparazzogo implements a caching proxy for
serving MJPEG-stream as JPG-images.
*/
package paparazzogo

import (
	"bytes"
	"io"
	"log"
	"mime/multipart"
	"net"
	"net/http"
	"strings"
	"sync"
	"time"
)

// A Mjpegproxy implements http.Handler	interface and generates
// JPG-images from a MJPEG-stream.
type Mjpegproxy struct {
	partbufsize int
	imgbufsize  int

	curImg     bytes.Buffer
	curImgLock sync.RWMutex
	conChan    chan time.Time
	running    bool
	l          net.Listener
	writer     io.Writer
	handler    http.Handler
}

// NewMjpegproxy returns a new Mjpegproxy with default buffer
// sizes.
func NewMjpegproxy() *Mjpegproxy {
	p := &Mjpegproxy{
		conChan: make(chan time.Time),
		// Max MJPEG-frame part size 1Mb.
		partbufsize: 125000,
		// Max MJPEG-frame size 5Mb.
		imgbufsize: 625000,
	}
	return p
}

// ServeHTTP uses w to serve current last MJPEG-frame
// as JPG. It also reopens MJPEG-stream
// if it was closed by idle timeout.
func (m *Mjpegproxy) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	m.curImgLock.RLock()
	w.Write(m.curImg.Bytes())
	m.curImgLock.RUnlock()

	select {
	case m.conChan <- time.Now():
	default:
	}
}

// CloseStream stops and closes MJPEG-stream.
func (m *Mjpegproxy) CloseStream() {
	m.running = false
}

// OpenStream creates a go-routine of openstream.
func (m *Mjpegproxy) OpenStream(mjpegStream, user, pass string, timeout time.Duration) {
	go m.openstream(mjpegStream, user, pass, timeout)
}

// OpenStream sends request to target and handles
// response. It opens MJPEG-stream and received frame
// to m.curImg. It closes stream if m.running is set
// to false or if difference between current time and
// lastconn (time of last request to ServeHTTP)
// is bigger than timeout.
func (m *Mjpegproxy) openstream(mjpegStream, user, pass string, timeout time.Duration) {
	m.running = true
	client := new(http.Client)
	request, err := http.NewRequest("GET", mjpegStream, nil)
	if user != "" && pass != "" {
		request.SetBasicAuth(user, pass)
	}
	if err != nil {
		log.Fatal(err)
	}

	buffer := make([]byte, m.partbufsize)
	img := bytes.Buffer{}

	var part *multipart.Part
	var lastconn time.Time
	var response *http.Response

	for m.running == true {
		lastconn = <-m.conChan
		if m.running && (time.Since(lastconn) < timeout || timeout == 0) {
			response, err = client.Do(request)
			if err != nil {
				log.Println(err.Error())
				continue
			}
			if response.StatusCode == 503 {
				response.Body.Close()
				log.Println(response.Status)
				continue
			}
			if response.StatusCode != 200 {
				response.Body.Close()
				log.Fatalln("Got invalid response status: ", response.Status)
			}
			defer response.Body.Close()
			// Get boundary string from response and clean it up
			split := strings.Split(response.Header.Get("Content-Type"), "boundary=")
			boundary := split[1]
			// TODO: Find out what happens when boundarystring ends in --
			boundary = strings.Replace(boundary, "--", "", 1)

			reader := io.ReadCloser(response.Body)
			mpread := multipart.NewReader(reader, boundary)
			for m.running && (time.Since(lastconn) < timeout || timeout == 0) {
				part, err = mpread.NextPart()
				if err != nil {
					log.Fatal(err)
				}
				// Get frame parts until err is EOF or running is false
				for err == nil && m.running {
					amnt := 0
					amnt, err = part.Read(buffer)
					if err != nil && err.Error() != "EOF" {
						if part != nil {
							part.Close()
						}
						log.Fatal(err)
					}
					img.Write(buffer[0:amnt])
				}
				part.Close()
				part = nil
				err = nil

				if img.Len() > m.imgbufsize {
					img.Truncate(m.imgbufsize)
				}
				m.curImgLock.Lock()
				m.curImg.Reset()
				_, err = m.curImg.Write(img.Bytes())
				if err != nil {
					m.curImgLock.Unlock()
					log.Fatal(err)
				}
				m.curImgLock.Unlock()
				img.Reset()
			}
			response.Body.Close()
		} else if response != nil {
			{
				response.Body.Close()
			}
		}
	}
}
<|endoftext|>


<commit_before>//  Copyright (c) 2013 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/couchbaselabs/dparval"
	tuqcollate "github.com/couchbaselabs/tuqtng/ast"
	"github.com/prataprc/collatejson"
	"github.com/prataprc/golib"
	"github.com/prataprc/monster"
	"math/rand"
	"os"
	"reflect"
	"sort"
	"strings"
	"time"
)

var options struct {
	prodfile string
	count    int
}

type codeList struct {
	kind  string
	jsons []string
}

var codec *collatejson.Codec

func argParse() {
	flag.StringVar(&options.prodfile, "p", "json.prod",
		"production file to use")
	flag.IntVar(&options.count, "c", 100,
		"Number samples")
	flag.Parse()
}

func main() {
	argParse()

	codec = collatejson.NewCodec()
	fmt.Printf("Generating %v json documents ...\n", options.count)
	jsons := generateJsons(options.prodfile, options.count)
	checkCodec(jsons)
	fmt.Println("Done")
}

func checkCodec(jsons []string) {
	var one, two map[string]interface{}

	fmt.Println("Checking Encoding and Decoding ...")
	for _, j := range jsons {
		out := codec.Decode(codec.Encode([]byte(j)))
		json.Unmarshal([]byte(j), &one)
		json.Unmarshal(out, &two)
		if !reflect.DeepEqual(one, two) {
			panic("monster check fails, did you change the encoding format ?")
		}
	}
	fmt.Println()
}

func compareWithTuq(jsons []string, count int) {
	for i := int(0); i < count; i++ {
		fmt.Printf(".")
		tuqjsons := make([]string, len(jsons))
		copy(tuqjsons, jsons)
		binjsons := make([]string, len(jsons))
		copy(binjsons, jsons)

		tuqcodes := codeList{"tuq", tuqjsons}
		sort.Sort(tuqcodes)
		fd, _ := os.Create("a")
		fd.Write([]byte(strings.Join(tuqcodes.jsons, "\n")))
		fd.Close()

		bincodes := codeList{"binary", binjsons}
		sort.Sort(bincodes)
		fd, _ = os.Create("b")
		fd.Write([]byte(strings.Join(bincodes.jsons, "\n")))
		fd.Close()
	}
	fmt.Println()
}

func generateJsons(prodfile string, count int) (jsons []string) {
	c, conf := make(monster.Context), make(golib.Config)
	start := monster.Parse(prodfile, conf)
	nonterminals, root := monster.Build(start)
	c["_nonterminals"] = nonterminals
	for i := 1; i < count; i++ {
		c["_random"] = rand.New(rand.NewSource(int64(time.Now().UnixNano())))
		monster.Initialize(c)
		text := root.Generate(c)
		jsons = append(jsons, text)
	}
	return
}

func (codes codeList) Len() int {
	return len(codes.jsons)
}

func (codes codeList) Less(i, j int) bool {
	key1, key2 := codes.jsons[i], codes.jsons[j]
	if codes.kind == "tuq" {
		value1 := dparval.NewValueFromBytes([]byte(key1)).Value()
		value2 := dparval.NewValueFromBytes([]byte(key2)).Value()
		return tuqcollate.CollateJSON(value1, value2) < 0
	} else if codes.kind == "binary" {
		value1 := codec.Encode([]byte(key1))
		value2 := codec.Encode([]byte(key2))
		return bytes.Compare(value1, value2) < 0
	} else {
		panic(fmt.Errorf("unknown kind"))
	}
	return false
}

func (codes codeList) Swap(i, j int) {
	codes.jsons[i], codes.jsons[j] = codes.jsons[j], codes.jsons[i]
}
<commit_msg>collatejson/monstercheck uses Generate() api.<commit_after>//  Copyright (c) 2013 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/couchbaselabs/dparval"
	tuqcollate "github.com/couchbaselabs/tuqtng/ast"
	"github.com/prataprc/collatejson"
	"github.com/prataprc/monster"
	"os"
	"reflect"
	"sort"
	"strings"
	"time"
)

var options struct {
	prodfile string
	count    int
}

type codeList struct {
	kind  string
	jsons []string
}

var codec *collatejson.Codec

func argParse() {
	flag.StringVar(&options.prodfile, "p", "json.prod",
		"production file to use")
	flag.IntVar(&options.count, "c", 100,
		"Number samples")
	flag.Parse()
}

func main() {
	argParse()

	codec = collatejson.NewCodec()
	fmt.Printf("Generating %v json documents ...\n", options.count)
	jsons := generateJsons(options.prodfile, options.count)
	checkCodec(jsons)
	fmt.Println("Done")
}

func checkCodec(jsons []string) {
	var one, two map[string]interface{}

	fmt.Println("Checking Encoding and Decoding ...")
	for _, j := range jsons {
		out := codec.Decode(codec.Encode([]byte(j)))
		json.Unmarshal([]byte(j), &one)
		json.Unmarshal(out, &two)
		if !reflect.DeepEqual(one, two) {
			panic("monster check fails, did you change the encoding format ?")
		}
	}
	fmt.Println()
}

func compareWithTuq(jsons []string, count int) {
	for i := int(0); i < count; i++ {
		fmt.Printf(".")
		tuqjsons := make([]string, len(jsons))
		copy(tuqjsons, jsons)
		binjsons := make([]string, len(jsons))
		copy(binjsons, jsons)

		tuqcodes := codeList{"tuq", tuqjsons}
		sort.Sort(tuqcodes)
		fd, _ := os.Create("a")
		fd.Write([]byte(strings.Join(tuqcodes.jsons, "\n")))
		fd.Close()

		bincodes := codeList{"binary", binjsons}
		sort.Sort(bincodes)
		fd, _ = os.Create("b")
		fd.Write([]byte(strings.Join(bincodes.jsons, "\n")))
		fd.Close()
	}
	fmt.Println()
}

func generateJsons(prodfile string, count int) []string {
	jsons, err :=
		monster.Generate(int(time.Now().UnixNano()), count, "", prodfile)
	if err != nil {
		panic(err)
	}
	return jsons
}

func (codes codeList) Len() int {
	return len(codes.jsons)
}

func (codes codeList) Less(i, j int) bool {
	key1, key2 := codes.jsons[i], codes.jsons[j]
	if codes.kind == "tuq" {
		value1 := dparval.NewValueFromBytes([]byte(key1)).Value()
		value2 := dparval.NewValueFromBytes([]byte(key2)).Value()
		return tuqcollate.CollateJSON(value1, value2) < 0
	} else if codes.kind == "binary" {
		value1 := codec.Encode([]byte(key1))
		value2 := codec.Encode([]byte(key2))
		return bytes.Compare(value1, value2) < 0
	} else {
		panic(fmt.Errorf("unknown kind"))
	}
	return false
}

func (codes codeList) Swap(i, j int) {
	codes.jsons[i], codes.jsons[j] = codes.jsons[j], codes.jsons[i]
}
<|endoftext|>


<commit_before>package producer

import (
	"bytes"
	"encoding/json"
	"github.com/bitly/go-nsq"
)

// Producer inherets the nsq Producer object
type Producer struct {
	*nsq.Producer
}

// PublishJSONAsync - sends message to nsq  topic in json format asynchronously
func (p *Producer) PublishJSONAsync(topic string, v interface{}, doneChan chan *nsq.ProducerTransaction, args ...interface{}) error {
	body, err := EncJSON(v)
	if err != nil {
		return err
	}
	return p.PublishAsync(topic, body, doneChan, args...)
}

// PublishJSON - sends message to nsq  topic in json format
func (p *Producer) PublishJSON(topic string, v interface{}) error {
	body, err := EncJSON(v)
	if err != nil {
		return err
	}
	return p.Publish(topic, body)
}

// Connect method initialize the connection to nsq
func (p *Producer) Connect(addr string) (err error) {
	config := nsq.NewConfig()
	p.Producer, err = nsq.NewProducer(addr, config)
	return
}

// EncJSON -
func EncJSON(v interface{}) ([]byte, error) {
	buf := new(bytes.Buffer)
	enc := json.NewEncoder(buf)
	err := enc.Encode(v)
	return buf.Bytes(), err
}
<commit_msg>Rename EncJSON to json.Marshal<commit_after>package producer

import (
	"encoding/json"
	"github.com/bitly/go-nsq"
)

// Producer inherets the nsq Producer object
type Producer struct {
	*nsq.Producer
}

// PublishJSONAsync - sends message to nsq  topic in json format asynchronously
func (p *Producer) PublishJSONAsync(topic string, v interface{}, doneChan chan *nsq.ProducerTransaction, args ...interface{}) error {
	body, err := json.Marshal(v)
	if err != nil {
		return err
	}
	return p.PublishAsync(topic, body, doneChan, args...)
}

// PublishJSON - sends message to nsq  topic in json format
func (p *Producer) PublishJSON(topic string, v interface{}) error {
	body, err := json.Marshal(v)
	if err != nil {
		return err
	}
	return p.Publish(topic, body)
}

// Connect method initialize the connection to nsq
func (p *Producer) Connect(addr string) (err error) {
	config := nsq.NewConfig()
	p.Producer, err = nsq.NewProducer(addr, config)
	return
}
<|endoftext|>


<commit_before>package hyperbahn

import (
	"fmt"
	"math/rand"
	"time"

	"golang.org/x/net/context"
)

const (
	// maxRegistrationFailures is the number of consecutive registration failures after
	// which we give up and trigger an OnError event.
	maxRegistrationFailures = 5
	// registrationInterval is the (approximate) time interval between re-registrations.
	// The interval is fuzzed to fuzzAmount.
	registrationInterval = 10 * time.Second
	// registrationRetryInterval is the duration to wait on failed registrations before retrying.
	registrationRetryInterval = 1 * time.Second
	// fuzzInterval is used to fuzz the registration interval.
	fuzzInterval = 10 * time.Second
)

// timeSleep is a variable for stubbing in unit tests.
var timeSleep = time.Sleep

// ErrRegistrationFailed is triggered when registration is failed.
type ErrRegistrationFailed struct {
	// WillRetry is set to true if registration will be retried.
	WillRetry bool
	// Cause is the underlying error returned from the register call.
	Cause error
}

func (e ErrRegistrationFailed) Error() string {
	return fmt.Sprintf("registration failed, retry: %v, cause: %v", e.WillRetry, e.Cause)
}

// The following parameters define the request/response for the Hyperbahn 'ad' call.
type service struct {
	Name string `json:"serviceName"`
	Cost int    `json:"cost"`
}

type adRequest struct {
	Services []service `json:"services"`
}

type adResponse struct {
	ConnectionCount int `json:"connectionCount"`
}

func (c *Client) sendRegistration() error {
	ctx, cancel := context.WithTimeout(context.Background(), c.opts.Timeout)
	defer cancel()

	sc := c.tchan.GetSubChannel(hyperbahnServiceName)
	arg := &adRequest{
		Services: []service{{
			Name: c.tchan.PeerInfo().ServiceName,
			Cost: 0,
		}},
	}
	var resp adResponse
	c.opts.Handler.On(RegistrationAttempt)
	if err := makeJSONCall(ctx, sc, "ad", arg, &resp); err != nil {
		return err
	}

	return nil
}

func (c *Client) fuzzedRegistrationInterval() time.Duration {
	// fuzz is a random value between -fuzzInterval and fuzzInterval
	fuzz := time.Duration(rand.Intn(int(fuzzInterval)*2)) - fuzzInterval
	return registrationInterval + fuzz
}

// registrationLoops re-registers the service approximately every minute (with some fuzzing).
func (c *Client) registrationLoop() {
	sleepFor := c.fuzzedRegistrationInterval()
	consecutiveFailures := 0

	for {
		timeSleep(sleepFor)

		if err := c.sendRegistration(); err != nil {
			consecutiveFailures++
			if consecutiveFailures >= maxRegistrationFailures {
				c.opts.Handler.OnError(ErrRegistrationFailed{Cause: err, WillRetry: false})
				if c.opts.FailStrategy == FailStrategyFatal {
					c.tchan.Logger().Fatalf("Hyperbahn client registration failed: %v", err)
				}
				return
			}
			c.opts.Handler.OnError(ErrRegistrationFailed{Cause: err, WillRetry: true})
			sleepFor = registrationRetryInterval
		} else {
			c.opts.Handler.On(RegistrationRefreshed)
			sleepFor = c.fuzzedRegistrationInterval()
			consecutiveFailures = 0
		}
	}
}
<commit_msg>switching from int to int64<commit_after>package hyperbahn

import (
	"fmt"
	"math/rand"
	"time"

	"golang.org/x/net/context"
)

const (
	// maxRegistrationFailures is the number of consecutive registration failures after
	// which we give up and trigger an OnError event.
	maxRegistrationFailures = 5
	// registrationInterval is the (approximate) time interval between re-registrations.
	// The interval is fuzzed to fuzzAmount.
	registrationInterval = 10 * time.Second
	// registrationRetryInterval is the duration to wait on failed registrations before retrying.
	registrationRetryInterval = 1 * time.Second
	// fuzzInterval is used to fuzz the registration interval.
	fuzzInterval = 10 * time.Second
)

// timeSleep is a variable for stubbing in unit tests.
var timeSleep = time.Sleep

// ErrRegistrationFailed is triggered when registration is failed.
type ErrRegistrationFailed struct {
	// WillRetry is set to true if registration will be retried.
	WillRetry bool
	// Cause is the underlying error returned from the register call.
	Cause error
}

func (e ErrRegistrationFailed) Error() string {
	return fmt.Sprintf("registration failed, retry: %v, cause: %v", e.WillRetry, e.Cause)
}

// The following parameters define the request/response for the Hyperbahn 'ad' call.
type service struct {
	Name string `json:"serviceName"`
	Cost int    `json:"cost"`
}

type adRequest struct {
	Services []service `json:"services"`
}

type adResponse struct {
	ConnectionCount int `json:"connectionCount"`
}

func (c *Client) sendRegistration() error {
	ctx, cancel := context.WithTimeout(context.Background(), c.opts.Timeout)
	defer cancel()

	sc := c.tchan.GetSubChannel(hyperbahnServiceName)
	arg := &adRequest{
		Services: []service{{
			Name: c.tchan.PeerInfo().ServiceName,
			Cost: 0,
		}},
	}
	var resp adResponse
	c.opts.Handler.On(RegistrationAttempt)
	if err := makeJSONCall(ctx, sc, "ad", arg, &resp); err != nil {
		return err
	}

	return nil
}

func (c *Client) fuzzedRegistrationInterval() time.Duration {
	// fuzz is a random value between -fuzzInterval and fuzzInterval
	fuzz := time.Duration(rand.Int63n(int64(fuzzInterval)*2)) - fuzzInterval
	return registrationInterval + fuzz
}

// registrationLoops re-registers the service approximately every minute (with some fuzzing).
func (c *Client) registrationLoop() {
	sleepFor := c.fuzzedRegistrationInterval()
	consecutiveFailures := 0

	for {
		timeSleep(sleepFor)

		if err := c.sendRegistration(); err != nil {
			consecutiveFailures++
			if consecutiveFailures >= maxRegistrationFailures {
				c.opts.Handler.OnError(ErrRegistrationFailed{Cause: err, WillRetry: false})
				if c.opts.FailStrategy == FailStrategyFatal {
					c.tchan.Logger().Fatalf("Hyperbahn client registration failed: %v", err)
				}
				return
			}
			c.opts.Handler.OnError(ErrRegistrationFailed{Cause: err, WillRetry: true})
			sleepFor = registrationRetryInterval
		} else {
			c.opts.Handler.On(RegistrationRefreshed)
			sleepFor = c.fuzzedRegistrationInterval()
			consecutiveFailures = 0
		}
	}
}
<|endoftext|>


<commit_before>// Package levenshtein is a Go implementation to calculate Levenshtein Distance.
//
// Implementation taken from
// https://gist.github.com/andrei-m/982927#gistcomment-1931258
package levenshtein

import "unicode/utf8"

// ComputeDistance computes the levenshtein distance between the two
// strings passed as an argument. The return value is the levenshtein distance
//
// Works on runes (Unicode code points) but does not normalize
// the input strings. See https://blog.golang.org/normalization
// and the golang.org/x/text/unicode/norm pacage.
func ComputeDistance(a, b string) int {
	if len(a) == 0 {
		return utf8.RuneCountInString(b)
	}

	if len(b) == 0 {
		return utf8.RuneCountInString(a)
	}

	if a == b {
		return 0
	}

	// We need to convert to []rune if the strings are non-ascii.
	// This could be avoided by using utf8.RuneCountInString
	// and then doing some juggling with rune indices.
	// The primary challenge is keeping track of the previous rune.
	// With a range loop, its not that easy. And with a for-loop
	// we need to keep track of the inter-rune width using utf8.DecodeRuneInString
	s1 := []rune(a)
	s2 := []rune(b)

	// swap to save some memory O(min(a,b)) instead of O(a)
	if len(s1) > len(s2) {
		s1, s2 = s2, s1
	}
	lenS1 := len(s1)
	lenS2 := len(s2)

	// init the row
	x := make([]int, lenS1+1)
	// we start from 1 because index 0 is already 0.
	for i := 1; i < len(x); i++ {
		x[i] = i
	}

	// make a dummy bounds check to prevent the 2 bounds check down below.
	// The one inside the loop is particularly costly.
	_ = x[lenS1]
	// fill in the rest
	for i := 1; i <= lenS2; i++ {
		prev := i
		var current int
		for j := 1; j <= lenS1; j++ {
			if s2[i-1] == s1[j-1] {
				current = x[j-1] // match
			} else {
				current = min(min(x[j-1]+1, prev+1), x[j]+1)
			}
			x[j-1] = prev
			prev = current
		}
		x[lenS1] = prev
	}
	return x[lenS1]
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}
<commit_msg>Clarify comment on []rune conversion.<commit_after>// Package levenshtein is a Go implementation to calculate Levenshtein Distance.
//
// Implementation taken from
// https://gist.github.com/andrei-m/982927#gistcomment-1931258
package levenshtein

import "unicode/utf8"

// ComputeDistance computes the levenshtein distance between the two
// strings passed as an argument. The return value is the levenshtein distance
//
// Works on runes (Unicode code points) but does not normalize
// the input strings. See https://blog.golang.org/normalization
// and the golang.org/x/text/unicode/norm pacage.
func ComputeDistance(a, b string) int {
	if len(a) == 0 {
		return utf8.RuneCountInString(b)
	}

	if len(b) == 0 {
		return utf8.RuneCountInString(a)
	}

	if a == b {
		return 0
	}

	// We need to convert to []rune if the strings are non-ASCII.
	// This could be avoided by using utf8.RuneCountInString
	// and then doing some juggling with rune indices,
	// but leads to far more bounds checks. It is a reasonable trade-off.
	s1 := []rune(a)
	s2 := []rune(b)

	// swap to save some memory O(min(a,b)) instead of O(a)
	if len(s1) > len(s2) {
		s1, s2 = s2, s1
	}
	lenS1 := len(s1)
	lenS2 := len(s2)

	// init the row
	x := make([]int, lenS1+1)
	// we start from 1 because index 0 is already 0.
	for i := 1; i < len(x); i++ {
		x[i] = i
	}

	// make a dummy bounds check to prevent the 2 bounds check down below.
	// The one inside the loop is particularly costly.
	_ = x[lenS1]
	// fill in the rest
	for i := 1; i <= lenS2; i++ {
		prev := i
		var current int
		for j := 1; j <= lenS1; j++ {
			if s2[i-1] == s1[j-1] {
				current = x[j-1] // match
			} else {
				current = min(min(x[j-1]+1, prev+1), x[j]+1)
			}
			x[j-1] = prev
			prev = current
		}
		x[lenS1] = prev
	}
	return x[lenS1]
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}
<|endoftext|>


<commit_before>package lexers

import (
	. "github.com/alecthomas/chroma" // nolint
)

var YAML = Register(MustNewLexer(
	&Config{
		Name:      "YAML",
		Aliases:   []string{"yaml"},
		Filenames: []string{"*.yaml", "*.yml"},
		MimeTypes: []string{"text/x-yaml"},
	},
	Rules{
		"root": {
			Include("whitespace"),
			{`#.*`, Comment, nil},
			{`![^\s]+`, CommentPreproc, nil},
			{`\s&[^\s]+`, CommentPreproc, nil},
			{`\s\*[^\s]+`, CommentPreproc, nil},
			{`^%include\s+[^\n\r]+`, CommentPreproc, nil},
			{`([>|])(\s+)((?:(?:.*?$)(?:[\n\r]*?\2)?)*)`, ByGroups(StringDoc, StringDoc, StringDoc), nil},
			Include("value"),
			{`[?:,\[\]]`, Punctuation, nil},
			{`.`, Text, nil},
		},
		"value": {
			{Words(``, `\b`, "true", "false", "null"), KeywordConstant, nil},
			{`"(?:\\.|[^"])*"`, StringDouble, nil},
			{`'(?:\\.|[^'])*'`, StringSingle, nil},
			{`\d\d\d\d-\d\d-\d\d([T ]\d\d:\d\d:\d\d(\.\d+)?(Z|\s+[-+]\d+)?)?`, LiteralDate, nil},
			{`\b[+\-]?(0x[\da-f]+|0o[0-7]+|(\d+\.?\d*|\.?\d+)(e[\+\-]?\d+)?|\.inf|\.nan)\b`, Number, nil},
		},
		"whitespace": {
			{`\s+`, Whitespace, nil},
		},
	},
))
<commit_msg>Slightly smarter YAML.<commit_after>package lexers

import (
	. "github.com/alecthomas/chroma" // nolint
)

var YAML = Register(MustNewLexer(
	&Config{
		Name:      "YAML",
		Aliases:   []string{"yaml"},
		Filenames: []string{"*.yaml", "*.yml"},
		MimeTypes: []string{"text/x-yaml"},
	},
	Rules{
		"root": {
			Include("whitespace"),
			{`#.*`, Comment, nil},
			{`!![^\s]+`, CommentPreproc, nil},
			{`&[^\s]+`, CommentPreproc, nil},
			{`\*[^\s]+`, CommentPreproc, nil},
			{`^%include\s+[^\n\r]+`, CommentPreproc, nil},
			{`([>|])(\s+)((?:(?:.*?$)(?:[\n\r]*?\2)?)*)`, ByGroups(StringDoc, StringDoc, StringDoc), nil},
			Include("value"),
			{`[?:,\[\]]`, Punctuation, nil},
			{`.`, Text, nil},
		},
		"value": {
			{Words(``, `\b`, "true", "false", "null"), KeywordConstant, nil},
			{`"(?:\\.|[^"])*"`, StringDouble, nil},
			{`'(?:\\.|[^'])*'`, StringSingle, nil},
			{`\d\d\d\d-\d\d-\d\d([T ]\d\d:\d\d:\d\d(\.\d+)?(Z|\s+[-+]\d+)?)?`, LiteralDate, nil},
			{`\b[+\-]?(0x[\da-f]+|0o[0-7]+|(\d+\.?\d*|\.?\d+)(e[\+\-]?\d+)?|\.inf|\.nan)\b`, Number, nil},
			{`\b[\w]+\b`, Text, nil},
		},
		"whitespace": {
			{`\s+`, Whitespace, nil},
		},
	},
))
<|endoftext|>