<commit_before>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package cmd

import (
	"fmt"
	"net/http"
	"net/url"
	"reflect"

	"github.com/TheThingsNetwork/ttn/ttnctl/util"
	"github.com/howeyc/gopass"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

// userCmd represents the users command
var userCmd = &cobra.Command{
	Use:   "user",
	Short: "Show the current user",
	Long:  `ttnctl user shows the current logged on user`,
	Run: func(cmd *cobra.Command, args []string) {
		t, err := util.LoadAuth(viper.GetString("ttn-account-server"))
		if err != nil {
			ctx.WithError(err).Fatal("Failed to load authentication token")
		}
		if t == nil {
			ctx.Warn("No login found. Please login with ttnctl user login [e-mail]")
			return
		}

		ctx.Infof("Logged on as %s", t.Email)
	},
}

var userCreateCmd = &cobra.Command{
	Use:   "create [e-mail]",
	Short: "Create a new user",
	Long:  `ttnctl user create allows you to create a new user`,
	Run: func(cmd *cobra.Command, args []string) {
		if len(args) < 1 {
			cmd.Help()
			return
		}

		email := args[0]
		fmt.Print("Password: ")
		password, err := gopass.GetPasswd()
		if err != nil {
			ctx.Fatal(err.Error())
		}
		fmt.Print("Retype password: ")
		password2, err := gopass.GetPasswd()
		if err != nil {
			ctx.Fatal(err.Error())
		}
		if ! reflect.DeepEqual(password,password2) {
			ctx.Fatal("Passwords do not match")
		}

		uri := fmt.Sprintf("%s/register", viper.GetString("ttn-account-server"))
		values := url.Values{
			"email":    {email},
			"password": {string(password)},
		}
		res, err := http.PostForm(uri, values)
		if err != nil {
			ctx.WithError(err).Fatal("Registration failed")
		}

		if res.StatusCode != http.StatusCreated {
			ctx.Fatalf("Registration failed: %d %s", res.StatusCode, res.Status)
		}

		ctx.Info("User created")
	},
}

var userLoginCmd = &cobra.Command{
	Use:   "login [e-mail]",
	Short: "Login",
	Long:  `ttnctl user login allows you to login`,
	Run: func(cmd *cobra.Command, args []string) {
		if len(args) < 1 {
			cmd.Help()
			return
		}

		email := args[0]
		fmt.Print("Password: ")
		password, err := gopass.GetPasswd()
		if err != nil {
			ctx.Fatal(err.Error())
		}

		_, err = util.Login(viper.GetString("ttn-account-server"), email, string(password))
		if err != nil {
			ctx.WithError(err).Fatal("Failed to login")
		}

		ctx.Infof("Logged in as %s and persisted token in %s", email, util.AuthsFileName)
	},
}

var userLogoutCmd = &cobra.Command{
	Use:   "logout",
	Short: "Logout the current user",
	Long:  `ttnctl user logout logs out the current user`,
	Run: func(cmd *cobra.Command, args []string) {
		if err := util.Logout(viper.GetString("ttn-account-server")); err != nil {
			ctx.WithError(err).Fatal("Failed to log out")
		}

		ctx.Info("Logged out")
	},
}

func init() {
	RootCmd.AddCommand(userCmd)
	userCmd.AddCommand(userCreateCmd)
	userCmd.AddCommand(userLoginCmd)
	userCmd.AddCommand(userLogoutCmd)
}
<commit_msg>Fix formatting<commit_after>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package cmd

import (
	"fmt"
	"net/http"
	"net/url"
	"reflect"

	"github.com/TheThingsNetwork/ttn/ttnctl/util"
	"github.com/howeyc/gopass"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

// userCmd represents the users command
var userCmd = &cobra.Command{
	Use:   "user",
	Short: "Show the current user",
	Long:  `ttnctl user shows the current logged on user`,
	Run: func(cmd *cobra.Command, args []string) {
		t, err := util.LoadAuth(viper.GetString("ttn-account-server"))
		if err != nil {
			ctx.WithError(err).Fatal("Failed to load authentication token")
		}
		if t == nil {
			ctx.Warn("No login found. Please login with ttnctl user login [e-mail]")
			return
		}

		ctx.Infof("Logged on as %s", t.Email)
	},
}

var userCreateCmd = &cobra.Command{
	Use:   "create [e-mail]",
	Short: "Create a new user",
	Long:  `ttnctl user create allows you to create a new user`,
	Run: func(cmd *cobra.Command, args []string) {
		if len(args) < 1 {
			cmd.Help()
			return
		}

		email := args[0]
		fmt.Print("Password: ")
		password, err := gopass.GetPasswd()
		if err != nil {
			ctx.Fatal(err.Error())
		}
		fmt.Print("Retype password: ")
		password2, err := gopass.GetPasswd()
		if err != nil {
			ctx.Fatal(err.Error())
		}
		if !reflect.DeepEqual(password, password2) {
			ctx.Fatal("Passwords do not match")
		}

		uri := fmt.Sprintf("%s/register", viper.GetString("ttn-account-server"))
		values := url.Values{
			"email":    {email},
			"password": {string(password)},
		}
		res, err := http.PostForm(uri, values)
		if err != nil {
			ctx.WithError(err).Fatal("Registration failed")
		}

		if res.StatusCode != http.StatusCreated {
			ctx.Fatalf("Registration failed: %d %s", res.StatusCode, res.Status)
		}

		ctx.Info("User created")
	},
}

var userLoginCmd = &cobra.Command{
	Use:   "login [e-mail]",
	Short: "Login",
	Long:  `ttnctl user login allows you to login`,
	Run: func(cmd *cobra.Command, args []string) {
		if len(args) < 1 {
			cmd.Help()
			return
		}

		email := args[0]
		fmt.Print("Password: ")
		password, err := gopass.GetPasswd()
		if err != nil {
			ctx.Fatal(err.Error())
		}

		_, err = util.Login(viper.GetString("ttn-account-server"), email, string(password))
		if err != nil {
			ctx.WithError(err).Fatal("Failed to login")
		}

		ctx.Infof("Logged in as %s and persisted token in %s", email, util.AuthsFileName)
	},
}

var userLogoutCmd = &cobra.Command{
	Use:   "logout",
	Short: "Logout the current user",
	Long:  `ttnctl user logout logs out the current user`,
	Run: func(cmd *cobra.Command, args []string) {
		if err := util.Logout(viper.GetString("ttn-account-server")); err != nil {
			ctx.WithError(err).Fatal("Failed to log out")
		}

		ctx.Info("Logged out")
	},
}

func init() {
	RootCmd.AddCommand(userCmd)
	userCmd.AddCommand(userCreateCmd)
	userCmd.AddCommand(userLoginCmd)
	userCmd.AddCommand(userLogoutCmd)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"

	"github.com/wx13/genesis/installer"
	"github.com/wx13/genesis/modules"
)

var inst *installer.Installer

func dotfiles() {

	inst.AddTask(modules.LineInFile{
		File:    "~/.bashrc",
		Line:    "source $HOME/.mybashrc",
		Pattern: "source $HOME/.mybashrc",
		Store:   inst.Store,
		Label:   "bashrc",
	})

	inst.AddTask(modules.CopyFile{
		DestFile: "~/.mybashrc",
		SrcFile:  "files/mybashrc",
		Store:    inst.Store,
	})

	inst.AddTask(modules.HttpGet{
		Url:   "https://raw.githubusercontent.com/git/git/master/contrib/completion/git-prompt.sh",
		Dest:  "~/.git-prompt.sh",
		Store: inst.Store,
	})

	inst.AddTask(modules.CopyFile{
		DestFile: "~/.gitconfig",
		SrcFile:  "files/gitconfig",
		Store:    inst.Store,
	})

	inst.AddTask(modules.CopyFile{
		DestFile: "~/.screenrc",
		SrcFile:  "files/screenrc",
		Store:    inst.Store,
	})

	sshConfig()

}

func sshConfig() {

	// Ensure SSH directory exists, but don't remove it.
	if !inst.Remove {
		inst.AddTask(modules.Mkdir{Path: "~/.ssh"})
	}

	// Enable SSH persistence.
	inst.AddTask(modules.BlockInFile{
		File:     "~/.ssh/config",
		Patterns: []string{`^Host \*`, "^ControlPersist"},
		Lines: []string{
			"Host *",
			"ControlMaster auto",
			"ControlPath ~/.ssh/master-%r@%h:%p",
			"ControlPersist 30m",
		},
		Store: inst.Store,
		Label: "ssh_persistence",
	})

	// Disable host key checking on select local networks.
	ips := []string{"10.0.0.*", "10.0.1.*", "192.168.1.*"}
	for _, ip := range ips {
		inst.AddTask(modules.BlockInFile{
			File: "~/.ssh/config",
			Patterns: []string{
				fmt.Sprintf("^Host %s", ip),
				"^UserKnownHostsFile",
			},
			Lines: []string{
				fmt.Sprintf("Host %s", ip),
				"StrictHostKeyChecking no",
				"UserKnownHostsFile=/dev/null",
			},
			Store: inst.Store,
			Label: "disable_ssh_host_key_checking" + ip,
		})
	}
}

func main() {

	inst = installer.New()
	if inst == nil {
		panic("Unable to create an installer.")
	}
	defer inst.Done()

	dotfiles()

}
<commit_msg>add sections to laptop<commit_after>package main

import (
	"fmt"

	"github.com/wx13/genesis/installer"
	"github.com/wx13/genesis/modules"
)

var inst *installer.Installer

func dotfiles() {

	sect := installer.NewSection("Configure dotfiles")
	defer inst.Add(sect)

	sect.AddTask(modules.LineInFile{
		File:    "~/.bashrc",
		Line:    "source $HOME/.mybashrc",
		Pattern: "source $HOME/.mybashrc",
		Store:   inst.Store,
		Label:   "bashrc",
	})

	sect.AddTask(modules.CopyFile{
		DestFile: "~/.mybashrc",
		SrcFile:  "files/mybashrc",
		Store:    inst.Store,
	})

	sect.AddTask(modules.HttpGet{
		Url:   "https://raw.githubusercontent.com/git/git/master/contrib/completion/git-prompt.sh",
		Dest:  "~/.git-prompt.sh",
		Store: inst.Store,
	})

	sect.AddTask(modules.CopyFile{
		DestFile: "~/.gitconfig",
		SrcFile:  "files/gitconfig",
		Store:    inst.Store,
	})

	sect.AddTask(modules.CopyFile{
		DestFile: "~/.screenrc",
		SrcFile:  "files/screenrc",
		Store:    inst.Store,
	})

}

func sshConfig() {

	sect := installer.NewSection("SSH configuration")
	defer inst.Add(sect)

	// Ensure SSH directory exists, but don't remove it.
	if !inst.Remove {
		sect.AddTask(modules.Mkdir{Path: "~/.ssh"})
	}

	// Enable SSH persistence.
	sect.AddTask(modules.BlockInFile{
		File:     "~/.ssh/config",
		Patterns: []string{`^Host \*`, "^ControlPersist"},
		Lines: []string{
			"Host *",
			"ControlMaster auto",
			"ControlPath ~/.ssh/master-%r@%h:%p",
			"ControlPersist 30m",
		},
		Store: inst.Store,
		Label: "ssh_persistence",
	})

	// Disable host key checking on select local networks.
	ips := []string{"10.0.0.*", "10.0.1.*", "192.168.1.*"}
	for _, ip := range ips {
		sect.AddTask(modules.BlockInFile{
			File: "~/.ssh/config",
			Patterns: []string{
				fmt.Sprintf("^Host %s", ip),
				"^UserKnownHostsFile",
			},
			Lines: []string{
				fmt.Sprintf("Host %s", ip),
				"StrictHostKeyChecking no",
				"UserKnownHostsFile=/dev/null",
			},
			Store: inst.Store,
			Label: "disable_ssh_host_key_checking" + ip,
		})
	}
}

func main() {

	inst = installer.New()
	if inst == nil {
		panic("Unable to create an installer.")
	}
	defer inst.Done()

	dotfiles()
	sshConfig()

}
<|endoftext|>


<commit_before>package pirc

import (
    "bytes"
    "strings"
    "log"
)

type IrcCmd struct {
    Cmd string
    Args []string
}

type CmdParser struct {
    commands []*IrcCmd
}

func (parser *CmdParser) Parse(buf []byte) (string, *CodePair) {
    log.Printf("In: %v", buf)
    cmd_list := bytes.Split(buf[0:], []byte("\n"))
    log.Printf("Split: %v", cmd_list)

    // Strip whitespace in case \r\n is sent
    var cmds []string
    if len(cmd_list[len(cmd_list)-1]) == 0 {
        cmds = make([]string, len(cmd_list)-1) // the last array from Split is empty
        for i := range cmd_list[0:len(cmd_list)-1] {
            cmds[i] = strings.TrimSpace(string(cmd_list[i]))
        }
    } else {
        // If somehow the last command did not end with a CRLF
        cmds = make([]string, len(cmd_list))
        for i := range cmd_list {
            cmds[i] = strings.TrimSpace(string(cmd_list[i]))
        }
    }

    for _, c := range cmds {
        log.Printf("Got command %v", c)
        cmd_split := strings.Split(c, " ")

        irc_cmd := IrcCmd {
            Cmd: cmd_split[0],
            Args: cmd_split[1:],
        }

        // Check if command is valid
        if _, ok := CmdDispatcher[irc_cmd.Cmd]; !ok {
            return irc_cmd.Cmd, &ERR.UNKNOWNCOMMAND
        }

    }
    return "", nil
}

var CmdDispatcher = make(map[string] func(*IrcCmd) ServerResponse)
<commit_msg>fix cmdparser<commit_after>package pirc

import (
    "bytes"
    "strings"
    "log"
)

type IrcCmd struct {
    Cmd string
    Args []string
}

type CmdParser struct {
    commands []*IrcCmd
}

func (parser *CmdParser) Parse(buf []byte) (string, *CodePair) {
    log.Printf("In: %v", buf)
    cmd_list := bytes.Split(buf[0:], []byte("\n"))
    log.Printf("Split: %v", cmd_list)

    // Strip whitespace in case \r\n is sent
    var cmds []string
    if len(cmd_list[len(cmd_list)-1]) == 0 {
        cmds = make([]string, len(cmd_list)-1) // the last array from Split is empty
        for i := range cmd_list[0:len(cmd_list)-1] {
            cmds[i] = strings.TrimSpace(string(cmd_list[i]))
        }
    } else {
        // If somehow the last command did not end with a CRLF
        cmds = make([]string, len(cmd_list))
        for i := range cmd_list {
            cmds[i] = strings.TrimSpace(string(cmd_list[i]))
        }
    }

    for _, c := range cmds {
        log.Printf("Got command %v", c)
        cmd_split := strings.Split(c, " ")

        irc_cmd := IrcCmd {
            Cmd: cmd_split[0],
            Args: cmd_split[1:],
        }

        // Check if command is valid
        if _, ok := CmdDispatcher[strings.ToLower(irc_cmd.Cmd)]; !ok {
            return irc_cmd.Cmd, &ERR.UNKNOWNCOMMAND
        }

    }
    return "", nil
}

var CmdDispatcher = make(map[string] func(*IrcCmd) ServerResponse)
// CmdDispatcher["NICK"] = func(*IrcCmd cmd) ServerResponse {
    // server.AddUser(
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package http

import (
	"bytes"
	"encoding/binary"
)

// Content-type sniffing algorithm.
// References in this file refer to this draft specification:
//   http://tools.ietf.org/html/draft-ietf-websec-mime-sniff-03

// The algorithm prefers to use sniffLen bytes to make its decision.
const sniffLen = 512

// DetectContentType returns the sniffed Content-Type string
// for the given data. This function always returns a valid MIME type.
func DetectContentType(data []byte) string {
	if len(data) > sniffLen {
		data = data[:sniffLen]
	}

	// Index of the first non-whitespace byte in data.
	firstNonWS := 0
	for ; firstNonWS < len(data) && isWS(data[firstNonWS]); firstNonWS++ {
	}

	for _, sig := range sniffSignatures {
		if ct := sig.match(data, firstNonWS); ct != "" {
			return ct
		}
	}

	return "application/octet-stream" // fallback
}

func isWS(b byte) bool {
	return bytes.IndexByte([]byte("\t\n\x0C\n "), b) != -1
}

type sniffSig interface {
	// match returns the MIME type of the data, or "" if unknown.
	match(data []byte, firstNonWS int) string
}

// Data matching the table in section 6.
var sniffSignatures = []sniffSig{
	htmlSig([]byte("<!DOCTYPE HTML")),
	htmlSig([]byte("<HTML")),
	htmlSig([]byte("<HEAD")),
	htmlSig([]byte("<SCRIPT")),
	htmlSig([]byte("<IFRAME")),
	htmlSig([]byte("<H1")),
	htmlSig([]byte("<DIV")),
	htmlSig([]byte("<FONT")),
	htmlSig([]byte("<TABLE")),
	htmlSig([]byte("<A")),
	htmlSig([]byte("<STYLE")),
	htmlSig([]byte("<TITLE")),
	htmlSig([]byte("<B")),
	htmlSig([]byte("<BODY")),
	htmlSig([]byte("<BR")),
	htmlSig([]byte("<P")),
	htmlSig([]byte("<!--")),

	&maskedSig{mask: []byte("\xFF\xFF\xFF\xFF\xFF"), pat: []byte("<?xml"), skipWS: true, ct: "text/xml; charset=utf-8"},

	&exactSig{[]byte("%PDF-"), "application/pdf"},
	&exactSig{[]byte("%!PS-Adobe-"), "application/postscript"},

	// UTF BOMs.
	&maskedSig{mask: []byte("\xFF\xFF\x00\x00"), pat: []byte("\xFE\xFF\x00\x00"), ct: "text/plain; charset=utf-16be"},
	&maskedSig{mask: []byte("\xFF\xFF\x00\x00"), pat: []byte("\xFF\xFE\x00\x00"), ct: "text/plain; charset=utf-16le"},
	&maskedSig{mask: []byte("\xFF\xFF\xFF\x00"), pat: []byte("\xEF\xBB\xBF\x00"), ct: "text/plain; charset=utf-8"},

	&exactSig{[]byte("GIF87a"), "image/gif"},
	&exactSig{[]byte("GIF89a"), "image/gif"},
	&exactSig{[]byte("\x89\x50\x4E\x47\x0D\x0A\x1A\x0A"), "image/png"},
	&exactSig{[]byte("\xFF\xD8\xFF"), "image/jpeg"},
	&exactSig{[]byte("BM"), "image/bmp"},
	&maskedSig{
		mask: []byte("\xFF\xFF\xFF\xFF\x00\x00\x00\x00\xFF\xFF\xFF\xFF\xFF\xFF"),
		pat:  []byte("RIFF\x00\x00\x00\x00WEBPVP"),
		ct:   "image/webp",
	},
	&exactSig{[]byte("\x00\x00\x01\x00"), "image/vnd.microsoft.icon"},
	&exactSig{[]byte("\x4F\x67\x67\x53\x00"), "application/ogg"},
	&maskedSig{
		mask: []byte("\xFF\xFF\xFF\xFF\x00\x00\x00\x00\xFF\xFF\xFF\xFF"),
		pat:  []byte("RIFF\x00\x00\x00\x00WAVE"),
		ct:   "audio/wave",
	},
	&exactSig{[]byte("\x1A\x45\xDF\xA3"), "video/webm"},
	&exactSig{[]byte("\x52\x61\x72\x20\x1A\x07\x00"), "application/x-rar-compressed"},
	&exactSig{[]byte("\x50\x4B\x03\x04"), "application/zip"},
	&exactSig{[]byte("\x1F\x8B\x08"), "application/x-gzip"},

	// TODO(dsymonds): Re-enable this when the spec is sorted w.r.t. MP4.
	//mp4Sig(0),

	textSig(0), // should be last
}

type exactSig struct {
	sig []byte
	ct  string
}

func (e *exactSig) match(data []byte, firstNonWS int) string {
	if bytes.HasPrefix(data, e.sig) {
		return e.ct
	}
	return ""
}

type maskedSig struct {
	mask, pat []byte
	skipWS    bool
	ct        string
}

func (m *maskedSig) match(data []byte, firstNonWS int) string {
	if m.skipWS {
		data = data[firstNonWS:]
	}
	if len(data) < len(m.mask) {
		return ""
	}
	for i, mask := range m.mask {
		db := data[i] & mask
		if db != m.pat[i] {
			return ""
		}
	}
	return m.ct
}

type htmlSig []byte

func (h htmlSig) match(data []byte, firstNonWS int) string {
	data = data[firstNonWS:]
	if len(data) < len(h)+1 {
		return ""
	}
	for i, b := range h {
		db := data[i]
		if 'A' <= b && b <= 'Z' {
			db &= 0xDF
		}
		if b != db {
			return ""
		}
	}
	// Next byte must be space or right angle bracket.
	if db := data[len(h)]; db != ' ' && db != '>' {
		return ""
	}
	return "text/html; charset=utf-8"
}

type mp4Sig int

func (mp4Sig) match(data []byte, firstNonWS int) string {
	// c.f. section 6.1.
	if len(data) < 8 {
		return ""
	}
	boxSize := int(binary.BigEndian.Uint32(data[:4]))
	if boxSize%4 != 0 || len(data) < boxSize {
		return ""
	}
	if !bytes.Equal(data[4:8], []byte("ftyp")) {
		return ""
	}
	for st := 8; st < boxSize; st += 4 {
		if st == 12 {
			// minor version number
			continue
		}
		seg := string(data[st : st+3])
		switch seg {
		case "mp4", "iso", "M4V", "M4P", "M4B":
			return "video/mp4"
			/* The remainder are not in the spec.
			case "M4A":
				return "audio/mp4"
			case "3gp":
				return "video/3gpp"
			case "jp2":
				return "image/jp2" // JPEG 2000
			*/
		}
	}
	return ""
}

type textSig int

func (textSig) match(data []byte, firstNonWS int) string {
	// c.f. section 5, step 4.
	for _, b := range data[firstNonWS:] {
		switch {
		case 0x00 <= b && b <= 0x08,
			b == 0x0B,
			0x0E <= b && b <= 0x1A,
			0x1C <= b && b <= 0x1F:
			return ""
		}
	}
	return "text/plain; charset=utf-8"
}
<commit_msg>http: update mime sniffing doc reference.<commit_after>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package http

import (
	"bytes"
	"encoding/binary"
)

// Content-type sniffing algorithm.
// References in this file refer to this draft specification:
//   http://mimesniff.spec.whatwg.org/

// The algorithm prefers to use sniffLen bytes to make its decision.
const sniffLen = 512

// DetectContentType returns the sniffed Content-Type string
// for the given data. This function always returns a valid MIME type.
func DetectContentType(data []byte) string {
	if len(data) > sniffLen {
		data = data[:sniffLen]
	}

	// Index of the first non-whitespace byte in data.
	firstNonWS := 0
	for ; firstNonWS < len(data) && isWS(data[firstNonWS]); firstNonWS++ {
	}

	for _, sig := range sniffSignatures {
		if ct := sig.match(data, firstNonWS); ct != "" {
			return ct
		}
	}

	return "application/octet-stream" // fallback
}

func isWS(b byte) bool {
	return bytes.IndexByte([]byte("\t\n\x0C\n "), b) != -1
}

type sniffSig interface {
	// match returns the MIME type of the data, or "" if unknown.
	match(data []byte, firstNonWS int) string
}

// Data matching the table in section 6.
var sniffSignatures = []sniffSig{
	htmlSig([]byte("<!DOCTYPE HTML")),
	htmlSig([]byte("<HTML")),
	htmlSig([]byte("<HEAD")),
	htmlSig([]byte("<SCRIPT")),
	htmlSig([]byte("<IFRAME")),
	htmlSig([]byte("<H1")),
	htmlSig([]byte("<DIV")),
	htmlSig([]byte("<FONT")),
	htmlSig([]byte("<TABLE")),
	htmlSig([]byte("<A")),
	htmlSig([]byte("<STYLE")),
	htmlSig([]byte("<TITLE")),
	htmlSig([]byte("<B")),
	htmlSig([]byte("<BODY")),
	htmlSig([]byte("<BR")),
	htmlSig([]byte("<P")),
	htmlSig([]byte("<!--")),

	&maskedSig{mask: []byte("\xFF\xFF\xFF\xFF\xFF"), pat: []byte("<?xml"), skipWS: true, ct: "text/xml; charset=utf-8"},

	&exactSig{[]byte("%PDF-"), "application/pdf"},
	&exactSig{[]byte("%!PS-Adobe-"), "application/postscript"},

	// UTF BOMs.
	&maskedSig{mask: []byte("\xFF\xFF\x00\x00"), pat: []byte("\xFE\xFF\x00\x00"), ct: "text/plain; charset=utf-16be"},
	&maskedSig{mask: []byte("\xFF\xFF\x00\x00"), pat: []byte("\xFF\xFE\x00\x00"), ct: "text/plain; charset=utf-16le"},
	&maskedSig{mask: []byte("\xFF\xFF\xFF\x00"), pat: []byte("\xEF\xBB\xBF\x00"), ct: "text/plain; charset=utf-8"},

	&exactSig{[]byte("GIF87a"), "image/gif"},
	&exactSig{[]byte("GIF89a"), "image/gif"},
	&exactSig{[]byte("\x89\x50\x4E\x47\x0D\x0A\x1A\x0A"), "image/png"},
	&exactSig{[]byte("\xFF\xD8\xFF"), "image/jpeg"},
	&exactSig{[]byte("BM"), "image/bmp"},
	&maskedSig{
		mask: []byte("\xFF\xFF\xFF\xFF\x00\x00\x00\x00\xFF\xFF\xFF\xFF\xFF\xFF"),
		pat:  []byte("RIFF\x00\x00\x00\x00WEBPVP"),
		ct:   "image/webp",
	},
	&exactSig{[]byte("\x00\x00\x01\x00"), "image/vnd.microsoft.icon"},
	&exactSig{[]byte("\x4F\x67\x67\x53\x00"), "application/ogg"},
	&maskedSig{
		mask: []byte("\xFF\xFF\xFF\xFF\x00\x00\x00\x00\xFF\xFF\xFF\xFF"),
		pat:  []byte("RIFF\x00\x00\x00\x00WAVE"),
		ct:   "audio/wave",
	},
	&exactSig{[]byte("\x1A\x45\xDF\xA3"), "video/webm"},
	&exactSig{[]byte("\x52\x61\x72\x20\x1A\x07\x00"), "application/x-rar-compressed"},
	&exactSig{[]byte("\x50\x4B\x03\x04"), "application/zip"},
	&exactSig{[]byte("\x1F\x8B\x08"), "application/x-gzip"},

	// TODO(dsymonds): Re-enable this when the spec is sorted w.r.t. MP4.
	//mp4Sig(0),

	textSig(0), // should be last
}

type exactSig struct {
	sig []byte
	ct  string
}

func (e *exactSig) match(data []byte, firstNonWS int) string {
	if bytes.HasPrefix(data, e.sig) {
		return e.ct
	}
	return ""
}

type maskedSig struct {
	mask, pat []byte
	skipWS    bool
	ct        string
}

func (m *maskedSig) match(data []byte, firstNonWS int) string {
	if m.skipWS {
		data = data[firstNonWS:]
	}
	if len(data) < len(m.mask) {
		return ""
	}
	for i, mask := range m.mask {
		db := data[i] & mask
		if db != m.pat[i] {
			return ""
		}
	}
	return m.ct
}

type htmlSig []byte

func (h htmlSig) match(data []byte, firstNonWS int) string {
	data = data[firstNonWS:]
	if len(data) < len(h)+1 {
		return ""
	}
	for i, b := range h {
		db := data[i]
		if 'A' <= b && b <= 'Z' {
			db &= 0xDF
		}
		if b != db {
			return ""
		}
	}
	// Next byte must be space or right angle bracket.
	if db := data[len(h)]; db != ' ' && db != '>' {
		return ""
	}
	return "text/html; charset=utf-8"
}

type mp4Sig int

func (mp4Sig) match(data []byte, firstNonWS int) string {
	// c.f. section 6.1.
	if len(data) < 8 {
		return ""
	}
	boxSize := int(binary.BigEndian.Uint32(data[:4]))
	if boxSize%4 != 0 || len(data) < boxSize {
		return ""
	}
	if !bytes.Equal(data[4:8], []byte("ftyp")) {
		return ""
	}
	for st := 8; st < boxSize; st += 4 {
		if st == 12 {
			// minor version number
			continue
		}
		seg := string(data[st : st+3])
		switch seg {
		case "mp4", "iso", "M4V", "M4P", "M4B":
			return "video/mp4"
			/* The remainder are not in the spec.
			case "M4A":
				return "audio/mp4"
			case "3gp":
				return "video/3gpp"
			case "jp2":
				return "image/jp2" // JPEG 2000
			*/
		}
	}
	return ""
}

type textSig int

func (textSig) match(data []byte, firstNonWS int) string {
	// c.f. section 5, step 4.
	for _, b := range data[firstNonWS:] {
		switch {
		case 0x00 <= b && b <= 0x08,
			b == 0x0B,
			0x0E <= b && b <= 0x1A,
			0x1C <= b && b <= 0x1F:
			return ""
		}
	}
	return "text/plain; charset=utf-8"
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"flag"
	"log"
	"net/http"
	"strings"

	"code.google.com/p/go.net/websocket"
	"github.com/GeertJohan/go.rice"
	"github.com/bmizerany/pat"
	_ "github.com/mattn/go-sqlite3"
	"github.com/russross/meddler"

	"github.com/drone/drone/pkg/channel"
	"github.com/drone/drone/pkg/database"
	"github.com/drone/drone/pkg/handler"
)

var (
	// local path where the SQLite database
	// should be stored. By default this is
	// in the current working directory.
	path string

	// port the server will run on
	port string

	// database driver used to connect to the database
	driver string

	// driver specific connection information. In this
	// case, it should be the location of the SQLite file
	datasource string
)

func main() {
	// parse command line flags
	flag.StringVar(&path, "path", "", "")
	flag.StringVar(&port, "port", ":8080", "")
	flag.StringVar(&driver, "driver", "sqlite3", "")
	flag.StringVar(&datasource, "datasource", "drone.sqlite", "")
	flag.Parse()

	// setup database and handlers
	setupDatabase()
	setupStatic()
	setupHandlers()

	// start the webserver on the default port.
	panic(http.ListenAndServe(port, nil))
}

// setup the database connection and register with the
// global database package.
func setupDatabase() {
	// inform meddler we're using sqlite
	meddler.Default = meddler.SQLite

	// connect to the SQLite database
	db, err := sql.Open(driver, datasource)
	if err != nil {
		log.Fatal(err)
	}

	database.Set(db)
}

// setup routes for static assets. These assets may
// be directly embedded inside the application using
// the `rice embed` command, else they are served from disk.
func setupStatic() {
	box := rice.MustFindBox("assets")
	http.Handle("/css/", http.FileServer(box.HTTPBox()))
	http.Handle("/img/", http.FileServer(box.HTTPBox()))
}

// setup routes for serving dynamic content.
func setupHandlers() {
	m := pat.New()
	m.Get("/login", handler.ErrorHandler(handler.Login))
	m.Post("/login", handler.ErrorHandler(handler.Authorize))
	m.Get("/logout", handler.ErrorHandler(handler.Logout))
	m.Get("/forgot", handler.ErrorHandler(handler.Forgot))
	m.Post("/forgot", handler.ErrorHandler(handler.ForgotPost))
	m.Get("/reset", handler.ErrorHandler(handler.Reset))
	m.Post("/reset", handler.ErrorHandler(handler.ResetPost))
	m.Get("/register", handler.ErrorHandler(handler.Register))
	m.Post("/register", handler.ErrorHandler(handler.RegisterPost))
	m.Get("/accept", handler.UserHandler(handler.TeamMemberAccept))

	// handlers for setting up your GitHub repository
	m.Post("/new/github.com", handler.UserHandler(handler.RepoCreateGithub))
	m.Get("/new/github.com", handler.UserHandler(handler.RepoAdd))

	// handlers for linking your GitHub account
	m.Get("/auth/login/github", handler.UserHandler(handler.LinkGithub))

	// handlers for dashboard pages
	m.Get("/dashboard/team/:team", handler.UserHandler(handler.TeamShow))
	m.Get("/dashboard", handler.UserHandler(handler.UserShow))

	// handlers for user account management
	m.Get("/account/user/profile", handler.UserHandler(handler.UserEdit))
	m.Post("/account/user/profile", handler.UserHandler(handler.UserUpdate))
	m.Get("/account/user/delete", handler.UserHandler(handler.UserDeleteConfirm))
	m.Post("/account/user/delete", handler.UserHandler(handler.UserDelete))
	m.Get("/account/user/password", handler.UserHandler(handler.UserPass))
	m.Post("/account/user/password", handler.UserHandler(handler.UserPassUpdate))
	m.Get("/account/user/teams/add", handler.UserHandler(handler.TeamAdd))
	m.Post("/account/user/teams/add", handler.UserHandler(handler.TeamCreate))
	m.Get("/account/user/teams", handler.UserHandler(handler.UserTeams))

	// handlers for team managements
	m.Get("/account/team/:team/profile", handler.UserHandler(handler.TeamEdit))
	m.Post("/account/team/:team/profile", handler.UserHandler(handler.TeamUpdate))
	m.Get("/account/team/:team/delete", handler.UserHandler(handler.TeamDeleteConfirm))
	m.Post("/account/team/:team/delete", handler.UserHandler(handler.TeamDelete))
	m.Get("/account/team/:team/members/add", handler.UserHandler(handler.TeamMemberAdd))
	m.Post("/account/team/:team/members/add", handler.UserHandler(handler.TeamMemberInvite))
	m.Get("/account/team/:team/members/edit", handler.UserHandler(handler.TeamMemberEdit))
	m.Post("/account/team/:team/members/edit", handler.UserHandler(handler.TeamMemberUpdate))
	m.Post("/account/team/:team/members/delete", handler.UserHandler(handler.TeamMemberDelete))
	m.Get("/account/team/:team/members", handler.UserHandler(handler.TeamMembers))

	// handlers for system administration
	m.Get("/account/admin/settings", handler.AdminHandler(handler.AdminSettings))
	m.Post("/account/admin/settings", handler.AdminHandler(handler.AdminSettingsUpdate))
	m.Get("/account/admin/users/edit", handler.AdminHandler(handler.AdminUserEdit))
	m.Post("/account/admin/users/edit", handler.AdminHandler(handler.AdminUserUpdate))
	m.Post("/account/admin/users/delete", handler.AdminHandler(handler.AdminUserDelete))
	m.Get("/account/admin/users/add", handler.AdminHandler(handler.AdminUserAdd))
	m.Post("/account/admin/users", handler.AdminHandler(handler.AdminUserInvite))
	m.Get("/account/admin/users", handler.AdminHandler(handler.AdminUserList))

	// handlers for GitHub post-commit hooks
	m.Post("/hook/github.com", handler.ErrorHandler(handler.Hook))

	// handlers for first-time installation
	m.Get("/install", handler.ErrorHandler(handler.Install))
	m.Post("/install", handler.ErrorHandler(handler.InstallPost))

	// handlers for repository, commits and build details
	m.Get("/:host/:owner/:name/commit/:commit/build/:label/out.txt", handler.RepoHandler(handler.BuildOut))
	m.Get("/:host/:owner/:name/commit/:commit/build/:label", handler.RepoHandler(handler.CommitShow))
	m.Get("/:host/:owner/:name/commit/:commit", handler.RepoHandler(handler.CommitShow))
	m.Get("/:host/:owner/:name/tree", handler.RepoHandler(handler.RepoDashboard))
	m.Get("/:host/:owner/:name/status.png", handler.ErrorHandler(handler.Badge))
	m.Get("/:host/:owner/:name/settings", handler.RepoAdminHandler(handler.RepoSettingsForm))
	m.Get("/:host/:owner/:name/params", handler.RepoAdminHandler(handler.RepoParamsForm))
	m.Get("/:host/:owner/:name/badges", handler.RepoAdminHandler(handler.RepoBadges))
	m.Get("/:host/:owner/:name/keys", handler.RepoAdminHandler(handler.RepoKeys))
	m.Get("/:host/:owner/:name/delete", handler.RepoAdminHandler(handler.RepoDeleteForm))
	m.Post("/:host/:owner/:name/delete", handler.RepoAdminHandler(handler.RepoDelete))
	m.Get("/:host/:owner/:name", handler.RepoHandler(handler.RepoDashboard))
	m.Post("/:host/:owner/:name", handler.RepoHandler(handler.RepoUpdate))
	http.Handle("/feed", websocket.Handler(channel.Read))

	// no routes are served at the root URL. Instead we will
	// redirect the user to his/her dashboard page.
	m.Get("/", http.RedirectHandler("/dashboard", http.StatusSeeOther))

	// the first time a page is requested we should record
	// the scheme and hostname.
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		// get the hostname and scheme

		// our multiplexer is a bit finnicky and therefore requires
		// us to strip any trailing slashes in order to correctly
		// find and match a route.
		if r.URL.Path != "/" && strings.HasSuffix(r.URL.Path, "/") {
			http.Redirect(w, r, r.URL.Path[:len(r.URL.Path)-1], http.StatusSeeOther)
			return
		}

		// standard header variables that should be set, for good measure.
		w.Header().Add("Cache-Control", "no-cache, no-store, max-age=0, must-revalidate")
		w.Header().Add("X-Frame-Options", "DENY")
		w.Header().Add("X-Content-Type-Options", "nosniff")
		w.Header().Add("X-XSS-Protection", "1; mode=block")

		// ok, now we're ready to serve the request.
		m.ServeHTTP(w, r)
	})
}
<commit_msg>fix for #70 to prevent caching build images<commit_after>package main

import (
	"database/sql"
	"flag"
	"log"
	"net/http"
	"strings"

	"code.google.com/p/go.net/websocket"
	"github.com/GeertJohan/go.rice"
	"github.com/bmizerany/pat"
	_ "github.com/mattn/go-sqlite3"
	"github.com/russross/meddler"

	"github.com/drone/drone/pkg/channel"
	"github.com/drone/drone/pkg/database"
	"github.com/drone/drone/pkg/handler"
)

var (
	// local path where the SQLite database
	// should be stored. By default this is
	// in the current working directory.
	path string

	// port the server will run on
	port string

	// database driver used to connect to the database
	driver string

	// driver specific connection information. In this
	// case, it should be the location of the SQLite file
	datasource string
)

func main() {
	// parse command line flags
	flag.StringVar(&path, "path", "", "")
	flag.StringVar(&port, "port", ":8080", "")
	flag.StringVar(&driver, "driver", "sqlite3", "")
	flag.StringVar(&datasource, "datasource", "drone.sqlite", "")
	flag.Parse()

	// setup database and handlers
	setupDatabase()
	setupStatic()
	setupHandlers()

	// start the webserver on the default port.
	panic(http.ListenAndServe(port, nil))
}

// setup the database connection and register with the
// global database package.
func setupDatabase() {
	// inform meddler we're using sqlite
	meddler.Default = meddler.SQLite

	// connect to the SQLite database
	db, err := sql.Open(driver, datasource)
	if err != nil {
		log.Fatal(err)
	}

	database.Set(db)
}

// setup routes for static assets. These assets may
// be directly embedded inside the application using
// the `rice embed` command, else they are served from disk.
func setupStatic() {
	box := rice.MustFindBox("assets")
	http.Handle("/css/", http.FileServer(box.HTTPBox()))

	// we need to intercept all attempts to serve images
	// so that we can add a cache-control settings
	var images = http.FileServer(box.HTTPBox())
	http.HandleFunc("/img/", func(w http.ResponseWriter, r *http.Request) {
		if strings.HasPrefix(r.URL.Path, "/img/build_") {
			w.Header().Add("Cache-Control", "no-cache")
		}

		// serce images
		images.ServeHTTP(w, r)
	})
}

// setup routes for serving dynamic content.
func setupHandlers() {
	m := pat.New()
	m.Get("/login", handler.ErrorHandler(handler.Login))
	m.Post("/login", handler.ErrorHandler(handler.Authorize))
	m.Get("/logout", handler.ErrorHandler(handler.Logout))
	m.Get("/forgot", handler.ErrorHandler(handler.Forgot))
	m.Post("/forgot", handler.ErrorHandler(handler.ForgotPost))
	m.Get("/reset", handler.ErrorHandler(handler.Reset))
	m.Post("/reset", handler.ErrorHandler(handler.ResetPost))
	m.Get("/register", handler.ErrorHandler(handler.Register))
	m.Post("/register", handler.ErrorHandler(handler.RegisterPost))
	m.Get("/accept", handler.UserHandler(handler.TeamMemberAccept))

	// handlers for setting up your GitHub repository
	m.Post("/new/github.com", handler.UserHandler(handler.RepoCreateGithub))
	m.Get("/new/github.com", handler.UserHandler(handler.RepoAdd))

	// handlers for linking your GitHub account
	m.Get("/auth/login/github", handler.UserHandler(handler.LinkGithub))

	// handlers for dashboard pages
	m.Get("/dashboard/team/:team", handler.UserHandler(handler.TeamShow))
	m.Get("/dashboard", handler.UserHandler(handler.UserShow))

	// handlers for user account management
	m.Get("/account/user/profile", handler.UserHandler(handler.UserEdit))
	m.Post("/account/user/profile", handler.UserHandler(handler.UserUpdate))
	m.Get("/account/user/delete", handler.UserHandler(handler.UserDeleteConfirm))
	m.Post("/account/user/delete", handler.UserHandler(handler.UserDelete))
	m.Get("/account/user/password", handler.UserHandler(handler.UserPass))
	m.Post("/account/user/password", handler.UserHandler(handler.UserPassUpdate))
	m.Get("/account/user/teams/add", handler.UserHandler(handler.TeamAdd))
	m.Post("/account/user/teams/add", handler.UserHandler(handler.TeamCreate))
	m.Get("/account/user/teams", handler.UserHandler(handler.UserTeams))

	// handlers for team managements
	m.Get("/account/team/:team/profile", handler.UserHandler(handler.TeamEdit))
	m.Post("/account/team/:team/profile", handler.UserHandler(handler.TeamUpdate))
	m.Get("/account/team/:team/delete", handler.UserHandler(handler.TeamDeleteConfirm))
	m.Post("/account/team/:team/delete", handler.UserHandler(handler.TeamDelete))
	m.Get("/account/team/:team/members/add", handler.UserHandler(handler.TeamMemberAdd))
	m.Post("/account/team/:team/members/add", handler.UserHandler(handler.TeamMemberInvite))
	m.Get("/account/team/:team/members/edit", handler.UserHandler(handler.TeamMemberEdit))
	m.Post("/account/team/:team/members/edit", handler.UserHandler(handler.TeamMemberUpdate))
	m.Post("/account/team/:team/members/delete", handler.UserHandler(handler.TeamMemberDelete))
	m.Get("/account/team/:team/members", handler.UserHandler(handler.TeamMembers))

	// handlers for system administration
	m.Get("/account/admin/settings", handler.AdminHandler(handler.AdminSettings))
	m.Post("/account/admin/settings", handler.AdminHandler(handler.AdminSettingsUpdate))
	m.Get("/account/admin/users/edit", handler.AdminHandler(handler.AdminUserEdit))
	m.Post("/account/admin/users/edit", handler.AdminHandler(handler.AdminUserUpdate))
	m.Post("/account/admin/users/delete", handler.AdminHandler(handler.AdminUserDelete))
	m.Get("/account/admin/users/add", handler.AdminHandler(handler.AdminUserAdd))
	m.Post("/account/admin/users", handler.AdminHandler(handler.AdminUserInvite))
	m.Get("/account/admin/users", handler.AdminHandler(handler.AdminUserList))

	// handlers for GitHub post-commit hooks
	m.Post("/hook/github.com", handler.ErrorHandler(handler.Hook))

	// handlers for first-time installation
	m.Get("/install", handler.ErrorHandler(handler.Install))
	m.Post("/install", handler.ErrorHandler(handler.InstallPost))

	// handlers for repository, commits and build details
	m.Get("/:host/:owner/:name/commit/:commit/build/:label/out.txt", handler.RepoHandler(handler.BuildOut))
	m.Get("/:host/:owner/:name/commit/:commit/build/:label", handler.RepoHandler(handler.CommitShow))
	m.Get("/:host/:owner/:name/commit/:commit", handler.RepoHandler(handler.CommitShow))
	m.Get("/:host/:owner/:name/tree", handler.RepoHandler(handler.RepoDashboard))
	m.Get("/:host/:owner/:name/status.png", handler.ErrorHandler(handler.Badge))
	m.Get("/:host/:owner/:name/settings", handler.RepoAdminHandler(handler.RepoSettingsForm))
	m.Get("/:host/:owner/:name/params", handler.RepoAdminHandler(handler.RepoParamsForm))
	m.Get("/:host/:owner/:name/badges", handler.RepoAdminHandler(handler.RepoBadges))
	m.Get("/:host/:owner/:name/keys", handler.RepoAdminHandler(handler.RepoKeys))
	m.Get("/:host/:owner/:name/delete", handler.RepoAdminHandler(handler.RepoDeleteForm))
	m.Post("/:host/:owner/:name/delete", handler.RepoAdminHandler(handler.RepoDelete))
	m.Get("/:host/:owner/:name", handler.RepoHandler(handler.RepoDashboard))
	m.Post("/:host/:owner/:name", handler.RepoHandler(handler.RepoUpdate))
	http.Handle("/feed", websocket.Handler(channel.Read))

	// no routes are served at the root URL. Instead we will
	// redirect the user to his/her dashboard page.
	m.Get("/", http.RedirectHandler("/dashboard", http.StatusSeeOther))

	// the first time a page is requested we should record
	// the scheme and hostname.
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		// our multiplexer is a bit finnicky and therefore requires
		// us to strip any trailing slashes in order to correctly
		// find and match a route.
		if r.URL.Path != "/" && strings.HasSuffix(r.URL.Path, "/") {
			http.Redirect(w, r, r.URL.Path[:len(r.URL.Path)-1], http.StatusSeeOther)
			return
		}

		// standard header variables that should be set, for good measure.
		w.Header().Add("Cache-Control", "no-cache, no-store, max-age=0, must-revalidate")
		w.Header().Add("X-Frame-Options", "DENY")
		w.Header().Add("X-Content-Type-Options", "nosniff")
		w.Header().Add("X-XSS-Protection", "1; mode=block")

		// ok, now we're ready to serve the request.
		m.ServeHTTP(w, r)
	})
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The math package provides basic constants and mathematical functions.
package math

// Mathematical constants.
// Reference: http://www.research.att.com/~njas/sequences/Axxxxxx
const (
	E	= 2.71828182845904523536028747135266249775724709369995957496696763;  // A001113
	Pi	= 3.14159265358979323846264338327950288419716939937510582097494459;  // A000796
	Phi	= 1.61803398874989484820458683436563811772030917980576286213544862;  // A001622

	Sqrt2	= 1.41421356237309504880168872420969807856967187537694807317667974;  // A002193
	SqrtE	= 1.64872127070012814684865078781416357165377610071014801157507931;  // A019774
	SqrtPi	= 1.77245385090551602729816748334114518279754945612238712821380779;  // A002161
	SqrtPhi	= 1.27201964951406896425242246173749149171560804184009624861664038;  // A139339

	Ln2	= 0.693147180559945309417232121458176568075500134360255254120680009; // A002162
	Log2E	= 1/Ln2;
	Ln10	= 2.30258509299404568401799145468436420760110148862877297603332790;  // A002392
	Log10E	= 1/Ln10;
)

// Limit values
const (
	MaxFloat32	= 3.40282346638528860e+38;
	MinFloat32	= 1.40129846432481707e-45;
	MaxFloat64	= 1.7976931348623157e+308;
	MinFloat64	= 5.0e-324;

	MaxInt8 = 1<<7 - 1;
	MinInt8 = -1<<7;
	MaxInt16 = 1<<15 - 1;
	MinInt16 = -1<<15;
	MaxInt32 = 1<<31 - 1;
	MinInt32 = -1<<31;
	MaxInt64 = 1<<63 - 1;
	MinInt64 = -1<<63;
	MaxUint8 = 1<<8 - 1;
	MaxUint16 = 1<<16 - 1;
	MaxUint32 = 1<<32 - 1;
	MaxUint64 = 1<<64 - 1;
)

// BUG(rsc): The manual should define the special cases for all of these functions.
<commit_msg>floating-point limits and equations<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The math package provides basic constants and mathematical functions.
package math

// Mathematical constants.
// Reference: http://www.research.att.com/~njas/sequences/Axxxxxx
const (
	E	= 2.71828182845904523536028747135266249775724709369995957496696763;  // A001113
	Pi	= 3.14159265358979323846264338327950288419716939937510582097494459;  // A000796
	Phi	= 1.61803398874989484820458683436563811772030917980576286213544862;  // A001622

	Sqrt2	= 1.41421356237309504880168872420969807856967187537694807317667974;  // A002193
	SqrtE	= 1.64872127070012814684865078781416357165377610071014801157507931;  // A019774
	SqrtPi	= 1.77245385090551602729816748334114518279754945612238712821380779;  // A002161
	SqrtPhi	= 1.27201964951406896425242246173749149171560804184009624861664038;  // A139339

	Ln2	= 0.693147180559945309417232121458176568075500134360255254120680009; // A002162
	Log2E	= 1/Ln2;
	Ln10	= 2.30258509299404568401799145468436420760110148862877297603332790;  // A002392
	Log10E	= 1/Ln10;
)

// Floating-point limit values.
// Max is the largest finite value representable by the type.
// Min is the smallest nonzero value representable by the type.
const (
	MaxFloat32 = 3.40282346638528859811704183484516925440e+38;	/* 2^127 * (2^24 - 1) / 2^23 */
	MinFloat32 = 1.401298464324817070923729583289916131280e-45;	/* 1 / 2^(127 - 1 + 23) */

	MaxFloat64 = 1.797693134862315708145274237317043567981e+308;	/* 2^1023 * (2^53 - 1) / 2^52 */
	MinFloat64 = 4.940656458412465441765687928682213723651e-324;	/* 1 / 2^(1023 - 1 + 52) */
)

// Integer limit values.
const (
	MaxInt8 = 1<<7 - 1;
	MinInt8 = -1<<7;
	MaxInt16 = 1<<15 - 1;
	MinInt16 = -1<<15;
	MaxInt32 = 1<<31 - 1;
	MinInt32 = -1<<31;
	MaxInt64 = 1<<63 - 1;
	MinInt64 = -1<<63;
	MaxUint8 = 1<<8 - 1;
	MaxUint16 = 1<<16 - 1;
	MaxUint32 = 1<<32 - 1;
	MaxUint64 = 1<<64 - 1;
)

// BUG(rsc): The manual should define the special cases for all of these functions.
<|endoftext|>


<commit_before>// Copyright 2014 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.

// ethtest executes Ethereum JSON tests.
package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/codegangsta/cli"
	"github.com/ethereum/go-ethereum/logger/glog"
	"github.com/ethereum/go-ethereum/tests"
)

var (
	continueOnError = false
	testExtension   = ".json"
	defaultTest     = "all"
	defaultDir      = "."
	allTests        = []string{"BlockTests", "StateTests", "TransactionTests", "VMTests", "RLPTests"}
	testDirMapping  = map[string]string{"BlockTests": "BlockchainTests"}
	skipTests       = []string{}

	TestFlag = cli.StringFlag{
		Name:  "test",
		Usage: "Test type (string): VMTests, TransactionTests, StateTests, BlockTests",
		Value: defaultTest,
	}
	FileFlag = cli.StringFlag{
		Name:   "file",
		Usage:  "Test file or directory. Directories are searched for .json files 1 level deep",
		Value:  defaultDir,
		EnvVar: "ETHEREUM_TEST_PATH",
	}
	ContinueOnErrorFlag = cli.BoolFlag{
		Name:  "continue",
		Usage: "Continue running tests on error (true) or [default] exit immediately (false)",
	}
	ReadStdInFlag = cli.BoolFlag{
		Name:  "stdin",
		Usage: "Accept input from stdin instead of reading from file",
	}
	SkipTestsFlag = cli.StringFlag{
		Name:  "skip",
		Usage: "Tests names to skip",
	}
)

func runTestWithReader(test string, r io.Reader) error {
	glog.Infoln("runTest", test)
	var err error
	switch strings.ToLower(test) {
	case "bk", "block", "blocktest", "blockchaintest", "blocktests", "blockchaintests":
		err = tests.RunBlockTestWithReader(r, skipTests)
	case "st", "state", "statetest", "statetests":
		err = tests.RunStateTestWithReader(r, skipTests)
	case "tx", "transactiontest", "transactiontests":
		err = tests.RunTransactionTestsWithReader(r, skipTests)
	case "vm", "vmtest", "vmtests":
		err = tests.RunVmTestWithReader(r, skipTests)
	case "rlp", "rlptest", "rlptests":
		err = tests.RunRLPTestWithReader(r, skipTests)
	default:
		err = fmt.Errorf("Invalid test type specified: %v", test)
	}

	if err != nil {
		return err
	}

	return nil
}

func getFiles(path string) ([]string, error) {
	glog.Infoln("getFiles", path)
	var files []string
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	fi, err := f.Stat()
	if err != nil {
		return nil, err
	}

	switch mode := fi.Mode(); {
	case mode.IsDir():
		fi, _ := ioutil.ReadDir(path)
		files = make([]string, len(fi))
		for i, v := range fi {
			// only go 1 depth and leave directory entires blank
			if !v.IsDir() && v.Name()[len(v.Name())-len(testExtension):len(v.Name())] == testExtension {
				files[i] = filepath.Join(path, v.Name())
				glog.Infoln("Found file", files[i])
			}
		}
	case mode.IsRegular():
		files = make([]string, 1)
		files[0] = path
	}

	return files, nil
}

func runSuite(test, file string) {
	var tests []string

	if test == defaultTest {
		tests = allTests
	} else {
		tests = []string{test}
	}

	for _, curTest := range tests {
		glog.Infoln("runSuite", curTest, file)
		var err error
		var files []string
		if test == defaultTest {
			// check if we have an explicit directory mapping for the test
			if _, ok := testDirMapping[curTest]; ok {
				files, err = getFiles(filepath.Join(file, testDirMapping[curTest]))
			} else {
				// otherwise assume test name
				files, err = getFiles(filepath.Join(file, curTest))
			}
		} else {
			files, err = getFiles(file)
		}
		if err != nil {
			glog.Fatalln(err)
		}

		if len(files) == 0 {
			glog.Warningln("No files matched path")
		}
		for _, curFile := range files {
			// Skip blank entries
			if len(curFile) == 0 {
				continue
			}

			r, err := os.Open(curFile)
			if err != nil {
				glog.Fatalln(err)
			}
			defer r.Close()

			err = runTestWithReader(curTest, r)
			if err != nil {
				if continueOnError {
					glog.Errorln(err)
				} else {
					glog.Fatalln(err)
				}
			}

		}
	}
}

func setupApp(c *cli.Context) {
	flagTest := c.GlobalString(TestFlag.Name)
	flagFile := c.GlobalString(FileFlag.Name)
	continueOnError = c.GlobalBool(ContinueOnErrorFlag.Name)
	useStdIn := c.GlobalBool(ReadStdInFlag.Name)
	skipTests = strings.Split(c.GlobalString(SkipTestsFlag.Name), " ")

	if !useStdIn {
		runSuite(flagTest, flagFile)
	} else {
		if err := runTestWithReader(flagTest, os.Stdin); err != nil {
			glog.Fatalln(err)
		}

	}
}

func main() {
	glog.SetToStderr(true)

	app := cli.NewApp()
	app.Name = "ethtest"
	app.Usage = "go-ethereum test interface"
	app.Action = setupApp
	app.Version = "0.2.0"
	app.Author = "go-ethereum team"

	app.Flags = []cli.Flag{
		TestFlag,
		FileFlag,
		ContinueOnErrorFlag,
		ReadStdInFlag,
		SkipTestsFlag,
	}

	if err := app.Run(os.Args); err != nil {
		glog.Fatalln(err)
	}

}
<commit_msg>cmd/ethtest: added trace flag for debugging<commit_after>// Copyright 2014 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.

// ethtest executes Ethereum JSON tests.
package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/codegangsta/cli"
	"github.com/ethereum/go-ethereum/core/vm"
	"github.com/ethereum/go-ethereum/logger/glog"
	"github.com/ethereum/go-ethereum/tests"
)

var (
	continueOnError = false
	testExtension   = ".json"
	defaultTest     = "all"
	defaultDir      = "."
	allTests        = []string{"BlockTests", "StateTests", "TransactionTests", "VMTests", "RLPTests"}
	testDirMapping  = map[string]string{"BlockTests": "BlockchainTests"}
	skipTests       = []string{}

	TestFlag = cli.StringFlag{
		Name:  "test",
		Usage: "Test type (string): VMTests, TransactionTests, StateTests, BlockTests",
		Value: defaultTest,
	}
	FileFlag = cli.StringFlag{
		Name:   "file",
		Usage:  "Test file or directory. Directories are searched for .json files 1 level deep",
		Value:  defaultDir,
		EnvVar: "ETHEREUM_TEST_PATH",
	}
	ContinueOnErrorFlag = cli.BoolFlag{
		Name:  "continue",
		Usage: "Continue running tests on error (true) or [default] exit immediately (false)",
	}
	ReadStdInFlag = cli.BoolFlag{
		Name:  "stdin",
		Usage: "Accept input from stdin instead of reading from file",
	}
	SkipTestsFlag = cli.StringFlag{
		Name:  "skip",
		Usage: "Tests names to skip",
	}
	TraceFlag = cli.BoolFlag{
		Name:  "trace",
		Usage: "Enable VM tracing",
	}
)

func runTestWithReader(test string, r io.Reader) error {
	glog.Infoln("runTest", test)
	var err error
	switch strings.ToLower(test) {
	case "bk", "block", "blocktest", "blockchaintest", "blocktests", "blockchaintests":
		err = tests.RunBlockTestWithReader(r, skipTests)
	case "st", "state", "statetest", "statetests":
		err = tests.RunStateTestWithReader(r, skipTests)
	case "tx", "transactiontest", "transactiontests":
		err = tests.RunTransactionTestsWithReader(r, skipTests)
	case "vm", "vmtest", "vmtests":
		err = tests.RunVmTestWithReader(r, skipTests)
	case "rlp", "rlptest", "rlptests":
		err = tests.RunRLPTestWithReader(r, skipTests)
	default:
		err = fmt.Errorf("Invalid test type specified: %v", test)
	}

	if err != nil {
		return err
	}

	return nil
}

func getFiles(path string) ([]string, error) {
	glog.Infoln("getFiles", path)
	var files []string
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	fi, err := f.Stat()
	if err != nil {
		return nil, err
	}

	switch mode := fi.Mode(); {
	case mode.IsDir():
		fi, _ := ioutil.ReadDir(path)
		files = make([]string, len(fi))
		for i, v := range fi {
			// only go 1 depth and leave directory entires blank
			if !v.IsDir() && v.Name()[len(v.Name())-len(testExtension):len(v.Name())] == testExtension {
				files[i] = filepath.Join(path, v.Name())
				glog.Infoln("Found file", files[i])
			}
		}
	case mode.IsRegular():
		files = make([]string, 1)
		files[0] = path
	}

	return files, nil
}

func runSuite(test, file string) {
	var tests []string

	if test == defaultTest {
		tests = allTests
	} else {
		tests = []string{test}
	}

	for _, curTest := range tests {
		glog.Infoln("runSuite", curTest, file)
		var err error
		var files []string
		if test == defaultTest {
			// check if we have an explicit directory mapping for the test
			if _, ok := testDirMapping[curTest]; ok {
				files, err = getFiles(filepath.Join(file, testDirMapping[curTest]))
			} else {
				// otherwise assume test name
				files, err = getFiles(filepath.Join(file, curTest))
			}
		} else {
			files, err = getFiles(file)
		}
		if err != nil {
			glog.Fatalln(err)
		}

		if len(files) == 0 {
			glog.Warningln("No files matched path")
		}
		for _, curFile := range files {
			// Skip blank entries
			if len(curFile) == 0 {
				continue
			}

			r, err := os.Open(curFile)
			if err != nil {
				glog.Fatalln(err)
			}
			defer r.Close()

			err = runTestWithReader(curTest, r)
			if err != nil {
				if continueOnError {
					glog.Errorln(err)
				} else {
					glog.Fatalln(err)
				}
			}
		}
	}
}

func setupApp(c *cli.Context) {
	flagTest := c.GlobalString(TestFlag.Name)
	flagFile := c.GlobalString(FileFlag.Name)
	continueOnError = c.GlobalBool(ContinueOnErrorFlag.Name)
	useStdIn := c.GlobalBool(ReadStdInFlag.Name)
	skipTests = strings.Split(c.GlobalString(SkipTestsFlag.Name), " ")
	vm.Debug = c.GlobalBool(TraceFlag.Name)

	if !useStdIn {
		runSuite(flagTest, flagFile)
	} else {
		if err := runTestWithReader(flagTest, os.Stdin); err != nil {
			glog.Fatalln(err)
		}

	}
}

func main() {
	glog.SetToStderr(true)

	app := cli.NewApp()
	app.Name = "ethtest"
	app.Usage = "go-ethereum test interface"
	app.Action = setupApp
	app.Version = "0.2.0"
	app.Author = "go-ethereum team"

	app.Flags = []cli.Flag{
		TestFlag,
		FileFlag,
		ContinueOnErrorFlag,
		ReadStdInFlag,
		SkipTestsFlag,
		TraceFlag,
	}

	if err := app.Run(os.Args); err != nil {
		glog.Fatalln(err)
	}

}
<|endoftext|>


<commit_before>

import (
	"github.com/astaxie/beego"
	"github.com/ulule/limiter"
	"github.com/ulule/limiter/drivers/store/memory"
	"github.com/astaxie/beego/context"
	"net/http"
	"strconv"
)

var rateLimiter *limiter.Limiter

func main() {
	rate, err := limiter.NewRateFromFormatted("2-H")
	if err != nil{
		panic(err)
	}
	store := memory.NewStore()

	rateLimiter = limiter.New(store, rate)

	//More on Beego filters here https://beego.me/docs/mvc/controller/filter.md
	beego.InsertFilter("*", beego.BeforeRouter, rateLimit)
	beego.Run()
}

func rateLimit(ctx *context.Context) {
	r := ctx.Request

	context, err := rateLimiter.Get(r.Context(), limiter.GetIPKey(r, false))
	if err != nil {
		ctx.Abort(http.StatusInternalServerError, "Internal Server Error")
		return
	}

	h := ctx.ResponseWriter.Header()
	h.Add("X-RateLimit-Limit", strconv.FormatInt(context.Limit, 10))
	h.Add("X-RateLimit-Remaining", strconv.FormatInt(context.Remaining, 10))
	h.Add("X-RateLimit-Reset", strconv.FormatInt(context.Reset, 10))

	if context.Reached {
		ctx.Abort(http.StatusTooManyRequests, "Too Many Requests")
		return
	}

}
<commit_msg>Add extra error handling help<commit_after>

import (
	"github.com/astaxie/beego"
	"github.com/ulule/limiter"
	"github.com/ulule/limiter/drivers/store/memory"
	"github.com/astaxie/beego/context"
	"net/http"
	"strconv"
)

var rateLimiter *limiter.Limiter

func main() {
	rate, err := limiter.NewRateFromFormatted("2-H")
	if err != nil{
		panic(err)
	}
	store := memory.NewStore()

	rateLimiter = limiter.New(store, rate)

	//More on Beego filters here https://beego.me/docs/mvc/controller/filter.md
	beego.InsertFilter("*", beego.BeforeRouter, rateLimit)
	beego.Run()
}

func rateLimit(ctx *context.Context) {
	r := ctx.Request

	context, err := rateLimiter.Get(r.Context(), limiter.GetIPKey(r, false))
	if err != nil {
		ctx.Abort(http.StatusInternalServerError, "Internal Server Error")
		return
	}

	h := ctx.ResponseWriter.Header()
	h.Add("X-RateLimit-Limit", strconv.FormatInt(context.Limit, 10))
	h.Add("X-RateLimit-Remaining", strconv.FormatInt(context.Remaining, 10))
	h.Add("X-RateLimit-Reset", strconv.FormatInt(context.Reset, 10))

	if context.Reached {
		//This will cause a panic on the logs. To avoid this, add the error string to
		//Beego.ErrorMaps as discribed here https://beego.me/docs/mvc/controller/errors.md
		ctx.Abort(http.StatusTooManyRequests, "Too Many Requests")
		return
	}

}
<|endoftext|>


<commit_before>// Copyright © 2017 Slotix s.r.o. <dm@slotix.sk>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package main

import (
	"fmt"
	"os"
	"os/signal"
	"time"

	"github.com/slotix/dataflowkit/fetch"
	"github.com/slotix/dataflowkit/healthcheck"

	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

var (
	//VERSION               string // VERSION is set during build
	//  DFKFetch represents address of DFK Fetch service
	DFKFetch              string //Fetch service address
	splashHost            string
	splashTimeout         int
	splashResourceTimeout int
	splashWait            float64

	storageType     string
	skipStorageMW   bool
	ignoreCacheInfo bool
	diskvBaseDir    string

	//Digital Ocean spaces configuration file
	spacesConfig string
	//Digital Ocean spaces endpoint address
	spacesEndpoint string
	s3Region       string
	//Bucket name for AWS S3 or DO Spaces
	DFKBucket string

	redisHost       string
	redisExpire     int
	redisNetwork    string
	redisPassword   string
	redisDB         int
	redisSocketPath string
)

// RootCmd represents the base command when called without any subcommands
var RootCmd = &cobra.Command{
	Use:   "dataflowkit",
	Short: "Dataflow Kit html fetcher",
	Long:  `Dataflow Kit fetch service retrieves html pages from websites and passes content to DFK parser service.`,
	Run: func(cmd *cobra.Command, args []string) {
		fmt.Println("Checking services ... ")
		services := []healthcheck.Checker{
			healthcheck.SplashConn{
				Host: viper.GetString("SPLASH"),
			},
		}
		if storageType == "Redis" {
			services = append(services, healthcheck.RedisConn{
				Network: redisNetwork,
				Host:    redisHost})
		}
		status := healthcheck.CheckServices(services...)
		allAlive := true

		for k, v := range status {
			fmt.Printf("%s: %s\n", k, v)
			if v != "Ok" {
				allAlive = false
			}
		}
		if allAlive {

			if skipStorageMW {
				fmt.Printf("Storage %s\n", "None")
			} else {
				fmt.Printf("Storage %s\n", storageType)
			}
			fetchServer := viper.GetString("DFK_FETCH")
			serverCfg := fetch.Config{
				Host:         fetchServer, //"localhost:5000",
				ReadTimeout:  5 * time.Second,
				WriteTimeout: 5 * time.Second,
			}
			htmlServer := fetch.Start(serverCfg)
			defer htmlServer.Stop()

			sigChan := make(chan os.Signal, 1)
			signal.Notify(sigChan, os.Interrupt)
			<-sigChan

			fmt.Println("main : shutting down")
		}
	},
}

// Execute adds all child commands to the root command sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute(version string) {
	VERSION = version

	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}
}

func init() {

	//flags and configuration settings. They are global for the application.

	RootCmd.Flags().StringVarP(&DFKFetch, "DFK_FETCH", "a", "127.0.0.1:8000", "HTTP listen address")
	RootCmd.Flags().StringVarP(&splashHost, "SPLASH", "s", "127.0.0.1:8050", "Splash host address")
	RootCmd.Flags().IntVarP(&splashTimeout, "SPLASH_TIMEOUT", "", 20, "Timeout (in seconds) for the render.")
	RootCmd.Flags().IntVarP(&splashResourceTimeout, "SPLASH_RESOURCE_TIMEOUT", "", 30, "A timeout (in seconds) for individual network requests.")
	RootCmd.Flags().Float64VarP(&splashWait, "SPLASH_WAIT", "", 0.5, "Time in seconds to wait until js scripts loaded.")

	//set here default type of storage
	RootCmd.Flags().StringVarP(&storageType, "STORAGE_TYPE", "", "Diskv", "Storage backend for intermediary data passed to html parser. Types: S3, Spaces, Redis, Diskv")
	RootCmd.Flags().BoolVarP(&skipStorageMW, "SKIP_STORAGE_MW", "", true, "If true no data will be saved to storage. This flag forces fetcher to bypass storage middleware.")
	RootCmd.Flags().BoolVarP(&ignoreCacheInfo, "IGNORE_CACHE_INFO", "", false, "If a website is not cachable by some reason, ignore this and use cached copy if any. Please don't set it to true in production")
	RootCmd.Flags().StringVarP(&diskvBaseDir, "DISKV_BASE_DIR", "", "diskv", "diskv base directory for storing fetch results")
	RootCmd.Flags().StringVarP(&spacesConfig, "SPACES_CONFIG", "", "$HOME/.spaces/credentials", "Digital Ocean Spaces Configuration file")
	RootCmd.Flags().StringVarP(&spacesEndpoint, "SPACES_ENDPOINT", "", "https://ams3.digitaloceanspaces.com", "Digital Ocean Spaces Endpoint Address")
	RootCmd.Flags().StringVarP(&s3Region, "S3_REGION", "", "us-east-1", "AWS S3 or Digital Ocean Spaces region")
	RootCmd.Flags().StringVarP(&DFKBucket, "DFK_BUCKET", "", "dfk-storage", "AWS S3 or Digital Ocean Spaces bucket name for storing fetch results")

	RootCmd.Flags().StringVarP(&redisHost, "REDIS", "r", "127.0.0.1:6379", "Redis host address")
	RootCmd.Flags().IntVarP(&redisExpire, "REDIS_EXPIRE", "", 3600, "Default Redis expire value in seconds")
	RootCmd.Flags().StringVarP(&redisNetwork, "REDIS_NETWORK", "", "tcp", "Redis Network")
	RootCmd.Flags().StringVarP(&redisPassword, "REDIS_PASSWORD", "", "", "Redis Password")
	RootCmd.Flags().IntVarP(&redisDB, "REDIS_DB", "", 0, "Redis DB")
	RootCmd.Flags().StringVarP(&redisSocketPath, "REDIS_SOCKET_PATH", "", "", "Redis Socket Path")

	//viper.AutomaticEnv() // read in environment variables that match

	//Environmoent variable takes precedence over flag value
	if os.Getenv("SPLASH") != "" {
		//viper.BindEnv("SPLASH")
		viper.Set("SPLASH", os.Getenv("SPLASH"))
	} else {
		viper.BindPFlag("SPLASH", RootCmd.Flags().Lookup("SPLASH"))
	}

	if os.Getenv("DFK_FETCH") != "" {
		viper.Set("DFK_FETCH", os.Getenv("DFK_FETCH"))
	} else {
		viper.BindPFlag("DFK_FETCH", RootCmd.Flags().Lookup("DFK_FETCH"))
		//os.Setenv("DFK_FETCH", DFKFetch)
	}

	if os.Getenv("DISKV_BASE_DIR") != "" {
		//viper.BindEnv("DISKV_BASE_DIR")
		viper.Set("DISKV_BASE_DIR", os.Getenv("DISKV_BASE_DIR"))
	} else {
		viper.BindPFlag("DISKV_BASE_DIR", RootCmd.Flags().Lookup("DISKV_BASE_DIR"))
	}

	viper.BindPFlag("SPLASH_TIMEOUT", RootCmd.Flags().Lookup("SPLASH_TIMEOUT"))
	viper.BindPFlag("SPLASH_RESOURCE_TIMEOUT", RootCmd.Flags().Lookup("SPLASH_RESOURCE_TIMEOUT"))
	viper.BindPFlag("SPLASH_WAIT", RootCmd.Flags().Lookup("SPLASH_WAIT"))

	viper.BindPFlag("STORAGE_TYPE", RootCmd.Flags().Lookup("STORAGE_TYPE"))
	//viper.BindPFlag("ITEM_EXPIRE_IN", RootCmd.Flags().Lookup("ITEM_EXPIRE_IN"))
	viper.BindPFlag("SKIP_STORAGE_MW", RootCmd.Flags().Lookup("SKIP_STORAGE_MW"))
	viper.BindPFlag("IGNORE_CACHE_INFO", RootCmd.Flags().Lookup("IGNORE_CACHE_INFO"))
	viper.BindPFlag("SPACES_CONFIG", RootCmd.Flags().Lookup("SPACES_CONFIG"))
	viper.BindPFlag("SPACES_ENDPOINT", RootCmd.Flags().Lookup("SPACES_ENDPOINT"))
	viper.BindPFlag("DISKV_BASE_DIR", RootCmd.Flags().Lookup("DISKV_BASE_DIR"))
	viper.BindPFlag("S3_REGION", RootCmd.Flags().Lookup("S3_REGION"))
	viper.BindPFlag("DFK_BUCKET", RootCmd.Flags().Lookup("DFK_BUCKET"))
	viper.BindPFlag("REDIS", RootCmd.Flags().Lookup("REDIS"))
	viper.BindPFlag("REDIS_EXPIRE", RootCmd.Flags().Lookup("REDIS_EXPIRE"))
	viper.BindPFlag("REDIS_NETWORK", RootCmd.Flags().Lookup("REDIS_NETWORK"))
	viper.BindPFlag("REDIS_PASSWORD", RootCmd.Flags().Lookup("REDIS_PASSWORD"))
	viper.BindPFlag("REDIS_DB", RootCmd.Flags().Lookup("REDIS_DB"))
	viper.BindPFlag("REDIS_SOCKET_PATH", RootCmd.Flags().Lookup("REDIS_SOCKET_PATH"))

	//viper.SetConfigType("yaml")
	//viper.SetConfigName("conf")
	//viper.AddConfigPath("$HOME/go/src/github.com/slotix/dataflowkit/fetch/fetch.d/")
	//err := viper.ReadInConfig() // Find and read the config file
	//if err != nil {             // Handle errors reading the config file
	//	panic(fmt.Errorf("Fatal error config file: %s \n", err))
	//}
	//err := viper.WriteConfig()
	//if err != nil {
	//	fmt.Println(err)
	//}
}
<commit_msg>formatted<commit_after>// Copyright © 2017 Slotix s.r.o. <dm@slotix.sk>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package main

import (
	"fmt"
	"os"
	"os/signal"
	"time"

	"github.com/slotix/dataflowkit/fetch"
	"github.com/slotix/dataflowkit/healthcheck"

	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

var (
	//VERSION               string // VERSION is set during build
	//  DFKFetch represents address of DFK Fetch service
	DFKFetch              string //Fetch service address
	splashHost            string
	splashTimeout         int
	splashResourceTimeout int
	splashWait            float64

	storageType     string
	skipStorageMW   bool
	ignoreCacheInfo bool
	diskvBaseDir    string

	//Digital Ocean spaces configuration file
	spacesConfig string
	//Digital Ocean spaces endpoint address
	spacesEndpoint string
	s3Region       string
	//Bucket name for AWS S3 or DO Spaces
	DFKBucket string

	redisHost       string
	redisExpire     int
	redisNetwork    string
	redisPassword   string
	redisDB         int
	redisSocketPath string
)

// RootCmd represents the base command when called without any subcommands
var RootCmd = &cobra.Command{
	Use:   "dataflowkit",
	Short: "Dataflow Kit html fetcher",
	Long:  `Dataflow Kit fetch service retrieves html pages from websites and passes content to DFK parser service.`,
	Run: func(cmd *cobra.Command, args []string) {
		fmt.Println("Checking services ... ")
		services := []healthcheck.Checker{
			healthcheck.SplashConn{
				Host: viper.GetString("SPLASH"),
			},
		}
		if storageType == "Redis" {
			services = append(services, healthcheck.RedisConn{
				Network: redisNetwork,
				Host:    redisHost})
		}
		status := healthcheck.CheckServices(services...)
		allAlive := true

		for k, v := range status {
			fmt.Printf("%s: %s\n", k, v)
			if v != "Ok" {
				allAlive = false
			}
		}
		if allAlive {

			if skipStorageMW {
				fmt.Printf("Storage %s\n", "None")
			} else {
				fmt.Printf("Storage %s\n", storageType)
			}
			fetchServer := viper.GetString("DFK_FETCH")
			serverCfg := fetch.Config{
				Host:         fetchServer, //"localhost:5000",
				ReadTimeout:  5 * time.Second,
				WriteTimeout: 5 * time.Second,
			}
			htmlServer := fetch.Start(serverCfg)
			defer htmlServer.Stop()

			sigChan := make(chan os.Signal, 1)
			signal.Notify(sigChan, os.Interrupt)
			<-sigChan

			fmt.Println("main : shutting down")
		}
	},
}

// Execute adds all child commands to the root command sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute(version string) {
	VERSION = version
	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}
}

func init() {
	//flags and configuration settings. They are global for the application.

	RootCmd.Flags().StringVarP(&DFKFetch, "DFK_FETCH", "a", "127.0.0.1:8000", "HTTP listen address")
	RootCmd.Flags().StringVarP(&splashHost, "SPLASH", "s", "127.0.0.1:8050", "Splash host address")
	RootCmd.Flags().IntVarP(&splashTimeout, "SPLASH_TIMEOUT", "", 20, "Timeout (in seconds) for the render.")
	RootCmd.Flags().IntVarP(&splashResourceTimeout, "SPLASH_RESOURCE_TIMEOUT", "", 30, "A timeout (in seconds) for individual network requests.")
	RootCmd.Flags().Float64VarP(&splashWait, "SPLASH_WAIT", "", 0.5, "Time in seconds to wait until js scripts loaded.")

	//set here default type of storage
	RootCmd.Flags().StringVarP(&storageType, "STORAGE_TYPE", "", "Diskv", "Storage backend for intermediary data passed to html parser. Types: S3, Spaces, Redis, Diskv")
	RootCmd.Flags().BoolVarP(&skipStorageMW, "SKIP_STORAGE_MW", "", true, "If true no data will be saved to storage. This flag forces fetcher to bypass storage middleware.")
	RootCmd.Flags().BoolVarP(&ignoreCacheInfo, "IGNORE_CACHE_INFO", "", false, "If a website is not cachable by some reason, ignore this and use cached copy if any. Please don't set it to true in production")
	RootCmd.Flags().StringVarP(&diskvBaseDir, "DISKV_BASE_DIR", "", "diskv", "diskv base directory for storing fetch results")
	RootCmd.Flags().StringVarP(&spacesConfig, "SPACES_CONFIG", "", "$HOME/.spaces/credentials", "Digital Ocean Spaces Configuration file")
	RootCmd.Flags().StringVarP(&spacesEndpoint, "SPACES_ENDPOINT", "", "https://ams3.digitaloceanspaces.com", "Digital Ocean Spaces Endpoint Address")
	RootCmd.Flags().StringVarP(&s3Region, "S3_REGION", "", "us-east-1", "AWS S3 or Digital Ocean Spaces region")
	RootCmd.Flags().StringVarP(&DFKBucket, "DFK_BUCKET", "", "dfk-storage", "AWS S3 or Digital Ocean Spaces bucket name for storing fetch results")

	RootCmd.Flags().StringVarP(&redisHost, "REDIS", "r", "127.0.0.1:6379", "Redis host address")
	RootCmd.Flags().IntVarP(&redisExpire, "REDIS_EXPIRE", "", 3600, "Default Redis expire value in seconds")
	RootCmd.Flags().StringVarP(&redisNetwork, "REDIS_NETWORK", "", "tcp", "Redis Network")
	RootCmd.Flags().StringVarP(&redisPassword, "REDIS_PASSWORD", "", "", "Redis Password")
	RootCmd.Flags().IntVarP(&redisDB, "REDIS_DB", "", 0, "Redis DB")
	RootCmd.Flags().StringVarP(&redisSocketPath, "REDIS_SOCKET_PATH", "", "", "Redis Socket Path")

	//viper.AutomaticEnv() // read in environment variables that match

	//Environmoent variable takes precedence over flag value
	if os.Getenv("SPLASH") != "" {
		//viper.BindEnv("SPLASH")
		viper.Set("SPLASH", os.Getenv("SPLASH"))
	} else {
		viper.BindPFlag("SPLASH", RootCmd.Flags().Lookup("SPLASH"))
	}

	if os.Getenv("DFK_FETCH") != "" {
		viper.Set("DFK_FETCH", os.Getenv("DFK_FETCH"))
	} else {
		viper.BindPFlag("DFK_FETCH", RootCmd.Flags().Lookup("DFK_FETCH"))
		//os.Setenv("DFK_FETCH", DFKFetch)
	}

	if os.Getenv("DISKV_BASE_DIR") != "" {
		//viper.BindEnv("DISKV_BASE_DIR")
		viper.Set("DISKV_BASE_DIR", os.Getenv("DISKV_BASE_DIR"))
	} else {
		viper.BindPFlag("DISKV_BASE_DIR", RootCmd.Flags().Lookup("DISKV_BASE_DIR"))
	}

	viper.BindPFlag("SPLASH_TIMEOUT", RootCmd.Flags().Lookup("SPLASH_TIMEOUT"))
	viper.BindPFlag("SPLASH_RESOURCE_TIMEOUT", RootCmd.Flags().Lookup("SPLASH_RESOURCE_TIMEOUT"))
	viper.BindPFlag("SPLASH_WAIT", RootCmd.Flags().Lookup("SPLASH_WAIT"))

	viper.BindPFlag("STORAGE_TYPE", RootCmd.Flags().Lookup("STORAGE_TYPE"))
	//viper.BindPFlag("ITEM_EXPIRE_IN", RootCmd.Flags().Lookup("ITEM_EXPIRE_IN"))
	viper.BindPFlag("SKIP_STORAGE_MW", RootCmd.Flags().Lookup("SKIP_STORAGE_MW"))
	viper.BindPFlag("IGNORE_CACHE_INFO", RootCmd.Flags().Lookup("IGNORE_CACHE_INFO"))
	viper.BindPFlag("SPACES_CONFIG", RootCmd.Flags().Lookup("SPACES_CONFIG"))
	viper.BindPFlag("SPACES_ENDPOINT", RootCmd.Flags().Lookup("SPACES_ENDPOINT"))
	viper.BindPFlag("DISKV_BASE_DIR", RootCmd.Flags().Lookup("DISKV_BASE_DIR"))
	viper.BindPFlag("S3_REGION", RootCmd.Flags().Lookup("S3_REGION"))
	viper.BindPFlag("DFK_BUCKET", RootCmd.Flags().Lookup("DFK_BUCKET"))
	viper.BindPFlag("REDIS", RootCmd.Flags().Lookup("REDIS"))
	viper.BindPFlag("REDIS_EXPIRE", RootCmd.Flags().Lookup("REDIS_EXPIRE"))
	viper.BindPFlag("REDIS_NETWORK", RootCmd.Flags().Lookup("REDIS_NETWORK"))
	viper.BindPFlag("REDIS_PASSWORD", RootCmd.Flags().Lookup("REDIS_PASSWORD"))
	viper.BindPFlag("REDIS_DB", RootCmd.Flags().Lookup("REDIS_DB"))
	viper.BindPFlag("REDIS_SOCKET_PATH", RootCmd.Flags().Lookup("REDIS_SOCKET_PATH"))

	//viper.SetConfigType("yaml")
	//viper.SetConfigName("conf")
	//viper.AddConfigPath("$HOME/go/src/github.com/slotix/dataflowkit/fetch/fetch.d/")
	//err := viper.ReadInConfig() // Find and read the config file
	//if err != nil {             // Handle errors reading the config file
	//	panic(fmt.Errorf("Fatal error config file: %s \n", err))
	//}
	//err := viper.WriteConfig()
	//if err != nil {
	//	fmt.Println(err)
	//}
}
<|endoftext|>


<commit_before>//  Copyright (c) 2015 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package main

import (
	"io/ioutil"
	"os"
	"strconv"
	"strings"
	"testing"

	log "github.com/couchbase/clog"

	"github.com/couchbaselabs/cbgt"
)

func TestRunRebalancer(t *testing.T) {
	testDir, _ := ioutil.TempDir("./tmp", "test")
	defer os.RemoveAll(testDir)

	log.Printf("testDir: %s", testDir)

	tests := []struct {
		label      string
		ops        string // Space separated "+a", "-x".
		params     map[string]string
		expNodes   string // Space separated list of nodes ("a"..."v").
		expIndexes string // Space separated list of indxes ("x"..."z").
		expChanged bool
		expErr     bool
	}{
		{"1st node",
			"+a", nil,
			"a",
			"",
			false, true,
		},
		{"add 1st index x",
			"+x", nil,
			"a",
			"x",
			true, false,
		},
		{"add 2nd node b",
			"+b", nil,
			"a b",
			"x",
			true, false,
		},
	}

	cfg := cbgt.NewCfgMem()

	mgrs := map[string]*cbgt.Manager{}

	var mgr0 *cbgt.Manager

	server := "."

	waitUntilEmptyCfgEvents := func(ch chan cbgt.CfgEvent) {
		for {
			select {
			case <-ch:
			default:
				return
			}
		}
	}

	cfgEventsNodeDefsWanted := make(chan cbgt.CfgEvent, 100)
	cfg.Subscribe(cbgt.NODE_DEFS_WANTED, cfgEventsNodeDefsWanted)

	waitUntilEmptyCfgEventsNodeDefsWanted := func() {
		waitUntilEmptyCfgEvents(cfgEventsNodeDefsWanted)
	}

	cfgEventsIndexDefs := make(chan cbgt.CfgEvent, 100)
	cfg.Subscribe(cbgt.INDEX_DEFS_KEY, cfgEventsIndexDefs)

	waitUntilEmptyCfgEventsIndexDefs := func() {
		waitUntilEmptyCfgEvents(cfgEventsIndexDefs)
	}

	for testi, test := range tests {
		log.Printf("testi: %d, label: %q", testi, test.label)

		for opi, op := range strings.Split(test.ops, " ") {
			log.Printf(" opi: %d, op: %s", opi, op)

			name := op[1:2]

			isIndexOp := name >= "x"
			if isIndexOp {
				indexName := name
				log.Printf(" indexOp: %s, indexName: %s", op[0:1], indexName)

				sourceType := "primary"
				if test.params["sourceType"] != "" {
					sourceType = test.params["sourceType"]
				}
				if test.params[indexName+".sourceType"] != "" {
					sourceType = test.params[indexName+".sourceType"]
				}

				sourceName := "default"
				if test.params["sourceName"] != "" {
					sourceName = test.params["sourceName"]
				}
				if test.params[indexName+".sourceName"] != "" {
					sourceName = test.params[indexName+".sourceName"]
				}

				sourceUUID := ""
				if test.params["sourceUUID"] != "" {
					sourceUUID = test.params["sourceUUID"]
				}
				if test.params[indexName+".sourceUUID"] != "" {
					sourceUUID = test.params[indexName+".sourceUUID"]
				}

				sourceParams := `{"numPartitions":4}`
				if test.params["sourceParams"] != "" {
					sourceParams = test.params["sourceParams"]
				}
				if test.params[indexName+".sourceParams"] != "" {
					sourceParams = test.params[indexName+".sourceParams"]
				}

				indexType := "blackhole"
				if test.params["indexType"] != "" {
					indexType = test.params["indexType"]
				}
				if test.params[indexName+".indexType"] != "" {
					indexType = test.params[indexName+".indexType"]
				}

				indexParams := ""
				if test.params["indexParams"] != "" {
					indexParams = test.params["indexParams"]
				}
				if test.params[indexName+".indexParams"] != "" {
					indexParams = test.params[indexName+".indexParams"]
				}

				prevIndexUUID := ""
				if test.params["prevIndexUUID"] != "" {
					prevIndexUUID = test.params["prevIndexUUID"]
				}
				if test.params[indexName+".prevIndexUUID"] != "" {
					prevIndexUUID = test.params[indexName+".prevIndexUUID"]
				}

				planParams := cbgt.PlanParams{
					MaxPartitionsPerPIndex: 1,
				}

				waitUntilEmptyCfgEventsIndexDefs()

				err := mgr0.CreateIndex(
					sourceType, sourceName, sourceUUID, sourceParams,
					indexType, indexName, indexParams,
					planParams,
					prevIndexUUID)
				if err != nil {
					t.Errorf("expected no err, got: %#v", err)
				}

				waitUntilEmptyCfgEventsIndexDefs()
			} else { // It's a node op.
				nodeName := name
				log.Printf(" nodeOp: %s, nodeName: %s", op[0:1], nodeName)

				register := "wanted"
				if op[0:1] == "-" {
					register = "unknown"
				}
				if test.params["register"] != "" {
					register = test.params["register"]
				}
				if test.params[nodeName+".register"] != "" {
					register = test.params[nodeName+".register"]
				}

				if mgrs[nodeName] != nil {
					mgrs[nodeName].Stop()
					delete(mgrs, nodeName)
				}

				waitUntilEmptyCfgEventsNodeDefsWanted()

				mgr, err := startNodeManager(testDir, cfg,
					name, register, test.params, server)
				if err != nil || mgr == nil {
					t.Errorf("expected no err, got: %#v", err)
				}
				if mgr0 == nil {
					mgr0 = mgr
				}

				if register != "unknown" {
					mgrs[nodeName] = mgr
				}

				mgr.Kick("kick")

				waitUntilEmptyCfgEventsNodeDefsWanted()
			}
		}

		changed, err := runRebalancer(cbgt.VERSION, cfg, ".")
		if changed != test.expChanged {
			t.Errorf("testi: %d, label: %q,"+
				" expChanged: %v, but got: %v",
				testi, test.label,
				test.expChanged, changed)
		}
		if (test.expErr && err == nil) ||
			(!test.expErr && err != nil) {
			t.Errorf("testi: %d, label: %q,"+
				" expErr: %v, but got: %v",
				testi, test.label,
				test.expErr, err)
		}
	}
}

func startNodeManager(testDir string, cfg cbgt.Cfg, node, register string,
	params map[string]string, server string) (
	mgr *cbgt.Manager, err error) {
	uuid := node
	if params["uuid"] != "" {
		uuid = params["uuid"]
	}
	if params[node+".uuid"] != "" {
		uuid = params[node+".uuid"]
	}

	// No planner in tags because mcp provides the planner.
	tags := []string{"feed", "pindex", "janitor", "queryer"}
	if params["tags"] != "" {
		tags = strings.Split(params["tags"], ",")
	}
	if params[node+".tags"] != "" {
		tags = strings.Split(params[node+".tags"], ",")
	}

	container := ""
	if params["container"] != "" {
		container = params["container"]
	}
	if params[node+".container"] != "" {
		container = params[node+".container"]
	}

	weight := 1
	if params["weight"] != "" {
		weight, err = strconv.Atoi(params["weight"])
	}
	if params[node+".weight"] != "" {
		weight, err = strconv.Atoi(params[node+".weight"])
	}
	if weight < 1 {
		weight = 1
	}

	extras := ""

	bindHttp := node

	dataDir := testDir + string(os.PathSeparator) + node

	os.MkdirAll(dataDir, 0700)

	meh := cbgt.ManagerEventHandlers(nil)

	mgr = cbgt.NewManager(cbgt.VERSION, cfg, uuid,
		tags, container, weight, extras,
		bindHttp, dataDir, server, meh)

	err = mgr.Start(register)
	if err != nil {
		mgr.Stop()

		return nil, err
	}

	return mgr, nil
}
<commit_msg>testCreateIndex helper func<commit_after>//  Copyright (c) 2015 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package main

import (
	"io/ioutil"
	"os"
	"strconv"
	"strings"
	"testing"

	log "github.com/couchbase/clog"

	"github.com/couchbaselabs/cbgt"
)

func TestRunRebalancer(t *testing.T) {
	testDir, _ := ioutil.TempDir("./tmp", "test")
	defer os.RemoveAll(testDir)

	log.Printf("testDir: %s", testDir)

	tests := []struct {
		label      string
		ops        string // Space separated "+a", "-x".
		params     map[string]string
		expNodes   string // Space separated list of nodes ("a"..."v").
		expIndexes string // Space separated list of indxes ("x"..."z").
		expChanged bool
		expErr     bool
	}{
		{"1st node",
			"+a", nil,
			"a",
			"",
			false, true,
		},
		{"add 1st index x",
			"+x", nil,
			"a",
			"x",
			true, false,
		},
		{"add 2nd node b",
			"+b", nil,
			"a b",
			"x",
			true, false,
		},
	}

	cfg := cbgt.NewCfgMem()

	mgrs := map[string]*cbgt.Manager{}

	var mgr0 *cbgt.Manager

	server := "."

	waitUntilEmptyCfgEvents := func(ch chan cbgt.CfgEvent) {
		for {
			select {
			case <-ch:
			default:
				return
			}
		}
	}

	cfgEventsNodeDefsWanted := make(chan cbgt.CfgEvent, 100)
	cfg.Subscribe(cbgt.NODE_DEFS_WANTED, cfgEventsNodeDefsWanted)

	waitUntilEmptyCfgEventsNodeDefsWanted := func() {
		waitUntilEmptyCfgEvents(cfgEventsNodeDefsWanted)
	}

	cfgEventsIndexDefs := make(chan cbgt.CfgEvent, 100)
	cfg.Subscribe(cbgt.INDEX_DEFS_KEY, cfgEventsIndexDefs)

	waitUntilEmptyCfgEventsIndexDefs := func() {
		waitUntilEmptyCfgEvents(cfgEventsIndexDefs)
	}

	for testi, test := range tests {
		log.Printf("testi: %d, label: %q", testi, test.label)

		for opi, op := range strings.Split(test.ops, " ") {
			log.Printf(" opi: %d, op: %s", opi, op)

			name := op[1:2]

			isIndexOp := name >= "x"
			if isIndexOp {
				indexName := name
				log.Printf(" indexOp: %s, indexName: %s", op[0:1], indexName)

				testCreateIndex(t, mgr0, indexName, test.params,
					waitUntilEmptyCfgEventsIndexDefs)
			} else { // It's a node op.
				nodeName := name
				log.Printf(" nodeOp: %s, nodeName: %s", op[0:1], nodeName)

				register := "wanted"
				if op[0:1] == "-" {
					register = "unknown"
				}
				if test.params["register"] != "" {
					register = test.params["register"]
				}
				if test.params[nodeName+".register"] != "" {
					register = test.params[nodeName+".register"]
				}

				if mgrs[nodeName] != nil {
					mgrs[nodeName].Stop()
					delete(mgrs, nodeName)
				}

				waitUntilEmptyCfgEventsNodeDefsWanted()

				mgr, err := startNodeManager(testDir, cfg,
					name, register, test.params, server)
				if err != nil || mgr == nil {
					t.Errorf("expected no err, got: %#v", err)
				}
				if mgr0 == nil {
					mgr0 = mgr
				}

				if register != "unknown" {
					mgrs[nodeName] = mgr
				}

				mgr.Kick("kick")

				waitUntilEmptyCfgEventsNodeDefsWanted()
			}
		}

		changed, err := runRebalancer(cbgt.VERSION, cfg, ".")
		if changed != test.expChanged {
			t.Errorf("testi: %d, label: %q,"+
				" expChanged: %v, but got: %v",
				testi, test.label,
				test.expChanged, changed)
		}
		if (test.expErr && err == nil) ||
			(!test.expErr && err != nil) {
			t.Errorf("testi: %d, label: %q,"+
				" expErr: %v, but got: %v",
				testi, test.label,
				test.expErr, err)
		}
	}
}

func testCreateIndex(t *testing.T,
	mgr *cbgt.Manager,
	indexName string,
	params map[string]string,
	waitUntilEmptyCfgEventsIndexDefs func()) {
	sourceType := "primary"
	if params["sourceType"] != "" {
		sourceType = params["sourceType"]
	}
	if params[indexName+".sourceType"] != "" {
		sourceType = params[indexName+".sourceType"]
	}

	sourceName := "default"
	if params["sourceName"] != "" {
		sourceName = params["sourceName"]
	}
	if params[indexName+".sourceName"] != "" {
		sourceName = params[indexName+".sourceName"]
	}

	sourceUUID := ""
	if params["sourceUUID"] != "" {
		sourceUUID = params["sourceUUID"]
	}
	if params[indexName+".sourceUUID"] != "" {
		sourceUUID = params[indexName+".sourceUUID"]
	}

	sourceParams := `{"numPartitions":4}`
	if params["sourceParams"] != "" {
		sourceParams = params["sourceParams"]
	}
	if params[indexName+".sourceParams"] != "" {
		sourceParams = params[indexName+".sourceParams"]
	}

	indexType := "blackhole"
	if params["indexType"] != "" {
		indexType = params["indexType"]
	}
	if params[indexName+".indexType"] != "" {
		indexType = params[indexName+".indexType"]
	}

	indexParams := ""
	if params["indexParams"] != "" {
		indexParams = params["indexParams"]
	}
	if params[indexName+".indexParams"] != "" {
		indexParams = params[indexName+".indexParams"]
	}

	prevIndexUUID := ""
	if params["prevIndexUUID"] != "" {
		prevIndexUUID = params["prevIndexUUID"]
	}
	if params[indexName+".prevIndexUUID"] != "" {
		prevIndexUUID = params[indexName+".prevIndexUUID"]
	}

	planParams := cbgt.PlanParams{
		MaxPartitionsPerPIndex: 1,
	}

	waitUntilEmptyCfgEventsIndexDefs()

	err := mgr.CreateIndex(
		sourceType, sourceName, sourceUUID, sourceParams,
		indexType, indexName, indexParams,
		planParams,
		prevIndexUUID)
	if err != nil {
		t.Errorf("expected no err, got: %#v", err)
	}

	waitUntilEmptyCfgEventsIndexDefs()
}

func startNodeManager(testDir string, cfg cbgt.Cfg, node, register string,
	params map[string]string, server string) (
	mgr *cbgt.Manager, err error) {
	uuid := node
	if params["uuid"] != "" {
		uuid = params["uuid"]
	}
	if params[node+".uuid"] != "" {
		uuid = params[node+".uuid"]
	}

	// No planner in tags because mcp provides the planner.
	tags := []string{"feed", "pindex", "janitor", "queryer"}
	if params["tags"] != "" {
		tags = strings.Split(params["tags"], ",")
	}
	if params[node+".tags"] != "" {
		tags = strings.Split(params[node+".tags"], ",")
	}

	container := ""
	if params["container"] != "" {
		container = params["container"]
	}
	if params[node+".container"] != "" {
		container = params[node+".container"]
	}

	weight := 1
	if params["weight"] != "" {
		weight, err = strconv.Atoi(params["weight"])
	}
	if params[node+".weight"] != "" {
		weight, err = strconv.Atoi(params[node+".weight"])
	}
	if weight < 1 {
		weight = 1
	}

	extras := ""

	bindHttp := node

	dataDir := testDir + string(os.PathSeparator) + node

	os.MkdirAll(dataDir, 0700)

	meh := cbgt.ManagerEventHandlers(nil)

	mgr = cbgt.NewManager(cbgt.VERSION, cfg, uuid,
		tags, container, weight, extras,
		bindHttp, dataDir, server, meh)

	err = mgr.Start(register)
	if err != nil {
		mgr.Stop()

		return nil, err
	}

	return mgr, nil
}
<|endoftext|>


<commit_before>package cmd

import (
	"encoding/json"
	"io"
	"os"
	"path/filepath"

	"github.com/codeclimate/test-reporter/formatters"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
)

type CoverageSummer struct {
	Output string
	Print  bool
}

var summerOptions = CoverageSummer{}

var sumCoverageCmd = &cobra.Command{
	Use:   "sum-coverage",
	Short: "Combine (sum) multiple pre-formatted coverage payloads into one.",
	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) == 0 {
			return errors.New("you must pass in one or more files to be summarized")
		}
		rep, err := formatters.NewReport()
		if err != nil {
			return errors.WithStack(err)
		}
		for _, n := range args {
			f, err := os.Open(n)
			if err != nil {
				return errors.WithStack(err)
			}
			rr, err := formatters.NewReport()
			if err != nil {
				return errors.WithStack(err)
			}
			err = json.NewDecoder(f).Decode(&rr)
			if err != nil {
				return errors.WithStack(err)
			}
			rep.Merge(&rr)
		}

		var out io.Writer
		if summerOptions.Print {
			out = os.Stdout
		} else {
			os.MkdirAll(filepath.Dir(summerOptions.Output), 0755)
			out, err = os.Create(summerOptions.Output)
			if err != nil {
				return errors.WithStack(err)
			}
		}

		err = rep.Save(out)
		if err != nil {
			return errors.WithStack(err)
		}
		return nil
	},
}

func init() {
	sumCoverageCmd.Flags().BoolVarP(&summerOptions.Print, "print", "p", false, "prints to standard out only")
	sumCoverageCmd.Flags().StringVarP(&summerOptions.Output, "output", "o", "codeclimate.json", "output path")
	RootCmd.AddCommand(sumCoverageCmd)
}
<commit_msg>changed the default output of the sum command as per #63<commit_after>package cmd

import (
	"encoding/json"
	"io"
	"os"
	"path/filepath"

	"github.com/codeclimate/test-reporter/formatters"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
)

type CoverageSummer struct {
	Output string
	Print  bool
}

var summerOptions = CoverageSummer{}

var sumCoverageCmd = &cobra.Command{
	Use:   "sum-coverage",
	Short: "Combine (sum) multiple pre-formatted coverage payloads into one.",
	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) == 0 {
			return errors.New("you must pass in one or more files to be summarized")
		}
		rep, err := formatters.NewReport()
		if err != nil {
			return errors.WithStack(err)
		}
		for _, n := range args {
			f, err := os.Open(n)
			if err != nil {
				return errors.WithStack(err)
			}
			rr, err := formatters.NewReport()
			if err != nil {
				return errors.WithStack(err)
			}
			err = json.NewDecoder(f).Decode(&rr)
			if err != nil {
				return errors.WithStack(err)
			}
			rep.Merge(&rr)
		}

		var out io.Writer
		if summerOptions.Print {
			out = os.Stdout
		} else {
			err = os.MkdirAll(filepath.Dir(summerOptions.Output), 0755)
			if err != nil {
				return errors.WithStack(err)
			}
			out, err = os.Create(summerOptions.Output)
			if err != nil {
				return errors.WithStack(err)
			}
		}

		err = rep.Save(out)
		if err != nil {
			return errors.WithStack(err)
		}
		return nil
	},
}

func init() {
	sumCoverageCmd.Flags().BoolVarP(&summerOptions.Print, "print", "p", false, "prints to standard out only")
	sumCoverageCmd.Flags().StringVarP(&summerOptions.Output, "output", "o", "coverage/codeclimate.json", "output path")
	RootCmd.AddCommand(sumCoverageCmd)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"
	_ "net/http/pprof"
	"os"
	"strings"

	"github.com/anacrolix/libtorgo/metainfo"

	"bitbucket.org/anacrolix/go.torrent"
	"bitbucket.org/anacrolix/go.torrent/util"
)

var (
	downloadDir = flag.String("downloadDir", "", "directory to store download torrent data")
	testPeer    = flag.String("testPeer", "", "bootstrap peer address")
	httpAddr    = flag.String("httpAddr", "localhost:6061", "http serve address")
	// TODO: Check the default torrent listen port.
	listenAddr      = flag.String("listenAddr", ":50007", "incoming connection address")
	disableTrackers = flag.Bool("disableTrackers", false, "disable trackers")
	seed            = flag.Bool("seed", false, "seed after downloading")
)

func main() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)
	flag.Parse()
	if *httpAddr != "" {
		util.LoggedHTTPServe(*httpAddr)
	}
	client, err := torrent.NewClient(&torrent.Config{
		DataDir:         *downloadDir,
		DisableTrackers: *disableTrackers,
		ListenAddr:      *listenAddr,
	})
	if err != nil {
		log.Fatalf("error creating client: %s", err)
	}
	http.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		client.WriteStatus(w)
	})
	defer client.Stop()
	if flag.NArg() == 0 {
		fmt.Fprintln(os.Stderr, "no torrents specified")
		return
	}
	for _, arg := range flag.Args() {
		var ih torrent.InfoHash
		if strings.HasPrefix(arg, "magnet:") {
			m, err := torrent.ParseMagnetURI(arg)
			if err != nil {
				log.Fatalf("error parsing magnet uri: %s", err)
			}
			ih = m.InfoHash
			_, err = client.AddMagnet(arg)
			if err != nil {
				log.Fatalf("error adding magnet: %s", err)
			}
		} else {
			metaInfo, err := metainfo.LoadFromFile(arg)
			if err != nil {
				log.Fatal(err)
			}
			err = client.AddTorrent(metaInfo)
			if err != nil {
				log.Fatal(err)
			}
			util.CopyExact(&ih, metaInfo.Info.Hash)
		}
		// client.PrioritizeDataRegion(ih, 0, 999999999)
		err := client.AddPeers(ih, func() []torrent.Peer {
			if *testPeer == "" {
				return nil
			}
			addr, err := net.ResolveTCPAddr("tcp", *testPeer)
			if err != nil {
				log.Fatal(err)
			}
			return []torrent.Peer{{
				IP:   addr.IP,
				Port: addr.Port,
			}}
		}())
		if err != nil {
			log.Fatal(err)
		}
	}
	if *seed {
		select {}
	}
	if client.WaitAll() {
		log.Print("downloaded ALL the torrents")
	} else {
		log.Fatal("y u no complete torrents?!")
	}
}
<commit_msg>cmd/torrent: Add -upload flag<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"
	_ "net/http/pprof"
	"os"
	"strings"

	"github.com/anacrolix/libtorgo/metainfo"

	"bitbucket.org/anacrolix/go.torrent"
	"bitbucket.org/anacrolix/go.torrent/util"
)

var (
	downloadDir = flag.String("downloadDir", "", "directory to store download torrent data")
	testPeer    = flag.String("testPeer", "", "bootstrap peer address")
	httpAddr    = flag.String("httpAddr", "localhost:6061", "http serve address")
	// TODO: Check the default torrent listen port.
	listenAddr      = flag.String("listenAddr", ":50007", "incoming connection address")
	disableTrackers = flag.Bool("disableTrackers", false, "disable trackers")
	seed            = flag.Bool("seed", false, "seed after downloading")
	upload          = flag.Bool("upload", true, "upload data to peers")
)

func main() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)
	flag.Parse()
	if *httpAddr != "" {
		util.LoggedHTTPServe(*httpAddr)
	}
	client, err := torrent.NewClient(&torrent.Config{
		DataDir:         *downloadDir,
		DisableTrackers: *disableTrackers,
		ListenAddr:      *listenAddr,
		NoUpload:        !*upload,
	})
	if err != nil {
		log.Fatalf("error creating client: %s", err)
	}
	http.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		client.WriteStatus(w)
	})
	defer client.Stop()
	if flag.NArg() == 0 {
		fmt.Fprintln(os.Stderr, "no torrents specified")
		return
	}
	for _, arg := range flag.Args() {
		var ih torrent.InfoHash
		if strings.HasPrefix(arg, "magnet:") {
			m, err := torrent.ParseMagnetURI(arg)
			if err != nil {
				log.Fatalf("error parsing magnet uri: %s", err)
			}
			ih = m.InfoHash
			_, err = client.AddMagnet(arg)
			if err != nil {
				log.Fatalf("error adding magnet: %s", err)
			}
		} else {
			metaInfo, err := metainfo.LoadFromFile(arg)
			if err != nil {
				log.Fatal(err)
			}
			err = client.AddTorrent(metaInfo)
			if err != nil {
				log.Fatal(err)
			}
			util.CopyExact(&ih, metaInfo.Info.Hash)
		}
		// client.PrioritizeDataRegion(ih, 0, 999999999)
		err := client.AddPeers(ih, func() []torrent.Peer {
			if *testPeer == "" {
				return nil
			}
			addr, err := net.ResolveTCPAddr("tcp", *testPeer)
			if err != nil {
				log.Fatal(err)
			}
			return []torrent.Peer{{
				IP:   addr.IP,
				Port: addr.Port,
			}}
		}())
		if err != nil {
			log.Fatal(err)
		}
	}
	if *seed {
		select {}
	}
	if client.WaitAll() {
		log.Print("downloaded ALL the torrents")
	} else {
		log.Fatal("y u no complete torrents?!")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bitbucket.org/anacrolix/go.torrent"
	"flag"
	metainfo "github.com/nsf/libtorgo/torrent"
	"log"
	"net"
)

var (
	downloadDir = flag.String("downloadDir", "", "directory to store download torrent data")
)

func init() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)
	flag.Parse()
}

func main() {
	client := torrent.NewClient(*downloadDir)
	for _, arg := range flag.Args() {
		metaInfo, err := metainfo.LoadFromFile(arg)
		if err != nil {
			log.Fatal(err)
		}
		err = client.AddTorrent(metaInfo)
		if err != nil {
			log.Fatal(err)
		}
		err = client.AddPeers(torrent.BytesInfoHash(metaInfo.InfoHash), []torrent.Peer{{
			IP:   net.IPv4(127, 0, 0, 1),
			Port: 50933,
		}})
		if err != nil {
			log.Fatal(err)
		}
	}
	client.WaitAll()
	client.Close()
}
<commit_msg>Bring the 'torrent' command up to speed with the client API, including profiling flags<commit_after>package main

import (
	"bitbucket.org/anacrolix/go.torrent"
	"flag"
	"fmt"
	metainfo "github.com/nsf/libtorgo/torrent"
	"log"
	"net"
	"net/http"
	_ "net/http/pprof"
	"os"
)

var (
	downloadDir = flag.String("downloadDir", "", "directory to store download torrent data")
	testPeer    = flag.String("testPeer", "", "bootstrap peer address")
	profAddr    = flag.String("profAddr", "", "http serve address")
)

func init() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)
	flag.Parse()
}

func main() {
	if *profAddr != "" {
		go http.ListenAndServe(*profAddr, nil)
	}
	client := torrent.Client{
		DataDir: *downloadDir,
	}
	client.Start()
	defer client.Stop()
	if flag.NArg() == 0 {
		fmt.Fprintln(os.Stderr, "no torrents specified")
		return
	}
	for _, arg := range flag.Args() {
		metaInfo, err := metainfo.LoadFromFile(arg)
		if err != nil {
			log.Fatal(err)
		}
		err = client.AddTorrent(metaInfo)
		if err != nil {
			log.Fatal(err)
		}
		err = client.AddPeers(torrent.BytesInfoHash(metaInfo.InfoHash), func() []torrent.Peer {
			if *testPeer == "" {
				return nil
			}
			addr, err := net.ResolveTCPAddr("tcp", *testPeer)
			if err != nil {
				log.Fatal(err)
			}
			return []torrent.Peer{{
				IP:   addr.IP,
				Port: addr.Port,
			}}
		}())
		if err != nil {
			log.Fatal(err)
		}
	}
	client.WaitAll()
}
<|endoftext|>