<commit_before>package main

import "fmt"
import "log"
import "os"
import "errors"
import "strings"
import "github.com/nlopes/slack"
import "github.com/tadgh/go-toggl"
import "encoding/gob"
import "time"
import "runtime"

const file = "./test.gob"

func Check(err error) {
	if err != nil {
		_, file, line, _ := runtime.Caller(1)
		fmt.Println(line, "\t", file, "\n", err)
		os.Exit(1)
	}
}

func pingTogglApi(apiKey string) error {
	ts := toggl.OpenSession(apiKey)
	_, err := ts.GetAccount()
	if err != nil {
		fmt.Println(os.Stderr, "Error: %s\n", err)
		return err
	}
	return nil
}

func Save(path string, object interface{}) error {
	file, err := os.Create(path)
	if err == nil {
		encoder := gob.NewEncoder(file)
		encoder.Encode(object)
	}
	file.Close()
	return err
}

func Load(path string, object interface{}) error {
	file, err := os.Open(path)
	if err == nil {
		decoder := gob.NewDecoder(file)
		err = decoder.Decode(object)
	}
	file.Close()
	return err
}

func createTimeEntry(apiKey, description string, start time.Time, duration time.Duration, pid, tid int) *toggl.TimeEntry {
	ts := toggl.OpenSession(apiKey)
	te, err := ts.CreateTimeEntry(pid, 0, start, duration, description)
	if err != nil {
		log.Fatal("Error uploading time entry! %v", err)
	}
	return &te
}

func stopTimer(apiKey string) (*toggl.TimeEntry, error) {
	ts := toggl.OpenSession(apiKey)
	te, err := ts.GetActiveTimeEntry()
	if err != nil {
		return nil, err
	}
	if te.ID <= 0 {
		return nil, errors.New("No timer is currently running!")
	}
	te, err = ts.StopTimeEntry(te)
	if err != nil {
		return nil, err
	}
	return &te, nil
}

func startTimer(apiKey, description string, pid int) *toggl.TimeEntry {
	ts := toggl.OpenSession(apiKey)
	te, err := ts.StartTimeEntryForProject(description, pid)

	if err != nil {
		fmt.Println(err)
	}
	sessionMap[apiKey] = te
	return &te
}

func getProjectWithName(apiKey, projectName string) int {
	ts := toggl.OpenSession(apiKey)
	acc, err := ts.GetAccount()
	var retVal int
	if err != nil {
		fmt.Println(err)
	}
	for _, project := range acc.Data.Projects {
		//case insensitive string comparison
		if strings.EqualFold(project.Name, projectName) {
			retVal = project.ID
		}
	}
	fmt.Println("FOUND PROJECT WITH NAME: " + projectName + ", id is: " + string(retVal))
	return retVal
}

type BotCommand struct {
	Channel string
	Event   *slack.MessageEvent
	UserId  string
}

type ReplyChannel struct {
	Channel      string
	Attachment   *slack.Attachment
	DisplayTitle string
}

var (
	api               *slack.Client
	botCommandChannel chan *BotCommand
	botReplyChannel   chan ReplyChannel
	botId             string
	userMap           map[string]string
	sessionMap        map[string]toggl.TimeEntry
)

func handleBotCommands(replyChannel chan ReplyChannel) {
	commands := map[string]string{
		"register": "Register yourself with togglbot. `@togglbot register MY_TOGGL_API_KEY`",
		"start":    "Start a timer for a given project and description. `@togglbot start <PROJECT_NAME> <EVERYTHING_ELSE_IS_DESCRIPTION>`",
		"stop":     "Stops any current timer session. `@togglbot stop`",
		"track":    "adds a toggl entry to a project for a given time range. `@togglbot track icancope 9am-5pm`",
	}

	for {
		incomingCommand := <-botCommandChannel
		commandArray := strings.Fields(incomingCommand.Event.Text)
		var reply ReplyChannel
		reply.Channel = incomingCommand.Channel

		switch commandArray[1] {
		case "help":
			reply.DisplayTitle = "Help!"
			fields := make([]slack.AttachmentField, 0)
			for k, v := range commands {
				fields = append(fields, slack.AttachmentField{
					Title: "<bot> " + k,
					Value: v,
				})
			}
			attachment := &slack.Attachment{
				Pretext:    "TogglBot Command List",
				Color:      "#B733FF",
				Fields:     fields,
				MarkdownIn: []string{"fields"},
			}
			reply.Attachment = attachment
			fmt.Println("SENDING REPLY TO CHANNEL")
			replyChannel <- reply
		case "register":
			togglApiKey := commandArray[2]
			err := pingTogglApi(togglApiKey)
			if err != nil {
				reply.DisplayTitle = "Failed to register. Bad api key?"
			} else {
				userMap[incomingCommand.Event.User] = togglApiKey
				err := Save(file, userMap)
				Check(err)
				reply.DisplayTitle = "Successfully registered!"
			}
			replyChannel <- reply
		}

		togglApiKey, ok := userMap[incomingCommand.Event.User]
		if !ok {
			reply.DisplayTitle = "You have not registered with togglbot yet. Try @togglbot register API_KEY_HERE"
			replyChannel <- reply
			break
		}

		switch commandArray[1] {

		case "start":
			if len(commandArray) <= 3 {
				reply.DisplayTitle = "Please provide a project name and description! `@togglbot start PROJECT_NAME DESCRIPTION`"
				replyChannel <- reply
				break
			}
			project := commandArray[2]
			description := strings.Join(commandArray[3:], " ")
			pid := getProjectWithName(togglApiKey, project)
			fmt.Printf("%v", pid)
			startTimer(togglApiKey, description, pid)
			reply.DisplayTitle = "Timer started! *get back to work peon*"
			replyChannel <- reply
		case "stop":
			te, err := stopTimer(togglApiKey)
			if err != nil {
				reply.DisplayTitle = "couldn't stop timer: " + err.Error()
				replyChannel <- reply
				break
			}
			dur, err := time.ParseDuration(fmt.Sprintf("%vs", te.Duration))
			if err != nil {
				log.Fatal("Unparseable duration! %v", te.Duration)
			}
			reply.DisplayTitle = fmt.Sprintf("Timer Stopped. Worked for %v.", dur.String())
			replyChannel <- reply
		case "track":
			if len(commandArray) < 4 {
				reply.DisplayTitle = "Sorry, I don't have enough information to make an event for you. try `@togglbot track PROJECT_NAME 9:00AM-5:00PM TASK_DESCRIPTION`"
				replyChannel <- reply
				break
			}
			projectName := commandArray[2]
			pid := getProjectWithName(togglApiKey, projectName)
			timeRange := commandArray[3]
			description := "no description provided"
			if len(commandArray) > 4 {
				description = strings.Join(commandArray[4:], " ")
			}

			startTime, duration, err := parseTimeRange(timeRange)
			if err != nil {
				reply.DisplayTitle = err.Error()
				replyChannel <- reply
				break
			}
			createTimeEntry(togglApiKey, description, *startTime, *duration, pid, 0)
			reply.DisplayTitle = "Time entry created!"
			replyChannel <- reply
		default:
			reply.DisplayTitle = "Sorry, i don't understand that command. Try `@Togglbot help`"
			replyChannel <- reply
		}
	}
}

func parseTimeRange(timeRange string) (*time.Time, *time.Duration, error) {
	fields := strings.Split(timeRange, "-")
	if len(fields) != 2 {
		return nil, nil, errors.New("Your date range is incorrectly formatted! Try something like: 9:00AM-5:00PM")
	}

	startTime, err := time.Parse(time.Kitchen, fields[0])
	if err != nil {
		return nil, nil, errors.New("Your date range is incorrectly formatted! Try something like: 9:00AM-5:00PM")
	}

	endTime, err := time.Parse(time.Kitchen, fields[1])
	if err != nil {
		return nil, nil, errors.New("Your date range is incorrectly formatted! Try something like: 9:00AM-5:00PM")
	}

	duration := endTime.Sub(startTime)

	// Kitchen time has only hours and PM/AM. Drop in today's date.
	now := time.Now()
	startTime = startTime.AddDate(now.Year(), int(now.Month())-1, now.Day()-1)
	return &startTime, &duration, nil
}

func handleBotReplies() {
	for {
		reply := <-botReplyChannel
		params := slack.PostMessageParameters{}
		params.AsUser = true
		if reply.Attachment != nil {
			params.Attachments = []slack.Attachment{*reply.Attachment}
		}
		_, _, err := api.PostMessage(reply.Channel, reply.DisplayTitle, params)
		if err != nil {
			log.Fatal(err)
		}
	}
}

func main() {

	if len(os.Args) != 2 {
		fmt.Printf("usage: togglbot slack-bot-token")
		os.Exit(1)
	}

	//First attempt to deserialize a saved registration file, otherwise
	//create new
	err := Load(file, &userMap)
	if err != nil {
		fmt.Println("ERROR WAS DETECTED")
		log.Fatal(err)
		userMap = make(map[string]string)
	}

	sessionMap = make(map[string]toggl.TimeEntry)
	token := os.Args[1]
	api = slack.New(token)
	rtm := api.NewRTM()

	botCommandChannel = make(chan *BotCommand)
	botReplyChannel = make(chan ReplyChannel)
	go rtm.ManageConnection()
	go handleBotCommands(botReplyChannel)
	go handleBotReplies()
	go fmt.Println("TogglBot ready, ^C exits")
Loop:
	for {
		select {
		case msg := <-rtm.IncomingEvents:
			switch event := msg.Data.(type) {
			case *slack.ConnectedEvent:
				botId = event.Info.User.ID
			case *slack.MessageEvent:

				botCommand := &BotCommand{
					Channel: event.Channel,
					Event:   event,
					UserId:  event.User,
				}

				if event.Type == "message" && strings.HasPrefix(event.Text, "<@"+botId+">") {
					fmt.Println("FOUND COMMAND")
					botCommandChannel <- botCommand
				}
			case *slack.RTMError:
				fmt.Printf("ERROR: %s\n", event.Error())
			case *slack.InvalidAuthEvent:
				fmt.Printf("Invalid credentials")
				break Loop
			}
		}
	}

}
<commit_msg>closes #2, adds ability to DM togglbot *without* mentioning him.<commit_after>package main

import "fmt"
import "log"
import "os"
import "errors"
import "strings"
import "github.com/nlopes/slack"
import "github.com/tadgh/go-toggl"
import "encoding/gob"
import "time"
import "runtime"

const file = "./test.gob"

func Check(err error) {
	if err != nil {
		_, file, line, _ := runtime.Caller(1)
		fmt.Println(line, "\t", file, "\n", err)
		os.Exit(1)
	}
}

func pingTogglApi(apiKey string) error {
	ts := toggl.OpenSession(apiKey)
	_, err := ts.GetAccount()
	if err != nil {
		fmt.Println(os.Stderr, "Error: %s\n", err)
		return err
	}
	return nil
}

func Save(path string, object interface{}) error {
	file, err := os.Create(path)
	if err == nil {
		encoder := gob.NewEncoder(file)
		encoder.Encode(object)
	}
	file.Close()
	return err
}

func Load(path string, object interface{}) error {
	file, err := os.Open(path)
	if err == nil {
		decoder := gob.NewDecoder(file)
		err = decoder.Decode(object)
	}
	file.Close()
	return err
}

func createTimeEntry(apiKey, description string, start time.Time, duration time.Duration, pid, tid int) *toggl.TimeEntry {
	ts := toggl.OpenSession(apiKey)
	te, err := ts.CreateTimeEntry(pid, 0, start, duration, description)
	if err != nil {
		log.Fatal("Error uploading time entry! %v", err)
	}
	return &te
}

func stopTimer(apiKey string) (*toggl.TimeEntry, error) {
	ts := toggl.OpenSession(apiKey)
	te, err := ts.GetActiveTimeEntry()
	if err != nil {
		return nil, err
	}
	if te.ID <= 0 {
		return nil, errors.New("No timer is currently running!")
	}
	te, err = ts.StopTimeEntry(te)
	if err != nil {
		return nil, err
	}
	return &te, nil
}

func startTimer(apiKey, description string, pid int) *toggl.TimeEntry {
	ts := toggl.OpenSession(apiKey)
	te, err := ts.StartTimeEntryForProject(description, pid)

	if err != nil {
		fmt.Println(err)
	}
	sessionMap[apiKey] = te
	return &te
}

func getProjectWithName(apiKey, projectName string) int {
	ts := toggl.OpenSession(apiKey)
	acc, err := ts.GetAccount()
	var retVal int
	if err != nil {
		fmt.Println(err)
	}
	for _, project := range acc.Data.Projects {
		//case insensitive string comparison
		if strings.EqualFold(project.Name, projectName) {
			retVal = project.ID
		}
	}
	return retVal
}

type BotCommand struct {
	Channel string
	Event   *slack.MessageEvent
	UserId  string
}

type ReplyChannel struct {
	Channel      string
	Attachment   *slack.Attachment
	DisplayTitle string
}

var (
	api               *slack.Client
	botCommandChannel chan *BotCommand
	botReplyChannel   chan ReplyChannel
	botId             string
	userMap           map[string]string
	sessionMap        map[string]toggl.TimeEntry
)

func handleBotCommands(replyChannel chan ReplyChannel) {
	commands := map[string]string{
		"register": "Register yourself with togglbot. `@togglbot register MY_TOGGL_API_KEY`",
		"start":    "Start a timer for a given project and description. `@togglbot start <PROJECT_NAME> <EVERYTHING_ELSE_IS_DESCRIPTION>`",
		"stop":     "Stops any current timer session. `@togglbot stop`",
		"track":    "adds a toggl entry to a project for a given time range. `@togglbot track icancope 9am-5pm`",
	}

	for {
		incomingCommand := <-botCommandChannel
		commandArray := strings.Fields(incomingCommand.Event.Text)
		if commandArray[0] == "<@"+botId+">" {
			commandArray = commandArray[1:]
		}
		var reply ReplyChannel
		reply.Channel = incomingCommand.Channel

		switch commandArray[0] {
		case "help":
			reply.DisplayTitle = "Help!"
			fields := make([]slack.AttachmentField, 0)
			for k, v := range commands {
				fields = append(fields, slack.AttachmentField{
					Title: "<bot> " + k,
					Value: v,
				})
			}
			attachment := &slack.Attachment{
				Pretext:    "TogglBot Command List",
				Color:      "#B733FF",
				Fields:     fields,
				MarkdownIn: []string{"fields"},
			}
			reply.Attachment = attachment
			replyChannel <- reply
		case "register":
			togglApiKey := commandArray[1]
			err := pingTogglApi(togglApiKey)
			if err != nil {
				reply.DisplayTitle = "Failed to register. Bad api key?"
			} else {
				userMap[incomingCommand.Event.User] = togglApiKey
				err := Save(file, userMap)
				Check(err)
				reply.DisplayTitle = "Successfully registered!"
			}
			replyChannel <- reply
		}

		if reply.DisplayTitle != "" {
			continue
		}
		togglApiKey, ok := userMap[incomingCommand.Event.User]
		if !ok {
			reply.DisplayTitle = "You have not registered with togglbot yet. Try @togglbot register API_KEY_HERE"
			replyChannel <- reply
			continue
		}

		switch commandArray[0] {
		case "start":
			if len(commandArray) <= 2 {
				reply.DisplayTitle = "Please provide a project name and description! `@togglbot start PROJECT_NAME DESCRIPTION`"
				replyChannel <- reply
				break
			}
			project := commandArray[1]
			description := strings.Join(commandArray[2:], " ")
			pid := getProjectWithName(togglApiKey, project)
			fmt.Printf("%v", pid)
			startTimer(togglApiKey, description, pid)
			reply.DisplayTitle = "Timer started! *get back to work peon*"
			replyChannel <- reply
		case "stop":
			te, err := stopTimer(togglApiKey)
			if err != nil {
				reply.DisplayTitle = "couldn't stop timer: " + err.Error()
				replyChannel <- reply
				break
			}
			dur, err := time.ParseDuration(fmt.Sprintf("%vs", te.Duration))
			if err != nil {
				log.Fatal("Unparseable duration! %v", te.Duration)
			}
			reply.DisplayTitle = fmt.Sprintf("Timer Stopped. Worked for %v.", dur.String())
			replyChannel <- reply
		case "track":
			if len(commandArray) < 3 {
				reply.DisplayTitle = "Sorry, I don't have enough information to make an event for you. try `@togglbot track PROJECT_NAME 9:00AM-5:00PM TASK_DESCRIPTION`"
				replyChannel <- reply
				break
			}
			projectName := commandArray[1]
			pid := getProjectWithName(togglApiKey, projectName)
			timeRange := commandArray[2]
			description := "no description provided"
			if len(commandArray) > 3 {
				description = strings.Join(commandArray[3:], " ")
			}

			startTime, duration, err := parseTimeRange(timeRange)
			if err != nil {
				reply.DisplayTitle = err.Error()
				replyChannel <- reply
				break
			}
			createTimeEntry(togglApiKey, description, *startTime, *duration, pid, 0)
			reply.DisplayTitle = "Time entry created!"
			replyChannel <- reply
		default:
			reply.DisplayTitle = "Sorry, i don't understand that command. Try `@Togglbot help`"
			replyChannel <- reply
		}
	}
}

func parseTimeRange(timeRange string) (*time.Time, *time.Duration, error) {
	fields := strings.Split(timeRange, "-")
	if len(fields) != 2 {
		return nil, nil, errors.New("Your date range is incorrectly formatted! Try something like: 9:00AM-5:00PM")
	}

	startTime, err := time.Parse(time.Kitchen, fields[0])
	if err != nil {
		return nil, nil, errors.New("Your date range is incorrectly formatted! Try something like: 9:00AM-5:00PM")
	}

	endTime, err := time.Parse(time.Kitchen, fields[1])
	if err != nil {
		return nil, nil, errors.New("Your date range is incorrectly formatted! Try something like: 9:00AM-5:00PM")
	}

	duration := endTime.Sub(startTime)

	// Kitchen time has only hours and PM/AM. Drop in today's date.
	now := time.Now()
	startTime = startTime.AddDate(now.Year(), int(now.Month())-1, now.Day()-1)
	return &startTime, &duration, nil
}

func handleBotReplies() {
	for {
		reply := <-botReplyChannel
		params := slack.PostMessageParameters{}
		params.AsUser = true
		if reply.Attachment != nil {
			params.Attachments = []slack.Attachment{*reply.Attachment}
		}
		_, _, err := api.PostMessage(reply.Channel, reply.DisplayTitle, params)
		if err != nil {
			fmt.Println("FATAL SHIT")
			log.Fatal(err)
		}
	}
}

func main() {

	if len(os.Args) != 2 {
		fmt.Printf("usage: togglbot slack-bot-token")
		os.Exit(1)
	}

	//First attempt to deserialize a saved registration file, otherwise
	//create new
	err := Load(file, &userMap)
	if err != nil {
		fmt.Println("ERROR WAS DETECTED")
		log.Fatal(err)
		userMap = make(map[string]string)
	}

	sessionMap = make(map[string]toggl.TimeEntry)
	token := os.Args[1]
	api = slack.New(token)
	rtm := api.NewRTM()
	botCommandChannel = make(chan *BotCommand)
	botReplyChannel = make(chan ReplyChannel)
	go rtm.ManageConnection()
	go handleBotCommands(botReplyChannel)
	go handleBotReplies()
	go fmt.Println("TogglBot ready, ^C exits")
Loop:
	for {
		select {
		case msg := <-rtm.IncomingEvents:
			switch event := msg.Data.(type) {
			case *slack.ConnectedEvent:
				botId = event.Info.User.ID
			case *slack.MessageEvent:

				botCommand := &BotCommand{
					Channel: event.Channel,
					Event:   event,
					UserId:  event.User,
				}

				if isValidMessageEvent(event) {
					fmt.Println("Received event: ", event)
					botCommandChannel <- botCommand
				}
			case *slack.RTMError:
				fmt.Printf("ERROR: %s\n", event.Error())
			case *slack.InvalidAuthEvent:
				fmt.Printf("Invalid credentials")
				break Loop
			}
		}
	}

}

func isValidMessageEvent(event *slack.MessageEvent) bool {
	if event.Type != "message" {
		return false
	}
	if event.User == botId {
		return false
	}
	if strings.HasPrefix(event.Text, "<@"+botId+">") {
		return true
	}
	if strings.HasPrefix(event.Channel, "D") {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"time"
)

func worker(id int, jobs <-chan int, results chan<- string) {
	os.Chdir("gittest")
	for j := range jobs {
		results <- runCommand("git show " + strconv.Itoa(j) + ":test.txt")
	}
}

func testWorkers() {
	start := time.Now()
	//In order to use our pool of workers we need to send them work and collect their results. We make 2 channels for this.
	jobs := make(chan int, 100)
	results := make(chan string, 100)
	//This starts up 50 workers, initially blocked because there are no jobs yet.
	for w := 1; w <= 50; w++ {
		go worker(w, jobs, results)
	}
	//Here we send 9 jobs and then close that channel to indicate that’s all the work we have.
	for j := 0; j < 100; j++ {
		jobs <- j
	}
	close(jobs)
	//Finally we collect all the results of the work.
	for a := 0; a < 100; a++ {
		<-results
	}
	elapsed := time.Since(start)
	log.Printf("testWorkers took %s", elapsed/101)
}

func createGithubRepo(username string, password string, reponame string) (bool, string) {
	// Generated by curl-to-Go: https://mholt.github.io/curl-to-go

	body := strings.NewReader(`{"name":"` + reponame + `"}`)
	req, err := http.NewRequest("POST", "https://api.github.com/user/repos", body)
	if err != nil {
		// handle err
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	req.SetBasicAuth(username, password)
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		// handle err
	}
	defer resp.Body.Close()
	jsonDataFromHttp, err := ioutil.ReadAll(resp.Body)
	response := string(jsonDataFromHttp)
	if strings.Contains(response, "Bad credentials") || strings.Contains(response, "Validation Failed") {
		return false, response
	}
	return true, response
}

func main() {
	// createBranches()
	// readBranches()
	// testWorkers()
	fmt.Println(runCommand("git push origin master"))
}

func readBranches() {
	os.Chdir("gittest")
	start := time.Now()
	for i := 0; i < 100; i++ {
		runCommand("git show " + strconv.Itoa(i) + ":test.txt")
	}
	elapsed := time.Since(start)
	log.Printf("readBranches took %s", elapsed/101)
	fmt.Println("Done")

}

func createBranches() {
	os.RemoveAll("./gittest")
	os.Mkdir("gittest", 0644)
	os.Chdir("gittest")
	runCommand("git init")
	d1 := []byte("hello, world")
	err := ioutil.WriteFile("test.txt", d1, 0644)
	if err != nil {
		log.Fatal(err)
	}
	runCommand("git add test.txt")
	runCommand("git commit -am 'added test.txt'")

	start := time.Now()
	for i := 0; i < 100; i++ {
		runCommand("git checkout --orphan " + strconv.Itoa(i))
		d1 = []byte("hello, world branch #" + strconv.Itoa(i))
		err = ioutil.WriteFile("test.txt", d1, 0644)
		if err != nil {
			log.Fatal(err)
		}
		runCommand("git add test.txt")
		runCommand("git commit -am 'added test.txt'")
		// 3 commands = 115 ms / command
	}

	elapsed := time.Since(start)
	log.Printf("createBranches took %s", elapsed)
	fmt.Println("Done")
}

func runCommand(fullCommand string) string {
	var (
		cmdOut []byte
		err    error
	)
	splitCommand := strings.Split(fullCommand, " ")
	if len(splitCommand) > 1 {
		if cmdOut, err = exec.Command(splitCommand[0], splitCommand[1:]...).Output(); err != nil {
			log.Println(splitCommand)
			log.Fatal(err)
		}
	} else {
		if cmdOut, err = exec.Command(splitCommand[0]).Output(); err != nil {
			log.Println(splitCommand)
			log.Fatal(err)
		}
	}
	return string(cmdOut)
}
<commit_msg>Test2<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"time"
)

func worker(id int, jobs <-chan int, results chan<- string) {
	os.Chdir("gittest")
	for j := range jobs {
		results <- runCommand("git show " + strconv.Itoa(j) + ":test.txt")
	}
}

func testWorkers() {
	start := time.Now()
	//In order to use our pool of workers we need to send them work and collect their results. We make 2 channels for this.
	jobs := make(chan int, 100)
	results := make(chan string, 100)
	//This starts up 50 workers, initially blocked because there are no jobs yet.
	for w := 1; w <= 50; w++ {
		go worker(w, jobs, results)
	}
	//Here we send 9 jobs and then close that channel to indicate that’s all the work we have.
	for j := 0; j < 100; j++ {
		jobs <- j
	}
	close(jobs)
	//Finally we collect all the results of the work.
	for a := 0; a < 100; a++ {
		<-results
	}
	elapsed := time.Since(start)
	log.Printf("testWorkers took %s", elapsed/101)
}

func createGithubRepo(username string, password string, reponame string) (bool, string) {
	// Generated by curl-to-Go: https://mholt.github.io/curl-to-go

	body := strings.NewReader(`{"name":"` + reponame + `"}`)
	req, err := http.NewRequest("POST", "https://api.github.com/user/repos", body)
	if err != nil {
		// handle err
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	req.SetBasicAuth(username, password)
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		// handle err
	}
	defer resp.Body.Close()
	jsonDataFromHttp, err := ioutil.ReadAll(resp.Body)
	response := string(jsonDataFromHttp)
	if strings.Contains(response, "Bad credentials") || strings.Contains(response, "Validation Failed") {
		return false, response
	}
	return true, response
}

func main() {
	// createBranches()
	// readBranches()
	// testWorkers()

}

func githubPush(username string, password string, reponame string) {
	fmt.Println(runCommand("git push https://" + username + ":" + password + "@github.com/" + username + "/" + reponame + ".git --all"))
}

func readBranches() {
	os.Chdir("gittest")
	start := time.Now()
	for i := 0; i < 100; i++ {
		runCommand("git show " + strconv.Itoa(i) + ":test.txt")
	}
	elapsed := time.Since(start)
	log.Printf("readBranches took %s", elapsed/101)
	fmt.Println("Done")

}

func createBranches() {
	os.RemoveAll("./gittest")
	os.Mkdir("gittest", 0644)
	os.Chdir("gittest")
	runCommand("git init")
	d1 := []byte("hello, world")
	err := ioutil.WriteFile("test.txt", d1, 0644)
	if err != nil {
		log.Fatal(err)
	}
	runCommand("git add test.txt")
	runCommand("git commit -am 'added test.txt'")

	start := time.Now()
	for i := 0; i < 100; i++ {
		runCommand("git checkout --orphan " + strconv.Itoa(i))
		d1 = []byte("hello, world branch #" + strconv.Itoa(i))
		err = ioutil.WriteFile("test.txt", d1, 0644)
		if err != nil {
			log.Fatal(err)
		}
		runCommand("git add test.txt")
		runCommand("git commit -am 'added test.txt'")
		// 3 commands = 115 ms / command
	}

	elapsed := time.Since(start)
	log.Printf("createBranches took %s", elapsed)
	fmt.Println("Done")
}

func runCommand(fullCommand string) string {
	var (
		cmdOut []byte
		err    error
	)
	splitCommand := strings.Split(fullCommand, " ")
	if len(splitCommand) > 1 {
		if cmdOut, err = exec.Command(splitCommand[0], splitCommand[1:]...).Output(); err != nil {
			log.Println(splitCommand)
			log.Fatal(err)
		}
	} else {
		if cmdOut, err = exec.Command(splitCommand[0]).Output(); err != nil {
			log.Println(splitCommand)
			log.Fatal(err)
		}
	}
	return string(cmdOut)
}
<|endoftext|>


<commit_before>package stats

import (
	"encoding/csv"
	"flag"
	"fmt"
	"io"
	"os"
	"strconv"
	"unicode/utf8"
)

var (
	ignore = flag.Bool("i", false, "ignore invalid numbers")
	behead = flag.Bool("b", false,
		"remove the first line (head) from calculations. Useful to ignore column names")
	separator = flag.String("s", " ",
		"define the SEPARATOR to use instead of whitespace for column separator")
	column = flag.Int("c", 1, "calculate stats based on the specified COLUMN")
)

func fail(format string, v ...interface{}) {
	fmt.Fprintf(os.Stderr, format, v...)
	os.Exit(1)
}

func calculate(s *Stats) {
	if len(flag.Args()) == 0 {
		parse("<stdin>", os.Stdin, s)
	}

	for _, filename := range flag.Args() {
		if filename == "-" {
			parse("<stdin>", os.Stdin, s)
			continue
		}
		file, err := os.Open(filename)
		if err != nil {
			fail("%s\n", err.Error())
		}
		parse(filename, file, s)
	}
}

func parse(filename string, input *os.File, s *Stats) {
	r := csv.NewReader(input)
	sep, _ := utf8.DecodeRuneInString(*separator)
	r.Comma = sep
	var line int64
	for {
		line += 1
		record, err := r.Read()
		if err == io.EOF {
			return
		}
		if err != nil {
			fail("An error occurred while reading the file %s: %+v\n", filename, err)
		}
		if line == 1 && *behead {
			continue
		}
		if *column > len(record) {
			fail("Invalid column number: %d", *column)
		}
		value, err := strconv.ParseFloat(record[*column-1], 64)
		if err != nil {
			if *ignore {
				continue
			} else {
				fail("Invalid number found in file %s at line %d: %s\n", filename, line, record[*column-1])
			}
		}
		s.Update(value)
	}
}

func Tool(desc string, outputter func(s *Stats)) {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s [OPTION]... [FILE]...\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "%s\n\n", desc)
		flag.PrintDefaults()
		fmt.Fprintln(os.Stderr, "\nWith no FILE, or when FILE is -, read standard input.")
	}
	flag.Parse()

	s := NewStats()
	calculate(s)
	outputter(s)
}
<commit_msg>replace go's builtin flag package by POSIX-compliant one<commit_after>package stats

import (
	"encoding/csv"
	"fmt"
	"io"
	"os"
	"strconv"
	"unicode/utf8"

	flag "github.com/ogier/pflag"
)

var (
	ignore = flag.BoolP("ignore", "i", false, "ignore invalid numbers")
	behead = flag.BoolP("behead", "b", false,
		"remove the first line (head) from calculations. Useful to ignore column names")
	separator = flag.StringP("separator", "s", " ",
		"define the SEPARATOR to use instead of whitespace for column separator")
	column = flag.IntP("column", "c", 1, "calculate stats based on the specified COLUMN")
)

func fail(format string, v ...interface{}) {
	fmt.Fprintf(os.Stderr, format, v...)
	os.Exit(1)
}

func calculate(s *Stats) {
	if len(flag.Args()) == 0 {
		parse("<stdin>", os.Stdin, s)
	}

	for _, filename := range flag.Args() {
		if filename == "-" {
			parse("<stdin>", os.Stdin, s)
			continue
		}
		file, err := os.Open(filename)
		if err != nil {
			fail("%s\n", err.Error())
		}
		parse(filename, file, s)
	}
}

func parse(filename string, input *os.File, s *Stats) {
	r := csv.NewReader(input)
	sep, _ := utf8.DecodeRuneInString(*separator)
	r.Comma = sep
	var line int64
	for {
		line += 1
		record, err := r.Read()
		if err == io.EOF {
			return
		}
		if err != nil {
			fail("An error occurred while reading the file %s: %+v\n", filename, err)
		}
		if line == 1 && *behead {
			continue
		}
		if *column > len(record) {
			fail("Invalid column number: %d", *column)
		}
		value, err := strconv.ParseFloat(record[*column-1], 64)
		if err != nil {
			if *ignore {
				continue
			} else {
				fail("Invalid number found in file %s at line %d: %s\n", filename, line, record[*column-1])
			}
		}
		s.Update(value)
	}
}

func Tool(desc string, outputter func(s *Stats)) {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s [OPTION]... [FILE]...\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "%s\n\n", desc)
		flag.PrintDefaults()
		fmt.Fprintln(os.Stderr, "\nWith no FILE, or when FILE is -, read standard input.")
	}
	flag.Parse()

	s := NewStats()
	calculate(s)
	outputter(s)
}
<|endoftext|>


<commit_before>package goserver

import (
	"regexp"
	"strings"
)

type (
	node struct {
		path     string
		regexp   *regexp.Regexp
		route    *route
		parent   *node
		children []*node
		params   uint8
	}
)

func (n *node) isRoot() bool {
	return n.parent == nil
}

func (n *node) isLeaf() bool {
	return len(n.children) == 0
}

func (n *node) regexpToString() string {
	if n.regexp == nil {
		return ""
	}
	return n.regexp.String()
}

func (n *node) setRegexp(exp string) {
	reg, err := regexp.Compile(exp)
	if err == nil {
		n.regexp = reg
	}
}

func (n *node) setRoute(r *route) {
	n.route = r
}

func (n *node) addChild(paths []string) *node {
	if len(paths) > 0 && paths[0] != "" {
		var cn *node
		for _, child := range n.children {
			if child.path == paths[0] {
				cn = child
				break
			}
		}
		if cn == nil {
			cn = newNode(n, paths[0])
			n.children = append(n.children, cn)
		}
		return cn.addChild(paths[1:])
	}
	return n
}

func (n *node) child(paths []string) (*node, Params) {
	return n.childRecursive(paths)
}

//childRecursive returns child node by path using recurency
func (n *node) childRecursive(paths []string) (*node, Params) {
	if len(paths) > 0 && paths[0] != "" {
		path := paths[0]
		for _, child := range n.children {
			if child.path == path {
				return child.child(paths[1:])
			}
			pl := len(child.path)
			if pl > 1 && child.path[0] == ':' {
				if child.regexp != nil && !child.regexp.MatchString(path) {
					continue
				}
				node, params := child.child(paths[1:])
				if node == nil {
					continue
				}
				if child.regexp != nil && child.regexp.MatchString(path) {
					for i := 1; i < pl; i++ {
						if child.path[i] == ':' {
							params[child.params-1].Key = child.path[1:i]
							break
						}
					}
				} else {
					params[child.params-1].Key = child.path[1:]
				}
				params[child.params-1].Value = path
				return node, params
			}
		}
	} else if len(paths) == 0 {
		return n, make(Params, n.params)
	}
	return nil, nil
}

//childNotRecursive returns child node by path not using recurency
func (n *node) childNotRecursive(paths []string) (*node, Params) {
	var params Params
st:
	for {
		if len(paths) > 0 && paths[0] != "" {
			path := paths[0]
			for _, child := range n.children {
				if child.path == path {
					n = child
					paths = paths[1:]
					continue st
				}
				pl := len(child.path)
				if pl > 0 && child.path[0] == ':' {
					if child.regexp != nil && !child.regexp.MatchString(path) {
						continue
					}
					if len(params) == 0 {
						params = make(Params, len(paths))
					}
					if child.regexp != nil && child.regexp.MatchString(path) {
						for i := 1; i < pl; i++ {
							if child.path[i] == ':' {
								params[child.params-1].Key = child.path[1:i]
								break
							}
						}
					} else {
						params[child.params-1].Key = child.path[1:]
					}
					params[child.params-1].Value = path
					n = child
					paths = paths[1:]
					continue st
				}
			}
			return nil, nil
		} else if len(paths) == 0 {
			return n, params
		}
		return nil, nil
	}
}

func newNode(root *node, path string) *node {
	n := &node{
		path:     path,
		parent:   root,
		children: make([]*node, 0),
	}

	if root != nil {
		n.params = root.params
	}

	if len(n.path) > 0 && path[:1] == ":" {
		n.params++
		if parts := strings.Split(n.path, ":"); len(parts) == 3 {
			n.setRegexp(parts[2])
		}
	}

	return n
}

func newRoot(path string) *node {
	return newNode(nil, path)
}
<commit_msg>Add benchmarks<commit_after>package goserver

import (
	"regexp"
	"strings"
)

type (
	node struct {
		path     string
		regexp   *regexp.Regexp
		route    *route
		parent   *node
		children []*node
		params   uint8
	}
)

func (n *node) isRoot() bool {
	return n.parent == nil
}

func (n *node) isLeaf() bool {
	return len(n.children) == 0
}

func (n *node) regexpToString() string {
	if n.regexp == nil {
		return ""
	}
	return n.regexp.String()
}

func (n *node) setRegexp(exp string) {
	reg, err := regexp.Compile(exp)
	if err == nil {
		n.regexp = reg
	}
}

func (n *node) setRoute(r *route) {
	n.route = r
}

func (n *node) addChild(paths []string) *node {
	if len(paths) > 0 && paths[0] != "" {
		var cn *node
		for _, child := range n.children {
			if child.path == paths[0] {
				cn = child
				break
			}
		}
		if cn == nil {
			cn = newNode(n, paths[0])
			n.children = append(n.children, cn)
		}
		return cn.addChild(paths[1:])
	}
	return n
}

func (n *node) child(paths []string) (*node, Params) {
	return n.childRecursive(paths)
}

//childRecursive returns child node by path using recurency
func (n *node) childRecursive(paths []string) (*node, Params) {
	pathsLen := len(paths)
	if pathsLen == 0 {
		return n, make(Params, n.params)
	}
	if pathsLen > 0 && paths[0] != "" {
		path := paths[0]
		for _, child := range n.children {
			if child.path == path {
				return child.child(paths[1:])
			}
			pathLen := len(child.path)
			if pathLen > 1 && child.path[0] == ':' {
				if child.regexp != nil && !child.regexp.MatchString(path) {
					continue
				}
				node, params := child.child(paths[1:])
				if node == nil {
					continue
				}
				if child.regexp != nil && child.regexp.MatchString(path) {
					for i := 1; i < pathLen; i++ {
						if child.path[i] == ':' {
							params[child.params-1].Key = child.path[1:i]
							break
						}
					}
				} else {
					params[child.params-1].Key = child.path[1:]
				}
				params[child.params-1].Value = path
				return node, params
			}
		}
	}
	return nil, nil
}

//childNotRecursive returns child node by path not using recurency
func (n *node) childNotRecursive(paths []string) (*node, Params) {
	var params Params
st:
	for {
		pathsLen := len(paths)
		if pathsLen == 0 {
			return n, params
		}
		if pathsLen > 0 && paths[0] != "" {
			path := paths[0]
			for _, child := range n.children {
				if child.path == path {
					n = child
					paths = paths[1:]
					continue st
				}
				pathLen := len(child.path)
				if pathLen > 0 && child.path[0] == ':' {
					if child.regexp != nil && !child.regexp.MatchString(path) {
						continue
					}
					if len(params) == 0 {
						params = make(Params, len(paths))
					}
					if child.regexp != nil && child.regexp.MatchString(path) {
						for i := 1; i < pathLen; i++ {
							if child.path[i] == ':' {
								params[child.params-1].Key = child.path[1:i]
								break
							}
						}
					} else {
						params[child.params-1].Key = child.path[1:]
					}
					params[child.params-1].Value = path
					n = child
					paths = paths[1:]
					continue st
				}
			}
			return nil, nil
		}
		return nil, nil
	}
}

func newNode(root *node, path string) *node {
	n := &node{
		path:     path,
		parent:   root,
		children: make([]*node, 0),
	}

	if root != nil {
		n.params = root.params
	}

	if len(n.path) > 0 && path[:1] == ":" {
		n.params++
		if parts := strings.Split(n.path, ":"); len(parts) == 3 {
			n.setRegexp(parts[2])
		}
	}

	return n
}

func newRoot(path string) *node {
	return newNode(nil, path)
}
<|endoftext|>


<commit_before>package CloudForest

import ()

type nodeAndCases struct {
	n          *Node
	start      int
	end        int
	nconstants int
	parent     *Node
	dead       bool
}

//Tree represents a single decision tree.
type Tree struct {
	//Tree int
	Root   *Node
	Target string
	Weight float64
}

//NewTree initializes one node tree.
func NewTree() *Tree {
	return &Tree{new(Node), "", -1.0}
}

//AddNode adds a node a the specified path with the specified pred value and/or
//Splitter. Paths are specified in the same format as in rf-aces sf files, as a
//string of 'L' and 'R'. Nodes must be added from the root up as the case where
//the path specifies a node whose parent does not already exist in the tree is
//not handled well.
func (t *Tree) AddNode(path string, pred string, splitter *Splitter) {
	n := new(Node)
	n.Pred = pred
	n.Splitter = splitter
	if t.Root == nil {
		t.Root = n
	} else {
		loc := t.Root
		for i := 0; i < len(path); i++ {
			switch path[i : i+1] {
			case "L":
				if loc.Left == nil {
					loc.Left = n
				}
				loc = loc.Left
			case "R":
				if loc.Right == nil {
					loc.Right = n
				}
				loc = loc.Right

			case "M":
				if loc.Missing == nil {
					loc.Missing = n
				}
				loc = loc.Missing
			}

		}
	}

}

//StripCodes removes all of the coded splits from a tree so that it can be used on new catagorical data.
func (t *Tree) StripCodes() {
	t.Root.Climb(func(n *Node) {
		if n.CodedSplit != nil {
			n.CodedSplit = nil

		}
	})
}

/*
Grow grows the receiver tree through recursion. It uses impurity decrease to select splitters at
each node as in Brieman's Random Forest. It should be called on a tree with only a root node defined.

fm is a feature matrix of training data.

target is the feature to predict via regression or classification as determined by feature type.

cases specifies the cases to calculate impurity decrease over and can contain repeated values
to allow for sampling of cases with replacement as in RF.

canidates specifies the potential features to use as splitters

mTry specifies the number of candidate features to evaluate for each split.

leafSize specifies the minimum number of cases at a leafNode.

splitmissing indicates if missing values should be split onto a third branch

vet indicates if splits should be penalized against a randomized version of them selves
*/
func (t *Tree) Grow(fm *FeatureMatrix,
	target Target,
	cases []int,
	candidates []int,
	oob []int,
	mTry int,
	leafSize int,
	splitmissing bool,
	force bool,
	vet bool,
	evaloob bool,
	extraRandom bool,
	importance *[]*RunningMean,
	depthUsed *[]int,
	allocs *BestSplitAllocs) {

	//var innercanidates []int
	var impDec float64
	// for i := 0; i < len(allocs.Weights); i++ {
	// 	allocs.Weights[i] = 0
	// }
	// allocs.Cases = allocs.Cases[0:0]
	// for _, i := range cases {
	// 	if allocs.Weights[i] == 0 {
	// 		allocs.Cases = append(allocs.Cases, i)
	// 	}
	// 	allocs.Weights[i]++
	// }
	t.Root.CodedRecurse(func(n *Node, innercases *[]int, depth int, nconstantsbefore int) (fi int, split interface{}, nconstants int) {

		nconstants = nconstantsbefore

		if (2 * leafSize) <= len(*innercases) {
			//SampleFirstN(&candidates, &innercanidates, mTry, 0)
			//innercanidates = candidates[:mTry]

			fi, split, impDec, nconstants = fm.BestSplitter(target, innercases, &candidates, mTry, &oob, leafSize, force, vet, evaloob, extraRandom, allocs, nconstantsbefore)

			// for i := mTry; i < len(candidates)-1 && impDec == minImp; i++ {
			// 	randi := i + rand.Intn(len(candidates)-i)
			// 	candidates[randi], candidates[i] = candidates[i], candidates[randi]
			// 	innercanidates = candidates[i : i+1]
			// 	fi, split, impDec, nconstants = fm.BestSplitter(target, innercases, &innercanidates, &oob, leafSize, vet, evaloob, allocs, nconstantsbefore)
			// }
			if split != nil {
				if importance != nil {
					(*importance)[fi].Add(impDec)
				}
				if depthUsed != nil && ((*depthUsed)[fi] == 0 || depth < (*depthUsed)[fi]) {
					(*depthUsed)[fi] = depth
				}
				//not a leaf node so define the splitter and left and right nodes
				//so recursion will continue
				n.CodedSplit = split
				n.Featurei = fi
				n.Splitter = fm.Data[fi].DecodeSplit(split)
				n.Pred = ""
				//is this check needed? is it safe to reuse?
				if n.Left == nil || n.Right == nil {
					n.Left = new(Node)
					n.Right = new(Node)
				}
				if splitmissing {
					n.Missing = new(Node)
				}
				return
			}

		}
		//fmt.Println("Terminating in tree grow.")

		//Leaf node so find the predictive value and set it in n.Pred
		split = nil
		n.CodedSplit = nil
		n.Splitter = nil
		n.Pred = target.FindPredicted(*innercases)
		return

	}, fm, &cases, 0, 0)
}

func (t *Tree) GrowJungle(fm *FeatureMatrix,
	target Target,
	cases []int,
	candidates []int,
	oob []int,
	mTry int,
	leafSize int,
	splitmissing bool,
	force bool,
	vet bool,
	evaloob bool,
	extraRandom bool,
	importance *[]*RunningMean,
	depthUsed *[]int,
	allocs *BestSplitAllocs) {

	//var innercanidates []int
	var impDec float64
	nodes := []nodeAndCases{nodeAndCases{t.Root, 0, len(cases), 0, nil, false}}
	var depth, nconstants, start, end, fi, firstThisLevel int
	var split interface{}

	innercases := cases[0:len(cases)]
	innercases2 := cases[0:len(cases)]

	for {
		//Extend Nodes
		lastThisLevel := len(nodes)

		for i := firstThisLevel; i < lastThisLevel; i++ {
			if nodes[i].dead {
				continue
			}

			node := nodes[i]
			n := node.n
			start = node.start
			end = node.end
			innercases = cases[node.start:node.end]
			nconstants = node.nconstants

			if (2 * leafSize) <= len(innercases) {

				fi, split, impDec, nconstants = fm.BestSplitter(target, &innercases, &candidates, mTry, &oob, leafSize, force, vet, evaloob, extraRandom, allocs, nconstants)

				if split != nil {
					if importance != nil {
						(*importance)[fi].Add(impDec)
					}
					if depthUsed != nil && ((*depthUsed)[fi] == 0 || depth < (*depthUsed)[fi]) {
						(*depthUsed)[fi] = depth
					}

					//not a leaf node so define the splitter and left and right nodes
					//so recursion will continue
					n.CodedSplit = split
					n.Featurei = fi
					n.Splitter = fm.Data[fi].DecodeSplit(split)
					n.Pred = ""

					li, ri := fm.Data[fi].SplitPoints(split, &innercases)

					//Left
					n.Left = new(Node)
					nodes = append(nodes, nodeAndCases{n.Left, start, start + li, nconstants, n, false})

					//Right
					n.Right = new(Node)
					nodes = append(nodes, nodeAndCases{n.Right, start + ri, end, nconstants, n, false})

					if splitmissing {
						n.Missing = new(Node)
						if li != ri {
							nodes = append(nodes, nodeAndCases{n.Missing, start + li, start + ri, nconstants, n, false})
						}
					}

					continue
				}

			}

			//Leaf node so find the predictive value and set it in n.Pred
			split = nil
			n.CodedSplit = nil
			n.Splitter = nil
			n.Pred = target.FindPredicted(innercases)
			continue

		}

		if len(nodes) == lastThisLevel {
			break
		}

		//Combine next level nodes here for jungles
		for i := lastThisLevel; i < len(nodes); i++ {
			if nodes[i].dead {
				continue
			}
			var maxImpDec, impDec float64
			combine := -1
			nodei := nodes[i]
			for j := lastThisLevel; j < len(nodes); j++ {
				if nodes[j].dead {
					continue
				}

				nodej := nodes[j]
				if nodei.end == nodej.start && nodei.parent != nodej.parent {
					//should we consider other harder combinations?
					innercases = cases[nodei.start:nodei.end]
					innercases2 = cases[nodej.start:nodej.end]
					impDec = target.SplitImpurity(&innercases, &innercases2, nil, allocs)

					innercases = cases[nodei.start:nodej.end]
					impDec -= target.Impurity(&innercases, allocs.Counter)

					if impDec > maxImpDec {
						maxImpDec = impDec
						combine = j
					}

				}
			}

			if combine != -1 {
				nodes[combine].dead = true
				nj := nodes[combine]
				nodei.end = nj.end
				parent := nj.parent
				switch nj.n {
				case parent.Left:
					parent.Left = nodei.n
				case parent.Right:
					parent.Right = nodei.n
				case parent.Missing:
					parent.Missing = nodei.n
				}

			}

		}

		firstThisLevel = lastThisLevel

		depth++
	}

}

//GetLeaves is called by the leaf count utility to
//gather statistics about the nodes of a tree including the sets of cases at
//"leaf" nodes that aren't split further and the number of times each feature
//is used to split away each case.
func (t *Tree) GetLeaves(fm *FeatureMatrix, fbycase *SparseCounter) []Leaf {
	leaves := make([]Leaf, 0)
	ncases := fm.Data[0].Length()
	cases := make([]int, 0, ncases)
	for i := 0; i < ncases; i++ {
		cases = append(cases, i)
	}

	t.Root.Recurse(func(n *Node, cases []int, depth int) {
		if n.Left == nil && n.Right == nil { // I'm in a leaf node
			leaves = append(leaves, Leaf{cases, n.Pred})
		}
		if fbycase != nil && n.Splitter != nil { //I'm not in a leaf node?
			for _, c := range cases {
				fbycase.Add(c, fm.Map[n.Splitter.Feature], 1)
			}
		}

	}, fm, cases, 0)
	return leaves

}

//Partition partitions all of the cases in a FeatureMatrix.
func (t *Tree) Partition(fm *FeatureMatrix) *[][]int {
	leaves := make([][]int, 0)
	ncases := fm.Data[0].Length()
	cases := make([]int, 0, ncases)
	for i := 0; i < ncases; i++ {
		cases = append(cases, i)
	}

	t.Root.Recurse(func(n *Node, cases []int, depth int) {
		if n.Left == nil && n.Right == nil { // I'm in a leaf node
			leaves = append(leaves, cases)
		}

	}, fm, cases, 0)
	return &leaves

}

//Leaf is a struct for storing the index of the cases at a terminal "Leaf" node
//along with the Numeric predicted value.
type Leaf struct {
	Cases []int
	Pred  string
}

//Vote casts a vote for the predicted value of each case in fm *FeatureMatrix.
//into bb *BallotBox. Since BallotBox is not thread safe trees should not vote
//into the same BallotBox in parallel.
func (t *Tree) Vote(fm *FeatureMatrix, bb VoteTallyer) {
	ncases := fm.Data[0].Length()
	cases := make([]int, 0, ncases)
	for i := 0; i < ncases; i++ {
		cases = append(cases, i)
	}

	t.VoteCases(fm, bb, cases)
}

//VoteCases casts a vote for the predicted value of each case in fm *FeatureMatrix.
//into bb *BallotBox. Since BallotBox is not thread safe trees should not vote
//into the same BallotBox in parallel.
func (t *Tree) VoteCases(fm *FeatureMatrix, bb VoteTallyer, cases []int) {

	weight := 1.0
	if t.Weight >= 0.0 {
		weight = t.Weight
	}

	t.Root.Recurse(func(n *Node, cases []int, depth int) {
		if n.Left == nil && n.Right == nil {
			// I'm in a leaf node
			for i := 0; i < len(cases); i++ {
				bb.Vote(cases[i], n.Pred, weight)
			}
		}
	}, fm, cases, 0)
}
<commit_msg>better decision jungle code<commit_after>package CloudForest

import ()

type nodeAndCases struct {
	n          *Node
	start      int
	end        int
	nconstants int
	parent     *Node
	dead       bool
}

//Tree represents a single decision tree.
type Tree struct {
	//Tree int
	Root   *Node
	Target string
	Weight float64
}

//NewTree initializes one node tree.
func NewTree() *Tree {
	return &Tree{new(Node), "", -1.0}
}

//AddNode adds a node a the specified path with the specified pred value and/or
//Splitter. Paths are specified in the same format as in rf-aces sf files, as a
//string of 'L' and 'R'. Nodes must be added from the root up as the case where
//the path specifies a node whose parent does not already exist in the tree is
//not handled well.
func (t *Tree) AddNode(path string, pred string, splitter *Splitter) {
	n := new(Node)
	n.Pred = pred
	n.Splitter = splitter
	if t.Root == nil {
		t.Root = n
	} else {
		loc := t.Root
		for i := 0; i < len(path); i++ {
			switch path[i : i+1] {
			case "L":
				if loc.Left == nil {
					loc.Left = n
				}
				loc = loc.Left
			case "R":
				if loc.Right == nil {
					loc.Right = n
				}
				loc = loc.Right

			case "M":
				if loc.Missing == nil {
					loc.Missing = n
				}
				loc = loc.Missing
			}

		}
	}

}

//StripCodes removes all of the coded splits from a tree so that it can be used on new catagorical data.
func (t *Tree) StripCodes() {
	t.Root.Climb(func(n *Node) {
		if n.CodedSplit != nil {
			n.CodedSplit = nil

		}
	})
}

/*
Grow grows the receiver tree through recursion. It uses impurity decrease to select splitters at
each node as in Brieman's Random Forest. It should be called on a tree with only a root node defined.

fm is a feature matrix of training data.

target is the feature to predict via regression or classification as determined by feature type.

cases specifies the cases to calculate impurity decrease over and can contain repeated values
to allow for sampling of cases with replacement as in RF.

canidates specifies the potential features to use as splitters

mTry specifies the number of candidate features to evaluate for each split.

leafSize specifies the minimum number of cases at a leafNode.

splitmissing indicates if missing values should be split onto a third branch

vet indicates if splits should be penalized against a randomized version of them selves
*/
func (t *Tree) Grow(fm *FeatureMatrix,
	target Target,
	cases []int,
	candidates []int,
	oob []int,
	mTry int,
	leafSize int,
	splitmissing bool,
	force bool,
	vet bool,
	evaloob bool,
	extraRandom bool,
	importance *[]*RunningMean,
	depthUsed *[]int,
	allocs *BestSplitAllocs) {

	//var innercanidates []int
	var impDec float64
	// for i := 0; i < len(allocs.Weights); i++ {
	// 	allocs.Weights[i] = 0
	// }
	// allocs.Cases = allocs.Cases[0:0]
	// for _, i := range cases {
	// 	if allocs.Weights[i] == 0 {
	// 		allocs.Cases = append(allocs.Cases, i)
	// 	}
	// 	allocs.Weights[i]++
	// }
	t.Root.CodedRecurse(func(n *Node, innercases *[]int, depth int, nconstantsbefore int) (fi int, split interface{}, nconstants int) {

		nconstants = nconstantsbefore

		if (2 * leafSize) <= len(*innercases) {
			//SampleFirstN(&candidates, &innercanidates, mTry, 0)
			//innercanidates = candidates[:mTry]

			fi, split, impDec, nconstants = fm.BestSplitter(target, innercases, &candidates, mTry, &oob, leafSize, force, vet, evaloob, extraRandom, allocs, nconstantsbefore)

			// for i := mTry; i < len(candidates)-1 && impDec == minImp; i++ {
			// 	randi := i + rand.Intn(len(candidates)-i)
			// 	candidates[randi], candidates[i] = candidates[i], candidates[randi]
			// 	innercanidates = candidates[i : i+1]
			// 	fi, split, impDec, nconstants = fm.BestSplitter(target, innercases, &innercanidates, &oob, leafSize, vet, evaloob, allocs, nconstantsbefore)
			// }
			if split != nil {
				if importance != nil {
					(*importance)[fi].Add(impDec)
				}
				if depthUsed != nil && ((*depthUsed)[fi] == 0 || depth < (*depthUsed)[fi]) {
					(*depthUsed)[fi] = depth
				}
				//not a leaf node so define the splitter and left and right nodes
				//so recursion will continue
				n.CodedSplit = split
				n.Featurei = fi
				n.Splitter = fm.Data[fi].DecodeSplit(split)
				n.Pred = ""
				//is this check needed? is it safe to reuse?
				if n.Left == nil || n.Right == nil {
					n.Left = new(Node)
					n.Right = new(Node)
				}
				if splitmissing {
					n.Missing = new(Node)
				}
				return
			}

		}
		//fmt.Println("Terminating in tree grow.")

		//Leaf node so find the predictive value and set it in n.Pred
		split = nil
		n.CodedSplit = nil
		n.Splitter = nil
		n.Pred = target.FindPredicted(*innercases)
		return

	}, fm, &cases, 0, 0)
}

func (t *Tree) GrowJungle(fm *FeatureMatrix,
	target Target,
	cases []int,
	candidates []int,
	oob []int,
	mTry int,
	leafSize int,
	splitmissing bool,
	force bool,
	vet bool,
	evaloob bool,
	extraRandom bool,
	importance *[]*RunningMean,
	depthUsed *[]int,
	allocs *BestSplitAllocs) {

	//var innercanidates []int
	var impDec float64
	nodes := []nodeAndCases{nodeAndCases{t.Root, 0, len(cases), 0, nil, false}}
	var depth, nconstants, start, end, fi, firstThisLevel int
	var split interface{}

	innercases := cases[0:len(cases)]
	innercases2 := cases[0:len(cases)]

	for {
		//Extend Nodes
		lastThisLevel := len(nodes)

		for i := firstThisLevel; i < lastThisLevel; i++ {
			if nodes[i].dead {
				continue
			}

			node := nodes[i]
			n := node.n
			start = node.start
			end = node.end
			innercases = cases[node.start:node.end]
			nconstants = node.nconstants

			if (2 * leafSize) <= len(innercases) {

				fi, split, impDec, nconstants = fm.BestSplitter(target, &innercases, &candidates, mTry, &oob, leafSize, force, vet, evaloob, extraRandom, allocs, nconstants)

				if split != nil {
					if importance != nil {
						(*importance)[fi].Add(impDec)
					}
					if depthUsed != nil && ((*depthUsed)[fi] == 0 || depth < (*depthUsed)[fi]) {
						(*depthUsed)[fi] = depth
					}

					//not a leaf node so define the splitter and left and right nodes
					//so recursion will continue
					n.CodedSplit = split
					n.Featurei = fi
					n.Splitter = fm.Data[fi].DecodeSplit(split)
					n.Pred = ""

					li, ri := fm.Data[fi].SplitPoints(split, &innercases)

					//Left
					n.Left = new(Node)
					nodes = append(nodes, nodeAndCases{n.Left, start, start + li, nconstants, n, false})

					//Right
					n.Right = new(Node)
					nodes = append(nodes, nodeAndCases{n.Right, start + ri, end, nconstants, n, false})

					if splitmissing {
						n.Missing = new(Node)
						if li != ri {
							nodes = append(nodes, nodeAndCases{n.Missing, start + li, start + ri, nconstants, n, false})
						}
					}

					continue
				}

			}

			//Leaf node so find the predictive value and set it in n.Pred
			split = nil
			n.CodedSplit = nil
			n.Splitter = nil
			n.Pred = target.FindPredicted(innercases)
			continue

		}

		if len(nodes) == lastThisLevel {
			break
		}

		//Combine next level nodes here for jungles
		madeChanges := true
		for madeChanges {
			madeChanges = false
			for i := lastThisLevel; i < len(nodes); i++ {
				if nodes[i].dead {
					continue
				}
				var maxImpDec, impDec float64
				combine := -1
				nodei := nodes[i]
				for j := lastThisLevel; j < len(nodes); j++ {
					if nodes[j].dead {
						continue
					}

					nodej := nodes[j]
					if nodei.end == nodej.start && nodei.parent != nodej.parent {
						//should we consider other harder noncontigeous combinations?
						innercases = cases[nodei.start:nodei.end]
						innercases2 = cases[nodej.start:nodej.end]
						impDec = target.SplitImpurity(&innercases, &innercases2, nil, allocs)

						innercases = cases[nodei.start:nodej.end]
						impDec -= target.Impurity(&innercases, allocs.Counter)

						if impDec > maxImpDec {
							maxImpDec = impDec
							combine = j
						}

					}
				}

				if combine != -1 {
					madeChanges = true
					nodes[combine].dead = true
					nj := nodes[combine]
					nodei.end = nj.end
					parent := nj.parent
					switch nj.n {
					case parent.Left:
						parent.Left = nodei.n
					case parent.Right:
						parent.Right = nodei.n
					case parent.Missing:
						parent.Missing = nodei.n
					}

				}

			}
		}

		firstThisLevel = lastThisLevel

		depth++
	}

}

//GetLeaves is called by the leaf count utility to
//gather statistics about the nodes of a tree including the sets of cases at
//"leaf" nodes that aren't split further and the number of times each feature
//is used to split away each case.
func (t *Tree) GetLeaves(fm *FeatureMatrix, fbycase *SparseCounter) []Leaf {
	leaves := make([]Leaf, 0)
	ncases := fm.Data[0].Length()
	cases := make([]int, 0, ncases)
	for i := 0; i < ncases; i++ {
		cases = append(cases, i)
	}

	t.Root.Recurse(func(n *Node, cases []int, depth int) {
		if n.Left == nil && n.Right == nil { // I'm in a leaf node
			leaves = append(leaves, Leaf{cases, n.Pred})
		}
		if fbycase != nil && n.Splitter != nil { //I'm not in a leaf node?
			for _, c := range cases {
				fbycase.Add(c, fm.Map[n.Splitter.Feature], 1)
			}
		}

	}, fm, cases, 0)
	return leaves

}

//Partition partitions all of the cases in a FeatureMatrix.
func (t *Tree) Partition(fm *FeatureMatrix) *[][]int {
	leaves := make([][]int, 0)
	ncases := fm.Data[0].Length()
	cases := make([]int, 0, ncases)
	for i := 0; i < ncases; i++ {
		cases = append(cases, i)
	}

	t.Root.Recurse(func(n *Node, cases []int, depth int) {
		if n.Left == nil && n.Right == nil { // I'm in a leaf node
			leaves = append(leaves, cases)
		}

	}, fm, cases, 0)
	return &leaves

}

//Leaf is a struct for storing the index of the cases at a terminal "Leaf" node
//along with the Numeric predicted value.
type Leaf struct {
	Cases []int
	Pred  string
}

//Vote casts a vote for the predicted value of each case in fm *FeatureMatrix.
//into bb *BallotBox. Since BallotBox is not thread safe trees should not vote
//into the same BallotBox in parallel.
func (t *Tree) Vote(fm *FeatureMatrix, bb VoteTallyer) {
	ncases := fm.Data[0].Length()
	cases := make([]int, 0, ncases)
	for i := 0; i < ncases; i++ {
		cases = append(cases, i)
	}

	t.VoteCases(fm, bb, cases)
}

//VoteCases casts a vote for the predicted value of each case in fm *FeatureMatrix.
//into bb *BallotBox. Since BallotBox is not thread safe trees should not vote
//into the same BallotBox in parallel.
func (t *Tree) VoteCases(fm *FeatureMatrix, bb VoteTallyer, cases []int) {

	weight := 1.0
	if t.Weight >= 0.0 {
		weight = t.Weight
	}

	t.Root.Recurse(func(n *Node, cases []int, depth int) {
		if n.Left == nil && n.Right == nil {
			// I'm in a leaf node
			for i := 0; i < len(cases); i++ {
				bb.Vote(cases[i], n.Pred, weight)
			}
		}
	}, fm, cases, 0)
}
<|endoftext|>


<commit_before>// tube.go
package beanstalkd

type BeanstalkdTube struct {
	clients   []*BeanstalkdClient //
	maxActive int                 //
	useTime   int                 //
	idleTime  int                 //
	addr      string              //
	tube      string              //
}

var (
	DEFAULT_MAX_ACTIVE = 30             // default pool size
	DEFAULT_USE_TIME   = 20 * 60 * 1000 // default checkout time is 20 minutes
	DEFAULT_IDEL_TIME  = 20 * 60 * 1000 // connection will be removed after not use
)

func NewBeanstalkdTube1(addr, tube string) *BeanstalkdTube {
	return &BeanstalkdTube{
		clients:   make([]*BeanstalkdClient, DEFAULT_MAX_ACTIVE),
		maxActive: DEFAULT_MAX_ACTIVE,
		useTime:   DEFAULT_USE_TIME,
		idleTime:  DEFAULT_IDEL_TIME,
		addr:      addr,
		tube:      tube,
	}
}

func NewBeanstalkdTube2(addr, tube string, poolSize, useTime, idleTime int) *BeanstalkdTube {
	return &BeanstalkdTube{
		clients:   make([]*BeanstalkdClient, poolSize),
		maxActive: poolSize,
		useTime:   useTime,
		idleTime:  idleTime,
		addr:      addr,
		tube:      tube,
	}
}
<commit_msg>Delete tube.go<commit_after><|endoftext|>


<commit_before>// Copyright 2015 mvbjrn. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !windows !cgo
// +build 386 amd64 arm

package serial

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"syscall"
	"time"
	"unsafe"
)

// TODO flow control

var (
	baudrates = map[Baud]uint32{
		Baud4800:   syscall.B4800,
		Baud9600:   syscall.B9600,
		Baud19200:  syscall.B19200,
		Baud38400:  syscall.B38400,
		Baud57600:  syscall.B57600,
		Baud115200: syscall.B115200,
	}
	databits = map[DataBit]uint32{
		DataBit5: syscall.CS5,
		DataBit6: syscall.CS6,
		DataBit7: syscall.CS7,
		DataBit8: syscall.CS8,
	}
)

// structs and its functions

// Connection represents a serial connection with all parameters.
type Connection struct {
	Port    string
	Baud    Baud
	DataBit DataBit
	StopBit StopBit
	Parity  Parity
	f       *os.File
	isOpen  bool
}

func (connection *Connection) check() error {

	match, _ := regexp.MatchString("/dev/tty.*", connection.Port)
	if !match {
		return errPort
	}

	switch connection.Baud {
	case Baud115200, Baud57600, Baud38400, Baud19200, Baud9600, Baud4800:
	default:
		return errBaud
	}

	switch connection.DataBit {
	case DataBit5, DataBit6, DataBit7, DataBit8:
	default:
		return errDataBit
	}

	switch connection.StopBit {
	case StopBit1, StopBit2:
	default:
		return errStopBit
	}

	switch connection.Parity {
	case ParityNone, ParityEven, ParityOdd:
	default:
		return errParity
	}

	return nil
}

// Save a connection to a json file.
func (connection *Connection) Save(path string) error {
	json, err := json.Marshal(connection)
	if err != nil {
		return err
	}
	err = ioutil.WriteFile(path, json, 0644)
	if err != nil {
		return err
	}

	return nil
}

func (connection *Connection) String() string {

	var parity string
	switch connection.Parity {
	case ParityNone:
		parity = "N"
	case ParityEven:
		parity = "E"
	case ParityOdd:
		parity = "O"
	}

	return fmt.Sprintf("port: %s, baud rate:%d, parameters: %d%s%d",
		connection.Port, connection.Baud, connection.DataBit, parity, connection.StopBit)
}

// Open a connection with a read timeout.
func (connection *Connection) Open(timeout time.Duration) error {

	var err error

	// The serial port is basically a file we are writing to and reading from.
	// 	O_RDWR allows the program to read and write the file.
	// 	O_NOCTTY prevents the device from controlling the terminal.
	// 	O_NONBLOCK prevents the system from blocking for a long time.
	connection.f, err = os.OpenFile(connection.Port, syscall.O_RDWR|syscall.O_NOCTTY|syscall.O_NONBLOCK, 0666)
	if err != nil {
		return err
	}

	// Close the file on error occurrence.
	defer func() {
		if err != nil && connection.f != nil {
			connection.f.Close()
		}
	}()

	// Create a plain termios, which allows the program to execute input/output operations.
	termios := syscall.Termios{}

	// Setup the baud rate in the termios structure.
	baudrate := baudrates[connection.Baud]

	termios.Cflag |= baudrate
	termios.Ispeed = baudrate
	termios.Ospeed = baudrate

	// Setup stop bits in the termios structure.
	switch connection.StopBit {
	case StopBit1:
		termios.Cflag &^= syscall.CSTOPB // CSTOPB = 0x40
	case StopBit2:
		termios.Cflag |= syscall.CSTOPB
	default:
		return errStopBit
	}

	// Setup data bits in the termios structure.
	databit := databits[connection.DataBit]
	termios.Cflag |= databit

	// Setup parity in the termios structure.
	switch connection.Parity {
	case ParityNone:
		termios.Cflag &^= syscall.PARENB // PARENB = 0x100
	case ParityEven:
		termios.Cflag |= syscall.PARENB
	case ParityOdd:
		termios.Cflag |= syscall.PARENB
		termios.Cflag |= syscall.PARODD // PARODD = 0x200
	default:
		return errParity
	}

	// Calculate timeout and attach it to the termios structure.
	vmin, vtime := calculateTimeout(timeout)
	termios.Cc = [32]uint8{
		syscall.VMIN:  vmin,  // min bytes per transfer
		syscall.VTIME: vtime, // actual read timeout in deciseconds
	}

	// Execute IOCTL with the modified termios structure to apply the changes.
	if _, _, errno := syscall.Syscall6(
		syscall.SYS_IOCTL,          // device-specific input/output operations
		uintptr(connection.f.Fd()), // open file descriptor
		uintptr(syscall.TCSETS),    // a request code number to set the current serial port settings
		//TODO: it looks like syscall.TCSETS is not available under freebsd and darwin. Is this a bug?
		uintptr(unsafe.Pointer(&termios)), // a pointer to the termios structure
		0,
		0,
		0,
	); errno != 0 {
		return errno
	}

	connection.isOpen = true
	return nil
}

// Write a byte array to an open connection.
func (connection *Connection) Write(b []byte) (int, error) {
	if connection.isOpen {
		return connection.f.Write(b)
	}

	return 0, errConnOpen
}

// Read from an open connection until the delimiter is reached.
func (connection *Connection) Read(delimiter byte) ([]byte, error) {
	if connection.isOpen {
		reader := bufio.NewReader(connection.f)
		return reader.ReadBytes(delimiter)
	}

	return nil, errConnOpen
}

// ReadToBuffer reads from an open connection into a []byte buffer with the given size.
func (connection *Connection) ReadToBuffer(size int) ([]byte, error) {
	buffer := make([]byte, size)
	//TODO do something with bytes read
	_, err := connection.f.Read(buffer)
	if err != nil {
		return nil, err
	}

	return buffer, nil
}

// Flush the connection, which causes untransmitted or not read data to be discarded.
func (connection *Connection) Flush() error {
	if connection.isOpen {
		_, _, err := syscall.Syscall(
			syscall.SYS_IOCTL,          // device-specific input/output operations
			uintptr(connection.f.Fd()), // open file descriptor
			uintptr(syscall.TCIOFLUSH), // a request code number to flush input/output
			uintptr(0),                 // a pointer to data, not needed here
		)
		return err
	}
	return errConnOpen
}

// Close a connection.
func (connection *Connection) Close() error {
	err := connection.f.Close()
	connection.isOpen = false

	return err
}

// functions

func calculateTimeout(timeout time.Duration) (uint8, uint8) {
	// calculate deciseconds from duration
	vtime := timeout.Seconds() / 10

	// Cap size of vtime (uint8 max value is 255)
	const MAXUINT8 = 1<<8 - 1 // 255
	if vtime < 1 {
		// 0.1s
		vtime = 1
	} else if vtime > MAXUINT8 {
		// 25.5s
		vtime = MAXUINT8
	}

	vmin := uint8(0)
	if timeout == 0 {
		vmin = 1
	}
	return vmin, uint8(vtime)
}

// createConnection is the entrence point for the Connection in unix-like operating systems.
func createConnection(port string, baudrate Baud, databit DataBit, stopbit StopBit, parity Parity) (*Connection, error) {
	connection := &Connection{Port: port, Baud: baudrate, DataBit: databit, StopBit: stopbit, Parity: parity}
	return connection, connection.check()
}
<commit_msg>cleaned up read timeout<commit_after>// Copyright 2015 mvbjrn. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !windows !cgo
// +build 386 amd64 arm

package serial

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"syscall"
	"unsafe"
)

// TODO flow control

var (
	baudrates = map[Baud]uint32{
		Baud4800:   syscall.B4800,
		Baud9600:   syscall.B9600,
		Baud19200:  syscall.B19200,
		Baud38400:  syscall.B38400,
		Baud57600:  syscall.B57600,
		Baud115200: syscall.B115200,
	}
	databits = map[DataBit]uint32{
		DataBit5: syscall.CS5,
		DataBit6: syscall.CS6,
		DataBit7: syscall.CS7,
		DataBit8: syscall.CS8,
	}
)

// structs and its functions

// Connection represents a serial connection with all parameters.
type Connection struct {
	Port    string
	Baud    Baud
	DataBit DataBit
	StopBit StopBit
	Parity  Parity
	f       *os.File
	isOpen  bool
}

func (connection *Connection) check() error {

	match, _ := regexp.MatchString("/dev/tty.*", connection.Port)
	if !match {
		return errPort
	}

	switch connection.Baud {
	case Baud115200, Baud57600, Baud38400, Baud19200, Baud9600, Baud4800:
	default:
		return errBaud
	}

	switch connection.DataBit {
	case DataBit5, DataBit6, DataBit7, DataBit8:
	default:
		return errDataBit
	}

	switch connection.StopBit {
	case StopBit1, StopBit2:
	default:
		return errStopBit
	}

	switch connection.Parity {
	case ParityNone, ParityEven, ParityOdd:
	default:
		return errParity
	}

	return nil
}

// Save a connection to a json file.
func (connection *Connection) Save(path string) error {
	json, err := json.Marshal(connection)
	if err != nil {
		return err
	}
	err = ioutil.WriteFile(path, json, 0644)
	if err != nil {
		return err
	}

	return nil
}

func (connection *Connection) String() string {

	var parity string
	switch connection.Parity {
	case ParityNone:
		parity = "N"
	case ParityEven:
		parity = "E"
	case ParityOdd:
		parity = "O"
	}

	return fmt.Sprintf("port: %s, baud rate:%d, parameters: %d%s%d",
		connection.Port, connection.Baud, connection.DataBit, parity, connection.StopBit)
}

// Open a connection with a read timeout.
func (connection *Connection) Open(timeout uint8) error {

	var err error

	// The serial port is basically a file we are writing to and reading from.
	// 	O_RDWR allows the program to read and write the file.
	// 	O_NOCTTY prevents the device from controlling the terminal.
	// 	O_NONBLOCK prevents the system from blocking for a long time.
	connection.f, err = os.OpenFile(connection.Port, syscall.O_RDWR|syscall.O_NOCTTY|syscall.O_NONBLOCK, 0666)
	if err != nil {
		return err
	}

	// Close the file on error occurrence.
	defer func() {
		if err != nil && connection.f != nil {
			connection.f.Close()
		}
	}()

	// Create a plain termios, which allows the program to execute input/output operations.
	termios := syscall.Termios{}

	// Setup the baud rate in the termios structure.
	baudrate := baudrates[connection.Baud]

	termios.Cflag |= baudrate
	termios.Ispeed = baudrate
	termios.Ospeed = baudrate

	// Setup stop bits in the termios structure.
	switch connection.StopBit {
	case StopBit1:
		termios.Cflag &^= syscall.CSTOPB // CSTOPB = 0x40
	case StopBit2:
		termios.Cflag |= syscall.CSTOPB
	default:
		return errStopBit
	}

	// Setup data bits in the termios structure.
	databit := databits[connection.DataBit]
	termios.Cflag |= databit

	// Setup parity in the termios structure.
	switch connection.Parity {
	case ParityNone:
		termios.Cflag &^= syscall.PARENB // PARENB = 0x100
	case ParityEven:
		termios.Cflag |= syscall.PARENB
	case ParityOdd:
		termios.Cflag |= syscall.PARENB
		termios.Cflag |= syscall.PARODD // PARODD = 0x200
	default:
		return errParity
	}

	// // set blocking / non-blocking read
	vmin := uint8(0)
	if timeout == 0 {
		vmin = 1
	}

	// Attach min bytes and timeout to the termios structure.
	termios.Cc = [32]uint8{
		syscall.VMIN:  vmin,    // min bytes per transfer
		syscall.VTIME: timeout, // actual read timeout in deciseconds
	}

	// Execute IOCTL with the modified termios structure to apply the changes.
	if _, _, errno := syscall.Syscall6(
		syscall.SYS_IOCTL,          // device-specific input/output operations
		uintptr(connection.f.Fd()), // open file descriptor
		uintptr(syscall.TCSETS),    // a request code number to set the current serial port settings
		//TODO: it looks like syscall.TCSETS is not available under freebsd and darwin. Is this a bug?
		uintptr(unsafe.Pointer(&termios)), // a pointer to the termios structure
		0,
		0,
		0,
	); errno != 0 {
		return errno
	}

	connection.isOpen = true
	return nil
}

// Write a byte array to an open connection.
func (connection *Connection) Write(b []byte) (int, error) {
	if connection.isOpen {
		return connection.f.Write(b)
	}

	return 0, errConnOpen
}

// Read from an open connection until the delimiter is reached.
func (connection *Connection) Read(delimiter byte) ([]byte, error) {
	if connection.isOpen {
		reader := bufio.NewReader(connection.f)
		return reader.ReadBytes(delimiter)
	}

	return nil, errConnOpen
}

// ReadToBuffer reads from an open connection into a []byte buffer with the given size.
func (connection *Connection) ReadToBuffer(size int) ([]byte, error) {
	buffer := make([]byte, size)
	//TODO do something with bytes read
	_, err := connection.f.Read(buffer)
	if err != nil {
		return nil, err
	}

	return buffer, nil
}

// Flush the connection, which causes untransmitted or not read data to be discarded.
func (connection *Connection) Flush() error {
	if connection.isOpen {
		_, _, err := syscall.Syscall(
			syscall.SYS_IOCTL,          // device-specific input/output operations
			uintptr(connection.f.Fd()), // open file descriptor
			uintptr(syscall.TCIOFLUSH), // a request code number to flush input/output
			uintptr(0),                 // a pointer to data, not needed here
		)
		return err
	}
	return errConnOpen
}

// Close a connection.
func (connection *Connection) Close() error {
	err := connection.f.Close()
	connection.isOpen = false

	return err
}

// functions

// createConnection is the entrence point for the Connection in unix-like operating systems.
func createConnection(port string, baudrate Baud, databit DataBit, stopbit StopBit, parity Parity) (*Connection, error) {
	connection := &Connection{Port: port, Baud: baudrate, DataBit: databit, StopBit: stopbit, Parity: parity}
	return connection, connection.check()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/garyburd/go-websocket/websocket"
	"github.com/ugorji/go/codec"
	"io/ioutil"
	"strconv"
	"time"
)

const (
	MAX_USERS_PER_LOCATION = 20
	MAX_NICKNAME_LENGTH    = 20
	SEND_CHANNEL_SIZE      = 256
	// Time allowed to write a message to the client.
	WRITE_WAIT = 5 * time.Second
	// Time allowed to read the next message from the client.
	READ_WAIT = 10 * time.Second
	// Send pings to client with this period. Must be less than READ_WAIT.
	PING_PERIOD = (READ_WAIT * 9) / 10
	// Maximum message size allowed from client.
	// must be at least 4 * MAX_CHAT_MESSAGE_LENGTH (in script.js)
	MAX_MESSAGE_SIZE = 1024
)

var (
	userIdGenerator chan int
	msgpackHandle   codec.MsgpackHandle
)

type User struct {
	Socket      *websocket.Conn
	sendData    chan<- *[]byte
	recv        <-chan *[]interface{}
	UserId      int
	Nickname    string
	MouseIsDown bool
	PositionX   int
	PositionY   int
	ColorRed    int
	ColorGreen  int
	ColorBlue   int
	UsePen      bool
	kick        chan string
}

func NewUser(ws *websocket.Conn) *User {
	user := new(User)
	user.Socket = ws
	user.UserId = <-userIdGenerator
	user.Nickname = strconv.Itoa(user.UserId)
	user.UsePen = true
	user.sendData = user.sender()
	user.recv = user.receiver()
	user.kick = make(chan string)
	return user
}

func init() {
	userIdGenerator = make(chan int)
	go func() {
		i := 1
		for {
			userIdGenerator <- i
			i += 1
		}
	}()
}

func encodeEvent(event *[]interface{}) (*[]byte, error) {
	var result []byte
	var err = codec.NewEncoderBytes(&result, &msgpackHandle).Encode(event)
	if err != nil {
		Log.Printf("Failed to encode event %v\n", event)
	}
	return &result, err
}

func (user *User) Kick(description string) {
	select {
	case user.kick <- description:
	default:
		// SocketHandler has already returned, avoid blocking
	}
}

func (user *User) Error(description string) {
	user.SendEvent(&[]interface{}{EventTypeError, description})
	user.Kick(description)
}

func (user *User) SendEvent(event *[]interface{}) {
	data, err := encodeEvent(event)
	if err != nil {
		return
	}
	user.SendData(data)
}

func (user *User) SendData(data *[]byte) {
	select {
	case user.sendData <- data:
	default:
		Log.Printf("Buffer full for user %v: kicking.\n", user.UserId)
		user.Kick("Buffer full")
	}
}

func (user *User) mouseMove(x int, y int, duration int) {
	user.PositionX = x
	user.PositionY = y
}

func (user *User) mouseUp() {
	user.MouseIsDown = false
}

func (user *User) mouseDown() {
	user.MouseIsDown = true
}

func (user *User) changeTool(use_pen bool) {
	user.UsePen = use_pen
}

func (user *User) changeColor(red, green, blue int) {
	user.ColorRed = red
	user.ColorGreen = green
	user.ColorBlue = blue
}

func (user *User) changeNickname(nickname string) {
	user.Nickname = nickname
}

func write(ws *websocket.Conn, opCode int, payload []byte) error {
	ws.SetWriteDeadline(time.Now().Add(WRITE_WAIT))
	return ws.WriteMessage(opCode, payload)
}

func (user *User) sender() chan<- *[]byte {
	ch := make(chan *[]byte, SEND_CHANNEL_SIZE)
	go func() {
		ticker := time.NewTicker(PING_PERIOD)
		defer func() {
			ticker.Stop()
		}()
		for {
			select {
			case data, ok := <-ch:
				if !ok {
					write(user.Socket, websocket.OpClose, []byte{})
					return
				}
				if err := write(user.Socket, websocket.OpBinary, *data); err != nil {
					user.Kick(err.Error())
					return
				}
			case <-ticker.C:
				if err := write(user.Socket, websocket.OpPing, []byte{}); err != nil {
					user.Kick(err.Error())
					return
				}
			}
		}
	}()
	return ch
}

func (user *User) receiver() <-chan *[]interface{} {
	// receives and decodes messages from users
	ch := make(chan *[]interface{})
	go func() {
		user.Socket.SetReadLimit(MAX_MESSAGE_SIZE)
		user.Socket.SetReadDeadline(time.Now().Add(READ_WAIT))
		for {
			op, r, err := user.Socket.NextReader()
			if err != nil {
				user.Kick(err.Error())
				break
			}
			switch op {
			case websocket.OpPong:
				user.Socket.SetReadDeadline(time.Now().Add(READ_WAIT))
			case websocket.OpBinary:
				data, err := ioutil.ReadAll(r)
				if err != nil {
					user.Kick(err.Error())
					break
				}
				var event []interface{}
				err = codec.NewDecoderBytes(data, &msgpackHandle).Decode(&event)
				if err != nil {
					user.Kick(err.Error())
					break
				}
				ch <- &event
			default:
				user.Kick(fmt.Sprintf("bad message type: %v", op))
				break
			}
		}
		close(ch)
	}()
	return ch
}

func (user *User) SocketHandler(location *Location) {
	defer func() {
		if location != nil {
			location.Quit <- user
			Log.Printf("user %v left\n", user.UserId)
		}
	}()
	for {
		select {
		case event, ok := <-user.recv:
			if !ok {
				return
			}
			if location != nil {
				location.Message <- &UserAndEvent{user, event}
			}
		case err_msg := <-user.kick:
			Log.Printf("user %v kicked for '%v'\n", user.UserId, err_msg)
			return
		}
	}
}
<commit_msg>longer read timeout<commit_after>package main

import (
	"fmt"
	"github.com/garyburd/go-websocket/websocket"
	"github.com/ugorji/go/codec"
	"io/ioutil"
	"strconv"
	"time"
)

const (
	MAX_USERS_PER_LOCATION = 20
	MAX_NICKNAME_LENGTH    = 20
	SEND_CHANNEL_SIZE      = 256
	// Time allowed to write a message to the client.
	WRITE_WAIT = 5 * time.Second
	// Time allowed to read the next message from the client.
	READ_WAIT = 15 * time.Second
	// Send pings to client with this period. Must be less than READ_WAIT.
	PING_PERIOD = (READ_WAIT * 9) / 10
	// Maximum message size allowed from client.
	// must be at least 4 * MAX_CHAT_MESSAGE_LENGTH (in script.js)
	MAX_MESSAGE_SIZE = 1024
)

var (
	userIdGenerator chan int
	msgpackHandle   codec.MsgpackHandle
)

type User struct {
	Socket      *websocket.Conn
	sendData    chan<- *[]byte
	recv        <-chan *[]interface{}
	UserId      int
	Nickname    string
	MouseIsDown bool
	PositionX   int
	PositionY   int
	ColorRed    int
	ColorGreen  int
	ColorBlue   int
	UsePen      bool
	kick        chan string
}

func NewUser(ws *websocket.Conn) *User {
	user := new(User)
	user.Socket = ws
	user.UserId = <-userIdGenerator
	user.Nickname = strconv.Itoa(user.UserId)
	user.UsePen = true
	user.sendData = user.sender()
	user.recv = user.receiver()
	user.kick = make(chan string)
	return user
}

func init() {
	userIdGenerator = make(chan int)
	go func() {
		i := 1
		for {
			userIdGenerator <- i
			i += 1
		}
	}()
}

func encodeEvent(event *[]interface{}) (*[]byte, error) {
	var result []byte
	var err = codec.NewEncoderBytes(&result, &msgpackHandle).Encode(event)
	if err != nil {
		Log.Printf("Failed to encode event %v\n", event)
	}
	return &result, err
}

func (user *User) Kick(description string) {
	select {
	case user.kick <- description:
	default:
		// SocketHandler has already returned, avoid blocking
	}
}

func (user *User) Error(description string) {
	user.SendEvent(&[]interface{}{EventTypeError, description})
	user.Kick(description)
}

func (user *User) SendEvent(event *[]interface{}) {
	data, err := encodeEvent(event)
	if err != nil {
		return
	}
	user.SendData(data)
}

func (user *User) SendData(data *[]byte) {
	select {
	case user.sendData <- data:
	default:
		Log.Printf("Buffer full for user %v: kicking.\n", user.UserId)
		user.Kick("Buffer full")
	}
}

func (user *User) mouseMove(x int, y int, duration int) {
	user.PositionX = x
	user.PositionY = y
}

func (user *User) mouseUp() {
	user.MouseIsDown = false
}

func (user *User) mouseDown() {
	user.MouseIsDown = true
}

func (user *User) changeTool(use_pen bool) {
	user.UsePen = use_pen
}

func (user *User) changeColor(red, green, blue int) {
	user.ColorRed = red
	user.ColorGreen = green
	user.ColorBlue = blue
}

func (user *User) changeNickname(nickname string) {
	user.Nickname = nickname
}

func write(ws *websocket.Conn, opCode int, payload []byte) error {
	ws.SetWriteDeadline(time.Now().Add(WRITE_WAIT))
	return ws.WriteMessage(opCode, payload)
}

func (user *User) sender() chan<- *[]byte {
	ch := make(chan *[]byte, SEND_CHANNEL_SIZE)
	go func() {
		ticker := time.NewTicker(PING_PERIOD)
		defer func() {
			ticker.Stop()
		}()
		for {
			select {
			case data, ok := <-ch:
				if !ok {
					write(user.Socket, websocket.OpClose, []byte{})
					return
				}
				if err := write(user.Socket, websocket.OpBinary, *data); err != nil {
					user.Kick(err.Error())
					return
				}
			case <-ticker.C:
				if err := write(user.Socket, websocket.OpPing, []byte{}); err != nil {
					user.Kick(err.Error())
					return
				}
			}
		}
	}()
	return ch
}

func (user *User) receiver() <-chan *[]interface{} {
	// receives and decodes messages from users
	ch := make(chan *[]interface{})
	go func() {
		user.Socket.SetReadLimit(MAX_MESSAGE_SIZE)
		user.Socket.SetReadDeadline(time.Now().Add(READ_WAIT))
		for {
			op, r, err := user.Socket.NextReader()
			if err != nil {
				user.Kick(err.Error())
				break
			}
			switch op {
			case websocket.OpPong:
				user.Socket.SetReadDeadline(time.Now().Add(READ_WAIT))
			case websocket.OpBinary:
				data, err := ioutil.ReadAll(r)
				if err != nil {
					user.Kick(err.Error())
					break
				}
				var event []interface{}
				err = codec.NewDecoderBytes(data, &msgpackHandle).Decode(&event)
				if err != nil {
					user.Kick(err.Error())
					break
				}
				ch <- &event
			default:
				user.Kick(fmt.Sprintf("bad message type: %v", op))
				break
			}
		}
		close(ch)
	}()
	return ch
}

func (user *User) SocketHandler(location *Location) {
	defer func() {
		if location != nil {
			location.Quit <- user
			Log.Printf("user %v left\n", user.UserId)
		}
	}()
	for {
		select {
		case event, ok := <-user.recv:
			if !ok {
				return
			}
			if location != nil {
				location.Message <- &UserAndEvent{user, event}
			}
		case err_msg := <-user.kick:
			Log.Printf("user %v kicked for '%v'\n", user.UserId, err_msg)
			return
		}
	}
}
<|endoftext|>


<commit_before>package irckit

import (
	"fmt"
	"net"
	"strings"
	"sync"

	"github.com/sorcix/irc"
)

// NewUser creates a *User, wrapping a connection with metadata we need for our server.
func NewUser(c Conn) *User {
	return &User{
		Conn:     c,
		Host:     "*",
		channels: map[Channel]struct{}{},
	}
}

// NewUserNet creates a *User from a net.Conn connection.
func NewUserNet(c net.Conn) *User {
	return NewUser(&conn{
		Conn:    c,
		Encoder: irc.NewEncoder(c),
		Decoder: irc.NewDecoder(c),
	})
}

const defaultCloseMsg = "Closed."

type User struct {
	Conn

	sync.RWMutex
	Nick        string // From NICK command
	User        string // From USER command
	Real        string // From USER command
	Pass        []string // From PASS command
	Host        string
	Roles       string
	DisplayName string

	channels map[Channel]struct{}

	MmInfo
	SlackInfo
}

func (u *User) ID() string {
	return strings.ToLower(u.Nick)
}

func (u *User) Prefix() *irc.Prefix {
	return &irc.Prefix{
		Name: u.Nick,
		User: u.User,
		Host: u.Host,
	}
}

func (u *User) Close() error {
	for ch := range u.channels {
		ch.Part(u, defaultCloseMsg)
	}
	return u.Conn.Close()
}

func (u *User) String() string {
	return u.Prefix().String()
}

func (u *User) NumChannels() int {
	u.RLock()
	defer u.RUnlock()
	return len(u.channels)
}

func (u *User) Channels() []Channel {
	u.RLock()
	channels := make([]Channel, 0, len(u.channels))
	for ch := range u.channels {
		channels = append(channels, ch)
	}
	u.RUnlock()
	return channels
}

func (u *User) VisibleTo() []*User {
	seen := map[*User]struct{}{}
	seen[u] = struct{}{}

	channels := u.Channels()
	num := 0
	for _, ch := range channels {
		// Don't include self
		num += ch.Len()
	}

	// Pre-allocate
	users := make([]*User, 0, num)
	if num == 0 {
		return users
	}

	// Get all unique users
	for _, ch := range channels {
		for _, other := range ch.Users() {
			if _, dupe := seen[other]; dupe {
				continue
			}
			seen[other] = struct{}{}
			// TODO: Check visibility (once it's implemented)
			users = append(users, other)
		}
	}
	return users
}

// Encode and send each msg until an error occurs, then returns.
func (user *User) Encode(msgs ...*irc.Message) (err error) {
	if user.MmGhostUser {
		return nil
	}
	for _, msg := range msgs {
		logger.Debugf("-> %s", msg)
		err := user.Conn.Encode(msg)
		if err != nil {
			return err
		}
	}
	return nil
}

// Decode will receive and return a decoded message, or an error.
func (user *User) Decode() (*irc.Message, error) {
	if user.MmGhostUser {
		// block
		c := make(chan struct{})
		<-c
	}
	msg, err := user.Conn.Decode()
	if err == nil && msg != nil {
		dmsg := fmt.Sprintf("<- %s", msg)
		if msg.Command == "PRIVMSG" && msg.Params != nil && (msg.Params[0] == "slack" || msg.Params[0] == "mattermost") {
			// Don't log sensitive information
			trail := strings.Split(msg.Trailing, " ")
			if (msg.Trailing != "" && trail[0] == "login") || (len(msg.Params) > 1 && msg.Params[1] == "login") {
				dmsg = fmt.Sprintf("<- PRIVMSG %s :login [redacted]", msg.Params[0])
			}
		}
		logger.Debug(dmsg)
	}
	return msg, err
}
<commit_msg>Redact slack token from debug log<commit_after>package irckit

import (
	"fmt"
	"net"
	"strings"
	"sync"

	"github.com/sorcix/irc"
)

// NewUser creates a *User, wrapping a connection with metadata we need for our server.
func NewUser(c Conn) *User {
	return &User{
		Conn:     c,
		Host:     "*",
		channels: map[Channel]struct{}{},
	}
}

// NewUserNet creates a *User from a net.Conn connection.
func NewUserNet(c net.Conn) *User {
	return NewUser(&conn{
		Conn:    c,
		Encoder: irc.NewEncoder(c),
		Decoder: irc.NewDecoder(c),
	})
}

const defaultCloseMsg = "Closed."

type User struct {
	Conn

	sync.RWMutex
	Nick        string   // From NICK command
	User        string   // From USER command
	Real        string   // From USER command
	Pass        []string // From PASS command
	Host        string
	Roles       string
	DisplayName string

	channels map[Channel]struct{}

	MmInfo
	SlackInfo
}

func (u *User) ID() string {
	return strings.ToLower(u.Nick)
}

func (u *User) Prefix() *irc.Prefix {
	return &irc.Prefix{
		Name: u.Nick,
		User: u.User,
		Host: u.Host,
	}
}

func (u *User) Close() error {
	for ch := range u.channels {
		ch.Part(u, defaultCloseMsg)
	}
	return u.Conn.Close()
}

func (u *User) String() string {
	return u.Prefix().String()
}

func (u *User) NumChannels() int {
	u.RLock()
	defer u.RUnlock()
	return len(u.channels)
}

func (u *User) Channels() []Channel {
	u.RLock()
	channels := make([]Channel, 0, len(u.channels))
	for ch := range u.channels {
		channels = append(channels, ch)
	}
	u.RUnlock()
	return channels
}

func (u *User) VisibleTo() []*User {
	seen := map[*User]struct{}{}
	seen[u] = struct{}{}

	channels := u.Channels()
	num := 0
	for _, ch := range channels {
		// Don't include self
		num += ch.Len()
	}

	// Pre-allocate
	users := make([]*User, 0, num)
	if num == 0 {
		return users
	}

	// Get all unique users
	for _, ch := range channels {
		for _, other := range ch.Users() {
			if _, dupe := seen[other]; dupe {
				continue
			}
			seen[other] = struct{}{}
			// TODO: Check visibility (once it's implemented)
			users = append(users, other)
		}
	}
	return users
}

// Encode and send each msg until an error occurs, then returns.
func (user *User) Encode(msgs ...*irc.Message) (err error) {
	if user.MmGhostUser {
		return nil
	}
	for _, msg := range msgs {
		if msg.Command == "PRIVMSG" && (msg.Prefix.Name == "slack" || msg.Prefix.Name == "mattermost") && msg.Prefix.Host == "service" && strings.Contains(msg.Trailing, "token") {
			logger.Debugf("-> %s %s %s", msg.Command, msg.Prefix.Name, "[token redacted]")
			err := user.Conn.Encode(msg)
			if err != nil {
				return err
			}
			continue
		}
		logger.Debugf("-> %s", msg)
		err := user.Conn.Encode(msg)
		if err != nil {
			return err
		}
	}
	return nil
}

// Decode will receive and return a decoded message, or an error.
func (user *User) Decode() (*irc.Message, error) {
	if user.MmGhostUser {
		// block
		c := make(chan struct{})
		<-c
	}
	msg, err := user.Conn.Decode()
	if err == nil && msg != nil {
		dmsg := fmt.Sprintf("<- %s", msg)
		if msg.Command == "PRIVMSG" && msg.Params != nil && (msg.Params[0] == "slack" || msg.Params[0] == "mattermost") {
			// Don't log sensitive information
			trail := strings.Split(msg.Trailing, " ")
			if (msg.Trailing != "" && trail[0] == "login") || (len(msg.Params) > 1 && msg.Params[1] == "login") {
				dmsg = fmt.Sprintf("<- PRIVMSG %s :login [redacted]", msg.Params[0])
			}
		}
		logger.Debug(dmsg)
	}
	return msg, err
}
<|endoftext|>