<commit_before>// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

//go:build !gogit
// +build !gogit

package pipeline

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"sort"
	"strings"
	"sync"
	"time"

	"code.gitea.io/gitea/modules/git"
)

// LFSResult represents commits found using a provided pointer file hash
type LFSResult struct {
	Name           string
	SHA            string
	Summary        string
	When           time.Time
	ParentHashes   []git.SHA1
	BranchName     string
	FullCommitName string
}

type lfsResultSlice []*LFSResult

func (a lfsResultSlice) Len() int           { return len(a) }
func (a lfsResultSlice) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a lfsResultSlice) Less(i, j int) bool { return a[j].When.After(a[i].When) }

// FindLFSFile finds commits that contain a provided pointer file hash
func FindLFSFile(repo *git.Repository, hash git.SHA1) ([]*LFSResult, error) {
	resultsMap := map[string]*LFSResult{}
	results := make([]*LFSResult, 0)

	basePath := repo.Path

	// Use rev-list to provide us with all commits in order
	revListReader, revListWriter := io.Pipe()
	defer func() {
		_ = revListWriter.Close()
		_ = revListReader.Close()
	}()

	go func() {
		stderr := strings.Builder{}
		err := git.NewCommand("rev-list", "--all").RunInDirPipeline(repo.Path, revListWriter, &stderr)
		if err != nil {
			_ = revListWriter.CloseWithError(git.ConcatenateError(err, (&stderr).String()))
		} else {
			_ = revListWriter.Close()
		}
	}()

	// Next feed the commits in order into cat-file --batch, followed by their trees and sub trees as necessary.
	// so let's create a batch stdin and stdout
	batchStdinWriter, batchReader, cancel := repo.CatFileBatch()
	defer cancel()

	// We'll use a scanner for the revList because it's simpler than a bufio.Reader
	scan := bufio.NewScanner(revListReader)
	trees := [][]byte{}
	paths := []string{}

	fnameBuf := make([]byte, 4096)
	modeBuf := make([]byte, 40)
	workingShaBuf := make([]byte, 20)

	for scan.Scan() {
		// Get the next commit ID
		commitID := scan.Bytes()

		// push the commit to the cat-file --batch process
		_, err := batchStdinWriter.Write(commitID)
		if err != nil {
			return nil, err
		}
		_, err = batchStdinWriter.Write([]byte{'\n'})
		if err != nil {
			return nil, err
		}

		var curCommit *git.Commit
		curPath := ""

	commitReadingLoop:
		for {
			_, typ, size, err := git.ReadBatchLine(batchReader)
			if err != nil {
				return nil, err
			}

			switch typ {
			case "tag":
				// This shouldn't happen but if it does well just get the commit and try again
				id, err := git.ReadTagObjectID(batchReader, size)
				if err != nil {
					return nil, err
				}
				_, err = batchStdinWriter.Write([]byte(id + "\n"))
				if err != nil {
					return nil, err
				}
				continue
			case "commit":
				// Read in the commit to get its tree and in case this is one of the last used commits
				curCommit, err = git.CommitFromReader(repo, git.MustIDFromString(string(commitID)), io.LimitReader(batchReader, int64(size)))
				if err != nil {
					return nil, err
				}
				if _, err := batchReader.Discard(1); err != nil {
					return nil, err
				}

				_, err := batchStdinWriter.Write([]byte(curCommit.Tree.ID.String() + "\n"))
				if err != nil {
					return nil, err
				}
				curPath = ""
			case "tree":
				var n int64
				for n < size {
					mode, fname, sha20byte, count, err := git.ParseTreeLine(batchReader, modeBuf, fnameBuf, workingShaBuf)
					if err != nil {
						return nil, err
					}
					n += int64(count)
					if bytes.Equal(sha20byte, hash[:]) {
						result := LFSResult{
							Name:         curPath + string(fname),
							SHA:          curCommit.ID.String(),
							Summary:      strings.Split(strings.TrimSpace(curCommit.CommitMessage), "\n")[0],
							When:         curCommit.Author.When,
							ParentHashes: curCommit.Parents,
						}
						resultsMap[curCommit.ID.String()+":"+curPath+string(fname)] = &result
					} else if string(mode) == git.EntryModeTree.String() {
						sha40Byte := make([]byte, 40)
						git.To40ByteSHA(sha20byte, sha40Byte)
						trees = append(trees, sha40Byte)
						paths = append(paths, curPath+string(fname)+"/")
					}
				}
				if _, err := batchReader.Discard(1); err != nil {
					return nil, err
				}
				if len(trees) > 0 {
					_, err := batchStdinWriter.Write(trees[len(trees)-1])
					if err != nil {
						return nil, err
					}
					_, err = batchStdinWriter.Write([]byte("\n"))
					if err != nil {
						return nil, err
					}
					curPath = paths[len(paths)-1]
					trees = trees[:len(trees)-1]
					paths = paths[:len(paths)-1]
				} else {
					break commitReadingLoop
				}
			}
		}
	}

	if err := scan.Err(); err != nil {
		return nil, err
	}

	for _, result := range resultsMap {
		hasParent := false
		for _, parentHash := range result.ParentHashes {
			if _, hasParent = resultsMap[parentHash.String()+":"+result.Name]; hasParent {
				break
			}
		}
		if !hasParent {
			results = append(results, result)
		}
	}

	sort.Sort(lfsResultSlice(results))

	// Should really use a go-git function here but name-rev is not completed and recapitulating it is not simple
	shasToNameReader, shasToNameWriter := io.Pipe()
	nameRevStdinReader, nameRevStdinWriter := io.Pipe()
	errChan := make(chan error, 1)
	wg := sync.WaitGroup{}
	wg.Add(3)

	go func() {
		defer wg.Done()
		scanner := bufio.NewScanner(nameRevStdinReader)
		i := 0
		for scanner.Scan() {
			line := scanner.Text()
			if len(line) == 0 {
				continue
			}
			result := results[i]
			result.FullCommitName = line
			result.BranchName = strings.Split(line, "~")[0]
			i++
		}
	}()
	go NameRevStdin(shasToNameReader, nameRevStdinWriter, &wg, basePath)
	go func() {
		defer wg.Done()
		defer shasToNameWriter.Close()
		for _, result := range results {
			i := 0
			if i < len(result.SHA) {
				n, err := shasToNameWriter.Write([]byte(result.SHA)[i:])
				if err != nil {
					errChan <- err
					break
				}
				i += n
			}
			var err error
			n := 0
			for n < 1 {
				n, err = shasToNameWriter.Write([]byte{'\n'})
				if err != nil {
					errChan <- err
					break
				}

			}

		}
	}()

	wg.Wait()

	select {
	case err, has := <-errChan:
		if has {
			return nil, fmt.Errorf("Unable to obtain name for LFS files. Error: %w", err)
		}
	default:
	}

	return results, nil
}
<commit_msg>Simplify code for wrting SHA to name-rev (#17696)<commit_after>// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

//go:build !gogit
// +build !gogit

package pipeline

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"sort"
	"strings"
	"sync"
	"time"

	"code.gitea.io/gitea/modules/git"
)

// LFSResult represents commits found using a provided pointer file hash
type LFSResult struct {
	Name           string
	SHA            string
	Summary        string
	When           time.Time
	ParentHashes   []git.SHA1
	BranchName     string
	FullCommitName string
}

type lfsResultSlice []*LFSResult

func (a lfsResultSlice) Len() int           { return len(a) }
func (a lfsResultSlice) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a lfsResultSlice) Less(i, j int) bool { return a[j].When.After(a[i].When) }

// FindLFSFile finds commits that contain a provided pointer file hash
func FindLFSFile(repo *git.Repository, hash git.SHA1) ([]*LFSResult, error) {
	resultsMap := map[string]*LFSResult{}
	results := make([]*LFSResult, 0)

	basePath := repo.Path

	// Use rev-list to provide us with all commits in order
	revListReader, revListWriter := io.Pipe()
	defer func() {
		_ = revListWriter.Close()
		_ = revListReader.Close()
	}()

	go func() {
		stderr := strings.Builder{}
		err := git.NewCommand("rev-list", "--all").RunInDirPipeline(repo.Path, revListWriter, &stderr)
		if err != nil {
			_ = revListWriter.CloseWithError(git.ConcatenateError(err, (&stderr).String()))
		} else {
			_ = revListWriter.Close()
		}
	}()

	// Next feed the commits in order into cat-file --batch, followed by their trees and sub trees as necessary.
	// so let's create a batch stdin and stdout
	batchStdinWriter, batchReader, cancel := repo.CatFileBatch()
	defer cancel()

	// We'll use a scanner for the revList because it's simpler than a bufio.Reader
	scan := bufio.NewScanner(revListReader)
	trees := [][]byte{}
	paths := []string{}

	fnameBuf := make([]byte, 4096)
	modeBuf := make([]byte, 40)
	workingShaBuf := make([]byte, 20)

	for scan.Scan() {
		// Get the next commit ID
		commitID := scan.Bytes()

		// push the commit to the cat-file --batch process
		_, err := batchStdinWriter.Write(commitID)
		if err != nil {
			return nil, err
		}
		_, err = batchStdinWriter.Write([]byte{'\n'})
		if err != nil {
			return nil, err
		}

		var curCommit *git.Commit
		curPath := ""

	commitReadingLoop:
		for {
			_, typ, size, err := git.ReadBatchLine(batchReader)
			if err != nil {
				return nil, err
			}

			switch typ {
			case "tag":
				// This shouldn't happen but if it does well just get the commit and try again
				id, err := git.ReadTagObjectID(batchReader, size)
				if err != nil {
					return nil, err
				}
				_, err = batchStdinWriter.Write([]byte(id + "\n"))
				if err != nil {
					return nil, err
				}
				continue
			case "commit":
				// Read in the commit to get its tree and in case this is one of the last used commits
				curCommit, err = git.CommitFromReader(repo, git.MustIDFromString(string(commitID)), io.LimitReader(batchReader, int64(size)))
				if err != nil {
					return nil, err
				}
				if _, err := batchReader.Discard(1); err != nil {
					return nil, err
				}

				_, err := batchStdinWriter.Write([]byte(curCommit.Tree.ID.String() + "\n"))
				if err != nil {
					return nil, err
				}
				curPath = ""
			case "tree":
				var n int64
				for n < size {
					mode, fname, sha20byte, count, err := git.ParseTreeLine(batchReader, modeBuf, fnameBuf, workingShaBuf)
					if err != nil {
						return nil, err
					}
					n += int64(count)
					if bytes.Equal(sha20byte, hash[:]) {
						result := LFSResult{
							Name:         curPath + string(fname),
							SHA:          curCommit.ID.String(),
							Summary:      strings.Split(strings.TrimSpace(curCommit.CommitMessage), "\n")[0],
							When:         curCommit.Author.When,
							ParentHashes: curCommit.Parents,
						}
						resultsMap[curCommit.ID.String()+":"+curPath+string(fname)] = &result
					} else if string(mode) == git.EntryModeTree.String() {
						sha40Byte := make([]byte, 40)
						git.To40ByteSHA(sha20byte, sha40Byte)
						trees = append(trees, sha40Byte)
						paths = append(paths, curPath+string(fname)+"/")
					}
				}
				if _, err := batchReader.Discard(1); err != nil {
					return nil, err
				}
				if len(trees) > 0 {
					_, err := batchStdinWriter.Write(trees[len(trees)-1])
					if err != nil {
						return nil, err
					}
					_, err = batchStdinWriter.Write([]byte("\n"))
					if err != nil {
						return nil, err
					}
					curPath = paths[len(paths)-1]
					trees = trees[:len(trees)-1]
					paths = paths[:len(paths)-1]
				} else {
					break commitReadingLoop
				}
			}
		}
	}

	if err := scan.Err(); err != nil {
		return nil, err
	}

	for _, result := range resultsMap {
		hasParent := false
		for _, parentHash := range result.ParentHashes {
			if _, hasParent = resultsMap[parentHash.String()+":"+result.Name]; hasParent {
				break
			}
		}
		if !hasParent {
			results = append(results, result)
		}
	}

	sort.Sort(lfsResultSlice(results))

	// Should really use a go-git function here but name-rev is not completed and recapitulating it is not simple
	shasToNameReader, shasToNameWriter := io.Pipe()
	nameRevStdinReader, nameRevStdinWriter := io.Pipe()
	errChan := make(chan error, 1)
	wg := sync.WaitGroup{}
	wg.Add(3)

	go func() {
		defer wg.Done()
		scanner := bufio.NewScanner(nameRevStdinReader)
		i := 0
		for scanner.Scan() {
			line := scanner.Text()
			if len(line) == 0 {
				continue
			}
			result := results[i]
			result.FullCommitName = line
			result.BranchName = strings.Split(line, "~")[0]
			i++
		}
	}()
	go NameRevStdin(shasToNameReader, nameRevStdinWriter, &wg, basePath)
	go func() {
		defer wg.Done()
		defer shasToNameWriter.Close()
		for _, result := range results {
			_, err := shasToNameWriter.Write([]byte(result.SHA))
			if err != nil {
				errChan <- err
				break
			}
			_, err = shasToNameWriter.Write([]byte{'\n'})
			if err != nil {
				errChan <- err
				break
			}

		}
	}()

	wg.Wait()

	select {
	case err, has := <-errChan:
		if has {
			return nil, fmt.Errorf("Unable to obtain name for LFS files. Error: %w", err)
		}
	default:
	}

	return results, nil
}
<|endoftext|>


<commit_before>// +build tools

/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package hack

// Add tools that hack scripts depend on here, to ensure they are vendored.
import (
	_ "github.com/bazelbuild/bazel-gazelle/cmd/gazelle"
	_ "github.com/client9/misspell/cmd/misspell"
	_ "k8s.io/code-generator/cmd/client-gen"
	_ "k8s.io/code-generator/cmd/deepcopy-gen"
	_ "k8s.io/code-generator/cmd/informer-gen"
	_ "k8s.io/code-generator/cmd/lister-gen"
	_ "k8s.io/repo-infra/cmd/kazel"
)
<commit_msg>Remove tools that repo-infra now handles<commit_after>// +build tools

/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package hack

// Add tools that hack scripts depend on here, to ensure they are vendored.
import (
	_ "github.com/client9/misspell/cmd/misspell"
	_ "k8s.io/code-generator/cmd/client-gen"
	_ "k8s.io/code-generator/cmd/deepcopy-gen"
	_ "k8s.io/code-generator/cmd/informer-gen"
	_ "k8s.io/code-generator/cmd/lister-gen"
)
<|endoftext|>


<commit_before>package LevenshteinTrie

import (
	"fmt"
	"sort"
	"unicode/utf8"
)

func Min(a ...int) int {
	min := int(^uint(0) >> 1) // largest int
	for _, i := range a {
		if i < min {
			min = i
		}
	}
	return min
}
func Max(a ...int) int {
	max := int(0)
	for _, i := range a {
		if i > max {
			max = i
		}
	}
	return max
}

type TrieNode struct {
	letter   rune //Equivalent to int32
	children map[rune]*TrieNode
	final    bool
	text     string
}

func NewTrie() *TrieNode {
	return &TrieNode{children: make(map[rune]*TrieNode)}
}

func (root *TrieNode) InsertText(text string) {

	if root == nil {
		return
	}

	currNode := root //Starts at root
	for i, w := 0, 0; i < len(text); i += w {
		runeValue, width := utf8.DecodeRuneInString(text[i:])
		final := false
		if width+i == len(text) {
			final = true
		}
		w = width

		currNode = NewTrieNode(currNode, runeValue, final, text)
	}
}

func NewTrieNode(t *TrieNode, runeValue rune, final bool, text string) *TrieNode {
	node, exists := t.children[runeValue]
	if exists {
		if final {
			node.final = true
			node.text = text
		}
		return node
	} else {
		node = &TrieNode{letter: runeValue, children: make(map[rune]*TrieNode)}
		t.children[runeValue] = node
		return node
	}
	return nil
}

func (t *TrieNode) SearchSuffix(query string) []string {

	var curr *TrieNode
	var ok bool
	//first, find the end of the prefix
	for _, letter := range query {
		if curr != nil {
			if curr, ok = curr.children[letter]; ok {
				//do nothing
			}

		}
	}

	candidates := make([]string, 0)

	var getAllSuffixes func(n *TrieNode)
	getAllSuffixes = func(n *TrieNode) {
		if n == nil {
			return
		}
		if n.final == true {
			candidates = append(candidates, n.text)
		}

		for _, childNode := range n.children {
			getAllSuffixes(childNode)
		}

	}
	getAllSuffixes(curr)

	return candidates
}

type QueryResult struct {
	Val      string
	Distance int
}

func (q QueryResult) String() string {
	return fmt.Sprintf("Val: %s\n", q.Val)
}

type ByDistance []QueryResult

func (a ByDistance) Len() int           { return len(a) }
func (a ByDistance) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByDistance) Less(i, j int) bool { return a[i].Distance < a[j].Distance }

func (n *TrieNode) SearchLevenshtein(text string, distance int) []QueryResult {

	//initialize the first row for the dynamic programming alg
	l := utf8.RuneCount([]byte(text))
	currentRow := make([]int, l+1)

	for i := 0; i < len(currentRow); i++ {
		currentRow[i] = i
	}

	candidates := make([]QueryResult, 0)

	var searchRecursive func(n *TrieNode, prevRow []int, letter rune, text []rune, maxDistance int)
	searchRecursive = func(n *TrieNode, prevRow []int, letter rune, text []rune, maxDistance int) {
		columns := len(text) + 1
		currentRow := make([]int, columns)

		currentRow[0] = prevRow[0] + 1

		for col := 1; col < columns; col++ {
			insertCost := currentRow[col-1] + 1
			deleteCost := currentRow[col] + 1
			var replaceCost int
			if text[col-1] != letter {
				if text[col-1] != letter {
					replaceCost = prevRow[col-1] + 1
				} else {
					replaceCost = prevRow[col-1]
				}
			}
			currentRow[col] = Min(insertCost, deleteCost, replaceCost)
		}

		distance := currentRow[len(currentRow)-1]
		if distance <= maxDistance && len(n.text) > 0 {
			candidates = append(candidates, QueryResult{Val: n.text, Distance: distance})
		}

		if Min(currentRow...) <= maxDistance {
			for letter, childNode := range n.children {
				searchRecursive(childNode, currentRow, letter, []rune(text), maxDistance)
			}
		}
	}

	for letter, childNode := range n.children {
		searchRecursive(childNode, currentRow, letter, []rune(text), distance)
	}
	sort.Sort(ByDistance(candidates))
	return candidates
}
<commit_msg>fixed issues with recursion. search works well now.<commit_after>package LevenshteinTrie

import (
	"fmt"
	"sort"
	"unicode/utf8"
)

func Min(a ...int) int {
	min := int(^uint(0) >> 1) // largest int
	for _, i := range a {
		if i < min {
			min = i
		}
	}
	return min
}
func Max(a ...int) int {
	max := int(0)
	for _, i := range a {
		if i > max {
			max = i
		}
	}
	return max
}

type TrieNode struct {
	letter   rune //Equivalent to int32
	children map[rune]*TrieNode
	final    bool
	text     string
}

func (t *TrieNode) String() string {
	s := fmt.Sprintf("%s\n", t.letter)
	for _, v := range t.children {
		s += fmt.Sprintf("-%s\n", v)
	}
	return s
}

func NewTrie() *TrieNode {
	return &TrieNode{children: make(map[rune]*TrieNode)}
}

func (root *TrieNode) InsertText(text string) {
	if root == nil {
		return
	}

	currNode := root //Starts at root
	for i, w := 0, 0; i < len(text); i += w {
		runeValue, width := utf8.DecodeRuneInString(text[i:])
		final := false
		if width+i == len(text) {
			final = true
		}
		w = width

		currNode = NewTrieNode(currNode, runeValue, final, text)
	}
}

func NewTrieNode(t *TrieNode, runeValue rune, final bool, text string) *TrieNode {
	node, exists := t.children[runeValue]
	if !exists {
		node = &TrieNode{letter: runeValue, children: make(map[rune]*TrieNode)}
		t.children[runeValue] = node
	}
	if final {
		node.final = true
		node.text = text
	}
	return node
}

func (t *TrieNode) SearchSuffix(query string) []string {
	var curr *TrieNode
	var ok bool

	curr = t
	//first, find the end of the prefix
	for _, letter := range query {
		if curr != nil {
			curr, ok = curr.children[letter]
			if ok {
				//do nothing
			}

		} else {
			return nil
		}
	}

	candidates := getsuffixr(curr)

	return candidates
}

func getsuffixr(n *TrieNode) []string {
	if n == nil {
		return nil
	}

	candidates := make([]string, 0)
	if n.final == true {
		candidates = append(candidates, n.text)
	}

	for _, childNode := range n.children {
		candidates = append(candidates, getsuffixr(childNode)...)
	}
	return candidates
}

type QueryResult struct {
	Val      string
	Distance int
}

func (q QueryResult) String() string {
	return fmt.Sprintf("Val: %s\n", q.Val)
}

type ByDistance []QueryResult

func (a ByDistance) Len() int           { return len(a) }
func (a ByDistance) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByDistance) Less(i, j int) bool { return a[i].Distance < a[j].Distance }

func (n *TrieNode) SearchLevenshtein(text string, distance int) []QueryResult {

	//initialize the first row for the dynamic programming alg
	l := utf8.RuneCount([]byte(text))
	currentRow := make([]int, l+1)

	for i := 0; i < len(currentRow); i++ {
		currentRow[i] = i
	}

	candidates := make([]QueryResult, 0)

	for letter, childNode := range n.children {
		candidates = append(candidates, searchlevr(childNode, currentRow, letter, []rune(text), distance)...)
	}

	sort.Sort(ByDistance(candidates))
	return candidates
}

func searchlevr(n *TrieNode, prevRow []int, letter rune, text []rune, maxDistance int) []QueryResult {
	columns := len(prevRow)
	currentRow := make([]int, columns)

	currentRow[0] = prevRow[0] + 1

	for col := 1; col < columns; col++ {
		if text[col-1] == letter {
			currentRow[col] = prevRow[col-1]
			continue
		}
		insertCost := currentRow[col-1] + 1
		deleteCost := prevRow[col] + 1
		replaceCost := prevRow[col-1] + 1

		currentRow[col] = Min(insertCost, deleteCost, replaceCost)
	}

	candidates := make([]QueryResult, 0)

	distance := currentRow[len(currentRow)-1]
	if distance <= maxDistance && n.final == true {
		candidates = append(candidates, QueryResult{Val: n.text, Distance: distance})
	}
	mi := Min(currentRow[1:]...)
	if mi <= maxDistance {
		for l, childNode := range n.children {
			candidates = append(candidates, searchlevr(childNode, currentRow, l, text, maxDistance)...)
		}
	}
	return candidates
}
<|endoftext|>


<commit_before>package handler

import (
	"encoding/json"
	"log"
	"net"
	"net/http"
	"sync"
	"time"

	"github.com/ghophp/buildbot-dashing/container"
	"github.com/gorilla/websocket"
)

type (
	WsHandler struct {
		c *container.ContainerBag
	}

	ClientConn struct {
		websocket *websocket.Conn
		clientIP  net.Addr
	}
)

var (
	ActiveClients = make(map[ClientConn]int)
	wsMutex       sync.RWMutex
)

func addClient(cc ClientConn) {
	wsMutex.Lock()
	ActiveClients[cc] = 0
	wsMutex.Unlock()
}

func deleteClient(cc ClientConn) {
	wsMutex.Lock()
	delete(ActiveClients, cc)
	wsMutex.Unlock()
}

func broadcastMessage(messageType int, message []byte) {
	for client, _ := range ActiveClients {
		if err := client.websocket.WriteMessage(messageType, message); err != nil {
			deleteClient(client)
		}
	}
}

func MonitorBuilders(c *container.ContainerBag) {
	for {
		if len(ActiveClients) > 0 {
			builders, err := GetBuilders(c)
			if err != nil || len(builders) <= 0 {
				return
			}

			for id, builder := range builders {
				if len(builder.CachedBuilds) > 0 {
					b, err := GetBuilder(c, id, builder)
					if err == nil {
						if r, err := json.Marshal(b); err == nil {
							broadcastMessage(websocket.TextMessage, r)
						}
					}
				}
			}
		}

		time.Sleep(time.Second * time.Duration(c.RefreshSec))
	}
}

func NewWsHandler(c *container.ContainerBag) *WsHandler {
	go MonitorBuilders(c)

	return &WsHandler{
		c: c,
	}
}

func (h WsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	ws, err := websocket.Upgrade(w, r, nil, 1024, 1024)
	if _, ok := err.(websocket.HandshakeError); ok {
		http.Error(w, "Not a websocket handshake", 400)
		return
	} else if err != nil {
		log.Println(err)
		return
	}

	sockCli := ClientConn{ws, ws.RemoteAddr()}
	addClient(sockCli)
}
<commit_msg>improvement on the performance, spanning the goroutines all at once to fetch the builders<commit_after>package handler

import (
	"encoding/json"
	"log"
	"net"
	"net/http"
	"sync"
	"time"

	"github.com/ghophp/buildbot-dashing/container"
	"github.com/gorilla/websocket"
)

type (
	WsHandler struct {
		c *container.ContainerBag
	}

	ClientConn struct {
		websocket *websocket.Conn
		clientIP  net.Addr
	}
)

var (
	ActiveClients = make(map[ClientConn]int)
	wsMutex       sync.RWMutex
)

func addClient(cc ClientConn) {
	wsMutex.Lock()
	ActiveClients[cc] = 0
	wsMutex.Unlock()
}

func deleteClient(cc ClientConn) {
	wsMutex.Lock()
	delete(ActiveClients, cc)
	wsMutex.Unlock()
}

func broadcastMessage(messageType int, message []byte) {
	wsMutex.RLock()
	for client, _ := range ActiveClients {
		client.websocket.WriteMessage(messageType, message)
	}
	wsMutex.RUnlock()
}

func MonitorBuilders(c *container.ContainerBag) {
	responses := make(chan string)

	for {
		if len(ActiveClients) > 0 {
			builders, err := GetBuilders(c)
			if err != nil || len(builders) <= 0 {
				return
			}

			filtered := make(map[string]Builder)
			for id, builder := range builders {
				if len(builder.CachedBuilds) > 0 {
					filtered[id] = builder
				}
			}

			var wg sync.WaitGroup
			wg.Add(len(filtered))

			for id, builder := range filtered {
				go func(id string, builder Builder) {
					defer wg.Done()

					b, err := GetBuilder(c, id, builder)
					if err == nil {
						if r, err := json.Marshal(b); err == nil {
							responses <- string(r)
						}
					}

				}(id, builder)
			}

			go func() {
				for response := range responses {
					broadcastMessage(websocket.TextMessage, []byte(response))
				}
			}()

			wg.Wait()
		}

		time.Sleep(time.Second * time.Duration(c.RefreshSec))
	}
}

func NewWsHandler(c *container.ContainerBag) *WsHandler {
	go MonitorBuilders(c)

	return &WsHandler{
		c: c,
	}
}

func (h WsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	ws, err := websocket.Upgrade(w, r, nil, 1024, 1024)
	if _, ok := err.(websocket.HandshakeError); ok {
		http.Error(w, "Not a websocket handshake", 400)
		return
	} else if err != nil {
		log.Println(err)
		return
	}

	sockCli := ClientConn{ws, ws.RemoteAddr()}
	addClient(sockCli)
}
<|endoftext|>


<commit_before>package main

import (
	"io"
	"net/http"
	"path"
	"bytes"
	"crypto/md5"
	"encoding/hex"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"
)

type DirEntry struct {
	pth      string
	fi       os.FileInfo
	checksum string
}

type DirEntries []DirEntry

func (a DirEntries) Len() int           { return len(a) }
func (a DirEntries) Less(i, j int) bool { return a[i].pth < a[j].pth }
func (a DirEntries) Swap(i, j int) {
	a[i], a[j] = a[j], a[i]
}

type AppConfig struct {
	Name        string
	InputRoot   string
	OutputDir   string
	BuildCmd    string
	ArchiveLocal string
	ArchiveRemote string
	Include		[]string
	Exclude		[]string
}

func countFullChecksum(ents *DirEntries) {
	for i, v := range *ents {
		st, err := os.Stat(v.pth)
		if err != nil {
			fmt.Printf("SKIP bad file [%s]\n", v.pth)
			continue
		}
		v.fi = st

		if v.fi.IsDir() {
			continue
		}

		if v.fi.Size() == 0 {
			v.checksum = "0000"
			continue
		}

		dat, err := ioutil.ReadFile(v.pth)
		if err != nil {
			panic(err)
		}
		var csum = md5.Sum(dat)
		v.checksum = hex.EncodeToString(csum[:])
		//fmt.Printf("%s\n %x", v.checksum, csum)
		(*ents)[i] = v
	}
}

func shouldIgnore(config *AppConfig, pth string) bool {
	for _,v := range config.Exclude {
		if strings.HasPrefix(pth, v) {
			return true
		}
	}

	if len(config.Include) == 0 {
		return false
	}

	for _, v := range config.Include {
		if strings.HasPrefix(pth, v) {
			return false
		}
	}
	return true
}

func collectWithGit(config *AppConfig) DirEntries{
	cmd := exec.Command("git", "ls-files")

	cmd.Dir = config.InputRoot
	out, err := cmd.Output()
	if (err != nil) {
		panic(err)
	}
	asStr := string(out)
	lines := strings.Split(asStr, "\n")
	var all DirEntries

	for _, v := range lines {
		if shouldIgnore(config, v) {
			continue
		}
		all = append(all, DirEntry { pth: path.Join(config.InputRoot ,v)})
	}
	return all
}

func normalizePaths(ents *DirEntries, rootPath string) {
	for i, v := range *ents {
		oldpath := v.pth
		newpath := strings.Replace(strings.TrimPrefix(oldpath, rootPath+"/"), "\\", "/", -1)
		v.pth = newpath
		(*ents)[i] = v
	}
}

func collectByConfig(config *AppConfig) DirEntries {
	return collectWithGit(config)
}

func getCheckSumForFiles(config *AppConfig) (DirEntries, string) {
	all := collectByConfig(config)
	sort.Sort(all)
	countFullChecksum(&all)
	normalizePaths(&all, config.InputRoot)
	var manifest bytes.Buffer
	for _, v := range all {
		manifest.WriteString(v.pth)
		manifest.WriteString(v.checksum)
		//fmt.Printf("%s %s\n", v.pth, v.checksum)
	}
	manifestSum := md5.Sum(manifest.Bytes())
	return all, hex.EncodeToString(manifestSum[:])
}

func run(bin string, arg ...string) {
	fmt.Printf("> %s %s", bin, arg)

	cmd := exec.Command(bin, arg...)
	out, err := cmd.CombinedOutput()
	fmt.Printf("%s", string(out))
	if err != nil {
		panic(err)
	}
}
func zipOutput(path string, zipfile string) {
	run("zip", "-r", zipfile, path+"/*")
}

func unzipOutput(pth string, zipfile string) {
	// we will replace the old path completely
	ensureDir(path.Dir(pth))
	os.RemoveAll(pth)
	run("unzip", zipfile, "-d"+pth)
}

func runBuildCommand(config *AppConfig) {
	fmt.Printf("Running build command '%s' in %s\n", config.BuildCmd, config.InputRoot)
	parts := strings.Fields(config.BuildCmd)
	cmd := exec.Command(parts[0], parts[1:]...)
	cmd.Dir = config.InputRoot
	out, err := cmd.CombinedOutput()
	fmt.Println(string(out))
	if err != nil {
		fmt.Printf("Build failed with error!")
		panic(err)
	}
}

func fetchTo(url string, to string) bool {
	fmt.Printf("GET %s\n", url)
	resp, err := http.Get(url)
	if err != nil || resp.StatusCode != 200 {
		fmt.Printf("Not available: %s\n", url)
		return false
	}
	defer resp.Body.Close()
	out, err := os.Create(to)
	if err != nil {
		panic(err)
	}
	defer out.Close()
	_, err  = io.Copy(out, resp.Body)
	if err != nil {
		panic(err)
	}
	return true
}

func discoverArchive(config *AppConfig, checksum string) (string,bool) {
	archiveRoot := config.ArchiveLocal
	zipName := config.Name + "_" + checksum + ".zip"

	// 1. just try local
	localZipName := filepath.Join(archiveRoot, zipName)
	_, err := os.Stat(localZipName)
	if err == nil {
		return localZipName, true
	}

	// 2. try remote if applicable

	remoteArchive := config.ArchiveRemote

	if remoteArchive == "" {
		return localZipName, false
	}
	if strings.Index(remoteArchive, "[ZIP]") == -1 {
		fmt.Printf("Error: remote archive template %s does not contain [ZIP]\n", remoteArchive)
		return "", false
	}
	remoteUrl := strings.Replace(remoteArchive, "[ZIP]", zipName, -1)
	fetched := fetchTo(remoteUrl, localZipName)
	if !fetched {
		return localZipName, false
	}
	return localZipName, true
}

func buildWithConfig(config *AppConfig) {
	// check input checksum
	fmt.Printf("Config %s\n", config)
	archiveRoot := config.ArchiveLocal

	if archiveRoot == "" {
		fmt.Println("HASHIBUILD_ARCHIVE not set, building without artifact caching")
		runBuildCommand(config)
		return
	}

	_, inputChecksum := getCheckSumForFiles(config)
	// if finding archive found, unzip it and we are ready
	ensureDir(archiveRoot)

	zipName, found := discoverArchive(config, inputChecksum)

	if found {
		fmt.Printf("Unzip %s to %s\n", zipName, config.OutputDir)
		unzipOutput(config.OutputDir, zipName)
		return
	}

	// run build if mismatch

	runBuildCommand(config)

	// zip the results
	fmt.Printf("Zipping %s to %s\n", config.OutputDir, zipName)
	zipOutput(config.OutputDir, zipName)
}

func checkDir(pth string) {
	if _, err := os.Stat(pth); os.IsNotExist(err) {
		fmt.Printf("Path does not exist: %s", pth)
		panic(err)
	}
}

func ensureDir(pth string) {
	if _, err := os.Stat(pth); os.IsNotExist(err) {
		fmt.Printf("Creating dir: %s\n", pth)
		os.MkdirAll(pth, 0777)
	}
}

func parseConfig(configPath string) AppConfig {
	cont, err := ioutil.ReadFile(configPath)
	if err != nil {
		panic(err)
	}
	config := AppConfig{}
	err = json.Unmarshal(cont, &config)
	if (err != nil) {
		panic(err)
	}
	// fixup paths to be relative to config file
	configDir, _ := filepath.Abs(filepath.Dir(configPath))
	config.InputRoot = filepath.Join(configDir, config.InputRoot)
	config.OutputDir = filepath.Join(configDir, config.OutputDir)

	checkDir(config.InputRoot)

	return config
}

func dumpManifest(config *AppConfig) {

	all, csum := getCheckSumForFiles(config)
	for _, v := range all {
		fmt.Printf("%s %s\n", v.pth, v.checksum)
	}
	fmt.Printf("Total: %s\n", csum)
}

func main() {
	manifest := flag.Bool("manifest", false, "Show manifest (requires --config)")
	treeHash := flag.String("treehash", "", "Show manifest for specified path (no config needed)")
	toParse := flag.String("config", "", "Json config file")
	startBuild := flag.Bool("build", false, "Run build")
	archiveDir := flag.String("archive", "", "Archive root dir (needed if HASHIBUILD_ARCHIVE env var is not set)")
	fetch := flag.String("fetch", "", "Fetch remote archive file to local archive")
	if len(os.Args) < 2 {
		flag.Usage()
		return
	}

	flag.Parse()

	var config AppConfig
	if (*toParse) != "" {
		config = parseConfig(*toParse)
		if *archiveDir != "" {
			config.ArchiveLocal = *archiveDir
		}
		if config.ArchiveLocal == "" {
			config.ArchiveLocal = os.Getenv("HASHIBUILD_ARCHIVE")
		}
		if config.ArchiveRemote == "" {
			config.ArchiveRemote = os.Getenv("HASHIBUILD_ARCHIVE_REMOTE")
		}
	}

	if *fetch != "" {
		_, inputChecksum := getCheckSumForFiles(&config)
		// if finding archive found, unzip it and we are ready
		zipName, _ := discoverArchive(&config, inputChecksum)
		fmt.Printf("%s", zipName)
	}

	if *manifest {
		dumpManifest(&config)
	}

	if *startBuild {
		buildWithConfig(&config)
	}

	if len(*treeHash) > 0 {
		pth, _ := filepath.Abs(*treeHash)

		config := AppConfig{InputRoot: pth }
		dumpManifest(&config)
	}

}
<commit_msg>change to zip / unzip<commit_after>package main

import (
	"io"
	"net/http"
	"path"
	"bytes"
	"crypto/md5"
	"encoding/hex"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"
)

type DirEntry struct {
	pth      string
	fi       os.FileInfo
	checksum string
}

type DirEntries []DirEntry

func (a DirEntries) Len() int           { return len(a) }
func (a DirEntries) Less(i, j int) bool { return a[i].pth < a[j].pth }
func (a DirEntries) Swap(i, j int) {
	a[i], a[j] = a[j], a[i]
}

type AppConfig struct {
	Name        string
	InputRoot   string
	OutputDir   string
	BuildCmd    string
	ArchiveLocal string
	ArchiveRemote string
	Include		[]string
	Exclude		[]string
}

func countFullChecksum(ents *DirEntries) {
	for i, v := range *ents {
		st, err := os.Stat(v.pth)
		if err != nil {
			fmt.Printf("SKIP bad file [%s]\n", v.pth)
			continue
		}
		v.fi = st

		if v.fi.IsDir() {
			continue
		}

		if v.fi.Size() == 0 {
			v.checksum = "0000"
			continue
		}

		dat, err := ioutil.ReadFile(v.pth)
		if err != nil {
			panic(err)
		}
		var csum = md5.Sum(dat)
		v.checksum = hex.EncodeToString(csum[:])
		//fmt.Printf("%s\n %x", v.checksum, csum)
		(*ents)[i] = v
	}
}

func shouldIgnore(config *AppConfig, pth string) bool {
	for _,v := range config.Exclude {
		if strings.HasPrefix(pth, v) {
			return true
		}
	}

	if len(config.Include) == 0 {
		return false
	}

	for _, v := range config.Include {
		if strings.HasPrefix(pth, v) {
			return false
		}
	}
	return true
}

func collectWithGit(config *AppConfig) DirEntries{
	cmd := exec.Command("git", "ls-files")

	cmd.Dir = config.InputRoot
	out, err := cmd.Output()
	if (err != nil) {
		panic(err)
	}
	asStr := string(out)
	lines := strings.Split(asStr, "\n")
	var all DirEntries

	for _, v := range lines {
		if shouldIgnore(config, v) {
			continue
		}
		all = append(all, DirEntry { pth: path.Join(config.InputRoot ,v)})
	}
	return all
}

func normalizePaths(ents *DirEntries, rootPath string) {
	for i, v := range *ents {
		oldpath := v.pth
		newpath := strings.Replace(strings.TrimPrefix(oldpath, rootPath+"/"), "\\", "/", -1)
		v.pth = newpath
		(*ents)[i] = v
	}
}

func collectByConfig(config *AppConfig) DirEntries {
	return collectWithGit(config)
}

func getCheckSumForFiles(config *AppConfig) (DirEntries, string) {
	all := collectByConfig(config)
	sort.Sort(all)
	countFullChecksum(&all)
	normalizePaths(&all, config.InputRoot)
	var manifest bytes.Buffer
	for _, v := range all {
		manifest.WriteString(v.pth)
		manifest.WriteString(v.checksum)
		//fmt.Printf("%s %s\n", v.pth, v.checksum)
	}
	manifestSum := md5.Sum(manifest.Bytes())
	return all, hex.EncodeToString(manifestSum[:])
}

func run(cwd string, bin string, arg ...string) {
	fmt.Printf("> %s %s", bin, arg)
	cmd := exec.Command(bin, arg...)
	cmd.Dir = cwd
	out, err := cmd.CombinedOutput()
	fmt.Printf("%s", string(out))
	if err != nil {
		panic(err)
	}
}

func zipOutput(path string, zipfile string) {
	run(path, "zip", "-r", zipfile, "*")
}

func unzipOutput(pth string, zipfile string) {
	// we will replace the old path completely
	ensureDir(pth)
	err := os.RemoveAll(pth)
	if err != nil {
		panic(err)
	}
	ensureDir(pth)
	run(".", "unzip", zipfile, "-d"+pth)
}

func runBuildCommand(config *AppConfig) {
	fmt.Printf("Running build command '%s' in %s\n", config.BuildCmd, config.InputRoot)
	parts := strings.Fields(config.BuildCmd)
	cmd := exec.Command(parts[0], parts[1:]...)
	cmd.Dir = config.InputRoot
	out, err := cmd.CombinedOutput()
	fmt.Println(string(out))
	if err != nil {
		fmt.Printf("Build failed with error!")
		panic(err)
	}
}

func fetchTo(url string, to string) bool {
	fmt.Printf("GET %s\n", url)
	resp, err := http.Get(url)
	if err != nil || resp.StatusCode != 200 {
		fmt.Printf("Not available: %s\n", url)
		return false
	}
	defer resp.Body.Close()
	out, err := os.Create(to)
	if err != nil {
		panic(err)
	}
	defer out.Close()
	_, err  = io.Copy(out, resp.Body)
	if err != nil {
		panic(err)
	}
	return true
}

func discoverArchive(config *AppConfig, checksum string) (string,bool) {
	archiveRoot := config.ArchiveLocal
	zipName := config.Name + "_" + checksum + ".zip"

	// 1. just try local
	localZipName := filepath.Join(archiveRoot, zipName)
	_, err := os.Stat(localZipName)
	if err == nil {
		return localZipName, true
	}

	// 2. try remote if applicable

	remoteArchive := config.ArchiveRemote

	if remoteArchive == "" {
		return localZipName, false
	}
	if strings.Index(remoteArchive, "[ZIP]") == -1 {
		fmt.Printf("Error: remote archive template %s does not contain [ZIP]\n", remoteArchive)
		return "", false
	}
	remoteUrl := strings.Replace(remoteArchive, "[ZIP]", zipName, -1)
	fetched := fetchTo(remoteUrl, localZipName)
	if !fetched {
		return localZipName, false
	}
	return localZipName, true
}

func buildWithConfig(config *AppConfig) {
	// check input checksum
	fmt.Printf("Config %s\n", config)
	archiveRoot := config.ArchiveLocal

	if archiveRoot == "" {
		fmt.Println("HASHIBUILD_ARCHIVE not set, building without artifact caching")
		runBuildCommand(config)
		return
	}

	_, inputChecksum := getCheckSumForFiles(config)
	// if finding archive found, unzip it and we are ready
	ensureDir(archiveRoot)

	zipName, found := discoverArchive(config, inputChecksum)

	if found {
		fmt.Printf("Unzip %s to %s\n", zipName, config.OutputDir)
		unzipOutput(config.OutputDir, zipName)
		return
	}

	// run build if mismatch

	runBuildCommand(config)

	// zip the results
	fmt.Printf("Zipping %s to %s\n", config.OutputDir, zipName)
	zipOutput(config.OutputDir, zipName)
}

func checkDir(pth string) {
	if _, err := os.Stat(pth); os.IsNotExist(err) {
		fmt.Printf("Path does not exist: %s", pth)
		panic(err)
	}
}

func ensureDir(pth string) {
	if _, err := os.Stat(pth); os.IsNotExist(err) {
		fmt.Printf("Creating dir: %s\n", pth)
		os.MkdirAll(pth, 0777)
	} else {
		fmt.Printf("Path exists: %s\n", pth)
	}

}

func parseConfig(configPath string) AppConfig {
	cont, err := ioutil.ReadFile(configPath)
	if err != nil {
		panic(err)
	}
	config := AppConfig{}
	err = json.Unmarshal(cont, &config)
	if (err != nil) {
		panic(err)
	}
	// fixup paths to be relative to config file
	configDir, _ := filepath.Abs(filepath.Dir(configPath))
	config.InputRoot = filepath.Join(configDir, config.InputRoot)
	config.OutputDir = filepath.Join(configDir, config.OutputDir)

	checkDir(config.InputRoot)

	return config
}

func dumpManifest(config *AppConfig) {

	all, csum := getCheckSumForFiles(config)
	for _, v := range all {
		fmt.Printf("%s %s\n", v.pth, v.checksum)
	}
	fmt.Printf("Total: %s\n", csum)
}

func main() {
	manifest := flag.Bool("manifest", false, "Show manifest (requires --config)")
	treeHash := flag.String("treehash", "", "Show manifest for specified path (no config needed)")
	toParse := flag.String("config", "", "Json config file")
	startBuild := flag.Bool("build", false, "Run build")
	archiveDir := flag.String("archive", "", "Archive root dir (needed if HASHIBUILD_ARCHIVE env var is not set)")
	fetch := flag.String("fetch", "", "Fetch remote archive file to local archive")
	if len(os.Args) < 2 {
		flag.Usage()
		return
	}

	flag.Parse()

	var config AppConfig
	if (*toParse) != "" {
		config = parseConfig(*toParse)
		if *archiveDir != "" {
			config.ArchiveLocal = *archiveDir
		}
		if config.ArchiveLocal == "" {
			config.ArchiveLocal = os.Getenv("HASHIBUILD_ARCHIVE")
		}
		if config.ArchiveRemote == "" {
			config.ArchiveRemote = os.Getenv("HASHIBUILD_ARCHIVE_REMOTE")
		}
	}

	if *fetch != "" {
		_, inputChecksum := getCheckSumForFiles(&config)
		// if finding archive found, unzip it and we are ready
		zipName, _ := discoverArchive(&config, inputChecksum)
		fmt.Printf("%s", zipName)
	}

	if *manifest {
		dumpManifest(&config)
	}

	if *startBuild {
		buildWithConfig(&config)
	}

	if len(*treeHash) > 0 {
		pth, _ := filepath.Abs(*treeHash)

		config := AppConfig{InputRoot: pth }
		dumpManifest(&config)
	}

}
<|endoftext|>


<commit_before>package trousseau

import (
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/oleiade/trousseau/crypto"
	"github.com/oleiade/trousseau/dsn"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"time"
)

func CreateAction(c *cli.Context) {
	if !hasExpectedArgs(c.Args(), 1) {
		log.Fatal("Incorrect number of arguments to 'configure' command")
	}

	recipients := strings.Split(c.Args()[0], ",")

	opts := &crypto.Options{
		Algorithm:  crypto.GPG_ENCRYPTION,
		Passphrase: gPasshphrase,
		Recipients: recipients,
	}

	meta := Meta{
		CreatedAt:        time.Now().String(),
		LastModifiedAt:   time.Now().String(),
		Recipients:       recipients,
		TrousseauVersion: TROUSSEAU_VERSION,
	}

	// Create and write empty store file
	err := CreateStoreFile(gStorePath, opts, &meta)
	if err != nil {
		log.Fatal(err)
	}

	fmt.Println("Trousseau data store succesfully created")
}

func PushAction(c *cli.Context) {
	if !hasExpectedArgs(c.Args(), 1) {
		log.Fatal("Incorrect number of arguments to 'push' command")
	}

	endpointDsn, err := dsn.Parse(c.Args()[0])
	if err != nil {
		log.Fatal(err)
	}

	switch endpointDsn.Scheme {
	case "s3":
		err := endpointDsn.SetDefaults(gS3Defaults)
		if err != nil {
			log.Fatal(err)
		}

		err = uploadUsingS3(endpointDsn)
		if err != nil {
			log.Fatal(err)
		}
		fmt.Printf("Trousseau data store succesfully pushed to s3")
	case "scp":
		privateKey := c.String("ssh-private-key")

		err := endpointDsn.SetDefaults(gScpDefaults)
		if err != nil {
			log.Fatal(err)
		}

		if c.Bool("ask-password") == true {
			password := PromptForPassword()
			endpointDsn.Secret = password
		}

		err = uploadUsingScp(endpointDsn, privateKey)
		if err != nil {
			log.Fatal(err)
		}
		fmt.Printf("Trousseau data store succesfully pushed to ssh remote storage")
	case "gist":
		err = uploadUsingGist(endpointDsn)
		if err != nil {
			log.Fatal(err)
		}
		fmt.Printf("Trousseau data store succesfully pushed to gist")
	}
}

func PullAction(c *cli.Context) {
	if !hasExpectedArgs(c.Args(), 1) {
		log.Fatal("Incorrect number of arguments to 'pull' command")
	}

	endpointDsn, err := dsn.Parse(c.Args()[0])
	if err != nil {
		log.Fatal(err)
	}

	switch endpointDsn.Scheme {
	case "s3":
		err := endpointDsn.SetDefaults(gS3Defaults)
		if err != nil {
			log.Fatal(err)
		}

		err = DownloadUsingS3(endpointDsn)
		if err != nil {
			log.Fatal(err)
		}
		fmt.Printf("Trousseau data store succesfully pulled from S3")
	case "scp":
		privateKey := c.String("ssh-private-key")

		err := endpointDsn.SetDefaults(gScpDefaults)
		if err != nil {
			log.Fatal(err)
		}

		if c.Bool("ask-password") == true {
			password := PromptForPassword()
			endpointDsn.Secret = password
		}

		err = DownloadUsingScp(endpointDsn, privateKey)
		if err != nil {
			log.Fatal(err)
		}
		fmt.Printf("Trousseau data store succesfully pulled from ssh remote storage")
	case "gist":
		err = DownloadUsingGist(endpointDsn)
		if err != nil {
			log.Fatal(err)
		}
		fmt.Printf("Trousseau data store succesfully pulled from gist")
	default:
		if endpointDsn.Scheme == "" {
			log.Fatalf("No dsn scheme supplied")
		} else {
			log.Fatalf("Invalid dsn scheme supplied: %s", endpointDsn.Scheme)
		}
	}

	fmt.Printf("Trousseau data store succesfully pulled from remote storage")
}

func ExportAction(c *cli.Context) {
	if !hasExpectedArgs(c.Args(), 1) {
		log.Fatal("Incorrect number of arguments to 'export' command")
	}

	var err error
	var inputFilePath string = gStorePath
	var outputFilePath string = c.Args()[0]

	inputFile, err := os.Open(inputFilePath)
	defer inputFile.Close()
	if err != nil {
		log.Fatal(err)
	}

	outputFile, err := os.Create(outputFilePath)
	defer outputFile.Close()
	if err != nil {
		log.Fatal(err)
	}

	_, err = io.Copy(outputFile, inputFile)
	if err != nil {
		log.Fatal(err)
	}

	fmt.Printf("Trousseau data store exported to %s", outputFilePath)
}

func ImportAction(c *cli.Context) {
	if !hasExpectedArgs(c.Args(), 1) {
		log.Fatal("Incorrect number of arguments to 'import' command")
	}

	var err error
	var importedFilePath string = c.Args()[0]
	var localFilePath string = gStorePath
	var strategy *ImportStrategy = new(ImportStrategy)

	// Transform provided merging startegy flags
	// into a proper ImportStrategy byte.
	err = strategy.FromCliContext(c)
	if err != nil {
		log.Fatal(err)
	}

	opts := &crypto.Options{
		Algorithm:  crypto.GPG_ENCRYPTION,
		Passphrase: gPasshphrase,
	}

	localStore, err := LoadStore(localFilePath, opts)
	if err != nil {
		log.Fatal(err)
	}

	importedStore, err := LoadStore(importedFilePath, opts)
	if err != nil {
		log.Fatal(err)
	}

	err = ImportStore(importedStore, localStore, *strategy)
	if err != nil {
		log.Fatal(err)
	}

	err = localStore.Sync()
	if err != nil {
		log.Fatal(err)
	}

	fmt.Println("Trousseau data store imported")
}

func AddRecipientAction(c *cli.Context) {
	if !hasExpectedArgs(c.Args(), 1) {
		log.Fatal("Incorrect number of arguments to 'add-recipient' command")
	}

	recipient := c.Args()[0]

	opts := &crypto.Options{
		Algorithm:  crypto.GPG_ENCRYPTION,
		Passphrase: gPasshphrase,
	}

	store, err := LoadStore(gStorePath, opts)
	if err != nil {
		log.Fatal(err)
	}

	err = store.DataStore.Meta.AddRecipient(recipient)

	err = store.Sync()
	if err != nil {
		log.Fatal(err)
	}

	fmt.Printf("%s recipient added to trousseau data store", recipient)
}

func RemoveRecipientAction(c *cli.Context) {
	if !hasExpectedArgs(c.Args(), 1) {
		log.Fatal("Incorrect number of arguments to 'remove-recipient' command")
	}

	recipient := c.Args()[0]

	opts := &crypto.Options{
		Algorithm:  crypto.GPG_ENCRYPTION,
		Passphrase: gPasshphrase,
	}

	store, err := LoadStore(gStorePath, opts)
	if err != nil {
		log.Fatal(err)
	}

	err = store.Meta.RemoveRecipient(recipient)

	err = store.Sync()
	if err != nil {
		log.Fatal(err)
	}

	fmt.Printf("%s recipient removed from trousseau data store", recipient)
}

func GetAction(c *cli.Context) {
	if !hasExpectedArgs(c.Args(), 1) {
		log.Fatal("Incorrect number of arguments to 'get' command")
	}

	opts := &crypto.Options{
		Algorithm:  crypto.GPG_ENCRYPTION,
		Passphrase: gPasshphrase,
	}

	store, err := LoadStore(gStorePath, opts)
	if err != nil {
		log.Fatal(err)
	}

	value, err := store.Get(c.Args()[0])
	if err != nil {
		log.Fatal(err)
	}

	fmt.Printf("%s key's value: %s\n", c.Args()[0], value)
}

func SetAction(c *cli.Context) {
	var key string
	var value interface{}
	var err error

	// If the --file flag is provided
	if c.String("file") != "" && hasExpectedArgs(c.Args(), 1) {
		// And the file actually exists on file system
		if pathExists(c.String("file")) {
			// Then load it's content
			value, err = ioutil.ReadFile(c.String("file"))
			if err != nil {
				log.Fatal(err)
			}
		} else {
			log.Fatalf("Cannot open %s because it doesn't exist", c.String("file"))
		}
	} else if c.String("file") == "" && hasExpectedArgs(c.Args(), 2) {
		value = c.Args()[1]
	} else {
		log.Fatal("Incorrect number of arguments to 'set' command")
	}

  key = c.Args()[0]

	opts := &crypto.Options{
		Algorithm:  crypto.GPG_ENCRYPTION,
		Passphrase: gPasshphrase,
	}

	store, err := LoadStore(gStorePath, opts)
	if err != nil {
		log.Fatal(err)
	}

	err = store.Set(key, value)
	if err != nil {
		log.Fatal(err)
	}

	err = store.Sync()
	if err != nil {
		log.Fatal(err)
	}

	fmt.Printf("key-value pair set: %s:%s\n", key, value)
}

func DelAction(c *cli.Context) {
	if !hasExpectedArgs(c.Args(), 1) {
		log.Fatal("Incorrect number of arguments to 'del' command")
	}

	opts := &crypto.Options{
		Algorithm:  crypto.GPG_ENCRYPTION,
		Passphrase: gPasshphrase,
	}

	store, err := LoadStore(gStorePath, opts)
	if err != nil {
		log.Fatal(err)
	}

	err = store.Del(c.Args()[0])
	if err != nil {
		log.Fatal(err)
	}

	err = store.Sync()
	if err != nil {
		log.Fatal(err)
	}

	fmt.Printf("%s key deleted\n", c.Args()[0])
}

func KeysAction(c *cli.Context) {
	if !hasExpectedArgs(c.Args(), 0) {
		log.Fatal("Incorrect number of arguments to 'keys' command")
	}

	opts := &crypto.Options{
		Algorithm:  crypto.GPG_ENCRYPTION,
		Passphrase: gPasshphrase,
	}

	store, err := LoadStore(gStorePath, opts)
	if err != nil {
		log.Fatal(err)
	}

	keys, err := store.Keys()
	if err != nil {
		log.Fatal(err)
	} else {
		for _, k := range keys {
			fmt.Println(k)
		}
	}
}

func ShowAction(c *cli.Context) {
	if !hasExpectedArgs(c.Args(), 0) {
		log.Fatal("Incorrect number of arguments to 'show' command")
	}

	opts := &crypto.Options{
		Algorithm:  crypto.GPG_ENCRYPTION,
		Passphrase: gPasshphrase,
	}

	store, err := LoadStore(gStorePath, opts)
	if err != nil {
		log.Fatal(err)
	}

	pairs, err := store.Items()
	if err != nil {
		log.Fatal(err)
	} else {
		for _, pair := range pairs {
			fmt.Printf("%s: %s\n", pair.Key, pair.Value)
		}
	}
}

func MetaAction(c *cli.Context) {
	if !hasExpectedArgs(c.Args(), 0) {
		log.Fatal("Incorrect number of arguments to 'meta' command")
	}

	opts := &crypto.Options{
		Algorithm:  crypto.GPG_ENCRYPTION,
		Passphrase: gPasshphrase,
	}

	store, err := LoadStore(gStorePath, opts)
	if err != nil {
		log.Fatal(err)
	}

	pairs, err := store.Metadata()
	if err != nil {
		log.Fatal(err)
	}

	for _, pair := range pairs {
		fmt.Printf("%s: %s\n", pair.Key, pair.Value)
	}
}

// hasExpectedArgs checks whether the number of args are as expected.
func hasExpectedArgs(args []string, expected int) bool {
	switch expected {
	case -1:
		if len(args) > 0 {
			return true
		} else {
			return false
		}
	default:
		if len(args) == expected {
			return true
		} else {
			return false
		}
	}
}
<commit_msg>Fix #69 clean command-line output for its parsing to be easier<commit_after>package trousseau

import (
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/oleiade/trousseau/crypto"
	"github.com/oleiade/trousseau/dsn"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"time"
)

func CreateAction(c *cli.Context) {
	if !hasExpectedArgs(c.Args(), 1) {
		log.Fatal("Incorrect number of arguments to 'configure' command")
	}

	recipients := strings.Split(c.Args()[0], ",")

	opts := &crypto.Options{
		Algorithm:  crypto.GPG_ENCRYPTION,
		Passphrase: gPasshphrase,
		Recipients: recipients,
	}

	meta := Meta{
		CreatedAt:        time.Now().String(),
		LastModifiedAt:   time.Now().String(),
		Recipients:       recipients,
		TrousseauVersion: TROUSSEAU_VERSION,
	}

	// Create and write empty store file
	err := CreateStoreFile(gStorePath, opts, &meta)
	if err != nil {
		log.Fatal(err)
	}

	fmt.Println("Trousseau data store succesfully created")
}

func PushAction(c *cli.Context) {
	if !hasExpectedArgs(c.Args(), 1) {
		log.Fatal("Incorrect number of arguments to 'push' command")
	}

	endpointDsn, err := dsn.Parse(c.Args()[0])
	if err != nil {
		log.Fatal(err)
	}

	switch endpointDsn.Scheme {
	case "s3":
		err := endpointDsn.SetDefaults(gS3Defaults)
		if err != nil {
			log.Fatal(err)
		}

		err = uploadUsingS3(endpointDsn)
		if err != nil {
			log.Fatal(err)
		}
		fmt.Printf("Trousseau data store succesfully pushed to s3")
	case "scp":
		privateKey := c.String("ssh-private-key")

		err := endpointDsn.SetDefaults(gScpDefaults)
		if err != nil {
			log.Fatal(err)
		}

		if c.Bool("ask-password") == true {
			password := PromptForPassword()
			endpointDsn.Secret = password
		}

		err = uploadUsingScp(endpointDsn, privateKey)
		if err != nil {
			log.Fatal(err)
		}
		fmt.Printf("Trousseau data store succesfully pushed to ssh remote storage")
	case "gist":
		err = uploadUsingGist(endpointDsn)
		if err != nil {
			log.Fatal(err)
		}
		fmt.Printf("Trousseau data store succesfully pushed to gist")
	}
}

func PullAction(c *cli.Context) {
	if !hasExpectedArgs(c.Args(), 1) {
		log.Fatal("Incorrect number of arguments to 'pull' command")
	}

	endpointDsn, err := dsn.Parse(c.Args()[0])
	if err != nil {
		log.Fatal(err)
	}

	switch endpointDsn.Scheme {
	case "s3":
		err := endpointDsn.SetDefaults(gS3Defaults)
		if err != nil {
			log.Fatal(err)
		}

		err = DownloadUsingS3(endpointDsn)
		if err != nil {
			log.Fatal(err)
		}
		fmt.Printf("Trousseau data store succesfully pulled from S3")
	case "scp":
		privateKey := c.String("ssh-private-key")

		err := endpointDsn.SetDefaults(gScpDefaults)
		if err != nil {
			log.Fatal(err)
		}

		if c.Bool("ask-password") == true {
			password := PromptForPassword()
			endpointDsn.Secret = password
		}

		err = DownloadUsingScp(endpointDsn, privateKey)
		if err != nil {
			log.Fatal(err)
		}
		fmt.Printf("Trousseau data store succesfully pulled from ssh remote storage")
	case "gist":
		err = DownloadUsingGist(endpointDsn)
		if err != nil {
			log.Fatal(err)
		}
		fmt.Printf("Trousseau data store succesfully pulled from gist")
	default:
		if endpointDsn.Scheme == "" {
			log.Fatalf("No dsn scheme supplied")
		} else {
			log.Fatalf("Invalid dsn scheme supplied: %s", endpointDsn.Scheme)
		}
	}

	fmt.Printf("Trousseau data store succesfully pulled from remote storage")
}

func ExportAction(c *cli.Context) {
	if !hasExpectedArgs(c.Args(), 1) {
		log.Fatal("Incorrect number of arguments to 'export' command")
	}

	var err error
	var inputFilePath string = gStorePath
	var outputFilePath string = c.Args()[0]

	inputFile, err := os.Open(inputFilePath)
	defer inputFile.Close()
	if err != nil {
		log.Fatal(err)
	}

	outputFile, err := os.Create(outputFilePath)
	defer outputFile.Close()
	if err != nil {
		log.Fatal(err)
	}

	_, err = io.Copy(outputFile, inputFile)
	if err != nil {
		log.Fatal(err)
	}

	fmt.Printf("Trousseau data store exported to: %s\n", outputFilePath)
}

func ImportAction(c *cli.Context) {
	if !hasExpectedArgs(c.Args(), 1) {
		log.Fatal("Incorrect number of arguments to 'import' command")
	}

	var err error
	var importedFilePath string = c.Args()[0]
	var localFilePath string = gStorePath
	var strategy *ImportStrategy = new(ImportStrategy)

	// Transform provided merging startegy flags
	// into a proper ImportStrategy byte.
	err = strategy.FromCliContext(c)
	if err != nil {
		log.Fatal(err)
	}

	opts := &crypto.Options{
		Algorithm:  crypto.GPG_ENCRYPTION,
		Passphrase: gPasshphrase,
	}

	localStore, err := LoadStore(localFilePath, opts)
	if err != nil {
		log.Fatal(err)
	}

	importedStore, err := LoadStore(importedFilePath, opts)
	if err != nil {
		log.Fatal(err)
	}

	err = ImportStore(importedStore, localStore, *strategy)
	if err != nil {
		log.Fatal(err)
	}

	err = localStore.Sync()
	if err != nil {
		log.Fatal(err)
	}

	fmt.Printf("Trousseau data store imported: %s\n", importedFilePath)
}

func AddRecipientAction(c *cli.Context) {
	if !hasExpectedArgs(c.Args(), 1) {
		log.Fatal("Incorrect number of arguments to 'add-recipient' command")
	}

	recipient := c.Args()[0]

	opts := &crypto.Options{
		Algorithm:  crypto.GPG_ENCRYPTION,
		Passphrase: gPasshphrase,
	}

	store, err := LoadStore(gStorePath, opts)
	if err != nil {
		log.Fatal(err)
	}

	err = store.DataStore.Meta.AddRecipient(recipient)

	err = store.Sync()
	if err != nil {
		log.Fatal(err)
	}

	fmt.Printf("Recipient added to trousseau data store: %s\n", recipient)
}

func RemoveRecipientAction(c *cli.Context) {
	if !hasExpectedArgs(c.Args(), 1) {
		log.Fatal("Incorrect number of arguments to 'remove-recipient' command")
	}

	recipient := c.Args()[0]

	opts := &crypto.Options{
		Algorithm:  crypto.GPG_ENCRYPTION,
		Passphrase: gPasshphrase,
	}

	store, err := LoadStore(gStorePath, opts)
	if err != nil {
		log.Fatal(err)
	}

	err = store.Meta.RemoveRecipient(recipient)

	err = store.Sync()
	if err != nil {
		log.Fatal(err)
	}

	fmt.Printf("Recipient removed from trousseau data store: %s\n", recipient)
}

func GetAction(c *cli.Context) {
	if !hasExpectedArgs(c.Args(), 1) {
		log.Fatal("Incorrect number of arguments to 'get' command")
	}

	opts := &crypto.Options{
		Algorithm:  crypto.GPG_ENCRYPTION,
		Passphrase: gPasshphrase,
	}

	store, err := LoadStore(gStorePath, opts)
	if err != nil {
		log.Fatal(err)
	}

	value, err := store.Get(c.Args()[0])
	if err != nil {
		log.Fatal(err)
	}

	fmt.Printf("%s\n", value)
}

func SetAction(c *cli.Context) {
	var key string
	var value interface{}
	var err error

	// If the --file flag is provided
	if c.String("file") != "" && hasExpectedArgs(c.Args(), 1) {
		// And the file actually exists on file system
		if pathExists(c.String("file")) {
			// Then load it's content
			value, err = ioutil.ReadFile(c.String("file"))
			if err != nil {
				log.Fatal(err)
			}
		} else {
			log.Fatalf("Cannot open %s because it doesn't exist", c.String("file"))
		}
	} else if c.String("file") == "" && hasExpectedArgs(c.Args(), 2) {
		value = c.Args()[1]
	} else {
		log.Fatal("Incorrect number of arguments to 'set' command")
	}

	key = c.Args()[0]

	opts := &crypto.Options{
		Algorithm:  crypto.GPG_ENCRYPTION,
		Passphrase: gPasshphrase,
	}

	store, err := LoadStore(gStorePath, opts)
	if err != nil {
		log.Fatal(err)
	}

	err = store.Set(key, value)
	if err != nil {
		log.Fatal(err)
	}

	err = store.Sync()
	if err != nil {
		log.Fatal(err)
	}

	fmt.Printf("%s:%s\n", key, value)
}

func DelAction(c *cli.Context) {
	if !hasExpectedArgs(c.Args(), 1) {
		log.Fatal("Incorrect number of arguments to 'del' command")
	}

	opts := &crypto.Options{
		Algorithm:  crypto.GPG_ENCRYPTION,
		Passphrase: gPasshphrase,
	}

	store, err := LoadStore(gStorePath, opts)
	if err != nil {
		log.Fatal(err)
	}

	err = store.Del(c.Args()[0])
	if err != nil {
		log.Fatal(err)
	}

	err = store.Sync()
	if err != nil {
		log.Fatal(err)
	}

	fmt.Printf("deleted: %s\n", c.Args()[0])
}

func KeysAction(c *cli.Context) {
	if !hasExpectedArgs(c.Args(), 0) {
		log.Fatal("Incorrect number of arguments to 'keys' command")
	}

	opts := &crypto.Options{
		Algorithm:  crypto.GPG_ENCRYPTION,
		Passphrase: gPasshphrase,
	}

	store, err := LoadStore(gStorePath, opts)
	if err != nil {
		log.Fatal(err)
	}

	keys, err := store.Keys()
	if err != nil {
		log.Fatal(err)
	} else {
		for _, k := range keys {
			fmt.Println(k)
		}
	}
}

func ShowAction(c *cli.Context) {
	if !hasExpectedArgs(c.Args(), 0) {
		log.Fatal("Incorrect number of arguments to 'show' command")
	}

	opts := &crypto.Options{
		Algorithm:  crypto.GPG_ENCRYPTION,
		Passphrase: gPasshphrase,
	}

	store, err := LoadStore(gStorePath, opts)
	if err != nil {
		log.Fatal(err)
	}

	pairs, err := store.Items()
	if err != nil {
		log.Fatal(err)
	} else {
		for _, pair := range pairs {
			fmt.Printf("%s : %s\n", pair.Key, pair.Value)
		}
	}
}

func MetaAction(c *cli.Context) {
	if !hasExpectedArgs(c.Args(), 0) {
		log.Fatal("Incorrect number of arguments to 'meta' command")
	}

	opts := &crypto.Options{
		Algorithm:  crypto.GPG_ENCRYPTION,
		Passphrase: gPasshphrase,
	}

	store, err := LoadStore(gStorePath, opts)
	if err != nil {
		log.Fatal(err)
	}

	pairs, err := store.Metadata()
	if err != nil {
		log.Fatal(err)
	}

	for _, pair := range pairs {
		fmt.Printf("%s : %s\n", pair.Key, pair.Value)
	}
}

// hasExpectedArgs checks whether the number of args are as expected.
func hasExpectedArgs(args []string, expected int) bool {
	switch expected {
	case -1:
		if len(args) > 0 {
			return true
		} else {
			return false
		}
	default:
		if len(args) == expected {
			return true
		} else {
			return false
		}
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 Niklas Wolber
// This file is licensed under the MIT license.
// See the LICENSE file for more information.
package main

import (
	"fmt"
	"log"
	"net/url"

	twitter "github.com/ChimeraCoder/anaconda"
	r "github.com/dancannon/gorethink"
)

type followerExplorer struct {
	id        int64
	followers []int64
}

func newFollowerTask(session *r.Session, api *twitter.TwitterApi) *task {
	selectUncrawledExplorers := func(row r.Term) interface{} {
		return row.HasFields("followers").Not()
	}
	query := r.
		Table("pi").
		Filter(selectUncrawledExplorers).
		OrderBy(r.Desc("tweets"))

	extractor := func(row map[string]interface{}) interface{} {
		var exp followerExplorer
		exp.id = int64(row["explorer"].(float64))
		return exp
	}

	processor := func(entity interface{}) {
		exp := entity.(followerExplorer)

		var followers []int64
		var nextCursor int64
		log.Println("Fetching followers of", exp.id)
		for {
			params := url.Values{}
			params.Add("user_id", fmt.Sprintf("%d", exp.id))
			params.Add("count", "5000")
			if nextCursor > 0 {
				params.Add("cursor", fmt.Sprintf("%d", nextCursor))
			}

			cursor, err := api.GetFollowersIds(params)

			if err != nil {
				log.Fatalln(err)
			}

			log.Println("Fetched", len(cursor.Ids), "followers of", exp.id)
			followers = append(followers, cursor.Ids...)

			nextCursor = cursor.Next_cursor

			if nextCursor == 0 {
				exp.storeFollowers(session, followers)
				return
			}
		}
	}
	return newTask(query, extractor, processor, false)
}

func (exp *followerExplorer) storeFollowers(session *r.Session, followers []int64) {
	result, err := r.
		Table("pi").
		Insert(map[string]interface{}{
		"explorer":  exp.id,
		"followers": followers,
	}, r.InsertOpts{Conflict: "update"}).
		RunWrite(session)

	if err != nil {
		log.Println(err)
	}

	if result.Errors > 0 {
		log.Println(result.FirstError)
	}
}
<commit_msg>Don't fail when a Twitter requests errrors<commit_after>// Copyright (c) 2015 Niklas Wolber
// This file is licensed under the MIT license.
// See the LICENSE file for more information.
package main

import (
	"fmt"
	"log"
	"net/url"
	"time"

	twitter "github.com/ChimeraCoder/anaconda"
	r "github.com/dancannon/gorethink"
)

type followerExplorer struct {
	id        int64
	followers []int64
}

func newFollowerTask(session *r.Session, api *twitter.TwitterApi) *task {
	selectUncrawledExplorers := func(row r.Term) interface{} {
		return row.HasFields("followers").Not()
	}
	query := r.
		Table("pi").
		Filter(selectUncrawledExplorers).
		OrderBy(r.Desc("tweets"))

	extractor := func(row map[string]interface{}) interface{} {
		var exp followerExplorer
		exp.id = int64(row["explorer"].(float64))
		return exp
	}

	processor := func(entity interface{}) {
		exp := entity.(followerExplorer)

		var followers []int64
		var nextCursor int64
		log.Println("Fetching followers of", exp.id)
		for {
			params := url.Values{}
			params.Add("user_id", fmt.Sprintf("%d", exp.id))
			params.Add("count", "5000")
			if nextCursor > 0 {
				params.Add("cursor", fmt.Sprintf("%d", nextCursor))
			}

			cursor, err := api.GetFollowersIds(params)

			if err != nil {
				log.Println(err)
				time.Sleep(time.Minute)
				continue
			}

			log.Println("Fetched", len(cursor.Ids), "followers of", exp.id)
			followers = append(followers, cursor.Ids...)

			nextCursor = cursor.Next_cursor

			if nextCursor == 0 {
				exp.storeFollowers(session, followers)
				return
			}
		}
	}
	return newTask(query, extractor, processor, false)
}

func (exp *followerExplorer) storeFollowers(session *r.Session, followers []int64) {
	result, err := r.
		Table("pi").
		Insert(map[string]interface{}{
		"explorer":  exp.id,
		"followers": followers,
	}, r.InsertOpts{Conflict: "update"}).
		RunWrite(session)

	if err != nil {
		log.Println(err)
	}

	if result.Errors > 0 {
		log.Println(result.FirstError)
	}
}
<|endoftext|>


<commit_before>// Package bot provides the internal machinery for most of Gopherbot.
package bot

/* bot.go defines core data structures and public methods for startup.
   handler.go has the methods for callbacks from the connector, */

import (
	"encoding/base64"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"net"
	"net/http"
	"os"
	"os/signal"
	"path/filepath"
	"regexp"
	"sync"
	"syscall"
	"time"

	"github.com/lnxjedi/gopherbot/robot"
)

// VersionInfo holds information about the version, duh. (stupid linter)
type VersionInfo struct {
	Version, Commit string
}

// global values for GOPHER_HOME, GOPHER_CONFIGDIR and GOPHER_INSTALLDIR
var homePath, configPath, installPath string

var botVersion VersionInfo

var random *rand.Rand

var connectors = make(map[string]func(robot.Handler, *log.Logger) robot.Connector)

// RegisterConnector should be called in an init function to register a type
// of connector. Currently only Slack is implemented.
func RegisterConnector(name string, connstarter func(robot.Handler, *log.Logger) robot.Connector) {
	if stopRegistrations {
		return
	}
	if connectors[name] != nil {
		log.Fatal("Attempted registration of duplicate connector:", name)
	}
	connectors[name] = connstarter
}

// Interfaces to external stuff, items should be set while single-threaded and never change
var interfaces struct {
	robot.Connector                       // Connector interface, implemented by each specific protocol
	brain           robot.SimpleBrain     // Interface for robot to Store and Retrieve data
	history         robot.HistoryProvider // Provider for storing and retrieving job / plugin histories
	stop            chan struct{}         // stop channel for stopping the connector
	done            chan bool             // shutdown channel, true to restart
}

// internal state tracking
var state struct {
	shuttingDown   bool // to prevent new plugins from starting
	restart        bool // indicate stop and restart vs. stop only, for bootstrapping
	pluginsRunning int  // a count of how many plugins are currently running
	sync.WaitGroup      // for keeping track of running plugins
	sync.RWMutex        // for safe updating of bot data structures
}

// regexes the bot uses to determine if it's being spoken to
var regexes struct {
	preRegex  *regexp.Regexp // regex for matching prefixed commands, e.g. "Gort, drop your weapon"
	postRegex *regexp.Regexp // regex for matching, e.g. "open the pod bay doors, hal"
	bareRegex *regexp.Regexp // regex for matching the robot's bare name, if you forgot it in the previous command
	sync.RWMutex
}

// configuration struct holds all the interal data relevant to the Bot. Most of it is digested
// and populated by loadConfig.
type configuration struct {
	adminUsers           []string            // List of users with access to administrative commands
	alias                rune                // single-char alias for addressing the bot
	botinfo              UserInfo            // robot's name, ID, email, etc.
	adminContact         string              // who to contact for problems with the bot
	mailConf             botMailer           // configuration to use when sending email
	ignoreUsers          []string            // list of users to never listen to, like other bots
	joinChannels         []string            // list of channels to join
	defaultAllowDirect   bool                // whether plugins are available in DM by default
	defaultMessageFormat robot.MessageFormat // Raw unless set to Variable or Fixed
	plugChannels         []string            // list of channels where plugins are available by default
	protocol             string              // Name of the protocol, e.g. "slack"
	brainProvider        string              // Type of Brain provider to use
	encryptionKey        string              // Key for encrypting data (unlocks "real" key in brain)
	historyProvider      string              // Name of the history provider to use
	workSpace            string              // Read/Write directory where the robot does work
	defaultElevator      string              // Plugin name for performing elevation
	defaultAuthorizer    string              // Plugin name for performing authorization
	externalPlugins      []TaskSettings      // List of external plugins to load
	externalJobs         []TaskSettings      // List of external jobs to load
	externalTasks        []TaskSettings      // List of external tasks to load
	goPlugins            []TaskSettings      // Settings for goPlugins: Name(match), Description, NameSpace, Parameters, Disabled
	goJobs               []TaskSettings      // Settings for goJobs: Name(match), Description, NameSpace, Parameters, Disabled
	goTasks              []TaskSettings      // Settings for goTasks: Name(match), Description, NameSpace, Parameters, Disabled
	nsList               []TaskSettings      // loaded NameSpaces for shared parameters
	loadableModules      []LoadableModule    // List of loadable modules to load
	ScheduledJobs        []ScheduledTask     // List of scheduled tasks
	port                 string              // Configured localhost port to listen on, or 0 for first open
	timeZone             *time.Location      // for forcing the TimeZone, Unix only
	defaultJobChannel    string              // where job statuses will post if not otherwise specified
}

// The current configuration and task list
var currentCfg = struct {
	*configuration
	*taskList
	sync.RWMutex
}{
	configuration: &configuration{},
	taskList: &taskList{
		t:          []interface{}{struct{}{}}, // initialize 0 to "nothing", for namespaces only
		nameMap:    make(map[string]int),
		idMap:      make(map[string]int),
		nameSpaces: make(map[string]NameSpace),
	},
	RWMutex: sync.RWMutex{},
}

var listening bool    // for tests where initBot runs multiple times
var listenPort string // actual listening port

// initBot sets up the global robot; when cli is false it also loads configuration.
// cli indicates that a CLI command is being processed, as opposed to actually running
// a robot.
func initBot(cpath, epath string, logger *log.Logger) {
	// Seed the pseudo-random number generator, for plugin IDs, RandomString, etc.
	random = rand.New(rand.NewSource(time.Now().UnixNano()))

	// Initialize current config with an empty struct (to be loaded)
	currentCfg.configuration = &configuration{}

	botLogger.l = logger

	var err error
	homePath, err = os.Getwd()
	if err != nil {
		Log(robot.Warn, "Unable to get cwd")
	}
	h := handler{}
	if err := h.GetDirectory(cpath); err != nil {
		Log(robot.Fatal, "Unable to get/create config path: %s", cpath)
	}
	configPath = cpath
	installPath = epath
	interfaces.stop = make(chan struct{})
	interfaces.done = make(chan bool)
	state.shuttingDown = false

	if cliOp {
		setLogLevel(robot.Warn)
	}

	encryptionInitialized := initCrypt()

	c := &botContext{
		environment: make(map[string]string),
	}
	if err := c.loadConfig(true); err != nil {
		Log(robot.Fatal, "Loading initial configuration: %v", err)
	}
	os.Unsetenv(keyEnv)

	if cliOp {
		if fileLog {
			setLogLevel(robot.Debug)
		} else {
			setLogLevel(robot.Warn)
		}
	}

	// loadModules for go loadable modules; a no-op for static builds
	loadModules()

	// All pluggables registered, ok to stop registrations
	stopRegistrations = true

	if len(currentCfg.brainProvider) > 0 {
		if bprovider, ok := brains[currentCfg.brainProvider]; !ok {
			Log(robot.Fatal, "No provider registered for brain: \"%s\"", currentCfg.brainProvider)
		} else {
			brain := bprovider(handle)
			interfaces.brain = brain
			Log(robot.Info, "Initialized brain provider '%s'", currentCfg.brainProvider)
		}
	} else {
		bprovider, _ := brains["mem"]
		interfaces.brain = bprovider(handle)
		Log(robot.Error, "No brain configured, falling back to default 'mem' brain - no memories will persist")
	}
	if !encryptionInitialized && len(currentCfg.encryptionKey) > 0 {
		if initializeEncryptionFromBrain(currentCfg.encryptionKey) {
			Log(robot.Info, "Successfully initialized encryption from configured key")
			encryptionInitialized = true
		} else {
			Log(robot.Error, "Failed to initialize brain encryption with configured EncryptionKey")
		}
	}
	if encryptBrain && !encryptionInitialized {
		Log(robot.Warn, "Brain encryption specified but not initialized; use 'initialize brain <key>' to initialize the encrypted brain interactively")
	}

	// cli commands don't need an http listener
	if cliOp {
		return
	}

	if !listening {
		listening = true
		listener, err := net.Listen("tcp4", fmt.Sprintf("127.0.0.1:%s", currentCfg.port))
		if err != nil {
			Log(robot.Fatal, "Listening on tcp4 port 127.0.0.1:%s: %v", currentCfg.port, err)
		}
		listenPort = listener.Addr().String()
		go func() {
			raiseThreadPriv("http handler")
			http.Handle("/json", handle)
			Log(robot.Info, "Listening for external plugin connections on http://%s", listenPort)
			Log(robot.Fatal, "Error serving '/json': %s", http.Serve(listener, nil))
		}()
	}
}

// set connector sets the connector, which should already be initialized
func setConnector(c robot.Connector) {
	interfaces.Connector = c
}

var keyEnv = "GOPHER_ENCRYPTION_KEY"

func initCrypt() bool {
	// Initialize encryption (new style for v2)
	keyFile := filepath.Join(configPath, encryptedKeyFile)
	encryptionInitialized := false
	if ek, ok := os.LookupEnv(keyEnv); ok {
		ik := []byte(ek)[0:32]
		if bkf, err := ioutil.ReadFile(keyFile); err == nil {
			if bke, err := base64.StdEncoding.DecodeString(string(bkf)); err == nil {
				if key, err := decrypt(bke, ik); err == nil {
					cryptKey.key = key
					cryptKey.initialized = true
					encryptionInitialized = true
					Log(robot.Info, "Successfully decrypted binary encryption key '%s'", keyFile)
				} else {
					Log(robot.Error, "Decrypting binary encryption key '%s' from environment key '%s': %v", keyFile, keyEnv, err)
				}
			} else {
				Log(robot.Error, "Base64 decoding '%s': %v", keyFile, err)
			}
		} else {
			Log(robot.Warn, "Binary encryption key not loaded from '%s': %v", keyFile, err)
		}
		os.Unsetenv(keyEnv)
	} else {
		Log(robot.Warn, "GOPHER_ENCRYPTION_KEY not set in environment")
	}
	return encryptionInitialized
}

// run starts all the loops and returns a channel that closes when the robot
// shuts down. It should return after the connector loop has started and
// plugins are initialized.
func run() <-chan bool {
	// Start the brain loop
	go runBrain()

	var cl []string
	cl = append(cl, currentCfg.joinChannels...)
	cl = append(cl, currentCfg.plugChannels...)
	cl = append(cl, currentCfg.defaultJobChannel)
	jc := make(map[string]bool)
	for _, channel := range cl {
		if _, ok := jc[channel]; !ok {
			jc[channel] = true
			interfaces.JoinChannel(channel)
		}
	}

	// signal handler
	go func() {
		done := interfaces.done
		sigs := make(chan os.Signal, 1)

		signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)

	loop:
		for {
			select {
			case sig := <-sigs:
				state.Lock()
				if state.shuttingDown {
					Log(robot.Warn, "Received SIGINT/SIGTERM while shutdown in progress")
					state.Unlock()
				} else {
					state.shuttingDown = true
					state.Unlock()
					signal.Stop(sigs)
					Log(robot.Info, "Exiting on signal: %s", sig)
					stop()
				}
			case <-done:
				break loop
			}
		}
	}()

	// connector loop
	go func(conn robot.Connector, stop <-chan struct{}, done chan<- bool) {
		raiseThreadPriv("connector loop")
		conn.Run(stop)
		state.RLock()
		restart := state.restart
		state.RUnlock()
		if restart {
			Log(robot.Info, "Restarting...")
		}
		done <- restart
		// NOTE!! Black Magic Ahead - for some reason, the read on the done channel
		// keeps blocking without this close.
		close(done)
	}(interfaces.Connector, interfaces.stop, interfaces.done)
	c := &botContext{
		environment: make(map[string]string),
	}
	c.registerActive(nil)
	c.loadConfig(false)
	c.deregister()
	return interfaces.done
}

// stop is called whenever the robot needs to shut down gracefully. All callers
// should lock the bot and check the value of botCfg.shuttingDown; see
// builtins.go.
func stop() {
	state.RLock()
	pr := state.pluginsRunning
	stop := interfaces.stop
	state.RUnlock()
	Log(robot.Debug, "stop called with %d plugins running", pr)
	state.Wait()
	brainQuit()
	close(stop)
}
<commit_msg>Generate new-style key if old-style not configured<commit_after>// Package bot provides the internal machinery for most of Gopherbot.
package bot

/* bot.go defines core data structures and public methods for startup.
   handler.go has the methods for callbacks from the connector, */

import (
	crand "crypto/rand"
	"encoding/base64"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"net"
	"net/http"
	"os"
	"os/signal"
	"path/filepath"
	"regexp"
	"sync"
	"syscall"
	"time"

	"github.com/lnxjedi/gopherbot/robot"
)

// VersionInfo holds information about the version, duh. (stupid linter)
type VersionInfo struct {
	Version, Commit string
}

// global values for GOPHER_HOME, GOPHER_CONFIGDIR and GOPHER_INSTALLDIR
var homePath, configPath, installPath string

var botVersion VersionInfo

var random *rand.Rand

var connectors = make(map[string]func(robot.Handler, *log.Logger) robot.Connector)

// RegisterConnector should be called in an init function to register a type
// of connector. Currently only Slack is implemented.
func RegisterConnector(name string, connstarter func(robot.Handler, *log.Logger) robot.Connector) {
	if stopRegistrations {
		return
	}
	if connectors[name] != nil {
		log.Fatal("Attempted registration of duplicate connector:", name)
	}
	connectors[name] = connstarter
}

// Interfaces to external stuff, items should be set while single-threaded and never change
var interfaces struct {
	robot.Connector                       // Connector interface, implemented by each specific protocol
	brain           robot.SimpleBrain     // Interface for robot to Store and Retrieve data
	history         robot.HistoryProvider // Provider for storing and retrieving job / plugin histories
	stop            chan struct{}         // stop channel for stopping the connector
	done            chan bool             // shutdown channel, true to restart
}

// internal state tracking
var state struct {
	shuttingDown   bool // to prevent new plugins from starting
	restart        bool // indicate stop and restart vs. stop only, for bootstrapping
	pluginsRunning int  // a count of how many plugins are currently running
	sync.WaitGroup      // for keeping track of running plugins
	sync.RWMutex        // for safe updating of bot data structures
}

// regexes the bot uses to determine if it's being spoken to
var regexes struct {
	preRegex  *regexp.Regexp // regex for matching prefixed commands, e.g. "Gort, drop your weapon"
	postRegex *regexp.Regexp // regex for matching, e.g. "open the pod bay doors, hal"
	bareRegex *regexp.Regexp // regex for matching the robot's bare name, if you forgot it in the previous command
	sync.RWMutex
}

// configuration struct holds all the interal data relevant to the Bot. Most of it is digested
// and populated by loadConfig.
type configuration struct {
	adminUsers           []string            // List of users with access to administrative commands
	alias                rune                // single-char alias for addressing the bot
	botinfo              UserInfo            // robot's name, ID, email, etc.
	adminContact         string              // who to contact for problems with the bot
	mailConf             botMailer           // configuration to use when sending email
	ignoreUsers          []string            // list of users to never listen to, like other bots
	joinChannels         []string            // list of channels to join
	defaultAllowDirect   bool                // whether plugins are available in DM by default
	defaultMessageFormat robot.MessageFormat // Raw unless set to Variable or Fixed
	plugChannels         []string            // list of channels where plugins are available by default
	protocol             string              // Name of the protocol, e.g. "slack"
	brainProvider        string              // Type of Brain provider to use
	encryptionKey        string              // Key for encrypting data (unlocks "real" key in brain)
	historyProvider      string              // Name of the history provider to use
	workSpace            string              // Read/Write directory where the robot does work
	defaultElevator      string              // Plugin name for performing elevation
	defaultAuthorizer    string              // Plugin name for performing authorization
	externalPlugins      []TaskSettings      // List of external plugins to load
	externalJobs         []TaskSettings      // List of external jobs to load
	externalTasks        []TaskSettings      // List of external tasks to load
	goPlugins            []TaskSettings      // Settings for goPlugins: Name(match), Description, NameSpace, Parameters, Disabled
	goJobs               []TaskSettings      // Settings for goJobs: Name(match), Description, NameSpace, Parameters, Disabled
	goTasks              []TaskSettings      // Settings for goTasks: Name(match), Description, NameSpace, Parameters, Disabled
	nsList               []TaskSettings      // loaded NameSpaces for shared parameters
	loadableModules      []LoadableModule    // List of loadable modules to load
	ScheduledJobs        []ScheduledTask     // List of scheduled tasks
	port                 string              // Configured localhost port to listen on, or 0 for first open
	timeZone             *time.Location      // for forcing the TimeZone, Unix only
	defaultJobChannel    string              // where job statuses will post if not otherwise specified
}

// The current configuration and task list
var currentCfg = struct {
	*configuration
	*taskList
	sync.RWMutex
}{
	configuration: &configuration{},
	taskList: &taskList{
		t:          []interface{}{struct{}{}}, // initialize 0 to "nothing", for namespaces only
		nameMap:    make(map[string]int),
		idMap:      make(map[string]int),
		nameSpaces: make(map[string]NameSpace),
	},
	RWMutex: sync.RWMutex{},
}

var listening bool    // for tests where initBot runs multiple times
var listenPort string // actual listening port

// initBot sets up the global robot; when cli is false it also loads configuration.
// cli indicates that a CLI command is being processed, as opposed to actually running
// a robot.
func initBot(cpath, epath string, logger *log.Logger) {
	// Seed the pseudo-random number generator, for plugin IDs, RandomString, etc.
	random = rand.New(rand.NewSource(time.Now().UnixNano()))

	// Initialize current config with an empty struct (to be loaded)
	currentCfg.configuration = &configuration{}

	botLogger.l = logger

	var err error
	homePath, err = os.Getwd()
	if err != nil {
		Log(robot.Warn, "Unable to get cwd")
	}
	h := handler{}
	if err := h.GetDirectory(cpath); err != nil {
		Log(robot.Fatal, "Unable to get/create config path: %s", cpath)
	}
	configPath = cpath
	installPath = epath
	interfaces.stop = make(chan struct{})
	interfaces.done = make(chan bool)
	state.shuttingDown = false

	if cliOp {
		setLogLevel(robot.Warn)
	}

	encryptionInitialized := initCrypt()

	c := &botContext{
		environment: make(map[string]string),
	}
	if err := c.loadConfig(true); err != nil {
		Log(robot.Fatal, "Loading initial configuration: %v", err)
	}
	os.Unsetenv(keyEnv)

	if cliOp {
		if fileLog {
			setLogLevel(robot.Debug)
		} else {
			setLogLevel(robot.Warn)
		}
	}

	// loadModules for go loadable modules; a no-op for static builds
	loadModules()

	// All pluggables registered, ok to stop registrations
	stopRegistrations = true

	if len(currentCfg.brainProvider) > 0 {
		if bprovider, ok := brains[currentCfg.brainProvider]; !ok {
			Log(robot.Fatal, "No provider registered for brain: \"%s\"", currentCfg.brainProvider)
		} else {
			brain := bprovider(handle)
			interfaces.brain = brain
			Log(robot.Info, "Initialized brain provider '%s'", currentCfg.brainProvider)
		}
	} else {
		bprovider, _ := brains["mem"]
		interfaces.brain = bprovider(handle)
		Log(robot.Error, "No brain configured, falling back to default 'mem' brain - no memories will persist")
	}
	if !encryptionInitialized && len(currentCfg.encryptionKey) > 0 {
		if initializeEncryptionFromBrain(currentCfg.encryptionKey) {
			Log(robot.Info, "Successfully initialized encryption from configured key")
			encryptionInitialized = true
		} else {
			Log(robot.Error, "Failed to initialize brain encryption with configured EncryptionKey")
		}
	}
	if encryptBrain && !encryptionInitialized {
		Log(robot.Warn, "Brain encryption specified but not initialized; use 'initialize brain <key>' to initialize the encrypted brain interactively")
	}

	// cli commands don't need an http listener
	if cliOp {
		return
	}

	if !listening {
		listening = true
		listener, err := net.Listen("tcp4", fmt.Sprintf("127.0.0.1:%s", currentCfg.port))
		if err != nil {
			Log(robot.Fatal, "Listening on tcp4 port 127.0.0.1:%s: %v", currentCfg.port, err)
		}
		listenPort = listener.Addr().String()
		go func() {
			raiseThreadPriv("http handler")
			http.Handle("/json", handle)
			Log(robot.Info, "Listening for external plugin connections on http://%s", listenPort)
			Log(robot.Fatal, "Error serving '/json': %s", http.Serve(listener, nil))
		}()
	}
}

// set connector sets the connector, which should already be initialized
func setConnector(c robot.Connector) {
	interfaces.Connector = c
}

var keyEnv = "GOPHER_ENCRYPTION_KEY"

func initCrypt() bool {
	// Initialize encryption (new style for v2)
	keyFile := filepath.Join(configPath, encryptedKeyFile)
	encryptionInitialized := false
	if ek, ok := os.LookupEnv(keyEnv); ok {
		ik := []byte(ek)[0:32]
		if bkf, err := ioutil.ReadFile(keyFile); err == nil {
			if bke, err := base64.StdEncoding.DecodeString(string(bkf)); err == nil {
				if key, err := decrypt(bke, ik); err == nil {
					cryptKey.key = key
					cryptKey.initialized = true
					encryptionInitialized = true
					Log(robot.Info, "Successfully decrypted binary encryption key '%s'", keyFile)
				} else {
					Log(robot.Error, "Decrypting binary encryption key '%s' from environment key '%s': %v", keyFile, keyEnv, err)
				}
			} else {
				Log(robot.Error, "Base64 decoding '%s': %v", keyFile, err)
			}
		} else {
			Log(robot.Warn, "Binary encryption key not loaded from '%s': %v", keyFile, err)
			if len(currentCfg.encryptionKey) == 0 {
				// No encryptionKey in config, create new-style key
				bk := make([]byte, 32)
				_, err := crand.Read(bk)
				if err != nil {
					Log(robot.Error, "Generating new random encryption key: %v", err)
					return false
				}
				bek, err := encrypt(bk, ik)
				if err != nil {
					Log(robot.Error, "Encrypting new random key: %v", err)
					return false
				}
				beks := base64.StdEncoding.EncodeToString(bek)
				err = ioutil.WriteFile(keyFile, []byte(beks), 0444)
				if err != nil {
					Log(robot.Error, "Writing out generated key: %v", err)
					return false
				}
				Log(robot.Info, "Successfully wrote new binary encryption key to '%s'", keyFile)
				return true
			}
		}
		os.Unsetenv(keyEnv)
	} else {
		Log(robot.Warn, "GOPHER_ENCRYPTION_KEY not set in environment")
	}
	return encryptionInitialized
}

// run starts all the loops and returns a channel that closes when the robot
// shuts down. It should return after the connector loop has started and
// plugins are initialized.
func run() <-chan bool {
	// Start the brain loop
	go runBrain()

	var cl []string
	cl = append(cl, currentCfg.joinChannels...)
	cl = append(cl, currentCfg.plugChannels...)
	cl = append(cl, currentCfg.defaultJobChannel)
	jc := make(map[string]bool)
	for _, channel := range cl {
		if _, ok := jc[channel]; !ok {
			jc[channel] = true
			interfaces.JoinChannel(channel)
		}
	}

	// signal handler
	go func() {
		done := interfaces.done
		sigs := make(chan os.Signal, 1)

		signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)

	loop:
		for {
			select {
			case sig := <-sigs:
				state.Lock()
				if state.shuttingDown {
					Log(robot.Warn, "Received SIGINT/SIGTERM while shutdown in progress")
					state.Unlock()
				} else {
					state.shuttingDown = true
					state.Unlock()
					signal.Stop(sigs)
					Log(robot.Info, "Exiting on signal: %s", sig)
					stop()
				}
			case <-done:
				break loop
			}
		}
	}()

	// connector loop
	go func(conn robot.Connector, stop <-chan struct{}, done chan<- bool) {
		raiseThreadPriv("connector loop")
		conn.Run(stop)
		state.RLock()
		restart := state.restart
		state.RUnlock()
		if restart {
			Log(robot.Info, "Restarting...")
		}
		done <- restart
		// NOTE!! Black Magic Ahead - for some reason, the read on the done channel
		// keeps blocking without this close.
		close(done)
	}(interfaces.Connector, interfaces.stop, interfaces.done)
	c := &botContext{
		environment: make(map[string]string),
	}
	c.registerActive(nil)
	c.loadConfig(false)
	c.deregister()
	return interfaces.done
}

// stop is called whenever the robot needs to shut down gracefully. All callers
// should lock the bot and check the value of botCfg.shuttingDown; see
// builtins.go.
func stop() {
	state.RLock()
	pr := state.pluginsRunning
	stop := interfaces.stop
	state.RUnlock()
	Log(robot.Debug, "stop called with %d plugins running", pr)
	state.Wait()
	brainQuit()
	close(stop)
}
<|endoftext|>


<commit_before>package golangQL

import (
	"reflect"
	"strings"
	"sync"
)

type filterFunc func(val reflect.Value, tree *node) (interface{}, error)

func (g *golangQL) filter(v interface{}, query string) (interface{}, error) {
	if len(query) == 0 {
		return v, nil
	}

	tree, err := parse(query)
	if err != nil {
		return nil, err
	}

	val := reflect.ValueOf(v)
	filter := g.getFilter(val, tree)

	result, err := filter(val, tree)
	if err != nil {
		return nil, err
	}

	return result, nil
}

func (g *golangQL) getFilter(val reflect.Value, tree *node) filterFunc {
	key := newCacheKey(val.Type(), tree)

	g.RLock()
	f := g.filters[key]
	g.RUnlock()
	if f != nil {
		return f
	}

	g.Lock()
	f = g.filters[key]
	if f != nil {
		g.Unlock()
		return f
	}

	recursive := sync.WaitGroup{}
	recursive.Add(1)
	g.filters[key] = func(val reflect.Value, tree *node) (interface{}, error) {
		recursive.Wait()
		return f(val, tree)
	}
	g.Unlock()

	f = g.newFilter(val.Type(), tree)
	g.Lock()
	g.filters[key] = f
	g.Unlock()

	recursive.Done()
	return f
}

func (g *golangQL) newFilter(typ reflect.Type, tree *node) filterFunc {
	switch typ.Kind() {
	case reflect.Ptr:
		return g.newPtrFilter(typ, tree)
	case reflect.Struct:
		return g.newStructFilter(typ, tree)
	case reflect.Slice:
		return g.newSliceFilter(typ, tree)
	default:
		return defaultFilter()
	}
}

// Filter constructors

func defaultFilter() filterFunc {
	return func(val reflect.Value, tree *node) (interface{}, error) {
		return val.Interface(), nil
	}
}

func (g *golangQL) newPtrFilter(typ reflect.Type, tree *node) filterFunc {
	elemFilter := g.newFilter(typ.Elem(), tree)

	return func(val reflect.Value, tree *node) (interface{}, error) {
		v := val.Elem()

		return elemFilter(v, tree)
	}
}

func (g *golangQL) newSliceFilter(typ reflect.Type, tree *node) filterFunc {
	return func(val reflect.Value, tree *node) (interface{}, error) {
		resultMap := make([]map[string]interface{}, val.Len())
		resultMapValue := reflect.ValueOf(resultMap)
		for i := 0; i < val.Len(); i++ {
			v := val.Index(i)

			elementType := v.Type()
			elemFilter := g.newFilter(elementType, tree)

			filteredStruct, err := elemFilter(v, tree)
			if err != nil {
				return nil, err
			}

			filteredValue := reflect.ValueOf(filteredStruct)

			resultMapValue.Index(i).Set(filteredValue)
		}

		return resultMap, nil
	}
}

func (g *golangQL) newStructFilter(typ reflect.Type, tree *node) filterFunc {

	return func(val reflect.Value, tree *node) (interface{}, error) {
		typ := val.Type()
		resultMap := make(map[string]interface{}, typ.NumField())

		for i := 0; i < typ.NumField(); i++ {
			field := typ.Field(i)
			tag, ok := field.Tag.Lookup("json")
			tagName := g.fieldParseTag(tag)

			if !ok {
				elemFilter := g.newFilter(field.Type, tree)

				embeded, err := elemFilter(val.Field(i), tree)
				if err != nil {
					return nil, err
				}

				if reflect.TypeOf(embeded).Kind() != reflect.Map {
					resultMap[tagName] = val
					continue
				}

				embededMapValue := reflect.ValueOf(embeded)
				for _, key := range embededMapValue.MapKeys() {
					resultMap[key.String()] = embededMapValue.MapIndex(key).Interface()
				}
				continue
			}

			child := tree.findChildByName(tagName)
			if child != nil {
				elemFilter := g.newFilter(typ, child)

				childStruct, err := elemFilter(val.Field(i).Elem(), child)
				if err != nil {
					return nil, err
				}

				resultMap[tagName] = childStruct
				continue
			}

			if tree.containsField(tagName) {
				resultMap[tagName] = val.Field(i).Interface()
			}
		}

		return resultMap, nil
	}
}

func (g *golangQL) fieldParseTag(tag string) string {
	if i := strings.Index(tag, ","); i != -1 {
		name := tag[:i]
		tag = tag[i+1:]
		return name
	}
	return tag
}
<commit_msg>fix cache<commit_after>package golangQL

import (
	"reflect"
	"strings"
	"sync"
)

type filterFunc func(val reflect.Value, tree *node) (interface{}, error)

func (g *golangQL) filter(v interface{}, query string) (interface{}, error) {
	if len(query) == 0 {
		return v, nil
	}

	tree, err := parse(query)
	if err != nil {
		return nil, err
	}

	val := reflect.ValueOf(v)
	filter := g.getFilter(val.Type(), tree)

	result, err := filter(val, tree)
	if err != nil {
		return nil, err
	}

	return result, nil
}

func (g *golangQL) getFilter(typ reflect.Type, tree *node) filterFunc {
	g.RLock()
	key := newCacheKey(typ, tree)
	f := g.filters[key]
	g.RUnlock()
	if f != nil {
		return f
	}

	g.Lock()
	f = g.filters[key]
	if f != nil {
		g.Unlock()
		return f
	}

	recursive := sync.WaitGroup{}
	recursive.Add(1)
	g.filters[key] = func(val reflect.Value, tree *node) (interface{}, error) {
		recursive.Wait()
		return f(val, tree)
	}
	g.Unlock()

	f = g.newFilter(typ, tree)
	g.Lock()
	g.filters[key] = f
	g.Unlock()

	recursive.Done()
	return f
}

func (g *golangQL) newFilter(typ reflect.Type, tree *node) filterFunc {
	switch typ.Kind() {
	case reflect.Ptr:
		return g.newPtrFilter(typ, tree)
	case reflect.Struct:
		return g.newStructFilter(typ, tree)
	case reflect.Slice:
		return g.newSliceFilter(typ, tree)
	default:
		return defaultFilter()
	}
}

// Filter constructors

func defaultFilter() filterFunc {
	return func(val reflect.Value, tree *node) (interface{}, error) {
		return val.Interface(), nil
	}
}

func (g *golangQL) newPtrFilter(typ reflect.Type, tree *node) filterFunc {
	elemFilter := g.getFilter(typ.Elem(), tree)

	return func(val reflect.Value, tree *node) (interface{}, error) {
		v := val.Elem()

		return elemFilter(v, tree)
	}
}

func (g *golangQL) newSliceFilter(typ reflect.Type, tree *node) filterFunc {
	return func(val reflect.Value, tree *node) (interface{}, error) {
		resultMap := make([]map[string]interface{}, val.Len())
		resultMapValue := reflect.ValueOf(resultMap)
		for i := 0; i < val.Len(); i++ {
			v := val.Index(i)

			elementType := v.Type()
			elemFilter := g.getFilter(elementType, tree)

			filteredStruct, err := elemFilter(v, tree)
			if err != nil {
				return nil, err
			}

			filteredValue := reflect.ValueOf(filteredStruct)

			resultMapValue.Index(i).Set(filteredValue)
		}

		return resultMap, nil
	}
}

func (g *golangQL) newStructFilter(typ reflect.Type, tree *node) filterFunc {

	return func(val reflect.Value, tree *node) (interface{}, error) {
		typ := val.Type()
		resultMap := make(map[string]interface{}, typ.NumField())

		for i := 0; i < typ.NumField(); i++ {
			field := typ.Field(i)
			tag, ok := field.Tag.Lookup("json")
			tagName := g.fieldParseTag(tag)

			if !ok {
				elemFilter := g.getFilter(field.Type, tree)

				embeded, err := elemFilter(val.Field(i), tree)
				if err != nil {
					return nil, err
				}

				if reflect.TypeOf(embeded).Kind() != reflect.Map {
					resultMap[tagName] = val
					continue
				}

				embededMapValue := reflect.ValueOf(embeded)
				for _, key := range embededMapValue.MapKeys() {
					resultMap[key.String()] = embededMapValue.MapIndex(key).Interface()
				}
				continue
			}

			child := tree.findChildByName(tagName)
			if child != nil {
				elemFilter := g.getFilter(typ, child)

				childStruct, err := elemFilter(val.Field(i).Elem(), child)
				if err != nil {
					return nil, err
				}

				resultMap[tagName] = childStruct
				continue
			}

			if tree.containsField(tagName) {
				resultMap[tagName] = val.Field(i).Interface()
			}
		}

		return resultMap, nil
	}
}

func (g *golangQL) fieldParseTag(tag string) string {
	if i := strings.Index(tag, ","); i != -1 {
		name := tag[:i]
		tag = tag[i+1:]
		return name
	}
	return tag
}
<|endoftext|>


<commit_before>package i

// Filter iterator
type FilterFunc func(Iterator) bool

type filter struct {
	wforward
	ff FilterFunc
}

func Filter(ff FilterFunc, itr Forward) Forward {
	f := filter{ff: ff}
	f.wforward = *(WrapForward(itr))
	return &f
}

func (f *filter) AtEnd() bool {
	for !f.wforward.AtEnd() {
		if f.ff(&f.wforward) {
			return false
		}
		f.wforward.Next()
	}
	return true
}
<commit_msg>i.Filter: remove AtEnd implementation, move it to Next. AtEnd is now idempotent as it should be<commit_after>package i

import "fmt"

// Filter iterator
type FilterFunc func(Iterator) bool

type filter struct {
	WForward
	ff FilterFunc
}

func Filter(ff FilterFunc, itr Forward) Forward {
	f := filter{ff: ff}
	f.WForward = *(WrapForward(itr))
	return &f
}

func (f *filter) Next() error {
	if f.WForward.AtEnd() {
		f.WForward.SetError(fmt.Errorf("Calling Next() after end"))
		return f.WForward.Error()
	}
	for !f.WForward.AtEnd() {
		f.WForward.Next()
		if !f.WForward.AtEnd() && f.ff(&f.WForward) {
			return nil
		}
	}
	return f.WForward.Error()
}
<|endoftext|>