<commit_before>/*
Copyright 2017 Eliott Teissonniere

Permission is hereby granted, free of charge, to any person
obtaining a copy of this software and associated documentation
files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge,
publish, distribute, sublicense, and/or sell copies of the Software,
and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:

The above copyright notice and this permission notice shall be included
in all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/

package repo

import (
	"fmt"
	"os"
	"strconv"
	"time"

	"crypto/md5"
	"encoding/base64"
	"encoding/json"
	"io/ioutil"

	"github.com/DeveloppSoft/go-ipfs-api"
)

type Ledger struct {
	Repo string

	sh *shell.Shell // IPFS api
}

func NewLedger(repo_path string, ipfs_api string) Ledger {
	// Create some files if needed
	checkAndMake(repo_path)
	checkAndMake(repo_path + "/feed")
	checkAndMake(repo_path + "/ressources")
	checkAndMakeFile(repo_path+"/lastseq", []byte("0"))
	checkAndMakeFile(repo_path+"/about.json", []byte("{}"))

	return Ledger{Repo: repo_path, sh: shell.NewShell(ipfs_api)}
}

// Recursively add stuff in the repo and do an `ipfs name publish`
func (l *Ledger) Sync() error {
	// First, add the repo to ipfs
	id, err := l.sh.AddDir(l.Repo)
	if err != nil {
		return err
	}

	// Do the ipfs name publish <id>
	// Publish for 365 days
	return l.sh.Publish(id, "8760h")
}

// Get a message, returned as a reader
func (l *Ledger) GetMessage(peer_name string, sequence string) (string, error) {
	peer, err := l.Resolve(peer_name)
	if err != nil {
		return "", err
	}

	reader, err := l.sh.Cat(peer + "/feed/" + sequence + ".json")
	if err != nil {
		return "", err
	}

	bytes, err := ioutil.ReadAll(reader)
	return string(bytes), err
}

// Get the last seq number, as a string (no need to convert)
func (l *Ledger) GetLastSeq(peer_name string) (string, error) {
	peer, err := l.Resolve(peer_name)
	if err != nil {
		return "", err
	}

	reader, err := l.sh.Cat(peer + "/lastseq")
	if err != nil {
		return "", err
	}

	bytes, err := ioutil.ReadAll(reader)
	return string(bytes), err
}

// Get all messages from a peer, return a slice of them, ordered from the more recent to the oldest
func (l *Ledger) GetFeed(peer_name string) ([]string, error) {
	result := make([]string, 0)

	seq_str, err := l.GetLastSeq(peer_name)
	if err != nil {
		return result, err
	}

	seq, err := strconv.Atoi(seq_str)
	if err != nil {
		return result, err
	}

	for i := seq; i > 0; i-- {
		msg, err := l.GetMessage(peer_name, strconv.Itoa(i))
		if err != nil {
			return result, err
		}

		result = append(result, msg)
	}

	return result, nil
}

// Return our id or ""
func (l *Ledger) Whoami() string {
	id, err := l.sh.ID()

	if err != nil {
		return ""
	} else {
		return id.ID
	}
}

// Just retrieve about.json
func (l *Ledger) About(peer_name string) (string, error) {
	peer, err := l.Resolve(peer_name)
	if err != nil {
		return "", err
	}

	reader, err := l.sh.Cat(peer + "/about.json")
	if err != nil {
		return "", err
	}

	bytes, err := ioutil.ReadAll(reader)
	return string(bytes), err
}

// Fill the profile of our user
func (l *Ledger) SetAbout(about About) error {
	bytes, err := about.Export()
	if err != nil {
		return err
	}

	// Write that to about.json
	return ioutil.WriteFile(l.Repo+"/about.json", bytes, os.ModePerm)
}

type Message struct {
	Seq       int
	Timestamp time.Time

	Data string
}

// Add a message and increase the lastseq
func (l *Ledger) Publish(data string) error {
	seq_str, err := l.GetLastSeq(l.Whoami())
	if err != nil {
		return err
	}

	seq, err := strconv.Atoi(seq_str)
	if err != nil {
		return err
	}

	seq++
	seq_str = strconv.Itoa(seq)

	// Build the message
	msg := Message{Seq: seq, Timestamp: time.Now(), Data: data}
	msg_byte, err := json.Marshal(msg)
	if err != nil {
		return err
	}

	// Just write it to the repo
	err = ioutil.WriteFile(l.Repo+"/feed/"+seq_str+".json", msg_byte, os.ModePerm) // TODO: better perm
	if err != nil {
		return err
	}

	// Increment lastseq
	return ioutil.WriteFile(l.Repo+"/lastseq", []byte(seq_str), os.ModePerm) // TODO: better perm
}

func (l *Ledger) AddRessource(b64 string) (string, error) {
	// Unpack data
	data, err := base64.StdEncoding.DecodeString(b64)
	if err != nil {
		return "", err
	}

	// Calculate checksum (no need for a mega high algo here, let's use md5)
	hash_bytes := md5.Sum(data)
	hash := fmt.Sprintf("%s", hash_bytes)

	err = ioutil.WriteFile(l.Repo+"/ressources/"+hash, data, os.ModePerm) // Need better perms
	return hash, err
}

func (l *Ledger) GetRessource(id string) (string, error) {
	reader, err := l.sh.Cat(id)
	if err != nil {
		return "", err
	}

	bytes, err := ioutil.ReadAll(reader)
	if err != nil {
		return "", err
	}

	return base64.StdEncoding.EncodeToString(bytes), nil
}

func (l *Ledger) Resolve(name string) (string, error) {
	return l.sh.Resolve(name)
}
<commit_msg>[repo] Change from md5 to blake2<commit_after>/*
Copyright 2017 Eliott Teissonniere

Permission is hereby granted, free of charge, to any person
obtaining a copy of this software and associated documentation
files (the "Software"), to deal in the Software without restriction,
including without limitation the rights to use, copy, modify, merge,
publish, distribute, sublicense, and/or sell copies of the Software,
and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:

The above copyright notice and this permission notice shall be included
in all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/

package repo

import (
	"fmt"
	"os"
	"strconv"
	"time"

	"encoding/base64"
	"encoding/json"
	"io/ioutil"

	"github.com/DeveloppSoft/go-ipfs-api"

	"golang.org/x/crypto/blake2b"
)

type Ledger struct {
	Repo string

	sh *shell.Shell // IPFS api
}

func NewLedger(repo_path string, ipfs_api string) Ledger {
	// Create some files if needed
	checkAndMake(repo_path)
	checkAndMake(repo_path + "/feed")
	checkAndMake(repo_path + "/ressources")
	checkAndMakeFile(repo_path+"/lastseq", []byte("0"))
	checkAndMakeFile(repo_path+"/about.json", []byte("{}"))

	return Ledger{Repo: repo_path, sh: shell.NewShell(ipfs_api)}
}

// Recursively add stuff in the repo and do an `ipfs name publish`
func (l *Ledger) Sync() error {
	// First, add the repo to ipfs
	id, err := l.sh.AddDir(l.Repo)
	if err != nil {
		return err
	}

	// Do the ipfs name publish <id>
	// Publish for 365 days
	return l.sh.Publish(id, "8760h")
}

// Get a message, returned as a reader
func (l *Ledger) GetMessage(peer_name string, sequence string) (string, error) {
	peer, err := l.Resolve(peer_name)
	if err != nil {
		return "", err
	}

	reader, err := l.sh.Cat(peer + "/feed/" + sequence + ".json")
	if err != nil {
		return "", err
	}

	bytes, err := ioutil.ReadAll(reader)
	return string(bytes), err
}

// Get the last seq number, as a string (no need to convert)
func (l *Ledger) GetLastSeq(peer_name string) (string, error) {
	peer, err := l.Resolve(peer_name)
	if err != nil {
		return "", err
	}

	reader, err := l.sh.Cat(peer + "/lastseq")
	if err != nil {
		return "", err
	}

	bytes, err := ioutil.ReadAll(reader)
	return string(bytes), err
}

// Get all messages from a peer, return a slice of them, ordered from the more recent to the oldest
func (l *Ledger) GetFeed(peer_name string) ([]string, error) {
	result := make([]string, 0)

	seq_str, err := l.GetLastSeq(peer_name)
	if err != nil {
		return result, err
	}

	seq, err := strconv.Atoi(seq_str)
	if err != nil {
		return result, err
	}

	for i := seq; i > 0; i-- {
		msg, err := l.GetMessage(peer_name, strconv.Itoa(i))
		if err != nil {
			return result, err
		}

		result = append(result, msg)
	}

	return result, nil
}

// Return our id or ""
func (l *Ledger) Whoami() string {
	id, err := l.sh.ID()

	if err != nil {
		return ""
	} else {
		return id.ID
	}
}

// Just retrieve about.json
func (l *Ledger) About(peer_name string) (string, error) {
	peer, err := l.Resolve(peer_name)
	if err != nil {
		return "", err
	}

	reader, err := l.sh.Cat(peer + "/about.json")
	if err != nil {
		return "", err
	}

	bytes, err := ioutil.ReadAll(reader)
	return string(bytes), err
}

// Fill the profile of our user
func (l *Ledger) SetAbout(about About) error {
	bytes, err := about.Export()
	if err != nil {
		return err
	}

	// Write that to about.json
	return ioutil.WriteFile(l.Repo+"/about.json", bytes, os.ModePerm)
}

type Message struct {
	Seq       int
	Timestamp time.Time

	Data string
}

// Add a message and increase the lastseq
func (l *Ledger) Publish(data string) error {
	seq_str, err := l.GetLastSeq(l.Whoami())
	if err != nil {
		return err
	}

	seq, err := strconv.Atoi(seq_str)
	if err != nil {
		return err
	}

	seq++
	seq_str = strconv.Itoa(seq)

	// Build the message
	msg := Message{Seq: seq, Timestamp: time.Now(), Data: data}
	msg_byte, err := json.Marshal(msg)
	if err != nil {
		return err
	}

	// Just write it to the repo
	err = ioutil.WriteFile(l.Repo+"/feed/"+seq_str+".json", msg_byte, os.ModePerm) // TODO: better perm
	if err != nil {
		return err
	}

	// Increment lastseq
	return ioutil.WriteFile(l.Repo+"/lastseq", []byte(seq_str), os.ModePerm) // TODO: better perm
}

func (l *Ledger) AddRessource(b64 string) (string, error) {
	// Unpack data
	data, err := base64.StdEncoding.DecodeString(b64)
	if err != nil {
		return "", err
	}

	// Calculate checksum (no need for a mega high algo here, let's use md5)
	hash_bytes := blake2b.Sum512(data)
	hash := fmt.Sprintf("%x", hash_bytes)

	err = ioutil.WriteFile(l.Repo+"/ressources/"+hash, data, os.ModePerm) // Need better perms
	return hash, err
}

func (l *Ledger) GetRessource(id string) (string, error) {
	reader, err := l.sh.Cat(id)
	if err != nil {
		return "", err
	}

	bytes, err := ioutil.ReadAll(reader)
	if err != nil {
		return "", err
	}

	return base64.StdEncoding.EncodeToString(bytes), nil
}

func (l *Ledger) Resolve(name string) (string, error) {
	return l.sh.Resolve(name)
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"
	"time"

	fcm "github.com/NaySoftware/go-fcm"
	"github.com/SaidinWoT/timespan"
	"github.com/heetch/sqalx"
	"github.com/jmoiron/sqlx"

	sq "github.com/gbl08ma/squirrel"

	"github.com/gbl08ma/disturbancesmlx/dataobjects"
	"github.com/gbl08ma/keybox"
)

var (
	rdb           *sqlx.DB
	sdb           sq.StatementBuilderType
	rootSqalxNode sqalx.Node
	secrets       *keybox.Keybox
	fcmcl         *fcm.FcmClient
	mainLog       = log.New(os.Stdout, "", log.Ldate|log.Ltime)
	webLog        = log.New(os.Stdout, "web", log.Ldate|log.Ltime)
	lastChange    time.Time
)

// MLcalculator implements resource.StatsCalculator
type MLcalculator struct{}

func (*MLcalculator) Availability(node sqalx.Node, line *dataobjects.Line, startTime time.Time, endTime time.Time) (float64, time.Duration, error) {
	return MLlineAvailability(node, line, startTime, endTime)
}

// MLlastDisturbanceTime returns the time of the latest Metro de Lisboa disturbance
func MLlastDisturbanceTime(node sqalx.Node) (t time.Time, err error) {
	tx, err := node.Beginx()
	if err != nil {
		return time.Now().UTC(), err
	}
	defer tx.Commit() // read-only tx

	n, err := dataobjects.GetNetwork(tx, MLnetworkID)
	if err != nil {
		return time.Now().UTC(), err
	}
	d, err := n.LastDisturbance(tx)
	if err != nil {
		return time.Now().UTC(), err
	}

	if !d.Ended {
		return time.Now().UTC(), nil
	}

	return d.EndTime, nil
}

// MLlineAvailability returns the availability for a Metro de Lisboa line
func MLlineAvailability(node sqalx.Node, line *dataobjects.Line, startTime time.Time, endTime time.Time) (float64, time.Duration, error) {
	// calculate closed time
	var closedDuration time.Duration
	ct := startTime
	wholeSpan := timespan.New(startTime, endTime.Sub(startTime))
	for ct.Before(endTime) {
		closeTime := time.Date(ct.Year(), ct.Month(), ct.Day(), 1, 0, 0, 0, ct.Location())
		openTime := time.Date(ct.Year(), ct.Month(), ct.Day(), 6, 30, 0, 0, ct.Location())

		closedSpan := timespan.New(closeTime, openTime.Sub(closeTime))
		d, hasIntersection := wholeSpan.Intersection(closedSpan)
		if hasIntersection {
			closedDuration += d.Duration()
		}
		ct = ct.AddDate(0, 0, 1)
	}

	return line.Availability(node, startTime, endTime, closedDuration)
}

func main() {
	var err error
	mainLog.Println("Server starting, opening keybox...")
	secrets, err = keybox.Open(SecretsPath)
	if err != nil {
		mainLog.Fatal(err)
	}
	mainLog.Println("Keybox opened")

	mainLog.Println("Opening database...")
	databaseURI, present := secrets.Get("databaseURI")
	if !present {
		mainLog.Fatal("Database connection string not present in keybox")
	}
	rdb, err = sqlx.Open("postgres", databaseURI)
	if err != nil {
		mainLog.Fatal(err)
	}
	defer rdb.Close()

	err = rdb.Ping()
	if err != nil {
		mainLog.Fatal(err)
	}
	sdb = sq.StatementBuilder.PlaceholderFormat(sq.Dollar).RunWith(rdb)

	rootSqalxNode, err = sqalx.New(rdb)
	if err != nil {
		mainLog.Fatal(err)
	}

	mainLog.Println("Database opened")

	fcmServerKey, present := secrets.Get("firebaseServerKey")
	if !present {
		mainLog.Fatal("Firebase server key not present in keybox")
	}
	fcmcl = fcm.NewFcmClient(fcmServerKey)

	SetUpScrapers()
	defer TearDownScrapers()

	facebookAccessToken, present := secrets.Get("facebookToken")
	if !present {
		mainLog.Fatal("Facebook API access token not present in keybox")
	}

	SetUpAnnouncements(facebookAccessToken)
	defer TearDownAnnouncements()

	go WebServer()

	certPath := DefaultClientCertPath
	if len(os.Args) > 1 {
		certPath = os.Args[1]
	}
	go APIserver(certPath)

	go func() {
		for {
			err := ComputeTypicalSeconds(rootSqalxNode)
			if err != nil {
				mainLog.Println(err)
			}
			time.Sleep(12 * time.Hour)
		}
	}()

	for {
		if DEBUG {
			printLatestDisturbance(rootSqalxNode)
			ld, err := MLlastDisturbanceTime(rootSqalxNode)
			if err != nil {
				mainLog.Println(err)
			}
			mainLog.Printf("Last disturbance: %s", ld.String())
		}
		time.Sleep(1 * time.Minute)
	}
}

func printLatestDisturbance(node sqalx.Node) {
	tx, err := node.Beginx()
	if err != nil {
		mainLog.Println(err)
		return
	}
	defer tx.Commit() // read-only tx

	n, err := dataobjects.GetNetwork(tx, MLnetworkID)
	if err != nil {
		mainLog.Println(err)
		return
	}
	d, err := n.LastDisturbance(tx)
	if err == nil {
		mainLog.Println("Network last disturbance at", d.StartTime, "description:", d.Description)
	} else {
		mainLog.Println(err)
	}
}
<commit_msg>Add delay before starting to compute time intervals<commit_after>package main

import (
	"log"
	"os"
	"time"

	fcm "github.com/NaySoftware/go-fcm"
	"github.com/SaidinWoT/timespan"
	"github.com/heetch/sqalx"
	"github.com/jmoiron/sqlx"

	sq "github.com/gbl08ma/squirrel"

	"github.com/gbl08ma/disturbancesmlx/dataobjects"
	"github.com/gbl08ma/keybox"
)

var (
	rdb           *sqlx.DB
	sdb           sq.StatementBuilderType
	rootSqalxNode sqalx.Node
	secrets       *keybox.Keybox
	fcmcl         *fcm.FcmClient
	mainLog       = log.New(os.Stdout, "", log.Ldate|log.Ltime)
	webLog        = log.New(os.Stdout, "web", log.Ldate|log.Ltime)
	lastChange    time.Time
)

// MLcalculator implements resource.StatsCalculator
type MLcalculator struct{}

func (*MLcalculator) Availability(node sqalx.Node, line *dataobjects.Line, startTime time.Time, endTime time.Time) (float64, time.Duration, error) {
	return MLlineAvailability(node, line, startTime, endTime)
}

// MLlastDisturbanceTime returns the time of the latest Metro de Lisboa disturbance
func MLlastDisturbanceTime(node sqalx.Node) (t time.Time, err error) {
	tx, err := node.Beginx()
	if err != nil {
		return time.Now().UTC(), err
	}
	defer tx.Commit() // read-only tx

	n, err := dataobjects.GetNetwork(tx, MLnetworkID)
	if err != nil {
		return time.Now().UTC(), err
	}
	d, err := n.LastDisturbance(tx)
	if err != nil {
		return time.Now().UTC(), err
	}

	if !d.Ended {
		return time.Now().UTC(), nil
	}

	return d.EndTime, nil
}

// MLlineAvailability returns the availability for a Metro de Lisboa line
func MLlineAvailability(node sqalx.Node, line *dataobjects.Line, startTime time.Time, endTime time.Time) (float64, time.Duration, error) {
	// calculate closed time
	var closedDuration time.Duration
	ct := startTime
	wholeSpan := timespan.New(startTime, endTime.Sub(startTime))
	for ct.Before(endTime) {
		closeTime := time.Date(ct.Year(), ct.Month(), ct.Day(), 1, 0, 0, 0, ct.Location())
		openTime := time.Date(ct.Year(), ct.Month(), ct.Day(), 6, 30, 0, 0, ct.Location())

		closedSpan := timespan.New(closeTime, openTime.Sub(closeTime))
		d, hasIntersection := wholeSpan.Intersection(closedSpan)
		if hasIntersection {
			closedDuration += d.Duration()
		}
		ct = ct.AddDate(0, 0, 1)
	}

	return line.Availability(node, startTime, endTime, closedDuration)
}

func main() {
	var err error
	mainLog.Println("Server starting, opening keybox...")
	secrets, err = keybox.Open(SecretsPath)
	if err != nil {
		mainLog.Fatal(err)
	}
	mainLog.Println("Keybox opened")

	mainLog.Println("Opening database...")
	databaseURI, present := secrets.Get("databaseURI")
	if !present {
		mainLog.Fatal("Database connection string not present in keybox")
	}
	rdb, err = sqlx.Open("postgres", databaseURI)
	if err != nil {
		mainLog.Fatal(err)
	}
	defer rdb.Close()

	err = rdb.Ping()
	if err != nil {
		mainLog.Fatal(err)
	}
	sdb = sq.StatementBuilder.PlaceholderFormat(sq.Dollar).RunWith(rdb)

	rootSqalxNode, err = sqalx.New(rdb)
	if err != nil {
		mainLog.Fatal(err)
	}

	mainLog.Println("Database opened")

	fcmServerKey, present := secrets.Get("firebaseServerKey")
	if !present {
		mainLog.Fatal("Firebase server key not present in keybox")
	}
	fcmcl = fcm.NewFcmClient(fcmServerKey)

	SetUpScrapers()
	defer TearDownScrapers()

	facebookAccessToken, present := secrets.Get("facebookToken")
	if !present {
		mainLog.Fatal("Facebook API access token not present in keybox")
	}

	SetUpAnnouncements(facebookAccessToken)
	defer TearDownAnnouncements()

	go WebServer()

	certPath := DefaultClientCertPath
	if len(os.Args) > 1 {
		certPath = os.Args[1]
	}
	go APIserver(certPath)

	go func() {
		time.Sleep(5 * time.Second)
		for {
			err := ComputeTypicalSeconds(rootSqalxNode)
			if err != nil {
				mainLog.Println(err)
			}
			time.Sleep(10 * time.Hour)
		}
	}()

	for {
		if DEBUG {
			printLatestDisturbance(rootSqalxNode)
			ld, err := MLlastDisturbanceTime(rootSqalxNode)
			if err != nil {
				mainLog.Println(err)
			}
			mainLog.Printf("Last disturbance: %s", ld.String())
		}
		time.Sleep(1 * time.Minute)
	}
}

func printLatestDisturbance(node sqalx.Node) {
	tx, err := node.Beginx()
	if err != nil {
		mainLog.Println(err)
		return
	}
	defer tx.Commit() // read-only tx

	n, err := dataobjects.GetNetwork(tx, MLnetworkID)
	if err != nil {
		mainLog.Println(err)
		return
	}
	d, err := n.LastDisturbance(tx)
	if err == nil {
		mainLog.Println("Network last disturbance at", d.StartTime, "description:", d.Description)
	} else {
		mainLog.Println(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"io/ioutil"
	"os"
	"regexp"
	"testing"
)

func TestGetRepoHash(t *testing.T) {
	if os.Getenv("TRAVIS") != "true" {
		t.Skip("skipping test on travis")
	}
	moduleConfig := ModuleConfig{
		Name:    "continuum",
		Url:     "git@github.com:c4s4/continuum.git",
		Branch:  "master",
		Command: "echo 'TEST'",
	}
	repoHash := GetRepoHash(moduleConfig)
	if match, _ := regexp.MatchString("^[0-9a-f]{40}$", repoHash); match != true {
		t.Errorf("GetRepoStatus() response '%s' doesn't look like a hash", repoHash)
	}
}

const testModulesInfo = `module1:
  repo-hash: dbe955d1d83ea4ec969656d1e002e25ca1382fd8
  build-ok: true
module2:
  repo-hash: c634c54781a89253167076ce102e588af8a60141
  build-ok: false
`

func TestLoadModulesInfo(t *testing.T) {
	tempFile, err := ioutil.TempFile("/tmp", "go-test-")
	if err != nil {
		panic(errors.New("Could not open temp file"))
	}
	_, err = tempFile.WriteString(testModulesInfo)
	if err != nil {
		panic(errors.New("Could not write temp file"))
	}
	defer os.Remove(tempFile.Name())
	modulesInfo := LoadModulesInfo(tempFile.Name())
	if modulesInfo["module1"].RepoHash != "dbe955d1d83ea4ec969656d1e002e25ca1382fd8" {
		t.Error("Bad repo hash")
	}
	if modulesInfo["module1"].BuildOK != true {
		t.Error("Bad build status")
	}
	if modulesInfo["module2"].RepoHash != "c634c54781a89253167076ce102e588af8a60141" {
		t.Error("Bad repo hash")
	}
	if modulesInfo["modules2"].BuildOK != false {
		t.Error("Bad build status")
	}
}

const TestModulesInfoFile = "/tmp/test-repo-hash.yml"

func TestSaveModulesInfo(t *testing.T) {
	modulesInfo := ModulesInfo{
		"module1": ModuleInfo{
			RepoHash: "dbe955d1d83ea4ec969656d1e002e25ca1382fd8",
			BuildOK:  true,
		},
		"module2": ModuleInfo{
			RepoHash: "c634c54781a89253167076ce102e588af8a60141",
			BuildOK:  false,
		},
	}
	SaveModulesInfo(modulesInfo, TestModulesInfoFile)
	defer os.Remove(TestModulesInfoFile)
	actual, _ := ioutil.ReadFile(TestModulesInfoFile)
	if string(actual) != testModulesInfo {
		t.Error("Error writing repo file")
	}
}
<commit_msg>Fixed test<commit_after>package main

import (
	"errors"
	"io/ioutil"
	"os"
	"regexp"
	"testing"
)

func TestGetRepoHash(t *testing.T) {
	if os.Getenv("TRAVIS") == "true" {
		t.Skip("skipping test on travis")
	}
	moduleConfig := ModuleConfig{
		Name:    "continuum",
		Url:     "git@github.com:c4s4/continuum.git",
		Branch:  "master",
		Command: "echo 'TEST'",
	}
	repoHash := GetRepoHash(moduleConfig)
	if match, _ := regexp.MatchString("^[0-9a-f]{40}$", repoHash); match != true {
		t.Errorf("GetRepoStatus() response '%s' doesn't look like a hash", repoHash)
	}
}

const testModulesInfo = `module1:
  repo-hash: dbe955d1d83ea4ec969656d1e002e25ca1382fd8
  build-ok: true
module2:
  repo-hash: c634c54781a89253167076ce102e588af8a60141
  build-ok: false
`

func TestLoadModulesInfo(t *testing.T) {
	tempFile, err := ioutil.TempFile("/tmp", "go-test-")
	if err != nil {
		panic(errors.New("Could not open temp file"))
	}
	_, err = tempFile.WriteString(testModulesInfo)
	if err != nil {
		panic(errors.New("Could not write temp file"))
	}
	defer os.Remove(tempFile.Name())
	modulesInfo := LoadModulesInfo(tempFile.Name())
	if modulesInfo["module1"].RepoHash != "dbe955d1d83ea4ec969656d1e002e25ca1382fd8" {
		t.Error("Bad repo hash")
	}
	if modulesInfo["module1"].BuildOK != true {
		t.Error("Bad build status")
	}
	if modulesInfo["module2"].RepoHash != "c634c54781a89253167076ce102e588af8a60141" {
		t.Error("Bad repo hash")
	}
	if modulesInfo["modules2"].BuildOK != false {
		t.Error("Bad build status")
	}
}

const TestModulesInfoFile = "/tmp/test-repo-hash.yml"

func TestSaveModulesInfo(t *testing.T) {
	modulesInfo := ModulesInfo{
		"module1": ModuleInfo{
			RepoHash: "dbe955d1d83ea4ec969656d1e002e25ca1382fd8",
			BuildOK:  true,
		},
		"module2": ModuleInfo{
			RepoHash: "c634c54781a89253167076ce102e588af8a60141",
			BuildOK:  false,
		},
	}
	SaveModulesInfo(modulesInfo, TestModulesInfoFile)
	defer os.Remove(TestModulesInfoFile)
	actual, _ := ioutil.ReadFile(TestModulesInfoFile)
	if string(actual) != testModulesInfo {
		t.Error("Error writing repo file")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"os/exec"
	"strings"

	"github.com/codegangsta/cli"
	"github.com/codegangsta/negroni"
	"github.com/davecgh/go-spew/spew"
	"github.com/gorilla/mux"
	logging "github.com/op/go-logging"
	"github.com/unrolled/render"
)

type GithubPushEventPayload struct {
	Hook struct {
		Config struct {
			Secret string `json:"secret"`
		} `json:"config"`
	} `json:"hook"`
	Repository struct {
		FullName string `json:"full_name"`
		HtmlUrl  string `json:"html_url"`
	} `json:"repository"`
}

var (
	log         = logging.MustGetLogger("streamLog")
	format      = "%{color}%{time:15:04:05} => %{color:reset} %{message}"
	buildStatus = map[string]string{}
)

func loadBuildStatus() error {
	return nil
}

func streamCommand(name string, args ...string) error {
	cmd := exec.Command(name, args...)
	cmdReader, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}
	scanner := bufio.NewScanner(cmdReader)
	go func() {
		for scanner.Scan() {
			log.Notice(scanner.Text())
		}
	}()
	err = cmd.Start()
	if err != nil {
		return err
	}
	err = cmd.Wait()
	if err != nil {
		return err
	}
	return nil
}

func BuildHookReceiver(c *cli.Context, r *render.Render, dockerBinary string) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, req *http.Request) {
		var (
			payload GithubPushEventPayload
		)
		decoder := json.NewDecoder(req.Body)
		err := decoder.Decode(&payload)
		if err != nil {
			fmt.Fprintln(os.Stderr, "Error decoding Github push payload:", err)
		}
		spew.Dump(payload)
		if c.String("secret") == "" || payload.Hook.Config.Secret == c.String("secret") {
			githubFullName := payload.Repository.FullName
			repoPath := fmt.Sprintf("./repos/%s", githubFullName)
			repoUrl := payload.Repository.HtmlUrl
			if _, err := os.Stat(repoPath); err != nil {
				if os.IsNotExist(err) {
					fmt.Println("Executing command", "git clone --recursive", repoUrl, repoPath)
					if err := exec.Command("git", "clone", "--recursive", repoUrl, repoPath).Run(); err != nil {
						fmt.Fprintln(os.Stderr, "Error cloning git repository:", err)
					}
				} else {
					fmt.Fprintln(os.Stderr, "Error stat-ing directory", repoPath, ":", err)
				}
				os.Chdir(repoPath)
			} else {
				os.Chdir(repoPath)
				fmt.Println("Pulling existing repository")
				if err := exec.Command("git", "pull").Run(); err != nil {
					fmt.Fprintln(os.Stderr, "Error pulling git repository:", err)
					r.JSON(w, http.StatusInternalServerError, "")
				}
			}

			namespacedImage := ""
			splitImage := strings.Split(githubFullName, "/")
			imageBase := splitImage[len(splitImage)-1]
			if c.String("hub-name") == "" {
				if c.String("alt-registry") == "" {
					namespacedImage = githubFullName
				} else {
					namespacedImage = fmt.Sprintf("%s/%s", c.String("alt-registry"), imageBase)
				}
			} else {
				namespacedImage = fmt.Sprintf("%s/%s", c.String("hub-name"), imageBase)
			}

			fmt.Println("Building docker image")
			err := streamCommand(dockerBinary, "build", "-t", namespacedImage, ".")
			if err != nil {
				fmt.Fprintln(os.Stderr, "Error building docker image for", namespacedImage, ":", err)
				r.JSON(w, http.StatusInternalServerError, map[string]interface{}{
					"Error": err,
				})
			}

			registryName := ""
			if c.String("alt-registry") != "" {
				registryName = c.String("alt-registry")
			} else {
				registryName = "Docker Hub"
			}

			fmt.Println(fmt.Sprintf("Pushing image back to specified registry (%s)...", registryName))
			err = streamCommand(dockerBinary, "push", namespacedImage)
			if err != nil {
				fmt.Fprintln(os.Stderr, "Error pushing docker image for", namespacedImage, ":", err)
				r.JSON(w, http.StatusInternalServerError, map[string]interface{}{
					"Error": err,
				})
			}
		} else {
			r.JSON(w, http.StatusInternalServerError, map[string]interface{}{
				"Error": "Secret from payload was invalid",
			})
		}
		r.JSON(w, http.StatusOK, "")
	}
}

func StatusHandler(w http.ResponseWriter, req *http.Request) {

}

func main() {
	logBackend := logging.NewLogBackend(os.Stderr, "", 0)
	logging.SetBackend(logBackend)
	logging.SetFormatter(logging.MustStringFormatter(format))
	logging.SetLevel(logging.NOTICE, "streamLog")
	homeDir := os.Getenv("HOME")

	app := cli.NewApp()
	app.Name = "tarzan"
	app.Usage = "naive cached automated build implementation"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "p,port",
			Value: "3000",
			Usage: "port to serve tarzan on",
		},
		cli.StringFlag{
			Name:  "alt-registry",
			Value: "",
			Usage: "alternative registry to push images to instead of Docker Hub",
		},
		cli.StringFlag{
			Name:  "secret",
			Value: "",
			Usage: "secret to use when receiving webhook payload",
		},
		cli.StringFlag{
			Name:  "hub-name",
			Value: "",
			Usage: "specify a username on Docker Hub which is different than your Github handle",
		},
		cli.StringFlag{
			Name:  "docker-binary-name",
			Value: "docker",
			Usage: "specify the docker binary name (if it is not docker in $PATH)",
		},
	}

	app.Action = func(c *cli.Context) {
		dockerBinary := c.String("docker-binary-name")
		if err := loadBuildStatus(); err != nil {
			fmt.Fprintln(os.Stderr, "Error loading build status (did your repos/ dir get corrupted?):", err)
			os.Exit(1)
		}
		if _, err := os.Stat(fmt.Sprintf("%s/.dockercfg", homeDir)); err != nil {
			if os.IsNotExist(err) {
				log.Warning("Detected no Docker Hub login.  Please log in now.")
				cmd := exec.Command(dockerBinary, "login")
				cmd.Stdin = os.Stdin
				cmd.Stdout = os.Stdout
				cmd.Stderr = os.Stderr
				if err := cmd.Run(); err != nil {
					fmt.Fprintln(os.Stderr, "Error running docker login")
					os.Exit(1)
				}
			}
		}
		r := render.New(render.Options{})
		router := mux.NewRouter()
		router.HandleFunc("/build", BuildHookReceiver(c, r, dockerBinary)).Methods("POST")
		router.HandleFunc("/buildList", StatusHandler).Methods("GET")

		n := negroni.Classic()
		n.Use(negroni.NewStatic(http.Dir("frontend/")))
		n.UseHandler(router)
		n.Run(fmt.Sprintf(":%s", c.String("port")))
	}

	app.Run(os.Args)
}
<commit_msg>Wrap in closure<commit_after>package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"os/exec"
	"strings"

	"github.com/codegangsta/cli"
	"github.com/codegangsta/negroni"
	"github.com/davecgh/go-spew/spew"
	"github.com/gorilla/mux"
	logging "github.com/op/go-logging"
	"github.com/unrolled/render"
)

type GithubPushEventPayload struct {
	Hook struct {
		Config struct {
			Secret string `json:"secret"`
		} `json:"config"`
	} `json:"hook"`
	Repository struct {
		FullName string `json:"full_name"`
		HtmlUrl  string `json:"html_url"`
	} `json:"repository"`
}

var (
	log         = logging.MustGetLogger("streamLog")
	format      = "%{color}%{time:15:04:05} => %{color:reset} %{message}"
	buildStatus = map[string]string{}
)

func loadBuildStatus() error {
	return nil
}

func streamCommand(name string, args ...string) error {
	cmd := exec.Command(name, args...)
	cmdReader, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}
	scanner := bufio.NewScanner(cmdReader)
	go func() {
		for scanner.Scan() {
			log.Notice(scanner.Text())
		}
	}()
	err = cmd.Start()
	if err != nil {
		return err
	}
	err = cmd.Wait()
	if err != nil {
		return err
	}
	return nil
}

func BuildHookReceiver(c *cli.Context, r *render.Render, dockerBinary string) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, req *http.Request) {
		var (
			payload GithubPushEventPayload
		)
		decoder := json.NewDecoder(req.Body)
		err := decoder.Decode(&payload)
		if err != nil {
			fmt.Fprintln(os.Stderr, "Error decoding Github push payload:", err)
		}
		spew.Dump(payload)
		if c.String("secret") == "" || payload.Hook.Config.Secret == c.String("secret") {
			githubFullName := payload.Repository.FullName
			repoPath := fmt.Sprintf("./repos/%s", githubFullName)
			repoUrl := payload.Repository.HtmlUrl
			if _, err := os.Stat(repoPath); err != nil {
				if os.IsNotExist(err) {
					fmt.Println("Executing command", "git clone --recursive", repoUrl, repoPath)
					if err := exec.Command("git", "clone", "--recursive", repoUrl, repoPath).Run(); err != nil {
						fmt.Fprintln(os.Stderr, "Error cloning git repository:", err)
					}
				} else {
					fmt.Fprintln(os.Stderr, "Error stat-ing directory", repoPath, ":", err)
				}
				os.Chdir(repoPath)
			} else {
				os.Chdir(repoPath)
				fmt.Println("Pulling existing repository")
				if err := exec.Command("git", "pull").Run(); err != nil {
					fmt.Fprintln(os.Stderr, "Error pulling git repository:", err)
					r.JSON(w, http.StatusInternalServerError, "")
				}
			}

			namespacedImage := ""
			splitImage := strings.Split(githubFullName, "/")
			imageBase := splitImage[len(splitImage)-1]
			if c.String("hub-name") == "" {
				if c.String("alt-registry") == "" {
					namespacedImage = githubFullName
				} else {
					namespacedImage = fmt.Sprintf("%s/%s", c.String("alt-registry"), imageBase)
				}
			} else {
				namespacedImage = fmt.Sprintf("%s/%s", c.String("hub-name"), imageBase)
			}

			fmt.Println("Building docker image")
			err := streamCommand(dockerBinary, "build", "-t", namespacedImage, ".")
			if err != nil {
				fmt.Fprintln(os.Stderr, "Error building docker image for", namespacedImage, ":", err)
				r.JSON(w, http.StatusInternalServerError, map[string]interface{}{
					"Error": err,
				})
			}

			registryName := ""
			if c.String("alt-registry") != "" {
				registryName = c.String("alt-registry")
			} else {
				registryName = "Docker Hub"
			}

			fmt.Println(fmt.Sprintf("Pushing image back to specified registry (%s)...", registryName))
			err = streamCommand(dockerBinary, "push", namespacedImage)
			if err != nil {
				fmt.Fprintln(os.Stderr, "Error pushing docker image for", namespacedImage, ":", err)
				r.JSON(w, http.StatusInternalServerError, map[string]interface{}{
					"Error": err,
				})
			}
		} else {
			r.JSON(w, http.StatusInternalServerError, map[string]interface{}{
				"Error": "Secret from payload was invalid",
			})
		}
		r.JSON(w, http.StatusOK, "")
	}
}

func MakeBuildListHandler(r *render.Render) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, req *http.Request) {

	}
}

func main() {
	logBackend := logging.NewLogBackend(os.Stderr, "", 0)
	logging.SetBackend(logBackend)
	logging.SetFormatter(logging.MustStringFormatter(format))
	logging.SetLevel(logging.NOTICE, "streamLog")
	homeDir := os.Getenv("HOME")

	app := cli.NewApp()
	app.Name = "tarzan"
	app.Usage = "naive cached automated build implementation"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "p,port",
			Value: "3000",
			Usage: "port to serve tarzan on",
		},
		cli.StringFlag{
			Name:  "alt-registry",
			Value: "",
			Usage: "alternative registry to push images to instead of Docker Hub",
		},
		cli.StringFlag{
			Name:  "secret",
			Value: "",
			Usage: "secret to use when receiving webhook payload",
		},
		cli.StringFlag{
			Name:  "hub-name",
			Value: "",
			Usage: "specify a username on Docker Hub which is different than your Github handle",
		},
		cli.StringFlag{
			Name:  "docker-binary-name",
			Value: "docker",
			Usage: "specify the docker binary name (if it is not docker in $PATH)",
		},
	}

	app.Action = func(c *cli.Context) {
		dockerBinary := c.String("docker-binary-name")
		if err := loadBuildStatus(); err != nil {
			fmt.Fprintln(os.Stderr, "Error loading build status (did your repos/ dir get corrupted?):", err)
			os.Exit(1)
		}
		if _, err := os.Stat(fmt.Sprintf("%s/.dockercfg", homeDir)); err != nil {
			if os.IsNotExist(err) {
				log.Warning("Detected no Docker Hub login.  Please log in now.")
				cmd := exec.Command(dockerBinary, "login")
				cmd.Stdin = os.Stdin
				cmd.Stdout = os.Stdout
				cmd.Stderr = os.Stderr
				if err := cmd.Run(); err != nil {
					fmt.Fprintln(os.Stderr, "Error running docker login")
					os.Exit(1)
				}
			}
		}
		r := render.New(render.Options{})
		router := mux.NewRouter()
		router.HandleFunc("/build", BuildHookReceiver(c, r, dockerBinary)).Methods("POST")
		router.HandleFunc("/buildList", MakeBuildListHandler(r)).Methods("GET")

		n := negroni.Classic()
		n.Use(negroni.NewStatic(http.Dir("frontend/")))
		n.UseHandler(router)
		n.Run(fmt.Sprintf(":%s", c.String("port")))
	}

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"time"

	"github.com/codegangsta/cli"
	"gopkg.in/olivere/elastic.v3"
)

func main() {
	app := cli.NewApp()
	app.Name = "eless"
	app.Usage = "cli for elasticsearch"

	var url string
	var prefix string
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:        "url, u",
			Value:       "http://127.0.0.1:9200",
			Usage:       "elasticsearch server url",
			Destination: &url,
		},
		cli.StringFlag{
			Name:        "prefix, p",
			Value:       "logstash-",
			Usage:       "indices prefix",
			Destination: &prefix,
		},
		cli.StringFlag{
			Name:  "separator, s",
			Value: " ",
			Usage: "output separator",
		},
		cli.StringSliceFlag{
			Name:  "field, f",
			Value: &cli.StringSlice{"@timestamp", "message"},
			Usage: "fields to return",
		},
		cli.StringSliceFlag{
			Name:  "date, d",
			Usage: "dates to return",
		},
	}

	app.Action = func(c *cli.Context) {
		client, err := elastic.NewSimpleClient(elastic.SetURL(url))
		if err != nil {
			panic(err)
		}

		dates := c.StringSlice("date")
		if dates == nil || len(dates) == 0 {
			dates = []string{currentDate()}
		}

		indices := make([]string, len(dates))
		for i, date := range dates {
			indices[i] = prefix + date
		}

		fields := c.StringSlice("field")

		searchResult, err := client.Search().
			Index(indices...).
			Sort("@timestamp", false).
			Sort("offset", false).
			Fields(fields...).
			From(0).
			Size(10).
			Do()

		if err != nil {
			panic(err)
		}

		if searchResult.Hits != nil {
			for _, hit := range searchResult.Hits.Hits {
				for _, field := range fields {
					fmt.Print(hit.Fields[field].(string))
					fmt.Print(c.String("separator"))
				}
				fmt.Print("\n")
			}
		}
	}

	app.Run(os.Args)
}

func currentDate() string {
	return time.Now().Format("2006.01.02")
}
<commit_msg>It works!<commit_after>package main

import (
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/codegangsta/cli"
	"gopkg.in/olivere/elastic.v3"
)

const defaultQuerySize = 1000

func main() {
	app := cli.NewApp()
	app.Name = "eless"
	app.Usage = "cli for elasticsearch"

	var url string
	var prefix string
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:        "url, u",
			Value:       "http://127.0.0.1:9200",
			Usage:       "elasticsearch server url",
			Destination: &url,
		},
		cli.StringFlag{
			Name:        "prefix, p",
			Value:       "logstash-",
			Usage:       "indices prefix",
			Destination: &prefix,
		},
		cli.StringFlag{
			Name:  "separator, s",
			Value: " ",
			Usage: "output separator",
		},
		cli.StringSliceFlag{
			Name:  "field, f",
			Value: &cli.StringSlice{"@timestamp", "message"},
			Usage: "fields to return",
		},
		cli.StringSliceFlag{
			Name:  "date, d",
			Usage: "dates to return",
		},
		cli.StringSliceFlag{
			Name:  "term, t",
			Usage: "define term query, example: FIELD:TERM",
		},
	}

	app.Action = func(c *cli.Context) {
		client, err := elastic.NewSimpleClient(elastic.SetURL(url))
		if err != nil {
			panic(err)
		}

		dates := c.StringSlice("date")
		if dates == nil || len(dates) == 0 {
			dates = []string{yesterdayDate(), currentDate()}
		}

		indices := make([]string, len(dates))
		for i, date := range dates {
			indices[i] = prefix + date
		}

		fields := c.StringSlice("field")

		queriesArray := c.StringSlice("term")
		globalQuery := elastic.NewBoolQuery()
		for _, queryString := range queriesArray {
			queryStringArray := strings.Split(queryString, ":")
			field, value := queryStringArray[0], queryStringArray[1]
			query := elastic.NewMatchPhraseQuery(field, value)
			globalQuery.Must(query)
		}

		from := 0
		for processPortion(client, indices, globalQuery, fields, c.String("separator"), from) {
			from += defaultQuerySize
		}
	}

	app.Run(os.Args)
}

func processPortion(client *elastic.Client, indices []string, globalQuery *elastic.BoolQuery, fields []string, separator string, from int) bool {
	searchResult, err := client.Search(indices...).
		Query(globalQuery).
		Sort("@timestamp", true).
		Sort("offset", true).
		Fields(fields...).
		From(from).
		Size(defaultQuerySize).
		Do()

	if err != nil {
		panic(err)
	}

	if searchResult.Hits != nil {
		if int64(from) > searchResult.TotalHits() {
			return false
		}
		for _, hit := range searchResult.Hits.Hits {
			for _, field := range fields {
				for _, fieldValue := range hit.Fields[field].([]interface{}) {
					fmt.Print(fieldValue)
				}
				fmt.Print(separator)
			}
			fmt.Print("\n")
		}
		return true
	}

	return false
}

func currentDate() string {
	return time.Now().Format("2006.01.02")
}

func yesterdayDate() string {
	return time.Now().AddDate(0, 0, -1).Format("2006.01.02")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/olebedev/config"
	"io/ioutil"
	"net/http"
)

func loadConfig(configFile string) (string, error) {
	configData, err := ioutil.ReadFile(configFile)
	if nil != err {
		return "", err
	}
	appConfig, err := config.ParseYaml(string(configData))
	if nil != err {
		return "", err
	}
	value, err := appConfig.String("services.0.url")
	if nil != err {
		return "", err
	}

	return value, err;
}

func handler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "echo is called: %s", r.URL.Path[1:])
}

func startServer() {
	http.HandleFunc("/echo", handler)
	err := http.ListenAndServe(":8080", nil)
	if nil != err {
		fmt.Println(err)
	}
}

func main() {
	configFile := "./config/shervice.yaml"
	value, err := loadConfig(configFile)
	if nil != err {
		fmt.Println(err)
	}

	fmt.Println(value)

	startServer()
}
<commit_msg>+exec command<commit_after>package main

import (
	"fmt"
	"github.com/olebedev/config"
	"io/ioutil"
	"net/http"
	"os/exec"
	"log"
)

func loadConfig(configFile string) (string, error) {
	configData, err := ioutil.ReadFile(configFile)
	if nil != err {
		return "", err
	}
	appConfig, err := config.ParseYaml(string(configData))
	if nil != err {
		return "", err
	}
	value, err := appConfig.String("services.0.url")
	if nil != err {
		return "", err
	}

	return value, err
}

func handler(w http.ResponseWriter, r *http.Request) {
	cmd := exec.Command("/bin/echo", "wow")
	out, err := cmd.Output()
	if err != nil {
		log.Fatal(err)
	}
	fmt.Fprint(w, string(out))
}

func startServer() {
	http.HandleFunc("/echo", handler)
	err := http.ListenAndServe(":8080", nil)
	if nil != err {
		fmt.Println(err)
	}
}

func main() {
	configFile := "./config/shervice.yaml"
	value, err := loadConfig(configFile)
	if nil != err {
		fmt.Println(err)
	}

	fmt.Println(value)

	startServer()
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/gob"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/user"
	"path/filepath"
	"time"

	"golang.org/x/oauth2"
)

type credentials struct {
	ClientID     string `json:"clientID"`
	ClientSecret string `json:"clientSecret"`
}

func main() {
	path := filepath.Join(userDir(), ".config", "youtube-emacs-search")

	var file *os.File
	var err error

	// Get value of last update from some file => ~/.config/youtube-emacs
	file, err = os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666)
	defer file.Close()

	t := time.Now()
	// Subtract 2 days from current date
	twoDaysAgo := t.AddDate(0, 0, -2)

	if err == nil {
		fmt.Printf("%s does not exist! Creating it...\n", path)
		file.WriteString(twoDaysAgo.Format(time.RFC3339))
	} else {
		file, err = os.OpenFile(path, os.O_RDWR, 0666)
		check(err)
	}

	cred := loadOauthCredentials()

	if cred.ClientID == "YOUR-CLIENTID" || cred.ClientSecret == "YOUR-CLIENTSECRET" {
		fmt.Println("Please setup your YouTube OAuth credentials.")
		os.Exit(1)
	}

	conf := &oauth2.Config{
		ClientID:     cred.ClientID,
		ClientSecret: cred.ClientSecret,
		Scopes:       []string{"https://www.googleapis.com/auth/youtube.readonly"},
		RedirectURL:  "urn:ietf:wg:oauth:2.0:oob",
		Endpoint: oauth2.Endpoint{
			AuthURL:  "https://accounts.google.com/o/oauth2/auth",
			TokenURL: "https://accounts.google.com/o/oauth2/token",
		},
	}

	tok, err := loadToken()

	if err != nil {
		url := conf.AuthCodeURL("state", oauth2.AccessTypeOffline)
		fmt.Printf("Visit the URL for the auth dialog then come back here and paste the token:\n%v\n\n", url)

		var code string
		if _, err := fmt.Scan(&code); err != nil {
			check(err)
		}

		tok, err = conf.Exchange(oauth2.NoContext, code)
		if err != nil {
			check(err)
		}

		saveToken(tok)
	}

	client := conf.Client(oauth2.NoContext, tok)

	// Construct URL to query with value of last update
	resp, err := client.Get("https://www.googleapis.com/youtube/v3/search?part=snippet&order=date&publishedAfter=2015-02-17T00%3A00%3A00Z&q=emacs&type=video&maxResults=50")
	check(err)

	defer resp.Body.Close()

	htmlData, err := ioutil.ReadAll(resp.Body)
	check(err)

	// If everything went fine write the current time into update value
	file.WriteString(twoDaysAgo.Format(time.RFC3339))

	// Show the result of the search
	// fmt.Println(string(htmlData))
	ppJSON(htmlData)

	// Send results via email
}

func ppJSON(data []byte) {
	var dat map[string]interface{}

	if err := json.Unmarshal(data, &dat); err != nil {
		panic(err)
	}

	b, err := json.MarshalIndent(dat, "", "  ")
	if err != nil {
		log.Println(err)
	}
	fmt.Printf("%s\n", b)
}

func check(e error) {
	if e != nil {
		panic(e)
	}
}

func userDir() (userDir string) {
	currentUser, _ := user.Current()
	userDir = currentUser.HomeDir
	return
}

func loadOauthCredentials() (cred credentials) {
	credentialsFile, err := os.Open(filepath.Join(userDir(), ".config", "youtube-oauth-credentials"))
	defer credentialsFile.Close()

	if err != nil {
		fmt.Println("Please add your YouTube API credentials to ~/.config/youtube-oauth-credentials")
		os.Exit(1)
	}

	dec := json.NewDecoder(credentialsFile)
	err = dec.Decode(&cred)
	check(err)

	return
}

func saveToken(token *oauth2.Token) {
	tokenFile, err := os.Create(filepath.Join(userDir(), ".config", "youtube-oauth-token"))
	defer tokenFile.Close()
	check(err)

	tokenEncoder := gob.NewEncoder(tokenFile)
	tokenEncoder.Encode(token)
}

func loadToken() (token *oauth2.Token, err error) {
	tokenFile, err := os.Open(filepath.Join(userDir(), ".config", "youtube-oauth-token"))
	defer tokenFile.Close()

	if err != nil {
		return nil, err
	}

	tokenDecoder := gob.NewDecoder(tokenFile)
	err = tokenDecoder.Decode(&token)
	check(err)
	return
}
<commit_msg>Put config files into subdirectory.<commit_after>package main

import (
	"encoding/gob"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/user"
	"path/filepath"
	"time"

	"golang.org/x/oauth2"
)

type credentials struct {
	ClientID     string `json:"clientID"`
	ClientSecret string `json:"clientSecret"`
}

var configPath = filepath.Join(userDir(), ".config", "youtube-emacs-search")

func main() {

	dirInfo, err := os.Stat(configPath)

	if err != nil || !dirInfo.IsDir() {
		fmt.Println("There's no config directory. Creating it now...")
		if err = os.MkdirAll(configPath, 0700); err != nil {
			fmt.Println("Something went wrong while creating the config directory")
			os.Exit(1)
		}
	}

	path := filepath.Join(configPath, "timestamp")

	var file *os.File

	file, err = os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666)
	defer file.Close()

	t := time.Now()
	// Subtract 2 days from current date
	twoDaysAgo := t.AddDate(0, 0, -2)

	if err == nil {
		fmt.Printf("%s does not exist! Creating it...\n", path)
		file.WriteString(twoDaysAgo.Format(time.RFC3339))
	} else {
		file, err = os.OpenFile(path, os.O_RDWR, 0666)
		check(err)
	}

	cred := loadOauthCredentials()

	if cred.ClientID == "YOUR-CLIENTID" || cred.ClientSecret == "YOUR-CLIENTSECRET" {
		fmt.Println("Please setup your YouTube OAuth credentials.")
		os.Exit(1)
	}

	conf := &oauth2.Config{
		ClientID:     cred.ClientID,
		ClientSecret: cred.ClientSecret,
		Scopes:       []string{"https://www.googleapis.com/auth/youtube.readonly"},
		RedirectURL:  "urn:ietf:wg:oauth:2.0:oob",
		Endpoint: oauth2.Endpoint{
			AuthURL:  "https://accounts.google.com/o/oauth2/auth",
			TokenURL: "https://accounts.google.com/o/oauth2/token",
		},
	}

	tok, err := loadToken()

	if err != nil {
		url := conf.AuthCodeURL("state", oauth2.AccessTypeOffline)
		fmt.Printf("Visit the URL for the auth dialog then come back here and paste the token:\n%v\n\n", url)

		var code string
		if _, err := fmt.Scan(&code); err != nil {
			check(err)
		}

		tok, err = conf.Exchange(oauth2.NoContext, code)
		if err != nil {
			check(err)
		}

		saveToken(tok)
	}

	client := conf.Client(oauth2.NoContext, tok)

	// Construct URL to query with value of last update
	resp, err := client.Get("https://www.googleapis.com/youtube/v3/search?part=snippet&order=date&publishedAfter=2015-02-17T00%3A00%3A00Z&q=emacs&type=video&maxResults=50")
	check(err)

	defer resp.Body.Close()

	htmlData, err := ioutil.ReadAll(resp.Body)
	check(err)

	// If everything went fine write the current time into update value
	file.WriteString(twoDaysAgo.Format(time.RFC3339))

	// Show the result of the search
	// fmt.Println(string(htmlData))
	ppJSON(htmlData)

	// Send results via email
}

func ppJSON(data []byte) {
	var dat map[string]interface{}

	if err := json.Unmarshal(data, &dat); err != nil {
		panic(err)
	}

	b, err := json.MarshalIndent(dat, "", "  ")
	if err != nil {
		log.Println(err)
	}
	fmt.Printf("%s\n", b)
}

func check(e error) {
	if e != nil {
		panic(e)
	}
}

func userDir() (userDir string) {
	currentUser, _ := user.Current()
	userDir = currentUser.HomeDir
	return
}

func loadOauthCredentials() (cred credentials) {
	credentialsFile, err := os.Open(filepath.Join(configPath, "youtube-oauth-credentials"))
	defer credentialsFile.Close()

	if err != nil {
		fmt.Printf("Please add your YouTube API credentials to %s\n", filepath.Join(configPath, "youtube-oauth-credentials"))
		os.Exit(1)
	}

	dec := json.NewDecoder(credentialsFile)
	err = dec.Decode(&cred)
	check(err)

	return
}

func saveToken(token *oauth2.Token) {
	tokenFile, err := os.Create(filepath.Join(configPath, "youtube-oauth-token"))
	defer tokenFile.Close()
	check(err)

	tokenEncoder := gob.NewEncoder(tokenFile)
	tokenEncoder.Encode(token)
}

func loadToken() (token *oauth2.Token, err error) {
	tokenFile, err := os.Open(filepath.Join(configPath, "youtube-oauth-token"))
	defer tokenFile.Close()

	if err != nil {
		return nil, err
	}

	tokenDecoder := gob.NewDecoder(tokenFile)
	err = tokenDecoder.Decode(&token)
	check(err)
	return
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"github.com/tomsteele/blacksheepwall/bsw"
	"log"
	"net"
	"os"
	"sort"
	"text/tabwriter"
)

var usage = `
  Usage: blacksheepwall [options] <ip address or CIDR>

  Options:
  -h, --help            Show Usage and exit.
  -version              Show version and exit.
  -debug                Enable debugging and show errors returned from tasks.
  -concurrency <int>    Max amount of concurrent tasks.    [default: 100]
  -server <string>      DNS server address.    [default: "8.8.8.8"]
  -input <string>       Line separated file of networks (CIDR) or
                        IP Addresses.
  -ipv6	                Look for additional AAAA records where applicable.
  -domain <string>      Target domain to use for certain tasks.
  -dictionary <string>  Attempt to retrieve the CNAME and A record for
                        each subdomain in the line separated file.
  -yandex <string>      Provided a Yandex search XML API url. Use the Yandex
                        search 'rhost:' operator to find subdomains of a
                        provided domain..
  -bing	<string>        Provided a base64 encoded API key. Use the Bing search
                        API's 'ip:' operator to lookup hostnames for each host.
  -headers              Perform HTTP(s) requests to each host and look for
                        hostnames in a possible Location header.
  -reverse              Retrieve the PTR for each host.
  -tls                  Attempt to retrieve names from TLS certificates
                        (CommonName and Subject Alternative Name).
  -viewdns              Lookup each host using viewdns.info's Reverse IP
                        Lookup function.
  -fcrdns               Verify results by attempting to retrieve the A or AAAA record for
                        each result previously identified hostname.
  -clean                Print results as unique hostnames for each host.
  -csv                  Print results in csv format.
  -json                 Print results as JSON.

`

// Returns all ip addresses from each CIDR range in a list.
func linesToIpList(lines []string) ([]string, error) {
	ipList := make([]string, 0)
	for _, line := range lines {
		if net.ParseIP(line) != nil {
			ipList = append(ipList, line)
		} else if ip, network, err := net.ParseCIDR(line); err == nil {
			for ip := ip.Mask(network.Mask); network.Contains(ip); increaseIp(ip) {
				ipList = append(ipList, ip.String())
			}
		} else {
			return ipList, errors.New("\"" + line + "\" is not an IP Address or CIDR Network")
		}
	}
	return ipList, nil
}

// Increases IP by a single address
func increaseIp(ip net.IP) {
	for j := len(ip) - 1; j >= 0; j-- {
		ip[j]++
		if ip[j] > 0 {
			break
		}
	}
}

// Reads lines from a file and returns as a slice.
func readFileLines(path string) ([]string, error) {
	file, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	lines := make([]string, 0)
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		lines = append(lines, scanner.Text())
	}
	return lines, scanner.Err()
}

type task func() (bsw.Results, error)
type empty struct{}

func main() {
	var (
		flVersion     = flag.Bool("version", false, "Show version and exit.")
		flConcurrency = flag.Int("concurrency", 100, "Max amount of concurrent tasks.")
		flDebug       = flag.Bool("debug", false, "Enable debugging and show errors returned from tasks.")
		flipv6        = flag.Bool("ipv6", false, "Look for AAAA records where applicable.")
		flServerAddr  = flag.String("server", "8.8.8.8", "DNS server address.")
		flIpFile      = flag.String("input", "", "Line separated file of networks (CIDR) or IP Addresses.")
		flReverse     = flag.Bool("reverse", false, "Retrieve the PTR for each host.")
		flHeader      = flag.Bool("headers", false, "Perform HTTP(s) requests to each host and look for hostnames in a possible Location header.")
		flTLS         = flag.Bool("tls", false, "Attempt to retrieve names from TLS certificates (CommonName and Subject Alternative Name).")
		flViewDnsInfo = flag.Bool("viewdns", false, "Lookup each host using viewdns.info's Reverse IP Lookup function.")
		flBing        = flag.String("bing", "", "Provided a base64 encoded API key. Use the Bing search API's 'ip:' operator to lookup hostnames for each host.")
		flYandex      = flag.String("yandex", "", "Provided a Yandex search XML API url. Use the Yandex search 'rhost:' operator to find subdomains of a provided domain.")
		flDomain      = flag.String("domain", "", "Target domain to use for certain tasks.")
		flDictFile    = flag.String("dictionary", "", "Attempt to retrieve the CNAME and A record for each subdomain in the line separated file.")
		flFcrdns      = flag.Bool("fcrdns", false, "Verify results by attempting to retrieve the A or AAAA record for each result previously identified hostname.")
		flClean       = flag.Bool("clean", false, "Print results as unique hostnames for each host.")
		flCsv         = flag.Bool("csv", false, "Print results in csv format.")
		flJson        = flag.Bool("json", false, "Print results as JSON.")
	)
	flag.Usage = func() { fmt.Print(usage) }
	flag.Parse()

	if *flVersion {
		fmt.Println("blacksheepwall version ", bsw.VERSION)
		os.Exit(0)
	}

	// Holds all ip addresses for testing
	ipAddrList := make([]string, 0)

	// Used to hold a ip or CIDR range passed as fl.Arg(0)
	var flNetwork string

	// Verify that some sort of work load was given in commands
	if *flIpFile == "" && *flDomain == "" && len(flag.Args()) < 1 {
		log.Fatal("You didn't provide any work for me to do")
	}
	if *flYandex != "" && *flDomain == "" {
		log.Fatal("Yandex API requires domain set with -domain")
	}
	if *flDictFile != "" && *flDomain == "" {
		log.Fatal("Dictionary lookup requires domain set with -domain")
	}
	if *flDomain != "" && *flYandex == "" && *flDictFile == "" {
		log.Fatal("-domain provided but no methods provided that use it")
	}

	// Get first argument that is not an option and turn it into a list of ips
	if len(flag.Args()) > 0 {
		flNetwork = flag.Arg(0)
		list, err := linesToIpList([]string{flNetwork})
		if err != nil {
			log.Fatal(err.Error())
		}
		ipAddrList = append(ipAddrList, list...)
	}

	// If file given as -input, read lines and turn each possible ip or network into
	// a list of ips. Appends list to ipAddrList. Will fail fatally if line in file
	// is not a valid ip or CIDR range.
	if *flIpFile != "" {
		lines, err := readFileLines(*flIpFile)
		if err != nil {
			log.Fatal("Error reading " + *flIpFile + " " + err.Error())
		}
		list, err := linesToIpList(lines)
		if err != nil {
			log.Fatal(err.Error())
		}
		ipAddrList = append(ipAddrList, list...)
	}

	// tracker: Chanel uses an empty struct to track when all goroutines in the pool
	//          have completed as well as a single call from the gatherer.
	//
	// tasks:   Chanel used in the goroutine pool to manage incoming work. A task is
	//          a function wrapper that returns a slice of results and a possible error.
	//
	// res:     When each task is called in the pool, it will send valid results to
	//          the res channel. A goroutine manages this channel and appends results
	//          to results slice.
	tracker := make(chan empty)
	tasks := make(chan task, *flConcurrency)
	res := make(chan bsw.Results, *flConcurrency)
	results := bsw.Results{}

	// Start up *flConcurrency amount of goroutines
	log.Printf("Spreading tasks across %d goroutines", *flConcurrency)
	for i := 0; i < *flConcurrency; i++ {
		go func() {
			var c = 0
			for def := range tasks {
				result, err := def()
				if m := c % 2; m == 0 {
					c = 3
					os.Stderr.WriteString("\rWorking \\")
				} else {
					c = 2
					os.Stderr.WriteString("\rWorking /")
				}
				if err != nil && *flDebug {
					log.Println(err.Error())
				}
				if err == nil {
					res <- result
				}
			}
			tracker <- empty{}
		}()
	}

	// Ingest incoming results
	go func() {
		for result := range res {
			if len(result) < 1 {
				continue
			}
			if *flFcrdns {
				for _, r := range result {
					ip, err := bsw.LookupName(r.Hostname, *flServerAddr)
					if err == nil && len(ip) > 0 {
						results = append(results, bsw.Result{Source: "fcrdns", IP: ip, Hostname: r.Hostname})
					}
					ip, err = bsw.LookupName6(r.Hostname, *flServerAddr)
					if err == nil && len(ip) > 0 {
						results = append(results, bsw.Result{Source: "fcrdns", IP: ip, Hostname: r.Hostname})
					}
				}
			} else {
				results = append(results, result...)
			}
		}
		e := empty{}
		tracker <- e
	}()

	// Bing has two possible search paths. We need to find which one is valid.
	var bingPath string
	if *flBing != "" {
		p, err := bsw.FindBingSearchPath(*flBing)
		if err != nil {
			log.Fatal(err.Error())
		}
		bingPath = p
	}

	// ip based functionality should be added to the pool here
	for _, h := range ipAddrList {
		host := h
		if *flReverse {
			tasks <- func() (bsw.Results, error) {
				return bsw.Reverse(host, *flServerAddr)
			}
		}
		if *flTLS {
			tasks <- func() (bsw.Results, error) {
				return bsw.TLS(host)
			}
		}
		if *flViewDnsInfo {
			tasks <- func() (bsw.Results, error) {
				return bsw.ViewDnsInfo(host)
			}
		}
		if *flBing != "" && bingPath != "" {
			tasks <- func() (bsw.Results, error) {
				return bsw.BingAPI(host, *flBing, bingPath)
			}
		}
		if *flHeader {
			tasks <- func() (bsw.Results, error) {
				return bsw.Headers(host)
			}
		}

	}

	// Domain based functions will likely require separate blocks

	// Subdomain dictionary guessing
	if *flDictFile != "" && *flDomain != "" {
		nameList, err := readFileLines(*flDictFile)
		if err != nil {
			log.Fatal("Error reading " + *flDictFile + " " + err.Error())
		}
		// Get an ip for a possible wildcard domain and use it as a blacklist
		blacklist := bsw.GetWildCard(*flDomain, *flServerAddr)
		var blacklist6 string
		if *flipv6 {
			blacklist6 = bsw.GetWildCard6(*flDomain, *flServerAddr)
		}
		for _, n := range nameList {
			sub := n
			tasks <- func() (bsw.Results, error) {
				return bsw.Dictionary(*flDomain, sub, blacklist, *flServerAddr)
			}
			if *flipv6 {
				tasks <- func() (bsw.Results, error) {
					return bsw.Dictionary6(*flDomain, sub, blacklist6, *flServerAddr)
				}
			}
		}
	}

	if *flYandex != "" && *flDomain != "" {
		tasks <- func() (bsw.Results, error) {
			return bsw.YandexAPI(*flDomain, *flYandex, *flServerAddr)
		}
	}

	// Close the tasks channel after all jobs have completed and for each
	// goroutine in the pool receive an empty message from  tracker.
	close(tasks)
	for i := 0; i < *flConcurrency; i++ {
		<-tracker
	}
	close(res)
	// Receive and empty message from the result gatherer
	<-tracker

	os.Stderr.WriteString("\r")
	log.Println("All tasks completed\n")
	sort.Sort(results)

	// Output options
	if *flJson {
		j, _ := json.MarshalIndent(results, "", "    ")
		fmt.Println(string(j))
	} else if *flCsv {
		for _, r := range results {
			fmt.Printf("%s,%s,%s\n", r.Hostname, r.IP, r.Source)
		}
	} else if *flClean {
		cleanSet := make(map[string][]string)
		for _, r := range results {
			cleanSet[r.Hostname] = append(cleanSet[r.Hostname], r.IP)
		}
		for k, v := range cleanSet {
			fmt.Printf("%s:\n", k)
			for _, ip := range v {
				fmt.Printf("\t%s\n", ip)
			}
		}
	} else {
		w := tabwriter.NewWriter(os.Stdout, 0, 8, 4, ' ', 0)
		fmt.Fprintln(w, "IP\tHostname\tSource")
		for _, r := range results {
			fmt.Fprintf(w, "%s\t%s\t%s\n", r.IP, r.Hostname, r.Source)
		}
		w.Flush()
	}
}
<commit_msg>Replace reportin if else with switch case<commit_after>package main

import (
	"bufio"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"github.com/tomsteele/blacksheepwall/bsw"
	"log"
	"net"
	"os"
	"sort"
	"text/tabwriter"
)

var usage = `
  Usage: blacksheepwall [options] <ip address or CIDR>

  Options:
  -h, --help            Show Usage and exit.
  -version              Show version and exit.
  -debug                Enable debugging and show errors returned from tasks.
  -concurrency <int>    Max amount of concurrent tasks.    [default: 100]
  -server <string>      DNS server address.    [default: "8.8.8.8"]
  -input <string>       Line separated file of networks (CIDR) or
                        IP Addresses.
  -ipv6	                Look for additional AAAA records where applicable.
  -domain <string>      Target domain to use for certain tasks.
  -dictionary <string>  Attempt to retrieve the CNAME and A record for
                        each subdomain in the line separated file.
  -yandex <string>      Provided a Yandex search XML API url. Use the Yandex
                        search 'rhost:' operator to find subdomains of a
                        provided domain..
  -bing	<string>        Provided a base64 encoded API key. Use the Bing search
                        API's 'ip:' operator to lookup hostnames for each host.
  -headers              Perform HTTP(s) requests to each host and look for
                        hostnames in a possible Location header.
  -reverse              Retrieve the PTR for each host.
  -tls                  Attempt to retrieve names from TLS certificates
                        (CommonName and Subject Alternative Name).
  -viewdns              Lookup each host using viewdns.info's Reverse IP
                        Lookup function.
  -fcrdns               Verify results by attempting to retrieve the A or AAAA record for
                        each result previously identified hostname.
  -clean                Print results as unique hostnames for each host.
  -csv                  Print results in csv format.
  -json                 Print results as JSON.

`

// Returns all ip addresses from each CIDR range in a list.
func linesToIpList(lines []string) ([]string, error) {
	ipList := make([]string, 0)
	for _, line := range lines {
		if net.ParseIP(line) != nil {
			ipList = append(ipList, line)
		} else if ip, network, err := net.ParseCIDR(line); err == nil {
			for ip := ip.Mask(network.Mask); network.Contains(ip); increaseIp(ip) {
				ipList = append(ipList, ip.String())
			}
		} else {
			return ipList, errors.New("\"" + line + "\" is not an IP Address or CIDR Network")
		}
	}
	return ipList, nil
}

// Increases IP by a single address
func increaseIp(ip net.IP) {
	for j := len(ip) - 1; j >= 0; j-- {
		ip[j]++
		if ip[j] > 0 {
			break
		}
	}
}

// Reads lines from a file and returns as a slice.
func readFileLines(path string) ([]string, error) {
	file, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	lines := make([]string, 0)
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		lines = append(lines, scanner.Text())
	}
	return lines, scanner.Err()
}

type task func() (bsw.Results, error)
type empty struct{}

func main() {
	var (
		flVersion     = flag.Bool("version", false, "Show version and exit.")
		flConcurrency = flag.Int("concurrency", 100, "Max amount of concurrent tasks.")
		flDebug       = flag.Bool("debug", false, "Enable debugging and show errors returned from tasks.")
		flipv6        = flag.Bool("ipv6", false, "Look for AAAA records where applicable.")
		flServerAddr  = flag.String("server", "8.8.8.8", "DNS server address.")
		flIpFile      = flag.String("input", "", "Line separated file of networks (CIDR) or IP Addresses.")
		flReverse     = flag.Bool("reverse", false, "Retrieve the PTR for each host.")
		flHeader      = flag.Bool("headers", false, "Perform HTTP(s) requests to each host and look for hostnames in a possible Location header.")
		flTLS         = flag.Bool("tls", false, "Attempt to retrieve names from TLS certificates (CommonName and Subject Alternative Name).")
		flViewDnsInfo = flag.Bool("viewdns", false, "Lookup each host using viewdns.info's Reverse IP Lookup function.")
		flBing        = flag.String("bing", "", "Provided a base64 encoded API key. Use the Bing search API's 'ip:' operator to lookup hostnames for each host.")
		flYandex      = flag.String("yandex", "", "Provided a Yandex search XML API url. Use the Yandex search 'rhost:' operator to find subdomains of a provided domain.")
		flDomain      = flag.String("domain", "", "Target domain to use for certain tasks.")
		flDictFile    = flag.String("dictionary", "", "Attempt to retrieve the CNAME and A record for each subdomain in the line separated file.")
		flFcrdns      = flag.Bool("fcrdns", false, "Verify results by attempting to retrieve the A or AAAA record for each result previously identified hostname.")
		flClean       = flag.Bool("clean", false, "Print results as unique hostnames for each host.")
		flCsv         = flag.Bool("csv", false, "Print results in csv format.")
		flJson        = flag.Bool("json", false, "Print results as JSON.")
	)
	flag.Usage = func() { fmt.Print(usage) }
	flag.Parse()

	if *flVersion {
		fmt.Println("blacksheepwall version ", bsw.VERSION)
		os.Exit(0)
	}

	// Holds all ip addresses for testing
	ipAddrList := make([]string, 0)

	// Used to hold a ip or CIDR range passed as fl.Arg(0)
	var flNetwork string

	// Verify that some sort of work load was given in commands
	if *flIpFile == "" && *flDomain == "" && len(flag.Args()) < 1 {
		log.Fatal("You didn't provide any work for me to do")
	}
	if *flYandex != "" && *flDomain == "" {
		log.Fatal("Yandex API requires domain set with -domain")
	}
	if *flDictFile != "" && *flDomain == "" {
		log.Fatal("Dictionary lookup requires domain set with -domain")
	}
	if *flDomain != "" && *flYandex == "" && *flDictFile == "" {
		log.Fatal("-domain provided but no methods provided that use it")
	}

	// Get first argument that is not an option and turn it into a list of ips
	if len(flag.Args()) > 0 {
		flNetwork = flag.Arg(0)
		list, err := linesToIpList([]string{flNetwork})
		if err != nil {
			log.Fatal(err.Error())
		}
		ipAddrList = append(ipAddrList, list...)
	}

	// If file given as -input, read lines and turn each possible ip or network into
	// a list of ips. Appends list to ipAddrList. Will fail fatally if line in file
	// is not a valid ip or CIDR range.
	if *flIpFile != "" {
		lines, err := readFileLines(*flIpFile)
		if err != nil {
			log.Fatal("Error reading " + *flIpFile + " " + err.Error())
		}
		list, err := linesToIpList(lines)
		if err != nil {
			log.Fatal(err.Error())
		}
		ipAddrList = append(ipAddrList, list...)
	}

	// tracker: Chanel uses an empty struct to track when all goroutines in the pool
	//          have completed as well as a single call from the gatherer.
	//
	// tasks:   Chanel used in the goroutine pool to manage incoming work. A task is
	//          a function wrapper that returns a slice of results and a possible error.
	//
	// res:     When each task is called in the pool, it will send valid results to
	//          the res channel. A goroutine manages this channel and appends results
	//          to results slice.
	tracker := make(chan empty)
	tasks := make(chan task, *flConcurrency)
	res := make(chan bsw.Results, *flConcurrency)
	results := bsw.Results{}

	// Start up *flConcurrency amount of goroutines
	log.Printf("Spreading tasks across %d goroutines", *flConcurrency)
	for i := 0; i < *flConcurrency; i++ {
		go func() {
			var c = 0
			for def := range tasks {
				result, err := def()
				if m := c % 2; m == 0 {
					c = 3
					os.Stderr.WriteString("\rWorking \\")
				} else {
					c = 2
					os.Stderr.WriteString("\rWorking /")
				}
				if err != nil && *flDebug {
					log.Println(err.Error())
				}
				if err == nil {
					res <- result
				}
			}
			tracker <- empty{}
		}()
	}

	// Ingest incoming results
	go func() {
		for result := range res {
			if len(result) < 1 {
				continue
			}
			if *flFcrdns {
				for _, r := range result {
					ip, err := bsw.LookupName(r.Hostname, *flServerAddr)
					if err == nil && len(ip) > 0 {
						results = append(results, bsw.Result{Source: "fcrdns", IP: ip, Hostname: r.Hostname})
					}
					ip, err = bsw.LookupName6(r.Hostname, *flServerAddr)
					if err == nil && len(ip) > 0 {
						results = append(results, bsw.Result{Source: "fcrdns", IP: ip, Hostname: r.Hostname})
					}
				}
			} else {
				results = append(results, result...)
			}
		}
		e := empty{}
		tracker <- e
	}()

	// Bing has two possible search paths. We need to find which one is valid.
	var bingPath string
	if *flBing != "" {
		p, err := bsw.FindBingSearchPath(*flBing)
		if err != nil {
			log.Fatal(err.Error())
		}
		bingPath = p
	}

	// ip based functionality should be added to the pool here
	for _, h := range ipAddrList {
		host := h
		if *flReverse {
			tasks <- func() (bsw.Results, error) {
				return bsw.Reverse(host, *flServerAddr)
			}
		}
		if *flTLS {
			tasks <- func() (bsw.Results, error) {
				return bsw.TLS(host)
			}
		}
		if *flViewDnsInfo {
			tasks <- func() (bsw.Results, error) {
				return bsw.ViewDnsInfo(host)
			}
		}
		if *flBing != "" && bingPath != "" {
			tasks <- func() (bsw.Results, error) {
				return bsw.BingAPI(host, *flBing, bingPath)
			}
		}
		if *flHeader {
			tasks <- func() (bsw.Results, error) {
				return bsw.Headers(host)
			}
		}

	}

	// Domain based functions will likely require separate blocks

	// Subdomain dictionary guessing
	if *flDictFile != "" && *flDomain != "" {
		nameList, err := readFileLines(*flDictFile)
		if err != nil {
			log.Fatal("Error reading " + *flDictFile + " " + err.Error())
		}
		// Get an ip for a possible wildcard domain and use it as a blacklist
		blacklist := bsw.GetWildCard(*flDomain, *flServerAddr)
		var blacklist6 string
		if *flipv6 {
			blacklist6 = bsw.GetWildCard6(*flDomain, *flServerAddr)
		}
		for _, n := range nameList {
			sub := n
			tasks <- func() (bsw.Results, error) {
				return bsw.Dictionary(*flDomain, sub, blacklist, *flServerAddr)
			}
			if *flipv6 {
				tasks <- func() (bsw.Results, error) {
					return bsw.Dictionary6(*flDomain, sub, blacklist6, *flServerAddr)
				}
			}
		}
	}

	if *flYandex != "" && *flDomain != "" {
		tasks <- func() (bsw.Results, error) {
			return bsw.YandexAPI(*flDomain, *flYandex, *flServerAddr)
		}
	}

	// Close the tasks channel after all jobs have completed and for each
	// goroutine in the pool receive an empty message from  tracker.
	close(tasks)
	for i := 0; i < *flConcurrency; i++ {
		<-tracker
	}
	close(res)
	// Receive and empty message from the result gatherer
	<-tracker

	os.Stderr.WriteString("\r")
	log.Println("All tasks completed\n")
	sort.Sort(results)

	// Output options
	switch {
	case *flJson:
		j, _ := json.MarshalIndent(results, "", "    ")
		fmt.Println(string(j))
	case *flCsv:
		for _, r := range results {
			fmt.Printf("%s,%s,%s\n", r.Hostname, r.IP, r.Source)
		}
	case *flClean:
		cleanSet := make(map[string][]string)
		for _, r := range results {
			cleanSet[r.Hostname] = append(cleanSet[r.Hostname], r.IP)
		}
		for k, v := range cleanSet {
			fmt.Printf("%s:\n", k)
			for _, ip := range v {
				fmt.Printf("\t%s\n", ip)
			}
		}
	default:
		w := tabwriter.NewWriter(os.Stdout, 0, 8, 4, ' ', 0)
		fmt.Fprintln(w, "IP\tHostname\tSource")
		for _, r := range results {
			fmt.Fprintf(w, "%s\t%s\t%s\n", r.IP, r.Hostname, r.Source)
		}
		w.Flush()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/captncraig/github-webhooks"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"time"
)

var scriptDir string
var scriptExt = ".sh"

func init() {
	scriptDir = os.Getenv("TINYCI-SCRIPT-DIR")

	if scriptDir == "" {
		var err error
		scriptDir, err = os.Getwd()
		if err != nil {
			log.Fatal(err)
		}
		scriptDir = filepath.Join(scriptDir, "scripts")
	}
	if runtime.GOOS == "windows" {
		scriptExt = ".bat"
	}
}

func main() {

	gitHooks := webhooks.WebhookListener{}
	gitHooks.OnPush = githubHook
	http.HandleFunc("/gh", gitHooks.GetHttpListener())
	http.HandleFunc("/dh", dockerHubHook)
	http.ListenAndServe(":4567", nil)
}

func githubHook(event *webhooks.PushEvent, _ *webhooks.WebhookContext) {
	repo := strings.Replace(event.Repository.FullName, "/", ".", -1)
	refPath := strings.Split(event.Ref, "/")
	ref := refPath[len(refPath)-1]
	runScriptIfExists(fmt.Sprintf("gh-%s", repo))
	runScriptIfExists(fmt.Sprintf("gh-%s~%s", repo, ref))
}

type DockerHubData struct {
	CallbackUrl string `json:"callback_url"`
	Repository  struct {
		Name string `json:"repo_name"`
	} `json:"repository"`
}

func dockerHubHook(w http.ResponseWriter, r *http.Request) {
	body, _ := ioutil.ReadAll(r.Body)
	data := DockerHubData{}
	json.Unmarshal(body, &data)
	runScriptIfExists(fmt.Sprintf("dh-%s", strings.Replace(data.Repository.Name, "/", ".", -1)))

	go func() {
		//wait for incoming request to finish before calling callback. The test on dockerhub is more consistent this way.
		time.Sleep(15 * time.Millisecond)
		resp, err := http.Post(data.CallbackUrl, "application/json", bytes.NewBuffer([]byte(`{"state": "success"}`)))
		if err != nil {
			log.Println(err)
			return
		}
		fmt.Println(resp.StatusCode)
	}()
}

func runScriptIfExists(name string) {
	filename := filepath.Join(scriptDir, name+scriptExt)
	if _, err := os.Stat(filename); os.IsNotExist(err) {
		log.Printf("Script does not exist: %s. Skipping.\n", filename)
		return
	}
	cmd := exec.Command(filename)
	log.Printf("Executing %s...\n", filename)
	output, err := cmd.CombinedOutput()
	if err != nil {
		log.Printf("Error executing %s: %s.", filename, err.Error())
	}
	log.Println(string(output))
}
<commit_msg>adding dockerfile<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/captncraig/github-webhooks"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"time"
)

var scriptDir string
var scriptExt = ".sh"

func init() {
	scriptDir = os.Getenv("TINYCI-SCRIPT-DIR")

	if scriptDir == "" {
		var err error
		scriptDir, err = os.Getwd()
		if err != nil {
			log.Fatal(err)
		}
		scriptDir = filepath.Join(scriptDir, "scripts")
	}
	if runtime.GOOS == "windows" {
		scriptExt = ".bat"
	}
}

func main() {
	gitHooks := webhooks.WebhookListener{}
	gitHooks.OnPush = githubHook
	http.HandleFunc("/gh", gitHooks.GetHttpListener())
	http.HandleFunc("/dh", dockerHubHook)
	go gitPoll()
	http.ListenAndServe(":4567", nil)
}

func githubHook(event *webhooks.PushEvent, _ *webhooks.WebhookContext) {
	repo := strings.Replace(event.Repository.FullName, "/", ".", -1)
	refPath := strings.Split(event.Ref, "/")
	ref := refPath[len(refPath)-1]
	runScriptIfExists(fmt.Sprintf("gh-%s", repo))
	runScriptIfExists(fmt.Sprintf("gh-%s~%s", repo, ref))
}

type DockerHubData struct {
	CallbackUrl string `json:"callback_url"`
	Repository  struct {
		Name string `json:"repo_name"`
	} `json:"repository"`
}

func dockerHubHook(w http.ResponseWriter, r *http.Request) {
	body, _ := ioutil.ReadAll(r.Body)
	data := DockerHubData{}
	json.Unmarshal(body, &data)
	runScriptIfExists(fmt.Sprintf("dh-%s", strings.Replace(data.Repository.Name, "/", ".", -1)))

	go func() {
		//wait for incoming request to finish before calling callback. The test on dockerhub is more consistent this way.
		time.Sleep(15 * time.Millisecond)
		resp, err := http.Post(data.CallbackUrl, "application/json", bytes.NewBuffer([]byte(`{"state": "success"}`)))
		if err != nil {
			log.Println(err)
			return
		}
		fmt.Println(resp.StatusCode)
	}()
}

func runScriptIfExists(name string) {
	filename := filepath.Join(scriptDir, name+scriptExt)
	if _, err := os.Stat(filename); os.IsNotExist(err) {
		log.Printf("Script does not exist: %s. Skipping.\n", filename)
		return
	}
	cmd := exec.Command(filename)
	log.Printf("Executing %s...\n", filename)
	output, err := cmd.CombinedOutput()
	if err != nil {
		log.Printf("Error executing %s: %s.", filename, err.Error())
	}
	log.Println(string(output))
}

func gitPoll() {

}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"strings"
)

type dependency struct {
	ImportPath string
	Rev        string // VCS-specific commit ID.
}

var (
	godepsJSON = flag.String("godeps", os.Getenv("PWD")+"/Godeps/Godeps.json", "path to Godeps.json")
	godeps     struct {
		ImportPath string
		Deps       []dependency
	}
	temporaryDir, git string
)

func init() {
	flag.StringVar(&temporaryDir, "temp", "", "temporary path for cloning the repositories")
	flag.Parse()
}

func main() {
	f, err := ioutil.ReadFile(*godepsJSON)

	if err != nil {
		fmt.Fprintf(os.Stderr, "opening %s: %s", *godepsJSON, err)
		os.Exit(1)
	}

	if err := json.Unmarshal(f, &godeps); err != nil {
		fmt.Fprintf(os.Stderr, "unmarshalling %s: %s", *godepsJSON, err)
		os.Exit(1)
	}

	if len(temporaryDir) == 0 {
		createTemporaryDir()
		defer os.RemoveAll(temporaryDir)
	}

	lookGITPath()
	results := processDependencies(godeps.Deps)

	for _, dep := range godeps.Deps {
		if commits := results[dep.ImportPath]; len(commits) > 1 {
			fmt.Println(dep.ImportPath)

			for _, commit := range commits {
				fmt.Println("  " + commit)
			}
		}
	}
}

func createTemporaryDir() {
	var err error
	temporaryDir, err = ioutil.TempDir("/tmp", "godeps-check")

	if err != nil {
		fmt.Fprintf(os.Stderr, "creating temporary dir: %s", err)
		os.Exit(1)
	}

	if err := os.Chmod(temporaryDir, 0777); err != nil {
		fmt.Fprintf(os.Stderr, "chmod %s: %s", temporaryDir, err)
		os.Exit(1)
	}

	if err := os.Chdir(temporaryDir); err != nil {
		fmt.Fprintf(os.Stderr, "chdir %s: %s", temporaryDir, err)
		os.Exit(1)
	}
}

func lookGITPath() {
	var err error
	git, err = exec.LookPath("git")

	if err != nil {
		fmt.Fprintf(os.Stderr, "looking up git path: %s", err)
		os.Exit(1)
	}
}

func processDependencies(deps []dependency) map[string][]string {
	results := make(map[string][]string)

	for _, d := range deps {
		importPath := d.ImportPath
		parts := strings.Split(importPath, "/")

		if len(parts) > 3 {
			importPath = strings.Join(parts[0:3], "/")
		}

		if !strings.Contains(parts[0], ".") {
			fmt.Fprintf(os.Stderr, "skipping %s: not go gettable\n", d.ImportPath)
			continue
		}

		projectDir := path.Join(temporaryDir, importPath)

		if err := os.MkdirAll(projectDir, 0777); err != nil {
			fmt.Fprintln(os.Stderr, err)
			continue
		}

		// defer os.RemoveAll(projectDir)

		url := "https://" + importPath

		if err := clone(url, projectDir); err != nil {
			continue
		}

		if err := os.Chdir(projectDir); err != nil {
			fmt.Fprintf(os.Stderr, "chdir %s: %s", temporaryDir, err)
			continue
		}

		results[d.ImportPath] = diff(d.Rev)
	}

	return results
}

func clone(url, projectDir string) error {
	cmd := exec.Command(git, "clone", url, projectDir)
	output, err := cmd.CombinedOutput()
	content := string(output)

	wd, _ := os.Getwd()
	fmt.Fprintln(os.Stderr, strings.Replace(wd, temporaryDir+"/", "", -1))
	fmt.Fprint(os.Stderr, strings.Join(cmd.Args, " "))

	if err != nil {
		fmt.Fprintf(os.Stderr, ": %s\n", err)
	} else {
		fmt.Fprintln(os.Stderr, "")
	}

	fmt.Fprintln(os.Stderr, "  "+strings.Replace(content, "\n", "\n  ", -1))

	return err
}

func diff(revision string) []string {
	cmd := exec.Command(git, "log", "--pretty=oneline", fmt.Sprintf("%s..master", revision))
	output, err := cmd.CombinedOutput()

	if err != nil {
		return []string{err.Error()}
	}

	parts := strings.Split(string(output), "\n")

	if len(parts) > 10 {
		size := len(parts) - 10

		parts = parts[0:9]
		parts = append(parts, fmt.Sprintf("[%d commits not shown]\n", size))
	}

	return parts
}
<commit_msg>Parallelize cloning and diff'ing<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"strings"
)

type dependency struct {
	ImportPath string
	Rev        string // VCS-specific commit ID.
}

type result struct {
	importPath string
	commits    []string
}

var (
	godepsJSON = flag.String("godeps", os.Getenv("PWD")+"/Godeps/Godeps.json", "path to Godeps.json")
	godeps     struct {
		ImportPath string
		Deps       []dependency
	}
	temporaryDir, git string
)

func init() {
	flag.StringVar(&temporaryDir, "temp", "", "temporary path for cloning the repositories")
	flag.Parse()
}

func main() {
	f, err := ioutil.ReadFile(*godepsJSON)

	if err != nil {
		fmt.Fprintf(os.Stderr, "opening %s: %s", *godepsJSON, err)
		os.Exit(1)
	}

	if err := json.Unmarshal(f, &godeps); err != nil {
		fmt.Fprintf(os.Stderr, "unmarshalling %s: %s", *godepsJSON, err)
		os.Exit(1)
	}

	if len(temporaryDir) == 0 {
		createTemporaryDir()
		defer os.RemoveAll(temporaryDir)
	}

	lookGITPath()
	results := processDependencies(godeps.Deps)

	for _, dep := range godeps.Deps {
		if commits := results[dep.ImportPath]; len(commits) > 1 {
			fmt.Println(dep.ImportPath)

			for _, commit := range commits {
				fmt.Println("  " + commit)
			}
		}
	}
}

func createTemporaryDir() {
	var err error
	temporaryDir, err = ioutil.TempDir("/tmp", "godeps-check")

	if err != nil {
		fmt.Fprintf(os.Stderr, "creating temporary dir: %s", err)
		os.Exit(1)
	}

	if err := os.Chmod(temporaryDir, 0777); err != nil {
		fmt.Fprintf(os.Stderr, "chmod %s: %s", temporaryDir, err)
		os.Exit(1)
	}
}

func lookGITPath() {
	var err error
	git, err = exec.LookPath("git")

	if err != nil {
		fmt.Fprintf(os.Stderr, "looking up git path: %s", err)
		os.Exit(1)
	}
}

func processDependencies(deps []dependency) map[string][]string {
	results := make(map[string][]string)
	ch := make(chan result, len(deps))

	for _, d := range deps {
		go func(d dependency, ch chan result) {
			ch <- result{d.ImportPath, processDependency(d)}
		}(d, ch)
	}

	done := 0

alldone:
	for {
		select {
		case r := <-ch:
			results[r.importPath] = r.commits
			done++

			if done == len(deps) {
				break alldone
			}
		}
	}

	return results
}

func processDependency(d dependency) []string {
	importPath := d.ImportPath
	parts := strings.Split(importPath, "/")

	if len(parts) > 3 {
		importPath = strings.Join(parts[0:3], "/")
	}

	if !strings.Contains(parts[0], ".") {
		fmt.Fprintf(os.Stderr, "skipping %s: not go gettable\n", d.ImportPath)
		return nil
	}

	projectDir := path.Join(temporaryDir, importPath)

	if err := os.MkdirAll(projectDir, 0777); err != nil {
		fmt.Fprintln(os.Stderr, err)
		return nil
	}

	if err := clone("https://"+importPath, projectDir); err != nil {
		return nil
	}

	return diff(projectDir, d.Rev)
}

func clone(url, projectDir string) error {
	cmd := exec.Command(git, "clone", url, projectDir)
	output, err := cmd.CombinedOutput()
	content := string(output)

	fmt.Fprint(os.Stderr, strings.Join(cmd.Args, " "))

	if err != nil {
		fmt.Fprintf(os.Stderr, ": %s\n", err)
	} else {
		fmt.Fprintln(os.Stderr, "")
	}

	fmt.Fprintln(os.Stderr, "  "+strings.Replace(content, "\n", "\n  ", -1))

	return err
}

func diff(path, revision string) []string {
	cmd := exec.Command(git, "-C", path, "log", "--pretty=oneline", revision+"..master")
	output, err := cmd.CombinedOutput()
	lines := strings.Split(string(output), "\n")

	if err != nil {
		return append(lines, err.Error()+"\n")
	}

	for i := range lines {
		if len(lines[i]) == 0 {
			continue
		}

		revision := strings.Split(lines[i], " ")[0]
		lines[i] = strings.Replace(lines[i], revision, revision[0:7], -1)
	}

	if len(lines) > 10 {
		size := len(lines) - 10

		lines = lines[0:9]
		lines = append(lines, fmt.Sprintf("[%d commits not shown]\n", size))
	}

	return lines
}
<|endoftext|>


<commit_before>// Copyright © 2016 NAME HERE <EMAIL ADDRESS>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import "github.com/eddyzags/kafka-cli/kafkactl/cmd"

func main() {
	cmd.Execute()
}
<commit_msg>main: Minor error fix (imports)<commit_after>// Copyright © 2016 NAME HERE <EMAIL ADDRESS>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import "github.com/eddyzags/kafka-cli/cmd"

func main() {
	cmd.Execute()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"strconv"

	"github.com/ogier/pflag"
	"github.com/yuya-takeyama/argf"
)

var (
	name    = "csvp"
	version = "0.8.1"
)

func printUsage() {
	fmt.Fprintf(os.Stderr, `
Usage: %s [OPTION]... [FILE]...
Print selected parts of CSV from each FILE to standard output.

Options:
  -i, --indexes=LIST
                 select only these indexes
  -h, --headers=LIST
                 select only these headers
  -t, --tsv
                 equivalent to -d'\t'
  -d, --delimiter=DELIM
                 use DELIM instead of comma for field delimiter
  -D, --output-delimiter=STRING
                 use STRING as the output delimiter (default: \t)
  --help
                 display this help text and exit
  --version
                 output version information and exit
`[1:], name)
}

func printVersion() {
	fmt.Fprintln(os.Stderr, version)
}

type Option struct {
	IndexesList     string
	HeadersList     string
	IsTSV           bool
	Delimiter       string
	OutputDelimiter string
	IsHelp          bool
	IsVersion       bool
	Files           []string
}

func parseOption(args []string) (opt *Option, err error) {
	flag := pflag.NewFlagSet(name, pflag.ContinueOnError)
	flag.SetOutput(ioutil.Discard)

	opt = &Option{}
	flag.StringVarP(&opt.IndexesList, "indexes", "i", "", "")
	flag.StringVarP(&opt.HeadersList, "headers", "h", "", "")
	flag.BoolVarP(&opt.IsTSV, "tsv", "t", false, "")
	flag.StringVarP(&opt.Delimiter, "delimiter", "d", ",", "")
	flag.StringVarP(&opt.OutputDelimiter, "output-delimiter", "D", "\t", "")
	flag.BoolVarP(&opt.IsHelp, "help", "", false, "")
	flag.BoolVarP(&opt.IsVersion, "version", "", false, "")

	if err = flag.Parse(args); err != nil {
		return nil, err
	}

	opt.Files = flag.Args()
	return opt, nil
}

func toDelimiter(s string) (r rune, err error) {
	s, err = strconv.Unquote(`"` + s + `"`)
	if err != nil {
		return 0, err
	}

	runes := []rune(s)
	if len(runes) != 1 {
		return 0, fmt.Errorf("the delimiter must be a single character")
	}
	return runes[0], nil
}

func newCSVScannerFromOption(opt *Option) (c *CSVScanner, err error) {
	var selector Selector
	switch {
	case opt.IndexesList != "" && opt.HeadersList != "":
		return nil, fmt.Errorf("only one type of list may be specified")
	case opt.IndexesList != "":
		selector = NewIndexes(opt.IndexesList)
	case opt.HeadersList != "":
		selector = NewHeaders(opt.HeadersList)
	default:
		selector = NewAll()
	}

	reader, err := argf.From(opt.Files)
	if err != nil {
		return nil, err
	}

	c = NewCSVScanner(selector, reader)
	c.SetOutputDelimiter(opt.OutputDelimiter)
	switch {
	case opt.IsTSV:
		c.SetDelimiter('\t')
	default:
		delimiter, err := toDelimiter(opt.Delimiter)
		if err != nil {
			return nil, err
		}
		c.SetDelimiter(delimiter)
	}
	return c, nil
}

func do(c *CSVScanner) error {
	for c.Scan() {
		fmt.Println(c.Text())
	}
	return c.Err()
}

func printErr(err error) {
	fmt.Fprintf(os.Stderr, "%s: %s\n", name, err)
}

func guideToHelp() {
	fmt.Fprintf(os.Stderr, "Try '%s --help' for more information.\n", name)
}

func _main() int {
	opt, err := parseOption(os.Args[1:])
	if err != nil {
		printErr(err)
		guideToHelp()
		return 2
	}
	switch {
	case opt.IsHelp:
		printUsage()
		return 0
	case opt.IsVersion:
		printVersion()
		return 0
	}

	c, err := newCSVScannerFromOption(opt)
	if err != nil {
		printErr(err)
		guideToHelp()
		return 2
	}
	if err = do(c); err != nil {
		printErr(err)
		return 1
	}
	return 0
}

func main() {
	e := _main()
	os.Exit(e)
}
<commit_msg>Declare printErr before Option<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"strconv"

	"github.com/ogier/pflag"
	"github.com/yuya-takeyama/argf"
)

var (
	name    = "csvp"
	version = "0.8.1"
)

func printUsage() {
	fmt.Fprintf(os.Stderr, `
Usage: %s [OPTION]... [FILE]...
Print selected parts of CSV from each FILE to standard output.

Options:
  -i, --indexes=LIST
                 select only these indexes
  -h, --headers=LIST
                 select only these headers
  -t, --tsv
                 equivalent to -d'\t'
  -d, --delimiter=DELIM
                 use DELIM instead of comma for field delimiter
  -D, --output-delimiter=STRING
                 use STRING as the output delimiter (default: \t)
  --help
                 display this help text and exit
  --version
                 output version information and exit
`[1:], name)
}

func printVersion() {
	fmt.Fprintln(os.Stderr, version)
}

func printErr(err error) {
	fmt.Fprintf(os.Stderr, "%s: %s\n", name, err)
}

type Option struct {
	IndexesList     string
	HeadersList     string
	IsTSV           bool
	Delimiter       string
	OutputDelimiter string
	IsHelp          bool
	IsVersion       bool
	Files           []string
}

func parseOption(args []string) (opt *Option, err error) {
	flag := pflag.NewFlagSet(name, pflag.ContinueOnError)
	flag.SetOutput(ioutil.Discard)

	opt = &Option{}
	flag.StringVarP(&opt.IndexesList, "indexes", "i", "", "")
	flag.StringVarP(&opt.HeadersList, "headers", "h", "", "")
	flag.BoolVarP(&opt.IsTSV, "tsv", "t", false, "")
	flag.StringVarP(&opt.Delimiter, "delimiter", "d", ",", "")
	flag.StringVarP(&opt.OutputDelimiter, "output-delimiter", "D", "\t", "")
	flag.BoolVarP(&opt.IsHelp, "help", "", false, "")
	flag.BoolVarP(&opt.IsVersion, "version", "", false, "")

	if err = flag.Parse(args); err != nil {
		return nil, err
	}

	opt.Files = flag.Args()
	return opt, nil
}

func toDelimiter(s string) (r rune, err error) {
	s, err = strconv.Unquote(`"` + s + `"`)
	if err != nil {
		return 0, err
	}

	runes := []rune(s)
	if len(runes) != 1 {
		return 0, fmt.Errorf("the delimiter must be a single character")
	}
	return runes[0], nil
}

func newCSVScannerFromOption(opt *Option) (c *CSVScanner, err error) {
	var selector Selector
	switch {
	case opt.IndexesList != "" && opt.HeadersList != "":
		return nil, fmt.Errorf("only one type of list may be specified")
	case opt.IndexesList != "":
		selector = NewIndexes(opt.IndexesList)
	case opt.HeadersList != "":
		selector = NewHeaders(opt.HeadersList)
	default:
		selector = NewAll()
	}

	reader, err := argf.From(opt.Files)
	if err != nil {
		return nil, err
	}

	c = NewCSVScanner(selector, reader)
	c.SetOutputDelimiter(opt.OutputDelimiter)
	switch {
	case opt.IsTSV:
		c.SetDelimiter('\t')
	default:
		delimiter, err := toDelimiter(opt.Delimiter)
		if err != nil {
			return nil, err
		}
		c.SetDelimiter(delimiter)
	}
	return c, nil
}

func do(c *CSVScanner) error {
	for c.Scan() {
		fmt.Println(c.Text())
	}
	return c.Err()
}

func guideToHelp() {
	fmt.Fprintf(os.Stderr, "Try '%s --help' for more information.\n", name)
}

func _main() int {
	opt, err := parseOption(os.Args[1:])
	if err != nil {
		printErr(err)
		guideToHelp()
		return 2
	}
	switch {
	case opt.IsHelp:
		printUsage()
		return 0
	case opt.IsVersion:
		printVersion()
		return 0
	}

	c, err := newCSVScannerFromOption(opt)
	if err != nil {
		printErr(err)
		guideToHelp()
		return 2
	}
	if err = do(c); err != nil {
		printErr(err)
		return 1
	}
	return 0
}

func main() {
	e := _main()
	os.Exit(e)
}
<|endoftext|>