<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
)

type PostCache struct {
	Url   string
	Items map[string]*item
}

var postCache = PostCache{Items: map[string]*item{}}

const DEFAULT_FOLDER = "<from title>"
const HOST_HACKER_NEWS = "news.ycombinator.com"

var Verbose *bool
var Quiet *bool

func main() {
	cacheFile := flag.String("readfrom", "", "Read posts from a json file instead of querying firebase (mainly used for testing)")
	dump := flag.Bool("dump", false, "Fetch the given url only and dump the response. Do not fetch its children")
	folder := flag.String("o", DEFAULT_FOLDER, "Save mails into the given directory instead of the default one")
	Verbose = flag.Bool("v", false, "Verbose")
	Quiet = flag.Bool("q", false, "Be quiet")
	flag.Parse()
	if *Quiet {
		*Verbose = false
	}

	var arg string
	if *cacheFile != "" {
		if fileExists(*cacheFile) {
			readCacheFile(*cacheFile)
		} else {
			bail("readfrom: file %s does not exist", *cacheFile)
		}
		arg = postCache.Url
	} else {
		if flag.NArg() != 1 {
			printError("Usage: fetchpost <url|dir>")
			os.Exit(1)
		}
		arg = flag.Arg(0)
	}

	var url string
	if strings.HasPrefix(arg, "https://") {
		url = arg
		postCache.Url = url
	} else {
		dir, file := filepath.Split(filepath.Clean(arg))
		if dir != "" {
			err := os.Chdir(dir)
			if err != nil {
				bail(err.Error())
			}
		}
		arg = file
		url = readUrlFile(arg)
		if strings.HasPrefix(url, "https://") {
			postCache.Url = url
		}
	}

	/**
	1. Get post
	2. If mail for post doesn't exist:
		2.1 write mail in maildir
	3. else if post body or title changed:
		3.1 Update subject or body and mark mail as new
	4. Repeat from step 1 for each kid in post.
	*/
	post := mustGetPost(url)
	if *dump {
		buf, err := json.Marshal(post)
		if err != nil {
			bail("dump: %v", err)
		}
		fmt.Printf("%s\n", string(buf))
		os.Exit(0)
	}

	added, updated, deleted := saveComments(post, arg, url, *folder)
	if !*Quiet {
		fmt.Printf("Comments: %d added, %d updated, %d deleted\n",
			added, updated, deleted)
	}

	postDir := getPostDir(post)
	fmt.Println(postDir)
	postTestFile := filepath.Join(postDir, postDir+".json")
	buf, err := json.Marshal(postCache)
	if err != nil {
		printError("Marshal postCache: %v", err)
	}
	err = ioutil.WriteFile(postTestFile, append(buf, '\n'), 0644)
	if err != nil {
		printError("Write postCache: %v", err)
	}
}

func saveComments(post Post, arg, url string, folder string) (added, updated, deleted int) {
	dir := getPostDir(post)
	if folder == DEFAULT_FOLDER && arg != url && arg != dir {
		renamePostDir(arg, dir)
	}

	mails, err := NewMails(dir)
	if err != nil {
		bail("saveComments(): error: %v", err)
	}
	maybeSavePostUrl(dir, url)
	maybeReportScoreChange(mails, post)
	return mails.SavePost(nil, post, 0)
}

func maybeReportScoreChange(mails *Mails, post Post) {
	mid := post.MessageId()
	key, exist := mails.keyForMessageId[mid]
	if !exist {
		return
	}

	msg := mails.mustGetMessage(key)
	score, exist := msg.Header["X-Score"]
	if !exist || len(score[0]) == 0 {
		printError("X-Score header missing in %s", key)
		return
	}
	oldScore, err := strconv.Atoi(score[0])
	if err != nil {
		printError("X-Score: %v", err)
		return
	}
	newScore := post.Points()

	if oldScore != newScore {
		fmt.Printf("Points changed from %d to %d\n", oldScore, newScore)
	}
}

func renamePostDir(olddir, newdir string) {
	if fileExists(newdir) {
		bail("renamePostDir(): post title changed but directory for new title"+
			" already exist\n\told title: %s\n\tnew title: %s",
			olddir, newdir)
	}
	err := os.Rename(olddir, newdir)
	if err != nil {
		bail("renamePostDir(): Rename: %v", err)
	}
	printNotice("post title has changed:\n\told: %s\n\tnew: %s", olddir, newdir)

}

func readCacheFile(f string) {
	buf, err := ioutil.ReadFile(f)
	if err != nil {
		bail("Read %s: %v", f, err)
	}
	err = json.Unmarshal(buf, &postCache)
	if err != nil {
		bail("Unmarshal %s: %v", f, err)
	}
}

func fileExists(path string) bool {
	_, err := os.Stat(path)
	if err != nil {
		if os.IsNotExist(err) {
			return false
		} else {
			bail("fileExists: %s: %v", path, err)
		}
	}
	return true
}

func (mails *Mails) SavePost(parent, child Post, threadLevel int) (added, updated, deleted int) {
	mid := child.MessageId()
	key, exist := mails.keyForMessageId[mid]
	if exist {
		msg, err := mails.md.Message(key)
		if err != nil {
			bail("SavePost(): Error getting mail for key %s: %v", key, err)
		}

		path, err := mails.md.Filename(key)
		if err != nil {
			bail("SavePost(): could not get Filename for key %s", key)
		}
		contentChanged := updateMail(parent, child, path, msg)
		if contentChanged {
			p := strings.Replace(path, "/cur/", "/new/", 1)
			if path != p {
				err = os.Rename(path, p)
				if err != nil {
					bail("SavePost(): Rename(%s, %s): %v", path, p, err)
				}
				verbose("Renamed %s to %s", path, p)
			}
			if child.Removed() {
				deleted++
			} else {
				updated++
			}
		}
		showThreadLevel(threadLevel, "*")
	} else {
		mail := NewMail(parent, child)
		err := mails.SaveMail(mail)
		if err != nil {
			bail("SavePost(): %v", err)
		}
		showThreadLevel(threadLevel, "+")
		added++
	}

	n, u, d := mails.SavePostChildren(child, threadLevel+1)
	added += n
	updated += u
	deleted += d
	return
}

func (mails *Mails) SavePostChildren(p Post, threadLevel int) (added, updated, deleted int) {
	for _, idStr := range p.Children() {
		child, err := fetchHNPost(idStr)
		if err != nil {
			printError("error: %v", err)
			continue
		}
		a, u, d := mails.SavePost(p, child, threadLevel)
		added += a
		updated += u
		deleted += d
	}
	return
}

func showThreadLevel(level int, symbol string) {
	for i := 0; i < level; i++ {
		fmt.Print(" ")
	}
	fmt.Println(symbol)
}

func maybeSavePostUrl(dir, url string) {
	if !fileExists(dir) {
		os.Mkdir(dir, 0755)
	}
	f := filepath.Join(dir, "url.txt")
	if fileExists(f) {
		return
	}
	err := ioutil.WriteFile(f, []byte(url), 0644)
	if err != nil {
		bail("%s: %v", f, err)
	}
}

func mustGetPost(href string) Post {
	url, err := url.Parse(href)
	if err != nil {
		bail("bad url: %s", err)
	}

	switch strings.ToLower(url.Host) {
	case HOST_HACKER_NEWS:
		return GetHNPost(url)
	default:
		bail("Unsupported url: %s", url.Host)
	}
	return nil
}

func bail(format string, args ...interface{}) {
	printError(format, args...)
	os.Exit(1)
}

func readUrlFile(arg string) string {
	urlFile := filepath.Join(arg, "url.txt")
	buf, err := ioutil.ReadFile(urlFile)
	if err != nil {
		bail("readUrlFile: error: %v", err)
	}
	return string(buf)
}

func verbose(format string, args ...interface{}) {
	if !*Verbose {
		return
	}
	fmt.Fprintf(os.Stderr, format+"\n", args...)
}

func printError(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, format+"\n", args...)
}

func printNotice(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, "notice:"+format+"\n", args...)
}
<commit_msg>Avoid panic on transient post retrieval anomaly<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
)

type PostCache struct {
	Url   string
	Items map[string]*item
}

var postCache = PostCache{Items: map[string]*item{}}

const DEFAULT_FOLDER = "<from title>"
const HOST_HACKER_NEWS = "news.ycombinator.com"

var Verbose *bool
var Quiet *bool

func main() {
	cacheFile := flag.String("readfrom", "", "Read posts from a json file instead of querying firebase (mainly used for testing)")
	dump := flag.Bool("dump", false, "Fetch the given url only and dump the response. Do not fetch its children")
	folder := flag.String("o", DEFAULT_FOLDER, "Save mails into the given directory instead of the default one")
	Verbose = flag.Bool("v", false, "Verbose")
	Quiet = flag.Bool("q", false, "Be quiet")
	flag.Parse()
	if *Quiet {
		*Verbose = false
	}

	var arg string
	if *cacheFile != "" {
		if fileExists(*cacheFile) {
			readCacheFile(*cacheFile)
		} else {
			bail("readfrom: file %s does not exist", *cacheFile)
		}
		arg = postCache.Url
	} else {
		if flag.NArg() != 1 {
			printError("Usage: fetchpost <url|dir>")
			os.Exit(1)
		}
		arg = flag.Arg(0)
	}

	var url string
	if strings.HasPrefix(arg, "https://") {
		url = arg
		postCache.Url = url
	} else {
		dir, file := filepath.Split(filepath.Clean(arg))
		if dir != "" {
			err := os.Chdir(dir)
			if err != nil {
				bail(err.Error())
			}
		}
		arg = file
		url = readUrlFile(arg)
		if strings.HasPrefix(url, "https://") {
			postCache.Url = url
		}
	}

	/**
	1. Get post
	2. If mail for post doesn't exist:
		2.1 write mail in maildir
	3. else if post body or title changed:
		3.1 Update subject or body and mark mail as new
	4. Repeat from step 1 for each kid in post.
	*/
	post := mustGetPost(url)
	if *dump {
		buf, err := json.Marshal(post)
		if err != nil {
			bail("dump: %v", err)
		}
		fmt.Printf("%s\n", string(buf))
		os.Exit(0)
	}

	added, updated, deleted := saveComments(post, arg, url, *folder)
	if !*Quiet {
		fmt.Printf("Comments: %d added, %d updated, %d deleted\n",
			added, updated, deleted)
	}

	postDir := getPostDir(post)
	fmt.Println(postDir)
	postTestFile := filepath.Join(postDir, postDir+".json")
	buf, err := json.Marshal(postCache)
	if err != nil {
		printError("Marshal postCache: %v", err)
	}
	err = ioutil.WriteFile(postTestFile, append(buf, '\n'), 0644)
	if err != nil {
		printError("Write postCache: %v", err)
	}
}

func saveComments(post Post, arg, url string, folder string) (added, updated, deleted int) {
	dir := getPostDir(post)
	if folder == DEFAULT_FOLDER && arg != url && arg != dir {
		renamePostDir(arg, dir)
	}

	mails, err := NewMails(dir)
	if err != nil {
		bail("saveComments(): error: %v", err)
	}
	maybeSavePostUrl(dir, url)
	maybeReportScoreChange(mails, post)
	return mails.SavePost(nil, post, 0)
}

func maybeReportScoreChange(mails *Mails, post Post) {
	mid := post.MessageId()
	key, exist := mails.keyForMessageId[mid]
	if !exist {
		return
	}

	msg := mails.mustGetMessage(key)
	score, exist := msg.Header["X-Score"]
	if !exist || len(score[0]) == 0 {
		printError("X-Score header missing in %s", key)
		return
	}
	oldScore, err := strconv.Atoi(score[0])
	if err != nil {
		printError("X-Score: %v", err)
		return
	}
	newScore := post.Points()

	if oldScore != newScore {
		fmt.Printf("Points changed from %d to %d\n", oldScore, newScore)
	}
}

func renamePostDir(olddir, newdir string) {
	if fileExists(newdir) {
		bail("renamePostDir(): post title changed but directory for new title"+
			" already exist\n\told title: %s\n\tnew title: %s",
			olddir, newdir)
	}
	err := os.Rename(olddir, newdir)
	if err != nil {
		bail("renamePostDir(): Rename: %v", err)
	}
	printNotice("post title has changed:\n\told: %s\n\tnew: %s", olddir, newdir)

}

func readCacheFile(f string) {
	buf, err := ioutil.ReadFile(f)
	if err != nil {
		bail("Read %s: %v", f, err)
	}
	err = json.Unmarshal(buf, &postCache)
	if err != nil {
		bail("Unmarshal %s: %v", f, err)
	}
}

func fileExists(path string) bool {
	_, err := os.Stat(path)
	if err != nil {
		if os.IsNotExist(err) {
			return false
		} else {
			bail("fileExists: %s: %v", path, err)
		}
	}
	return true
}

func (mails *Mails) SavePost(parent, child Post, threadLevel int) (added, updated, deleted int) {
	mid := child.MessageId()
	key, exist := mails.keyForMessageId[mid]
	if exist {
		msg, err := mails.md.Message(key)
		if err != nil {
			bail("SavePost(): Error getting mail for key %s: %v", key, err)
		}

		path, err := mails.md.Filename(key)
		if err != nil {
			bail("SavePost(): could not get Filename for key %s", key)
		}
		contentChanged := updateMail(parent, child, path, msg)
		if contentChanged {
			p := strings.Replace(path, "/cur/", "/new/", 1)
			if path != p {
				err = os.Rename(path, p)
				if err != nil {
					bail("SavePost(): Rename(%s, %s): %v", path, p, err)
				}
				verbose("Renamed %s to %s", path, p)
			}
			if child.Removed() {
				deleted++
			} else {
				updated++
			}
		}
		showThreadLevel(threadLevel, "*")
	} else {
		mail := NewMail(parent, child)
		err := mails.SaveMail(mail)
		if err != nil {
			bail("SavePost(): %v", err)
		}
		showThreadLevel(threadLevel, "+")
		added++
	}

	n, u, d := mails.SavePostChildren(child, threadLevel+1)
	added += n
	updated += u
	deleted += d
	return
}

func (mails *Mails) SavePostChildren(p Post, threadLevel int) (added, updated, deleted int) {
	for _, idStr := range p.Children() {
		child, err := fetchHNPost(idStr)
		if err != nil {
			printError("error: %v", err)
			continue
		}
		if child == nil {
			printError("WARN p.Children(%s) gave nil child (could be temporary error). Parent: %+v", idStr, p)
			continue
		}
		a, u, d := mails.SavePost(p, child, threadLevel)
		added += a
		updated += u
		deleted += d
	}
	return
}

func showThreadLevel(level int, symbol string) {
	for i := 0; i < level; i++ {
		fmt.Print(" ")
	}
	fmt.Println(symbol)
}

func maybeSavePostUrl(dir, url string) {
	if !fileExists(dir) {
		os.Mkdir(dir, 0755)
	}
	f := filepath.Join(dir, "url.txt")
	if fileExists(f) {
		return
	}
	err := ioutil.WriteFile(f, []byte(url), 0644)
	if err != nil {
		bail("%s: %v", f, err)
	}
}

func mustGetPost(href string) Post {
	url, err := url.Parse(href)
	if err != nil {
		bail("bad url: %s", err)
	}

	switch strings.ToLower(url.Host) {
	case HOST_HACKER_NEWS:
		return GetHNPost(url)
	default:
		bail("Unsupported url: %s", url.Host)
	}
	return nil
}

func bail(format string, args ...interface{}) {
	printError(format, args...)
	os.Exit(1)
}

func readUrlFile(arg string) string {
	urlFile := filepath.Join(arg, "url.txt")
	buf, err := ioutil.ReadFile(urlFile)
	if err != nil {
		bail("readUrlFile: error: %v", err)
	}
	return string(buf)
}

func verbose(format string, args ...interface{}) {
	if !*Verbose {
		return
	}
	fmt.Fprintf(os.Stderr, format+"\n", args...)
}

func printError(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, format+"\n", args...)
}

func printNotice(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, "notice:"+format+"\n", args...)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/DrItanium/fakku"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
)

var category = flag.String("category", fakku.Manga, "the type of the content")
var name = flag.String("name", "", "the name of the content itself, this is usually what you would find in the URL")

func main() {
	flag.Parse()
	if *name == "" {
		log.Fatal("Did not provide a name")
	}
	if !fakku.LegalCategory(*category) {
		log.Fatalf("Illegal category %s provided", *category)
	}
	content, err := fakku.GetContent(*category, *name)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println("Name:", content.Name)
	covUrl, errcu := content.CoverUrl()
	if errcu != nil {
		log.Fatal(errcu)
	}
	fmt.Println("Cover URL:", covUrl.String())
	comments, errcm := content.Comments()
	if errcm != nil {
		log.Fatal(errcm)
	}
	fmt.Println("Comments")
	for _, comment := range comments.Comments {
		fmt.Printf("\t[%s] %s - %s\n", comment.Date(), comment.Poster, comment.Text)
	}

	// try getting the covUrl's contents
	derr0 := DownloadFile(covUrl, "cover-thumbnail", 0644)
	if derr0 != nil {
		log.Fatal(derr0)
	}
	// dump the read-online stuff
	pages, err1 := content.ReadOnline()
	if err1 != nil {
		log.Fatal(err1)
	}
	for _, page := range pages {
		purl, perr := page.ImageUrl()
		if perr != nil {
			log.Fatal(perr)
		}
		dfErr := DownloadFile(purl, fmt.Sprintf("%s.jpg", page.Id), 0644)
		if dfErr != nil {
			log.Print(dfErr)
		}
	}
}

func DownloadFile(url *url.URL, outputDir string, perms os.FileMode) error {
	resp, rerr := http.Get(url.String())
	if rerr != nil {
		return rerr
	}
	defer resp.Body.Close()
	img, ierr := ioutil.ReadAll(resp.Body)

	if ierr != nil {
		return ierr
	}
	werr := ioutil.WriteFile(outputDir, img, perms)
	if werr != nil {
		return werr
	}
	return nil
}
<commit_msg>Reverted previous changes for the index name<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/DrItanium/fakku"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
)

var category = flag.String("category", fakku.Manga, "the type of the content")
var name = flag.String("name", "", "the name of the content itself, this is usually what you would find in the URL")

func main() {
	flag.Parse()
	if *name == "" {
		log.Fatal("Did not provide a name")
	}
	if !fakku.LegalCategory(*category) {
		log.Fatalf("Illegal category %s provided", *category)
	}
	content, err := fakku.GetContent(*category, *name)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println("Name:", content.Name)
	covUrl, errcu := content.CoverUrl()
	if errcu != nil {
		log.Fatal(errcu)
	}
	fmt.Println("Cover URL:", covUrl.String())
	comments, errcm := content.Comments()
	if errcm != nil {
		log.Fatal(errcm)
	}
	fmt.Println("Comments")
	for _, comment := range comments.Comments {
		fmt.Printf("\t[%s] %s - %s\n", comment.Date(), comment.Poster, comment.Text)
	}

	// try getting the covUrl's contents
	derr0 := DownloadFile(covUrl, "cover-thumbnail", 0644)
	if derr0 != nil {
		log.Fatal(derr0)
	}
	// dump the read-online stuff
	pages, err1 := content.ReadOnline()
	if err1 != nil {
		log.Fatal(err1)
	}
	for ind, page := range pages {
		purl, perr := page.ImageUrl()
		if perr != nil {
			log.Fatal(perr)
		}
		dfErr := DownloadFile(purl, fmt.Sprintf("%d.jpg", ind), 0644)
		if dfErr != nil {
			log.Print(dfErr)
		}
	}
}

func DownloadFile(url *url.URL, outputDir string, perms os.FileMode) error {
	resp, rerr := http.Get(url.String())
	if rerr != nil {
		return rerr
	}
	defer resp.Body.Close()
	img, ierr := ioutil.ReadAll(resp.Body)

	if ierr != nil {
		return ierr
	}
	werr := ioutil.WriteFile(outputDir, img, perms)
	if werr != nil {
		return werr
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"time"
)

const TWSEURL string = "http://mis.tse.com.tw/"

//STOCKPATH = '/stock/api/getStockInfo.jsp?ex_ch=%(exchange)s_%(no)s.tw_%(date)s&json=1&delay=%(delay)s&_=%(timestamp)s'

type StockOption struct {
	no        string
	timestamp int64
}

func GenStockUrl(params StockOption) string {
	return fmt.Sprintf("%sstock/api/getStockInfo.jsp?ex_ch=%s_%s.tw_%s&json=1&delay=0&_=%d", TWSEURL, "tse", params.no, "20140811", params.timestamp)
}

type jsonBlob struct {
	Args    map[string]interface{}
	Headers map[string]interface{}
}

func main() {
	//url := "http://httpbin.org/get?name=Toomore"
	//resp, _ := http.Get(url)
	//defer resp.Body.Close()
	//resp_data, _ := ioutil.ReadAll(resp.Body)
	//var json_blob jsonBlob
	//json.Unmarshal(resp_data, &json_blob)
	//fmt.Printf("%+v", json_blob)
	//fmt.Println(json_blob.Args["name"], TWSEURL)
	option := StockOption{no: "2618", timestamp: time.Now().Unix()}
	fmt.Println(option)
	fmt.Println(GenStockUrl(option))
}
<commit_msg>Add date.<commit_after>package main

import (
	"fmt"
	"time"
)

const TWSEURL string = "http://mis.tse.com.tw/"

//STOCKPATH = '/stock/api/getStockInfo.jsp?ex_ch=%(exchange)s_%(no)s.tw_%(date)s&json=1&delay=%(delay)s&_=%(timestamp)s'

type StockOption struct {
	no        string
	timestamp int64
	date      time.Time
}

func GenStockUrl(params StockOption) string {
	return fmt.Sprintf(
		"%sstock/api/getStockInfo.jsp?ex_ch=%s_%s.tw_%s&json=1&delay=0&_=%d",
		TWSEURL,
		"tse",
		params.no,
		fmt.Sprintf(
			"%d%02d%02d",
			params.date.Year(),
			int(params.date.Month()),
			params.date.Day(),
		),
		params.timestamp,
	)
}

type jsonBlob struct {
	Args    map[string]interface{}
	Headers map[string]interface{}
}

func main() {
	//url := "http://httpbin.org/get?name=Toomore"
	//resp, _ := http.Get(url)
	//defer resp.Body.Close()
	//resp_data, _ := ioutil.ReadAll(resp.Body)
	//var json_blob jsonBlob
	//json.Unmarshal(resp_data, &json_blob)
	//fmt.Printf("%+v", json_blob)
	//fmt.Println(json_blob.Args["name"], TWSEURL)
	option := StockOption{
		no:        "2618",
		timestamp: time.Now().Unix(),
		date:      time.Date(2014, time.August, 11, 0, 0, 0, 0, time.Local),
	}
	fmt.Println(option)
	fmt.Println(GenStockUrl(option))
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"

	"github.com/yarbelk/refasta/formats"
	"github.com/yarbelk/refasta/sequence"
	"gopkg.in/urfave/cli.v1"
)

type FakeReadCloser struct {
	io.Reader
}

func (f FakeReadCloser) Close() error {
	return nil
}

type FakeWriteCloser struct {
	io.Writer
}

type TNTContext struct {
	Title    string
	Outgroup string
}

func (f FakeWriteCloser) Close() error {
	return nil
}

func getOutputFilePointer(filename string) (io.WriteCloser, error) {
	if filename == "--" {
		return FakeWriteCloser{os.Stdout}, nil
	}
	return os.Create(filename)
}

func getInputFilePointer(filename string) (io.ReadCloser, error) {
	if filename == "--" {
		return FakeReadCloser{os.Stdin}, nil
	}
	return os.Open(filename)
}

func IsDirectory(path string) (bool, error) {
	fileInfo, err := os.Stat(path)
	return fileInfo.IsDir(), err
}

func isFormat(file, format string) bool {
	switch path.Ext(file) {
	case ".fas", ".fasta":
		return format == formats.FASTA_FORMAT
	default:
		return false
	}
}

func dirInput(dir, format string, recurse bool) ([]string, error) {
	absDir, err := filepath.Abs(dir)
	if err != nil {
		return nil, err
	}
	files, err := ioutil.ReadDir(absDir)
	if err != nil {
		return nil, err
	}
	filteredFiles := make([]string, 0, 10)
	for _, file := range files {
		if file.IsDir() && recurse {
			f, err := dirInput(filepath.Join(absDir, file.Name()), format, true)
			if err != nil {
				return nil, err
			}
			filteredFiles = append(filteredFiles, f...)
		} else if isFormat(file.Name(), format) {
			filteredFiles = append(filteredFiles, filepath.Join(absDir, file.Name()))
		}
	}
	return filteredFiles, nil
}

func isDirectory(path string) (bool, error) {
	fileInfo, err := os.Stat(path)
	if err != nil {
		return false, err
	}
	return fileInfo.IsDir(), err
}

func handleFastaInput(input string) ([]sequence.Sequence, error) {
	var files []string
	var sequences []sequence.Sequence

	if isDir, err := isDirectory(input); isDir && err == nil {
		files, err = dirInput(input, formats.FASTA_FORMAT, true)
		if err != nil {
			// Some error in walking the directory tree
			return nil, err
		}
	} else {
		files = []string{input}
	}

	for _, file := range files {
		ext := path.Ext(file)
		geneName := filepath.Base(file[:len(file)-len(ext)])
		err := func() error {
			fasta := formats.Fasta{SpeciesFromID: true}
			fd, err := os.Open(file)
			if err != nil {
				// probably an Access Control issue, or race condition
				return err
			}
			defer fd.Close()
			err = fasta.Parse(fd, geneName)
			if err != nil {
				// Some parsing error...
				return err
			}
			sequences = append(sequences, fasta.Sequences...)
			return nil
		}()
		if err != nil {
			return nil, err
		}
	}
	return sequences, nil
}

func handleFastaOutput(sequences []sequence.Sequence, output string) error {
	fasta := formats.Fasta{}
	fasta.AddSequence(sequences...)
	fd, err := os.Create(output)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Issue opening output file;\n%s", err.Error())
	}
	defer fd.Close()
	return fasta.WriteSequences(fd)
}

func handleTNTOutput(context TNTContext, sequences []sequence.Sequence, output string) error {
	tnt := formats.TNT{Title: context.Title}
	tnt.AddSequence(sequences...)
	tnt.SetOutgroup(context.Outgroup)
	fd, err := os.Create(output)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Issue opening output file;\n%s", err.Error())
	}
	defer fd.Close()
	return tnt.WriteSequences(fd)
}

func main() {

	app := cli.NewApp()
	app.Name = "refasta"
	app.Usage = `Convert various genitics data formats into other formats.
	Currently only fasta and tnt are supported, and in an opinionated way.`

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "input-format, f",
			Value: formats.FASTA_FORMAT,
			Usage: "INPUT_FORMAT must be one of the supported input types. Currently supported is 'fasta'",
		},
		cli.StringFlag{
			Name:  "output-format, F",
			Value: formats.FASTA_FORMAT,
			Usage: "OUTPUT_FORMAT must be one of the supported output types. Currently supported are 'fasta' and 'tnt'",
		},
		cli.StringFlag{
			Name:  "input-file, i",
			Value: "--",
			Usage: "INPUT_FILE, it must be a valid input, or '--', or blank.  if blank. or '--', will read from stdin",
		},
		cli.StringFlag{
			Name:  "output-file, o",
			Value: "--",
			Usage: "OUTPUT_FILE, it must be a valid input, or '--', or blank.  if blank. or '--', will write to stdout",
		},
		cli.StringFlag{
			Name:  "tnt-title, t",
			Value: "",
			Usage: "title for TNT output",
		},
		cli.StringFlag{
			Name:  "outgroup",
			Value: "",
			Usage: "Optional OUTGROUP for TNT output.  If specified, this species will be used as the outgroup for TNT. " +
				"Otherwise the first (alphabetically) will be used.  This must be left blank, or be a valid species name " +
				"from the input",
		},
	}
	fatalWithUsage := func(c *cli.Context, err error) {
		if err != nil {
			fmt.Fprintln(c.App.Writer, err.Error())
			cli.ShowAppHelp(c)
			os.Exit(1)
		}
	}

	app.Action = func(c *cli.Context) {
		var sequences []sequence.Sequence
		var err error

		input := c.String("input-file")
		output := c.String("output-file")
		inputFormat := c.String("input-format")
		outputFormat := c.String("output-format")

		switch inputFormat {
		case formats.FASTA_FORMAT:
			sequences, err = handleFastaInput(input)
		default:
			err = fmt.Errorf("Unknown intput format '%s'", inputFormat)
		}
		fatalWithUsage(c, err)

		switch outputFormat {
		case formats.FASTA_FORMAT:
			err = handleFastaOutput(sequences, output)
		case formats.TNT_FORMAT:
			fmt.Fprintf(os.Stderr, "Output format is TNT; serializing\n")
			context := TNTContext{
				Title:    c.String("title"),
				Outgroup: c.String("outgroup"),
			}
			err = handleTNTOutput(context, sequences, output)
		default:
			err = fmt.Errorf("Unknown output format '%s'", inputFormat)
		}
		fatalWithUsage(c, err)
	}

	app.Run(os.Args)
}
<commit_msg>Use subcommands for output file types.<commit_after>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"

	"github.com/yarbelk/refasta/formats"
	"github.com/yarbelk/refasta/sequence"
	"gopkg.in/urfave/cli.v1"
)

var sequences []sequence.Sequence

type CommandError struct {
	error
	c *cli.Context
}

func (c CommandError) UsageAndFail() {
	if c.error != nil {
		fmt.Fprintln(c.c.App.Writer, c.error.Error())
		cli.ShowAppHelp(c.c)
		os.Exit(1)
	}

}

type FakeReadCloser struct {
	io.Reader
}

func (f FakeReadCloser) Close() error {
	return nil
}

type FakeWriteCloser struct {
	io.Writer
}

type TNTContext struct {
	Title    string
	Outgroup string
}

func (f FakeWriteCloser) Close() error {
	return nil
}

func getOutputFilePointer(filename string) (io.WriteCloser, error) {
	if filename == "--" {
		return FakeWriteCloser{os.Stdout}, nil
	}
	return os.Create(filename)
}

func getInputFilePointer(filename string) (io.ReadCloser, error) {
	if filename == "--" {
		return FakeReadCloser{os.Stdin}, nil
	}
	return os.Open(filename)
}

func IsDirectory(path string) (bool, error) {
	fileInfo, err := os.Stat(path)
	return fileInfo.IsDir(), err
}

func isFormat(file, format string) bool {
	switch path.Ext(file) {
	case ".fas", ".fasta":
		return format == formats.FASTA_FORMAT
	default:
		return false
	}
}

func dirInput(dir, format string, recurse bool) ([]string, error) {
	absDir, err := filepath.Abs(dir)
	if err != nil {
		return nil, err
	}
	files, err := ioutil.ReadDir(absDir)
	if err != nil {
		return nil, err
	}
	filteredFiles := make([]string, 0, 10)
	for _, file := range files {
		if file.IsDir() && recurse {
			f, err := dirInput(filepath.Join(absDir, file.Name()), format, true)
			if err != nil {
				return nil, err
			}
			filteredFiles = append(filteredFiles, f...)
		} else if isFormat(file.Name(), format) {
			filteredFiles = append(filteredFiles, filepath.Join(absDir, file.Name()))
		}
	}
	return filteredFiles, nil
}

func isDirectory(path string) (bool, error) {
	fileInfo, err := os.Stat(path)
	if err != nil {
		return false, err
	}
	return fileInfo.IsDir(), err
}

func handleFastaInput(input string) ([]sequence.Sequence, error) {
	var files []string
	var sequences []sequence.Sequence

	if isDir, err := isDirectory(input); isDir && err == nil {
		files, err = dirInput(input, formats.FASTA_FORMAT, true)
		if err != nil {
			// Some error in walking the directory tree
			return nil, err
		}
	} else {
		files = []string{input}
	}

	for _, file := range files {
		ext := path.Ext(file)
		geneName := filepath.Base(file[:len(file)-len(ext)])
		err := func() error {
			fasta := formats.Fasta{SpeciesFromID: true}
			fd, err := os.Open(file)
			if err != nil {
				// probably an Access Control issue, or race condition
				return err
			}
			defer fd.Close()
			err = fasta.Parse(fd, geneName)
			if err != nil {
				// Some parsing error...
				return err
			}
			sequences = append(sequences, fasta.Sequences...)
			return nil
		}()
		if err != nil {
			return nil, err
		}
	}
	return sequences, nil
}

func handleFastaOutput(sequences []sequence.Sequence, output string) error {
	fasta := formats.Fasta{}
	fasta.AddSequence(sequences...)
	fd, err := os.Create(output)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Issue opening output file;\n%s", err.Error())
	}
	defer fd.Close()
	return fasta.WriteSequences(fd)
}

func handleTNTOutput(context TNTContext, sequences []sequence.Sequence, output string) error {
	tnt := formats.TNT{Title: context.Title}
	tnt.AddSequence(sequences...)
	tnt.SetOutgroup(context.Outgroup)
	fd, err := os.Create(output)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Issue opening output file;\n%s", err.Error())
	}
	defer fd.Close()
	return tnt.WriteSequences(fd)
}

func parseInput(c *cli.Context) error {
	var err error
	var inputFormat string = c.String("intput-format")
	switch inputFormat {
	case formats.FASTA_FORMAT:
		sequences, err = handleFastaInput(c.String("input"))
	default:
		err = CommandError{fmt.Errorf("Unknown intput format '%s'", inputFormat), c}
	}
	return err
}

func main() {

	app := cli.NewApp()
	app.Name = "refasta"
	app.Usage = "Convert various genitics data formats into other formats. " +
		"Currently only fasta and tnt are supported, and in an opinionated way."
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "input, i",
			Value: "",
			Usage: "`INPUT`, it must be either a file or directory.  If blank, stdin will be used",
		},
		cli.StringFlag{
			Name:  "input-format, f",
			Value: formats.FASTA_FORMAT,
			Usage: "`INPUT_FORMAT` must be one of the supported input types. Currently only 'fasta' is supported",
		},
	}

	app.Commands = []cli.Command{
		cli.Command{
			Name:        "fasta",
			Usage:       "Convert to `fasta` format",
			UsageText:   "This will convert the input to a fasta formatted file.",
			Description: "This requires an input file or directory, and an input format.  If you do not specify an OUTPUT_FILE, then the output will be written to stdout",
			ArgsUsage:   "[OUTPUT_FILE]",
			Before:      parseInput,
			Action: func(c *cli.Context) error {
				return handleFastaOutput(sequences, c.Args().First())
			},
		},
	}

	app.Commands = []cli.Command{
		cli.Command{
			Name:        "tnt",
			Usage:       "Convert to `TNT` format",
			UsageText:   "This will convert the input to a TNT formatted file.",
			Description: "This requires an input file or directory, and an input format.  You can specify the outgroup and title of the file.  If you do not specify an OUTPUT_FILE, then the output will be written to stdout",
			ArgsUsage:   "[OUTPUT_FILE]",
			Before:      parseInput,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "outgroup",
					Value: "",
					Usage: "Optional `OUTGROUP` for TNT output.  If specified, this species will be used as the outgroup for TNT. " +
						"Otherwise the first (alphabetically) will be used.  This must be left blank, or be a valid species name " +
						"from the input",
				},
				cli.StringFlag{
					Name:  "tnt-title, t",
					Value: "",
					Usage: "`TITLE` for TNT output",
				},
			},
			Action: func(c *cli.Context) error {
				fmt.Fprintf(os.Stderr, "Output format is TNT; serializing\n")
				context := TNTContext{
					Title:    c.String("title"),
					Outgroup: c.String("outgroup"),
				}
				return handleTNTOutput(context, sequences, c.Args().First())
			},
		},
	}

	if err := app.Run(os.Args); err != nil {
		switch e := err.(type) {
		case CommandError:
			e.UsageAndFail()
		default:
			fmt.Fprintln(os.Stderr, err.Error())
			os.Exit(1)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"github.com/viktorbenei/depman/depman"
	"log"
	"os"
)

type Command struct {
	Name  string
	Usage string
	Run   func() error
}

var (
	availableCommands = []Command{
		Command{
			Name:  "update",
			Usage: "update - updates",
			Run:   doUpdateCommand,
		},
		Command{
			Name:  "init",
			Usage: "init - creates a deplist.json file in the current folder",
			Run:   doInitCommand,
		},
	}
)

func usage() {
	fmt.Fprintf(os.Stderr, "Usage: %s command [FLAGS]", os.Args[0])
	flag.PrintDefaults()
	fmt.Println("Available commands:")
	for _, cmd := range availableCommands {
		fmt.Println(" *", cmd.Name)
		fmt.Println("    ", cmd.Usage)
	}
}

func readDepListFile() (depman.DepList, error) {
	deplist, err := depman.ReadDepListFromFile("./deplist.json")
	if err != nil {
		return depman.DepList{}, errors.New(fmt.Sprintf("Failed to load deplist: %s", err))
	}
	return deplist, nil
}

func doUpdateCommand() error {
	deplist, err := readDepListFile()
	if err != nil {
		return err
	}

	log.Printf("Updating %d dependencies...\n", len(deplist.Deps))
	deplocks, err := depman.PerformUpdateOnDepList(deplist)
	if err != nil {
		return errors.New(fmt.Sprintf("Update failed: %s", err))
	}
	// write DepLocks
	log.Println("Writing deplock...")
	if err := depman.WriteDepLocksToFile("./deplock.json", deplocks); err != nil {
		return errors.New(fmt.Sprintf("Failed to write deplock.json: %s", err))
	}
	log.Println("Update finished!")
	return nil
}

func doInitCommand() error {
	deplist := depman.DepList{
		Deps: []depman.DepStruct{},
	}
	err := depman.WriteDepListToFile("./deplist.json", deplist)
	if err != nil {
		return err
	}
	fmt.Println("deplist.json file saved")
	return nil
}

func main() {
	fmt.Println("DepMan")

	flag.Usage = usage
	flag.Parse()

	args := flag.Args()
	// fmt.Println(args)
	if len(args) < 1 {
		flag.Usage()
		os.Exit(1)
	}

	theCommandName := args[0]

	for _, cmd := range availableCommands {
		if cmd.Name == theCommandName {
			// cmd.Flag.Usage = func() { cmd.UsageExit() }
			// cmd.Flag.Parse(args[1:])
			err := cmd.Run()
			if err != nil {
				log.Fatalln(err)
			}
			return
		}
	}
}
<commit_msg>depman init: instead of empty deplist.json it creates an example item<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"github.com/viktorbenei/depman/depman"
	"log"
	"os"
)

type Command struct {
	Name  string
	Usage string
	Run   func() error
}

var (
	availableCommands = []Command{
		Command{
			Name:  "update",
			Usage: "update - updates",
			Run:   doUpdateCommand,
		},
		Command{
			Name:  "init",
			Usage: "init - creates a deplist.json file in the current folder",
			Run:   doInitCommand,
		},
	}
)

func usage() {
	fmt.Fprintf(os.Stderr, "Usage: %s command [FLAGS]", os.Args[0])
	flag.PrintDefaults()
	fmt.Println("Available commands:")
	for _, cmd := range availableCommands {
		fmt.Println(" *", cmd.Name)
		fmt.Println("    ", cmd.Usage)
	}
}

func readDepListFile() (depman.DepList, error) {
	deplist, err := depman.ReadDepListFromFile("./deplist.json")
	if err != nil {
		return depman.DepList{}, errors.New(fmt.Sprintf("Failed to load deplist: %s", err))
	}
	return deplist, nil
}

func doUpdateCommand() error {
	deplist, err := readDepListFile()
	if err != nil {
		return err
	}

	log.Printf("Updating %d dependencies...\n", len(deplist.Deps))
	deplocks, err := depman.PerformUpdateOnDepList(deplist)
	if err != nil {
		return errors.New(fmt.Sprintf("Update failed: %s", err))
	}
	// write DepLocks
	log.Println("Writing deplock...")
	if err := depman.WriteDepLocksToFile("./deplock.json", deplocks); err != nil {
		return errors.New(fmt.Sprintf("Failed to write deplock.json: %s", err))
	}
	log.Println("Update finished!")
	return nil
}

func doInitCommand() error {
	deplist := depman.DepList{
		Deps: []depman.DepStruct{
			depman.DepStruct{
				URL:       "http://repo.url",
				StorePath: "relative/store/path",
			},
		},
	}
	err := depman.WriteDepListToFile("./deplist.json", deplist)
	if err != nil {
		return err
	}
	fmt.Println("deplist.json file saved")
	return nil
}

func main() {
	fmt.Println("DepMan")

	flag.Usage = usage
	flag.Parse()

	args := flag.Args()
	// fmt.Println(args)
	if len(args) < 1 {
		flag.Usage()
		os.Exit(1)
	}

	theCommandName := args[0]

	for _, cmd := range availableCommands {
		if cmd.Name == theCommandName {
			// cmd.Flag.Usage = func() { cmd.UsageExit() }
			// cmd.Flag.Parse(args[1:])
			err := cmd.Run()
			if err != nil {
				log.Fatalln(err)
			}
			return
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/hex"
	"encoding/json"
	_ "expvar"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	_ "net/http/pprof"
	"net/url"
	"runtime"
	"sync"

	pickle "github.com/kisielk/og-rek"
)

var Debug int

var Config = struct {
	Backends []string
	MaxProcs int
	Port     int

	mu          sync.RWMutex
	metricPaths map[string][]string
}{
	MaxProcs:    1,
	Port:        8080,
	metricPaths: make(map[string][]string),
}

type serverResponse struct {
	server   string
	response []byte
}

func multiGet(servers []string, uri string) []serverResponse {

	ch := make(chan serverResponse)

	if Debug > 0 {
		log.Println("querying servers=", servers, "uri=", uri)
	}

	for _, server := range servers {
		go func(server string, ch chan<- serverResponse) {

			u, err := url.Parse(server + uri)
			if err != nil {
				log.Fatal(err)
			}
			req := http.Request{
				URL:    u,
				Header: make(http.Header),
			}

			resp, err := http.DefaultClient.Do(&req)

			if err != nil || resp.StatusCode != 200 {
				log.Println("got status code", resp.StatusCode, "while querying", server, "/", uri)
				ch <- serverResponse{server, nil}
			}
			defer resp.Body.Close()

			body, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				ch <- serverResponse{server, nil}
			}

			ch <- serverResponse{server, body}
		}(server, ch)
	}

	var response []serverResponse

	for i := 0; i < len(servers); i++ {
		r := <-ch
		if r.response != nil {
			response = append(response, r)
		}
	}

	return response
}

func findHandler(w http.ResponseWriter, req *http.Request) {

	if Debug > 0 {
		log.Println("request: ", req.URL.RequestURI())
	}

	responses := multiGet(Config.Backends, req.URL.RequestURI())

	if responses == nil || len(responses) == 0 {
		log.Println("error querying backends for: ", req.URL.RequestURI())
		http.Error(w, "error querying backends", http.StatusInternalServerError)
		return
	}

	// metric -> [server1, ... ]
	paths := make(map[string][]string)

	var metrics []map[interface{}]interface{}
	for _, r := range responses {
		d := pickle.NewDecoder(bytes.NewReader(r.response))
		metric, err := d.Decode()
		if err != nil {
			log.Printf("error decoding response from server:%s: req:%s: err=%s", r.server, req.URL.RequestURI(), err)
			if Debug > 1 {
				log.Println("\n" + hex.Dump(r.response))
			}
			continue
		}

		marray, ok := metric.([]interface{})
		if !ok {
			log.Printf("bad type for metric:%t from server:%s: req:%s", metric, r.server, req.URL.RequestURI())
			http.Error(w, fmt.Sprintf("bad type for metric: %t", metric), http.StatusInternalServerError)
			return
		}

		for i, m := range marray {
			mm, ok := m.(map[interface{}]interface{})
			if !ok {
				log.Printf("bad type for metric[%d]:%t from server:%s: req:%s", i, m, r.server, req.URL.RequestURI())
				http.Error(w, fmt.Sprintf("bad type for metric[%d]:%t", i, m), http.StatusInternalServerError)
				return
			}
			name, ok := mm["metric_path"].(string)
			if !ok {
				log.Printf("bad type for metric_path:%t from server:%s: req:%s", mm["metric_path"], r.server, req.URL.RequestURI())
				http.Error(w, fmt.Sprintf("bad type for metric_path: %t", mm["metric_path"]), http.StatusInternalServerError)
				return
			}
			p, ok := paths[name]
			if !ok {
				// we haven't seen this name yet
				// add the metric to the list of metrics to return
				metrics = append(metrics, mm)
			}
			// add the server to the list of servers that know about this metric
			p = append(p, r.server)
			paths[name] = p
		}
	}

	// update our cache of which servers have which metrics
	Config.mu.Lock()
	for k, v := range paths {
		Config.metricPaths[k] = v
	}
	Config.mu.Unlock()

	w.Header().Set("Content-Type", "application/pickle")

	pEnc := pickle.NewEncoder(w)
	pEnc.Encode(metrics)
}

func renderHandler(w http.ResponseWriter, req *http.Request) {

	if Debug > 0 {
		log.Println("request: ", req.URL.RequestURI())
	}

	req.ParseForm()
	target := req.FormValue("target")

	if target == "" {
		http.Error(w, "empty target", http.StatusBadRequest)
		return
	}

	var serverList []string
	var ok bool

	Config.mu.RLock()
	// lookup the server list for this metric, or use all the servers if it's unknown
	if serverList, ok = Config.metricPaths[target]; !ok {
		serverList = Config.Backends
	}
	Config.mu.RUnlock()

	responses := multiGet(serverList, req.URL.RequestURI())

	if responses == nil || len(responses) == 0 {
		log.Println("error querying backends for: ", req.URL.RequestURI())
		http.Error(w, "error querying backends", http.StatusInternalServerError)
		return
	}

	// nothing to merge
	if len(responses) == 1 {
		w.Header().Set("Content-Type", "application/pickle")
		w.Write(responses[0].response)
	}

	// decode everything
	var decoded [][]interface{}
	for _, r := range responses {
		d := pickle.NewDecoder(bytes.NewReader(r.response))
		metric, err := d.Decode()
		if err != nil {
			log.Printf("error decoding response from server:%s: req:%s: err=%s", r.server, req.URL.RequestURI(), err)
			if Debug > 1 {
				log.Println("\n" + hex.Dump(r.response))
			}
			continue
		}

		marray, ok := metric.([]interface{})
		if !ok {
			err := fmt.Sprintf("bad type for metric:%d from server:%s req:%s", metric, r.server, req.URL.RequestURI())
			log.Println(err)
			http.Error(w, err, http.StatusInternalServerError)
			return
		}
		if len(marray) == 0 {
			continue
		}
		decoded = append(decoded, marray)
	}

	if len(decoded) == 0 {
		log.Printf("no decoded responses to merge for req:%s", req.URL.RequestURI())
		w.Header().Set("Content-Type", "application/pickle")
		w.Write(responses[0].response)
		return
	}

	if len(decoded) == 1 {
		log.Printf("only one decoded responses to merge for req:%s", req.URL.RequestURI())
		w.Header().Set("Content-Type", "application/pickle")
		// send back whatever data we have
		e := pickle.NewEncoder(w)
		e.Encode(decoded[0])
		return
	}

	if len(decoded[0]) != 1 {
		err := fmt.Sprintf("bad length for decoded[]:%d from req:%s", len(decoded[0]), req.URL.RequestURI())
		log.Println(err)
		http.Error(w, err, http.StatusInternalServerError)
		return
	}

	base, ok := decoded[0][0].(map[interface{}]interface{})
	if !ok {
		err := fmt.Sprintf("bad type for decoded:%t from req:%s", decoded[0][0], req.URL.RequestURI())
		log.Println(err)
		http.Error(w, err, http.StatusInternalServerError)
		return
	}

	values, ok := base["values"].([]interface{})
	if !ok {
		err := fmt.Sprintf("bad type for values:%t from req:%s", base["values"], req.URL.RequestURI())
		log.Println(err)
		http.Error(w, err, http.StatusInternalServerError)
		return
	}

	for i := 0; i < len(values); i++ {
		if _, ok := values[i].(pickle.None); ok {
			// find one in the other values arrays
		replacenone:
			for other := 1; other < len(decoded); other++ {
				m, ok := decoded[other][0].(map[interface{}]interface{})
				if !ok {
					log.Println(fmt.Sprintf("bad type for decoded[%d][0]: %t", other, decoded[other][0]))
					continue
				}

				ovalues, ok := m["values"].([]interface{})
				if !ok {
					log.Printf("bad type for ovalues:%t from req:%s (skipping)", m["values"], req.URL.RequestURI())
					continue
				}

				if len(ovalues) <= i {
					log.Printf("bad ovalues length for %s: need offset %d but len(ovalues)=%d", req.URL.RequestURI(), i, len(ovalues))
					continue
				}

				if _, ok := ovalues[i].(pickle.None); !ok {
					values[i] = ovalues[i]
					break replacenone
				}
			}
		}
	}

	// the first response is where we've been filling in our data, so we're ok just to serialize it as our response
	w.Header().Set("Content-Type", "application/pickle")
	e := pickle.NewEncoder(w)
	e.Encode(decoded[0])
}

func stripCommentHeader(cfg []byte) []byte {

	// strip out the comment header block that begins with '#' characters
	// as soon as we see a line that starts with something _other_ than '#', we're done

	idx := 0
	for cfg[0] == '#' {
		idx = bytes.Index(cfg, []byte("\n"))
		if idx == -1 || idx+1 == len(cfg) {
			return nil
		}
		cfg = cfg[idx+1:]
	}

	return cfg
}

func main() {

	configFile := flag.String("c", "", "config file (json)")
	port := flag.Int("p", 0, "port to listen on")
	maxprocs := flag.Int("maxprocs", 0, "GOMAXPROCS")
	flag.IntVar(&Debug, "d", 0, "enable debug logging")

	flag.Parse()

	if *configFile == "" {
		log.Fatal("missing config file")
	}

	cfgjs, err := ioutil.ReadFile(*configFile)
	if err != nil {
		log.Fatal("unable to load config file:", err)
	}

	cfgjs = stripCommentHeader(cfgjs)

	if cfgjs == nil {
		log.Fatal("error removing header comment from ", *configFile)
	}

	err = json.Unmarshal(cfgjs, &Config)
	if err != nil {
		log.Fatal("error parsing config file: ", err)
	}

	if len(Config.Backends) == 0 {
		log.Fatal("no Backends loaded -- exiting")
	}

	// command line overrides config file

	if *port != 0 {
		Config.Port = *port
	}

	if *maxprocs != 0 {
		Config.MaxProcs = *maxprocs
	}

	log.Println("setting GOMAXPROCS=", Config.MaxProcs)
	runtime.GOMAXPROCS(Config.MaxProcs)

	http.HandleFunc("/metrics/find/", findHandler)
	http.HandleFunc("/render/", renderHandler)

	portStr := fmt.Sprintf(":%d", Config.Port)
	log.Println("listening on", portStr)
	log.Fatal(http.ListenAndServe(portStr, nil))
}
<commit_msg>add more debug logging<commit_after>package main

import (
	"bytes"
	"encoding/hex"
	"encoding/json"
	_ "expvar"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	_ "net/http/pprof"
	"net/url"
	"runtime"
	"sync"

	pickle "github.com/kisielk/og-rek"
)

var Debug int

var Config = struct {
	Backends []string
	MaxProcs int
	Port     int

	mu          sync.RWMutex
	metricPaths map[string][]string
}{
	MaxProcs:    1,
	Port:        8080,
	metricPaths: make(map[string][]string),
}

type serverResponse struct {
	server   string
	response []byte
}

func multiGet(servers []string, uri string) []serverResponse {

	ch := make(chan serverResponse)

	if Debug > 0 {
		log.Println("querying servers=", servers, "uri=", uri)
	}

	for _, server := range servers {
		go func(server string, ch chan<- serverResponse) {

			u, err := url.Parse(server + uri)
			if err != nil {
				log.Fatal(err)
			}
			req := http.Request{
				URL:    u,
				Header: make(http.Header),
			}

			resp, err := http.DefaultClient.Do(&req)

			if err != nil || resp.StatusCode != 200 {
				log.Println("got status code", resp.StatusCode, "while querying", server, "/", uri)
				ch <- serverResponse{server, nil}
			}
			defer resp.Body.Close()

			body, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				ch <- serverResponse{server, nil}
			}

			ch <- serverResponse{server, body}
		}(server, ch)
	}

	var response []serverResponse

	for i := 0; i < len(servers); i++ {
		r := <-ch
		if r.response != nil {
			response = append(response, r)
		}
	}

	return response
}

func findHandler(w http.ResponseWriter, req *http.Request) {

	if Debug > 0 {
		log.Println("request: ", req.URL.RequestURI())
	}

	responses := multiGet(Config.Backends, req.URL.RequestURI())

	if responses == nil || len(responses) == 0 {
		log.Println("error querying backends for: ", req.URL.RequestURI())
		http.Error(w, "error querying backends", http.StatusInternalServerError)
		return
	}

	// metric -> [server1, ... ]
	paths := make(map[string][]string)

	var metrics []map[interface{}]interface{}
	for _, r := range responses {
		d := pickle.NewDecoder(bytes.NewReader(r.response))
		metric, err := d.Decode()
		if err != nil {
			log.Printf("error decoding response from server:%s: req:%s: err=%s", r.server, req.URL.RequestURI(), err)
			if Debug > 1 {
				log.Println("\n" + hex.Dump(r.response))
			}
			continue
		}

		marray, ok := metric.([]interface{})
		if !ok {
			log.Printf("bad type for metric:%t from server:%s: req:%s", metric, r.server, req.URL.RequestURI())
			http.Error(w, fmt.Sprintf("bad type for metric: %t", metric), http.StatusInternalServerError)
			return
		}

		for i, m := range marray {
			mm, ok := m.(map[interface{}]interface{})
			if !ok {
				log.Printf("bad type for metric[%d]:%t from server:%s: req:%s", i, m, r.server, req.URL.RequestURI())
				http.Error(w, fmt.Sprintf("bad type for metric[%d]:%t", i, m), http.StatusInternalServerError)
				return
			}
			name, ok := mm["metric_path"].(string)
			if !ok {
				log.Printf("bad type for metric_path:%t from server:%s: req:%s", mm["metric_path"], r.server, req.URL.RequestURI())
				http.Error(w, fmt.Sprintf("bad type for metric_path: %t", mm["metric_path"]), http.StatusInternalServerError)
				return
			}
			p, ok := paths[name]
			if !ok {
				// we haven't seen this name yet
				// add the metric to the list of metrics to return
				metrics = append(metrics, mm)
			}
			// add the server to the list of servers that know about this metric
			p = append(p, r.server)
			paths[name] = p
		}
	}

	// update our cache of which servers have which metrics
	Config.mu.Lock()
	for k, v := range paths {
		Config.metricPaths[k] = v
	}
	Config.mu.Unlock()

	w.Header().Set("Content-Type", "application/pickle")

	pEnc := pickle.NewEncoder(w)
	pEnc.Encode(metrics)
}

func renderHandler(w http.ResponseWriter, req *http.Request) {

	if Debug > 0 {
		log.Println("request: ", req.URL.RequestURI())
	}

	req.ParseForm()
	target := req.FormValue("target")

	if target == "" {
		http.Error(w, "empty target", http.StatusBadRequest)
		return
	}

	var serverList []string
	var ok bool

	Config.mu.RLock()
	// lookup the server list for this metric, or use all the servers if it's unknown
	if serverList, ok = Config.metricPaths[target]; !ok {
		serverList = Config.Backends
	}
	Config.mu.RUnlock()

	responses := multiGet(serverList, req.URL.RequestURI())

	if responses == nil || len(responses) == 0 {
		log.Println("error querying backends for: ", req.URL.RequestURI())
		http.Error(w, "error querying backends", http.StatusInternalServerError)
		return
	}

	// nothing to merge
	if len(responses) == 1 {
		w.Header().Set("Content-Type", "application/pickle")
		w.Write(responses[0].response)
	}

	// decode everything
	var decoded [][]interface{}
	for _, r := range responses {
		d := pickle.NewDecoder(bytes.NewReader(r.response))
		metric, err := d.Decode()
		if err != nil {
			log.Printf("error decoding response from server:%s: req:%s: err=%s", r.server, req.URL.RequestURI(), err)
			if Debug > 1 {
				log.Println("\n" + hex.Dump(r.response))
			}
			continue
		}

		marray, ok := metric.([]interface{})
		if !ok {
			err := fmt.Sprintf("bad type for metric:%d from server:%s req:%s", metric, r.server, req.URL.RequestURI())
			log.Println(err)
			http.Error(w, err, http.StatusInternalServerError)
			return
		}
		if len(marray) == 0 {
			continue
		}
		decoded = append(decoded, marray)
	}

	if Debug > 2 {
		log.Printf("request: %s: %v", req.URL.RequestURI(), decoded)
	}

	if len(decoded) == 0 {
		log.Printf("no decoded responses to merge for req:%s", req.URL.RequestURI())
		w.Header().Set("Content-Type", "application/pickle")
		w.Write(responses[0].response)
		return
	}

	if len(decoded) == 1 {
		log.Printf("only one decoded responses to merge for req:%s", req.URL.RequestURI())
		w.Header().Set("Content-Type", "application/pickle")
		// send back whatever data we have
		e := pickle.NewEncoder(w)
		e.Encode(decoded[0])
		return
	}

	if len(decoded[0]) != 1 {
		err := fmt.Sprintf("bad length for decoded[]:%d from req:%s", len(decoded[0]), req.URL.RequestURI())
		log.Println(err)
		http.Error(w, err, http.StatusInternalServerError)
		return
	}

	base, ok := decoded[0][0].(map[interface{}]interface{})
	if !ok {
		err := fmt.Sprintf("bad type for decoded:%t from req:%s", decoded[0][0], req.URL.RequestURI())
		log.Println(err)
		http.Error(w, err, http.StatusInternalServerError)
		return
	}

	values, ok := base["values"].([]interface{})
	if !ok {
		err := fmt.Sprintf("bad type for values:%t from req:%s", base["values"], req.URL.RequestURI())
		log.Println(err)
		http.Error(w, err, http.StatusInternalServerError)
		return
	}

	for i := 0; i < len(values); i++ {
		if _, ok := values[i].(pickle.None); ok {
			// find one in the other values arrays
		replacenone:
			for other := 1; other < len(decoded); other++ {
				m, ok := decoded[other][0].(map[interface{}]interface{})
				if !ok {
					log.Println(fmt.Sprintf("bad type for decoded[%d][0]: %t", other, decoded[other][0]))
					continue
				}

				ovalues, ok := m["values"].([]interface{})
				if !ok {
					log.Printf("bad type for ovalues:%t from req:%s (skipping)", m["values"], req.URL.RequestURI())
					continue
				}

				if len(ovalues) <= i {
					log.Printf("bad ovalues length for %s: need offset %d but len(ovalues)=%d", req.URL.RequestURI(), i, len(ovalues))
					continue
				}

				if _, ok := ovalues[i].(pickle.None); !ok {
					values[i] = ovalues[i]
					break replacenone
				}
			}
		}
	}

	// the first response is where we've been filling in our data, so we're ok just to serialize it as our response
	w.Header().Set("Content-Type", "application/pickle")
	e := pickle.NewEncoder(w)
	e.Encode(decoded[0])
}

func stripCommentHeader(cfg []byte) []byte {

	// strip out the comment header block that begins with '#' characters
	// as soon as we see a line that starts with something _other_ than '#', we're done

	idx := 0
	for cfg[0] == '#' {
		idx = bytes.Index(cfg, []byte("\n"))
		if idx == -1 || idx+1 == len(cfg) {
			return nil
		}
		cfg = cfg[idx+1:]
	}

	return cfg
}

func main() {

	configFile := flag.String("c", "", "config file (json)")
	port := flag.Int("p", 0, "port to listen on")
	maxprocs := flag.Int("maxprocs", 0, "GOMAXPROCS")
	flag.IntVar(&Debug, "d", 0, "enable debug logging")

	flag.Parse()

	if *configFile == "" {
		log.Fatal("missing config file")
	}

	cfgjs, err := ioutil.ReadFile(*configFile)
	if err != nil {
		log.Fatal("unable to load config file:", err)
	}

	cfgjs = stripCommentHeader(cfgjs)

	if cfgjs == nil {
		log.Fatal("error removing header comment from ", *configFile)
	}

	err = json.Unmarshal(cfgjs, &Config)
	if err != nil {
		log.Fatal("error parsing config file: ", err)
	}

	if len(Config.Backends) == 0 {
		log.Fatal("no Backends loaded -- exiting")
	}

	// command line overrides config file

	if *port != 0 {
		Config.Port = *port
	}

	if *maxprocs != 0 {
		Config.MaxProcs = *maxprocs
	}

	log.Println("setting GOMAXPROCS=", Config.MaxProcs)
	runtime.GOMAXPROCS(Config.MaxProcs)

	http.HandleFunc("/metrics/find/", findHandler)
	http.HandleFunc("/render/", renderHandler)

	portStr := fmt.Sprintf(":%d", Config.Port)
	log.Println("listening on", portStr)
	log.Fatal(http.ListenAndServe(portStr, nil))
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"os/user"
	"path"
	"strconv"
	"strings"
	"syscall"
)

func checkError(err error) {
	if err != nil {
		fmt.Println(err)
		os.Exit(111)
	}
}

func main() {

	username := os.Args[1]
	program := os.Args[2]

	user, err := user.Lookup(username)
	checkError(err)

	uid, err := strconv.Atoi(user.Uid)
	checkError(err)

	gid, err := strconv.Atoi(user.Gid)
	checkError(err)

	err = syscall.Setuid(uid)
	checkError(err)

	err = syscall.Setgid(gid)
	checkError(err)

	if path.IsAbs(program) {
		err := syscall.Exec(program, os.Args[2:], os.Environ())
		checkError(err)
	}

	for _, p := range strings.Split(os.Getenv("PATH"), ":") {
		absPath := path.Join(p, program)
		err = syscall.Exec(absPath, os.Args[2:], os.Environ())
	}

	checkError(err)
}
<commit_msg>Set gid before uid<commit_after>package main

import (
	"fmt"
	"os"
	"os/user"
	"path"
	"strconv"
	"strings"
	"syscall"
)

func checkError(err error) {
	if err != nil {
		fmt.Println(err)
		os.Exit(111)
	}
}

func main() {

	username := os.Args[1]
	program := os.Args[2]

	user, err := user.Lookup(username)
	checkError(err)

	uid, err := strconv.Atoi(user.Uid)
	checkError(err)

	gid, err := strconv.Atoi(user.Gid)
	checkError(err)

	err = syscall.Setgid(gid)
	checkError(err)
	err = syscall.Setuid(uid)
	checkError(err)

	if path.IsAbs(program) {
		err := syscall.Exec(program, os.Args[2:], os.Environ())
		checkError(err)
	}

	for _, p := range strings.Split(os.Getenv("PATH"), ":") {
		absPath := path.Join(p, program)
		err = syscall.Exec(absPath, os.Args[2:], os.Environ())
	}

	checkError(err)
}
<|endoftext|>


<commit_before>// Package adapt provides an algorithm for adaptive hierarchical interpolation
// with local refinements.
package adapt

import (
	"runtime"
)

const (
	LEVEL_MASK = 0x3F
	LEVEL_SIZE = 6
)

// Basis is a functional basis.
type Basis interface {
	// Compute evaluates the value of a basis function at a point.
	Compute([]uint64, []float64) float64

	// Integrate computes the integral of a basis function over the whole
	// domain.
	Integrate([]uint64) float64
}

// Grid is a sparse grid.
type Grid interface {
	// Compute returns the nodes corresponding to the given indices.
	Compute([]uint64) []float64

	// Refine returns the child indices corresponding to a set of parent
	// indices.
	Refine([]uint64) []uint64

	// Parent transforms an index into its parent index in the ith dimension.
	Parent([]uint64, uint)

	// Sibling transforms an index into its sibling index in the ith dimension.
	Sibling([]uint64, uint)
}

// Interpolator is an instance of the algorithm.
type Interpolator struct {
	config Config
	basis  Basis
	grid   Grid
}

// Progress contains information about the interpolation process.
type Progress struct {
	Level     uint      // Reached level
	Iteration uint      // Iteration number
	Active    uint      // Number of active nodes
	Passive   uint      // Number of passive nodes
	Integral  []float64 // Integral over the whole domain
}

// New creates a new interpolator.
func New(grid Grid, basis Basis, config *Config) *Interpolator {
	interpolator := &Interpolator{
		config: *config,
		basis:  basis,
		grid:   grid,
	}

	config = &interpolator.config
	if config.Workers == 0 {
		config.Workers = uint(runtime.GOMAXPROCS(0))
	}
	if config.Rate == 0.0 {
		config.Rate = 1.0
	}

	return interpolator
}

// Compute constructs an interpolant for a function.
func (self *Interpolator) Compute(target Target) *Surrogate {
	config := &self.config

	ni, no := target.Dimensions()
	nw := config.Workers

	surrogate := newSurrogate(ni, no)
	tracker := newQueue(ni, config)
	history := newHash(ni)

	na, np := uint(1), uint(0)

	iteration := uint(0)
	integral := make([]float64, no)
	indices := make([]uint64, na*ni)
	for na > 0 && np+na <= config.MaxEvaluations && iteration < config.MaxIterations {
		progress := Progress{
			Level:     tracker.lnow,
			Iteration: iteration,
			Active:    na,
			Passive:   np,
			Integral:  integral,
		}

		target.Monitor(&progress)

		nodes := self.grid.Compute(indices)
		surpluses := subtract(
			invoke(target.Compute, nodes, ni, no, nw),
			approximate(self.basis, surrogate.Indices, surrogate.Surpluses, nodes, ni, no, nw),
		)

		scores := assess(self.basis, target, &progress, indices, nodes, surpluses, ni, no)
		tracker.push(indices, scores)
		surrogate.push(indices, surpluses)
		surrogate.step(tracker.lnow, na)

		cumulate(self.basis, indices, surpluses, ni, no, na, integral)

		indices = history.unseen(self.grid.Refine(tracker.pull()))
		if config.Balance {
			indices = append(indices, balance(self.grid, history, indices)...)
		}

		np += na
		na = uint(len(indices)) / ni
		iteration++
	}

	return surrogate
}

// Evaluate computes the values of an interpolant at a set of points.
func (self *Interpolator) Evaluate(surrogate *Surrogate, points []float64) []float64 {
	return approximate(self.basis, surrogate.Indices, surrogate.Surpluses, points,
		surrogate.Inputs, surrogate.Outputs, self.config.Workers)
}

// Integrate computes the integral of an interpolant over the whole domain.
func (self *Interpolator) Integrate(surrogate *Surrogate) []float64 {
	ni, no, nn := surrogate.Inputs, surrogate.Outputs, surrogate.Nodes

	integral := make([]float64, no)
	cumulate(self.basis, surrogate.Indices, surrogate.Surpluses, ni, no, nn, integral)

	return integral
}
<commit_msg>Rely on the Progress struct<commit_after>// Package adapt provides an algorithm for adaptive hierarchical interpolation
// with local refinements.
package adapt

import (
	"runtime"
)

const (
	LEVEL_MASK = 0x3F
	LEVEL_SIZE = 6
)

// Basis is a functional basis.
type Basis interface {
	// Compute evaluates the value of a basis function at a point.
	Compute([]uint64, []float64) float64

	// Integrate computes the integral of a basis function over the whole
	// domain.
	Integrate([]uint64) float64
}

// Grid is a sparse grid.
type Grid interface {
	// Compute returns the nodes corresponding to the given indices.
	Compute([]uint64) []float64

	// Refine returns the child indices corresponding to a set of parent
	// indices.
	Refine([]uint64) []uint64

	// Parent transforms an index into its parent index in the ith dimension.
	Parent([]uint64, uint)

	// Sibling transforms an index into its sibling index in the ith dimension.
	Sibling([]uint64, uint)
}

// Interpolator is an instance of the algorithm.
type Interpolator struct {
	config Config
	basis  Basis
	grid   Grid
}

// Progress contains information about the interpolation process.
type Progress struct {
	Iteration uint      // Iteration number
	Level     uint      // Reached level
	Active    uint      // Number of active nodes
	Passive   uint      // Number of passive nodes
	Integral  []float64 // Integral over the whole domain
}

// New creates a new interpolator.
func New(grid Grid, basis Basis, config *Config) *Interpolator {
	interpolator := &Interpolator{
		config: *config,
		basis:  basis,
		grid:   grid,
	}

	config = &interpolator.config
	if config.Workers == 0 {
		config.Workers = uint(runtime.GOMAXPROCS(0))
	}
	if config.Rate == 0.0 {
		config.Rate = 1.0
	}

	return interpolator
}

// Compute constructs an interpolant for a function.
func (self *Interpolator) Compute(target Target) *Surrogate {
	config := &self.config

	ni, no := target.Dimensions()
	nw := config.Workers

	surrogate := newSurrogate(ni, no)
	tracker := newQueue(ni, config)
	history := newHash(ni)

	indices := make([]uint64, 1*ni)
	progress := Progress{Integral: make([]float64, no)}
	for {
		progress.Level = tracker.lnow
		progress.Passive += progress.Active
		progress.Active = uint(len(indices)) / ni

		if progress.Iteration > config.MaxIterations {
			break
		}
		if progress.Active == 0 || progress.Active+progress.Passive > config.MaxEvaluations {
			break
		}
		target.Monitor(&progress)

		nodes := self.grid.Compute(indices)
		surpluses := subtract(
			invoke(target.Compute, nodes, ni, no, nw),
			approximate(self.basis, surrogate.Indices, surrogate.Surpluses, nodes, ni, no, nw),
		)

		scores := assess(self.basis, target, &progress, indices, nodes, surpluses, ni, no)
		tracker.push(indices, scores)
		surrogate.push(indices, surpluses)
		surrogate.step(tracker.lnow, progress.Active)

		cumulate(self.basis, indices, surpluses, ni, no, progress.Active, progress.Integral)

		indices = history.unseen(self.grid.Refine(tracker.pull()))
		if config.Balance {
			indices = append(indices, balance(self.grid, history, indices)...)
		}

		progress.Iteration++
	}

	return surrogate
}

// Evaluate computes the values of an interpolant at a set of points.
func (self *Interpolator) Evaluate(surrogate *Surrogate, points []float64) []float64 {
	return approximate(self.basis, surrogate.Indices, surrogate.Surpluses, points,
		surrogate.Inputs, surrogate.Outputs, self.config.Workers)
}

// Integrate computes the integral of an interpolant over the whole domain.
func (self *Interpolator) Integrate(surrogate *Surrogate) []float64 {
	ni, no, nn := surrogate.Inputs, surrogate.Outputs, surrogate.Nodes

	integral := make([]float64, no)
	cumulate(self.basis, surrogate.Indices, surrogate.Surpluses, ni, no, nn, integral)

	return integral
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"os/user"
	"path"
	"path/filepath"
	"sort"
	"strings"
	"syscall"

	"github.com/codegangsta/cli"
	"github.com/hpcloud/tail"
	"github.com/yext/errgo"
)

type EdwardConfiguration struct {
	Dir       string
	LogDir    string
	PidDir    string
	ScriptDir string
}

var EdwardConfig EdwardConfiguration = EdwardConfiguration{}

func createDirIfNeeded(path string) {
	if _, err := os.Stat(path); os.IsNotExist(err) {
		os.MkdirAll(path, 0777)
	}
}

func (e *EdwardConfiguration) initialize() error {
	user, err := user.Current()
	if err != nil {
		return err
	}
	e.Dir = path.Join(user.HomeDir, ".edward")
	e.LogDir = path.Join(e.Dir, "logs")
	e.PidDir = path.Join(e.Dir, "pidFiles")
	e.ScriptDir = path.Join(e.Dir, "scriptFiles")
	createDirIfNeeded(e.Dir)
	createDirIfNeeded(e.LogDir)
	createDirIfNeeded(e.PidDir)
	createDirIfNeeded(e.ScriptDir)
	return nil
}

var groups map[string]*ServiceGroupConfig
var services map[string]*ServiceConfig

func thirdPartyService(name string, startCommand string, stopCommand string, started string) *ServiceConfig {
	pathStr := "$ALPHA"
	return &ServiceConfig{
		Name: name,
		Path: &pathStr,
		Env:  []string{"YEXT_RABBITMQ=localhost"},
		Commands: ServiceConfigCommands{
			Launch: startCommand,
			Stop:   stopCommand,
		},
		Properties: ServiceConfigProperties{
			Started: started,
		},
	}
}

func getAlpha() string {
	for _, env := range os.Environ() {
		pair := strings.Split(env, "=")
		if pair[0] == "ALPHA" {
			return pair[1]
		}
	}
	return ""
}

func addFoundServices() {
	foundServices, _, err := generateServices(getAlpha())
	if err != nil {
		log.Fatal(err)
	}

	for _, s := range foundServices {
		if _, found := services[s.Name]; !found {
			services[s.Name] = s
		}
	}
}

// getConfigPath identifies the location of edward.json, if any exists
func getConfigPath() string {

	if flags.config != "" {
		return flags.config
	}

	var pathOptions []string

	// Config file in current working directory
	wd, err := os.Getwd()
	if err == nil {
		pathOptions = append(pathOptions, filepath.Join(wd, "edward.json"))
	}

	// Config file at root of working dir's git repo, if any
	gitRoot, err := gitRoot()
	if err == nil {
		pathOptions = append(pathOptions, filepath.Join(gitRoot, "edward.json"))

	}

	// Config file in Edward Config dir
	pathOptions = append(pathOptions, filepath.Join(EdwardConfig.Dir, "edward.json"))

	for _, path := range pathOptions {
		if _, err := os.Stat(path); err == nil {
			return path
		}
	}

	return ""
}

func gitRoot() (string, error) {
	output, err := exec.Command("git", "rev-parse", "--show-toplevel").CombinedOutput()
	if err != nil {
		return "", fmt.Errorf("%v\n%v", string(output), err)
	}
	return strings.TrimSpace(string(output)), nil
}

func initEmptyConfig() {
	groups = make(map[string]*ServiceGroupConfig)
	services = make(map[string]*ServiceConfig)
}

func loadConfig() error {
	initEmptyConfig()

	configPath := getConfigPath()
	if configPath != "" {
		println("Loading configuration from", configPath)
		r, err := os.Open(configPath)
		if err != nil {
			return errgo.Mask(err)
		}
		config, err := LoadConfig(r)
		if err != nil {
			return errgo.Mask(err)
		}

		services = config.ServiceMap
		groups = config.GroupMap
		return nil
	} else {
		addFoundServices()
		applyHardCodedServicesAndGroups()
	}

	return nil
}

func getServicesOrGroups(names []string) ([]ServiceOrGroup, error) {
	var outSG []ServiceOrGroup
	for _, name := range names {
		sg, err := getServiceOrGroup(name)
		if err != nil {
			return nil, err
		}
		outSG = append(outSG, sg)
	}
	return outSG, nil
}

func getServiceOrGroup(name string) (ServiceOrGroup, error) {
	if group, ok := groups[name]; ok {
		return group, nil
	}
	if service, ok := services[name]; ok {
		return service, nil
	}
	return nil, errors.New("Service or group not found")
}

func list(c *cli.Context) error {

	var groupNames []string
	var serviceNames []string
	for name, _ := range groups {
		groupNames = append(groupNames, name)
	}
	for name, _ := range services {
		serviceNames = append(serviceNames, name)
	}

	sort.Strings(groupNames)
	sort.Strings(serviceNames)

	println("Services and groups")
	println("Groups:")
	for _, name := range groupNames {
		println("\t", name)
	}
	println("Services:")
	for _, name := range serviceNames {
		println("\t", name)
	}

	return nil
}

func generate(c *cli.Context) error {

	// Add any new services to the config as appropriate
	addFoundServices()

	configPath := getConfigPath()
	if configPath == "" {
		wd, err := os.Getwd()
		if err == nil {
			configPath = filepath.Join(wd, "edward.json")
		}
	}

	if err := generateConfigFile(configPath); err != nil {
		return err
	}
	println("Wrote to", configPath)

	return nil
}

func allStatus() {
	var statuses []ServiceStatus
	for _, service := range services {
		statuses = append(statuses, service.GetStatus()...)
	}
	for _, status := range statuses {
		if status.Status != "STOPPED" {
			println(status.Service.Name, ":", status.Status)
		}
	}
}

func status(c *cli.Context) error {

	if len(c.Args()) == 0 {
		allStatus()
		return nil
	}

	sgs, err := getServicesOrGroups(c.Args())
	if err != nil {
		return err
	}
	for _, s := range sgs {
		statuses := s.GetStatus()
		for _, status := range statuses {
			println(status.Service.Name, ":", status.Status)
		}
	}
	return nil
}

func messages(c *cli.Context) error {
	return errors.New("Unimplemented")
}

func start(c *cli.Context) error {
	sgs, err := getServicesOrGroups(c.Args())
	if err != nil {
		return err
	}
	for _, s := range sgs {
		println("==== Build Phase ====")
		err = s.Build()
		if err != nil {
			return errors.New("Error building " + s.GetName() + ": " + err.Error())
		}
		println("==== Launch Phase ====")
		err = s.Start()
		if err != nil {
			return errors.New("Error launching " + s.GetName() + ": " + err.Error())
		}
	}
	return nil
}

func allServices() []ServiceOrGroup {
	var as []ServiceOrGroup
	for _, service := range services {
		as = append(as, service)
	}
	return as
}

func stop(c *cli.Context) error {
	var sgs []ServiceOrGroup
	var err error
	if len(c.Args()) == 0 {
		sgs = allServices()
	} else {
		sgs, err = getServicesOrGroups(c.Args())
		if err != nil {
			return err
		}
	}
	for _, s := range sgs {
		_ = s.Stop()
	}
	return nil
}

func restart(c *cli.Context) error {
	sgs, err := getServicesOrGroups(c.Args())
	if err != nil {
		return err
	}
	for _, s := range sgs {
		_ = s.Stop()
		err = s.Build()
		if err != nil {
			return err
		}
		err = s.Start()
		if err != nil {
			return err
		}
	}
	return nil
}

func doLog(c *cli.Context) error {
	if len(c.Args()) > 1 {
		return errors.New("Cannot output multiple service logs")
	}
	name := c.Args()[0]
	if _, ok := groups[name]; ok {
		return errors.New("Cannot output group logs")
	}
	if service, ok := services[name]; ok {
		command := service.GetCommand()
		runLog := command.Logs.Run
		t, err := tail.TailFile(runLog, tail.Config{Follow: true})
		if err != nil {
			return nil
		}
		for line := range t.Lines {
			println(line.Text)
		}
		return nil
	}
	return errors.New("Service not found: " + name)
}

func checkNotSudo() {
	user, err := user.Current()
	if err != nil {
		log.Fatal(err)
	}
	if user.Uid == "0" {
		log.Fatal("edward should not be run with sudo")
	}
}

func createScriptFile(suffix string, content string) (*os.File, error) {
	file, err := ioutil.TempFile(os.TempDir(), suffix)
	if err != nil {
		return nil, err
	}
	file.WriteString(content)
	file.Close()

	err = os.Chmod(file.Name(), 0777)
	if err != nil {
		return nil, err
	}

	return file, nil
}

func ensureSudoAble() {
	var buffer bytes.Buffer

	buffer.WriteString("#!/bin/bash\n")
	buffer.WriteString("sudo echo Test > /dev/null\n")
	buffer.WriteString("ISCHILD=YES ")
	buffer.WriteString(strings.Join(os.Args, " "))
	buffer.WriteString("\n")

	file, err := createScriptFile("sudoAbility", buffer.String())
	if err != nil {
		log.Fatal(err)
	}

	err = syscall.Exec(file.Name(), []string{file.Name()}, os.Environ())
	if err != nil {
		log.Fatal(err)
	}
}

func prepareForSudo() {
	checkNotSudo()

	isChild := os.Getenv("ISCHILD")
	if isChild == "" {
		ensureSudoAble()
		return
	}
}

func RemoveContents(dir string) error {
	d, err := os.Open(dir)
	if err != nil {
		return err
	}
	defer d.Close()
	names, err := d.Readdirnames(-1)
	if err != nil {
		return err
	}
	for _, name := range names {
		err = os.RemoveAll(filepath.Join(dir, name))
		if err != nil {
			return err
		}
	}
	return nil
}

func refreshForReboot() error {
	rebooted, err := hasRebooted(EdwardConfig.Dir)
	if err != nil {
		return errgo.Mask(err)
	}

	if rebooted {
		err = RemoveContents(EdwardConfig.PidDir)
		if err != nil {
			return errgo.Mask(err)
		}
		err = setRebootMarker(EdwardConfig.Dir)
		if err != nil {
			return errgo.Mask(err)
		}
	}

	return nil
}

var flags = struct {
	config string
}{}

func main() {

	app := cli.NewApp()
	app.Name = "Edward"
	app.Usage = "Manage local microservices"
	app.Before = func(c *cli.Context) error {
		command := c.Args().First()
		if command == "start" || command == "stop" || command == "restart" {
			prepareForSudo()
		}

		err := EdwardConfig.initialize()
		if err != nil {
			return errgo.Mask(err)
		}
		err = refreshForReboot()
		if err != nil {
			return errgo.Mask(err)
		}

		if command != "generate" {
			err = loadConfig()
			if err != nil {
				return errgo.Mask(err)
			}
		} else {
			initEmptyConfig()
		}

		return nil
	}
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:        "config, c",
			Usage:       "Use service configuration file at `PATH`",
			Destination: &(flags.config),
		},
	}
	app.Commands = []cli.Command{
		{
			Name:   "list",
			Usage:  "List available services",
			Action: list,
		},
		{
			Name:   "generate",
			Usage:  "Generate Edward config for a source tree",
			Action: generate,
		},
		{
			Name:   "status",
			Usage:  "Display service status",
			Action: status,
		},
		{
			Name:   "messages",
			Usage:  "Show messages from services",
			Action: messages,
		},
		{
			Name:   "start",
			Usage:  "Build and launch a service",
			Action: start,
		},
		{
			Name:   "stop",
			Usage:  "Stop a service",
			Action: stop,
		},
		{
			Name:   "restart",
			Usage:  "Rebuild and relaunch a service",
			Action: restart,
		},
		{
			Name:    "log",
			Aliases: []string{"tail"},
			Usage:   "Tail the log for a service",
			Action:  doLog,
		},
	}

	err := app.Run(os.Args)
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Allow the .edward folder's config to override any local ones for now<commit_after>package main

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"os/user"
	"path"
	"path/filepath"
	"sort"
	"strings"
	"syscall"

	"github.com/codegangsta/cli"
	"github.com/hpcloud/tail"
	"github.com/yext/errgo"
)

type EdwardConfiguration struct {
	Dir       string
	LogDir    string
	PidDir    string
	ScriptDir string
}

var EdwardConfig EdwardConfiguration = EdwardConfiguration{}

func createDirIfNeeded(path string) {
	if _, err := os.Stat(path); os.IsNotExist(err) {
		os.MkdirAll(path, 0777)
	}
}

func (e *EdwardConfiguration) initialize() error {
	user, err := user.Current()
	if err != nil {
		return err
	}
	e.Dir = path.Join(user.HomeDir, ".edward")
	e.LogDir = path.Join(e.Dir, "logs")
	e.PidDir = path.Join(e.Dir, "pidFiles")
	e.ScriptDir = path.Join(e.Dir, "scriptFiles")
	createDirIfNeeded(e.Dir)
	createDirIfNeeded(e.LogDir)
	createDirIfNeeded(e.PidDir)
	createDirIfNeeded(e.ScriptDir)
	return nil
}

var groups map[string]*ServiceGroupConfig
var services map[string]*ServiceConfig

func thirdPartyService(name string, startCommand string, stopCommand string, started string) *ServiceConfig {
	pathStr := "$ALPHA"
	return &ServiceConfig{
		Name: name,
		Path: &pathStr,
		Env:  []string{"YEXT_RABBITMQ=localhost"},
		Commands: ServiceConfigCommands{
			Launch: startCommand,
			Stop:   stopCommand,
		},
		Properties: ServiceConfigProperties{
			Started: started,
		},
	}
}

func getAlpha() string {
	for _, env := range os.Environ() {
		pair := strings.Split(env, "=")
		if pair[0] == "ALPHA" {
			return pair[1]
		}
	}
	return ""
}

func addFoundServices() {
	foundServices, _, err := generateServices(getAlpha())
	if err != nil {
		log.Fatal(err)
	}

	for _, s := range foundServices {
		if _, found := services[s.Name]; !found {
			services[s.Name] = s
		}
	}
}

// getConfigPath identifies the location of edward.json, if any exists
func getConfigPath() string {

	if flags.config != "" {
		return flags.config
	}

	var pathOptions []string

	// Config file in Edward Config dir
	pathOptions = append(pathOptions, filepath.Join(EdwardConfig.Dir, "edward.json"))

	// Config file in current working directory
	wd, err := os.Getwd()
	if err == nil {
		pathOptions = append(pathOptions, filepath.Join(wd, "edward.json"))
	}

	// Config file at root of working dir's git repo, if any
	gitRoot, err := gitRoot()
	if err == nil {
		pathOptions = append(pathOptions, filepath.Join(gitRoot, "edward.json"))

	}

	for _, path := range pathOptions {
		if _, err := os.Stat(path); err == nil {
			return path
		}
	}

	return ""
}

func gitRoot() (string, error) {
	output, err := exec.Command("git", "rev-parse", "--show-toplevel").CombinedOutput()
	if err != nil {
		return "", fmt.Errorf("%v\n%v", string(output), err)
	}
	return strings.TrimSpace(string(output)), nil
}

func initEmptyConfig() {
	groups = make(map[string]*ServiceGroupConfig)
	services = make(map[string]*ServiceConfig)
}

func loadConfig() error {
	initEmptyConfig()

	configPath := getConfigPath()
	if configPath != "" {
		println("Loading configuration from", configPath)
		r, err := os.Open(configPath)
		if err != nil {
			return errgo.Mask(err)
		}
		config, err := LoadConfig(r)
		if err != nil {
			return errgo.Mask(err)
		}

		services = config.ServiceMap
		groups = config.GroupMap
		return nil
	} else {
		addFoundServices()
		applyHardCodedServicesAndGroups()
	}

	return nil
}

func getServicesOrGroups(names []string) ([]ServiceOrGroup, error) {
	var outSG []ServiceOrGroup
	for _, name := range names {
		sg, err := getServiceOrGroup(name)
		if err != nil {
			return nil, err
		}
		outSG = append(outSG, sg)
	}
	return outSG, nil
}

func getServiceOrGroup(name string) (ServiceOrGroup, error) {
	if group, ok := groups[name]; ok {
		return group, nil
	}
	if service, ok := services[name]; ok {
		return service, nil
	}
	return nil, errors.New("Service or group not found")
}

func list(c *cli.Context) error {

	var groupNames []string
	var serviceNames []string
	for name, _ := range groups {
		groupNames = append(groupNames, name)
	}
	for name, _ := range services {
		serviceNames = append(serviceNames, name)
	}

	sort.Strings(groupNames)
	sort.Strings(serviceNames)

	println("Services and groups")
	println("Groups:")
	for _, name := range groupNames {
		println("\t", name)
	}
	println("Services:")
	for _, name := range serviceNames {
		println("\t", name)
	}

	return nil
}

func generate(c *cli.Context) error {

	// Add any new services to the config as appropriate
	addFoundServices()

	configPath := getConfigPath()
	if configPath == "" {
		wd, err := os.Getwd()
		if err == nil {
			configPath = filepath.Join(wd, "edward.json")
		}
	}

	if err := generateConfigFile(configPath); err != nil {
		return err
	}
	println("Wrote to", configPath)

	return nil
}

func allStatus() {
	var statuses []ServiceStatus
	for _, service := range services {
		statuses = append(statuses, service.GetStatus()...)
	}
	for _, status := range statuses {
		if status.Status != "STOPPED" {
			println(status.Service.Name, ":", status.Status)
		}
	}
}

func status(c *cli.Context) error {

	if len(c.Args()) == 0 {
		allStatus()
		return nil
	}

	sgs, err := getServicesOrGroups(c.Args())
	if err != nil {
		return err
	}
	for _, s := range sgs {
		statuses := s.GetStatus()
		for _, status := range statuses {
			println(status.Service.Name, ":", status.Status)
		}
	}
	return nil
}

func messages(c *cli.Context) error {
	return errors.New("Unimplemented")
}

func start(c *cli.Context) error {
	sgs, err := getServicesOrGroups(c.Args())
	if err != nil {
		return err
	}
	for _, s := range sgs {
		println("==== Build Phase ====")
		err = s.Build()
		if err != nil {
			return errors.New("Error building " + s.GetName() + ": " + err.Error())
		}
		println("==== Launch Phase ====")
		err = s.Start()
		if err != nil {
			return errors.New("Error launching " + s.GetName() + ": " + err.Error())
		}
	}
	return nil
}

func allServices() []ServiceOrGroup {
	var as []ServiceOrGroup
	for _, service := range services {
		as = append(as, service)
	}
	return as
}

func stop(c *cli.Context) error {
	var sgs []ServiceOrGroup
	var err error
	if len(c.Args()) == 0 {
		sgs = allServices()
	} else {
		sgs, err = getServicesOrGroups(c.Args())
		if err != nil {
			return err
		}
	}
	for _, s := range sgs {
		_ = s.Stop()
	}
	return nil
}

func restart(c *cli.Context) error {
	sgs, err := getServicesOrGroups(c.Args())
	if err != nil {
		return err
	}
	for _, s := range sgs {
		_ = s.Stop()
		err = s.Build()
		if err != nil {
			return err
		}
		err = s.Start()
		if err != nil {
			return err
		}
	}
	return nil
}

func doLog(c *cli.Context) error {
	if len(c.Args()) > 1 {
		return errors.New("Cannot output multiple service logs")
	}
	name := c.Args()[0]
	if _, ok := groups[name]; ok {
		return errors.New("Cannot output group logs")
	}
	if service, ok := services[name]; ok {
		command := service.GetCommand()
		runLog := command.Logs.Run
		t, err := tail.TailFile(runLog, tail.Config{Follow: true})
		if err != nil {
			return nil
		}
		for line := range t.Lines {
			println(line.Text)
		}
		return nil
	}
	return errors.New("Service not found: " + name)
}

func checkNotSudo() {
	user, err := user.Current()
	if err != nil {
		log.Fatal(err)
	}
	if user.Uid == "0" {
		log.Fatal("edward should not be run with sudo")
	}
}

func createScriptFile(suffix string, content string) (*os.File, error) {
	file, err := ioutil.TempFile(os.TempDir(), suffix)
	if err != nil {
		return nil, err
	}
	file.WriteString(content)
	file.Close()

	err = os.Chmod(file.Name(), 0777)
	if err != nil {
		return nil, err
	}

	return file, nil
}

func ensureSudoAble() {
	var buffer bytes.Buffer

	buffer.WriteString("#!/bin/bash\n")
	buffer.WriteString("sudo echo Test > /dev/null\n")
	buffer.WriteString("ISCHILD=YES ")
	buffer.WriteString(strings.Join(os.Args, " "))
	buffer.WriteString("\n")

	file, err := createScriptFile("sudoAbility", buffer.String())
	if err != nil {
		log.Fatal(err)
	}

	err = syscall.Exec(file.Name(), []string{file.Name()}, os.Environ())
	if err != nil {
		log.Fatal(err)
	}
}

func prepareForSudo() {
	checkNotSudo()

	isChild := os.Getenv("ISCHILD")
	if isChild == "" {
		ensureSudoAble()
		return
	}
}

func RemoveContents(dir string) error {
	d, err := os.Open(dir)
	if err != nil {
		return err
	}
	defer d.Close()
	names, err := d.Readdirnames(-1)
	if err != nil {
		return err
	}
	for _, name := range names {
		err = os.RemoveAll(filepath.Join(dir, name))
		if err != nil {
			return err
		}
	}
	return nil
}

func refreshForReboot() error {
	rebooted, err := hasRebooted(EdwardConfig.Dir)
	if err != nil {
		return errgo.Mask(err)
	}

	if rebooted {
		err = RemoveContents(EdwardConfig.PidDir)
		if err != nil {
			return errgo.Mask(err)
		}
		err = setRebootMarker(EdwardConfig.Dir)
		if err != nil {
			return errgo.Mask(err)
		}
	}

	return nil
}

var flags = struct {
	config string
}{}

func main() {

	app := cli.NewApp()
	app.Name = "Edward"
	app.Usage = "Manage local microservices"
	app.Before = func(c *cli.Context) error {
		command := c.Args().First()
		if command == "start" || command == "stop" || command == "restart" {
			prepareForSudo()
		}

		err := EdwardConfig.initialize()
		if err != nil {
			return errgo.Mask(err)
		}
		err = refreshForReboot()
		if err != nil {
			return errgo.Mask(err)
		}

		if command != "generate" {
			err = loadConfig()
			if err != nil {
				return errgo.Mask(err)
			}
		} else {
			initEmptyConfig()
		}

		return nil
	}
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:        "config, c",
			Usage:       "Use service configuration file at `PATH`",
			Destination: &(flags.config),
		},
	}
	app.Commands = []cli.Command{
		{
			Name:   "list",
			Usage:  "List available services",
			Action: list,
		},
		{
			Name:   "generate",
			Usage:  "Generate Edward config for a source tree",
			Action: generate,
		},
		{
			Name:   "status",
			Usage:  "Display service status",
			Action: status,
		},
		{
			Name:   "messages",
			Usage:  "Show messages from services",
			Action: messages,
		},
		{
			Name:   "start",
			Usage:  "Build and launch a service",
			Action: start,
		},
		{
			Name:   "stop",
			Usage:  "Stop a service",
			Action: stop,
		},
		{
			Name:   "restart",
			Usage:  "Rebuild and relaunch a service",
			Action: restart,
		},
		{
			Name:    "log",
			Aliases: []string{"tail"},
			Usage:   "Tail the log for a service",
			Action:  doLog,
		},
	}

	err := app.Run(os.Args)
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"strings"
	"sync"
)

const version = "v0.2.1"

var (
	f      = flag.String("f", ".*", "Filter directories by RE2 regexp")
	dir    = flag.String("d", ".", "Working directory - search from here")
	dryRun = flag.Bool("dry-run", false, "Print what would be run where, without actually doing it")
	ver    = flag.Bool("version", false, "Print version and exit")
)

func main() {
	flag.Parse()

	if *ver {
		fmt.Printf("shotgun %s\n", version)
		os.Exit(0)
	}

	// no arguments passed or invalid directory filter
	fRegexp, err := regexp.Compile(*f)
	if len(flag.Args()) < 1 || err != nil {
		usage()
		os.Exit(1)
	}

	// invalid working directory
	f, err := ioutil.ReadDir(*dir)
	if err != nil {
		panic(err)
	}

	var (
		c  = strings.Join(flag.Args(), " ")
		wg = sync.WaitGroup{}
	)

	for _, d := range f {
		wg.Add(1)
		go func(d os.FileInfo) {
			defer wg.Done()
			if d.IsDir() && fRegexp.MatchString(d.Name()) {
				if *dryRun {
					fmt.Printf("Would run '%s' in '%s'\n", c, d.Name())
					return
				}
				cmd(d.Name(), c).Run()
				fmt.Printf("Running '%s' in '%s'\n", c, d.Name())
			}
		}(d)
	}
	wg.Wait()
}

var usage = func() {
	fmt.Fprintf(os.Stderr, "Shotgun is a tool for running commands in parallel on a set of directories.\n")
	fmt.Fprintf(os.Stderr, "\nUsage:\n")
	fmt.Fprintf(os.Stderr, "  shotgun [options] command\n")
	fmt.Fprintf(os.Stderr, "\nOptions:\n")
	flag.PrintDefaults()
	fmt.Fprintf(os.Stderr, "\nExamples:\n")
	fmt.Fprintf(os.Stderr, "  %s\n    \t%s\n",
		"shotgun git pull",
		"Run a command in each child directory of the current")
	fmt.Fprintf(os.Stderr, "  %s\n    \t%s\n",
		"shotgun -f '^a' git pull",
		"Run a command in each directory beginning with the letter 'a'")
	fmt.Fprintf(os.Stderr, "  %s\n    \t%s\n",
		"shotgun -dir $GOPATH/src/github.com/bbrks git status --short",
		"Run a command for each directory in '$GOPATH/src/github.com/bbrks'")
	fmt.Fprintf(os.Stderr, "  %s\n    \t%s\n",
		"shotgun 'git checkout -- .; git checkout develop; git fetch; git pull'",
		"Wrap commands in quotes and separate by semicolons to chain sequentially")
	fmt.Fprintf(os.Stderr, "  %s\n    \t%s\n",
		"shotgun -dry-run 'rm .travis.yml'",
		"Print what would be run where")

}
<commit_msg>Add throttle and default to GOMAXPROCS*64<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"runtime"
	"strings"
	"sync"
)

const version = "v0.2.2"

var (
	f        = flag.String("f", ".*", "Filter directories by RE2 regexp")
	dir      = flag.String("d", ".", "Working directory - search from here")
	throttle = flag.Int("c", runtime.GOMAXPROCS(0)*64, "Maximum number of concurrent commands")
	dryRun   = flag.Bool("dry-run", false, "Print what would be run where, without actually doing it")
	ver      = flag.Bool("version", false, "Print version and exit")
)

func main() {
	flag.Parse()

	if *ver {
		fmt.Printf("shotgun %s\n", version)
		os.Exit(0)
	}

	// no arguments passed or invalid directory filter
	fRegexp, err := regexp.Compile(*f)
	if len(flag.Args()) < 1 || err != nil {
		usage()
		os.Exit(1)
	}

	// invalid working directory
	f, err := ioutil.ReadDir(*dir)
	if err != nil {
		panic(err)
	}

	var (
		c  = strings.Join(flag.Args(), " ")
		wg = sync.WaitGroup{}
		t  = make(chan struct{}, *throttle)
	)

	for _, d := range f {
		wg.Add(1)
		t <- struct{}{}
		go func(d os.FileInfo) {
			defer wg.Done()
			defer func() { <-t }()
			if d.IsDir() && fRegexp.MatchString(d.Name()) {
				if *dryRun {
					fmt.Printf("Would run '%s' in '%s'\n", c, d.Name())
					return
				}
				cmd(d.Name(), c).Run()
				fmt.Printf("Running '%s' in '%s'\n", c, d.Name())
			}
		}(d)
	}
	wg.Wait()
}

var usage = func() {
	fmt.Fprintf(os.Stderr, "Shotgun is a tool for running commands in parallel on a set of directories.\n")
	fmt.Fprintf(os.Stderr, "\nUsage:\n")
	fmt.Fprintf(os.Stderr, "  shotgun [options] command\n")
	fmt.Fprintf(os.Stderr, "\nOptions:\n")
	flag.PrintDefaults()
	fmt.Fprintf(os.Stderr, "\nExamples:\n")
	fmt.Fprintf(os.Stderr, "  %s\n    \t%s\n",
		"shotgun git pull",
		"Run a command in each child directory of the current")
	fmt.Fprintf(os.Stderr, "  %s\n    \t%s\n",
		"shotgun -f '^a' git pull",
		"Run a command in each directory beginning with the letter 'a'")
	fmt.Fprintf(os.Stderr, "  %s\n    \t%s\n",
		"shotgun -dir $GOPATH/src/github.com/bbrks git status --short",
		"Run a command for each directory in '$GOPATH/src/github.com/bbrks'")
	fmt.Fprintf(os.Stderr, "  %s\n    \t%s\n",
		"shotgun 'git checkout -- .; git checkout develop; git fetch; git pull'",
		"Wrap commands in quotes and separate by semicolons to chain sequentially")
	fmt.Fprintf(os.Stderr, "  %s\n    \t%s\n",
		"shotgun -dry-run 'rm .travis.yml'",
		"Print what would be run where")

}
<|endoftext|>


<commit_before>package main

import (
	"runtime"

	"github.com/Ariemeth/quantum-pulse/engine"
)

func init() {
	// GLFW event handling must run on the main OS thread
	runtime.LockOSThread()
}

func main() {

	engine := new(engine.Engine)

	engine.Init()

	engine.LoadScene(engine.LoadSceneFile("scene1.json"))

	engine.Run()
}
<commit_msg>updated to use restructured quantum pulse engine<commit_after>package main

import (
	"runtime"

	qp "github.com/Ariemeth/quantum-pulse"
)

func init() {
	// GLFW event handling must run on the main OS thread
	runtime.LockOSThread()
}

func main() {

	engine := new(qp.Engine)

	engine.Init()

	engine.LoadScene(engine.LoadSceneFile("scene1.json"))

	engine.Run()
}
<|endoftext|>