<commit_before>package photomgr

import (
	"fmt"
	"log"
	"os"
	"strconv"
	"strings"
	"sync"

	"github.com/PuerkitoBio/goquery"
)

type CK101 struct {
	//Inherit
	baseCrawler

	//To store current CK101 post result
	BaseDir string
}

func NewCK101() *CK101 {
	c := new(CK101)
	c.baseAddress = "https://www.CK101.cc"
	c.entryAddress = "http://ck101.com/forum-3465-1.html"
	return c
}
func (b *CK101) HasValidURL(url string) bool {
	log.Println("url=", url)
	return true
}

func (p *CK101) GetUrlPhotos(target string) []string {
	var resultSlice []string

	doc, err := goquery.NewDocument(target)
	if err != nil {
		panic(err)
	}

	doc.Find("div[itemprop=articleBody] img").Each(func(i int, img *goquery.Selection) {
		imgUrl, _ := img.Attr("file")
		resultSlice = append(resultSlice, imgUrl)
	})
	return resultSlice
}

func (p *CK101) Crawler(target string, workerNum int) {

	doc, err := goquery.NewDocument(target)
	if err != nil {
		panic(err)
	}

	title := doc.Find("h1#thread_subject").Text()

	log.Println("[CK101]:", title, " starting downloading...")
	dir := fmt.Sprintf("%v/%v - %v", p.BaseDir, "CK101", title)
	if exist, _ := exists(dir); exist {
		//fmt.Println("Already download")
		return
	}
	os.MkdirAll(dir, 0755)

	linkChan := make(chan string)
	wg := new(sync.WaitGroup)
	for i := 0; i < workerNum; i++ {
		wg.Add(1)
		go p.worker(dir, linkChan, wg)
	}

	doc.Find("div[itemprop=articleBody] img").Each(func(i int, img *goquery.Selection) {
		imgUrl, _ := img.Attr("file")
		linkChan <- imgUrl
	})

	close(linkChan)
	wg.Wait()
}

//Set CK101 board page index, fetch all post and return article count back
func (p *CK101) ParseCK101PageByIndex(page int) int {
	doc, err := goquery.NewDocument(p.entryAddress)
	if err != nil {
		log.Fatal(err)
	}

	urlList := make([]string, 0)
	postList := make([]string, 0)
	starList := make([]int, 0)

	var PageWebSide string
	page = page + 1 //one base
	if page > 1 {
		// Find page result
		PageWebSide = fmt.Sprintf("http://ck101.com/forum-3465-%d.html", page)
	} else {
		PageWebSide = p.entryAddress
	}
	//fmt.Println("Page", PageWebSide)

	doc, err = goquery.NewDocument(PageWebSide)
	if err != nil {
		log.Fatal(err)
	}
	doc.Find(".threadrow").Each(func(i int, s *goquery.Selection) {

		star := ""
		title := ""
		url := ""
		starInt := 0
		s.Find(".blockTitle a").Each(func(i int, tQ *goquery.Selection) {
			title, _ = tQ.Attr("title")
			url, _ = tQ.Attr("href")
		})
		s.Find(".icoPage img").Each(func(i int, starC *goquery.Selection) {
			star_c, _ := starC.Attr("title")
			if strings.Contains(star_c, "熱度") {
				star = strings.TrimPrefix(star_c, "熱度:")
				star = strings.TrimSpace(star)
				starInt, _ = strconv.Atoi(star)
			}
			//}
		})
		urlList = append(urlList, url)
		starList = append(starList, starInt)
		postList = append(postList, title)
	})

	p.storedPostURLList = urlList
	p.storedStarList = starList
	p.storedPostTitleList = postList

	return len(p.storedPostTitleList)
}
<commit_msg>Fixed ck101 format change<commit_after>package photomgr

import (
	"fmt"
	"log"
	"os"
	"strconv"
	"strings"
	"sync"

	"github.com/PuerkitoBio/goquery"
)

type CK101 struct {
	//Inherit
	baseCrawler

	//To store current CK101 post result
	BaseDir string
}

func NewCK101() *CK101 {
	c := new(CK101)
	c.baseAddress = "https://www.CK101.cc"
	c.entryAddress = "http://ck101.com/forum-1345-1.html"
	return c
}
func (b *CK101) HasValidURL(url string) bool {
	log.Println("url=", url)
	return true
}

func (p *CK101) GetUrlPhotos(target string) []string {
	var resultSlice []string

	doc, err := goquery.NewDocument(target)
	if err != nil {
		panic(err)
	}

	doc.Find("div[itemprop=articleBody] img").Each(func(i int, img *goquery.Selection) {
		imgUrl, _ := img.Attr("file")
		resultSlice = append(resultSlice, imgUrl)
	})
	return resultSlice
}

func (p *CK101) Crawler(target string, workerNum int) {

	doc, err := goquery.NewDocument(target)
	if err != nil {
		panic(err)
	}

	title := doc.Find("h1").Text()
	log.Println("[CK101]:", title, " starting downloading...")
	dir := fmt.Sprintf("%v/%v - %v", p.BaseDir, "CK101", title)
	if exist, _ := exists(dir); exist {
		//fmt.Println("Already download")
		return
	}
	os.MkdirAll(dir, 0755)

	linkChan := make(chan string)
	wg := new(sync.WaitGroup)
	for i := 0; i < workerNum; i++ {
		wg.Add(1)
		go p.worker(dir, linkChan, wg)
	}

	doc.Find("div[itemprop=articleBody] img").Each(func(i int, img *goquery.Selection) {
		imgUrl, _ := img.Attr("file")
		linkChan <- imgUrl
	})

	close(linkChan)
	wg.Wait()
}

//Set CK101 board page index, fetch all post and return article count back
func (p *CK101) ParseCK101PageByIndex(page int) int {
	doc, err := goquery.NewDocument(p.entryAddress)
	if err != nil {
		log.Fatal(err)
	}

	urlList := make([]string, 0)
	postList := make([]string, 0)
	starList := make([]int, 0)

	var PageWebSide string
	page = page + 1 //one base
	if page > 1 {
		// Find page result
		PageWebSide = fmt.Sprintf("http://ck101.com/forum-1345-%d.html", page)
	} else {
		PageWebSide = p.entryAddress
	}
	//fmt.Println("Page", PageWebSide)

	doc, err = goquery.NewDocument(PageWebSide)
	if err != nil {
		log.Fatal(err)
	}
	doc.Find(".cl_box").Each(func(i int, s *goquery.Selection) {
		star := ""
		title := ""
		url := ""
		starInt := 0
		s.Find("a").Each(func(i int, tQ *goquery.Selection) {
			title, _ = tQ.Attr("title")
			url, _ = tQ.Attr("href")
		})
		s.Find("em").Each(func(i int, starC *goquery.Selection) {
			star_c, _ := starC.Attr("title")
			fmt.Println("star_c:", star_c)
			if strings.Contains(star_c, "查看") {
				star = strings.Replace(star_c, "查看", "", -1)
				fmt.Println("star:", star)
				star = strings.TrimSpace(star)
				starInt, _ = strconv.Atoi(star)
			}
			//}
		})
		urlList = append(urlList, url)
		starList = append(starList, starInt)
		postList = append(postList, title)
	})

	p.storedPostURLList = urlList
	p.storedStarList = starList
	p.storedPostTitleList = postList

	return len(p.storedPostTitleList)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"
)

type AppConfig struct {
	verbose     bool
	newRelicKey string
	database    string
	interval    int
	user        string
	host        string
}

func HandleUserOptions() AppConfig {

	var config AppConfig

	flag.BoolVar(&config.verbose, "verbose", false, "Verbose mode")
	flag.StringVar(&config.newRelicKey, "key", "", "Newrelic license key (required)")
	flag.StringVar(&config.database, "database", "", "Database name (required)")
	flag.IntVar(&config.interval, "interval", 1, "Sampling interval [min]")
	flag.StringVar(&config.user, "user", "postgres", "Database user name")
	flag.StringVar(&config.host, "host", "localhost:5432", "Database host")

	flag.Parse()

	if config.newRelicKey == "" ||
		config.database == "" {
		flag.PrintDefaults()
		log.Fatal("Required parameter missing.")
	}

	return config
}
<commit_msg>Add version option for printing build number<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
)

// build number set on during linking
var minversion string

type AppConfig struct {
	verbose     bool
	newRelicKey string
	database    string
	interval    int
	user        string
	host        string
	version     bool
}

func HandleUserOptions() AppConfig {

	var config AppConfig

	flag.BoolVar(&config.verbose, "verbose", false, "Verbose mode")
	flag.StringVar(&config.newRelicKey, "key", "", "Newrelic license key (required)")
	flag.StringVar(&config.database, "database", "", "Database name (required)")
	flag.IntVar(&config.interval, "interval", 1, "Sampling interval [min]")
	flag.StringVar(&config.user, "user", "postgres", "Database user name")
	flag.StringVar(&config.host, "host", "localhost:5432", "Database host")
	flag.BoolVar(&config.version, "version", false, "Print version")

	flag.Parse()

	if config.version {
		fmt.Printf("Build: %s\n", minversion)
		os.Exit(0)
	}

	if config.newRelicKey == "" ||
		config.database == "" {
		flag.PrintDefaults()
		log.Fatal("Required parameter missing.")
	}

	return config
}
<|endoftext|>


<commit_before>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package metadata

import (
	"os/exec"
	"reflect"
	"runtime"
	"testing"
)

var allTests = []struct {
	MD  *T
	XML string
	B64 string
}{
	{
		MD:  FromMap(nil),
		XML: `<metadata></metadata>`,
		B64: `eJyyyU0tSUxJLEm0s9GHMwEBAAD//1RmB6Y=`,
	},
	{
		MD: FromMap(map[string]string{
			"A": `a value`,
			"B": `b value`,
			"C": `c value`,
		}),
		XML: `<metadata>
  <md id="A">a value</md>
  <md id="B">b value</md>
  <md id="C">c value</md>
</metadata>`,
		B64: `eJyyyU0tSUxJLEm0s8lNUchMsVVyVLJLVChLzClNtdHPTYELOynZJWERdlayS0YW1oebBwgAAP//H2Qc4g==`,
	},
	{
		MD: FromMap(map[string]string{
			"A": `a value`,
			"B": `
b value
   has newlines
 galore
`,
			"C": `c value`,
		}),
		XML: `<metadata>
  <md id="A">a value</md>
  <md id="B"><![CDATA[
b value
   has newlines
 galore
  ]]></md>
  <md id="C">c value</md>
</metadata>`,
		B64: `eJyyyU0tSUxJLEm0s8lNUchMsVVyVLJLVChLzClNtdHPTYELOynZ2ShGO7s4hjhGcyVBFHApKChkJBYr5KWW52TmpRZzKaQn5uQXgcRjY+1QtDsr2SUjm6oPtxcQAAD//0qVKG0=`,
	},
	{
		MD: FromMap(map[string]string{
			"BuildPlatform": `amd64unknown-linux-unknown`,
			"BuildTime":     `2015-05-01T01:33:35Z`,
			"Manifest": `
<manifest label="">
  <projects>
    <project exclude="false" name="release.go.v23" path="release/go/src/v.io/v23" protocol="git" remote="https://vanadium.googlesource.com/release.go.v23" revision="16889ec00eb4008849057a5e9014a025a231f836"></project>
    <project exclude="false" name="release.go.x.devtools" path="release/go/src/v.io/x/devtools" protocol="git" remote="https://vanadium.googlesource.com/release.go.x.devtools" revision="de0bd3a5f0f9b30532c41bdb01661cfe8c24df76"></project>
  </projects>
</manifest>
`,
			"ZZZ": `zzz`,
		}),
		XML: `<metadata>
  <md id="BuildPlatform">amd64unknown-linux-unknown</md>
  <md id="BuildTime">2015-05-01T01:33:35Z</md>
  <md id="Manifest"><![CDATA[
<manifest label="">
  <projects>
    <project exclude="false" name="release.go.v23" path="release/go/src/v.io/v23" protocol="git" remote="https://vanadium.googlesource.com/release.go.v23" revision="16889ec00eb4008849057a5e9014a025a231f836"></project>
    <project exclude="false" name="release.go.x.devtools" path="release/go/src/v.io/x/devtools" protocol="git" remote="https://vanadium.googlesource.com/release.go.x.devtools" revision="de0bd3a5f0f9b30532c41bdb01661cfe8c24df76"></project>
  </projects>
</manifest>
  ]]></md>
  <md id="ZZZ">zzz</md>
</metadata>`,
		B64: `eJyskjFv2zAQhff8CpW7RVKUFNmQBaTtWqCDJwcZTuTJYUvqDJFSDf/6MojhxBkKFAjA5d0d7+HDvdZjBAMRutabzJot+zpbZ346iANNnnXgTV3O4++R/owrZ8f5tLqolntz+2tnPbKuELJaifTkTsiNUhtV7W9mf8BoBwyRde2Xx2/fH3YPj3etvxQzBz26LWPdXZa1x4l+oY7hRbzJDE/azQa3bAAXkGUj+CQmdAgB8wPlS6FYdoT4fK3yA/Ewab7klvhre6JImpLXwUaWTegppi3PMR7DhvMFRjB29mkdHRwGmieNuSbPP/pMuNhgadwyWTfNGrUQ2JdCNE25FtU9VLgWsgRRVFAoOTSqTuT8wvK/ZKfc4BKJXPgX4Im/m/oEzveub7gGRW8UVIMY1r0SlSp0KXvTC1nXUg/Y6KI0w/1H3KtIZ03BuBz+pfP01N0kZb/fs+58Pr8W+TWrfwMAAP//M7rlfg==`,
	},
}

func TestToMap(t *testing.T) {
	for _, test := range allTests {
		if got, want := test.MD.ToMap(), test.MD.entries; !reflect.DeepEqual(got, want) {
			t.Errorf("got %v, want %v", got, want)
		}
	}
}

func TestFromMap(t *testing.T) {
	for _, test := range allTests {
		if got, want := FromMap(test.MD.entries), test.MD; !reflect.DeepEqual(got, want) {
			t.Errorf("got %#v, want %#v", got, want)
		}
	}
}

func TestToXML(t *testing.T) {
	for _, test := range allTests {
		if got, want := test.MD.ToXML(), test.XML; got != want {
			t.Errorf("got %v, want %v", got, want)
		}
	}
}

func TestFromXML(t *testing.T) {
	for _, test := range allTests {
		got, err := FromXML([]byte(test.XML))
		if err != nil {
			t.Errorf("%v FromXML failed: %v", test.XML, err)
		}
		if want := test.MD; !reflect.DeepEqual(got, want) {
			t.Errorf("got %#v, want %#v", got, want)
		}
		// Add some garbage and make sure it fails.
		bad := "<notclosed" + test.XML
		if got, err := FromXML([]byte(bad)); got != nil || err == nil {
			t.Errorf("%v FromXML should have failed: (%#v, %v)", bad, got, err)
		}
	}
}

func TestToBase64(t *testing.T) {
	for _, test := range allTests {
		if got, want := test.MD.ToBase64(), test.B64; got != want {
			t.Errorf("got %v, want %v", got, want)
		}
	}
}

func TestFromBase64(t *testing.T) {
	for _, test := range allTests {
		got, err := FromBase64([]byte(test.B64))
		if err != nil {
			t.Errorf("%v FromBase64 failed: %v", test.B64, err)
		}
		if want := test.MD; !reflect.DeepEqual(got, want) {
			t.Errorf("got %#v, want %#v", got, want)
		}
		// Add some garbage and make sure it fails.
		bad := "!@#$%^&*()_+" + test.B64
		if got, err := FromBase64([]byte(bad)); got != nil || err == nil {
			t.Errorf("%v FromBase64 should have failed: (%#v, %v)", bad, got, err)
		}
	}
}

func TestInsertLookup(t *testing.T) {
	tests := []struct {
		ID, Value, Old string
		Map            map[string]string
	}{
		{"A", "abc", "", map[string]string{"A": "abc"}},
		{"B", "123", "", map[string]string{"A": "abc", "B": "123"}},
		{"A", "xyz", "abc", map[string]string{"A": "xyz", "B": "123"}},
		{"C", "s p a c e s", "", map[string]string{"A": "xyz", "B": "123", "C": "s p a c e s"}},
	}
	var x T
	for _, test := range tests {
		if got, want := x.Lookup(test.ID), test.Old; got != want {
			t.Errorf("(%q, %q) Lookup got %q, want %q", test.ID, test.Value, got, want)
		}
		if got, want := x.Insert(test.ID, test.Value), test.Old; got != want {
			t.Errorf("(%q, %q) Insert got %q, want %q", test.ID, test.Value, got, want)
		}
		if got, want := x.Lookup(test.ID), test.Value; got != want {
			t.Errorf("(%q, %q) Lookup got %q, want %q", test.ID, test.Value, got, want)
		}
		// Add some leading and trailing spaces, which will be stripped.
		value2 := "ZZ" + test.Value + "ZZ"
		if got, want := x.Insert(test.ID, " \n\t"+value2+"\n\t "), test.Value; got != want {
			t.Errorf("(%q, %q) Insert got %q, want %q", test.ID, value2, got, want)
		}
		if got, want := x.Lookup(test.ID), value2; got != want {
			t.Errorf("(%q, %q) Lookup got %q, want %q", test.ID, value2, got, want)
		}
		// Set the value back.
		if got, want := x.Insert(test.ID, test.Value), value2; got != want {
			t.Errorf("(%q, %q) Insert got %q, want %q", test.ID, test.Value, got, want)
		}
		// Check the map form.
		if got, want := x.ToMap(), test.Map; !reflect.DeepEqual(got, want) {
			t.Errorf("(%q, %q) ToMap got %q, want %q", test.ID, test.Value, got, want)
		}
	}
}

func TestLDFlag(t *testing.T) {
	for _, test := range allTests {
		got, want := LDFlag(test.MD), "-X "+thisPkgPath+".initBuiltIn="+test.B64
		if got != want {
			t.Errorf("got %q, want %q", got, want)
		}
	}
}

// TestBuiltIn tests the package-level functions that operate on BuiltIn.
func TestBuiltIn(t *testing.T) {
	const id, value1, value2 = "TestID", "testvalue1", "testvalue2"
	if got, want := Lookup(id), ""; got != want {
		t.Errorf("Lookup %s got %q, want %q", id, got, want)
	}
	if got, want := Insert(id, value1), ""; got != want {
		t.Errorf("Insert %s got %q, want %q", id, got, want)
	}
	if got, want := Lookup(id), value1; got != want {
		t.Errorf("Lookup %s got %q, want %q", id, got, want)
	}
	if got, want := Insert(id, value2), value1; got != want {
		t.Errorf("Insert %s got %q, want %q", id, got, want)
	}
	wantMap := map[string]string{
		id:           value2,
		"go.Arch":    runtime.GOARCH,
		"go.OS":      runtime.GOOS,
		"go.Version": runtime.Version(),
	}
	if got, want := ToMap(), wantMap; !reflect.DeepEqual(got, want) {
		t.Errorf("got map %q, want %q", got, want)
	}
	if got, want := ToXML(), FromMap(wantMap).ToXML(); got != want {
		t.Errorf("got xml %q, want %q", got, want)
	}
	if got, want := ToBase64(), FromMap(wantMap).ToBase64(); got != want {
		t.Errorf("got base64 %q, want %q", got, want)
	}
}

// TestInitAndFlag builds a test binary with some metadata, and invokes the
// -metadata flag to make sure it dumps the expected metadata.
func TestInitAndFlag(t *testing.T) {
	// Run the test binary.
	const id, value = "zzzTestID", "abcdefg"
	x := FromMap(map[string]string{id: value})
	cmdRun := exec.Command("go", "run", "-ldflags="+LDFlag(x), "./testdata/testbin.go", "-metadata")
	outXML, err := cmdRun.CombinedOutput()
	if err != nil {
		t.Errorf("%v failed: %v\n%s", cmdRun.Args, err, outXML)
	}
	wantXML := `<metadata>
  <md id="go.Arch">` + runtime.GOARCH + `</md>
  <md id="go.OS">` + runtime.GOOS + `</md>
  <md id="go.Version">` + runtime.Version() + `</md>
  <md id="` + id + `">` + value + `</md>
</metadata>
`
	if got, want := string(outXML), wantXML; got != want {
		t.Errorf("got %q, want %q", got, want)
	}
}
<commit_msg>Get the tests for the metadata package to work for go 1.7<commit_after>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package metadata

import (
	"os/exec"
	"reflect"
	"runtime"
	"testing"
)

var allTests = []struct {
	MD  *T
	XML string
	B64 []string
}{
	{
		MD:  FromMap(nil),
		XML: `<metadata></metadata>`,
		B64: []string{`eJyyyU0tSUxJLEm0s9GHMwEBAAD//1RmB6Y=`},
	},
	{
		MD: FromMap(map[string]string{
			"A": `a value`,
			"B": `b value`,
			"C": `c value`,
		}),
		XML: `<metadata>
  <md id="A">a value</md>
  <md id="B">b value</md>
  <md id="C">c value</md>
</metadata>`,
		B64: []string{`eJyyyU0tSUxJLEm0s8lNUchMsVVyVLJLVChLzClNtdHPTYELOynZJWERdlayS0YW1oebBwgAAP//H2Qc4g==`},
	},
	{
		MD: FromMap(map[string]string{
			"A": `a value`,
			"B": `
b value
   has newlines
 galore
`,
			"C": `c value`,
		}),
		XML: `<metadata>
  <md id="A">a value</md>
  <md id="B"><![CDATA[
b value
   has newlines
 galore
  ]]></md>
  <md id="C">c value</md>
</metadata>`,
		B64: []string{`eJyyyU0tSUxJLEm0s8lNUchMsVVyVLJLVChLzClNtdHPTYELOynZ2ShGO7s4hjhGcyVBFHApKChkJBYr5KWW52TmpRZzKaQn5uQXgcRjY+1QtDsr2SUjm6oPtxcQAAD//0qVKG0=`},
	},
	{
		MD: FromMap(map[string]string{
			"BuildPlatform": `amd64unknown-linux-unknown`,
			"BuildTime":     `2015-05-01T01:33:35Z`,
			"Manifest": `
<manifest label="">
  <projects>
    <project exclude="false" name="release.go.v23" path="release/go/src/v.io/v23" protocol="git" remote="https://vanadium.googlesource.com/release.go.v23" revision="16889ec00eb4008849057a5e9014a025a231f836"></project>
    <project exclude="false" name="release.go.x.devtools" path="release/go/src/v.io/x/devtools" protocol="git" remote="https://vanadium.googlesource.com/release.go.x.devtools" revision="de0bd3a5f0f9b30532c41bdb01661cfe8c24df76"></project>
  </projects>
</manifest>
`,
			"ZZZ": `zzz`,
		}),
		XML: `<metadata>
  <md id="BuildPlatform">amd64unknown-linux-unknown</md>
  <md id="BuildTime">2015-05-01T01:33:35Z</md>
  <md id="Manifest"><![CDATA[
<manifest label="">
  <projects>
    <project exclude="false" name="release.go.v23" path="release/go/src/v.io/v23" protocol="git" remote="https://vanadium.googlesource.com/release.go.v23" revision="16889ec00eb4008849057a5e9014a025a231f836"></project>
    <project exclude="false" name="release.go.x.devtools" path="release/go/src/v.io/x/devtools" protocol="git" remote="https://vanadium.googlesource.com/release.go.x.devtools" revision="de0bd3a5f0f9b30532c41bdb01661cfe8c24df76"></project>
  </projects>
</manifest>
  ]]></md>
  <md id="ZZZ">zzz</md>
</metadata>`,
		B64: []string{
			`eJyskrFu2zAQhvc8hcrdIilKimxIBNJ2LdDBk4MMJ/KksCV1hkiphp++COI6cYYCBTp+/x1x+MC/DZjAQgLdBps527HPi/P2u4c00ByYhmDrcpl+TvRr2ng3LafNhVoe7O2rvQvIdCFktRHVRsi9kDuldqo63Ox+g8kNGBPT7afHL18f9g+Pd224hJmHHn3HmL7LsvY40w80Kb7AG2Z4Mn6x2LEBfESWTRCwYzN6hIj5SPlaKJYdIT1fUz4Sj7Pha+6Iv45nSmTId2x0iWUzBkrYseeUjnHH+QoTWLeEfCQaPUZaZoO5ocA/3plxddHR1DFZN80WjRDYl0I0TbkV1T1UuBWyBFFUUCg5NKpmuuUXl381O+UW10Tk498ET/zd1n/wfH/1Tdei6K2CahDDtleiUoUpZW97IetamgEbU5R2uP+oe4Wo71r+5+NfJk9P+qYph8OB6fP5/Brya1d/BwAA//8zuuV+`,
			`eJyskjFv2zAQhff8CpW7RVKUFNmQBaTtWqCDJwcZTuTJYUvqDJFSDf/6MojhxBkKFAjA5d0d7+HDvdZjBAMRutabzJot+zpbZ346iANNnnXgTV3O4++R/owrZ8f5tLqolntz+2tnPbKuELJaifTkTsiNUhtV7W9mf8BoBwyRde2Xx2/fH3YPj3etvxQzBz26LWPdXZa1x4l+oY7hRbzJDE/azQa3bAAXkGUj+CQmdAgB8wPlS6FYdoT4fK3yA/Ewab7klvhre6JImpLXwUaWTegppi3PMR7DhvMFRjB29mkdHRwGmieNuSbPP/pMuNhgadwyWTfNGrUQ2JdCNE25FtU9VLgWsgRRVFAoOTSqTuT8wvK/ZKfc4BKJXPgX4Im/m/oEzveub7gGRW8UVIMY1r0SlSp0KXvTC1nXUg/Y6KI0w/1H3KtIZ03BuBz+pfP01N0kZb/fs+58Pr8W+TWrfwMAAP//M7rlfg==`,
		},
	},
}

func TestToMap(t *testing.T) {
	for _, test := range allTests {
		if got, want := test.MD.ToMap(), test.MD.entries; !reflect.DeepEqual(got, want) {
			t.Errorf("got %v, want %v", got, want)
		}
	}
}

func TestFromMap(t *testing.T) {
	for _, test := range allTests {
		if got, want := FromMap(test.MD.entries), test.MD; !reflect.DeepEqual(got, want) {
			t.Errorf("got %#v, want %#v", got, want)
		}
	}
}

func TestToXML(t *testing.T) {
	for _, test := range allTests {
		if got, want := test.MD.ToXML(), test.XML; got != want {
			t.Errorf("got %v, want %v", got, want)
		}
	}
}

func TestFromXML(t *testing.T) {
	for _, test := range allTests {
		got, err := FromXML([]byte(test.XML))
		if err != nil {
			t.Errorf("%v FromXML failed: %v", test.XML, err)
		}
		if want := test.MD; !reflect.DeepEqual(got, want) {
			t.Errorf("got %#v, want %#v", got, want)
		}
		// Add some garbage and make sure it fails.
		bad := "<notclosed" + test.XML
		if got, err := FromXML([]byte(bad)); got != nil || err == nil {
			t.Errorf("%v FromXML should have failed: (%#v, %v)", bad, got, err)
		}
	}
}

func TestToBase64(t *testing.T) {
	for _, test := range allTests {
		got := test.MD.ToBase64()
		found := false
		for _, want := range test.B64 {
			if got == want {
				found = true
				break
			}
		}
		if !found {
			t.Errorf("got %q, want one of %q", got, test.B64)
		}
	}
}

func TestFromBase64(t *testing.T) {
	for _, test := range allTests {
		for _, b64 := range test.B64 {
			got, err := FromBase64([]byte(b64))
			if err != nil {
				t.Errorf("%v FromBase64 failed: %v", b64, err)
			}
			if want := test.MD; !reflect.DeepEqual(got, want) {
				t.Errorf("got %#v, want %#v", got, want)
			}
			// Add some garbage and make sure it fails.
			bad := "!@#$%^&*()_+" + b64
			if got, err := FromBase64([]byte(bad)); got != nil || err == nil {
				t.Errorf("%v FromBase64 should have failed: (%#v, %v)", bad, got, err)
			}
		}
	}
}

func TestInsertLookup(t *testing.T) {
	tests := []struct {
		ID, Value, Old string
		Map            map[string]string
	}{
		{"A", "abc", "", map[string]string{"A": "abc"}},
		{"B", "123", "", map[string]string{"A": "abc", "B": "123"}},
		{"A", "xyz", "abc", map[string]string{"A": "xyz", "B": "123"}},
		{"C", "s p a c e s", "", map[string]string{"A": "xyz", "B": "123", "C": "s p a c e s"}},
	}
	var x T
	for _, test := range tests {
		if got, want := x.Lookup(test.ID), test.Old; got != want {
			t.Errorf("(%q, %q) Lookup got %q, want %q", test.ID, test.Value, got, want)
		}
		if got, want := x.Insert(test.ID, test.Value), test.Old; got != want {
			t.Errorf("(%q, %q) Insert got %q, want %q", test.ID, test.Value, got, want)
		}
		if got, want := x.Lookup(test.ID), test.Value; got != want {
			t.Errorf("(%q, %q) Lookup got %q, want %q", test.ID, test.Value, got, want)
		}
		// Add some leading and trailing spaces, which will be stripped.
		value2 := "ZZ" + test.Value + "ZZ"
		if got, want := x.Insert(test.ID, " \n\t"+value2+"\n\t "), test.Value; got != want {
			t.Errorf("(%q, %q) Insert got %q, want %q", test.ID, value2, got, want)
		}
		if got, want := x.Lookup(test.ID), value2; got != want {
			t.Errorf("(%q, %q) Lookup got %q, want %q", test.ID, value2, got, want)
		}
		// Set the value back.
		if got, want := x.Insert(test.ID, test.Value), value2; got != want {
			t.Errorf("(%q, %q) Insert got %q, want %q", test.ID, test.Value, got, want)
		}
		// Check the map form.
		if got, want := x.ToMap(), test.Map; !reflect.DeepEqual(got, want) {
			t.Errorf("(%q, %q) ToMap got %q, want %q", test.ID, test.Value, got, want)
		}
	}
}

func TestLDFlag(t *testing.T) {
	for _, test := range allTests {
		got := LDFlag(test.MD)
		found := false
		for _, b64 := range test.B64 {
			want := "-X " + thisPkgPath + ".initBuiltIn=" + b64
			if got == want {
				found = true
				break
			}
		}
		if !found {
			t.Errorf("got %q, want one of %v", got, test.B64)
		}
	}
}

// TestBuiltIn tests the package-level functions that operate on BuiltIn.
func TestBuiltIn(t *testing.T) {
	const id, value1, value2 = "TestID", "testvalue1", "testvalue2"
	if got, want := Lookup(id), ""; got != want {
		t.Errorf("Lookup %s got %q, want %q", id, got, want)
	}
	if got, want := Insert(id, value1), ""; got != want {
		t.Errorf("Insert %s got %q, want %q", id, got, want)
	}
	if got, want := Lookup(id), value1; got != want {
		t.Errorf("Lookup %s got %q, want %q", id, got, want)
	}
	if got, want := Insert(id, value2), value1; got != want {
		t.Errorf("Insert %s got %q, want %q", id, got, want)
	}
	wantMap := map[string]string{
		id:           value2,
		"go.Arch":    runtime.GOARCH,
		"go.OS":      runtime.GOOS,
		"go.Version": runtime.Version(),
	}
	if got, want := ToMap(), wantMap; !reflect.DeepEqual(got, want) {
		t.Errorf("got map %q, want %q", got, want)
	}
	if got, want := ToXML(), FromMap(wantMap).ToXML(); got != want {
		t.Errorf("got xml %q, want %q", got, want)
	}
	if got, want := ToBase64(), FromMap(wantMap).ToBase64(); got != want {
		t.Errorf("got base64 %q, want %q", got, want)
	}
}

// TestInitAndFlag builds a test binary with some metadata, and invokes the
// -metadata flag to make sure it dumps the expected metadata.
func TestInitAndFlag(t *testing.T) {
	// Run the test binary.
	const id, value = "zzzTestID", "abcdefg"
	x := FromMap(map[string]string{id: value})
	cmdRun := exec.Command("go", "run", "-ldflags="+LDFlag(x), "./testdata/testbin.go", "-metadata")
	outXML, err := cmdRun.CombinedOutput()
	if err != nil {
		t.Errorf("%v failed: %v\n%s", cmdRun.Args, err, outXML)
	}
	wantXML := `<metadata>
  <md id="go.Arch">` + runtime.GOARCH + `</md>
  <md id="go.OS">` + runtime.GOOS + `</md>
  <md id="go.Version">` + runtime.Version() + `</md>
  <md id="` + id + `">` + value + `</md>
</metadata>
`
	if got, want := string(outXML), wantXML; got != want {
		t.Errorf("got %q, want %q", got, want)
	}
}
<|endoftext|>


<commit_before><commit_msg>Improved cli output<commit_after><|endoftext|>


<commit_before><commit_msg>go-bindata changes to dba.go.<commit_after><|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"os"
)

var welcome = `
 __      __.__            __
/  \    /  \  |__ _____ _/  |_ __ __  ____    _________
\   \/\/   /  |  \\__  \\   __\  |  \/    \  / ___\__  \
 \        /|   Y  \/ __ \|  | |  |  /   |  \/ /_/  > __ \_
  \__/\  / |___|  (____  /__| |____/|___|  /\___  (____  /
       \/       \/     \/                \//_____/     \/
`

// State constants
type state uint

const (
	COMMANDS      state = iota
	HELP                = iota
	SHOW                = iota
	ADD_TYPE            = iota
	ADD_PATH            = iota
	ADD_VALUE           = iota
	ADD_PARAMETER       = iota
	SET_PATH            = iota
	RM                  = iota
	VALIDATE            = iota
)

var currentState state = COMMANDS
var workingDir string = "/"

// A function which returns a list of possible options for a state
type autoComplete func(from state, input string) []string

func shell(info string, project *Project) {
	fmt.Printf("%s\n\n%s\n\n%s\n\n", info, welcome, Version())
	scanner := bufio.NewScanner(os.Stdin)
	prompt(project)
	for scanner.Scan() {
		cmd := scanner.Text()
		if cmd == "exit" {
			break
		}
		fmt.Println(cmd)
		prompt(project)
	}
	if err := scanner.Err(); err != nil {
		fmt.Fprintln(os.Stderr, "reading standard input:", err)
	}
}

func prompt(project *Project) {
	fmt.Printf("[%s %s @ %s ]> ", project.Name, project.Version, workingDir)
}
<commit_msg>Shell - wip<commit_after>package main

import (
	"bufio"
	"fmt"
	"os"
)

var welcome = `
 __      __.__            __
/  \    /  \  |__ _____ _/  |_ __ __  ____    _________
\   \/\/   /  |  \\__  \\   __\  |  \/    \  / ___\__  \
 \        /|   Y  \/ __ \|  | |  |  /   |  \/ /_/  > __ \_
  \__/\  / |___|  (____  /__| |____/|___|  /\___  (____  /
       \/       \/     \/                \//_____/     \/
`

// State constants
type state uint

const (
	COMMANDS      state = iota
	HELP                = iota
	SHOW                = iota
	ADD_TYPE            = iota
	ADD_PATH            = iota
	ADD_VALUE           = iota
	ADD_PARAMETER       = iota
	SET_PATH            = iota
	RM                  = iota
	VALIDATE            = iota
)

//var currentState state = COMMANDS
var workingDir string = "/"

// A function which returns a list of possible options for a state
//type autoComplete func(from state, input string) []string

func shell(info string, project *Project) {
	fmt.Printf("%s\n\n%s\n\n%s\n\n", info, welcome, Version())
	scanner := bufio.NewScanner(os.Stdin)
	prompt(project)
	for scanner.Scan() {
		cmd := scanner.Text()
		if cmd == "exit" {
			break
		}
		fmt.Println(cmd)
		prompt(project)
	}
	if err := scanner.Err(); err != nil {
		fmt.Fprintln(os.Stderr, "Error reading command:", err)
	}
}

func prompt(project *Project) {
	fmt.Printf("[%s %s @ %s ]> ", project.Name, project.Version, workingDir)
}
<|endoftext|>


<commit_before>package scipipe

import (
	"errors"
	"fmt"
	"os"
	"os/exec"
	re "regexp"
	str "strings"
)

type ShellProcess struct {
	process
	InPorts          map[string]chan *FileTarget
	OutPorts         map[string]chan *FileTarget
	OutPortsDoStream map[string]bool
	OutPathFuncs     map[string]func(*ShellTask) string
	ParamPorts       map[string]chan string
	Prepend          string
	CommandPattern   string
	Spawn            bool
}

func NewShellProcess(command string) *ShellProcess {
	return &ShellProcess{
		CommandPattern:   command,
		InPorts:          make(map[string]chan *FileTarget),
		OutPorts:         make(map[string]chan *FileTarget),
		OutPortsDoStream: make(map[string]bool),
		OutPathFuncs:     make(map[string]func(*ShellTask) string),
		ParamPorts:       make(map[string]chan string),
		Spawn:            true,
	}
}

func Shell(cmd string) *ShellProcess {
	if !LogExists {
		InitLogInfo()
	}
	p := NewShellProcess(cmd)
	p.initPortsFromCmdPattern(cmd, nil)
	return p
}

func Sh(cmd string) *ShellProcess {
	return Shell(cmd)
}

func (p *ShellProcess) Run() {
	defer p.closeOutPorts()

	tasks := []*ShellTask{}
	Debug.Printf("[%s] Starting to loop over tasks\n", p.CommandPattern)
	for t := range p.createTasks() {
		tasks = append(tasks, t)
		Debug.Println("Now processing task", t.Command, "...")

		// Sending FIFOs for the task
		for oname, otgt := range t.OutTargets {
			if otgt.doStream {
				Debug.Printf("[%s] Sending FIFO target on outport %s ...\n", p.CommandPattern, oname)
				p.OutPorts[oname] <- otgt
			}
		}

		// Run the task
		go t.Run()
	}

	// Wait for finish, and send out targets in arrival order
	for _, t := range tasks {
		Debug.Printf("[%s] Waiting for Done from task: %s\n", p.CommandPattern, t.Command)
		<-t.Done
		Debug.Printf("[%s] Receiving Done from task: %s\n", p.CommandPattern, t.Command)
		for oname, otgt := range t.OutTargets {
			if !otgt.doStream {
				Debug.Printf("[%s] Sending target on outport %s ...\n", p.CommandPattern, oname)
				p.OutPorts[oname] <- otgt
			}
		}
	}
}

func (p *ShellProcess) initPortsFromCmdPattern(cmd string, params map[string]string) {
	// Find in/out port names and Params and set up in struct fields
	r := getPlaceHolderRegex()
	ms := r.FindAllStringSubmatch(cmd, -1)
	for _, m := range ms {
		if len(m) < 3 {
			Check(errors.New("Too few matches"))
		}
		typ := m[1]
		name := m[2]
		if typ == "o" || typ == "os" {
			p.OutPorts[name] = make(chan *FileTarget, BUFSIZE)
			if typ == "os" {
				p.OutPortsDoStream[name] = true
			}
		} else if typ == "i" {
			// Set up a channel on the inports, even though this is
			// often replaced by another processes output port channel.
			// It might be nice to have it init'ed with a channel
			// anyways, for use cases when we want to send FileTargets
			// on the inport manually.
			p.InPorts[name] = make(chan *FileTarget, BUFSIZE)
		} else if typ == "p" {
			if params == nil || params[name] == "" {
				p.ParamPorts[name] = make(chan string, BUFSIZE)
			}
		}
	}
}

func (p *ShellProcess) createTasks() (ch chan *ShellTask) {
	ch = make(chan *ShellTask)
	go func() {
		defer close(ch)
		for {
			inTargets, inPortsOpen := p.receiveInputs()
			Debug.Printf("[%s] Got inTargets: %s", p.CommandPattern, inTargets)
			params, paramPortsOpen := p.receiveParams()
			Debug.Printf("[%s] Got params: %s", p.CommandPattern, params)
			if !inPortsOpen && !paramPortsOpen {
				Debug.Printf("[%s] Breaking: Both inPorts and paramPorts closed", p.CommandPattern)
				break
			}
			if len(p.InPorts) == 0 && !paramPortsOpen {
				Debug.Printf("[%s] Breaking: No inports, and params closed", p.CommandPattern)
				break
			}
			if len(p.ParamPorts) == 0 && !inPortsOpen {
				Debug.Printf("[%s] Breaking: No params, and inPorts closed", p.CommandPattern)
				break
			}
			t := NewShellTask(p.CommandPattern, inTargets, p.OutPathFuncs, p.OutPortsDoStream, params, p.Prepend)
			ch <- t
			if len(p.InPorts) == 0 && len(p.ParamPorts) == 0 {
				Debug.Printf("[%s] Breaking: No inports nor params", p.CommandPattern)
				break
			}
		}
	}()
	return ch
}

func (p *ShellProcess) receiveInputs() (inTargets map[string]*FileTarget, inPortsOpen bool) {
	inPortsOpen = true
	inTargets = make(map[string]*FileTarget)
	// Read input targets on in-ports and set up path mappings
	for iname, ichan := range p.InPorts {
		Debug.Printf("[%s] Receieving on inPort %s ...", p.CommandPattern, iname)
		inTarget, open := <-ichan
		if !open {
			inPortsOpen = false
			continue
		}
		Debug.Printf("[%s] Got inTarget %s ...", p.CommandPattern, inTarget.GetPath())
		inTargets[iname] = inTarget
	}
	return
}

func (p *ShellProcess) receiveParams() (params map[string]string, paramPortsOpen bool) {
	paramPortsOpen = true
	params = make(map[string]string)
	// Read input targets on in-ports and set up path mappings
	for pname, pchan := range p.ParamPorts {
		pval, open := <-pchan
		if !open {
			paramPortsOpen = false
			continue
		}
		Debug.Println("Receiving param:", pname, "with value", pval)
		params[pname] = pval
	}
	return
}

func (p *ShellProcess) closeOutPorts() {
	for oname, oport := range p.OutPorts {
		Debug.Printf("[%s] Closing port %s ...\n", p.CommandPattern, oname)
		close(oport)
	}
}

func getPlaceHolderRegex() *re.Regexp {
	r, err := re.Compile("{(o|os|i|is|p):([^{}:]+)}")
	Check(err)
	return r
}

// ------- ShellTask -------

type ShellTask struct {
	InTargets  map[string]*FileTarget
	OutTargets map[string]*FileTarget
	Params     map[string]string
	Command    string
	Done       chan int
}

func NewShellTask(cmdPat string, inTargets map[string]*FileTarget, outPathFuncs map[string]func(*ShellTask) string, outPortsDoStream map[string]bool, params map[string]string, prepend string) *ShellTask {
	t := &ShellTask{
		InTargets:  inTargets,
		OutTargets: make(map[string]*FileTarget),
		Params:     params,
		Command:    "",
		Done:       make(chan int),
	}
	// Create out targets
	Debug.Printf("[%s] Creating outTargets now ...", cmdPat)
	outTargets := make(map[string]*FileTarget)
	for oname, ofun := range outPathFuncs {
		opath := ofun(t)
		otgt := NewFileTarget(opath)
		if outPortsDoStream[oname] {
			otgt.doStream = true
		}
		Debug.Printf("[%s] Creating outTarget with path %s ...", cmdPat, opath)
		outTargets[oname] = otgt
	}
	t.OutTargets = outTargets
	t.Command = formatCommand(cmdPat, inTargets, outTargets, params, prepend)
	Debug.Printf("[%s] Created formatted command: %s", cmdPat, t.Command)
	return t
}

func (t *ShellTask) Run() {
	defer close(t.Done) // TODO: Is this needed?
	if !t.anyOutputExists() {
		t.executeCommand(t.Command)
		t.atomizeTargets()
		t.cleanUpFifos()
	}
	Debug.Printf("[%s] Starting to send Done in t.Run() ...)\n", t.Command)
	t.Done <- 1
	Debug.Printf("[%s] Done sending Done, in t.Run()\n", t.Command)
}

func (t *ShellTask) executeCommand(cmd string) {
	Info.Printf("[%s] Executing command: %s \n", t.Command, cmd)
	_, err := exec.Command("bash", "-c", cmd).Output()
	Check(err)
}

func (t *ShellTask) GetInPath(inPort string) string {
	return t.InTargets[inPort].GetPath()
}

func formatCommand(cmd string, inTargets map[string]*FileTarget, outTargets map[string]*FileTarget, params map[string]string, prepend string) string {

	// Debug.Println("Formatting command with the following data:")
	// Debug.Println("prepend:", prepend)
	// Debug.Println("cmd:", cmd)
	// Debug.Println("inTargets:", inTargets)
	// Debug.Println("outTargets:", outTargets)
	// Debug.Println("params:", params)

	r := getPlaceHolderRegex()
	ms := r.FindAllStringSubmatch(cmd, -1)
	for _, m := range ms {
		whole := m[0]
		typ := m[1]
		name := m[2]
		var newstr string
		if typ == "o" || typ == "os" {
			if outTargets[name] == nil {
				msg := fmt.Sprint("Missing outpath for outport '", name, "' for command '", cmd, "'")
				Check(errors.New(msg))
			} else {
				if typ == "o" {
					newstr = outTargets[name].GetTempPath() // Means important to Atomize afterwards!
				} else if typ == "os" {
					newstr = outTargets[name].GetFifoPath()
				}
			}
		} else if typ == "i" {
			if inTargets[name] == nil {
				msg := fmt.Sprint("Missing intarget for inport '", name, "' for command '", cmd, "'")
				Check(errors.New(msg))
			} else if inTargets[name].GetPath() == "" {
				msg := fmt.Sprint("Missing inpath for inport '", name, "' for command '", cmd, "'")
				Check(errors.New(msg))
			} else {
				if typ == "i" {
					if inTargets[name].doStream {
						newstr = inTargets[name].GetFifoPath()
					} else {
						newstr = inTargets[name].GetPath()
					}
				}
			}
		} else if typ == "p" {
			if params[name] == "" {
				msg := fmt.Sprint("Missing param value param '", name, "' for command '", cmd, "'")
				Check(errors.New(msg))
			} else {
				newstr = params[name]
			}
		}
		if newstr == "" {
			msg := fmt.Sprint("Replace failed for port ", name, " forcommand '", cmd, "'")
			Check(errors.New(msg))
		}
		cmd = str.Replace(cmd, whole, newstr, -1)
	}
	// Add prepend string to the command
	if prepend != "" {
		cmd = fmt.Sprintf("%s %s", prepend, cmd)
	}
	return cmd
}

func (t *ShellTask) atomizeTargets() {
	for _, tgt := range t.OutTargets {
		if !tgt.doStream {
			Debug.Printf("Atomizing file: %s -> %s", tgt.GetTempPath(), tgt.GetPath())
			tgt.Atomize()
			Debug.Printf("Done atomizing file: %s -> %s", tgt.GetTempPath(), tgt.GetPath())
		} else {
			Debug.Printf("Target is streaming, so not atomizing: %s", tgt.GetPath())
		}
	}
}

func (t *ShellTask) cleanUpFifos() {
	for _, tgt := range t.OutTargets {
		if tgt.doStream {
			Debug.Printf("[%s] Cleaning up FIFO for input target: %s\n", t.Command, tgt.GetFifoPath())
			tgt.RemoveFifo()
		} else {
			Debug.Printf("[%s] input target is not FIFO, so not removing any FIFO: %s\n", t.Command, tgt.GetPath())
		}
	}
}

func (t *ShellTask) anyOutputExists() (anyFileExists bool) {
	anyFileExists = false
	for _, tgt := range t.OutTargets {
		opath := tgt.GetPath()
		otmpPath := tgt.GetTempPath()
		ofifoPath := tgt.GetFifoPath()
		if _, err := os.Stat(opath); err == nil {
			Warn.Printf("[%s] Output file already exists: %s. Check your workflow for correctness!\n", t.Command, opath)
			anyFileExists = true
		}
		if _, err := os.Stat(otmpPath); err == nil {
			Warn.Printf("[%s] Temporary Output file already exists: %s. Check your workflow for correctness!\n", t.Command, otmpPath)
			anyFileExists = true
		}
		if _, err := os.Stat(ofifoPath); err == nil {
			Warn.Printf("[%s] FIFO Output file already exists: %s. Check your workflow for correctness!\n", t.Command, ofifoPath)
			anyFileExists = true
		}
	}
	return
}
<commit_msg>Create Fifos. Don't stop on existing FIFO, require it!<commit_after>package scipipe

import (
	"errors"
	"fmt"
	"os"
	"os/exec"
	re "regexp"
	str "strings"
)

type ShellProcess struct {
	process
	InPorts          map[string]chan *FileTarget
	OutPorts         map[string]chan *FileTarget
	OutPortsDoStream map[string]bool
	OutPathFuncs     map[string]func(*ShellTask) string
	ParamPorts       map[string]chan string
	Prepend          string
	CommandPattern   string
	Spawn            bool
}

func NewShellProcess(command string) *ShellProcess {
	return &ShellProcess{
		CommandPattern:   command,
		InPorts:          make(map[string]chan *FileTarget),
		OutPorts:         make(map[string]chan *FileTarget),
		OutPortsDoStream: make(map[string]bool),
		OutPathFuncs:     make(map[string]func(*ShellTask) string),
		ParamPorts:       make(map[string]chan string),
		Spawn:            true,
	}
}

func Shell(cmd string) *ShellProcess {
	if !LogExists {
		InitLogInfo()
	}
	p := NewShellProcess(cmd)
	p.initPortsFromCmdPattern(cmd, nil)
	return p
}

func Sh(cmd string) *ShellProcess {
	return Shell(cmd)
}

func (p *ShellProcess) Run() {
	defer p.closeOutPorts()

	tasks := []*ShellTask{}
	Debug.Printf("[%s] Starting to loop over tasks\n", p.CommandPattern)
	for t := range p.createTasks() {
		tasks = append(tasks, t)
		Debug.Println("Now processing task", t.Command, "...")

		t.createFifos()

		// Sending FIFOs for the task
		for oname, otgt := range t.OutTargets {
			if otgt.doStream {
				Debug.Printf("[%s] Sending FIFO target on outport %s ...\n", p.CommandPattern, oname)
				p.OutPorts[oname] <- otgt
			}
		}

		// Run the task
		go t.Run()
	}

	// Wait for finish, and send out targets in arrival order
	for _, t := range tasks {
		Debug.Printf("[%s] Waiting for Done from task: %s\n", p.CommandPattern, t.Command)
		<-t.Done
		Debug.Printf("[%s] Receiving Done from task: %s\n", p.CommandPattern, t.Command)
		for oname, otgt := range t.OutTargets {
			if !otgt.doStream {
				Debug.Printf("[%s] Sending target on outport %s ...\n", p.CommandPattern, oname)
				p.OutPorts[oname] <- otgt
			}
		}
	}
}

func (p *ShellProcess) initPortsFromCmdPattern(cmd string, params map[string]string) {
	// Find in/out port names and Params and set up in struct fields
	r := getPlaceHolderRegex()
	ms := r.FindAllStringSubmatch(cmd, -1)
	for _, m := range ms {
		if len(m) < 3 {
			Check(errors.New("Too few matches"))
		}
		typ := m[1]
		name := m[2]
		if typ == "o" || typ == "os" {
			p.OutPorts[name] = make(chan *FileTarget, BUFSIZE)
			if typ == "os" {
				p.OutPortsDoStream[name] = true
			}
		} else if typ == "i" {
			// Set up a channel on the inports, even though this is
			// often replaced by another processes output port channel.
			// It might be nice to have it init'ed with a channel
			// anyways, for use cases when we want to send FileTargets
			// on the inport manually.
			p.InPorts[name] = make(chan *FileTarget, BUFSIZE)
		} else if typ == "p" {
			if params == nil || params[name] == "" {
				p.ParamPorts[name] = make(chan string, BUFSIZE)
			}
		}
	}
}

func (p *ShellProcess) createTasks() (ch chan *ShellTask) {
	ch = make(chan *ShellTask)
	go func() {
		defer close(ch)
		for {
			inTargets, inPortsOpen := p.receiveInputs()
			Debug.Printf("[%s] Got inTargets: %s", p.CommandPattern, inTargets)
			params, paramPortsOpen := p.receiveParams()
			Debug.Printf("[%s] Got params: %s", p.CommandPattern, params)
			if !inPortsOpen && !paramPortsOpen {
				Debug.Printf("[%s] Breaking: Both inPorts and paramPorts closed", p.CommandPattern)
				break
			}
			if len(p.InPorts) == 0 && !paramPortsOpen {
				Debug.Printf("[%s] Breaking: No inports, and params closed", p.CommandPattern)
				break
			}
			if len(p.ParamPorts) == 0 && !inPortsOpen {
				Debug.Printf("[%s] Breaking: No params, and inPorts closed", p.CommandPattern)
				break
			}
			t := NewShellTask(p.CommandPattern, inTargets, p.OutPathFuncs, p.OutPortsDoStream, params, p.Prepend)
			ch <- t
			if len(p.InPorts) == 0 && len(p.ParamPorts) == 0 {
				Debug.Printf("[%s] Breaking: No inports nor params", p.CommandPattern)
				break
			}
		}
	}()
	return ch
}

func (p *ShellProcess) receiveInputs() (inTargets map[string]*FileTarget, inPortsOpen bool) {
	inPortsOpen = true
	inTargets = make(map[string]*FileTarget)
	// Read input targets on in-ports and set up path mappings
	for iname, ichan := range p.InPorts {
		Debug.Printf("[%s] Receieving on inPort %s ...", p.CommandPattern, iname)
		inTarget, open := <-ichan
		if !open {
			inPortsOpen = false
			continue
		}
		Debug.Printf("[%s] Got inTarget %s ...", p.CommandPattern, inTarget.GetPath())
		inTargets[iname] = inTarget
	}
	return
}

func (p *ShellProcess) receiveParams() (params map[string]string, paramPortsOpen bool) {
	paramPortsOpen = true
	params = make(map[string]string)
	// Read input targets on in-ports and set up path mappings
	for pname, pchan := range p.ParamPorts {
		pval, open := <-pchan
		if !open {
			paramPortsOpen = false
			continue
		}
		Debug.Println("Receiving param:", pname, "with value", pval)
		params[pname] = pval
	}
	return
}

func (p *ShellProcess) closeOutPorts() {
	for oname, oport := range p.OutPorts {
		Debug.Printf("[%s] Closing port %s ...\n", p.CommandPattern, oname)
		close(oport)
	}
}

func getPlaceHolderRegex() *re.Regexp {
	r, err := re.Compile("{(o|os|i|is|p):([^{}:]+)}")
	Check(err)
	return r
}

// ------- ShellTask -------

type ShellTask struct {
	InTargets  map[string]*FileTarget
	OutTargets map[string]*FileTarget
	Params     map[string]string
	Command    string
	Done       chan int
}

func NewShellTask(cmdPat string, inTargets map[string]*FileTarget, outPathFuncs map[string]func(*ShellTask) string, outPortsDoStream map[string]bool, params map[string]string, prepend string) *ShellTask {
	t := &ShellTask{
		InTargets:  inTargets,
		OutTargets: make(map[string]*FileTarget),
		Params:     params,
		Command:    "",
		Done:       make(chan int),
	}
	// Create out targets
	Debug.Printf("[%s] Creating outTargets now ...", cmdPat)
	outTargets := make(map[string]*FileTarget)
	for oname, ofun := range outPathFuncs {
		opath := ofun(t)
		otgt := NewFileTarget(opath)
		if outPortsDoStream[oname] {
			otgt.doStream = true
		}
		Debug.Printf("[%s] Creating outTarget with path %s ...", cmdPat, opath)
		outTargets[oname] = otgt
	}
	t.OutTargets = outTargets
	t.Command = formatCommand(cmdPat, inTargets, outTargets, params, prepend)
	Debug.Printf("[%s] Created formatted command: %s", cmdPat, t.Command)
	return t
}

func (t *ShellTask) Run() {
	defer close(t.Done) // TODO: Is this needed?
	if !t.anyOutputExists() && !t.fifosInOutTargetsMissing() {
		t.executeCommand(t.Command)
		t.atomizeTargets()
		t.cleanUpFifos()
	}
	Debug.Printf("[%s] Starting to send Done in t.Run() ...)\n", t.Command)
	t.Done <- 1
	Debug.Printf("[%s] Done sending Done, in t.Run()\n", t.Command)
}

func (t *ShellTask) executeCommand(cmd string) {
	Info.Printf("[%s] Executing command: %s \n", t.Command, cmd)
	_, err := exec.Command("bash", "-c", cmd).Output()
	Check(err)
}

func (t *ShellTask) GetInPath(inPort string) string {
	return t.InTargets[inPort].GetPath()
}

func formatCommand(cmd string, inTargets map[string]*FileTarget, outTargets map[string]*FileTarget, params map[string]string, prepend string) string {

	// Debug.Println("Formatting command with the following data:")
	// Debug.Println("prepend:", prepend)
	// Debug.Println("cmd:", cmd)
	// Debug.Println("inTargets:", inTargets)
	// Debug.Println("outTargets:", outTargets)
	// Debug.Println("params:", params)

	r := getPlaceHolderRegex()
	ms := r.FindAllStringSubmatch(cmd, -1)
	for _, m := range ms {
		whole := m[0]
		typ := m[1]
		name := m[2]
		var newstr string
		if typ == "o" || typ == "os" {
			if outTargets[name] == nil {
				msg := fmt.Sprint("Missing outpath for outport '", name, "' for command '", cmd, "'")
				Check(errors.New(msg))
			} else {
				if typ == "o" {
					newstr = outTargets[name].GetTempPath() // Means important to Atomize afterwards!
				} else if typ == "os" {
					newstr = outTargets[name].GetFifoPath()
				}
			}
		} else if typ == "i" {
			if inTargets[name] == nil {
				msg := fmt.Sprint("Missing intarget for inport '", name, "' for command '", cmd, "'")
				Check(errors.New(msg))
			} else if inTargets[name].GetPath() == "" {
				msg := fmt.Sprint("Missing inpath for inport '", name, "' for command '", cmd, "'")
				Check(errors.New(msg))
			} else {
				if typ == "i" {
					if inTargets[name].doStream {
						newstr = inTargets[name].GetFifoPath()
					} else {
						newstr = inTargets[name].GetPath()
					}
				}
			}
		} else if typ == "p" {
			if params[name] == "" {
				msg := fmt.Sprint("Missing param value param '", name, "' for command '", cmd, "'")
				Check(errors.New(msg))
			} else {
				newstr = params[name]
			}
		}
		if newstr == "" {
			msg := fmt.Sprint("Replace failed for port ", name, " forcommand '", cmd, "'")
			Check(errors.New(msg))
		}
		cmd = str.Replace(cmd, whole, newstr, -1)
	}
	// Add prepend string to the command
	if prepend != "" {
		cmd = fmt.Sprintf("%s %s", prepend, cmd)
	}
	return cmd
}

func (t *ShellTask) atomizeTargets() {
	for _, tgt := range t.OutTargets {
		if !tgt.doStream {
			Debug.Printf("Atomizing file: %s -> %s", tgt.GetTempPath(), tgt.GetPath())
			tgt.Atomize()
			Debug.Printf("Done atomizing file: %s -> %s", tgt.GetTempPath(), tgt.GetPath())
		} else {
			Debug.Printf("Target is streaming, so not atomizing: %s", tgt.GetPath())
		}
	}
}

func (t *ShellTask) cleanUpFifos() {
	for _, tgt := range t.OutTargets {
		if tgt.doStream {
			Debug.Printf("[%s] Cleaning up FIFO for input target: %s\n", t.Command, tgt.GetFifoPath())
			tgt.RemoveFifo()
		} else {
			Debug.Printf("[%s] input target is not FIFO, so not removing any FIFO: %s\n", t.Command, tgt.GetPath())
		}
	}
}

func (t *ShellTask) anyOutputExists() (anyFileExists bool) {
	anyFileExists = false
	for _, tgt := range t.OutTargets {
		opath := tgt.GetPath()
		otmpPath := tgt.GetTempPath()
		if !tgt.doStream {
			if _, err := os.Stat(opath); err == nil {
				Warn.Printf("[%s] Output file already exists: %s. Check your workflow for correctness!\n", t.Command, opath)
				anyFileExists = true
			}
			if _, err := os.Stat(otmpPath); err == nil {
				Warn.Printf("[%s] Temporary Output file already exists: %s. Check your workflow for correctness!\n", t.Command, otmpPath)
				anyFileExists = true
			}
		}
	}
	return
}

func (t *ShellTask) fifosInOutTargetsMissing() (fifosInOutTargetsMissing bool) {
	fifosInOutTargetsMissing = false
	for _, tgt := range t.OutTargets {
		if tgt.doStream {
			ofifoPath := tgt.GetFifoPath()
			if _, err := os.Stat(ofifoPath); err != nil {
				Warn.Printf("[%s] FIFO Output file missing, for streaming output: %s. Check your workflow for correctness!\n", t.Command, ofifoPath)
				fifosInOutTargetsMissing = true
			}
		}
	}
	return
}

func (t *ShellTask) createFifos() {
	for _, otgt := range t.OutTargets {
		if otgt.doStream {
			Debug.Println("Creating FIFO:", otgt.GetFifoPath())
			otgt.CreateFifo()
		}
	}
}
<|endoftext|>


<commit_before><commit_msg>Updated release instructions<commit_after><|endoftext|>


<commit_before><commit_msg>add more types<commit_after><|endoftext|>


<commit_before>package sudoku

import (
	"math/rand"
	"sync"
)

func (self *mutableGridImpl) Solve() bool {

	//Special case; check if it's already solved.
	//TODO: removing this causes Solve, when called on an already solved grid, to sometimes fail. Why is that?
	if self.Solved() {
		return true
	}

	//TODO: Optimization: we only need one, so we can bail as soon as we find a single one.
	solutions := self.Solutions()
	if len(solutions) == 0 {
		return false
	}
	self.LoadSDK(solutions[0].DataString())
	return true
}

func (self *gridImpl) NumSolutions() int {
	//TODO: implement this!
	return 0
}

func (self *mutableGridImpl) NumSolutions() int {
	return len(self.Solutions())
}

func (self *gridImpl) HasSolution() bool {
	//TODO: implement this!
	return false
}

func (self *mutableGridImpl) HasSolution() bool {
	//TODO: optimize this to bail as soon as we find a single solution.
	return len(self.nOrFewerSolutions(1)) > 0
}

func (self *gridImpl) HasMultipleSolutions() bool {
	//TODO: implement this!
	return false
}

func (self *mutableGridImpl) HasMultipleSolutions() bool {
	return len(self.nOrFewerSolutions(2)) >= 2
}

func (self *gridImpl) Solutions() []Grid {
	//TODO: implement this
	return nil
}

func (self *mutableGridImpl) Solutions() (solutions []Grid) {
	return self.nOrFewerSolutions(0)
}

//The actual workhorse of solutions generating. 0 means "as many as you can
//find". It might return more than you asked for, if it already had more
//results than requested sitting around.
func (self *mutableGridImpl) nOrFewerSolutions(max int) []Grid {

	self.cachedSolutionsLockRef.RLock()
	hasNoCachedSolutions := self.cachedSolutionsRef == nil
	cachedSolutionsLen := self.cachedSolutionsRequestedLengthRef
	self.cachedSolutionsLockRef.RUnlock()

	if hasNoCachedSolutions || (max == 0 && cachedSolutionsLen != 0) || (max > 0 && cachedSolutionsLen < max && cachedSolutionsLen > 0) {

		queueDone := make(chan bool, 1)

		queue := newSyncedFiniteQueue(0, DIM*DIM, queueDone)

		queue.In <- self.Copy()

		//In the past this wasn't buffered, but then when we finished early other items would try to go into it
		//and block, which prevented them from looping back up and getting the signal to shut down.
		//Since there's only a known number of threads, we'll make sure they all ahve a place to leave their work
		//without blocking so they can get the signal to shut down.
		incomingSolutions := make(chan Grid, _NUM_SOLVER_THREADS)

		//Using a pattern for closing fan in style receivers from http://blog.golang.org/pipelines
		var wg sync.WaitGroup

		var solutions []Grid

		//TODO: figure out a way to kill all of these threads when necessary.

		//TODO: don't use a constant here, use someting around the lines of
		//numCPU

		for i := 0; i < _NUM_SOLVER_THREADS; i++ {
			go func() {
				//Sovler thread.
				firstRun := true
				defer wg.Done()
				for {
					grid, ok := <-queue.Out
					if !ok {
						return
					}
					result := grid.(Grid).searchSolutions(queue, firstRun, max)
					if result != nil {
						incomingSolutions <- result
					}
					queue.ItemDone <- true
					firstRun = false
				}
			}()
		}

		wg.Add(_NUM_SOLVER_THREADS)

		go func() {
			wg.Wait()
			close(incomingSolutions)
		}()

	OuterLoop:
		for {
			select {
			case solution := <-incomingSolutions:
				//incomingSolutions must have been closed because no more work to do.
				if solution == nil {
					break OuterLoop
				}
				//Add it to results
				solutions = append(solutions, solution)
				if max > 0 && len(solutions) >= max {
					break OuterLoop
				}
			case <-queueDone:
				//Well, that's as good as it's going to get.
				break OuterLoop
			}
		}

		//In some cases that previous select would have something incoming on
		//incomingsolutions, as well as on queueDone, and queueDone would have
		//just so happened to have won. Check for one last remaining item
		//coming in from incomingSolutions. Technically it's possible (how?)
		//to have multiple items waiting on incomingSolutions, so read as many
		//as we can get without blocking.(Not checking for this was the reason
		//for bug #134.)

	DoneReading:
		for {
			select {
			case solution := <-incomingSolutions:
				if solution != nil {
					solutions = append(solutions, solution)
				}
			default:
				//Nope, guess there wasn't one left.
				break DoneReading
			}
		}

		//There might be some things waiting to go into incomingSolutions here, but because it has a slot
		//for every thread to be buffered, it's OK, we can just stop now.

		//TODO: the grids waiting in the queue will never have their .Done called. This isn't a big deal--GC should reclaim them--
		//but we won't have as many that we could reuse.
		queue.Exit <- true

		self.cachedSolutionsLockRef.Lock()
		self.cachedSolutionsRef = solutions
		self.cachedSolutionsRequestedLengthRef = max
		self.cachedSolutionsLockRef.Unlock()

	}

	//TODO: rejigger this to not need a write lock then a read lock when setting.
	self.cachedSolutionsLockRef.RLock()
	result := self.cachedSolutionsRef
	self.cachedSolutionsLockRef.RUnlock()
	return result

}

func (self *gridImpl) searchSolutions(queue *syncedFiniteQueue, isFirstRun bool, numSoughtSolutions int) Grid {
	//TODO: implement this!
	return nil
}

func (self *mutableGridImpl) searchSolutions(queue *syncedFiniteQueue, isFirstRun bool, numSoughtSolutions int) Grid {
	//This will only be called by Solutions.
	//We will return ourselves if we are a solution, and if not we will return nil.
	//If there are any sub children, we will send them to counter before we're done.

	if self.Invalid() {
		return nil
	}

	self.fillSimpleCells()
	//Have any cells noticed they were invalid while solving forward?
	if self.cellsInvalid() {
		return nil
	}

	if self.Solved() {
		return self
	}

	//Well, looks like we're going to have to branch.
	rankedObject := self.queue().NewGetter().Get()
	if rankedObject == nil {
		panic("Queue didn't have any cells.")
	}

	cell, ok := rankedObject.(Cell)
	if !ok {
		panic("We got back a non-cell from the grid's queue")
	}

	unshuffledPossibilities := cell.Possibilities()

	possibilities := make([]int, len(unshuffledPossibilities))

	for i, j := range rand.Perm(len(unshuffledPossibilities)) {
		possibilities[i] = unshuffledPossibilities[j]
	}

	var result Grid

	for i, num := range possibilities {
		//TODO: this seems like a natural place to use CopyWithModifications,
		//but gridImpl.fillSimpleCells will be called on it.
		copy := self.MutableCopy()
		cell.MutableInGrid(copy).SetNumber(num)
		//As an optimization for cases where there are many solutions, we'll just continue a DFS until we barf then unroll back up.
		//It doesn't appear to slow things down in the general case
		if i == 0 && !isFirstRun {
			result = copy.searchSolutions(queue, false, numSoughtSolutions)
			if result != nil && numSoughtSolutions == 1 {
				//No need to spin off other branches, just return up.
				return result
			}
		} else {
			queue.In <- copy
		}
	}

	return result

}

//Fills in all of the cells it can without branching or doing any advanced
//techniques that require anything more than a single cell's possibles list.
func (self *mutableGridImpl) fillSimpleCells() int {
	count := 0
	getter := self.queue().NewGetter()
	obj := getter.GetSmallerThan(2)
	for obj != nil && !self.cellsInvalid() {
		cell, ok := obj.(MutableCell)
		if !ok {
			continue
		}
		cell.SetNumber(cell.implicitNumber())
		count++
		obj = getter.GetSmallerThan(2)
	}
	return count
}
<commit_msg>Added a note on what to do with fillSimpleCells.<commit_after>package sudoku

import (
	"math/rand"
	"sync"
)

func (self *mutableGridImpl) Solve() bool {

	//Special case; check if it's already solved.
	//TODO: removing this causes Solve, when called on an already solved grid, to sometimes fail. Why is that?
	if self.Solved() {
		return true
	}

	//TODO: Optimization: we only need one, so we can bail as soon as we find a single one.
	solutions := self.Solutions()
	if len(solutions) == 0 {
		return false
	}
	self.LoadSDK(solutions[0].DataString())
	return true
}

func (self *gridImpl) NumSolutions() int {
	//TODO: implement this!
	return 0
}

func (self *mutableGridImpl) NumSolutions() int {
	return len(self.Solutions())
}

func (self *gridImpl) HasSolution() bool {
	//TODO: implement this!
	return false
}

func (self *mutableGridImpl) HasSolution() bool {
	//TODO: optimize this to bail as soon as we find a single solution.
	return len(self.nOrFewerSolutions(1)) > 0
}

func (self *gridImpl) HasMultipleSolutions() bool {
	//TODO: implement this!
	return false
}

func (self *mutableGridImpl) HasMultipleSolutions() bool {
	return len(self.nOrFewerSolutions(2)) >= 2
}

func (self *gridImpl) Solutions() []Grid {
	//TODO: implement this
	return nil
}

func (self *mutableGridImpl) Solutions() (solutions []Grid) {
	return self.nOrFewerSolutions(0)
}

//The actual workhorse of solutions generating. 0 means "as many as you can
//find". It might return more than you asked for, if it already had more
//results than requested sitting around.
func (self *mutableGridImpl) nOrFewerSolutions(max int) []Grid {

	self.cachedSolutionsLockRef.RLock()
	hasNoCachedSolutions := self.cachedSolutionsRef == nil
	cachedSolutionsLen := self.cachedSolutionsRequestedLengthRef
	self.cachedSolutionsLockRef.RUnlock()

	if hasNoCachedSolutions || (max == 0 && cachedSolutionsLen != 0) || (max > 0 && cachedSolutionsLen < max && cachedSolutionsLen > 0) {

		queueDone := make(chan bool, 1)

		queue := newSyncedFiniteQueue(0, DIM*DIM, queueDone)

		queue.In <- self.Copy()

		//In the past this wasn't buffered, but then when we finished early other items would try to go into it
		//and block, which prevented them from looping back up and getting the signal to shut down.
		//Since there's only a known number of threads, we'll make sure they all ahve a place to leave their work
		//without blocking so they can get the signal to shut down.
		incomingSolutions := make(chan Grid, _NUM_SOLVER_THREADS)

		//Using a pattern for closing fan in style receivers from http://blog.golang.org/pipelines
		var wg sync.WaitGroup

		var solutions []Grid

		//TODO: figure out a way to kill all of these threads when necessary.

		//TODO: don't use a constant here, use someting around the lines of
		//numCPU

		for i := 0; i < _NUM_SOLVER_THREADS; i++ {
			go func() {
				//Sovler thread.
				firstRun := true
				defer wg.Done()
				for {
					grid, ok := <-queue.Out
					if !ok {
						return
					}
					result := grid.(Grid).searchSolutions(queue, firstRun, max)
					if result != nil {
						incomingSolutions <- result
					}
					queue.ItemDone <- true
					firstRun = false
				}
			}()
		}

		wg.Add(_NUM_SOLVER_THREADS)

		go func() {
			wg.Wait()
			close(incomingSolutions)
		}()

	OuterLoop:
		for {
			select {
			case solution := <-incomingSolutions:
				//incomingSolutions must have been closed because no more work to do.
				if solution == nil {
					break OuterLoop
				}
				//Add it to results
				solutions = append(solutions, solution)
				if max > 0 && len(solutions) >= max {
					break OuterLoop
				}
			case <-queueDone:
				//Well, that's as good as it's going to get.
				break OuterLoop
			}
		}

		//In some cases that previous select would have something incoming on
		//incomingsolutions, as well as on queueDone, and queueDone would have
		//just so happened to have won. Check for one last remaining item
		//coming in from incomingSolutions. Technically it's possible (how?)
		//to have multiple items waiting on incomingSolutions, so read as many
		//as we can get without blocking.(Not checking for this was the reason
		//for bug #134.)

	DoneReading:
		for {
			select {
			case solution := <-incomingSolutions:
				if solution != nil {
					solutions = append(solutions, solution)
				}
			default:
				//Nope, guess there wasn't one left.
				break DoneReading
			}
		}

		//There might be some things waiting to go into incomingSolutions here, but because it has a slot
		//for every thread to be buffered, it's OK, we can just stop now.

		//TODO: the grids waiting in the queue will never have their .Done called. This isn't a big deal--GC should reclaim them--
		//but we won't have as many that we could reuse.
		queue.Exit <- true

		self.cachedSolutionsLockRef.Lock()
		self.cachedSolutionsRef = solutions
		self.cachedSolutionsRequestedLengthRef = max
		self.cachedSolutionsLockRef.Unlock()

	}

	//TODO: rejigger this to not need a write lock then a read lock when setting.
	self.cachedSolutionsLockRef.RLock()
	result := self.cachedSolutionsRef
	self.cachedSolutionsLockRef.RUnlock()
	return result

}

func (self *gridImpl) searchSolutions(queue *syncedFiniteQueue, isFirstRun bool, numSoughtSolutions int) Grid {
	//TODO: implement this!
	return nil
}

func (self *mutableGridImpl) searchSolutions(queue *syncedFiniteQueue, isFirstRun bool, numSoughtSolutions int) Grid {
	//This will only be called by Solutions.
	//We will return ourselves if we are a solution, and if not we will return nil.
	//If there are any sub children, we will send them to counter before we're done.

	if self.Invalid() {
		return nil
	}

	self.fillSimpleCells()
	//Have any cells noticed they were invalid while solving forward?
	if self.cellsInvalid() {
		return nil
	}

	if self.Solved() {
		return self
	}

	//Well, looks like we're going to have to branch.
	rankedObject := self.queue().NewGetter().Get()
	if rankedObject == nil {
		panic("Queue didn't have any cells.")
	}

	cell, ok := rankedObject.(Cell)
	if !ok {
		panic("We got back a non-cell from the grid's queue")
	}

	unshuffledPossibilities := cell.Possibilities()

	possibilities := make([]int, len(unshuffledPossibilities))

	for i, j := range rand.Perm(len(unshuffledPossibilities)) {
		possibilities[i] = unshuffledPossibilities[j]
	}

	var result Grid

	for i, num := range possibilities {
		//TODO: this seems like a natural place to use CopyWithModifications,
		//but gridImpl.fillSimpleCells will be called on it.
		copy := self.MutableCopy()
		cell.MutableInGrid(copy).SetNumber(num)
		//As an optimization for cases where there are many solutions, we'll just continue a DFS until we barf then unroll back up.
		//It doesn't appear to slow things down in the general case
		if i == 0 && !isFirstRun {
			result = copy.searchSolutions(queue, false, numSoughtSolutions)
			if result != nil && numSoughtSolutions == 1 {
				//No need to spin off other branches, just return up.
				return result
			}
		} else {
			queue.In <- copy
		}
	}

	return result

}

//Fills in all of the cells it can without branching or doing any advanced
//techniques that require anything more than a single cell's possibles list.
func (self *mutableGridImpl) fillSimpleCells() int {

	//TODO: make it so fillSimpleCells returns a Grid that has simplecells
	//filled. To make it work on readOnlyGrids, we can batch up all of the
	//items at 1 and apply in one batch, then do it again, repeating until
	//none are left.

	count := 0
	getter := self.queue().NewGetter()
	obj := getter.GetSmallerThan(2)
	for obj != nil && !self.cellsInvalid() {
		cell, ok := obj.(MutableCell)
		if !ok {
			continue
		}
		cell.SetNumber(cell.implicitNumber())
		count++
		obj = getter.GetSmallerThan(2)
	}
	return count
}
<|endoftext|>


<commit_before>package fakku

import (
	"testing"
)

const (
	// Test the manga listed in the API docs
	ContentTestingCategory     = "manga"
	ContentTestingName         = "right-now-while-cleaning-the-pool"
	ContentTestingUrl          = "http://www.fakku.net/manga/right-now-while-cleaning-the-pool"
	ContentTestingTag          = "Vanilla"
	ContentTestingDisplayName  = "Right now, while cleaning the pool"
	ContentTestingPoster       = "Jacob"
	ContentTestingRelatedTotal = 11236
	ContentTestingRelatedPages = 1124
)

func TestContentGetSimple_1(t *testing.T) {
	c, err := GetContentInformation(ContentTestingCategory, ContentTestingName)
	if err != nil {
		t.Error(err)
	}
	if c.Name != ContentTestingDisplayName {
		t.Errorf("Expected: %s, Got: %s", ContentTestingDisplayName, c.Name)
	}
	if c.Url != ContentTestingUrl {
		t.Errorf("Expected: %s, Got: %s", ContentTestingUrl, c.Url)
	}
	foundTag := false
	for i := 0; i < len(c.Tags); i++ {
		if c.Tags[i].Attribute == ContentTestingTag {
			foundTag = true
			break
		}
	}
	if !foundTag {
		t.Errorf("Didn't find tag %s but should have!", ContentTestingTag)
	}
}

func TestContentGetComments_1(t *testing.T) {
	/* tests to make sure that the functions work */
	_, err := GetContentComments(ContentTestingCategory, ContentTestingName)
	if err != nil {
		t.Error(err)
	}
}

func TestContentGetReadOnline_1(t *testing.T) {
	onlineContent, err := GetContentReadOnline(ContentTestingCategory, ContentTestingName)
	if err != nil {
		t.Error(err)
	}

	if len(onlineContent.Pages) != 25 {
		t.Errorf("Expected 25 pages, got %d pages", len(onlineContent.Pages))
	}
}

func TestGetContentDownloads_1(t *testing.T) {
	downloads, err := GetContentDownloads(ContentTestingCategory, ContentTestingName)
	if err != nil {
		t.Error(err)
	}
	if downloads.Total != 1 {
		t.Errorf("Expected 1 page, got %d pages", downloads.Total)
	} else {
		if len(downloads.Downloads) != 1 {
			t.Errorf("Expected 1 download, got %d downloads", len(downloads.Downloads))
		} else {
			if downloads.Downloads[0].Poster != ContentTestingPoster {
				t.Errorf("Expected download poster: %s, got %s", ContentTestingPoster, downloads.Downloads[0].Poster)
			}
		}
	}
}

func TestGetContentRelated_1(t *testing.T) {
	related, err := GetRelatedContentAll(ContentTestingCategory, ContentTestingName)
	if err != nil {
		t.Error(err)
	}
	// Since this can change, we need to do at least checks
	if related.Total < ContentTestingRelatedTotal {
		t.Errorf("Was unable to find an adequte enough of related content! Expected at least: %d, Got: %d", ContentTestingRelatedTotal, related.Total)
	}

	if related.Pages < ContentTestingRelatedPages {
		t.Errorf("Was unable to find an adequte enough of related content pages! Expected at least: %d, Got: %d", ContentTestingRelatedPages, related.Total)
	}
}
<commit_msg>Fixed the content_test.go test for GetRelatedContent<commit_after>package fakku

import (
	"testing"
)

const (
	// Test the manga listed in the API docs
	ContentTestingCategory           = "manga"
	ContentTestingName               = "right-now-while-cleaning-the-pool"
	ContentTestingUrl                = "http://www.fakku.net/manga/right-now-while-cleaning-the-pool"
	ContentTestingTag                = "Vanilla"
	ContentTestingDisplayName        = "Right now, while cleaning the pool"
	ContentTestingDisplayNameRelated = "Before the Pool Opens"
	ContentTestingPoster             = "Jacob"
	ContentTestingRelatedTotal       = 11236
	ContentTestingRelatedPages       = 1124
)

func TestContentGetSimple_1(t *testing.T) {
	c, err := GetContentInformation(ContentTestingCategory, ContentTestingName)
	if err != nil {
		t.Error(err)
	}
	if c.Name != ContentTestingDisplayName {
		t.Errorf("Expected: %s, Got: %s", ContentTestingDisplayName, c.Name)
	}
	if c.Url != ContentTestingUrl {
		t.Errorf("Expected: %s, Got: %s", ContentTestingUrl, c.Url)
	}
	foundTag := false
	for i := 0; i < len(c.Tags); i++ {
		if c.Tags[i].Attribute == ContentTestingTag {
			foundTag = true
			break
		}
	}
	if !foundTag {
		t.Errorf("Didn't find tag %s but should have!", ContentTestingTag)
	}
}

func TestContentGetComments_1(t *testing.T) {
	/* tests to make sure that the functions work */
	_, err := GetContentComments(ContentTestingCategory, ContentTestingName)
	if err != nil {
		t.Error(err)
	}
}

func TestContentGetReadOnline_1(t *testing.T) {
	onlineContent, err := GetContentReadOnline(ContentTestingCategory, ContentTestingName)
	if err != nil {
		t.Error(err)
	}

	if len(onlineContent.Pages) != 25 {
		t.Errorf("Expected 25 pages, got %d pages", len(onlineContent.Pages))
	}
}

func TestGetContentDownloads_1(t *testing.T) {
	downloads, err := GetContentDownloads(ContentTestingCategory, ContentTestingName)
	if err != nil {
		t.Error(err)
	}
	if downloads.Total != 1 {
		t.Errorf("Expected 1 page, got %d pages", downloads.Total)
	} else {
		if len(downloads.Downloads) != 1 {
			t.Errorf("Expected 1 download, got %d downloads", len(downloads.Downloads))
		} else {
			if downloads.Downloads[0].Poster != ContentTestingPoster {
				t.Errorf("Expected download poster: %s, got %s", ContentTestingPoster, downloads.Downloads[0].Poster)
			}
		}
	}
}

func TestGetContentRelated_1(t *testing.T) {
	related, err := GetRelatedContentAll(ContentTestingCategory, ContentTestingName)
	if err != nil {
		t.Error(err)
	}
	if related.Related[0].Name != ContentTestingDisplayNameRelated {
		t.Errorf("First entry in related content was %s not %s as expected!", related.Related[0].Name, ContentTestingDisplayNameRelated)
	}
	// Since this can change, we need to do at least checks
	if related.Total < ContentTestingRelatedTotal {
		t.Errorf("Was unable to find an adequte enough of related content! Expected at least: %d, Got: %d", ContentTestingRelatedTotal, related.Total)
	}

	if related.Pages < ContentTestingRelatedPages {
		t.Errorf("Was unable to find an adequte enough of related content pages! Expected at least: %d, Got: %d", ContentTestingRelatedPages, related.Total)
	}
}
<|endoftext|>


<commit_before>package gb

import (
	"strings"
	"testing"
	"time"
)

func testImportCycle(pkg string, t *testing.T) {
	ctx := testContext(t)

	// one of two goroutines should return to finish the test
	done := make(chan bool, 2)

	timeConstraint := time.AfterFunc(1*time.Second, func() {
		t.Error("ctx.ResolvePackage have not finished in 1s")
		done <- true
	})

	go func() {
		_, err := ctx.ResolvePackage(pkg)
		timeConstraint.Stop()

		if strings.Index(err.Error(), "cycle detected") == -1 {
			t.Errorf("ctx.ResolvePackage returned wrong error. Expected cycle detection, got: %v", err)
		}

		if err == nil {
			t.Errorf("ctx.ResolvePackage should have returned an error for cycle, returned nil")
		}
		done <- true
	}()

	<-done
}

func TestOneElementCycleDetection(t *testing.T) {
	testImportCycle("cycle0", t)
}

func TestSimpleCycleDetection(t *testing.T) {
	testImportCycle("cycle1/a", t)
}

func TestLongCycleDetection(t *testing.T) {
	testImportCycle("cycle2/a", t)
}
<commit_msg>move timed tests to limiting call stack size<commit_after>package gb

import (
	"runtime/debug"
	"strings"
	"testing"
)

func testImportCycle(pkg string, t *testing.T) {
	ctx := testContext(t)

	debug.SetMaxStack(1 << 18)

	_, err := ctx.ResolvePackage(pkg)
	if strings.Index(err.Error(), "cycle detected") == -1 {
		t.Errorf("ctx.ResolvePackage returned wrong error. Expected cycle detection, got: %v", err)
	}

	if err == nil {
		t.Errorf("ctx.ResolvePackage should have returned an error for cycle, returned nil")
	}
}

func TestOneElementCycleDetection(t *testing.T) {
	testImportCycle("cycle0", t)
}

func TestSimpleCycleDetection(t *testing.T) {
	testImportCycle("cycle1/a", t)
}

func TestLongCycleDetection(t *testing.T) {
	testImportCycle("cycle2/a", t)
}
<|endoftext|>


<commit_before>// go:generate statik -src=./static

package main

import (
	"html/template"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"time"

	"github.com/alecthomas/kingpin"
	"github.com/gorilla/handlers"
	"github.com/jakdept/dandler"
	_ "github.com/jakdept/sp9k1/statik"
	"github.com/rakyll/statik/fs"
)

// now would this be shitposting if there were _tests_?

var serverBanner = `
'______________________________________________________________________________
/                                                                              \
|                 '.'                           .-:::::::::::::::::::::-'      |
|                 -///-'                      '/+++++++++++++++++++++++++-     |
|                ':+++++/-                    -++//////////////////////+++     |
|                /++++++//:.                  -+/----------------------:++     |
|               '/++++///:::.                 -+/---:dddddddds+ymmms----++     |
|             .://+////:::::-                 -+/...:mmmmmdyoymNNNNy...-++     |
|           '://////::::::::::-.              -++::-:mmmdyoymNNNNNNy--:/++     |
|           ://:::::::::::/+++//:'            -+++++ommmhoymNNNNNNNh++++++     |
|           :/syys+:::///++syhyo:.            -+++++ommmmdsodNNNNNNh++++++     |
|         '.+mdo+hmh++++/ommo+yNh-''          -+++++ommmmmmhosmNNNNh++++++     |
|       .//+mN.'''hNs///:mN:'''sNy//-         -+++++ommmmmmmdyohNNNh++++++     |
|      '///+NN.'''hNs::::mM-'''sMy/::-        -++++++dmmmmmmmmdssdNh++++++     |
|      '::::yNd+/yMd::://sNmo/sMm/:::-        -++++++oyyyyyyyyyyo+s+++++++     |
|      '-::::+hmmds//++++/+ydmds/::::-'       -+++++++++++++++++++++++++++     |
|   '-///++++++++++++++++++///:::::////:.     '/+++++++++++++++++++++++++-     |
|  .///////////+hmmmmmmmmmmmh+::://+///::-      .:::::::+++++++++/:::::-'      |
|  ::////////::::+shmmNmmhs+:://++//::::::.             +++++++++:             |
|  -:::::::::::::::::::::///+++///:::::::-'             +++++++++:             |
|   .-:::::::::::::::////+////:::::::::-.'              +++++++++:             |
|      '''''''    ''''''''''''''''''''                  .........'             |
\______________________________________________________________________________/
`

var (
	listenAddress = kingpin.Flag("listen", "addresses to listen for incoming non-TLS connections").
			Short('l').Default("127.0.0.1:8080").TCP()

	hostname = kingpin.Flag("hostname", "hostname to register").Short('h').String()

	imgDir = kingpin.Flag("images", "directory of images to serve").
		Short('i').Default("./").ExistingDir()

	cacheMinDays = kingpin.Flag("cacheMin", "minimum days to cache images in browser").
			Default("30").Int()

	cacheVariation = kingpin.Flag("cacheVariation", "difference between minimum and maximum length to cache images").
			Default("7").Int()

	thumbWidth  = kingpin.Flag("width", "maximum thumbnail width").Default("310").Int()
	thumbHeight = kingpin.Flag("height", "thumbnail height").Default("200").Int()

	staticDir = kingpin.Flag("static", "alternate static directory to serve").Short('s').ExistingDir()

	templateFile = kingpin.Flag("template", "alternate index template to serve").Short('t').ExistingFile()

	canonicalURL        = kingpin.Flag("canonicalURL", "default redirect to serve").Default("localhost:80").String()
	canonicalDisableTLS = kingpin.Flag("canonicalDisableTLS", "force unencrypted protocol").Default("false").Bool()
	canonicalForceTLS   = kingpin.Flag("canonicalForceTLS", "force encrypted protocol").Default("true").Bool()
	canonicalForceHost  = kingpin.Flag("canonicalForceHost", "force a specific hostname").Default("true").Bool()
	canonicalForcePort  = kingpin.Flag("canonicalForcePort", "force a specific port").Default("false").Bool()
)

func parseTemplate(logger *log.Logger, fs http.FileSystem) *template.Template {
	if *templateFile != "" {
		// if an alternate template was provided, i can use that instead
		return template.Must(template.ParseFiles(*templateFile))
	}
	// have to do it the hard way because it comes from fs
	templFile, err := fs.Open("/page.template")
	if err != nil {
		logger.Fatal(err)
	}
	templData, err := ioutil.ReadAll(templFile)
	if err != nil {
		logger.Fatal(err)
	}
	return template.Must(template.New("page.template").Parse(string(templData)))
}

func createStaticFS(logger *log.Logger, path string) http.FileSystem {
	if path != "" {
		return http.Dir(path)
	}
	filesystem, err := fs.New()
	if err != nil {
		logger.Fatal(err)
	}
	return filesystem
}

func buildMuxer(logger *log.Logger,
	fs http.FileSystem,
	templ *template.Template,
	done chan struct{},
) http.Handler {

	day := time.Hour * time.Duration(64)
	var h http.Handler
	mux := http.NewServeMux()

	// building the static handler
	h = http.FileServer(fs)
	// split the main folder off into a redirect
	h = dandler.Split(http.RedirectHandler("/", 302), h)
	// add a prefix before the handler
	h = http.StripPrefix("/static/", h)
	// add some expiration
	h = dandler.ExpiresRange(day*time.Duration(*cacheMinDays),
		day*time.Duration(*cacheVariation), h)
	// add the static handler to the muxer
	mux.Handle("/static/", h)

	// create a caching handler
	h = dandler.ThumbCache(logger, *thumbWidth, *thumbHeight, 32<<20, *imgDir, "thumbs", "jpg")
	// split the folder itself into a redirect
	h = dandler.Split(http.RedirectHandler("/", 302), h)
	// strip the prefix
	h = http.StripPrefix("/thumb/", h)
	// add some expiration
	h = dandler.ExpiresRange(day*time.Duration(*cacheMinDays),
		day*time.Duration(*cacheVariation), h)
	// add the thumbnail handler to the muxer
	mux.Handle("/thumb/", h)

	h = dandler.DirSplit(logger, *imgDir, done,
		dandler.Index(logger, *imgDir, done, templ),
		dandler.ContentType(logger, *imgDir),
	)
	mux.Handle("/", h)

	h = dandler.ASCIIHeader("shit\nposting\n9001", serverBanner, " ", mux)
	h = handlers.CombinedLoggingHandler(os.Stdout, h)

	// add canonical header if required
	if *canonicalForceHost ||
		*canonicalForcePort ||
		*canonicalForceTLS ||
		*canonicalDisableTLS {
		options := 0
		if *canonicalForceHost {
			options += dandler.ForceHost
		}
		if *canonicalForcePort {
			options += dandler.ForcePort
		}
		if *canonicalForceTLS {
			options += dandler.ForceHTTPS
		} else if *canonicalDisableTLS {
			options += dandler.ForceHTTP
		}

		h = dandler.CanonicalHost(*canonicalURL, options, h)
	}

	// compress responses
	h = handlers.CompressHandler(h)

	return h
}

func main() {

	kingpin.Parse()

	logger := log.New(os.Stderr, "", log.Ldate|log.Ltime|log.Llongfile)

	fs := createStaticFS(logger, *staticDir)

	templ := parseTemplate(logger, fs)

	done := make(chan struct{})
	defer close(done)

	srvHandlers := buildMuxer(logger, fs, templ, done)

	logger.Fatal(http.ListenAndServe((*listenAddress).String(), srvHandlers))
}
<commit_msg>changed the library package reference<commit_after>// go:generate statik -src=./static

package main

import (
	"html/template"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"time"

	"github.com/gorilla/handlers"
	"github.com/jakdept/dandler"
	_ "github.com/jakdept/sp9k1/statik"
	"github.com/rakyll/statik/fs"
	kingpin "gopkg.in/alecthomas/kingpin.v2"
)

// now would this be shitposting if there were _tests_?

var serverBanner = `
'______________________________________________________________________________
/                                                                              \
|                 '.'                           .-:::::::::::::::::::::-'      |
|                 -///-'                      '/+++++++++++++++++++++++++-     |
|                ':+++++/-                    -++//////////////////////+++     |
|                /++++++//:.                  -+/----------------------:++     |
|               '/++++///:::.                 -+/---:dddddddds+ymmms----++     |
|             .://+////:::::-                 -+/...:mmmmmdyoymNNNNy...-++     |
|           '://////::::::::::-.              -++::-:mmmdyoymNNNNNNy--:/++     |
|           ://:::::::::::/+++//:'            -+++++ommmhoymNNNNNNNh++++++     |
|           :/syys+:::///++syhyo:.            -+++++ommmmdsodNNNNNNh++++++     |
|         '.+mdo+hmh++++/ommo+yNh-''          -+++++ommmmmmhosmNNNNh++++++     |
|       .//+mN.'''hNs///:mN:'''sNy//-         -+++++ommmmmmmdyohNNNh++++++     |
|      '///+NN.'''hNs::::mM-'''sMy/::-        -++++++dmmmmmmmmdssdNh++++++     |
|      '::::yNd+/yMd::://sNmo/sMm/:::-        -++++++oyyyyyyyyyyo+s+++++++     |
|      '-::::+hmmds//++++/+ydmds/::::-'       -+++++++++++++++++++++++++++     |
|   '-///++++++++++++++++++///:::::////:.     '/+++++++++++++++++++++++++-     |
|  .///////////+hmmmmmmmmmmmh+::://+///::-      .:::::::+++++++++/:::::-'      |
|  ::////////::::+shmmNmmhs+:://++//::::::.             +++++++++:             |
|  -:::::::::::::::::::::///+++///:::::::-'             +++++++++:             |
|   .-:::::::::::::::////+////:::::::::-.'              +++++++++:             |
|      '''''''    ''''''''''''''''''''                  .........'             |
\______________________________________________________________________________/
`

var (
	listenAddress = kingpin.Flag("listen", "addresses to listen for incoming non-TLS connections").
			Short('l').Default("127.0.0.1:8080").TCP()

	hostname = kingpin.Flag("hostname", "hostname to register").Short('h').String()

	imgDir = kingpin.Flag("images", "directory of images to serve").
		Short('i').Default("./").ExistingDir()

	cacheMinDays = kingpin.Flag("cacheMin", "minimum days to cache images in browser").
			Default("30").Int()

	cacheVariation = kingpin.Flag("cacheVariation", "difference between minimum and maximum length to cache images").
			Default("7").Int()

	thumbWidth  = kingpin.Flag("width", "maximum thumbnail width").Default("310").Int()
	thumbHeight = kingpin.Flag("height", "thumbnail height").Default("200").Int()

	staticDir = kingpin.Flag("static", "alternate static directory to serve").Short('s').ExistingDir()

	templateFile = kingpin.Flag("template", "alternate index template to serve").Short('t').ExistingFile()

	canonicalURL        = kingpin.Flag("canonicalURL", "default redirect to serve").Default("localhost:80").String()
	canonicalDisableTLS = kingpin.Flag("canonicalDisableTLS", "force unencrypted protocol").Default("false").Bool()
	canonicalForceTLS   = kingpin.Flag("canonicalForceTLS", "force encrypted protocol").Default("true").Bool()
	canonicalForceHost  = kingpin.Flag("canonicalForceHost", "force a specific hostname").Default("true").Bool()
	canonicalForcePort  = kingpin.Flag("canonicalForcePort", "force a specific port").Default("false").Bool()
)

func parseTemplate(logger *log.Logger, fs http.FileSystem) *template.Template {
	if *templateFile != "" {
		// if an alternate template was provided, i can use that instead
		return template.Must(template.ParseFiles(*templateFile))
	}
	// have to do it the hard way because it comes from fs
	templFile, err := fs.Open("/page.template")
	if err != nil {
		logger.Fatal(err)
	}
	templData, err := ioutil.ReadAll(templFile)
	if err != nil {
		logger.Fatal(err)
	}
	return template.Must(template.New("page.template").Parse(string(templData)))
}

func createStaticFS(logger *log.Logger, path string) http.FileSystem {
	if path != "" {
		return http.Dir(path)
	}
	filesystem, err := fs.New()
	if err != nil {
		logger.Fatal(err)
	}
	return filesystem
}

func buildMuxer(logger *log.Logger,
	fs http.FileSystem,
	templ *template.Template,
	done chan struct{},
) http.Handler {

	day := time.Hour * time.Duration(64)
	var h http.Handler
	mux := http.NewServeMux()

	// building the static handler
	h = http.FileServer(fs)
	// split the main folder off into a redirect
	h = dandler.Split(http.RedirectHandler("/", 302), h)
	// add a prefix before the handler
	h = http.StripPrefix("/static/", h)
	// add some expiration
	h = dandler.ExpiresRange(day*time.Duration(*cacheMinDays),
		day*time.Duration(*cacheVariation), h)
	// add the static handler to the muxer
	mux.Handle("/static/", h)

	// create a caching handler
	h = dandler.ThumbCache(logger, *thumbWidth, *thumbHeight, 32<<20, *imgDir, "thumbs", "jpg")
	// split the folder itself into a redirect
	h = dandler.Split(http.RedirectHandler("/", 302), h)
	// strip the prefix
	h = http.StripPrefix("/thumb/", h)
	// add some expiration
	h = dandler.ExpiresRange(day*time.Duration(*cacheMinDays),
		day*time.Duration(*cacheVariation), h)
	// add the thumbnail handler to the muxer
	mux.Handle("/thumb/", h)

	h = dandler.DirSplit(logger, *imgDir, done,
		dandler.Index(logger, *imgDir, done, templ),
		dandler.ContentType(logger, *imgDir),
	)
	mux.Handle("/", h)

	h = dandler.ASCIIHeader("shit\nposting\n9001", serverBanner, " ", mux)
	h = handlers.CombinedLoggingHandler(os.Stdout, h)

	// add canonical header if required
	if *canonicalForceHost ||
		*canonicalForcePort ||
		*canonicalForceTLS ||
		*canonicalDisableTLS {
		options := 0
		if *canonicalForceHost {
			options += dandler.ForceHost
		}
		if *canonicalForcePort {
			options += dandler.ForcePort
		}
		if *canonicalForceTLS {
			options += dandler.ForceHTTPS
		} else if *canonicalDisableTLS {
			options += dandler.ForceHTTP
		}

		h = dandler.CanonicalHost(*canonicalURL, options, h)
	}

	// compress responses
	h = handlers.CompressHandler(h)

	return h
}

func main() {

	kingpin.Parse()

	logger := log.New(os.Stderr, "", log.Ldate|log.Ltime|log.Llongfile)

	fs := createStaticFS(logger, *staticDir)

	templ := parseTemplate(logger, fs)

	done := make(chan struct{})
	defer close(done)

	srvHandlers := buildMuxer(logger, fs, templ, done)

	logger.Fatal(http.ListenAndServe((*listenAddress).String(), srvHandlers))
}
<|endoftext|>


<commit_before>package mixpanel

import (
	"fmt"
	"strings"
	"testing"
)

func TestAddSignature(t *testing.T) {
	// TODO: write me
}

func TestMakeArgs(t *testing.T) {
	// TODO: write me
}

func TestTransformEventData(t *testing.T) {
	mix := New("product", "", "")
	input := strings.NewReader(`
{"event": "a0", "properties": {"a": null, "b": "b0", "c": true, "d": ["foo"]}}
{"event": "a1", "properties": {"a": null, "b": "b1", "c": true, "d": ["foo"]}}
{"event": "a2", "properties": {"a": null, "b": "b2", "c": true, "d": ["foo"]}}`)

	output := make(chan EventData)

	go mix.TransformEventData(input, output)

	for i := 0; i < 3; i++ {
		event := <-output

		expected := []struct {
			Name  string
			Value interface{}
		}{
			{"event", fmt.Sprintf("a%d", i)},
			{"a", nil},
			{"b", fmt.Sprintf("b%d", i)},
			{"c", true},
		}

		for _, e := range expected {
			if v, ok := event[e.Name]; !ok || v != e.Value {
				t.Errorf("bad value: expected %s=(%v) got %s=(%v)", e.Name, e.Value,
					e.Name, v)
			}
		}

	}

	if _, ok := <-output; ok {
		t.Error("expected output chan to be closed here")
	}
}

func TestExportDate(t *testing.T) {
	// TODO: write me
}

func BenchmarkTransformEventData(b *testing.B) {
	mix := New("product", "", "")
	input := strings.NewReader(
		strings.Repeat(`{"event": "a2", "properties": {"a": null, "b": "b2", "c": true, "d": ["foo"]}}`, b.N))
	output := make(chan EventData)

	b.ResetTimer()

	go mix.TransformEventData(input, output)
	for _ = range output {
	}
}
<commit_msg>Fix mixpanel_test.<commit_after>package mixpanel

import (
	"fmt"
	"strings"
	"testing"
)

func TestAddSignature(t *testing.T) {
	// TODO: write me
}

func TestMakeArgs(t *testing.T) {
	// TODO: write me
}

func TestTransformEventData(t *testing.T) {
	mix := New("product", "", "")
	input := strings.NewReader(`
{"event": "a0", "properties": {"a": null, "b": "b0", "c": true, "d": ["foo"]}}
{"event": "a1", "properties": {"a": null, "b": "b1", "c": true, "d": ["foo"]}}
{"event": "a2", "properties": {"a": null, "b": "b2", "c": true, "d": ["foo"]}}`)

	output := make(chan EventData)

	go mix.TransformEventData(input, output)

	for i := 0; i < 3; i++ {
		event := <-output

		expected := []struct {
			Name  string
			Value interface{}
		}{
			{"event", fmt.Sprintf("a%d", i)},
			{"a", nil},
			{"b", fmt.Sprintf("b%d", i)},
			{"c", true},
		}

		for _, e := range expected {
			if v, ok := event[e.Name]; !ok || v != e.Value {
				t.Errorf("bad value: expected %s=(%v) got %s=(%v)", e.Name, e.Value,
					e.Name, v)
			}
		}
	}
	close(output)
}

func TestExportDate(t *testing.T) {
	// TODO: write me
}

func BenchmarkTransformEventData(b *testing.B) {
	mix := New("product", "", "")
	input := strings.NewReader(
		strings.Repeat(`{"event": "a2", "properties": {"a": null, "b": "b2", "c": true, "d": ["foo"]}}`, b.N))
	output := make(chan EventData)

	b.ResetTimer()

	go mix.TransformEventData(input, output)
	for i := 0; i < b.N; i++ {
		<-output
	}
	close(output)
}
<|endoftext|>


<commit_before>package mnemosyned

import (
	"net"
	"os"
	"strconv"
	"sync"
	"testing"
	"time"

	"github.com/piotrkowalczuk/mnemosyne"
	"github.com/piotrkowalczuk/sklog"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
)

func TestDaemon_Run(t *testing.T) {
	if testing.Short() {
		t.Skip("this test takes to long to run it in short mode")
	}

	ttl := 5 * time.Second
	ttc := 1 * time.Second
	nb := 10

	l, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	d := NewDaemon(&DaemonOpts{
		Environment: EnvironmentTest,
		SessionTTL:  ttl,
		SessionTTC:  ttc,
		RPCListener: l,
		// Use this logger to debug issues
		Logger: sklog.NewHumaneLogger(os.Stdout, sklog.DefaultHTTPFormatter),
		//Logger:                 sklog.NewTestLogger(t),
		StoragePostgresAddress: testPostgresAddress,
	})
	if err := d.Run(); err != nil {
		t.Fatal(err)
	}
	defer d.Close()

	conn, err := grpc.Dial(d.Addr().String(), grpc.WithInsecure(), grpc.WithBlock())
	if err != nil {
		t.Fatal(err)
	}
	defer conn.Close()

	m := mnemosyne.New(conn, mnemosyne.MnemosyneOpts{})
	ats := make([]mnemosyne.AccessToken, 0, nb)
	wg := &sync.WaitGroup{}
	for i := 0; i < nb; i++ {
		go func(i int, wg *sync.WaitGroup) {
			wg.Add(1)
			defer wg.Done()

			ctx, _ := context.WithTimeout(context.Background(), 1*time.Second)
			ses, err := m.Start(ctx, strconv.Itoa(i), "daemon test client", nil)
			if err != nil {
				t.Fatal(err)
			}
			t.Logf("session created, it expires at: %s", time.Unix(ses.ExpireAt.Seconds, int64(ses.ExpireAt.Nanos)).Format(time.RFC3339))
			ats = append(ats, *ses.AccessToken)
		}(i, wg)
	}
	wg.Wait()

	// BUG: this assertion can fail on travis because of cpu lag.
	<-time.After(ttl + ttc + ttc)

	for i, at := range ats {
		go func(i int, at mnemosyne.AccessToken, wg *sync.WaitGroup) {
			wg.Add(1)
			defer wg.Done()

			ctx, _ := context.WithTimeout(context.Background(), 1*time.Second)
			_, err := m.Get(ctx, at)
			if err == nil {
				t.Error("%d: missing error", i)
				return
			}
			if grpc.Code(err) != codes.NotFound {
				t.Errorf("%d: wrong error code, expected %s", i, codes.NotFound, grpc.Code(err))
			}

			t.Logf("%d: as expected, session does not exists anymore", i)
		}(i, at, wg)
	}
	wg.Wait()
}
<commit_msg>removed potential issue in test, usage of t.Fatal inside grouting<commit_after>package mnemosyned

import (
	"net"
	"os"
	"strconv"
	"sync"
	"testing"
	"time"

	"github.com/piotrkowalczuk/mnemosyne"
	"github.com/piotrkowalczuk/sklog"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
)

func TestDaemon_Run(t *testing.T) {
	if testing.Short() {
		t.Skip("this test takes to long to run it in short mode")
	}

	ttl := 5 * time.Second
	ttc := 1 * time.Second
	nb := 10

	l, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	d := NewDaemon(&DaemonOpts{
		Environment: EnvironmentTest,
		SessionTTL:  ttl,
		SessionTTC:  ttc,
		RPCListener: l,
		// Use this logger to debug issues
		Logger: sklog.NewHumaneLogger(os.Stdout, sklog.DefaultHTTPFormatter),
		//Logger:                 sklog.NewTestLogger(t),
		StoragePostgresAddress: testPostgresAddress,
	})
	if err := d.Run(); err != nil {
		t.Fatal(err)
	}
	defer d.Close()

	conn, err := grpc.Dial(d.Addr().String(), grpc.WithInsecure(), grpc.WithBlock())
	if err != nil {
		t.Fatal(err)
	}
	defer conn.Close()

	m := mnemosyne.New(conn, mnemosyne.MnemosyneOpts{})
	ats := make([]mnemosyne.AccessToken, 0, nb)
	wg1 := &sync.WaitGroup{}
	for i := 0; i < nb; i++ {
		go func(i int, wg *sync.WaitGroup) {
			wg.Add(1)
			defer wg.Done()

			ctx, _ := context.WithTimeout(context.Background(), 1*time.Second)
			ses, err := m.Start(ctx, strconv.Itoa(i), "daemon test client", nil)
			if err != nil {
				t.Errorf("session could not be started:", err)
				return
			}
			t.Logf("session created, it expires at: %s", time.Unix(ses.ExpireAt.Seconds, int64(ses.ExpireAt.Nanos)).Format(time.RFC3339))
			ats = append(ats, *ses.AccessToken)
		}(i, wg1)
	}
	wg1.Wait()

	// BUG: this assertion can fail on travis because of cpu lag.
	<-time.After(ttl + ttc + ttc)

	wg2 := &sync.WaitGroup{}
	for i, at := range ats {
		go func(i int, at mnemosyne.AccessToken, wg *sync.WaitGroup) {
			wg.Add(1)
			defer wg.Done()

			ctx, _ := context.WithTimeout(context.Background(), 1*time.Second)
			_, err := m.Get(ctx, at)
			if err == nil {
				t.Error("%d: missing error", i)
				return
			}
			if grpc.Code(err) != codes.NotFound {
				t.Errorf("%d: wrong error code, expected %s", i, codes.NotFound, grpc.Code(err))
				return
			}

			t.Logf("%d: as expected, session does not exists anymore", i)
		}(i, at, wg2)
	}
	wg2.Wait()
}
<|endoftext|>