<commit_before>// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package ulog

import (
	"io"
	"os"
	"path"

	"go.uber.org/fx/core/config"

	"github.com/uber-go/zap"
)

// Configuration for logging with uberfx
type Configuration struct {
	Level   string
	File    *FileConfiguration
	Stdout  bool
	Verbose bool

	prefixWithFileLine *bool `yaml:"prefix_with_fileline"`
	TextFormatter      *bool // use TextFormatter (default json)
}

// FileConfiguration describes the properties needed to log to a file
type FileConfiguration struct {
	Enabled   bool
	Directory string
	FileName  string
}

// LogBuilder is the struct containing logger
type LogBuilder struct {
	log       zap.Logger
	logConfig Configuration
}

// Builder creates an empty builder for building ulog.Log object
func Builder() *LogBuilder {
	return &LogBuilder{}
}

// New instance of ulog.Log is returned with the default setup
func New() Log {
	return Builder().Build()
}

// WithConfiguration sets up configuration for the log builder
func (lb *LogBuilder) WithConfiguration(logConfig Configuration) *LogBuilder {
	lb.logConfig = logConfig
	return lb
}

// SetLogger allows users to set their own initialized logger to work with ulog APIs
func (lb *LogBuilder) SetLogger(zap zap.Logger) *LogBuilder {
	lb.log = zap
	return lb
}

// Build the ulog logger for use
func (lb *LogBuilder) Build() Log {
	var log zap.Logger

	// When setLogger is called, we will always use logger that has been set
	if lb.log != nil {
		return &baselogger{
			log: lb.log,
		}
	}

	if config.IsDevelopment() {
		log = lb.devLogger()
	} else {
		log = lb.Configure()
	}

	return &baselogger{
		log: log,
	}
}

func (lb *LogBuilder) devLogger() zap.Logger {
	return zap.New(zap.NewTextEncoder(), zap.DebugLevel)
}

func (lb *LogBuilder) defaultLogger() zap.Logger {
	return zap.New(zap.NewJSONEncoder(), zap.InfoLevel, zap.Output(zap.AddSync(os.Stdout)))
}

// Configure Log object with the provided log.Configuration
func (lb *LogBuilder) Configure() zap.Logger {
	lb.log = lb.defaultLogger()

	var options []zap.Option
	if lb.logConfig.Verbose {
		options = append(options, zap.DebugLevel)
	} else {
		lb.log.Debug("Setting log level", zap.String("level", lb.logConfig.Level))
		var lv zap.Level
		err := lv.UnmarshalText([]byte(lb.logConfig.Level))
		if err != nil {
			lb.log.Debug(
				"Cannot parse log level. Setting to Debug as default",
				zap.String("level", lb.logConfig.Level),
			)
		} else {
			options = append(options, lv)
		}
	}

	if lb.logConfig.File == nil || !lb.logConfig.File.Enabled {
		options = append(options, zap.Output(zap.AddSync(os.Stdout)))
	} else {
		options = append(options, zap.Output(lb.fileOutput(lb.logConfig.File, lb.logConfig.Stdout, lb.logConfig.Verbose)))
	}

	if lb.logConfig.TextFormatter != nil && *lb.logConfig.TextFormatter {
		return zap.New(zap.NewTextEncoder(), options...)
	}
	return zap.New(zap.NewJSONEncoder(), options...)
}

func (lb *LogBuilder) fileOutput(cfg *FileConfiguration, stdout bool, verbose bool) zap.WriteSyncer {
	fileLoc := path.Join(cfg.Directory, cfg.FileName)
	lb.log.Debug("adding log file output to zap")
	err := os.MkdirAll(cfg.Directory, os.FileMode(0755))
	if err != nil {
		lb.log.Fatal("Failed to create log directory: ", zap.Error(err))
	}
	file, err := os.OpenFile(fileLoc, os.O_WRONLY|os.O_APPEND|os.O_CREATE, os.FileMode(0755))
	if err != nil {
		lb.log.Fatal("Failed to open log file for writing.", zap.Error(err))
	}
	lb.log.Debug("Logfile created successfully", zap.String("filename", fileLoc))
	if verbose || stdout {
		return zap.AddSync(io.MultiWriter(os.Stdout, file))
	}
	return zap.AddSync(file)
}
<commit_msg>Upgrade log message to info (#95)<commit_after>// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package ulog

import (
	"io"
	"os"
	"path"

	"go.uber.org/fx/core/config"

	"github.com/uber-go/zap"
)

// Configuration for logging with uberfx
type Configuration struct {
	Level   string
	File    *FileConfiguration
	Stdout  bool
	Verbose bool

	prefixWithFileLine *bool `yaml:"prefix_with_fileline"`
	TextFormatter      *bool // use TextFormatter (default json)
}

// FileConfiguration describes the properties needed to log to a file
type FileConfiguration struct {
	Enabled   bool
	Directory string
	FileName  string
}

// LogBuilder is the struct containing logger
type LogBuilder struct {
	log       zap.Logger
	logConfig Configuration
}

// Builder creates an empty builder for building ulog.Log object
func Builder() *LogBuilder {
	return &LogBuilder{}
}

// New instance of ulog.Log is returned with the default setup
func New() Log {
	return Builder().Build()
}

// WithConfiguration sets up configuration for the log builder
func (lb *LogBuilder) WithConfiguration(logConfig Configuration) *LogBuilder {
	lb.logConfig = logConfig
	return lb
}

// SetLogger allows users to set their own initialized logger to work with ulog APIs
func (lb *LogBuilder) SetLogger(zap zap.Logger) *LogBuilder {
	lb.log = zap
	return lb
}

// Build the ulog logger for use
func (lb *LogBuilder) Build() Log {
	var log zap.Logger

	// When setLogger is called, we will always use logger that has been set
	if lb.log != nil {
		return &baselogger{
			log: lb.log,
		}
	}

	if config.IsDevelopment() {
		log = lb.devLogger()
	} else {
		log = lb.Configure()
	}

	return &baselogger{
		log: log,
	}
}

func (lb *LogBuilder) devLogger() zap.Logger {
	return zap.New(zap.NewTextEncoder(), zap.DebugLevel)
}

func (lb *LogBuilder) defaultLogger() zap.Logger {
	return zap.New(zap.NewJSONEncoder(), zap.InfoLevel, zap.Output(zap.AddSync(os.Stdout)))
}

// Configure Log object with the provided log.Configuration
func (lb *LogBuilder) Configure() zap.Logger {
	lb.log = lb.defaultLogger()

	var options []zap.Option
	if lb.logConfig.Verbose {
		options = append(options, zap.DebugLevel)
	} else {
		lb.log.Info("Setting log level", zap.String("level", lb.logConfig.Level))
		var lv zap.Level
		err := lv.UnmarshalText([]byte(lb.logConfig.Level))
		if err != nil {
			lb.log.Debug(
				"Cannot parse log level. Setting to Debug as default",
				zap.String("level", lb.logConfig.Level),
			)
		} else {
			options = append(options, lv)
		}
	}

	if lb.logConfig.File == nil || !lb.logConfig.File.Enabled {
		options = append(options, zap.Output(zap.AddSync(os.Stdout)))
	} else {
		options = append(options, zap.Output(lb.fileOutput(lb.logConfig.File, lb.logConfig.Stdout, lb.logConfig.Verbose)))
	}

	if lb.logConfig.TextFormatter != nil && *lb.logConfig.TextFormatter {
		return zap.New(zap.NewTextEncoder(), options...)
	}
	return zap.New(zap.NewJSONEncoder(), options...)
}

func (lb *LogBuilder) fileOutput(cfg *FileConfiguration, stdout bool, verbose bool) zap.WriteSyncer {
	fileLoc := path.Join(cfg.Directory, cfg.FileName)
	lb.log.Debug("adding log file output to zap")
	err := os.MkdirAll(cfg.Directory, os.FileMode(0755))
	if err != nil {
		lb.log.Fatal("Failed to create log directory: ", zap.Error(err))
	}
	file, err := os.OpenFile(fileLoc, os.O_WRONLY|os.O_APPEND|os.O_CREATE, os.FileMode(0755))
	if err != nil {
		lb.log.Fatal("Failed to open log file for writing.", zap.Error(err))
	}
	lb.log.Debug("Logfile created successfully", zap.String("filename", fileLoc))
	if verbose || stdout {
		return zap.AddSync(io.MultiWriter(os.Stdout, file))
	}
	return zap.AddSync(file)
}
<|endoftext|>


<commit_before>package tagging

import (
	"context"
	"encoding/json"

	"github.com/olivere/elastic"

	"github.com/trackit/trackit/db"
	"github.com/trackit/trackit/es"
	"github.com/trackit/trackit/models"
)

type compliance struct {
	Total           int64 `json:"total"`
	TotallyTagged   int64 `json:"totallyTagged"`
	PartiallyTagged int64 `json:"partiallyTagged"`
	NotTagged       int64 `json:"notTagged"`
}

// UpdateTaggingComplianceForAccount updates tagging compliance based on latest tagging reports and latest most used tags reports
func UpdateTaggingComplianceForAccount(ctx context.Context, accountID int) error {
	mostUsedTags, err := getMostUsedTagsFromDb(accountID)
	if err != nil {
		return err
	}

	count, err := getReportsCount(ctx, accountID)
	if err != nil {
		return err
	}

	totallyTagged, err := getTotallyTagged(ctx, accountID, mostUsedTags)
	if err != nil {
		return err
	}

	untagged, err := getUntagged(ctx, accountID, mostUsedTags)
	if err != nil {
		return err
	}

	partiallyTagged := count - totallyTagged - untagged

	return pushComplianceToEs(ctx, accountID, compliance{
		Total:           count,
		TotallyTagged:   totallyTagged,
		PartiallyTagged: partiallyTagged,
		NotTagged:       untagged,
	})
}

func getMostUsedTagsFromDb(accountID int) ([]string, error) {
	mostUsedTags, err := models.LatestMostUsedTagsByAwsAccountID(db.Db, accountID)
	if err != nil {
		return nil, err
	}

	res := []string{}
	err = json.Unmarshal([]byte(mostUsedTags.Tags), &res)
	if err != nil {
		return nil, err
	}
	return res, nil
}

func mostUsedTagsToTermQueries(mostUsedTags []string) []elastic.Query {
	termQueries := []elastic.Query{}
	for _, mostUsedTag := range mostUsedTags {
		termQueries = append(termQueries, elastic.NewNestedQuery("tags", elastic.NewBoolQuery().Must(elastic.NewTermQuery("tags.key", mostUsedTag))))
	}
	return termQueries
}

func handleComplianceEsReponse(res *elastic.SearchResult, err error) (int64, error) {
	if err != nil {
		return 0, err
	}

	reportDateRes, found := res.Aggregations.Terms("reportDate")
	if !found || len(reportDateRes.Buckets) <= 0 {
		return 0, nil
	}

	return reportDateRes.Buckets[0].DocCount, nil
}

func getReportsCount(ctx context.Context, accountID int) (int64, error) {
	client := es.Client
	indexName := es.IndexNameForUserId(accountID, "tagging-reports")
	index := client.Search().Index(indexName)

	reportDateAgg := elastic.NewTermsAggregation().Field("reportDate").Order("_term", false).Size(1)
	res, err := index.Size(0).Query(elastic.NewMatchAllQuery()).Aggregation("reportDate", reportDateAgg).Do(ctx)
	return handleComplianceEsReponse(res, err)
}

func getTotallyTagged(ctx context.Context, accountID int, mostUsedTags []string) (int64, error) {
	client := es.Client
	indexName := es.IndexNameForUserId(accountID, "tagging-reports")
	index := client.Search().Index(indexName)

	termQueries := mostUsedTagsToTermQueries(mostUsedTags)
	query := elastic.NewBoolQuery().Must(termQueries...)

	reportDateAgg := elastic.NewTermsAggregation().Field("reportDate").Order("_term", false).Size(1)
	res, err := index.Size(0).Query(query).Aggregation("reportDate", reportDateAgg).Do(ctx)
	return handleComplianceEsReponse(res, err)
}

func getUntagged(ctx context.Context, accountID int, mostUsedTags []string) (int64, error) {
	client := es.Client
	indexName := es.IndexNameForUserId(accountID, "tagging-reports")
	index := client.Search().Index(indexName)

	termQueries := mostUsedTagsToTermQueries(mostUsedTags)
	query := elastic.NewBoolQuery().MustNot(termQueries...)

	reportDateAgg := elastic.NewTermsAggregation().Field("reportDate").Order("_term", false).Size(1)
	res, err := index.Size(0).Query(query).Aggregation("reportDate", reportDateAgg).Do(ctx)
	return handleComplianceEsReponse(res, err)
}

func pushComplianceToEs(ctx context.Context, accountID int, compliance compliance) error {
	client := es.Client
	indexName := es.IndexNameForUserId(accountID, "tagging-compliance")
	_, err := client.Index().Index(indexName).Type("tagging-compliance").BodyJson(compliance).Do(ctx)
	return err
}
<commit_msg>Gave more explicit names to some functions<commit_after>package tagging

import (
	"context"
	"encoding/json"

	"github.com/olivere/elastic"

	"github.com/trackit/trackit/db"
	"github.com/trackit/trackit/es"
	"github.com/trackit/trackit/models"
)

type compliance struct {
	Total           int64 `json:"total"`
	TotallyTagged   int64 `json:"totallyTagged"`
	PartiallyTagged int64 `json:"partiallyTagged"`
	NotTagged       int64 `json:"notTagged"`
}

// UpdateTaggingComplianceForAccount updates tagging compliance based on latest tagging reports and latest most used tags reports
func UpdateTaggingComplianceForAccount(ctx context.Context, accountID int) error {
	mostUsedTags, err := getMostUsedTagsFromDb(accountID)
	if err != nil {
		return err
	}

	count, err := getReportsCount(ctx, accountID)
	if err != nil {
		return err
	}

	totallyTagged, err := getTotallyTaggedReportsCount(ctx, accountID, mostUsedTags)
	if err != nil {
		return err
	}

	untagged, err := getNotTaggedReportsCount(ctx, accountID, mostUsedTags)
	if err != nil {
		return err
	}

	partiallyTagged := count - totallyTagged - untagged

	return pushComplianceToEs(ctx, accountID, compliance{
		Total:           count,
		TotallyTagged:   totallyTagged,
		PartiallyTagged: partiallyTagged,
		NotTagged:       untagged,
	})
}

func getMostUsedTagsFromDb(accountID int) ([]string, error) {
	mostUsedTags, err := models.LatestMostUsedTagsByAwsAccountID(db.Db, accountID)
	if err != nil {
		return nil, err
	}

	res := []string{}
	err = json.Unmarshal([]byte(mostUsedTags.Tags), &res)
	if err != nil {
		return nil, err
	}
	return res, nil
}

func mostUsedTagsToTermQueries(mostUsedTags []string) []elastic.Query {
	termQueries := []elastic.Query{}
	for _, mostUsedTag := range mostUsedTags {
		termQueries = append(termQueries, elastic.NewNestedQuery("tags", elastic.NewBoolQuery().Must(elastic.NewTermQuery("tags.key", mostUsedTag))))
	}
	return termQueries
}

func handleComplianceEsReponse(res *elastic.SearchResult, err error) (int64, error) {
	if err != nil {
		return 0, err
	}

	reportDateRes, found := res.Aggregations.Terms("reportDate")
	if !found || len(reportDateRes.Buckets) <= 0 {
		return 0, nil
	}

	return reportDateRes.Buckets[0].DocCount, nil
}

func getReportsCount(ctx context.Context, accountID int) (int64, error) {
	client := es.Client
	indexName := es.IndexNameForUserId(accountID, "tagging-reports")
	index := client.Search().Index(indexName)

	reportDateAgg := elastic.NewTermsAggregation().Field("reportDate").Order("_term", false).Size(1)
	res, err := index.Size(0).Query(elastic.NewMatchAllQuery()).Aggregation("reportDate", reportDateAgg).Do(ctx)
	return handleComplianceEsReponse(res, err)
}

func getTotallyTaggedReportsCount(ctx context.Context, accountID int, mostUsedTags []string) (int64, error) {
	client := es.Client
	indexName := es.IndexNameForUserId(accountID, "tagging-reports")
	index := client.Search().Index(indexName)

	termQueries := mostUsedTagsToTermQueries(mostUsedTags)
	query := elastic.NewBoolQuery().Must(termQueries...)

	reportDateAgg := elastic.NewTermsAggregation().Field("reportDate").Order("_term", false).Size(1)
	res, err := index.Size(0).Query(query).Aggregation("reportDate", reportDateAgg).Do(ctx)
	return handleComplianceEsReponse(res, err)
}

func getNotTaggedReportsCount(ctx context.Context, accountID int, mostUsedTags []string) (int64, error) {
	client := es.Client
	indexName := es.IndexNameForUserId(accountID, "tagging-reports")
	index := client.Search().Index(indexName)

	termQueries := mostUsedTagsToTermQueries(mostUsedTags)
	query := elastic.NewBoolQuery().MustNot(termQueries...)

	reportDateAgg := elastic.NewTermsAggregation().Field("reportDate").Order("_term", false).Size(1)
	res, err := index.Size(0).Query(query).Aggregation("reportDate", reportDateAgg).Do(ctx)
	return handleComplianceEsReponse(res, err)
}

func pushComplianceToEs(ctx context.Context, accountID int, compliance compliance) error {
	client := es.Client
	indexName := es.IndexNameForUserId(accountID, "tagging-compliance")
	_, err := client.Index().Index(indexName).Type("tagging-compliance").BodyJson(compliance).Do(ctx)
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"os/exec"
)

func readStdin(out chan string, in chan bool) {
	//no buffering
	exec.Command("stty", "-f", "/dev/tty", "cbreak", "min", "1").Run()
	//no visible output
	//exec.Command("stty", "-f", "/dev/tty", "-echo").Run()

	var b []byte = make([]byte, 1)
	for {
		select {
		case <-in:
			return
		default:
			os.Stdin.Read(b)
			out <- string(b)
		}
	}
}

func main() {
	stdin := make(chan string, 1)
	kill := make(chan bool, 1)
	var count int = 0
	var input string

	fmt.Print("this program don't wait for a enter-key\nyou can now start typing\npress \"q\" to quit\n\n")

	go readStdin(stdin, kill)
	for {
		str := <-stdin

		if str == "q" {
			kill <- true
			close(stdin)
			break
		}

		input += str
		count++

		if count%10 == 0 {
			fmt.Printf("\n\nyou typed %v keys, quit the demo by pressing \"q\"\n\n", count)
			if count == 600 {
				fmt.Printf("Don't waste your time! Do something meaningful!")
				break
			}
		}
	}
	fmt.Printf("\n\nyou typed %v keys before you quit the program\n", count)
	fmt.Println(input)
}
<commit_msg>hide input stty - change and print terminal line settings Local settings:        [-]echo 	      echo input characters<commit_after>package main

import (
	"fmt"
	"os"
	"os/exec"
)

func readStdin(out chan string, in chan bool) {
	//no buffering
	exec.Command("stty", "-f", "/dev/tty", "cbreak", "min", "1").Run()
	//no visible output
	exec.Command("stty", "-f", "/dev/tty", "-echo").Run()

	var b []byte = make([]byte, 1)
	for {
		select {
		case <-in:
			return
		default:
			os.Stdin.Read(b)
			out <- string(b)
		}
	}
}

func main() {
	defer func() {
		exec.Command("stty", "-f", "/dev/tty", "echo").Run()
	}()
	stdin := make(chan string, 1)
	kill := make(chan bool, 1)
	var count int = 0
	var input string

	fmt.Print("this program don't wait for a enter-key\nyou can now start typing\npress \"q\" to quit\n\n")

	go readStdin(stdin, kill)
	for {
		str := <-stdin

		if str == "q" {
			kill <- true
			close(stdin)
			break
		}

		fmt.Print(str)
		input += str
		count++

		if count%10 == 0 {
			fmt.Printf("\n\nyou typed %v keys, quit the demo by pressing \"q\"\n\n", count)
			if count == 600 {
				fmt.Printf("Don't waste your time! Do something meaningful!")
				break
			}
		}
	}
	fmt.Printf("\n\nyou typed %v keys before you quit the program\n", count)
	fmt.Println(input)
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/md5"
	"encoding/hex"
	"flag"
	csv "github.com/whosonfirst/go-whosonfirst-csv"
	log "github.com/whosonfirst/go-whosonfirst-log"
	golog "log"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"sync/atomic"
	"time"
)

type WOFClone struct {
	Source         string
	Dest           string
	Count          int64
	Success        int64
	Error          int64
	Skipped        int64
	Scheduled      int64
	Completed      int64
	client         *http.Client
	connections    int64
	maxconnections int64
	cond           *sync.Cond
	logger         *log.WOFLogger
}

func NewWOFClone(source string, dest string, logger *log.WOFLogger) *WOFClone {

	// to do - add logging

	cd := &sync.Cond{L: &sync.Mutex{}}

	cl := &http.Client{}

	c := WOFClone{
		Count:          0,
		Success:        0,
		Error:          0,
		Skipped:        0,
		Source:         source,
		Dest:           dest,
		logger:         logger,
		client:         cl,
		connections:    0,
		maxconnections: 200,
		cond:           cd,
	}

	return &c
}

func (c *WOFClone) ParseMetaFile(file string) error {

	abs_path, _ := filepath.Abs(file)
	c.logger.Debug("Parse meta file %s", abs_path)

	reader, read_err := csv.NewDictReader(abs_path)

	if read_err != nil {
		c.logger.Error("Failed to read %s, because %v", abs_path, read_err)
		return read_err
	}

	wg := new(sync.WaitGroup)

	for {
		row, err := reader.Read()

		if err == io.EOF {
			break
		}

		if err != nil {
			return err
		}

		rel_path, ok := row["path"]

		if !ok {
			continue
		}

		wg.Add(1)

		go func() {

			atomic.AddInt64(&c.Scheduled, 1)

			defer wg.Done()
			complete, _ := c.Clone(rel_path)

			c.logger.Debug("Finised cloning %s: %t", rel_path, complete)

			atomic.AddInt64(&c.Completed, 1)
		}()
	}

	go func() {

		for c.Completed < c.Scheduled {
			c.logger.Info("scheduled: %d completed: %d connections: %d", c.Scheduled, c.Completed, c.connections)
			time.Sleep(5 * time.Second)
		}
	}()

	wg.Wait()

	return nil
}

func (c *WOFClone) Clone(rel_path string) (bool, error) {

	c.logger.Debug("Pre-process %s", rel_path)

	atomic.AddInt64(&c.Count, 1)

	remote := c.Source + rel_path
	local := path.Join(c.Dest, rel_path)

	_, err := os.Stat(local)

	if !os.IsNotExist(err) {

		change, _ := c.HasChanged(local, remote)

		if !change {

			c.logger.Debug("%s has not changed so skipping", local)

			atomic.AddInt64(&c.Success, 1)
			atomic.AddInt64(&c.Skipped, 1)
			return true, nil
		}

	}

	process_err := c.Process(remote, local)

	if process_err != nil {
		atomic.AddInt64(&c.Error, 1)
		return false, process_err
	}

	atomic.AddInt64(&c.Success, 1)
	return true, nil
}

/*
Okay, I figured out the problem I was seeing yesterday when checking
ETags (now that the WOF meta files contain an MD5 hash of the records
they list).

For example:

$> md5 /usr/local/mapzen/whosonfirst-data/data/404/529/391/404529391.geojson
71714a9cb8d286eb50b9a300b7faa1c3

If I request the same file from AWS proper everything is All Good (tm) :

$> curl -v https://s3.amazonaws.com/whosonfirst.mapzen.com/data/404/529/391/404529391.geojson
> /dev/null
* Server certificate: s3.amazonaws.com
< ETag: "71714a9cb8d286eb50b9a300b7faa1c3"

OTOH if I request the same file from wof.mz.com/data I get this:

$> curl -v https://whosonfirst.mapzen.com/data/404/529/391/404529391.geojson
> /dev/null
* Server certificate: whosonfirst.mapzen.com
< ETag: "93ad7d36a39f0e98f4aa777f22c73c03193fe390"

Which seems all weird-and-wtf until you remember that the nginx config
for the spelunker tries to proxy files from the wof-data GH repository
first and fall backs on AWS as a last resort:

https://github.com/whosonfirst/whosonfirst-www-spelunker/blob/master/nginx/whosonfirst-www-spelunker.conf.example

Who knows how GH is setting their ETags. I guess I don't really
understand why they aren't just doing MD5 hashes too but it's their
party so...

So.

The short answer is to also do a HEAD / ETag check against:

https://s3.amazonaws.com/whosonfirst.mapzen.com/data/

(20151027/thisisaaronland)
*/

func (c *WOFClone) HasChanged(local string, remote string) (bool, error) {

	change := true

	c.Throttle()

	body, err := ioutil.ReadFile(local)

	atomic.AddInt64(&c.connections, -1)

	if err != nil {
		c.logger.Error("Failed to read %s, becase %v", local, err)
		golog.Fatal(err)
		return change, err
	}

	hash := md5.Sum(body)
	local_hash := hex.EncodeToString(hash[:])

	rsp, err := c.Fetch("HEAD", remote)

	if err != nil {
		return change, err
	}

	defer rsp.Body.Close()

	etag := rsp.Header.Get("Etag")
	remote_hash := strings.Replace(etag, "\"", "", -1)

	if local_hash == remote_hash {
		change = false
	}

	// log.Printf("hash %s etag %s change %t\n", local_hash, remote_hash, change)

	return change, nil
}

func (c *WOFClone) Process(remote string, local string) error {

	c.logger.Debug("fetch %s and store in %s", remote, local)

	local_root := path.Dir(local)

	_, err := os.Stat(local_root)

	if os.IsNotExist(err) {
		c.logger.Info("create %s", local_root)
		os.MkdirAll(local_root, 0755)
	}

	rsp, fetch_err := c.Fetch("GET", remote)

	if fetch_err != nil {
		return fetch_err
	}

	defer rsp.Body.Close()

	contents, read_err := ioutil.ReadAll(rsp.Body)

	if read_err != nil {
		c.logger.Error("failed to read body for %s, because %v", remote, read_err)
		return read_err
	}

	go func() error {

		write_err := ioutil.WriteFile(local, contents, 0644)

		if write_err != nil {
			c.logger.Error("Failed to write %s, because %v", local, write_err)

			// See the way we're in a goroutine? That means the parent function
			// will return 'okay' without an error. So we're just going to account
			// for that here...

			atomic.AddInt64(&c.Success, -1)
			atomic.AddInt64(&c.Error, 1)

			return write_err
		}

		c.logger.Debug("Wrote %s to disk", local)
		return nil
	}()

	return nil
}

func (c *WOFClone) Fetch(method string, url string) (*http.Response, error) {

	c.logger.Debug("%s %s", method, url)

	c.Throttle()

	req, _ := http.NewRequest(method, url, nil)
	req.Close = true

	rsp, err := c.client.Do(req)

	atomic.AddInt64(&c.connections, -1)

	if err != nil {
		c.logger.Error("Failed to %s %s, because %v", method, url, err)
		golog.Fatal(err)
		return nil, err
	}

	return rsp, err
}

func (c *WOFClone) Throttle() {

	for {
		c.cond.L.Lock()

		for c.connections > c.maxconnections {
			c.logger.Debug("%d still > %d", c.connections, c.maxconnections)
			c.cond.Wait()
		}

		atomic.AddInt64(&c.connections, 1)

		c.cond.L.Unlock()
		c.cond.Broadcast()
		break
	}

}

func main() {

	// See notes inre source and Etags in the `HasChanged` method (20151027/thisisaaronland)

	var source = flag.String("source", "https://s3.amazonaws.com/whosonfirst.mapzen.com/data/", "Where to look for files")
	var dest = flag.String("dest", "", "Where to write files")
	var loglevel = flag.String("loglevel", "debug", "The level of detail for logging")

	flag.Parse()
	args := flag.Args()

	writer := io.MultiWriter(os.Stdout)

	lg := log.NewWOFLogger(writer, "[clone] ", *loglevel)

	cl := NewWOFClone(*source, *dest, lg)

	start := time.Now()

	wg := new(sync.WaitGroup)

	for _, file := range args {

		wg.Add(1)

		go func() {
			defer wg.Done()
			cl.ParseMetaFile(file)
		}()
	}

	wg.Wait()

	since := time.Since(start)
	secs := float64(since) / 1e9

	cl.logger.Info("processed %d files (ok: %d error: %d skipped: %d) in %f seconds\n", cl.Count, cl.Success, cl.Error, cl.Skipped, secs)
}
<commit_msg>add tunny PoolWorker hooks<commit_after>package main

import (
	"crypto/md5"
	"encoding/hex"
	"flag"
	csv "github.com/whosonfirst/go-whosonfirst-csv"
	log "github.com/whosonfirst/go-whosonfirst-log"
	"github.com/jeffail/tunny"
	golog "log"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"sync/atomic"
	"time"
)

type WOFClone struct {
	Source         string
	Dest           string
	Count          int64
	Success        int64
	Error          int64
	Skipped        int64
	Scheduled      int64
	Completed      int64
	client         *http.Client
	connections    int64
	maxconnections int64
	cond           *sync.Cond
	logger         *log.WOFLogger
	pool	       *tunny.WorkPool
}

func NewWOFClone(source string, dest string, logger *log.WOFLogger) *WOFClone {

	cd := &sync.Cond{L: &sync.Mutex{}}

	cl := &http.Client{}

	numCPUs := runtime.NumCPU() * 4
	runtime.GOMAXPROCS(numCPUs)

	pool, _ := tunny.CreatePoolGeneric(numCPUs).Open()

	c := WOFClone{
		Count:          0,
		Success:        0,
		Error:          0,
		Skipped:        0,
		Source:         source,
		Dest:           dest,
		logger:         logger,
		client:         cl,
		connections:    0,
		maxconnections: 200,
		cond:           cd,
		pool:		pool,
	}

	return &c
}

func (c *WOFClone) ParseMetaFile(file string) error {

	abs_path, _ := filepath.Abs(file)
	c.logger.Debug("Parse meta file %s", abs_path)

	reader, read_err := csv.NewDictReader(abs_path)

	if read_err != nil {
		c.logger.Error("Failed to read %s, because %v", abs_path, read_err)
		return read_err
	}

	wg := new(sync.WaitGroup)

	for {
		row, err := reader.Read()

		if err == io.EOF {
			break
		}

		if err != nil {
			return err
		}

		rel_path, ok := row["path"]

		if !ok {
			continue
		}

		wg.Add(1)

		go func() {

		   defer wg.Done()

		   complete, _ := c.Clone(rel_path)
		   c.logger.Debug("Finised cloning %s: %t", rel_path, complete)
		}()

		wg.Wait()
	}

	return nil
}

func (c *WOFClone) Clone(rel_path string) (bool, error) {

        defer c.pool.Close()

	c.logger.Debug("Pre-process %s", rel_path)

	atomic.AddInt64(&c.Count, 1)

	remote := c.Source + rel_path
	local := path.Join(c.Dest, rel_path)

	_, err := os.Stat(local)

	if !os.IsNotExist(err) {

		change, _ := c.HasChanged(local, remote)

		if !change {

			c.logger.Debug("%s has not changed so skipping", local)

			atomic.AddInt64(&c.Success, 1)
			atomic.AddInt64(&c.Skipped, 1)
			return true, nil
		}

	}

	_, err = c.pool.SendWork(func(){

	       c.logger.Debug("WORKING")

		atomic.AddInt64(&c.Scheduled, 1)

		process_err := c.Process(remote, local)

		atomic.AddInt64(&c.Completed, 1)

		if process_err != nil {
		   atomic.AddInt64(&c.Error, 1)
		} else {
		   atomic.AddInt64(&c.Success, 1)
		}
	})

	return true, nil
}

/*
Okay, I figured out the problem I was seeing yesterday when checking
ETags (now that the WOF meta files contain an MD5 hash of the records
they list).

For example:

$> md5 /usr/local/mapzen/whosonfirst-data/data/404/529/391/404529391.geojson
71714a9cb8d286eb50b9a300b7faa1c3

If I request the same file from AWS proper everything is All Good (tm) :

$> curl -v https://s3.amazonaws.com/whosonfirst.mapzen.com/data/404/529/391/404529391.geojson
> /dev/null
* Server certificate: s3.amazonaws.com
< ETag: "71714a9cb8d286eb50b9a300b7faa1c3"

OTOH if I request the same file from wof.mz.com/data I get this:

$> curl -v https://whosonfirst.mapzen.com/data/404/529/391/404529391.geojson
> /dev/null
* Server certificate: whosonfirst.mapzen.com
< ETag: "93ad7d36a39f0e98f4aa777f22c73c03193fe390"

Which seems all weird-and-wtf until you remember that the nginx config
for the spelunker tries to proxy files from the wof-data GH repository
first and fall backs on AWS as a last resort:

https://github.com/whosonfirst/whosonfirst-www-spelunker/blob/master/nginx/whosonfirst-www-spelunker.conf.example

Who knows how GH is setting their ETags. I guess I don't really
understand why they aren't just doing MD5 hashes too but it's their
party so...

So.

The short answer is to also do a HEAD / ETag check against:

https://s3.amazonaws.com/whosonfirst.mapzen.com/data/

(20151027/thisisaaronland)
*/

func (c *WOFClone) HasChanged(local string, remote string) (bool, error) {

	change := true
	return change, nil

	c.Throttle()

	body, err := ioutil.ReadFile(local)

	atomic.AddInt64(&c.connections, -1)

	if err != nil {
		c.logger.Error("Failed to read %s, becase %v", local, err)
		golog.Fatal(err)
		return change, err
	}

	hash := md5.Sum(body)
	local_hash := hex.EncodeToString(hash[:])

	rsp, err := c.Fetch("HEAD", remote)

	if err != nil {
		return change, err
	}

	defer rsp.Body.Close()

	etag := rsp.Header.Get("Etag")
	remote_hash := strings.Replace(etag, "\"", "", -1)

	if local_hash == remote_hash {
		change = false
	}

	// log.Printf("hash %s etag %s change %t\n", local_hash, remote_hash, change)

	return change, nil
}

func (c *WOFClone) Process(remote string, local string) error {

	c.logger.Debug("fetch %s and store in %s", remote, local)

	local_root := path.Dir(local)

	_, err := os.Stat(local_root)

	if os.IsNotExist(err) {
		c.logger.Info("create %s", local_root)
		os.MkdirAll(local_root, 0755)
	}

	rsp, fetch_err := c.Fetch("GET", remote)

	if fetch_err != nil {
		return fetch_err
	}

	defer rsp.Body.Close()

	contents, read_err := ioutil.ReadAll(rsp.Body)

	if read_err != nil {
		c.logger.Error("failed to read body for %s, because %v", remote, read_err)
		return read_err
	}

	go func() error {

		write_err := ioutil.WriteFile(local, contents, 0644)

		if write_err != nil {
			c.logger.Error("Failed to write %s, because %v", local, write_err)

			// See the way we're in a goroutine? That means the parent function
			// will return 'okay' without an error. So we're just going to account
			// for that here...

			atomic.AddInt64(&c.Success, -1)
			atomic.AddInt64(&c.Error, 1)

			return write_err
		}

		c.logger.Debug("Wrote %s to disk", local)
		return nil
	}()

	return nil
}

func (c *WOFClone) Fetch(method string, url string) (*http.Response, error) {

	c.logger.Debug("%s %s", method, url)

	c.Throttle()

	req, _ := http.NewRequest(method, url, nil)
	req.Close = true

	rsp, err := c.client.Do(req)

	atomic.AddInt64(&c.connections, -1)

	if err != nil {
		c.logger.Error("Failed to %s %s, because %v", method, url, err)
		golog.Fatal(err)
		return nil, err
	}

	return rsp, err
}

func (c *WOFClone) Throttle() {

	for {
		c.cond.L.Lock()

		for c.connections > c.maxconnections {
			c.logger.Debug("%d still > %d", c.connections, c.maxconnections)
			c.cond.Wait()
		}

		atomic.AddInt64(&c.connections, 1)

		c.cond.L.Unlock()
		c.cond.Broadcast()
		break
	}

}

func (c *WOFClone) Monitor () {

	go func() {

		for c.Completed < c.Scheduled {
			c.logger.Info("scheduled: %d completed: %d connections: %d", c.Scheduled, c.Completed, c.connections)
			time.Sleep(5 * time.Second)
		}
	}()
}

func main() {

	// See notes inre source and Etags in the `HasChanged` method (20151027/thisisaaronland)

	var source = flag.String("source", "https://s3.amazonaws.com/whosonfirst.mapzen.com/data/", "Where to look for files")
	var dest = flag.String("dest", "", "Where to write files")
	var loglevel = flag.String("loglevel", "debug", "The level of detail for logging")

	flag.Parse()
	args := flag.Args()

	writer := io.MultiWriter(os.Stdout)

	lg := log.NewWOFLogger(writer, "[clone] ", *loglevel)

	cl := NewWOFClone(*source, *dest, lg)

	start := time.Now()

	wg := new(sync.WaitGroup)

	for _, file := range args {

		wg.Add(1)

		go func() {
			defer wg.Done()
			cl.ParseMetaFile(file)
		}()
	}

	cl.Monitor()

	wg.Wait()

	since := time.Since(start)
	secs := float64(since) / 1e9

	cl.logger.Info("processed %d files (ok: %d error: %d skipped: %d) in %f seconds\n", cl.Count, cl.Success, cl.Error, cl.Skipped, secs)
}
<|endoftext|>


<commit_before>package genetics

import (
	"github.com/yaricom/goNEAT/neat/network"
	"fmt"
)

// Organisms are Genotypes (Genomes) and Phenotypes (Networks) with fitness information,
// i.e. the genotype and phenotype together.
type Organism struct {
	// A measure of fitness for the Organism
	Fitness                   float64
	// A fitness measure that won't change during adjustments
	OriginalFitness           float64

	// Win marker (if needed for a particular task)
	IsWinner                  bool

	// The Organism's phenotype
	Phenotype                 *network.Network
	// The Organism's genotype
	Genotype                  *Genome
	// The Species of the Organism
	Species                   *Species

	// Number of children this Organism may have
	ExpectedOffspring         float64
	// Tells which generation this Organism is from
	Generation                int

	// Marker for destruction of inferior Organisms
	toEliminate               bool
	// Marks the species champion
	isChampion                bool

	// Number of reserved offspring for a population leader
	superChampOffspring       int
	// Marks the best in population
	isPopulationChampion      bool
	// Marks the duplicate child of a champion (for tracking purposes)
	isPopulationChampionChild bool

	// DEBUG variable - highest fitness of champ
	highestFitness            float64

	// Track its origin - for debugging or analysis - we can tell how the organism was born
	mutationStructBaby        bool
	mateBaby                  bool

	// Used just for reporting purposes
	Error                     float64
}

// Creates new organism with specified genome, fitness and given generation number
func NewOrganism(fit float64, g *Genome, generation int) *Organism {
	return &Organism{
		Fitness:fit,
		Genotype:g,
		Phenotype:g.genesis(g.Id),
		Generation:generation,
	}
}

// Regenerate the network based on a change in the genotype
func (o *Organism) UpdatePhenotype() {
	// First, delete the old phenotype (net)
	o.Phenotype = nil

	// Now, recreate the phenotype off the new genotype
	o.Phenotype = o.Genotype.genesis(o.Genotype.Id)
}

func (o *Organism) String() string {
	champStr := ""
	if o.isChampion {
		champStr = " - CHAMPION - "
	}
	eliminStr := ""
	if o.toEliminate {
		eliminStr = " - TO BE ELIMINATED - "
	}
	return fmt.Sprintf("[Organism generation: %d, fitness: %.3f, original fitness: %.3f%s%s]",
		o.Generation, o.Fitness, o.OriginalFitness, champStr, eliminStr)
}

// Organisms is sortable list of organisms by fitness
type Organisms []*Organism

func (f Organisms) Len() int {
	return len(f)
}
func (f Organisms) Swap(i, j int) {
	f[i], f[j] = f[j], f[i]
}
func (f Organisms) Less(i, j int) bool {
	if f[i].Fitness < f[j].Fitness {
		// try to promote most fit organisms
		return true  // lower fitness is less
	} else if f[i].Fitness == f[j].Fitness {
		// try to promote less complex organisms
		ci := f[i].Phenotype.Complexity()
		cj := f[j].Phenotype.Complexity()
		if ci > cj {
			return true // higher complexity is less
		} else if ci == cj {
			return f[i].Genotype.Id < f[j].Genotype.Id // least recent (older) is less
		}
	}
	return false
}
<commit_msg>Added method to check if champion child organism is damaged<commit_after>package genetics

import (
	"github.com/yaricom/goNEAT/neat/network"
	"fmt"
)

// Organisms are Genotypes (Genomes) and Phenotypes (Networks) with fitness information,
// i.e. the genotype and phenotype together.
type Organism struct {
	// A measure of fitness for the Organism
	Fitness                   float64
	// A fitness measure that won't change during adjustments
	OriginalFitness           float64

	// Win marker (if needed for a particular task)
	IsWinner                  bool

	// The Organism's phenotype
	Phenotype                 *network.Network
	// The Organism's genotype
	Genotype                  *Genome
	// The Species of the Organism
	Species                   *Species

	// Number of children this Organism may have
	ExpectedOffspring         float64
	// Tells which generation this Organism is from
	Generation                int

	// Marker for destruction of inferior Organisms
	toEliminate               bool
	// Marks the species champion
	isChampion                bool

	// Number of reserved offspring for a population leader
	superChampOffspring       int
	// Marks the best in population
	isPopulationChampion      bool
	// Marks the duplicate child of a champion (for tracking purposes)
	isPopulationChampionChild bool

	// DEBUG variable - highest fitness of champ
	highestFitness            float64

	// Track its origin - for debugging or analysis - we can tell how the organism was born
	mutationStructBaby        bool
	mateBaby                  bool

	// Used just for reporting purposes
	Error                     float64
}

// Creates new organism with specified genome, fitness and given generation number
func NewOrganism(fit float64, g *Genome, generation int) *Organism {
	return &Organism{
		Fitness:fit,
		Genotype:g,
		Phenotype:g.genesis(g.Id),
		Generation:generation,
	}
}

// Regenerate the network based on a change in the genotype
func (o *Organism) UpdatePhenotype() {
	// First, delete the old phenotype (net)
	o.Phenotype = nil

	// Now, recreate the phenotype off the new genotype
	o.Phenotype = o.Genotype.genesis(o.Genotype.Id)
}

// Method to check if this algorithm is champion child and if so than if it's damaged
func (o *Organism) CheckChampionChildDamaged() bool {
	if o.isPopulationChampionChild && o.highestFitness > o.Fitness {
		return true
	}
	return false
}

func (o *Organism) String() string {
	champStr := ""
	if o.isChampion {
		champStr = " - CHAMPION - "
	}
	eliminStr := ""
	if o.toEliminate {
		eliminStr = " - TO BE ELIMINATED - "
	}
	return fmt.Sprintf("[Organism generation: %d, fitness: %.3f, original fitness: %.3f%s%s]",
		o.Generation, o.Fitness, o.OriginalFitness, champStr, eliminStr)
}

// Organisms is sortable list of organisms by fitness
type Organisms []*Organism

func (f Organisms) Len() int {
	return len(f)
}
func (f Organisms) Swap(i, j int) {
	f[i], f[j] = f[j], f[i]
}
func (f Organisms) Less(i, j int) bool {
	if f[i].Fitness < f[j].Fitness {
		// try to promote most fit organisms
		return true  // lower fitness is less
	} else if f[i].Fitness == f[j].Fitness {
		// try to promote less complex organisms
		ci := f[i].Phenotype.Complexity()
		cj := f[j].Phenotype.Complexity()
		if ci > cj {
			return true // higher complexity is less
		} else if ci == cj {
			return f[i].Genotype.Id < f[j].Genotype.Id // least recent (older) is less
		}
	}
	return false
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/mmcquillan/jane/configs"
	"github.com/mmcquillan/jane/inputs"
	"github.com/mmcquillan/jane/listeners"
	"sync"
)

var wg sync.WaitGroup

func main() {
	config := configs.Load()
	configs.Flags(&config)
	configs.Logging(&config)
	wg.Add(3)
	go commandLoop(&config)
	go listenLoop(&config)
	wg.Wait()
}

func runListener(config *configs.Config) {
	defer wg.Done()
	for _, listener := range config.Listeners {
		if listener.Active {
			switch listener.Type {
			case "rss":
				listeners.Rss(config, listener)
			}
		}
	}
}

func commandLoop(config *configs.Config) {
	defer wg.Done()
	inputs.Input(config)
}

func listenLoop(config *configs.Config) {
	defer wg.Done()
	listeners.Listen(config)
}
<commit_msg>Forgot to startp the new listener loop<commit_after>package main

import (
	"github.com/mmcquillan/jane/configs"
	"github.com/mmcquillan/jane/inputs"
	"github.com/mmcquillan/jane/listeners"
	"sync"
)

var wg sync.WaitGroup

func main() {
	config := configs.Load()
	configs.Flags(&config)
	configs.Logging(&config)
	wg.Add(3)
	go runListener(&config)
	go commandLoop(&config)
	go listenLoop(&config)
	wg.Wait()
}

func runListener(config *configs.Config) {
	defer wg.Done()
	for _, listener := range config.Listeners {
		if listener.Active {
			switch listener.Type {
			case "rss":
				listeners.Rss(config, listener)
			}
		}
	}
}

func commandLoop(config *configs.Config) {
	defer wg.Done()
	inputs.Input(config)
}

func listenLoop(config *configs.Config) {
	defer wg.Done()
	listeners.Listen(config)
}
<|endoftext|>


<commit_before>// Copyright 2013-2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package semver

import (
	"bytes"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"math/rand"
	"reflect"
	"testing"
	"time"

	"gopkg.in/yaml.v2"
)

type fixture struct {
	GreaterVersion string
	LesserVersion  string
}

var fixtures = []fixture{
	fixture{"0.0.0", "0.0.0-foo"},
	fixture{"0.0.1", "0.0.0"},
	fixture{"1.0.0", "0.9.9"},
	fixture{"0.10.0", "0.9.0"},
	fixture{"0.99.0", "0.10.0"},
	fixture{"2.0.0", "1.2.3"},
	fixture{"0.0.0", "0.0.0-foo"},
	fixture{"0.0.1", "0.0.0"},
	fixture{"1.0.0", "0.9.9"},
	fixture{"0.10.0", "0.9.0"},
	fixture{"0.99.0", "0.10.0"},
	fixture{"2.0.0", "1.2.3"},
	fixture{"0.0.0", "0.0.0-foo"},
	fixture{"0.0.1", "0.0.0"},
	fixture{"1.0.0", "0.9.9"},
	fixture{"0.10.0", "0.9.0"},
	fixture{"0.99.0", "0.10.0"},
	fixture{"2.0.0", "1.2.3"},
	fixture{"1.2.3", "1.2.3-asdf"},
	fixture{"1.2.3", "1.2.3-4"},
	fixture{"1.2.3", "1.2.3-4-foo"},
	fixture{"1.2.3-5-foo", "1.2.3-5"},
	fixture{"1.2.3-5", "1.2.3-4"},
	fixture{"1.2.3-5-foo", "1.2.3-5-Foo"},
	fixture{"3.0.0", "2.7.2+asdf"},
	fixture{"3.0.0+foobar", "2.7.2"},
	fixture{"1.2.3-a.10", "1.2.3-a.5"},
	fixture{"1.2.3-a.b", "1.2.3-a.5"},
	fixture{"1.2.3-a.b", "1.2.3-a"},
	fixture{"1.2.3-a.b.c.10.d.5", "1.2.3-a.b.c.5.d.100"},
	fixture{"1.0.0", "1.0.0-rc.1"},
	fixture{"1.0.0-rc.2", "1.0.0-rc.1"},
	fixture{"1.0.0-rc.1", "1.0.0-beta.11"},
	fixture{"1.0.0-beta.11", "1.0.0-beta.2"},
	fixture{"1.0.0-beta.2", "1.0.0-beta"},
	fixture{"1.0.0-beta", "1.0.0-alpha.beta"},
	fixture{"1.0.0-alpha.beta", "1.0.0-alpha.1"},
	fixture{"1.0.0-alpha.1", "1.0.0-alpha"},
	fixture{"1.2.3-rc.1-1-1hash", "1.2.3-rc.2"},
}

func TestCompare(t *testing.T) {
	for _, v := range fixtures {
		gt, err := NewVersion(v.GreaterVersion)
		if err != nil {
			t.Error(err)
		}

		lt, err := NewVersion(v.LesserVersion)
		if err != nil {
			t.Error(err)
		}

		if gt.LessThan(*lt) {
			t.Errorf("%s should not be less than %s", gt, lt)
		}
		if gt.Equal(*lt) {
			t.Errorf("%s should not be equal to %s", gt, lt)
		}
		if gt.Compare(*lt) <= 0 {
			t.Errorf("%s should be greater than %s", gt, lt)
		}
		if !lt.LessThan(*gt) {
			t.Errorf("%s should be less than %s", lt, gt)
		}
		if !lt.Equal(*lt) {
			t.Errorf("%s should be equal to %s", lt, lt)
		}
		if lt.Compare(*gt) > 0 {
			t.Errorf("%s should not be greater than %s", lt, gt)
		}
	}
}

func testString(t *testing.T, orig string, version *Version) {
	if orig != version.String() {
		t.Errorf("%s != %s", orig, version)
	}
}

func TestString(t *testing.T) {
	for _, v := range fixtures {
		gt, err := NewVersion(v.GreaterVersion)
		if err != nil {
			t.Error(err)
		}
		testString(t, v.GreaterVersion, gt)

		lt, err := NewVersion(v.LesserVersion)
		if err != nil {
			t.Error(err)
		}
		testString(t, v.LesserVersion, lt)
	}
}

func shuffleStringSlice(src []string) []string {
	dest := make([]string, len(src))
	rand.Seed(time.Now().Unix())
	perm := rand.Perm(len(src))
	for i, v := range perm {
		dest[v] = src[i]
	}
	return dest
}

func TestSort(t *testing.T) {
	sortedVersions := []string{"1.0.0", "1.0.2", "1.2.0", "3.1.1"}
	unsortedVersions := shuffleStringSlice(sortedVersions)

	semvers := []*Version{}
	for _, v := range unsortedVersions {
		sv, err := NewVersion(v)
		if err != nil {
			t.Fatal(err)
		}
		semvers = append(semvers, sv)
	}

	Sort(semvers)

	for idx, sv := range semvers {
		if sv.String() != sortedVersions[idx] {
			t.Fatalf("incorrect sort at index %v", idx)
		}
	}
}

func TestBumpMajor(t *testing.T) {
	version, _ := NewVersion("1.0.0")
	version.BumpMajor()
	if version.Major != 2 {
		t.Fatalf("bumping major on 1.0.0 resulted in %v", version)
	}

	version, _ = NewVersion("1.5.2")
	version.BumpMajor()
	if version.Minor != 0 && version.Patch != 0 {
		t.Fatalf("bumping major on 1.5.2 resulted in %v", version)
	}

	version, _ = NewVersion("1.0.0+build.1-alpha.1")
	version.BumpMajor()
	if version.PreRelease != "" && version.Metadata != "" {
		t.Fatalf("bumping major on 1.0.0+build.1-alpha.1 resulted in %v", version)
	}
}

func TestBumpMinor(t *testing.T) {
	version, _ := NewVersion("1.0.0")
	version.BumpMinor()

	if version.Major != 1 {
		t.Fatalf("bumping minor on 1.0.0 resulted in %v", version)
	}

	if version.Minor != 1 {
		t.Fatalf("bumping major on 1.0.0 resulted in %v", version)
	}

	version, _ = NewVersion("1.0.0+build.1-alpha.1")
	version.BumpMinor()
	if version.PreRelease != "" && version.Metadata != "" {
		t.Fatalf("bumping major on 1.0.0+build.1-alpha.1 resulted in %v", version)
	}
}

func TestBumpPatch(t *testing.T) {
	version, _ := NewVersion("1.0.0")
	version.BumpPatch()

	if version.Major != 1 {
		t.Fatalf("bumping minor on 1.0.0 resulted in %v", version)
	}

	if version.Minor != 0 {
		t.Fatalf("bumping major on 1.0.0 resulted in %v", version)
	}

	if version.Patch != 1 {
		t.Fatalf("bumping major on 1.0.0 resulted in %v", version)
	}

	version, _ = NewVersion("1.0.0+build.1-alpha.1")
	version.BumpPatch()
	if version.PreRelease != "" && version.Metadata != "" {
		t.Fatalf("bumping major on 1.0.0+build.1-alpha.1 resulted in %v", version)
	}
}

func TestMust(t *testing.T) {
	tests := []struct {
		versionStr string

		version *Version
		recov   interface{}
	}{
		{
			versionStr: "1.0.0",
			version:    &Version{Major: 1},
		},
		{
			versionStr: "version number",
			recov:      errors.New("version number is not in dotted-tri format"),
		},
	}

	for _, tt := range tests {
		func() {
			defer func() {
				recov := recover()
				if !reflect.DeepEqual(tt.recov, recov) {
					t.Fatalf("incorrect panic for %q: want %v, got %v", tt.versionStr, tt.recov, recov)
				}
			}()

			version := Must(NewVersion(tt.versionStr))
			if !reflect.DeepEqual(tt.version, version) {
				t.Fatalf("incorrect version for %q: want %+v, got %+v", tt.versionStr, tt.version, version)
			}
		}()
	}
}

type fixtureJSON struct {
	GreaterVersion *Version
	LesserVersion  *Version
}

func TestJSON(t *testing.T) {
	fj := make([]fixtureJSON, len(fixtures))
	for i, v := range fixtures {
		var err error
		fj[i].GreaterVersion, err = NewVersion(v.GreaterVersion)
		if err != nil {
			t.Fatal(err)
		}
		fj[i].LesserVersion, err = NewVersion(v.LesserVersion)
		if err != nil {
			t.Fatal(err)
		}
	}

	fromStrings, err := json.Marshal(fixtures)
	if err != nil {
		t.Fatal(err)
	}
	fromVersions, err := json.Marshal(fj)
	if err != nil {
		t.Fatal(err)
	}
	if !bytes.Equal(fromStrings, fromVersions) {
		t.Errorf("Expected:   %s", fromStrings)
		t.Errorf("Unexpected: %s", fromVersions)
	}

	fromJson := make([]fixtureJSON, 0, len(fj))
	err = json.Unmarshal(fromStrings, &fromJson)
	if err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(fromJson, fj) {
		t.Error("Expected:   ", fj)
		t.Error("Unexpected: ", fromJson)
	}
}

func TestYAML(t *testing.T) {
	document, err := yaml.Marshal(fixtures)
	if err != nil {
		t.Fatal(err)
	}

	expected := make([]fixtureJSON, len(fixtures))
	for i, v := range fixtures {
		var err error
		expected[i].GreaterVersion, err = NewVersion(v.GreaterVersion)
		if err != nil {
			t.Fatal(err)
		}
		expected[i].LesserVersion, err = NewVersion(v.LesserVersion)
		if err != nil {
			t.Fatal(err)
		}
	}

	fromYAML := make([]fixtureJSON, 0, len(fixtures))
	err = yaml.Unmarshal(document, &fromYAML)
	if err != nil {
		t.Fatal(err)
	}

	if !reflect.DeepEqual(fromYAML, expected) {
		t.Error("Expected:   ", expected)
		t.Error("Unexpected: ", fromYAML)
	}
}

func TestBadInput(t *testing.T) {
	bad := []string{
		"1.2",
		"1.2.3x",
		"0x1.3.4",
		"-1.2.3",
		"1.2.3.4",
		"0.88.0-11_e4e5dcabb",
		"0.88.0+11_e4e5dcabb",
	}
	for _, b := range bad {
		if _, err := NewVersion(b); err == nil {
			t.Error("Improperly accepted value: ", b)
		}
	}
}

func TestFlag(t *testing.T) {
	v := Version{}
	f := flag.NewFlagSet("version", flag.ContinueOnError)
	f.Var(&v, "version", "set version")

	if err := f.Set("version", "1.2.3"); err != nil {
		t.Fatal(err)
	}

	if v.String() != "1.2.3" {
		t.Errorf("Set wrong value %q", v)
	}
}

func ExampleVersion_LessThan() {
	vA := New("1.2.3")
	vB := New("3.2.1")

	fmt.Printf("%s < %s == %t\n", vA, vB, vA.LessThan(*vB))
	// Output:
	// 1.2.3 < 3.2.1 == true
}
<commit_msg>*: gofmt -s<commit_after>// Copyright 2013-2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package semver

import (
	"bytes"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"math/rand"
	"reflect"
	"testing"
	"time"

	"gopkg.in/yaml.v2"
)

type fixture struct {
	GreaterVersion string
	LesserVersion  string
}

var fixtures = []fixture{
	{"0.0.0", "0.0.0-foo"},
	{"0.0.1", "0.0.0"},
	{"1.0.0", "0.9.9"},
	{"0.10.0", "0.9.0"},
	{"0.99.0", "0.10.0"},
	{"2.0.0", "1.2.3"},
	{"0.0.0", "0.0.0-foo"},
	{"0.0.1", "0.0.0"},
	{"1.0.0", "0.9.9"},
	{"0.10.0", "0.9.0"},
	{"0.99.0", "0.10.0"},
	{"2.0.0", "1.2.3"},
	{"0.0.0", "0.0.0-foo"},
	{"0.0.1", "0.0.0"},
	{"1.0.0", "0.9.9"},
	{"0.10.0", "0.9.0"},
	{"0.99.0", "0.10.0"},
	{"2.0.0", "1.2.3"},
	{"1.2.3", "1.2.3-asdf"},
	{"1.2.3", "1.2.3-4"},
	{"1.2.3", "1.2.3-4-foo"},
	{"1.2.3-5-foo", "1.2.3-5"},
	{"1.2.3-5", "1.2.3-4"},
	{"1.2.3-5-foo", "1.2.3-5-Foo"},
	{"3.0.0", "2.7.2+asdf"},
	{"3.0.0+foobar", "2.7.2"},
	{"1.2.3-a.10", "1.2.3-a.5"},
	{"1.2.3-a.b", "1.2.3-a.5"},
	{"1.2.3-a.b", "1.2.3-a"},
	{"1.2.3-a.b.c.10.d.5", "1.2.3-a.b.c.5.d.100"},
	{"1.0.0", "1.0.0-rc.1"},
	{"1.0.0-rc.2", "1.0.0-rc.1"},
	{"1.0.0-rc.1", "1.0.0-beta.11"},
	{"1.0.0-beta.11", "1.0.0-beta.2"},
	{"1.0.0-beta.2", "1.0.0-beta"},
	{"1.0.0-beta", "1.0.0-alpha.beta"},
	{"1.0.0-alpha.beta", "1.0.0-alpha.1"},
	{"1.0.0-alpha.1", "1.0.0-alpha"},
	{"1.2.3-rc.1-1-1hash", "1.2.3-rc.2"},
}

func TestCompare(t *testing.T) {
	for _, v := range fixtures {
		gt, err := NewVersion(v.GreaterVersion)
		if err != nil {
			t.Error(err)
		}

		lt, err := NewVersion(v.LesserVersion)
		if err != nil {
			t.Error(err)
		}

		if gt.LessThan(*lt) {
			t.Errorf("%s should not be less than %s", gt, lt)
		}
		if gt.Equal(*lt) {
			t.Errorf("%s should not be equal to %s", gt, lt)
		}
		if gt.Compare(*lt) <= 0 {
			t.Errorf("%s should be greater than %s", gt, lt)
		}
		if !lt.LessThan(*gt) {
			t.Errorf("%s should be less than %s", lt, gt)
		}
		if !lt.Equal(*lt) {
			t.Errorf("%s should be equal to %s", lt, lt)
		}
		if lt.Compare(*gt) > 0 {
			t.Errorf("%s should not be greater than %s", lt, gt)
		}
	}
}

func testString(t *testing.T, orig string, version *Version) {
	if orig != version.String() {
		t.Errorf("%s != %s", orig, version)
	}
}

func TestString(t *testing.T) {
	for _, v := range fixtures {
		gt, err := NewVersion(v.GreaterVersion)
		if err != nil {
			t.Error(err)
		}
		testString(t, v.GreaterVersion, gt)

		lt, err := NewVersion(v.LesserVersion)
		if err != nil {
			t.Error(err)
		}
		testString(t, v.LesserVersion, lt)
	}
}

func shuffleStringSlice(src []string) []string {
	dest := make([]string, len(src))
	rand.Seed(time.Now().Unix())
	perm := rand.Perm(len(src))
	for i, v := range perm {
		dest[v] = src[i]
	}
	return dest
}

func TestSort(t *testing.T) {
	sortedVersions := []string{"1.0.0", "1.0.2", "1.2.0", "3.1.1"}
	unsortedVersions := shuffleStringSlice(sortedVersions)

	semvers := []*Version{}
	for _, v := range unsortedVersions {
		sv, err := NewVersion(v)
		if err != nil {
			t.Fatal(err)
		}
		semvers = append(semvers, sv)
	}

	Sort(semvers)

	for idx, sv := range semvers {
		if sv.String() != sortedVersions[idx] {
			t.Fatalf("incorrect sort at index %v", idx)
		}
	}
}

func TestBumpMajor(t *testing.T) {
	version, _ := NewVersion("1.0.0")
	version.BumpMajor()
	if version.Major != 2 {
		t.Fatalf("bumping major on 1.0.0 resulted in %v", version)
	}

	version, _ = NewVersion("1.5.2")
	version.BumpMajor()
	if version.Minor != 0 && version.Patch != 0 {
		t.Fatalf("bumping major on 1.5.2 resulted in %v", version)
	}

	version, _ = NewVersion("1.0.0+build.1-alpha.1")
	version.BumpMajor()
	if version.PreRelease != "" && version.Metadata != "" {
		t.Fatalf("bumping major on 1.0.0+build.1-alpha.1 resulted in %v", version)
	}
}

func TestBumpMinor(t *testing.T) {
	version, _ := NewVersion("1.0.0")
	version.BumpMinor()

	if version.Major != 1 {
		t.Fatalf("bumping minor on 1.0.0 resulted in %v", version)
	}

	if version.Minor != 1 {
		t.Fatalf("bumping major on 1.0.0 resulted in %v", version)
	}

	version, _ = NewVersion("1.0.0+build.1-alpha.1")
	version.BumpMinor()
	if version.PreRelease != "" && version.Metadata != "" {
		t.Fatalf("bumping major on 1.0.0+build.1-alpha.1 resulted in %v", version)
	}
}

func TestBumpPatch(t *testing.T) {
	version, _ := NewVersion("1.0.0")
	version.BumpPatch()

	if version.Major != 1 {
		t.Fatalf("bumping minor on 1.0.0 resulted in %v", version)
	}

	if version.Minor != 0 {
		t.Fatalf("bumping major on 1.0.0 resulted in %v", version)
	}

	if version.Patch != 1 {
		t.Fatalf("bumping major on 1.0.0 resulted in %v", version)
	}

	version, _ = NewVersion("1.0.0+build.1-alpha.1")
	version.BumpPatch()
	if version.PreRelease != "" && version.Metadata != "" {
		t.Fatalf("bumping major on 1.0.0+build.1-alpha.1 resulted in %v", version)
	}
}

func TestMust(t *testing.T) {
	tests := []struct {
		versionStr string

		version *Version
		recov   interface{}
	}{
		{
			versionStr: "1.0.0",
			version:    &Version{Major: 1},
		},
		{
			versionStr: "version number",
			recov:      errors.New("version number is not in dotted-tri format"),
		},
	}

	for _, tt := range tests {
		func() {
			defer func() {
				recov := recover()
				if !reflect.DeepEqual(tt.recov, recov) {
					t.Fatalf("incorrect panic for %q: want %v, got %v", tt.versionStr, tt.recov, recov)
				}
			}()

			version := Must(NewVersion(tt.versionStr))
			if !reflect.DeepEqual(tt.version, version) {
				t.Fatalf("incorrect version for %q: want %+v, got %+v", tt.versionStr, tt.version, version)
			}
		}()
	}
}

type fixtureJSON struct {
	GreaterVersion *Version
	LesserVersion  *Version
}

func TestJSON(t *testing.T) {
	fj := make([]fixtureJSON, len(fixtures))
	for i, v := range fixtures {
		var err error
		fj[i].GreaterVersion, err = NewVersion(v.GreaterVersion)
		if err != nil {
			t.Fatal(err)
		}
		fj[i].LesserVersion, err = NewVersion(v.LesserVersion)
		if err != nil {
			t.Fatal(err)
		}
	}

	fromStrings, err := json.Marshal(fixtures)
	if err != nil {
		t.Fatal(err)
	}
	fromVersions, err := json.Marshal(fj)
	if err != nil {
		t.Fatal(err)
	}
	if !bytes.Equal(fromStrings, fromVersions) {
		t.Errorf("Expected:   %s", fromStrings)
		t.Errorf("Unexpected: %s", fromVersions)
	}

	fromJson := make([]fixtureJSON, 0, len(fj))
	err = json.Unmarshal(fromStrings, &fromJson)
	if err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(fromJson, fj) {
		t.Error("Expected:   ", fj)
		t.Error("Unexpected: ", fromJson)
	}
}

func TestYAML(t *testing.T) {
	document, err := yaml.Marshal(fixtures)
	if err != nil {
		t.Fatal(err)
	}

	expected := make([]fixtureJSON, len(fixtures))
	for i, v := range fixtures {
		var err error
		expected[i].GreaterVersion, err = NewVersion(v.GreaterVersion)
		if err != nil {
			t.Fatal(err)
		}
		expected[i].LesserVersion, err = NewVersion(v.LesserVersion)
		if err != nil {
			t.Fatal(err)
		}
	}

	fromYAML := make([]fixtureJSON, 0, len(fixtures))
	err = yaml.Unmarshal(document, &fromYAML)
	if err != nil {
		t.Fatal(err)
	}

	if !reflect.DeepEqual(fromYAML, expected) {
		t.Error("Expected:   ", expected)
		t.Error("Unexpected: ", fromYAML)
	}
}

func TestBadInput(t *testing.T) {
	bad := []string{
		"1.2",
		"1.2.3x",
		"0x1.3.4",
		"-1.2.3",
		"1.2.3.4",
		"0.88.0-11_e4e5dcabb",
		"0.88.0+11_e4e5dcabb",
	}
	for _, b := range bad {
		if _, err := NewVersion(b); err == nil {
			t.Error("Improperly accepted value: ", b)
		}
	}
}

func TestFlag(t *testing.T) {
	v := Version{}
	f := flag.NewFlagSet("version", flag.ContinueOnError)
	f.Var(&v, "version", "set version")

	if err := f.Set("version", "1.2.3"); err != nil {
		t.Fatal(err)
	}

	if v.String() != "1.2.3" {
		t.Errorf("Set wrong value %q", v)
	}
}

func ExampleVersion_LessThan() {
	vA := New("1.2.3")
	vB := New("3.2.1")

	fmt.Printf("%s < %s == %t\n", vA, vB, vA.LessThan(*vB))
	// Output:
	// 1.2.3 < 3.2.1 == true
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/gographics/imagick/imagick"
)

func Join(tiles matrix) *imagick.MagickWand {
	imagick.Initialize()
	defer imagick.Terminate()

	for i := tiles.TL.x; i < tiles.TR.x; i++ {
		col(tiles.BL.y, tiles.TL.y, i)
	}

	return row(tiles.TL.x, tiles.TR.x)
}

func row(from, to coord) *imagick.MagickWand {
	base := newBase()
	defer base.Destroy()

	for i := from; i < to; i++ {
		addImage(base, fmt.Sprintf("%d.png", i))
	}

	wand := base.MontageImage(imagick.NewDrawingWand(), "x1", "", imagick.MONTAGE_MODE_CONCATENATE, "0x0")
	return wand
}

func col(from, to, x coord) {
	base := newBase()
	defer base.Destroy()

	for i := from; i < to; i++ {
		addImage(base, fmt.Sprintf("tile_%d:%d.png", x, i))
	}

	wand := base.MontageImage(imagick.NewDrawingWand(), "1x", "", imagick.MONTAGE_MODE_CONCATENATE, "0x0")
	wand.WriteImage(fmt.Sprintf("%d.png", x))
}

/**
 * Add image to base wand, I know there is way to
 * add them more at time, but that would constitute
 * making an array
 */
func addImage(base *imagick.MagickWand, filename string) {
	additive := imagick.NewMagickWand()
	err := additive.ReadImage(filename)
	if err != nil {
		panic(err)
	}
	defer additive.Destroy()
	base.AddImage(additive)
}

/**
 * Common place for config
 */
func newBase() *imagick.MagickWand {
	base := imagick.NewMagickWand()
	base.SetImageCompressionQuality(100)
	return base
}
<commit_msg>joining using go instead of magick<commit_after>package main

import (
	"fmt"
	"github.com/nfnt/resize"
	"image"
	"image/png"
	"os"
)

func Join(tiles matrix) image.Image {
	cols := []string{}

	// Itinerate over Xes
	for dx := tiles.TL.x; dx < tiles.TR.x; dx++ {
		col := []string{}
		// All y for this x
		for dy := tiles.BL.y; dy < tiles.TL.y; dy++ {
			col = append(col, fmt.Sprintf("tile_%d:%d.png", dx, dy))
		}
		img := joinByName(col, true)
		filename := fmt.Sprintf("column_%d.png", dx)
		fmt.Print("Created: ", filename, "                   \r")
		Save(img, filename)
		cols = append(cols, filename)
	}

	return joinByName(cols, false)
}

func Save(img image.Image, filename string) {
	outfile, err := os.Create(filename)
	handle(err)
	png.Encode(outfile, img)
	outfile.Close()
}

func joinByName(filenames []string, vertical bool) image.Image {

	var output *image.RGBA
	var tile_size struct{ dx, dy int }
	var offset int

	for i, fname := range filenames {
		tile := open(fname)
		if i == 0 {
			offset = 0
			tile_size.dx = tile.Bounds().Dx()
			tile_size.dy = tile.Bounds().Dy()

			if vertical {
				output = createHost(tile_size.dx, tile_size.dy*len(filenames))
			} else {
				output = createHost(tile_size.dx*len(filenames), tile_size.dy)
			}
		}

		for dx := 0; dx < tile_size.dx; dx++ {
			for dy := 0; dy < tile_size.dy; dy++ {
				if vertical {
					output.Set(dx, dy+(offset*tile_size.dy), tile.At(dx, dy))
				} else {
					output.Set(dx+(offset*tile_size.dx), dy, tile.At(dx, dy))
				}
			}
		}

		offset++
	}

	return output
}

func createHost(xb, yb int) *image.RGBA {
	return image.NewRGBA(image.Rect(0, 0, xb, yb))
}

func open(filename string) image.Image {
	file, err := os.Open(filename)
	handle(err)
	img, _, err := image.Decode(file)
	handle(err)

	file.Close()

	img = resize.Resize(0, 150, img, resize.NearestNeighbor)
	return img
}

func handle(err error) {
	if err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package models

import (
  "github.com/spacedock-io/index/couch"
  "github.com/fjl/go-couchdb"
)

var prefix string = "user:"

type User struct {
  Username string `json:"username"`
  Rev string `json:"_rev,omitempty"`
}

func NewUser() *User {
  return &User{}
}

func GetUser(name string) (*User, error) {
  ret := &User{}
  err := couch.Global.Get(prefix + name, nil, ret)

  if err != nil {
    dberr, ok := err.(couchdb.DatabaseError)
    if ok && dberr.StatusCode == 404 {
      err = nil
      ret = nil
    }
  }
  return ret, err
}

func CreateUser(user *User) error {
  _, err := couch.Global.Put(prefix + user.Username, user)
  if err != nil {
    dberr, ok := err.(couchdb.DatabaseError)
    if ok && dberr.StatusCode == 409 {
      err = AlreadyExistsError{}
    }
  }

  return err
}

func DeleteUser(name string) error {
  user, err := GetUser(name)
  if err != nil {
    return err
  }

  _, err = couch.Global.Delete(prefix + user.Username, user.Rev)
  return err
}
<commit_msg>Add user emails and pbkdf2 password hashing<commit_after>package models

import (
  "crypto/sha256"
  "github.com/spacedock-io/index/couch"
  "github.com/fjl/go-couchdb"
  "github.com/gokyle/pbkdf2"
)

var prefix string = "user:"

type User struct {
  Username string `json:"username"`
  Email string `json:"email"`
  Salt string `json:"salt"`
  Pass string `json:"pass"`
  Rev string `json:"_rev,omitempty"`
}

func init() {
  pbkdf2.HashFunc = sha256.New
}

func NewUser() *User {
  return &User{}
}

func GetUser(name string) (*User, error) {
  ret := &User{}
  err := couch.Global.Get(prefix + name, nil, ret)

  if err != nil {
    dberr, ok := err.(couchdb.DatabaseError)
    if ok && dberr.StatusCode == 404 {
      err = nil
      ret = nil
    }
  }
  return ret, err
}

func CreateUser(user *User, password string) error {
  ph := pbkdf2.HashPassword(password)

  user.Salt = string(ph.Salt)
  user.Pass = string(ph.Hash)

  _, err := couch.Global.Put(prefix + user.Username, user)
  if err != nil {
    dberr, ok := err.(couchdb.DatabaseError)
    if ok && dberr.StatusCode == 409 {
      err = AlreadyExistsError{}
    }
  }

  return err
}

func DeleteUser(name string) error {
  user, err := GetUser(name)
  if err != nil {
    return err
  }

  _, err = couch.Global.Delete(prefix + user.Username, user.Rev)
  return err
}

func AuthUser(name string, pass string) bool {
  ret, err := GetUser(name)
  if err != nil { return false }
  ph := &pbkdf2.PasswordHash{[]byte(ret.Pass), []byte(ret.Salt)}
  return pbkdf2.MatchPassword(pass, ph)
}

<|endoftext|>


<commit_before>package main

import (
	"regexp"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/codegangsta/cli"
	"github.com/docker/swarm/discovery"
)

func checkAddrFormat(addr string) bool {
	m, _ := regexp.MatchString("^[0-9a-zA-Z._-]+:[0-9]{1,5}$", addr)
	return m
}

func join(c *cli.Context) {
	dflag := getDiscovery(c)
	if dflag == "" {
		log.Fatal("discovery required to join a cluster. See 'swarm join --help'.")
	}

	d, err := discovery.New(dflag, c.Int("heartbeat"))
	if err != nil {
		log.Fatal(err)
	}

	addr := c.String("addr")

	if !checkAddrFormat(addr) {
		log.Fatal("--addr should be of the form ip:port or hostname:port")
	}

	if err := d.Register(addr); err != nil {
		log.Fatal(err)
	}

	hb := time.Duration(c.Int("heartbeat"))
	for {
		time.Sleep(hb * time.Second)
		if err := d.Register(c.String("addr")); err != nil {
			log.Error(err)
		}
	}
}
<commit_msg>add info in join<commit_after>package main

import (
	"regexp"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/codegangsta/cli"
	"github.com/docker/swarm/discovery"
)

func checkAddrFormat(addr string) bool {
	m, _ := regexp.MatchString("^[0-9a-zA-Z._-]+:[0-9]{1,5}$", addr)
	return m
}

func join(c *cli.Context) {
	dflag := getDiscovery(c)
	if dflag == "" {
		log.Fatal("discovery required to join a cluster. See 'swarm join --help'.")
	}

	d, err := discovery.New(dflag, c.Int("heartbeat"))
	if err != nil {
		log.Fatal(err)
	}

	addr := c.String("addr")

	if !checkAddrFormat(addr) {
		log.Fatal("--addr should be of the form ip:port or hostname:port")
	}

	if err := d.Register(addr); err != nil {
		log.Fatal(err)
	}

	hb := time.Duration(c.Int("heartbeat"))
	for {
		log.Infof("Registering %q on the discovery service %q every %d seconds...", addr, dflag, hb)
		time.Sleep(hb * time.Second)
		if err := d.Register(addr); err != nil {
			log.Error(err)
		}
	}
}
<|endoftext|>


<commit_before>/*
Command jqsh provides an interactive wrapper to the jq command line utility.

Shell syntax

The current shell syntax is rudimentory but it suffices.  Commands are prefixed
with a colon ':' and a command name followed by a space separated list of
arguments.

	> :load test.json

The above loads the file "test.json" into the jqsh cache for inspection.  There
is no quoting of arguments.  A plus '+' may be used on the last argument to
include all charactors up to (but excluding) the next newline character.

	> :push +.items[] | select(.name | contains("hello"))

The above pushes the filter `.items[] | select(.name | contains("hello"))` on
to the jqsh filter stack. This is such a common operation that it has a special
shorthand.  A non-empty line that does not start with a colon causes the line's
contents to be pushed on the filter stack. So the above line could be
simplified.

	> .[] | select(.gender == "Female")

Blank lines are also a shorthand, printing the working filter stack applied to
the input, equivalent to the "write" command.

	> :write

That is all of the syntax in jqsh.

The filter stack

The core concept in jqsh in a stack of jq filters.  Filters create a larger
filter when joined with pipes "|".  And maintaining a stack of filters allows
exploritory querying of JSON structures.

In the previous section the "push" command was demonstrated.  Pushing filters
will effectively drill down into JSON structures.  To zoom out (or undo) the
application of filters the "pop" command is used.

	> :pop

Called without arguments, "pop" will remove the most recent filter pushed onto
the stack.  Multiple items can be popped off the stack by passing "pop" an
integer argument (the number of items to pop).

	> :pop 3

Command reference

A list of commands and other interactive help topics can be found through the
"help" command.

	> :help

Individual commands respond to the "-h" flag for usage documentation.
*/
package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"runtime"
	"strings"
	"sync"
	"sync/atomic"
	"unicode"
)

var ErrStackEmpty = fmt.Errorf("the stack is empty")

func main() {
	flag.Parse()
	args := flag.Args()

	jqbin, err := LocateJQ("")
	if err == ErrJQNotFound {
		fmt.Fprintln(os.Stderr, "Unable to locate the jq executable. Make sure it's installed.")
		fmt.Fprintln(os.Stderr)
		switch runtime.GOOS {
		case "darwin":
			fmt.Fprintln(os.Stderr, "The easiest way to install jq on OS X is with homebrew.")
			fmt.Fprintln(os.Stderr)
			fmt.Fprintln(os.Stderr, "\tbrew install jq")
		default:
			fmt.Fprintln(os.Stderr, "See the jq homepage for download and install instructions")
			fmt.Fprintln(os.Stderr)
			fmt.Fprintln(os.Stderr, "\thttp://stedolan.github.io/jq/")
		}
		os.Exit(1)
	}
	if err != nil {
		fmt.Fprintln(os.Stderr, "locating jq:", err)
		os.Exit(1)
	}
	_, err = CheckJQVersion(jqbin)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	// setup initial commands to play before reading input.  single files are
	// loaded with :load, multple files are loaded with :exec cat
	var initcmds [][]string
	doexec := func(cache bool, name string, args ...string) {
		cmd := make([]string, 0, 3+len(args))
		cmd = append(cmd, "exec")
		if !cache {
			cmd = append(cmd, "-c")
		}
		cmd = append(cmd, name)
		cmd = append(cmd, args...)
		initcmds = append(initcmds, cmd)
	}
	switch {
	case len(args) == 1:
		initcmds = [][]string{
			{"load", args[0]},
		}
	case len(args) > 1:
		doexec(false, "cat", args...)
	}

	// create a shell environment and wait for it to receive EOF or a 'quit'
	// command.
	fmt.Println("Welcome to jqsh!")
	fmt.Println()
	fmt.Println("To learn more about the environment type \":help\"")
	fmt.Println()
	fmt.Println("To learn more about jqsh see the online documentation")
	fmt.Println()
	fmt.Println("\thttps://github.com/bmatsuo/jqsh")
	fmt.Println()
	sh := NewInitShellReader(nil, initcmds)
	jq := NewJQShell(jqbin, sh)
	err = jq.Wait()
	if err != nil {
		log.Fatal(err)
	}
}

// Page returns an io.Writer whose input will be written to the pager program.
// The returned channel should be checked for an error using select before the
// writer is used.
//	w, errch := Page("less")
//	select {
//	case err := <-errch:
//		return err
//	default:
//		w.Write([]byte("boom"))
//	}
func Page(pager []string) (io.WriteCloser, <-chan error) {
	errch := make(chan error, 1)
	if len(pager) == 0 {
		pager = []string{"more", "-r"}
	}
	pagercmd := pager[0]
	pagerargs := pager[1:]
	cmd := exec.Command(pagercmd, pagerargs...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	stdin, err := cmd.StdinPipe()
	if err != nil {
		errch <- err
		return nil, errch
	}
	go func() {
		err := cmd.Run()
		stdin.Close()
		if err != nil {
			errch <- err
		}
		close(errch)
		fmt.Print("\033[0m")
	}()
	return stdin, errch
}

type writeCounter struct {
	n int64
	w io.Writer
}

func (w *writeCounter) Write(bs []byte) (int, error) {
	n, err := w.w.Write(bs)
	if n > 0 {
		atomic.AddInt64(&w.n, int64(n))
	}
	return n, err
}

type InvalidCommandError struct {
	Message string
}

func (err InvalidCommandError) Error() string {
	return err.Message
}

type ShellReader interface {
	ReadCommand() (cmd []string, eof bool, err error)
}

type SimpleShellReader struct {
	r  io.Reader
	br *bufio.Reader
}

func NewShellReader(r io.Reader) *SimpleShellReader {
	if r == nil {
		r = os.Stdin
	}
	br := bufio.NewReader(r)
	return &SimpleShellReader{r, br}
}

func (s *SimpleShellReader) ReadCommand() (cmd []string, eof bool, err error) {
	fmt.Print("> ")
	bs, err := s.br.ReadBytes('\n')
	eof = err == io.EOF
	if err != nil {
		if err == io.EOF && len(bs) > 0 {
			// this is ok
		} else {
			return nil, eof, err
		}
	}
	bs = bytes.TrimFunc(bs, unicode.IsSpace)

	if len(bs) == 0 {
		return []string{}, eof, nil
	} else if bs[0] != ':' {
		str := string(bs)
		cmd := []string{"push", str}
		return cmd, eof, nil
	}

	bs = bs[1:]
	plusi := bytes.Index(bs, []byte{'+'})
	var last *[]byte
	if plusi > 0 {
		lastp := bs[plusi+1:]
		last = &lastp
		bs = bs[:plusi]
	}
	cmd = strings.Fields(string(bs))
	if last != nil {
		cmd = append(cmd, string(*last))
	}
	if len(cmd) == 0 {
		cmd = []string{"write"}
	}
	return cmd, eof, nil
}

// An InitShellReader works like a SimpleShellReader but runs an init script
// before reading any input.
type InitShellReader struct {
	i    int
	init [][]string
	r    *SimpleShellReader
}

func NewInitShellReader(r io.Reader, initcmds [][]string) *InitShellReader {
	return &InitShellReader{0, initcmds, NewShellReader(r)}
}

func (sh *InitShellReader) ReadCommand() ([]string, bool, error) {
	if sh == nil {
		panic("nil shell")
	}
	if sh.i < len(sh.init) {
		cmd := sh.init[sh.i]
		sh.i++
		return cmd, false, nil
	}
	return sh.r.ReadCommand()
}

type JQShell struct {
	Log      *log.Logger
	Stack    *JQStack
	bin      string
	inputfn  func() (io.ReadCloser, error)
	filename string
	istmp    bool // the filename at path should be deleted when changed
	lib      *Lib
	sh       ShellReader
	err      error
	wg       sync.WaitGroup
}

func NewJQShell(bin string, sh ShellReader) *JQShell {
	if sh == nil {
		sh = NewShellReader(nil)
	}
	st := new(JQStack)
	jq := &JQShell{
		Log:   log.New(os.Stderr, "jqsh: ", 0),
		Stack: st,
		bin:   bin,
		sh:    sh,
	}
	jq.lib = Library()
	jq.lib.Register("push", JQShellCommandFunc(cmdPush))
	jq.lib.Register("pop", JQShellCommandFunc(cmdPop))
	jq.lib.Register("filter", JQShellCommandFunc(cmdFilter))
	jq.lib.Register("script", JQShellCommandFunc(cmdScript))
	jq.lib.Register("load", JQShellCommandFunc(cmdLoad))
	jq.lib.Register("exec", JQShellCommandFunc(cmdExec))
	jq.lib.Register("write", JQShellCommandFunc(cmdWrite))
	jq.lib.Register("raw", JQShellCommandFunc(cmdRaw))
	jq.lib.Register("quit", JQShellCommandFunc(cmdQuit))
	jq.wg.Add(1)
	go jq.loop()
	return jq
}

func (jq *JQShell) SetInputFile(path string, istmp bool) {
	jq.ClearInput()
	jq.inputfn = nil
	jq.filename = path
	jq.istmp = istmp
}

func (jq *JQShell) SetInput(fn func() (io.ReadCloser, error)) {
	jq.ClearInput()
	jq.inputfn = fn
}

func (jq *JQShell) Input() (io.ReadCloser, error) {
	switch {
	case jq.filename != "":
		jq.Log.Println("open", jq.filename)
		return os.Open(jq.filename)
	case jq.inputfn != nil:
		return jq.inputfn()
	default:
		return nil, fmt.Errorf("no input")
	}
}

func (jq *JQShell) Wait() error {
	jq.wg.Wait()
	return jq.err
}

func isShellExit(err error) bool {
	if err == nil {
		return false
	}
	if err == ShellExit {
		return true
	}
	if err, ok := err.(ExecError); ok {
		return isShellExit(err.err)
	}
	return false
}

func (jq *JQShell) ClearInput() {
	if jq.inputfn != nil {
		jq.inputfn = nil
	}
	if jq.filename != "" && jq.istmp {
		err := os.Remove(jq.filename)
		if err != nil {
			// not a critical error
			jq.Log.Printf("removingtemporary file %v: %v", jq.filename, err)
		}
	}
}

func (jq *JQShell) loop() {
	stop := make(chan struct{})
	_stop := func() { close(stop) }
	ready := make(chan struct{}, 1)
	ready <- struct{}{}
	type cmdin struct {
		cmd []string
		eof bool
		err error
	}
	cmdch := make(chan cmdin)
	for {
		select {
		case <-stop:
			// remove any temporary file
			if jq.filename != "" && jq.istmp {
				err := os.Remove(jq.filename)
				if err != nil {
					// not a critical error
					jq.Log.Printf("removingtemporary file %v: %v", jq.filename, err)
				}
			}
			jq.wg.Done()
			return
		case <-ready:
			go func() {
				cmd, eof, err := jq.sh.ReadCommand()
				cmdch <- cmdin{cmd, eof, err}
			}()
		case cmd := <-cmdch:
			if err, ok := cmd.err.(InvalidCommandError); ok {
				jq.Log.Println(err)
				ready <- struct{}{}
				continue
			}
			go func() {
				err := cmd.err
				if err == io.EOF {
					err = nil
				}
				err = jq.execute(cmd.cmd, err)
				if isShellExit(err) {
					_stop()
					return
				}
				if cmd.eof {
					_stop()
					return
				}
				if err != nil {
					jq.Log.Print(err)
				} else if len(cmd.cmd) == 0 {
					jq.Log.Println("empty command")
				} else if cmd.cmd[0] != "write" && cmd.cmd[0] != "raw" && cmd.cmd[0] != "filter" && cmd.cmd[0] != "script" && cmd.cmd[0] != "help" {
					// TODO clean this up. (cmdPushInteractive, cmdPeek)
					err := jq.execute([]string{"write"}, nil)
					if err != nil {
						jq.Log.Print(err)
						if cmd.cmd[0] == "push" {
							npush := len(cmd.cmd) - 1
							if npush == 0 {
								npush = 1
							}
							jq.Log.Print("reverting push operation")
							err := jq.execute([]string{"pop", fmt.Sprint(npush)}, nil)
							if err != nil {
								jq.Log.Print(err)
							}
						}
					}
				}
				ready <- struct{}{}
			}()
		}
	}
	panic("unreachable")
}

func (jq *JQShell) log(v ...interface{}) {
	jq.Log.Print(v...)
}

func (jq *JQShell) logf(format string, v ...interface{}) {
	jq.Log.Printf(format, v...)
}

type ExecError struct {
	cmd []string
	err error
}

func (err ExecError) Error() string {
	return fmt.Sprintf("%s: %v", err.cmd[0], err.err)
}

func (jq *JQShell) execute(cmd []string, err error) error {
	if isShellExit(err) {
		return err
	}
	if err, ok := err.(InvalidCommandError); ok {
		jq.Log.Print(err)
		return nil
	}
	if err != nil {
		return err
	}
	if len(cmd) > 0 {
		name, args := cmd[0], cmd[1:]
		return jq.lib.Execute(jq, name, args)
	}
	return nil
}
<commit_msg>godoc: link to filter documentation<commit_after>/*
Command jqsh provides an interactive wrapper to the jq command line utility.

Shell syntax

The current shell syntax is rudimentory but it suffices.  Commands are prefixed
with a colon ':' and a command name followed by a space separated list of
arguments.

	> :load test.json

The above loads the file "test.json" into the jqsh cache for inspection.  There
is no quoting of arguments.  A plus '+' may be used on the last argument to
include all charactors up to (but excluding) the next newline character.

	> :push +.items[] | select(.name | contains("hello"))

The above pushes the filter `.items[] | select(.name | contains("hello"))` on
to the jqsh filter stack. This is such a common operation that it has a special
shorthand.  A non-empty line that does not start with a colon causes the line's
contents to be pushed on the filter stack. So the above line could be
simplified.

	> .[] | select(.gender == "Female")

Blank lines are also a shorthand, printing the working filter stack applied to
the input, equivalent to the "write" command.

	> :write

That is all of the syntax in jqsh.

The filter stack

The core concept in jqsh is a stack of jq "filters".  Filters create a larger
filter when joined with pipes "|".  And maintaining a stack of filters allows
exploritory querying of JSON structures.

In the previous section the "push" command was demonstrated.  Pushing filters
will effectively drill down into JSON structures.  To zoom out (or undo) the
application of filters the "pop" command is used.

	> :pop

Called without arguments, "pop" will remove the most recent filter pushed onto
the stack.  Multiple items can be popped off the stack by passing "pop" an
integer argument (the number of items to pop).

	> :pop 3

Read more about jq filters at the tool's online manual.

	http://stedolan.github.io/jq/manual/

Command reference

A list of commands and other interactive help topics can be found through the
"help" command.

	> :help

Individual commands respond to the "-h" flag for usage documentation.
*/
package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"runtime"
	"strings"
	"sync"
	"sync/atomic"
	"unicode"
)

var ErrStackEmpty = fmt.Errorf("the stack is empty")

func main() {
	flag.Parse()
	args := flag.Args()

	jqbin, err := LocateJQ("")
	if err == ErrJQNotFound {
		fmt.Fprintln(os.Stderr, "Unable to locate the jq executable. Make sure it's installed.")
		fmt.Fprintln(os.Stderr)
		switch runtime.GOOS {
		case "darwin":
			fmt.Fprintln(os.Stderr, "The easiest way to install jq on OS X is with homebrew.")
			fmt.Fprintln(os.Stderr)
			fmt.Fprintln(os.Stderr, "\tbrew install jq")
		default:
			fmt.Fprintln(os.Stderr, "See the jq homepage for download and install instructions")
			fmt.Fprintln(os.Stderr)
			fmt.Fprintln(os.Stderr, "\thttp://stedolan.github.io/jq/")
		}
		os.Exit(1)
	}
	if err != nil {
		fmt.Fprintln(os.Stderr, "locating jq:", err)
		os.Exit(1)
	}
	_, err = CheckJQVersion(jqbin)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	// setup initial commands to play before reading input.  single files are
	// loaded with :load, multple files are loaded with :exec cat
	var initcmds [][]string
	doexec := func(cache bool, name string, args ...string) {
		cmd := make([]string, 0, 3+len(args))
		cmd = append(cmd, "exec")
		if !cache {
			cmd = append(cmd, "-c")
		}
		cmd = append(cmd, name)
		cmd = append(cmd, args...)
		initcmds = append(initcmds, cmd)
	}
	switch {
	case len(args) == 1:
		initcmds = [][]string{
			{"load", args[0]},
		}
	case len(args) > 1:
		doexec(false, "cat", args...)
	}

	// create a shell environment and wait for it to receive EOF or a 'quit'
	// command.
	fmt.Println("Welcome to jqsh!")
	fmt.Println()
	fmt.Println("To learn more about the environment type \":help\"")
	fmt.Println()
	fmt.Println("To learn more about jqsh see the online documentation")
	fmt.Println()
	fmt.Println("\thttps://github.com/bmatsuo/jqsh")
	fmt.Println()
	sh := NewInitShellReader(nil, initcmds)
	jq := NewJQShell(jqbin, sh)
	err = jq.Wait()
	if err != nil {
		log.Fatal(err)
	}
}

// Page returns an io.Writer whose input will be written to the pager program.
// The returned channel should be checked for an error using select before the
// writer is used.
//	w, errch := Page("less")
//	select {
//	case err := <-errch:
//		return err
//	default:
//		w.Write([]byte("boom"))
//	}
func Page(pager []string) (io.WriteCloser, <-chan error) {
	errch := make(chan error, 1)
	if len(pager) == 0 {
		pager = []string{"more", "-r"}
	}
	pagercmd := pager[0]
	pagerargs := pager[1:]
	cmd := exec.Command(pagercmd, pagerargs...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	stdin, err := cmd.StdinPipe()
	if err != nil {
		errch <- err
		return nil, errch
	}
	go func() {
		err := cmd.Run()
		stdin.Close()
		if err != nil {
			errch <- err
		}
		close(errch)
		fmt.Print("\033[0m")
	}()
	return stdin, errch
}

type writeCounter struct {
	n int64
	w io.Writer
}

func (w *writeCounter) Write(bs []byte) (int, error) {
	n, err := w.w.Write(bs)
	if n > 0 {
		atomic.AddInt64(&w.n, int64(n))
	}
	return n, err
}

type InvalidCommandError struct {
	Message string
}

func (err InvalidCommandError) Error() string {
	return err.Message
}

type ShellReader interface {
	ReadCommand() (cmd []string, eof bool, err error)
}

type SimpleShellReader struct {
	r  io.Reader
	br *bufio.Reader
}

func NewShellReader(r io.Reader) *SimpleShellReader {
	if r == nil {
		r = os.Stdin
	}
	br := bufio.NewReader(r)
	return &SimpleShellReader{r, br}
}

func (s *SimpleShellReader) ReadCommand() (cmd []string, eof bool, err error) {
	fmt.Print("> ")
	bs, err := s.br.ReadBytes('\n')
	eof = err == io.EOF
	if err != nil {
		if err == io.EOF && len(bs) > 0 {
			// this is ok
		} else {
			return nil, eof, err
		}
	}
	bs = bytes.TrimFunc(bs, unicode.IsSpace)

	if len(bs) == 0 {
		return []string{}, eof, nil
	} else if bs[0] != ':' {
		str := string(bs)
		cmd := []string{"push", str}
		return cmd, eof, nil
	}

	bs = bs[1:]
	plusi := bytes.Index(bs, []byte{'+'})
	var last *[]byte
	if plusi > 0 {
		lastp := bs[plusi+1:]
		last = &lastp
		bs = bs[:plusi]
	}
	cmd = strings.Fields(string(bs))
	if last != nil {
		cmd = append(cmd, string(*last))
	}
	if len(cmd) == 0 {
		cmd = []string{"write"}
	}
	return cmd, eof, nil
}

// An InitShellReader works like a SimpleShellReader but runs an init script
// before reading any input.
type InitShellReader struct {
	i    int
	init [][]string
	r    *SimpleShellReader
}

func NewInitShellReader(r io.Reader, initcmds [][]string) *InitShellReader {
	return &InitShellReader{0, initcmds, NewShellReader(r)}
}

func (sh *InitShellReader) ReadCommand() ([]string, bool, error) {
	if sh == nil {
		panic("nil shell")
	}
	if sh.i < len(sh.init) {
		cmd := sh.init[sh.i]
		sh.i++
		return cmd, false, nil
	}
	return sh.r.ReadCommand()
}

type JQShell struct {
	Log      *log.Logger
	Stack    *JQStack
	bin      string
	inputfn  func() (io.ReadCloser, error)
	filename string
	istmp    bool // the filename at path should be deleted when changed
	lib      *Lib
	sh       ShellReader
	err      error
	wg       sync.WaitGroup
}

func NewJQShell(bin string, sh ShellReader) *JQShell {
	if sh == nil {
		sh = NewShellReader(nil)
	}
	st := new(JQStack)
	jq := &JQShell{
		Log:   log.New(os.Stderr, "jqsh: ", 0),
		Stack: st,
		bin:   bin,
		sh:    sh,
	}
	jq.lib = Library()
	jq.lib.Register("push", JQShellCommandFunc(cmdPush))
	jq.lib.Register("pop", JQShellCommandFunc(cmdPop))
	jq.lib.Register("filter", JQShellCommandFunc(cmdFilter))
	jq.lib.Register("script", JQShellCommandFunc(cmdScript))
	jq.lib.Register("load", JQShellCommandFunc(cmdLoad))
	jq.lib.Register("exec", JQShellCommandFunc(cmdExec))
	jq.lib.Register("write", JQShellCommandFunc(cmdWrite))
	jq.lib.Register("raw", JQShellCommandFunc(cmdRaw))
	jq.lib.Register("quit", JQShellCommandFunc(cmdQuit))
	jq.wg.Add(1)
	go jq.loop()
	return jq
}

func (jq *JQShell) SetInputFile(path string, istmp bool) {
	jq.ClearInput()
	jq.inputfn = nil
	jq.filename = path
	jq.istmp = istmp
}

func (jq *JQShell) SetInput(fn func() (io.ReadCloser, error)) {
	jq.ClearInput()
	jq.inputfn = fn
}

func (jq *JQShell) Input() (io.ReadCloser, error) {
	switch {
	case jq.filename != "":
		jq.Log.Println("open", jq.filename)
		return os.Open(jq.filename)
	case jq.inputfn != nil:
		return jq.inputfn()
	default:
		return nil, fmt.Errorf("no input")
	}
}

func (jq *JQShell) Wait() error {
	jq.wg.Wait()
	return jq.err
}

func isShellExit(err error) bool {
	if err == nil {
		return false
	}
	if err == ShellExit {
		return true
	}
	if err, ok := err.(ExecError); ok {
		return isShellExit(err.err)
	}
	return false
}

func (jq *JQShell) ClearInput() {
	if jq.inputfn != nil {
		jq.inputfn = nil
	}
	if jq.filename != "" && jq.istmp {
		err := os.Remove(jq.filename)
		if err != nil {
			// not a critical error
			jq.Log.Printf("removingtemporary file %v: %v", jq.filename, err)
		}
	}
}

func (jq *JQShell) loop() {
	stop := make(chan struct{})
	_stop := func() { close(stop) }
	ready := make(chan struct{}, 1)
	ready <- struct{}{}
	type cmdin struct {
		cmd []string
		eof bool
		err error
	}
	cmdch := make(chan cmdin)
	for {
		select {
		case <-stop:
			// remove any temporary file
			if jq.filename != "" && jq.istmp {
				err := os.Remove(jq.filename)
				if err != nil {
					// not a critical error
					jq.Log.Printf("removingtemporary file %v: %v", jq.filename, err)
				}
			}
			jq.wg.Done()
			return
		case <-ready:
			go func() {
				cmd, eof, err := jq.sh.ReadCommand()
				cmdch <- cmdin{cmd, eof, err}
			}()
		case cmd := <-cmdch:
			if err, ok := cmd.err.(InvalidCommandError); ok {
				jq.Log.Println(err)
				ready <- struct{}{}
				continue
			}
			go func() {
				err := cmd.err
				if err == io.EOF {
					err = nil
				}
				err = jq.execute(cmd.cmd, err)
				if isShellExit(err) {
					_stop()
					return
				}
				if cmd.eof {
					_stop()
					return
				}
				if err != nil {
					jq.Log.Print(err)
				} else if len(cmd.cmd) == 0 {
					jq.Log.Println("empty command")
				} else if cmd.cmd[0] != "write" && cmd.cmd[0] != "raw" && cmd.cmd[0] != "filter" && cmd.cmd[0] != "script" && cmd.cmd[0] != "help" {
					// TODO clean this up. (cmdPushInteractive, cmdPeek)
					err := jq.execute([]string{"write"}, nil)
					if err != nil {
						jq.Log.Print(err)
						if cmd.cmd[0] == "push" {
							npush := len(cmd.cmd) - 1
							if npush == 0 {
								npush = 1
							}
							jq.Log.Print("reverting push operation")
							err := jq.execute([]string{"pop", fmt.Sprint(npush)}, nil)
							if err != nil {
								jq.Log.Print(err)
							}
						}
					}
				}
				ready <- struct{}{}
			}()
		}
	}
	panic("unreachable")
}

func (jq *JQShell) log(v ...interface{}) {
	jq.Log.Print(v...)
}

func (jq *JQShell) logf(format string, v ...interface{}) {
	jq.Log.Printf(format, v...)
}

type ExecError struct {
	cmd []string
	err error
}

func (err ExecError) Error() string {
	return fmt.Sprintf("%s: %v", err.cmd[0], err.err)
}

func (jq *JQShell) execute(cmd []string, err error) error {
	if isShellExit(err) {
		return err
	}
	if err, ok := err.(InvalidCommandError); ok {
		jq.Log.Print(err)
		return nil
	}
	if err != nil {
		return err
	}
	if len(cmd) > 0 {
		name, args := cmd[0], cmd[1:]
		return jq.lib.Execute(jq, name, args)
	}
	return nil
}
<|endoftext|>