<commit_before>package main

import (
	"bytes"
	"crypto/sha256"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"

	"github.com/fd0/khepri"
)

func hash(filename string) (khepri.ID, error) {
	h := sha256.New()
	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}

	io.Copy(h, f)
	return h.Sum([]byte{}), nil
}

func archive_dir(repo *khepri.Repository, path string) (khepri.ID, error) {
	log.Printf("archiving dir %q", path)

	dir, err := os.Open(path)
	if err != nil {
		log.Printf("open(%q): %v\n", path, err)
		return nil, err
	}

	entries, err := dir.Readdir(-1)
	if err != nil {
		log.Printf("readdir(%q): %v\n", path, err)
		return nil, err
	}

	// use nil ID for empty directories
	if len(entries) == 0 {
		return nil, nil
	}

	t := khepri.NewTree()
	for _, e := range entries {
		node := khepri.NodeFromFileInfo(e)

		var id khepri.ID
		var err error

		if e.IsDir() {
			id, err = archive_dir(repo, filepath.Join(path, e.Name()))
		} else {
			id, err = repo.PutFile(filepath.Join(path, e.Name()))
		}

		node.Content = id

		t.Nodes = append(t.Nodes, node)

		if err != nil {
			log.Printf("  error storing %q: %v\n", e.Name(), err)
			continue
		}
	}

	log.Printf("  dir %q: %v entries", path, len(t.Nodes))

	var buf bytes.Buffer
	t.Save(&buf)
	id, err := repo.PutRaw(khepri.TYPE_BLOB, buf.Bytes())

	if err != nil {
		log.Printf("error saving tree to repo: %v", err)
	}

	log.Printf("tree for %q saved at %s", path, id)

	return id, nil
}

func commandBackup(repo *khepri.Repository, args []string) error {
	if len(args) != 1 {
		return errors.New("usage: backup dir")
	}

	target := args[0]

	id, err := archive_dir(repo, target)
	if err != nil {
		return err
	}

	sn := repo.NewSnapshot(target)
	sn.Tree = id
	sn.Save()

	fmt.Printf("%q archived as %v\n", target, sn.ID())

	return nil
}
<commit_msg>Fix backup command<commit_after>package main

import (
	"crypto/sha256"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"

	"github.com/fd0/khepri"
)

func hash(filename string) (khepri.ID, error) {
	h := sha256.New()
	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}

	io.Copy(h, f)
	return h.Sum([]byte{}), nil
}

func store_file(repo *khepri.Repository, path string) (khepri.ID, error) {
	obj, err := repo.NewObject(khepri.TYPE_BLOB)
	if err != nil {
		return nil, err
	}

	file, err := os.Open(path)
	defer func() {
		file.Close()
	}()

	_, err = io.Copy(obj, file)
	if err != nil {
		return nil, err
	}

	err = obj.Close()
	if err != nil {
		return nil, err
	}

	return obj.ID(), nil
}

func archive_dir(repo *khepri.Repository, path string) (khepri.ID, error) {
	log.Printf("archiving dir %q", path)

	dir, err := os.Open(path)
	if err != nil {
		log.Printf("open(%q): %v\n", path, err)
		return nil, err
	}

	entries, err := dir.Readdir(-1)
	if err != nil {
		log.Printf("readdir(%q): %v\n", path, err)
		return nil, err
	}

	// use nil ID for empty directories
	if len(entries) == 0 {
		return nil, nil
	}

	t := khepri.NewTree()
	for _, e := range entries {
		node := khepri.NodeFromFileInfo(e)

		var id khepri.ID
		var err error

		if e.IsDir() {
			id, err = archive_dir(repo, filepath.Join(path, e.Name()))
		} else {
			id, err = store_file(repo, filepath.Join(path, e.Name()))
		}

		node.Content = id

		t.Nodes = append(t.Nodes, node)

		if err != nil {
			log.Printf("  error storing %q: %v\n", e.Name(), err)
			continue
		}
	}

	log.Printf("  dir %q: %v entries", path, len(t.Nodes))

	obj, err := repo.NewObject(khepri.TYPE_BLOB)

	if err != nil {
		log.Printf("error creating object for tree: %v", err)
		return nil, err
	}

	err = t.Save(obj)
	if err != nil {
		log.Printf("error saving tree to repo: %v", err)
	}

	obj.Close()

	id := obj.ID()
	log.Printf("tree for %q saved at %s", path, id)

	return id, nil
}

func commandBackup(repo *khepri.Repository, args []string) error {
	if len(args) != 1 {
		return errors.New("usage: backup dir")
	}

	target := args[0]

	id, err := archive_dir(repo, target)
	if err != nil {
		return err
	}

	sn := repo.NewSnapshot(target)
	sn.Tree = id
	sn.Save()

	fmt.Printf("%q archived as %v\n", target, sn.ID())

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"github.com/grafana/metrictank/stacktest/graphite"
	"github.com/grafana/metrictank/stats"
	log "github.com/sirupsen/logrus"
	"math"
	"net/http"
	"regexp"
	"strconv"
	"time"
)

type seriesStats struct {
	lastTs uint32
	//the partition currently being checked
	partition int32
	//the number of nans present in the time series
	nans int32
	//the sum of abs(value - ts) across the time series
	deltaSum float64
	//the number of timestamps where value != ts
	numNonMatching int32

	//tracks the last seen non-NaN time stamp (useful for lag
	lastSeen uint32
}

func monitor() {
	for tick := range time.NewTicker(queryInterval).C {

		query := graphite.ExecuteRenderQuery(buildRequest(tick))
		if query.HTTPErr != nil {
			stats.NewCounter32("parrot.monitoring.error;error=http").Inc()
		}
		if query.DecodeErr != nil {
			stats.NewCounter32("parrot.monitoring.error;error=decode").Inc()
		}

		for _, s := range query.Decoded {
			log.Infof("%d - %d", s.Datapoints[0].Ts, s.Datapoints[len(s.Datapoints)-1].Ts)

			partition, err := extractPartition(s.Target)
			if err != nil {
				log.Debug("unable to extract partition", err)
				stats.NewCounter32("parrot.monitoring.error;error=parsePartition").Inc()
				continue
			}
			serStats := seriesStats{}
			serStats.lastTs = s.Datapoints[len(s.Datapoints)-1].Ts

			for _, dp := range s.Datapoints {

				if math.IsNaN(dp.Val) {
					serStats.nans += 1
					continue
				}
				if diff := dp.Val - float64(dp.Ts); diff != 0 {
					log.Debugf("partition=%d dp.Val=%f dp.Ts=%d diff=%f", partition, dp.Val, dp.Ts, diff)
					serStats.lastSeen = dp.Ts
					serStats.deltaSum += diff
					serStats.numNonMatching += 1
				}
			}

			//number of missing values for each series
			stats.NewGauge32(fmt.Sprintf("parrot.monitoring.nancount;partition=%d", serStats.partition)).Set(int(serStats.nans))
			//time since the last value was recorded
			stats.NewGauge32(fmt.Sprintf("parrot.monitoring.lag;partition=%d", serStats.partition)).Set(int(serStats.lastTs - serStats.lastSeen))
			//total amount of drift between expected value and actual values
			stats.NewGauge32(fmt.Sprintf("parrot.monitoring.deltaSum;partition=%d", serStats.partition)).Set(int(serStats.deltaSum))
			//total number of entries where drift occurred
			stats.NewGauge32(fmt.Sprintf("parrot.monitoring.nonMatching;partition=%d", serStats.partition)).Set(int(serStats.numNonMatching))
		}
	}
}

var pattern = regexp.MustCompile(`parrot.testdata.(\d+).generated.\w+`)

func extractPartition(target string) (int32, error) {
	submatch := pattern.FindStringSubmatch(target)
	if len(submatch) < 2 {
		return -1, errors.New(fmt.Sprintf("target [%s] did not match pattern", target))
	}
	partition, err := strconv.Atoi(submatch[1])
	if err != nil {
		return -1, err
	}
	return int32(partition), nil

}

func buildRequest(now time.Time) *http.Request {
	req, _ := http.NewRequest("GET", fmt.Sprintf("%s/render", gatewayAddress), nil)
	q := req.URL.Query()
	q.Set("target", "parrot.testdata.*.generated.*")
	q.Set("from", strconv.FormatInt(now.Add(-5*time.Minute).Unix(), 10))
	q.Set("until", strconv.FormatInt(now.Unix(), 10))
	q.Set("format", "json")
	q.Set("X-Org-Id", strconv.Itoa(orgId))
	req.URL.RawQuery = q.Encode()
	if len(gatewayKey) != 0 {
		req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", gatewayKey))
	}
	return req
}
<commit_msg>FormatInt -> Itoa<commit_after>package main

import (
	"errors"
	"fmt"
	"github.com/grafana/metrictank/stacktest/graphite"
	"github.com/grafana/metrictank/stats"
	log "github.com/sirupsen/logrus"
	"math"
	"net/http"
	"regexp"
	"strconv"
	"time"
)

type seriesStats struct {
	lastTs uint32
	//the partition currently being checked
	partition int32
	//the number of nans present in the time series
	nans int32
	//the sum of abs(value - ts) across the time series
	deltaSum float64
	//the number of timestamps where value != ts
	numNonMatching int32

	//tracks the last seen non-NaN time stamp (useful for lag
	lastSeen uint32
}

func monitor() {
	for tick := range time.NewTicker(queryInterval).C {

		query := graphite.ExecuteRenderQuery(buildRequest(tick))
		if query.HTTPErr != nil {
			stats.NewCounter32("parrot.monitoring.error;error=http").Inc()
		}
		if query.DecodeErr != nil {
			stats.NewCounter32("parrot.monitoring.error;error=decode").Inc()
		}

		for _, s := range query.Decoded {
			log.Infof("%d - %d", s.Datapoints[0].Ts, s.Datapoints[len(s.Datapoints)-1].Ts)

			partition, err := extractPartition(s.Target)
			if err != nil {
				log.Debug("unable to extract partition", err)
				stats.NewCounter32("parrot.monitoring.error;error=parsePartition").Inc()
				continue
			}
			serStats := seriesStats{}
			serStats.lastTs = s.Datapoints[len(s.Datapoints)-1].Ts

			for _, dp := range s.Datapoints {

				if math.IsNaN(dp.Val) {
					serStats.nans += 1
					continue
				}
				if diff := dp.Val - float64(dp.Ts); diff != 0 {
					log.Debugf("partition=%d dp.Val=%f dp.Ts=%d diff=%f", partition, dp.Val, dp.Ts, diff)
					serStats.lastSeen = dp.Ts
					serStats.deltaSum += diff
					serStats.numNonMatching += 1
				}
			}

			//number of missing values for each series
			stats.NewGauge32(fmt.Sprintf("parrot.monitoring.nancount;partition=%d", serStats.partition)).Set(int(serStats.nans))
			//time since the last value was recorded
			stats.NewGauge32(fmt.Sprintf("parrot.monitoring.lag;partition=%d", serStats.partition)).Set(int(serStats.lastTs - serStats.lastSeen))
			//total amount of drift between expected value and actual values
			stats.NewGauge32(fmt.Sprintf("parrot.monitoring.deltaSum;partition=%d", serStats.partition)).Set(int(serStats.deltaSum))
			//total number of entries where drift occurred
			stats.NewGauge32(fmt.Sprintf("parrot.monitoring.nonMatching;partition=%d", serStats.partition)).Set(int(serStats.numNonMatching))
		}
	}
}

var pattern = regexp.MustCompile(`parrot.testdata.(\d+).generated.\w+`)

func extractPartition(target string) (int32, error) {
	submatch := pattern.FindStringSubmatch(target)
	if len(submatch) < 2 {
		return -1, errors.New(fmt.Sprintf("target [%s] did not match pattern", target))
	}
	partition, err := strconv.Atoi(submatch[1])
	if err != nil {
		return -1, err
	}
	return int32(partition), nil

}

func buildRequest(now time.Time) *http.Request {
	req, _ := http.NewRequest("GET", fmt.Sprintf("%s/render", gatewayAddress), nil)
	q := req.URL.Query()
	q.Set("target", "parrot.testdata.*.generated.*")
	q.Set("from", strconv.Itoa(int(now.Add(-5*time.Minute).Unix())))
	q.Set("until", strconv.Itoa(int(now.Unix())))
	q.Set("format", "json")
	q.Set("X-Org-Id", strconv.Itoa(orgId))
	req.URL.RawQuery = q.Encode()
	if len(gatewayKey) != 0 {
		req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", gatewayKey))
	}
	return req
}
<|endoftext|>


<commit_before>// Copyright (c) 2018 Frank Braun <frank@cryptogroup.net>
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

// randpass prints a random password (128-bit, 192-bit, or 256-bit) in base64
// encoding to stdout.
package main

import (
	"crypto/rand"
	"encoding/base64"
	"flag"
	"fmt"
	"io"
	"os"
)

func randPass(size int) (string, error) {
	pass := make([]byte, size)
	if _, err := io.ReadFull(rand.Reader, pass[:]); err != nil {
		return "", err
	}
	return base64.RawStdEncoding.EncodeToString(pass[:]), nil
}

func fatal(err error) {
	fmt.Fprintf(os.Stderr, "%s: error: %s\n", os.Args[0], err)
	os.Exit(1)
}

func main() {
	low := flag.Bool("l", false, "use low security (128-bit)")
	high := flag.Bool("h", false, "use high security (256-bit)")
	flag.Parse()
	size := 24 // 192-bit
	if *low {
		size = 16 // 128-bit
	}
	if *high {
		size = 32 // 256-bit
	}
	pw, err := randPass(size)
	if err != nil {
		fatal(err)
	}
	fmt.Println(pw)
}
<commit_msg>randpass: add -url option to use URL encoding<commit_after>// Copyright (c) 2018 Frank Braun <frank@cryptogroup.net>
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

// randpass prints a random password (128-bit, 192-bit, or 256-bit) in base64
// encoding to stdout.
package main

import (
	"crypto/rand"
	"encoding/base64"
	"flag"
	"fmt"
	"io"
	"os"
)

func randPass(size int, url bool) (string, error) {
	pass := make([]byte, size)
	if _, err := io.ReadFull(rand.Reader, pass[:]); err != nil {
		return "", err
	}
	if url {
		return base64.RawURLEncoding.EncodeToString(pass[:]), nil
	}
	return base64.RawStdEncoding.EncodeToString(pass[:]), nil
}

func fatal(err error) {
	fmt.Fprintf(os.Stderr, "%s: error: %s\n", os.Args[0], err)
	os.Exit(1)
}

func main() {
	low := flag.Bool("l", false, "use low security (128-bit)")
	high := flag.Bool("h", false, "use high security (256-bit)")
	url := flag.Bool("url", false, "use URL encoding")
	flag.Parse()
	size := 24 // 192-bit
	if *low {
		size = 16 // 128-bit
	}
	if *high {
		size = 32 // 256-bit
	}
	pw, err := randPass(size, *url)
	if err != nil {
		fatal(err)
	}
	fmt.Println(pw)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"log"
	"net/url"
	"strconv"
	"time"

	"github.com/taskcluster/generic-worker/livelog"
	"github.com/taskcluster/generic-worker/process"
	"github.com/taskcluster/stateless-dns-go/hostname"
	"github.com/taskcluster/taskcluster-base-go/scopes"
	tcclient "github.com/taskcluster/taskcluster-client-go"
)

type LiveLogFeature struct {
}

func (feature *LiveLogFeature) Name() string {
	return "Live Log"
}

func (feature *LiveLogFeature) Initialise() error {
	return nil
}

// livelog is always enabled
func (feature *LiveLogFeature) IsEnabled(fl EnabledFeatures) bool {
	return true
}

type LiveLogTask struct {
	// The canonical name of the log file as reported to the Queue, which
	// is typically the relative location of the log file to the user home
	// directory
	liveLog *livelog.LiveLog
	task    *TaskRun
}

func (feature *LiveLogFeature) NewTaskFeature(task *TaskRun) TaskFeature {
	return &LiveLogTask{
		task: task,
	}
}

func (l *LiveLogTask) RequiredScopes() scopes.Required {
	// let's not require any scopes, as I see no reason to control access to this feature
	return scopes.Required{}
}

func (l *LiveLogTask) Start() *CommandExecutionError {
	liveLog, err := livelog.New(config.LiveLogExecutable, config.LiveLogCertificate, config.LiveLogKey, config.LiveLogPUTPort, config.LiveLogGETPort)
	if err != nil {
		log.Printf("WARN: could not create livelog: %s", err)
		// then run without livelog, is only a "best effort" service
		return nil
	}
	l.liveLog = liveLog
	l.task.logWriter = io.MultiWriter(liveLog.LogWriter, l.task.logWriter)
	setCommandLogWriters(l.task.Commands, l.task.logWriter)

	err = l.uploadLiveLog()
	if err != nil {
		log.Printf("WARN: could not upload livelog: %s", err)
	}
	return nil
}

func (l *LiveLogTask) Stop() *CommandExecutionError {
	// if livelog couldn't be started, nothing to do here...
	if l.liveLog == nil {
		return nil
	}
	errClose := l.liveLog.LogWriter.Close()
	if errClose != nil {
		// no need to raise an exception
		log.Printf("WARN: could not close livelog writer: %s", errClose)
	}
	errTerminate := l.liveLog.Terminate()
	if errTerminate != nil {
		// no need to raise an exception
		log.Printf("WARN: could not terminate livelog writer: %s", errTerminate)
	}
	log.Print("Redirecting live.log to live_backing.log")
	logURL := fmt.Sprintf("%v/task/%v/runs/%v/artifacts/%v", Queue.BaseURL, l.task.TaskID, l.task.RunID, "public/logs/live_backing.log")
	err := l.task.uploadArtifact(
		&RedirectArtifact{
			BaseArtifact: &BaseArtifact{
				CanonicalPath: "public/logs/live.log",
				Name:          "public/logs/live.log",
				// same expiry as underlying log it points to
				Expires: l.task.Definition.Expires,
			},
			MimeType: "text/plain; charset=utf-8",
			URL:      logURL,
		},
	)
	if err != nil {
		return ResourceUnavailable(err)
	}
	return nil
}

func (l *LiveLogTask) uploadLiveLog() error {
	maxRunTimeDeadline := time.Time(l.task.TaskClaimResponse.Status.Runs[l.task.RunID].Started).Add(time.Duration(l.task.Payload.MaxRunTime) * time.Second)
	// deduce stateless DNS name to use
	statelessHostname := hostname.New(config.PublicIP, config.Subdomain, maxRunTimeDeadline, config.LiveLogSecret)
	getURL, err := url.Parse(l.liveLog.GetURL)
	if err != nil {
		return err
	}
	if l.liveLog.SSLCert != "" && l.liveLog.SSLKey != "" {
		getURL.Scheme = "https"
	} else {
		getURL.Scheme = "http"
	}
	getURL.Host = statelessHostname + ":" + strconv.Itoa(int(l.liveLog.GETPort))
	uploadErr := l.task.uploadArtifact(
		&RedirectArtifact{
			BaseArtifact: &BaseArtifact{
				CanonicalPath: "public/logs/live.log",
				Name:          "public/logs/live.log",
				// livelog expires when task must have completed
				Expires: tcclient.Time(maxRunTimeDeadline),
			},
			MimeType: "text/plain; charset=utf-8",
			URL:      getURL.String(),
		},
	)
	if uploadErr != nil {
		return uploadErr
	}
	// note this will be error(nil) not *CommandExecutionError(nil)
	return nil
}

func setCommandLogWriters(commands []*process.Command, logWriter io.Writer) {
	for i := range commands {
		commands[i].DirectOutput(logWriter)
	}
}
<commit_msg>Enable writing to log after livelog closes, by restoring backing log<commit_after>package main

import (
	"fmt"
	"io"
	"log"
	"net/url"
	"strconv"
	"time"

	"github.com/taskcluster/generic-worker/livelog"
	"github.com/taskcluster/generic-worker/process"
	"github.com/taskcluster/stateless-dns-go/hostname"
	"github.com/taskcluster/taskcluster-base-go/scopes"
	tcclient "github.com/taskcluster/taskcluster-client-go"
)

type LiveLogFeature struct {
}

func (feature *LiveLogFeature) Name() string {
	return "Live Log"
}

func (feature *LiveLogFeature) Initialise() error {
	return nil
}

// livelog is always enabled
func (feature *LiveLogFeature) IsEnabled(fl EnabledFeatures) bool {
	return true
}

type LiveLogTask struct {
	// The canonical name of the log file as reported to the Queue, which
	// is typically the relative location of the log file to the user home
	// directory
	liveLog   *livelog.LiveLog
	task      *TaskRun
	logWriter io.Writer
}

func (feature *LiveLogFeature) NewTaskFeature(task *TaskRun) TaskFeature {
	return &LiveLogTask{
		task: task,
	}
}

func (l *LiveLogTask) RequiredScopes() scopes.Required {
	// let's not require any scopes, as I see no reason to control access to this feature
	return scopes.Required{}
}

func (l *LiveLogTask) Start() *CommandExecutionError {
	liveLog, err := livelog.New(config.LiveLogExecutable, config.LiveLogCertificate, config.LiveLogKey, config.LiveLogPUTPort, config.LiveLogGETPort)
	if err != nil {
		log.Printf("WARN: could not create livelog: %s", err)
		// then run without livelog, is only a "best effort" service
		return nil
	}
	l.liveLog = liveLog
	// store current writer so it can be reinstated later when stopping livelog
	l.logWriter = l.task.logWriter
	l.task.logWriter = io.MultiWriter(liveLog.LogWriter, l.logWriter)
	setCommandLogWriters(l.task.Commands, l.task.logWriter)

	err = l.uploadLiveLog()
	if err != nil {
		log.Printf("WARN: could not upload livelog: %s", err)
	}
	return nil
}

func (l *LiveLogTask) Stop() *CommandExecutionError {
	// if livelog couldn't be started, nothing to do here...
	if l.liveLog == nil {
		return nil
	}
	// reinstate direct log
	l.task.logWriter = l.logWriter
	errClose := l.liveLog.LogWriter.Close()
	if errClose != nil {
		// no need to raise an exception
		log.Printf("WARN: could not close livelog writer: %s", errClose)
	}
	errTerminate := l.liveLog.Terminate()
	if errTerminate != nil {
		// no need to raise an exception
		log.Printf("WARN: could not terminate livelog writer: %s", errTerminate)
	}
	log.Print("Redirecting live.log to live_backing.log")
	logURL := fmt.Sprintf("%v/task/%v/runs/%v/artifacts/%v", Queue.BaseURL, l.task.TaskID, l.task.RunID, "public/logs/live_backing.log")
	err := l.task.uploadArtifact(
		&RedirectArtifact{
			BaseArtifact: &BaseArtifact{
				CanonicalPath: "public/logs/live.log",
				Name:          "public/logs/live.log",
				// same expiry as underlying log it points to
				Expires: l.task.Definition.Expires,
			},
			MimeType: "text/plain; charset=utf-8",
			URL:      logURL,
		},
	)
	if err != nil {
		return ResourceUnavailable(err)
	}
	return nil
}

func (l *LiveLogTask) uploadLiveLog() error {
	maxRunTimeDeadline := time.Time(l.task.TaskClaimResponse.Status.Runs[l.task.RunID].Started).Add(time.Duration(l.task.Payload.MaxRunTime) * time.Second)
	// deduce stateless DNS name to use
	statelessHostname := hostname.New(config.PublicIP, config.Subdomain, maxRunTimeDeadline, config.LiveLogSecret)
	getURL, err := url.Parse(l.liveLog.GetURL)
	if err != nil {
		return err
	}
	if l.liveLog.SSLCert != "" && l.liveLog.SSLKey != "" {
		getURL.Scheme = "https"
	} else {
		getURL.Scheme = "http"
	}
	getURL.Host = statelessHostname + ":" + strconv.Itoa(int(l.liveLog.GETPort))
	uploadErr := l.task.uploadArtifact(
		&RedirectArtifact{
			BaseArtifact: &BaseArtifact{
				CanonicalPath: "public/logs/live.log",
				Name:          "public/logs/live.log",
				// livelog expires when task must have completed
				Expires: tcclient.Time(maxRunTimeDeadline),
			},
			MimeType: "text/plain; charset=utf-8",
			URL:      getURL.String(),
		},
	)
	if uploadErr != nil {
		return uploadErr
	}
	// note this will be error(nil) not *CommandExecutionError(nil)
	return nil
}

func setCommandLogWriters(commands []*process.Command, logWriter io.Writer) {
	for i := range commands {
		commands[i].DirectOutput(logWriter)
	}
}
<|endoftext|>


<commit_before>package log

import (
	"io"
	"log"
	"os"
	"runtime"

	"github.com/anonx/sunplate/internal/github.com/agtorre/gocolorize"
)

// A list of loggers that are used by "sunplate" packages.
var (
	Error *log.Logger
	Warn  *log.Logger
	Info  *log.Logger
	Trace *log.Logger
)

// context stores information about logger output and its color.
type context struct {
	c gocolorize.Colorize
	w io.Writer
}

// Write is used for writing to predefined output using
// foreground color we want.
func (c *context) Write(p []byte) (n int, err error) {
	return c.w.Write([]byte(c.c.Paint(string(p))))
}

func init() {
	// Initialize default loggers.
	Error = log.New(
		&context{
			c: gocolorize.NewColor("red"),
			w: os.Stderr,
		}, "", 0,
	)
	Warn = log.New(
		&context{
			c: gocolorize.NewColor("yellow"),
			w: os.Stderr,
		}, "", 0,
	)
	Info = log.New(
		&context{
			c: gocolorize.NewColor("green"),
			w: os.Stdout,
		}, "", 0,
	)
	Trace = log.New(
		&context{
			c: gocolorize.NewColor("blue"),
			w: os.Stdout,
		}, "", 0,
	)

	// Do not use colorize when on windows.
	if runtime.GOOS == "windows" {
		gocolorize.SetPlain(true)
	}
}
<commit_msg>Update package comment of log<commit_after>// Package log is a simple wrapper under golang standard log
// package and agtorre/gocolorize (that colorizes the result on *nix systems).
// There are four predefined loggers.
// They are Error, Warn, Trace, and Info.
package log

import (
	"io"
	"log"
	"os"
	"runtime"

	"github.com/anonx/sunplate/internal/github.com/agtorre/gocolorize"
)

// A list of loggers that are used by "sunplate" packages.
var (
	Error *log.Logger
	Warn  *log.Logger
	Info  *log.Logger
	Trace *log.Logger
)

// context stores information about logger output and its color.
type context struct {
	c gocolorize.Colorize
	w io.Writer
}

// Write is used for writing to predefined output using
// foreground color we want.
func (c *context) Write(p []byte) (n int, err error) {
	return c.w.Write([]byte(c.c.Paint(string(p))))
}

func init() {
	// Initialize default loggers.
	Error = log.New(
		&context{
			c: gocolorize.NewColor("red"),
			w: os.Stderr,
		}, "", 0,
	)
	Warn = log.New(
		&context{
			c: gocolorize.NewColor("yellow"),
			w: os.Stderr,
		}, "", 0,
	)
	Info = log.New(
		&context{
			c: gocolorize.NewColor("green"),
			w: os.Stdout,
		}, "", 0,
	)
	Trace = log.New(
		&context{
			c: gocolorize.NewColor("blue"),
			w: os.Stdout,
		}, "", 0,
	)

	// Do not use colorize when on windows.
	if runtime.GOOS == "windows" {
		gocolorize.SetPlain(true)
	}
}
<|endoftext|>


<commit_before>package log

import (
	"encoding/json"
	"fmt"
	"io"
	"time"
)

type loggingLevel int

const (
	DEBUG loggingLevel = 1 << iota
	INFO
	CRITICAL
)

var restrictedFields = []string{"level", "time", "message"}

func NewLogger(out io.Writer, level loggingLevel, fields map[string]interface{}) *logger {
	return &logger{out, fields, level}
}

type logger struct {
	out io.Writer
	// List of default fields that will be serialized
	// along with the message and written to `out`
	defaultFields map[string]interface{}
	level         loggingLevel
}

func (l *logger) Debug(message string, fields map[string]interface{}) {
	if l.level > DEBUG {
		return
	}
	f := l.createMessage(message, fields)
	f["level"] = "debug"
	l.Write(f)
}

func (l *logger) Info(message string, fields map[string]interface{}) {
	if l.level > INFO {
		return
	}
	f := l.createMessage(message, fields)
	f["level"] = "info"
	l.Write(f)
}

func (l *logger) Critical(message string, fields map[string]interface{}) {
	f := l.createMessage(message, fields)
	f["level"] = "critical"
	f["message"] = fmt.Sprintf("[alert-operator] %s", f["message"])
	l.Write(f)
}
func (l *logger) Write(message map[string]interface{}) {
	// ignore if there is an error, logging failure should not cause something fatal
	if output, err := json.Marshal(message); err == nil {
		l.out.Write(output)
		l.out.Write([]byte("\n"))
	}
}

func (l *logger) createMessage(message string, fields map[string]interface{}) map[string]interface{} {
	f := map[string]interface{}{
		"message": message,
		"time":    time.Now().Unix(),
	}
	for k, v := range fields {
		fieldName := k
		if contains(restrictedFields, k) {
			fieldName = fmt.Sprintf("field_%s", k)
		}
		f[fieldName] = v
	}

	for k, v := range l.defaultFields {
		f[k] = v
	}

	return f
}

func contains(list []string, value interface{}) bool {
	for _, v := range list {
		if v == value {
			return true
		}
	}
	return false
}
<commit_msg>Add mutex and doc strings<commit_after>package log

import (
	"encoding/json"
	"fmt"
	"io"
	"sync"
	"time"
)

type loggingLevel int

// Log messages can have one of 3 possible states.
const (
	DEBUG loggingLevel = 1 << iota
	INFO
	CRITICAL
)

// List of fields that should not be overwritten by fields specified when logging
// a message
var restrictedFields = []string{"level", "time", "message"}

// Creates a new logging instance
func NewLogger(out io.Writer, level loggingLevel, fields map[string]interface{}) *logger {
	return &logger{
		out:           out,
		defaultFields: fields,
		level:         level,
	}
}

type logger struct {
	// Create a lock when writing to `out` so output is not intermingled
	mu  sync.Mutex
	out io.Writer
	// List of default fields that will be serialized
	// along with the message and written to `out`
	defaultFields map[string]interface{}
	level         loggingLevel
}

// Log a debug message as long as the logger's debug level is set to at least Debug
func (l *logger) Debug(message string, fields map[string]interface{}) {
	if l.level > DEBUG {
		return
	}
	f := l.createMessage(message, fields)
	f["level"] = "debug"
	l.Write(f)
}

// Log an informational message as long as the logger's debug level is set to at least Info
func (l *logger) Info(message string, fields map[string]interface{}) {
	if l.level > INFO {
		return
	}
	f := l.createMessage(message, fields)
	f["level"] = "info"
	l.Write(f)
}

// Log a critical message.  Critical Messages will have "[alert-operator]" prepended to them
// for alerting purposes.
func (l *logger) Critical(message string, fields map[string]interface{}) {
	f := l.createMessage(message, fields)
	f["level"] = "critical"
	f["message"] = fmt.Sprintf("[alert-operator] %s", f["message"])
	l.Write(f)
}
func (l *logger) Write(message map[string]interface{}) {
	// ignore if there is an error, logging failure should not cause something fatal
	if output, err := json.Marshal(message); err == nil {
		l.mu.Lock()
		defer l.mu.Unlock()
		if _, err := l.out.Write(output); err != nil {
			fmt.Println("Error writing to out")
		}
		if _, err := l.out.Write([]byte("\n")); err != nil {
			fmt.Println("Error writing to out")
		}
	}
}

// Creates a map with the default fields along with any fields that were added at
// the time of logging the message.
func (l *logger) createMessage(message string, fields map[string]interface{}) map[string]interface{} {
	f := map[string]interface{}{
		"message": message,
		"time":    time.Now().Unix(),
	}
	for k, v := range fields {
		fieldName := k
		if contains(restrictedFields, k) {
			fieldName = fmt.Sprintf("field_%s", k)
		}
		f[fieldName] = v
	}

	for k, v := range l.defaultFields {
		f[k] = v
	}

	return f
}

func contains(list []string, value interface{}) bool {
	for _, v := range list {
		if v == value {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package buffer

import (
	"fmt"
	"regexp"
	"strings"
	"sync"
)

func (buffer *Buffer) Justify(startRow, endRow, lineLen int, comStrs []string) error {

	// Get indents and remainders.
	subBuffer := buffer.InclSlice(startRow, endRow)
	indents, remainders, err := subBuffer.getIndentsRemainders(comStrs)
	if err != nil {
		return err
	}

	// Find paragraphs.
	paragraphs, err := findParagraphs(indents, remainders)
	if err != nil {
		return fmt.Errorf("error dividing text into paragraphs: %s", err)
	}

	// Justify each paragraph and append to a buffer.
	fBuffer := MakeBuffer([]string{})
	for i, p := range paragraphs {
		start := p
		end := subBuffer.Length() - 1
		if i+1 < len(paragraphs) {
			end = paragraphs[i+1] - 1
		}
		bigStr := strings.Join(remainders[start:end+1], " ")
		splitBuf := MakeSplitBuffer(bigStr, lineLen-len(indents[p]))

		// Put indentation back
		splitBuf.IndentByStr(indents[p], 0, -1)

		fBuffer.Append(splitBuf.Lines()...)
	}

	// Replace the original buffer lines with the justified ones.
	buffer.ReplaceLines(fBuffer.Lines(), startRow, endRow)

	return nil

}

func (buffer *Buffer) justify(lineLen int) *Buffer {

	// Make one long string out of the remainder of the lines.
	bigStr := buffer.ToString(" ")

	// Justify.
	splitBuf := MakeSplitBuffer(bigStr, lineLen)

	return &splitBuf

}

func (buffer *Buffer) getIndentsRemainders(comStrs []string) (indents, remainders []string, err error) {

	indents = make([]string, buffer.Length())
	remainders = make([]string, buffer.Length())

	// Construct the regexp to find indentation (including comments strings).
	pattern := "^[ \t]*(//|#)*[ \t]*"
	re, err := regexp.Compile(pattern)
	if err != nil {
		return indents, remainders, err
	}

	// Separete lines into indent + remainder.
	for i, line := range buffer.Lines() {
		lineStr := line.ToString()
		indent := re.FindString(lineStr)
		indents[i] = indent
		remainders[i] = lineStr[len(indent):]
	}

	return indents, remainders, nil

}

// Separate lines of text into paragraphs based on indentation.
// Return a list of paragraph starts.
func findParagraphs(indents []string, remainders []string) ([]int, error) {
	paragraphs := []int{0}

	// Sanity check
	if len(indents) != len(remainders) {
		return paragraphs, fmt.Errorf("indents and remainders are different lengths")
	}

	// Trivial case
	if len(indents) == 0 {
		return paragraphs, nil
	}

	lastIndent := indents[0]
	lastRemainder := remainders[0]
	for i, indent := range indents {
		remainder := remainders[i]
		if indent != lastIndent {
			// If indentation changes, then start a new paragraph.
			paragraphs = append(paragraphs, i)
		} else if remainder != "" && lastRemainder == "" {
			paragraphs = append(paragraphs, i)
		} else if remainder == "" && lastRemainder != "" {
			paragraphs = append(paragraphs, i)
		}
		lastRemainder = remainder
		lastIndent = indent
	}
	return paragraphs, nil
}

func (buffer *Buffer) getParagraphs() [][]int {
	offsets := [][]int{}
	start := 0
	prevIsBlank := true
	for i, line := range buffer.Lines() {
		// Blank line after non blanks.
		if line.Length() == 0 && !prevIsBlank && i > 0 {
			offsets = append(offsets, []int{start, i - 1})
		}
		// Non blank line after a blank.
		if line.Length() > 0 && prevIsBlank {
			start = i
		}
		prevIsBlank = line.Length() == 0
	}
	if !prevIsBlank {
		offsets = append(offsets, []int{start, buffer.Length() - 1})
	}
	return offsets
}

// MakeSplitBuffer creates a buffer from a long string by splitting
// the string at a certain length.
func MakeSplitBuffer(bigString string, lineLen int) Buffer {
	words := strings.Fields(bigString)
	if len(words) == 0 {
		return MakeBuffer([]string{""})
	}
	lines := []Line{}
	lineStr := words[0]
	for _, word := range words[1:] {
		if lineLen > 0 && len(lineStr)+len(word) > lineLen {
			lines = append(lines, MakeLine(lineStr))
			lineStr = word
		} else {
			lineStr += " " + word
		}
	}
	lines = append(lines, MakeLine(lineStr))
	return Buffer{
		lines: lines,
		mutex: &sync.Mutex{},
	}
}
<commit_msg>bugfix: justify was one char too long<commit_after>package buffer

import (
	"fmt"
	"regexp"
	"strings"
	"sync"
)

func (buffer *Buffer) Justify(startRow, endRow, lineLen int, comStrs []string) error {

	// Get indents and remainders.
	subBuffer := buffer.InclSlice(startRow, endRow)
	indents, remainders, err := subBuffer.getIndentsRemainders(comStrs)
	if err != nil {
		return err
	}

	// Find paragraphs.
	paragraphs, err := findParagraphs(indents, remainders)
	if err != nil {
		return fmt.Errorf("error dividing text into paragraphs: %s", err)
	}

	// Justify each paragraph and append to a buffer.
	fBuffer := MakeBuffer([]string{})
	for i, p := range paragraphs {
		start := p
		end := subBuffer.Length() - 1
		if i+1 < len(paragraphs) {
			end = paragraphs[i+1] - 1
		}
		bigStr := strings.Join(remainders[start:end+1], " ")
		splitBuf := MakeSplitBuffer(bigStr, lineLen-len(indents[p]))

		// Put indentation back
		splitBuf.IndentByStr(indents[p], 0, -1)

		fBuffer.Append(splitBuf.Lines()...)
	}

	// Replace the original buffer lines with the justified ones.
	buffer.ReplaceLines(fBuffer.Lines(), startRow, endRow)

	return nil

}

func (buffer *Buffer) justify(lineLen int) *Buffer {

	// Make one long string out of the remainder of the lines.
	bigStr := buffer.ToString(" ")

	// Justify.
	splitBuf := MakeSplitBuffer(bigStr, lineLen)

	return &splitBuf

}

func (buffer *Buffer) getIndentsRemainders(comStrs []string) (indents, remainders []string, err error) {

	indents = make([]string, buffer.Length())
	remainders = make([]string, buffer.Length())

	// Construct the regexp to find indentation (including comments strings).
	pattern := "^[ \t]*(//|#)*[ \t]*"
	re, err := regexp.Compile(pattern)
	if err != nil {
		return indents, remainders, err
	}

	// Separete lines into indent + remainder.
	for i, line := range buffer.Lines() {
		lineStr := line.ToString()
		indent := re.FindString(lineStr)
		indents[i] = indent
		remainders[i] = lineStr[len(indent):]
	}

	return indents, remainders, nil

}

// Separate lines of text into paragraphs based on indentation.
// Return a list of paragraph starts.
func findParagraphs(indents []string, remainders []string) ([]int, error) {
	paragraphs := []int{0}

	// Sanity check
	if len(indents) != len(remainders) {
		return paragraphs, fmt.Errorf("indents and remainders are different lengths")
	}

	// Trivial case
	if len(indents) == 0 {
		return paragraphs, nil
	}

	lastIndent := indents[0]
	lastRemainder := remainders[0]
	for i, indent := range indents {
		remainder := remainders[i]
		if indent != lastIndent {
			// If indentation changes, then start a new paragraph.
			paragraphs = append(paragraphs, i)
		} else if remainder != "" && lastRemainder == "" {
			paragraphs = append(paragraphs, i)
		} else if remainder == "" && lastRemainder != "" {
			paragraphs = append(paragraphs, i)
		}
		lastRemainder = remainder
		lastIndent = indent
	}
	return paragraphs, nil
}

func (buffer *Buffer) getParagraphs() [][]int {
	offsets := [][]int{}
	start := 0
	prevIsBlank := true
	for i, line := range buffer.Lines() {
		// Blank line after non blanks.
		if line.Length() == 0 && !prevIsBlank && i > 0 {
			offsets = append(offsets, []int{start, i - 1})
		}
		// Non blank line after a blank.
		if line.Length() > 0 && prevIsBlank {
			start = i
		}
		prevIsBlank = line.Length() == 0
	}
	if !prevIsBlank {
		offsets = append(offsets, []int{start, buffer.Length() - 1})
	}
	return offsets
}

// MakeSplitBuffer creates a buffer from a long string by splitting
// the string at a certain length.
func MakeSplitBuffer(bigString string, lineLen int) Buffer {
	words := strings.Fields(bigString)
	if len(words) == 0 {
		return MakeBuffer([]string{""})
	}
	lines := []Line{}
	lineStr := words[0]
	for _, word := range words[1:] {
		if lineLen > 0 && len(lineStr)+len(word) >= lineLen {
			lines = append(lines, MakeLine(lineStr))
			lineStr = word
		} else {
			lineStr += " " + word
		}
	}
	lines = append(lines, MakeLine(lineStr))
	return Buffer{
		lines: lines,
		mutex: &sync.Mutex{},
	}
}
<|endoftext|>


<commit_before>package lua

/*
use
	Win32_dllw4: DLL and import library built with MingW gcc 4.3,
	creates dependency with MSVCRT.DLL
from http://sourceforge.net/projects/luabinaries/files/5.2.3/Windows%20Libraries/Dynamic/
*/

/*
#cgo CFLAGS: -I ./include
#cgo LDFLAGS: -llua52 -L.
#include "stdlib.h"
#include "lua.h"
#include "lualib.h"
#include "lauxlib.h"

static int gLua_pcall(lua_State* L,int x,int y,int z)
{ return lua_pcall(L,x,y,z); }

static int gLuaL_loadfile(lua_State* L,const char *filename)
{ return luaL_loadfile(L,filename); }

static int gLua_isfunction(lua_State* L,int i)
{ return lua_isfunction(L,i); }

static void gLua_pop(lua_State* L,int n)
{ lua_pop(L,n); }

extern int luaToGoBridge(lua_State *);
static void gLua_pushbridge(lua_State*L)
{ return lua_pushcfunction(L,luaToGoBridge);}
*/
import "C"

import "errors"
import "fmt"
import "unsafe"

type Lua struct {
	lua *C.lua_State
}

func (this *Lua) Id() uintptr {
	return uintptr(unsafe.Pointer(this.lua))
}

const REGISTORYINDEX = C.LUA_REGISTRYINDEX

type goFunctionT struct {
	function func(*Lua) int
}

func New() *Lua {
	return &Lua{C.luaL_newstate()}
}

func (this *Lua) OpenLibs() {
	C.luaL_openlibs(this.lua)
}

func (this *Lua) Close() {
	C.lua_close(this.lua)
}

func (this *Lua) ToString(index int) (string, error) {
	if !this.IsString(index) {
		return "", fmt.Errorf("Lua.ToString(%d): Not String", index)
	}
	var length C.size_t
	p := C.lua_tolstring(this.lua, C.int(index), &length)
	if p == nil {
		return "", fmt.Errorf("Lua.ToString(%d): Empty", index)
	} else {
		return C.GoStringN(p, C.int(length)), nil
	}
}

func (this *Lua) ToAnsiString(index int) []byte {
	var length C.size_t
	p := C.lua_tolstring(this.lua, C.int(index), &length)
	if p == nil || length <= 0 {
		return []byte{}
	} else {
		return C.GoBytes(unsafe.Pointer(p), C.int(length))
	}
}

func (this *Lua) IsString(index int) bool {
	return C.lua_isstring(this.lua, C.int(index)) != 0
}

func (this *Lua) IsFunction(index int) bool {
	return C.gLua_isfunction(this.lua, C.int(index)) != 0
}

const (
	TNIL           = int(C.LUA_TNIL)
	TNUMBER        = int(C.LUA_TNUMBER)
	TBOOLEAN       = int(C.LUA_TBOOLEAN)
	TSTRING        = int(C.LUA_TSTRING)
	TTABLE         = int(C.LUA_TTABLE)
	TFUNCTION      = int(C.LUA_TFUNCTION)
	TUSERDATA      = int(C.LUA_TUSERDATA)
	TTHREAD        = int(C.LUA_TTHREAD)
	TLIGHTUSERDATA = int(C.LUA_TLIGHTUSERDATA)
)

func (this *Lua) GetType(index int) int {
	return int(C.lua_type(this.lua, C.int(index)))
}

func (this *Lua) PushValue(index int) {
	C.lua_pushvalue(this.lua, C.int(index))
}

func (this *Lua) Load(fname string) error {
	if C.gLuaL_loadfile(this.lua, C.CString(fname)) != 0 {
		msg, err := this.ToString(-1)
		if err == nil {
			return fmt.Errorf("%s: %s", fname, msg)
		} else {
			return err
		}
	}
	return nil
}

func (this *Lua) Call(nargs, nresult int) error {
	if C.gLua_pcall(this.lua, C.int(nargs), C.int(nresult), 0) != 0 {
		if this.IsString(-1) {
			msg, err := this.ToString(-1)
			if err == nil {
				return errors.New(msg)
			} else {
				return err
			}
		} else {
			return errors.New("<Lua Error>")
		}
	}
	return nil
}

func (this *Lua) Source(fname string) error {
	if err := this.Load(fname); err != nil {
		return err
	}
	return this.Call(0, 0)
}

func (this *Lua) NewTable() {
	C.lua_createtable(this.lua, 0, 0)
}

func (this *Lua) PushString(str string) {
	cstr := C.CString(str)
	C.lua_pushstring(this.lua, cstr)
	C.free(unsafe.Pointer(cstr))
}

func (this *Lua) PushAnsiString(data []byte) {
	if data != nil && len(data) > 0 {
		C.lua_pushlstring(this.lua,
			(*C.char)(unsafe.Pointer(&data[0])),
			C.size_t(len(data)))
	} else {
		this.PushString("")
	}
}

func (this *Lua) PushInteger(n int) {
	C.lua_pushinteger(this.lua, C.lua_Integer(n))
}

//export luaToGoBridge
func luaToGoBridge(lua *C.lua_State) int {
	f := *(*goFunctionT)(C.lua_touserdata(lua, 1))
	C.lua_remove(lua, 1)
	L := Lua{lua}
	return int(f.function(&L))
}

func (this *Lua) SetTable(index int) {
	C.lua_settable(this.lua, C.int(index))
}

func (this *Lua) GetTable(index int) {
	C.lua_gettable(this.lua, C.int(index))
}

func (this *Lua) SetField(index int, str string) {
	cstr := C.CString(str)
	C.lua_setfield(this.lua, C.int(index), cstr)
	C.free(unsafe.Pointer(cstr))
}

func (this *Lua) GetField(index int, str string) {
	cstr := C.CString(str)
	C.lua_getfield(this.lua, C.int(index), cstr)
	C.free(unsafe.Pointer(cstr))
}

func (this *Lua) SetGlobal(str string) {
	cstr := C.CString(str)
	C.lua_setglobal(this.lua, cstr)
	C.free(unsafe.Pointer(cstr))
}

func (this *Lua) GetGlobal(str string) {
	cstr := C.CString(str)
	C.lua_getglobal(this.lua, cstr)
	C.free(unsafe.Pointer(cstr))
}

func (this *Lua) SetMetaTable(index int) {
	C.lua_setmetatable(this.lua, C.int(index))
}

func (this *Lua) PushGoFunction(f func(L *Lua) int) {
	f_ := goFunctionT{f}
	voidptr := C.lua_newuserdata(this.lua, C.size_t(unsafe.Sizeof(f_)))
	*(*goFunctionT)(voidptr) = f_
	this.NewTable()
	C.gLua_pushbridge(this.lua)
	this.SetField(-2, "__call")
	this.SetMetaTable(-2)
}

func (this *Lua) GetTop() int {
	return int(C.lua_gettop(this.lua))
}

func (this *Lua) SetTop(index int) {
	C.lua_settop(this.lua, C.int(index))
}

func (this *Lua) Pop(n int) {
	C.gLua_pop(this.lua, C.int(n))
}

func (this *Lua) PushLightUserData(p unsafe.Pointer) {
	C.lua_pushlightuserdata(this.lua, p)
}

func (this *Lua) ToUserData(index int) unsafe.Pointer {
	return C.lua_touserdata(this.lua, C.int(index))
}

func (this *Lua) PushNil() {
	C.lua_pushnil(this.lua)
}

func (this *Lua) PushBool(value bool) {
	if value {
		C.lua_pushboolean(this.lua, 1)
	} else {
		C.lua_pushboolean(this.lua, 0)
	}
}

func (this *Lua) ToBool(index int) bool {
	if C.lua_toboolean(this.lua, C.int(index)) != 0 {
		return true
	} else {
		return false
	}
}
<commit_msg>lua\lua.go: ToInteger() を追加<commit_after>package lua

/*
use
	Win32_dllw4: DLL and import library built with MingW gcc 4.3,
	creates dependency with MSVCRT.DLL
from http://sourceforge.net/projects/luabinaries/files/5.2.3/Windows%20Libraries/Dynamic/
*/

/*
#cgo CFLAGS: -I ./include
#cgo LDFLAGS: -llua52 -L.
#include "stdlib.h"
#include "lua.h"
#include "lualib.h"
#include "lauxlib.h"

static int gLua_pcall(lua_State* L,int x,int y,int z)
{ return lua_pcall(L,x,y,z); }

static int gLuaL_loadfile(lua_State* L,const char *filename)
{ return luaL_loadfile(L,filename); }

static int gLua_isfunction(lua_State* L,int i)
{ return lua_isfunction(L,i); }

static void gLua_pop(lua_State* L,int n)
{ lua_pop(L,n); }

extern int luaToGoBridge(lua_State *);
static void gLua_pushbridge(lua_State*L)
{ return lua_pushcfunction(L,luaToGoBridge);}
*/
import "C"

import "errors"
import "fmt"
import "unsafe"

type Lua struct {
	lua *C.lua_State
}

func (this *Lua) Id() uintptr {
	return uintptr(unsafe.Pointer(this.lua))
}

const REGISTORYINDEX = C.LUA_REGISTRYINDEX

type goFunctionT struct {
	function func(*Lua) int
}

func New() *Lua {
	return &Lua{C.luaL_newstate()}
}

func (this *Lua) OpenLibs() {
	C.luaL_openlibs(this.lua)
}

func (this *Lua) Close() {
	C.lua_close(this.lua)
}

func (this *Lua) ToString(index int) (string, error) {
	if !this.IsString(index) {
		return "", fmt.Errorf("Lua.ToString(%d): Not String", index)
	}
	var length C.size_t
	p := C.lua_tolstring(this.lua, C.int(index), &length)
	if p == nil {
		return "", fmt.Errorf("Lua.ToString(%d): Empty", index)
	} else {
		return C.GoStringN(p, C.int(length)), nil
	}
}

func (this *Lua) ToAnsiString(index int) []byte {
	var length C.size_t
	p := C.lua_tolstring(this.lua, C.int(index), &length)
	if p == nil || length <= 0 {
		return []byte{}
	} else {
		return C.GoBytes(unsafe.Pointer(p), C.int(length))
	}
}

func (this *Lua) IsString(index int) bool {
	return C.lua_isstring(this.lua, C.int(index)) != 0
}

func (this *Lua) IsFunction(index int) bool {
	return C.gLua_isfunction(this.lua, C.int(index)) != 0
}

const (
	TNIL           = int(C.LUA_TNIL)
	TNUMBER        = int(C.LUA_TNUMBER)
	TBOOLEAN       = int(C.LUA_TBOOLEAN)
	TSTRING        = int(C.LUA_TSTRING)
	TTABLE         = int(C.LUA_TTABLE)
	TFUNCTION      = int(C.LUA_TFUNCTION)
	TUSERDATA      = int(C.LUA_TUSERDATA)
	TTHREAD        = int(C.LUA_TTHREAD)
	TLIGHTUSERDATA = int(C.LUA_TLIGHTUSERDATA)
)

func (this *Lua) GetType(index int) int {
	return int(C.lua_type(this.lua, C.int(index)))
}

func (this *Lua) PushValue(index int) {
	C.lua_pushvalue(this.lua, C.int(index))
}

func (this *Lua) Load(fname string) error {
	if C.gLuaL_loadfile(this.lua, C.CString(fname)) != 0 {
		msg, err := this.ToString(-1)
		if err == nil {
			return fmt.Errorf("%s: %s", fname, msg)
		} else {
			return err
		}
	}
	return nil
}

func (this *Lua) Call(nargs, nresult int) error {
	if C.gLua_pcall(this.lua, C.int(nargs), C.int(nresult), 0) != 0 {
		if this.IsString(-1) {
			msg, err := this.ToString(-1)
			if err == nil {
				return errors.New(msg)
			} else {
				return err
			}
		} else {
			return errors.New("<Lua Error>")
		}
	}
	return nil
}

func (this *Lua) Source(fname string) error {
	if err := this.Load(fname); err != nil {
		return err
	}
	return this.Call(0, 0)
}

func (this *Lua) NewTable() {
	C.lua_createtable(this.lua, 0, 0)
}

func (this *Lua) PushString(str string) {
	cstr := C.CString(str)
	C.lua_pushstring(this.lua, cstr)
	C.free(unsafe.Pointer(cstr))
}

func (this *Lua) PushAnsiString(data []byte) {
	if data != nil && len(data) > 0 {
		C.lua_pushlstring(this.lua,
			(*C.char)(unsafe.Pointer(&data[0])),
			C.size_t(len(data)))
	} else {
		this.PushString("")
	}
}

func (this *Lua) PushInteger(n int) {
	C.lua_pushinteger(this.lua, C.lua_Integer(n))
}

func (this *Lua) ToInteger(index int) (int, error) {
	var issucceeded C.int
	value := C.lua_tointegerx(this.lua, C.int(index), &issucceeded)
	if issucceeded != 0 {
		return int(value), nil
	} else {
		return 0, errors.New("ToInteger: the value in not integer on the stack")
	}
}

//export luaToGoBridge
func luaToGoBridge(lua *C.lua_State) int {
	f := *(*goFunctionT)(C.lua_touserdata(lua, 1))
	C.lua_remove(lua, 1)
	L := Lua{lua}
	return int(f.function(&L))
}

func (this *Lua) SetTable(index int) {
	C.lua_settable(this.lua, C.int(index))
}

func (this *Lua) GetTable(index int) {
	C.lua_gettable(this.lua, C.int(index))
}

func (this *Lua) SetField(index int, str string) {
	cstr := C.CString(str)
	C.lua_setfield(this.lua, C.int(index), cstr)
	C.free(unsafe.Pointer(cstr))
}

func (this *Lua) GetField(index int, str string) {
	cstr := C.CString(str)
	C.lua_getfield(this.lua, C.int(index), cstr)
	C.free(unsafe.Pointer(cstr))
}

func (this *Lua) SetGlobal(str string) {
	cstr := C.CString(str)
	C.lua_setglobal(this.lua, cstr)
	C.free(unsafe.Pointer(cstr))
}

func (this *Lua) GetGlobal(str string) {
	cstr := C.CString(str)
	C.lua_getglobal(this.lua, cstr)
	C.free(unsafe.Pointer(cstr))
}

func (this *Lua) SetMetaTable(index int) {
	C.lua_setmetatable(this.lua, C.int(index))
}

func (this *Lua) PushGoFunction(f func(L *Lua) int) {
	f_ := goFunctionT{f}
	voidptr := C.lua_newuserdata(this.lua, C.size_t(unsafe.Sizeof(f_)))
	*(*goFunctionT)(voidptr) = f_
	this.NewTable()
	C.gLua_pushbridge(this.lua)
	this.SetField(-2, "__call")
	this.SetMetaTable(-2)
}

func (this *Lua) GetTop() int {
	return int(C.lua_gettop(this.lua))
}

func (this *Lua) SetTop(index int) {
	C.lua_settop(this.lua, C.int(index))
}

func (this *Lua) Pop(n int) {
	C.gLua_pop(this.lua, C.int(n))
}

func (this *Lua) PushLightUserData(p unsafe.Pointer) {
	C.lua_pushlightuserdata(this.lua, p)
}

func (this *Lua) ToUserData(index int) unsafe.Pointer {
	return C.lua_touserdata(this.lua, C.int(index))
}

func (this *Lua) PushNil() {
	C.lua_pushnil(this.lua)
}

func (this *Lua) PushBool(value bool) {
	if value {
		C.lua_pushboolean(this.lua, 1)
	} else {
		C.lua_pushboolean(this.lua, 0)
	}
}

func (this *Lua) ToBool(index int) bool {
	if C.lua_toboolean(this.lua, C.int(index)) != 0 {
		return true
	} else {
		return false
	}
}
<|endoftext|>


<commit_before>package monitor

import (
	"bufio"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	_ "io/ioutil"
	"log"
	"os"
	"os/exec"
	"regexp"
	"strings"
)

type ipdata struct {
	Epoch       string `json:"Epoch"`
	SourceIP    string `json:"SourceIP"`
	SourcePort  string `json:"SourcePort"`
	ReceiveIP   string `json:"ReceiveIP"`
	ReceivePort string `json:"ReceivePort"`
}

/* The Monitor interface defines a series of methods which will be defined on
 * monitor structs. The Start method takes a channel to send messages over,
 * back to the configurator. The Stop method kills the process which is
 * performing the actual monitoring.
 */
type Monitor interface {
	Start(messages chan<- []byte, dockerComposeName string)
	Stop()
}

/* Store the MonitorName, which is the name of the program to execute, the
 * DockerDirs which are the directories to monitor which our Docker containers
 * of interest live in, and a pointer to the exec.Cmd struct which describes
 * the running command.
 */
type FSMonitor struct {
	MonitorName   string
	DockerDirs    []string
	fsWatcherProc *exec.Cmd
}

type NetMonitor struct {
	MonitorName  string
	ContainerIds []string
}

// Memoize this. It's kind of expensive to get.
var dockerContainerIds = []string{}

func runCommandAndSlurpOutput(commandname string, args []string) ([]string, error) {
	command := exec.Command(commandname, args...)
	fmt.Print("running the command: ")
	fmt.Println(commandname, args)
	stdout, err := command.StdoutPipe()
	if err != nil {
		return nil, err
	}
	stderr, err := command.StderrPipe()
	if err != nil {
		return nil, err
	}

	go io.Copy(os.Stderr, stderr)
	command.Start()
	defer command.Wait()

	output := []string{}
	stdoutreader := bufio.NewReader(stdout)
	slurp := true
	for slurp {
		fetch := true
		line := []byte{}
		for fetch {
			partial_line, f, err := stdoutreader.ReadLine()
			fetch = f
			line = append(line, partial_line...)
			if err == io.EOF {
				slurp = false
				break
			} else if err != nil {
				return nil, err
			}
		}
		if len(line) > 0 {
			output = append(output, string(line))
		}
	}
	return output, nil
}

func runCommandAndChannelOutput(commandname string, args []string, output chan<- []byte) error {
	command := exec.Command(commandname, args...)
	fmt.Print("running the command: ")
	fmt.Println(commandname, args)
	stdout, err := command.StdoutPipe()
	if err != nil {
		return err
	}
	stderr, err := command.StderrPipe()
	if err != nil {
		return err
	}

	go io.Copy(os.Stderr, stderr)
	command.Start()
	defer command.Wait()

	stdoutreader := bufio.NewReader(stdout)
	slurp := true
	for slurp {
		fetch := true
		line := []byte{}
		for fetch {
			partial_line, f, err := stdoutreader.ReadLine()
			fetch = f
			line = append(line, partial_line...)
			if err == io.EOF {
				slurp = false
				break
			} else if err != nil {
				return err
			}
		}
		if len(line) > 0 {
			output <- line
		}
	}
	return nil
}

func getDockerContainerIds(dockerComposeName string) []string {
	if len(dockerContainerIds) != 0 {
		return dockerContainerIds
	}
	ids, err := runCommandAndSlurpOutput("docker-compose", []string{"ps", "-q"})
	if err != nil {
		panic(err)
	}
	dockerContainerIds = ids
	return ids
}

func (n NetMonitor) Start(messages chan<- []byte, dockerComposeName string) {

	ids := getDockerContainerIds(dockerComposeName)
	arguments := []string{"inspect", "-f", "'{{ .State.Pid }}'"}
	arguments = append(arguments, ids...)
	output, err := runCommandAndSlurpOutput("docker", arguments)
	if err != nil {
		panic(err)
	}
	nmProcessorChan := make(chan []byte)
	go networkMonitorProcessor(messages, nmProcessorChan)
	for _, procId := range output {
		// FIXME: do we want to throw an error instead?
		if procId == "'0'" {
			fmt.Println("Container isn't running")
			continue
		}
		// Replace all quotes
		scrubbedProcId := strings.Replace(procId, "'", "", -1)
		// TODO: Check for ids named 0, which means no proc id
		err := setSymlink(scrubbedProcId, scrubbedProcId)
		if err != nil {
			panic(err)
		}
		go startIPProcess(nmProcessorChan, scrubbedProcId, "tcpdump", "-tt", "-n", "-i", "any", "-l")
	}
}
func networkMonitorProcessor(sending chan<- []byte, receiving <-chan []byte) error {
	platform := []byte{}
	carlson := []byte{}
	pattern := "(\\d+\\.\\d+) IP (\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3})\\.(\\d+) > (\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3})\\.(\\d+)"
	compiled_pattern, err := regexp.Compile(pattern)
	if err != nil {
		return err
	}
	for {
		platform = <-receiving
		matches := compiled_pattern.FindSubmatch(platform)
		fmt.Println(string(matches[2]), string(matches[1]))
		carl := ipdata{
			Epoch:       string(matches[1]),
			SourceIP:    string(matches[2]),
			SourcePort:  string(matches[3]),
			ReceiveIP:   string(matches[4]),
			ReceivePort: string(matches[5]),
		}
		carlson, err = json.Marshal(carl)
		if err != nil {
			return err
		}
		fmt.Println("Sending json: ", string(carlson))
		sending <- carlson
	}

	return nil
}
func startIPProcess(messages chan<- []byte, procId string, watcherName string,
	watcherArgs ...string) {
	arguments := []string{"netns", "exec", procId, watcherName}
	arguments = append(arguments, watcherArgs...)
	err := runCommandAndChannelOutput("ip", arguments, messages)
	if err != nil {
		panic(err)
	}
}

func setSymlink(procId string, destination string) error {
	nameSpaceDir := "/var/run/netns/"
	target := nameSpaceDir + destination
	source := "/proc/" + procId + "/ns/net"

	err := os.MkdirAll(nameSpaceDir, 0777)
	if err != nil {
		return err
	}
	_, err = os.Stat(target)
	if err == nil {
		err = os.Remove(target)
		if err != nil {
			return err
		}
	}

	_, err = os.Stat(source)
	if os.IsNotExist(err) {
		return errors.New("Could not set symlink: Target directory " +
			target + " does not exist")
	}

	err = os.Symlink(source, target)
	return err
}

/* This function is going to need some comments describing why we chose this
 * approach, because it will be hairy.
 * FIXME: Break out functionality into multiple functions so we can test this
 * easily!
 */
func (m FSMonitor) getDockerFSDirectory(dockerComposeName string) []string {
	ids := getDockerContainerIds(dockerComposeName)
	/*
		dockerInfoCommand := exec.Command("docker", "info")
		infoOutPipe, err := dockerInfoCommand.StdoutPipe()

		if err != nil {
			panic(err)
		}

		dockerInfoCommand.Start()
		defer dockerInfoCommand.Wait()

		infoBuf, err := ioutil.ReadAll(infoOutPipe)

		if err != nil {
			panic(err)
		}

		info := string(infoBuf)

		re := regexp.MustCompile("Docker Root Dir: (.*)")

		submatch := re.FindStringSubmatch(info)

		if len(submatch) < 2 {
			fmt.Println(submatch, info)
			panic("Couldn't find the docker root directory")
		}
		dockerRootPath := submatch[1]
	*/
	dockerRootPath := "/var/lib/docker/aufs"

	for i := 0; i < len(ids); i++ {
		ids[i] = dockerRootPath + "/mnt/" + ids[i]
	}
	return ids
}

/* Start the process running on the honeypot host to monitor the Docker
 * container. The Docker container's filesystem is mounted on the host. Find
 * the location of this filesystem with the getDockerFSDirectory function and
 * store it in the struct. Then create and start the process and forward
 * the output of the process on to the messages channel.
 */
func (m FSMonitor) Start(messages chan<- []byte, dockerComposeName string) {
	m.DockerDirs = m.getDockerFSDirectory(dockerComposeName)
	// FIXME Make arguments configurable
	//arguments := append([]string{"-wte"}, m.DockerDirs...)
	arguments := m.DockerDirs
	fmt.Println(m.MonitorName, arguments)
	m.fsWatcherProc = exec.Command(m.MonitorName, arguments...)
	defer m.fsWatcherProc.Wait()

	outpipe, err := m.fsWatcherProc.StdoutPipe()
	if err != nil {
		log.Println("Could not open the ", m.MonitorName, "stdout pipe")
		panic(err)
	}
	stderr, err := m.fsWatcherProc.StderrPipe()
	if err != nil {
		log.Println("Could not open the ", m.MonitorName, "stderr pipe")
		panic(err)
	}
	go io.Copy(os.Stderr, stderr)

	m.fsWatcherProc.Start()

	stdoutReader := bufio.NewReader(outpipe)

	for {
		fetch := true
		line := []byte{}
		for fetch {
			partial_line, f, err := stdoutReader.ReadLine()
			if err != nil {
				fmt.Println("File monitor stopped")
				panic(err)
			}
			fetch = f
			line = append(line, partial_line...)
		}
		messages <- line

	}
}

/* Stop the filesystem monitor. Kill the process monitoring the Docker
 * container's filesysem.
 */
func (m FSMonitor) Stop() {
	err := m.fsWatcherProc.Process.Kill()
	if err != nil {
		log.Fatal("Could not kill ", m.MonitorName, err)
	}
}
<commit_msg>Add function to get Docker container pids<commit_after>package monitor

import (
	"bufio"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	_ "io/ioutil"
	"log"
	"os"
	"os/exec"
	"regexp"
	"strings"
)

type ipdata struct {
	Epoch       string `json:"Epoch"`
	SourceIP    string `json:"SourceIP"`
	SourcePort  string `json:"SourcePort"`
	ReceiveIP   string `json:"ReceiveIP"`
	ReceivePort string `json:"ReceivePort"`
}

/* The Monitor interface defines a series of methods which will be defined on
 * monitor structs. The Start method takes a channel to send messages over,
 * back to the configurator. The Stop method kills the process which is
 * performing the actual monitoring.
 */
type Monitor interface {
	Start(messages chan<- []byte, dockerComposeName string)
	Stop()
}

/* Store the MonitorName, which is the name of the program to execute, the
 * DockerDirs which are the directories to monitor which our Docker containers
 * of interest live in, and a pointer to the exec.Cmd struct which describes
 * the running command.
 */
type FSMonitor struct {
	MonitorName   string
	DockerDirs    []string
	fsWatcherProc *exec.Cmd
}

type NetMonitor struct {
	MonitorName  string
	ContainerIds []string
}

// Memoize these. They're kind of expensive to get.
var dockerContainerIds = []string{}
var dockerContainerProcessIds = []string{}

func runCommandAndSlurpOutput(commandname string, args []string) ([]string, error) {
	command := exec.Command(commandname, args...)
	fmt.Print("running the command: ")
	fmt.Println(commandname, args)
	stdout, err := command.StdoutPipe()
	if err != nil {
		return nil, err
	}
	stderr, err := command.StderrPipe()
	if err != nil {
		return nil, err
	}

	go io.Copy(os.Stderr, stderr)
	command.Start()
	defer command.Wait()

	output := []string{}
	stdoutreader := bufio.NewReader(stdout)
	slurp := true
	for slurp {
		fetch := true
		line := []byte{}
		for fetch {
			partial_line, f, err := stdoutreader.ReadLine()
			fetch = f
			line = append(line, partial_line...)
			if err == io.EOF {
				slurp = false
				break
			} else if err != nil {
				return nil, err
			}
		}
		if len(line) > 0 {
			output = append(output, string(line))
		}
	}
	return output, nil
}

func runCommandAndChannelOutput(commandname string, args []string, output chan<- []byte) error {
	command := exec.Command(commandname, args...)
	fmt.Print("running the command: ")
	fmt.Println(commandname, args)
	stdout, err := command.StdoutPipe()
	if err != nil {
		return err
	}
	stderr, err := command.StderrPipe()
	if err != nil {
		return err
	}

	go io.Copy(os.Stderr, stderr)
	command.Start()
	defer command.Wait()

	stdoutreader := bufio.NewReader(stdout)
	slurp := true
	for slurp {
		fetch := true
		line := []byte{}
		for fetch {
			partial_line, f, err := stdoutreader.ReadLine()
			fetch = f
			line = append(line, partial_line...)
			if err == io.EOF {
				slurp = false
				break
			} else if err != nil {
				return err
			}
		}
		if len(line) > 0 {
			output <- line
		}
	}
	return nil
}

func getDockerContainerIds(dockerComposeName string) []string {
	if len(dockerContainerIds) != 0 {
		return dockerContainerIds
	}
	ids, err := runCommandAndSlurpOutput("docker-compose", []string{"ps", "-q"})
	if err != nil {
		panic(err)
	}
	dockerContainerIds = ids
	return ids
}

func getDockerContainerProcessIds(dockerComposeName string) []string {
	// Memoize this function
	if len(dockerContainerProcessIds) > 0 {
		return dockerContainerProcessIds
	}
	ids := getDockerContainerIds(dockerComposeName)
	arguments := []string{"inspect", "-f", "'{{ .State.Pid }}'"}
	arguments = append(arguments, ids...)
	output, err := runCommandAndSlurpOutput("docker", arguments)
	for i, out := range output {
		// FIXME: do we want to throw an error instead?
		if out == "'0'" {
			fmt.Println("Container isn't running: ", out)
			output = append(output[:i], output[i+1:]...)
			continue
		}
		// Replace all quotes
		scrubbedProcId := strings.Replace(out, "'", "", -1)
		output[i] = scrubbedProcId
	}

	if err != nil {
		panic(err)
	}
	dockerContainerProcessIds = output
	return output
}

func (n NetMonitor) Start(messages chan<- []byte, dockerComposeName string) {
	output := getDockerContainerProcessIds(dockerComposeName)
	nmProcessorChan := make(chan []byte)
	go networkMonitorProcessor(messages, nmProcessorChan)
	for _, procId := range output {
		err := setSymlink(procId, procId)
		if err != nil {
			panic(err)
		}
		go startIPProcess(nmProcessorChan, procId, "tcpdump", "-tt", "-n", "-i", "any", "-l")
	}
}
func networkMonitorProcessor(sending chan<- []byte, receiving <-chan []byte) error {
	platform := []byte{}
	carlson := []byte{}
	pattern := "(\\d+\\.\\d+) IP (\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3})\\.(\\d+) > (\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3})\\.(\\d+)"
	compiled_pattern, err := regexp.Compile(pattern)
	if err != nil {
		return err
	}
	for {
		platform = <-receiving
		matches := compiled_pattern.FindSubmatch(platform)
		fmt.Println(string(matches[2]), string(matches[1]))
		carl := ipdata{
			Epoch:       string(matches[1]),
			SourceIP:    string(matches[2]),
			SourcePort:  string(matches[3]),
			ReceiveIP:   string(matches[4]),
			ReceivePort: string(matches[5]),
		}
		carlson, err = json.Marshal(carl)
		if err != nil {
			return err
		}
		fmt.Println("Sending json: ", string(carlson))
		sending <- carlson
	}

	return nil
}
func startIPProcess(messages chan<- []byte, procId string, watcherName string,
	watcherArgs ...string) {
	arguments := []string{"netns", "exec", procId, watcherName}
	arguments = append(arguments, watcherArgs...)
	err := runCommandAndChannelOutput("ip", arguments, messages)
	if err != nil {
		panic(err)
	}
}

func setSymlink(procId string, destination string) error {
	nameSpaceDir := "/var/run/netns/"
	target := nameSpaceDir + destination
	source := "/proc/" + procId + "/ns/net"

	err := os.MkdirAll(nameSpaceDir, 0777)
	if err != nil {
		return err
	}
	_, err = os.Stat(target)
	if err == nil {
		err = os.Remove(target)
		if err != nil {
			return err
		}
	}

	_, err = os.Stat(source)
	if os.IsNotExist(err) {
		return errors.New("Could not set symlink: Target directory " +
			target + " does not exist")
	}

	err = os.Symlink(source, target)
	return err
}

/* Returns the root directories of all of the Docker containers being monitored
 * as a list of strings.
 */
func (m FSMonitor) getDockerFSDirectory(dockerComposeName string) []string {
	pids := getDockerContainerProcessIds(dockerComposeName)
	roots := make([]string, len(pids))
	for i, pid := range pids {
		roots[i] = "/proc/" + pid + "/root"
	}
	return roots
}

/* Start the process running on the honeypot host to monitor the Docker
 * container. The Docker container's filesystem is mounted on the host. Find
 * the location of this filesystem with the getDockerFSDirectory function and
 * store it in the struct. Then create and start the process and forward
 * the output of the process on to the messages channel.
 */
func (m FSMonitor) Start(messages chan<- []byte, dockerComposeName string) {
	m.DockerDirs = m.getDockerFSDirectory(dockerComposeName)
	// FIXME Make arguments configurable
	//arguments := append([]string{"-wte"}, m.DockerDirs...)
	arguments := m.DockerDirs
	fmt.Println(m.MonitorName, arguments)
	m.fsWatcherProc = exec.Command(m.MonitorName, arguments...)
	defer m.fsWatcherProc.Wait()

	outpipe, err := m.fsWatcherProc.StdoutPipe()
	if err != nil {
		log.Println("Could not open the ", m.MonitorName, "stdout pipe")
		panic(err)
	}
	stderr, err := m.fsWatcherProc.StderrPipe()
	if err != nil {
		log.Println("Could not open the ", m.MonitorName, "stderr pipe")
		panic(err)
	}
	go io.Copy(os.Stderr, stderr)

	m.fsWatcherProc.Start()

	stdoutReader := bufio.NewReader(outpipe)

	for {
		fetch := true
		line := []byte{}
		for fetch {
			partial_line, f, err := stdoutReader.ReadLine()
			if err != nil {
				fmt.Println("File monitor stopped")
				panic(err)
			}
			fetch = f
			line = append(line, partial_line...)
		}
		messages <- line

	}
}

/* Stop the filesystem monitor. Kill the process monitoring the Docker
 * container's filesysem.
 */
func (m FSMonitor) Stop() {
	err := m.fsWatcherProc.Process.Kill()
	if err != nil {
		log.Fatal("Could not kill ", m.MonitorName, err)
	}
}
<|endoftext|>


<commit_before>// The maildir package provides an interface to mailboxes in the Maildir format.
package maildir

import (
	"bufio"
	"bytes"
	"crypto/rand"
	"encoding/hex"
	"io"
	"net/mail"
	"net/textproto"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"sync/atomic"
	"time"
)

// The Separator separates a messages unique key from its flags in the filename.
// This should only be changed on operating systems where the colon isn't
// allowed in filenames.
var Separator rune = ':'

var id int64 = 10000

// CreateMode holds the permissions used when creating a directory.
const CreateMode = 0700

// A KeyError occurs when a key matches more or less than one message.
type KeyError struct {
	Key string // the (invalid) key
	N   int    // number of matches (!= 1)
}

func (e *KeyError) Error() string {
	return "maildir: key " + e.Key + " matches " + strconv.Itoa(e.N) + " files."
}

// A FlagError occurs when a non-standard info section is encountered.
type FlagError struct {
	Info         string // the encountered info section
	Experimental bool   // info section starts with 1
}

func (e *FlagError) Error() string {
	if e.Experimental {
		return "maildir: experimental info section encountered: " + e.Info[2:]
	}
	return "maildir: bad info section encountered: " + e.Info
}

// A Dir represents a single directory in a Maildir mailbox.
type Dir string

// Unseen moves messages from new to cur (they are now "seen") and returns their keys.
func (d Dir) Unseen() ([]string, error) {
	f, err := os.Open(filepath.Join(string(d), "new"))
	if err != nil {
		return nil, err
	}
	defer f.Close()
	names, err := f.Readdirnames(0)
	if err != nil {
		return nil, err
	}
	var keys []string
	for _, n := range names {
		if n[0] != '.' {
			split := strings.FieldsFunc(n, func(r rune) bool {
				return r == Separator
			})
			key := split[0]
			info := "2,"
			// Messages in new shouldn't have an info section but
			// we act as if, in case some other program didn't
			// follow the spec.
			if len(split) > 1 {
				info = split[1]
			}
			keys = append(keys, key)
			err = os.Rename(filepath.Join(string(d), "new", n),
				filepath.Join(string(d), "cur", key+string(Separator)+info))
		}
	}
	return keys, err
}

// Keys returns a slice of valid keys to access messages by.
func (d Dir) Keys() ([]string, error) {
	f, err := os.Open(filepath.Join(string(d), "cur"))
	if err != nil {
		return nil, err
	}
	defer f.Close()
	names, err := f.Readdirnames(0)
	if err != nil {
		return nil, err
	}
	var keys []string
	for _, n := range names {
		if n[0] != '.' {
			split := strings.FieldsFunc(n, func(r rune) bool {
				return r == Separator
			})
			keys = append(keys, split[0])
		}
	}
	return keys, nil
}

// Filename returns the path to the file corresponding to the key.
func (d Dir) Filename(key string) (string, error) {
	matches, err := filepath.Glob(filepath.Join(string(d), "cur", key+"*"))
	if err != nil {
		return "", err
	}
	if n := len(matches); n != 1 {
		return "", &KeyError{key, n}
	}
	return matches[0], nil
}

// Header returns the corresponding mail header to a key.
func (d Dir) Header(key string) (header mail.Header, err error) {
	filename, err := d.Filename(key)
	if err != nil {
		return
	}
	file, err := os.Open(filename)
	if err != nil {
		return
	}
	defer file.Close()
	tp := textproto.NewReader(bufio.NewReader(file))
	hdr, err := tp.ReadMIMEHeader()
	if err != nil {
		return
	}
	header = mail.Header(hdr)
	return
}

// Message returns a Message by key.
func (d Dir) Message(key string) (*mail.Message, error) {
	filename, err := d.Filename(key)
	if err != nil {
		return &mail.Message{}, err
	}
	r, err := os.Open(filename)
	if err != nil {
		return &mail.Message{}, err
	}
	defer r.Close()
	buf := new(bytes.Buffer)
	_, err = io.Copy(buf, r)
	if err != nil {
		return &mail.Message{}, err
	}
	msg, err := mail.ReadMessage(buf)
	if err != nil {
		return msg, err
	}
	return msg, nil
}

type runeSlice []rune

func (s runeSlice) Len() int           { return len(s) }
func (s runeSlice) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
func (s runeSlice) Less(i, j int) bool { return s[i] < s[j] }

// Flags returns the flags for a message sorted in ascending order.
// See the documentation of SetFlags for details.
func (d Dir) Flags(key string) (string, error) {
	filename, err := d.Filename(key)
	if err != nil {
		return "", err
	}
	split := strings.FieldsFunc(filename, func(r rune) bool {
		return r == Separator
	})
	switch {
	case len(split[1]) < 2,
		split[1][1] != ',':
		return "", &FlagError{split[1], false}
	case split[1][0] == '1':
		return "", &FlagError{split[1], true}
	case split[1][0] != '2':
		return "", &FlagError{split[1], false}
	}
	rs := runeSlice(split[1][2:])
	sort.Sort(rs)
	return string(rs), nil
}

// SetFlags appends an info section to the filename according to the given flags.
// This function removes duplicates and sorts the flags, but doesn't check
// whether they conform with the Maildir specification.
//
// The following flags are listed in the specification
// (http://cr.yp.to/proto/maildir.html):
//
//	Flag "P" (passed): the user has resent/forwarded/bounced this message to someone else.
//	Flag "R" (replied): the user has replied to this message.
//	Flag "S" (seen): the user has viewed this message, though perhaps he didn't read all the way through it.
//	Flag "T" (trashed): the user has moved this message to the trash; the trash will be emptied by a later user action.
//	Flag "D" (draft): the user considers this message a draft; toggled at user discretion.
//	Flag "F" (flagged): user-defined flag; toggled at user discretion.
func (d Dir) SetFlags(key string, flags string) error {
	info := "2,"
	rs := runeSlice(flags)
	sort.Sort(rs)
	for _, r := range rs {
		if []rune(info)[len(info)-1] != r {
			info += string(r)
		}
	}
	return d.SetInfo(key, info)
}

// Set the info part of the filename.
// Only use this if you plan on using a non-standard info part.
func (d Dir) SetInfo(key, info string) error {
	filename, err := d.Filename(key)
	if err != nil {
		return err
	}
	err = os.Rename(filename, filepath.Join(string(d), "cur", key+
		string(Separator)+info))
	return err
}

// Key generates a new unique key as described in the Maildir specification.
// For the third part of the key (delivery identifier) it uses an internal
// counter, the process id and a cryptographical random number to ensure
// uniqueness among messages delivered in the same second.
func Key() (string, error) {
	var key string
	key += strconv.FormatInt(time.Now().Unix(), 10)
	key += "."
	host, err := os.Hostname()
	if err != err {
		return "", err
	}
	host = strings.Replace(host, "/", "\057", -1)
	host = strings.Replace(host, string(Separator), "\072", -1)
	key += host
	key += "."
	key += strconv.FormatInt(int64(os.Getpid()), 10)
	key += strconv.FormatInt(id, 10)
	atomic.AddInt64(&id, 1)
	bs := make([]byte, 10)
	_, err = io.ReadFull(rand.Reader, bs)
	if err != nil {
		return "", err
	}
	key += hex.EncodeToString(bs)
	return key, nil
}

// Create creates the directory structure for a Maildir.
// If the main directory already exists, it tries to create the subdirectories
// in there. If an error occurs while creating one of the subdirectories, this
// function may leave a partially created directory structure.
func (d Dir) Create() error {
	err := os.Mkdir(string(d), os.ModeDir|CreateMode)
	if err != nil {
		if !os.IsExist(err) {
			return err
		}
	}
	err = os.Mkdir(filepath.Join(string(d), "tmp"), os.ModeDir|CreateMode)
	if err != nil {
		return err
	}
	err = os.Mkdir(filepath.Join(string(d), "new"), os.ModeDir|CreateMode)
	if err != nil {
		return err
	}
	err = os.Mkdir(filepath.Join(string(d), "cur"), os.ModeDir|CreateMode)
	if err != nil {
		return err
	}
	return nil
}

// Delivery represents an ongoing message delivery to the mailbox.
// It implements the WriteCloser interface. On closing the underlying file is
// moved/relinked to new.
type Delivery struct {
	file *os.File
	d    Dir
	key  string
}

// NewDelivery creates a new Delivery.
func (d Dir) NewDelivery() (*Delivery, error) {
	key, err := Key()
	if err != nil {
		return nil, err
	}
	del := &Delivery{}
	time.AfterFunc(24*time.Hour, func() { del.Abort() })
	file, err := os.Create(filepath.Join(string(d), "tmp", key))
	if err != nil {
		return nil, err
	}
	del.file = file
	del.d = d
	del.key = key
	return del, nil
}

func (d *Delivery) Write(p []byte) (int, error) {
	return d.file.Write(p)
}

// Close closes the underlying file and moves it to new.
func (d *Delivery) Close() error {
	err := d.file.Close()
	if err != nil {
		return err
	}
	err = os.Link(filepath.Join(string(d.d), "tmp", d.key),
		filepath.Join(string(d.d), "new", d.key))
	if err != nil {
		return err
	}
	err = os.Remove(filepath.Join(string(d.d), "tmp", d.key))
	if err != nil {
		return err
	}
	return nil
}

// Abort closes the underlying file and removes it completely.
func (d *Delivery) Abort() error {
	err := d.file.Close()
	if err != nil {
		return err
	}
	err = os.Remove(filepath.Join(string(d.d), "tmp", d.key))
	if err != nil {
		return err
	}
	return nil
}

// Move moves a message from this Maildir to another.
func (d Dir) Move(target Dir, key string) error {
	path, err := d.Filename(key)
	if err != nil {
		return err
	}
	return os.Rename(path, filepath.Join(string(target), "cur", filepath.Base(path)))
}

// Purge removes the actual file behind this message.
func (d Dir) Purge(key string) error {
	f, err := d.Filename(key)
	if err != nil {
		return err
	}
	return os.Remove(f)
}

// Clean removes old files from tmp and should be run periodically.
// This does not use access time but modification time for portability reasons.
func (d Dir) Clean() error {
	f, err := os.Open(filepath.Join(string(d), "tmp"))
	if err != nil {
		return err
	}
	defer f.Close()
	names, err := f.Readdirnames(0)
	if err != nil {
		return err
	}
	now := time.Now()
	for _, n := range names {
		fi, err := os.Stat(filepath.Join(string(d), "tmp", n))
		if err != nil {
			continue
		}
		if now.Sub(fi.ModTime()).Hours() > 36 {
			err = os.Remove(filepath.Join(string(d), "tmp", n))
			if err != nil {
				return err
			}
		}
	}
	return nil
}
<commit_msg>Improve Unseen() documentation<commit_after>// The maildir package provides an interface to mailboxes in the Maildir format.
package maildir

import (
	"bufio"
	"bytes"
	"crypto/rand"
	"encoding/hex"
	"io"
	"net/mail"
	"net/textproto"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"sync/atomic"
	"time"
)

// The Separator separates a messages unique key from its flags in the filename.
// This should only be changed on operating systems where the colon isn't
// allowed in filenames.
var Separator rune = ':'

var id int64 = 10000

// CreateMode holds the permissions used when creating a directory.
const CreateMode = 0700

// A KeyError occurs when a key matches more or less than one message.
type KeyError struct {
	Key string // the (invalid) key
	N   int    // number of matches (!= 1)
}

func (e *KeyError) Error() string {
	return "maildir: key " + e.Key + " matches " + strconv.Itoa(e.N) + " files."
}

// A FlagError occurs when a non-standard info section is encountered.
type FlagError struct {
	Info         string // the encountered info section
	Experimental bool   // info section starts with 1
}

func (e *FlagError) Error() string {
	if e.Experimental {
		return "maildir: experimental info section encountered: " + e.Info[2:]
	}
	return "maildir: bad info section encountered: " + e.Info
}

// A Dir represents a single directory in a Maildir mailbox.
type Dir string

// Unseen moves messages from new to cur and returns their keys.
// This means the messages are now known to the application. To find out whether
// a user has seen a message, use Flags().
func (d Dir) Unseen() ([]string, error) {
	f, err := os.Open(filepath.Join(string(d), "new"))
	if err != nil {
		return nil, err
	}
	defer f.Close()
	names, err := f.Readdirnames(0)
	if err != nil {
		return nil, err
	}
	var keys []string
	for _, n := range names {
		if n[0] != '.' {
			split := strings.FieldsFunc(n, func(r rune) bool {
				return r == Separator
			})
			key := split[0]
			info := "2,"
			// Messages in new shouldn't have an info section but
			// we act as if, in case some other program didn't
			// follow the spec.
			if len(split) > 1 {
				info = split[1]
			}
			keys = append(keys, key)
			err = os.Rename(filepath.Join(string(d), "new", n),
				filepath.Join(string(d), "cur", key+string(Separator)+info))
		}
	}
	return keys, err
}

// Keys returns a slice of valid keys to access messages by.
func (d Dir) Keys() ([]string, error) {
	f, err := os.Open(filepath.Join(string(d), "cur"))
	if err != nil {
		return nil, err
	}
	defer f.Close()
	names, err := f.Readdirnames(0)
	if err != nil {
		return nil, err
	}
	var keys []string
	for _, n := range names {
		if n[0] != '.' {
			split := strings.FieldsFunc(n, func(r rune) bool {
				return r == Separator
			})
			keys = append(keys, split[0])
		}
	}
	return keys, nil
}

// Filename returns the path to the file corresponding to the key.
func (d Dir) Filename(key string) (string, error) {
	matches, err := filepath.Glob(filepath.Join(string(d), "cur", key+"*"))
	if err != nil {
		return "", err
	}
	if n := len(matches); n != 1 {
		return "", &KeyError{key, n}
	}
	return matches[0], nil
}

// Header returns the corresponding mail header to a key.
func (d Dir) Header(key string) (header mail.Header, err error) {
	filename, err := d.Filename(key)
	if err != nil {
		return
	}
	file, err := os.Open(filename)
	if err != nil {
		return
	}
	defer file.Close()
	tp := textproto.NewReader(bufio.NewReader(file))
	hdr, err := tp.ReadMIMEHeader()
	if err != nil {
		return
	}
	header = mail.Header(hdr)
	return
}

// Message returns a Message by key.
func (d Dir) Message(key string) (*mail.Message, error) {
	filename, err := d.Filename(key)
	if err != nil {
		return &mail.Message{}, err
	}
	r, err := os.Open(filename)
	if err != nil {
		return &mail.Message{}, err
	}
	defer r.Close()
	buf := new(bytes.Buffer)
	_, err = io.Copy(buf, r)
	if err != nil {
		return &mail.Message{}, err
	}
	msg, err := mail.ReadMessage(buf)
	if err != nil {
		return msg, err
	}
	return msg, nil
}

type runeSlice []rune

func (s runeSlice) Len() int           { return len(s) }
func (s runeSlice) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
func (s runeSlice) Less(i, j int) bool { return s[i] < s[j] }

// Flags returns the flags for a message sorted in ascending order.
// See the documentation of SetFlags for details.
func (d Dir) Flags(key string) (string, error) {
	filename, err := d.Filename(key)
	if err != nil {
		return "", err
	}
	split := strings.FieldsFunc(filename, func(r rune) bool {
		return r == Separator
	})
	switch {
	case len(split[1]) < 2,
		split[1][1] != ',':
		return "", &FlagError{split[1], false}
	case split[1][0] == '1':
		return "", &FlagError{split[1], true}
	case split[1][0] != '2':
		return "", &FlagError{split[1], false}
	}
	rs := runeSlice(split[1][2:])
	sort.Sort(rs)
	return string(rs), nil
}

// SetFlags appends an info section to the filename according to the given flags.
// This function removes duplicates and sorts the flags, but doesn't check
// whether they conform with the Maildir specification.
//
// The following flags are listed in the specification
// (http://cr.yp.to/proto/maildir.html):
//
//	Flag "P" (passed): the user has resent/forwarded/bounced this message to someone else.
//	Flag "R" (replied): the user has replied to this message.
//	Flag "S" (seen): the user has viewed this message, though perhaps he didn't read all the way through it.
//	Flag "T" (trashed): the user has moved this message to the trash; the trash will be emptied by a later user action.
//	Flag "D" (draft): the user considers this message a draft; toggled at user discretion.
//	Flag "F" (flagged): user-defined flag; toggled at user discretion.
func (d Dir) SetFlags(key string, flags string) error {
	info := "2,"
	rs := runeSlice(flags)
	sort.Sort(rs)
	for _, r := range rs {
		if []rune(info)[len(info)-1] != r {
			info += string(r)
		}
	}
	return d.SetInfo(key, info)
}

// Set the info part of the filename.
// Only use this if you plan on using a non-standard info part.
func (d Dir) SetInfo(key, info string) error {
	filename, err := d.Filename(key)
	if err != nil {
		return err
	}
	err = os.Rename(filename, filepath.Join(string(d), "cur", key+
		string(Separator)+info))
	return err
}

// Key generates a new unique key as described in the Maildir specification.
// For the third part of the key (delivery identifier) it uses an internal
// counter, the process id and a cryptographical random number to ensure
// uniqueness among messages delivered in the same second.
func Key() (string, error) {
	var key string
	key += strconv.FormatInt(time.Now().Unix(), 10)
	key += "."
	host, err := os.Hostname()
	if err != err {
		return "", err
	}
	host = strings.Replace(host, "/", "\057", -1)
	host = strings.Replace(host, string(Separator), "\072", -1)
	key += host
	key += "."
	key += strconv.FormatInt(int64(os.Getpid()), 10)
	key += strconv.FormatInt(id, 10)
	atomic.AddInt64(&id, 1)
	bs := make([]byte, 10)
	_, err = io.ReadFull(rand.Reader, bs)
	if err != nil {
		return "", err
	}
	key += hex.EncodeToString(bs)
	return key, nil
}

// Create creates the directory structure for a Maildir.
// If the main directory already exists, it tries to create the subdirectories
// in there. If an error occurs while creating one of the subdirectories, this
// function may leave a partially created directory structure.
func (d Dir) Create() error {
	err := os.Mkdir(string(d), os.ModeDir|CreateMode)
	if err != nil {
		if !os.IsExist(err) {
			return err
		}
	}
	err = os.Mkdir(filepath.Join(string(d), "tmp"), os.ModeDir|CreateMode)
	if err != nil {
		return err
	}
	err = os.Mkdir(filepath.Join(string(d), "new"), os.ModeDir|CreateMode)
	if err != nil {
		return err
	}
	err = os.Mkdir(filepath.Join(string(d), "cur"), os.ModeDir|CreateMode)
	if err != nil {
		return err
	}
	return nil
}

// Delivery represents an ongoing message delivery to the mailbox.
// It implements the WriteCloser interface. On closing the underlying file is
// moved/relinked to new.
type Delivery struct {
	file *os.File
	d    Dir
	key  string
}

// NewDelivery creates a new Delivery.
func (d Dir) NewDelivery() (*Delivery, error) {
	key, err := Key()
	if err != nil {
		return nil, err
	}
	del := &Delivery{}
	time.AfterFunc(24*time.Hour, func() { del.Abort() })
	file, err := os.Create(filepath.Join(string(d), "tmp", key))
	if err != nil {
		return nil, err
	}
	del.file = file
	del.d = d
	del.key = key
	return del, nil
}

func (d *Delivery) Write(p []byte) (int, error) {
	return d.file.Write(p)
}

// Close closes the underlying file and moves it to new.
func (d *Delivery) Close() error {
	err := d.file.Close()
	if err != nil {
		return err
	}
	err = os.Link(filepath.Join(string(d.d), "tmp", d.key),
		filepath.Join(string(d.d), "new", d.key))
	if err != nil {
		return err
	}
	err = os.Remove(filepath.Join(string(d.d), "tmp", d.key))
	if err != nil {
		return err
	}
	return nil
}

// Abort closes the underlying file and removes it completely.
func (d *Delivery) Abort() error {
	err := d.file.Close()
	if err != nil {
		return err
	}
	err = os.Remove(filepath.Join(string(d.d), "tmp", d.key))
	if err != nil {
		return err
	}
	return nil
}

// Move moves a message from this Maildir to another.
func (d Dir) Move(target Dir, key string) error {
	path, err := d.Filename(key)
	if err != nil {
		return err
	}
	return os.Rename(path, filepath.Join(string(target), "cur", filepath.Base(path)))
}

// Purge removes the actual file behind this message.
func (d Dir) Purge(key string) error {
	f, err := d.Filename(key)
	if err != nil {
		return err
	}
	return os.Remove(f)
}

// Clean removes old files from tmp and should be run periodically.
// This does not use access time but modification time for portability reasons.
func (d Dir) Clean() error {
	f, err := os.Open(filepath.Join(string(d), "tmp"))
	if err != nil {
		return err
	}
	defer f.Close()
	names, err := f.Readdirnames(0)
	if err != nil {
		return err
	}
	now := time.Now()
	for _, n := range names {
		fi, err := os.Stat(filepath.Join(string(d), "tmp", n))
		if err != nil {
			continue
		}
		if now.Sub(fi.ModTime()).Hours() > 36 {
			err = os.Remove(filepath.Join(string(d), "tmp", n))
			if err != nil {
				return err
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package flow

import (
	"bufio"
	"log"
	"os"
	"reflect"
	"sync"
)

// Inputs: f(chan A), shardCount
func (fc *FlowContext) Source(f interface{}, shard int) (ret *Dataset) {
	ret = fc.newNextDataset(shard, guessFunctionOutputType(f))
	step := fc.AddOneToAllStep(nil, ret)
	step.Name = "Source"
	step.Function = func(task *Task) {
		ctype := reflect.ChanOf(reflect.BothDir, ret.Type)
		outChan := reflect.MakeChan(ctype, 0)
		fn := reflect.ValueOf(f)
		var wg sync.WaitGroup
		wg.Add(1)
		go func() {
			defer wg.Done()
			defer outChan.Close()
			fn.Call([]reflect.Value{outChan})
		}()

		wg.Add(1)
		go func() {
			defer wg.Done()

			var t reflect.Value
			i := 0
			for ok := true; ok; {
				if t, ok = outChan.Recv(); ok {
					task.Outputs[i].WriteChan.Send(t)
					i++
					if i == shard {
						i = 0
					}
				}
			}
		}()

		wg.Wait()

	}
	return
}

func (fc *FlowContext) TextFile(fname string, shard int) (ret *Dataset) {
	fn := func(out chan string) {
		file, err := os.Open(fname)
		if err != nil {
			// FIXME collect errors
			log.Panicf("Can not open file %s: %v", fname, err)
			return
		}
		defer file.Close()

		scanner := bufio.NewScanner(file)
		for scanner.Scan() {
			out <- scanner.Text()
		}

		if err := scanner.Err(); err != nil {
			log.Printf("Scan file %s: %v", fname, err)
		}
	}
	return fc.Source(fn, shard)
}

func (fc *FlowContext) Channel(ch interface{}) (ret *Dataset) {
	chValue, chType := reflect.ValueOf(ch), reflect.TypeOf(ch)

	return fc.doChannel(chValue, chType)
}

func (fc *FlowContext) doChannel(chValue reflect.Value, chType reflect.Type) (ret *Dataset) {
	ret = fc.newNextDataset(1, chType.Elem())
	ret.ExternalInputChans = append(ret.ExternalInputChans, reflect.Indirect(chValue))
	step := fc.AddOneToOneStep(nil, ret)
	step.Name = "Input"
	step.Function = func(task *Task) {
		for t := range task.MergedInputChan() {
			task.Outputs[0].WriteChan.Send(t)
		}
	}
	return
}

// Slice accepts a slice and send values to tasks via Channel()
func (fc *FlowContext) Slice(slice interface{}) (ret *Dataset) {
	sliceValue, sliceType := reflect.ValueOf(slice), reflect.TypeOf(slice)
	sliceLen := sliceValue.Len()
	chType := reflect.ChanOf(reflect.BothDir, sliceType.Elem())
	chValue := reflect.MakeChan(chType, 0)

	go func() {
		for i := 0; i < sliceLen; i++ {
			chValue.Send(sliceValue.Index(i))
		}
		chValue.Close()
	}()

	return fc.doChannel(chValue, chType)
}
<commit_msg>add buffer for slice channel<commit_after>package flow

import (
	"bufio"
	"log"
	"os"
	"reflect"
	"sync"
)

// Inputs: f(chan A), shardCount
func (fc *FlowContext) Source(f interface{}, shard int) (ret *Dataset) {
	ret = fc.newNextDataset(shard, guessFunctionOutputType(f))
	step := fc.AddOneToAllStep(nil, ret)
	step.Name = "Source"
	step.Function = func(task *Task) {
		ctype := reflect.ChanOf(reflect.BothDir, ret.Type)
		outChan := reflect.MakeChan(ctype, 0)
		fn := reflect.ValueOf(f)
		var wg sync.WaitGroup
		wg.Add(1)
		go func() {
			defer wg.Done()
			defer outChan.Close()
			fn.Call([]reflect.Value{outChan})
		}()

		wg.Add(1)
		go func() {
			defer wg.Done()

			var t reflect.Value
			i := 0
			for ok := true; ok; {
				if t, ok = outChan.Recv(); ok {
					task.Outputs[i].WriteChan.Send(t)
					i++
					if i == shard {
						i = 0
					}
				}
			}
		}()

		wg.Wait()

	}
	return
}

func (fc *FlowContext) TextFile(fname string, shard int) (ret *Dataset) {
	fn := func(out chan string) {
		file, err := os.Open(fname)
		if err != nil {
			// FIXME collect errors
			log.Panicf("Can not open file %s: %v", fname, err)
			return
		}
		defer file.Close()

		scanner := bufio.NewScanner(file)
		for scanner.Scan() {
			out <- scanner.Text()
		}

		if err := scanner.Err(); err != nil {
			log.Printf("Scan file %s: %v", fname, err)
		}
	}
	return fc.Source(fn, shard)
}

func (fc *FlowContext) Channel(ch interface{}) (ret *Dataset) {
	chValue, chType := reflect.ValueOf(ch), reflect.TypeOf(ch)

	return fc.doChannel(chValue, chType)
}

func (fc *FlowContext) doChannel(chValue reflect.Value, chType reflect.Type) (ret *Dataset) {
	ret = fc.newNextDataset(1, chType.Elem())
	ret.ExternalInputChans = append(ret.ExternalInputChans, reflect.Indirect(chValue))
	step := fc.AddOneToOneStep(nil, ret)
	step.Name = "Input"
	step.Function = func(task *Task) {
		for t := range task.MergedInputChan() {
			task.Outputs[0].WriteChan.Send(t)
		}
	}
	return
}

// Slice accepts a slice and send values to tasks via Channel()
func (fc *FlowContext) Slice(slice interface{}) (ret *Dataset) {
	sliceValue, sliceType := reflect.ValueOf(slice), reflect.TypeOf(slice)
	sliceLen := sliceValue.Len()
	chType := reflect.ChanOf(reflect.BothDir, sliceType.Elem())
	chValue := reflect.MakeChan(chType, 16)

	go func() {
		for i := 0; i < sliceLen; i++ {
			chValue.Send(sliceValue.Index(i))
		}
		chValue.Close()
	}()

	return fc.doChannel(chValue, chType)
}
<|endoftext|>