<commit_before>// xbuild debug

package main

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"os"

	"github.com/spf13/cobra"

	"github.com/restic/restic/internal/errors"
	"github.com/restic/restic/internal/pack"
	"github.com/restic/restic/internal/repository"
	"github.com/restic/restic/internal/restic"

	"github.com/restic/restic/internal/worker"
)

var cmdDump = &cobra.Command{
	Use:   "dump [indexes|snapshots|trees|all|packs]",
	Short: "Dump data structures",
	Long: `
The "dump" command dumps data structures from the repository as JSON objects. It
is used for debugging purposes only.`,
	DisableAutoGenTag: true,
	RunE: func(cmd *cobra.Command, args []string) error {
		return runDump(globalOptions, args)
	},
}

func init() {
	cmdRoot.AddCommand(cmdDump)
}

func prettyPrintJSON(wr io.Writer, item interface{}) error {
	buf, err := json.MarshalIndent(item, "", "  ")
	if err != nil {
		return err
	}

	_, err = wr.Write(append(buf, '\n'))
	return err
}

func debugPrintSnapshots(repo *repository.Repository, wr io.Writer) error {
	for id := range repo.List(context.TODO(), restic.SnapshotFile) {
		snapshot, err := restic.LoadSnapshot(context.TODO(), repo, id)
		if err != nil {
			fmt.Fprintf(os.Stderr, "LoadSnapshot(%v): %v", id.Str(), err)
			continue
		}

		fmt.Fprintf(wr, "snapshot_id: %v\n", id)

		err = prettyPrintJSON(wr, snapshot)
		if err != nil {
			return err
		}
	}

	return nil
}

const dumpPackWorkers = 10

// Pack is the struct used in printPacks.
type Pack struct {
	Name string `json:"name"`

	Blobs []Blob `json:"blobs"`
}

// Blob is the struct used in printPacks.
type Blob struct {
	Type   restic.BlobType `json:"type"`
	Length uint            `json:"length"`
	ID     restic.ID       `json:"id"`
	Offset uint            `json:"offset"`
}

func printPacks(repo *repository.Repository, wr io.Writer) error {
	f := func(ctx context.Context, job worker.Job) (interface{}, error) {
		name := job.Data.(string)

		h := restic.Handle{Type: restic.DataFile, Name: name}

		blobInfo, err := repo.Backend().Stat(ctx, h)
		if err != nil {
			return nil, err
		}

		blobs, err := pack.List(repo.Key(), restic.ReaderAt(repo.Backend(), h), blobInfo.Size)
		if err != nil {
			return nil, err
		}

		return blobs, nil
	}

	jobCh := make(chan worker.Job)
	resCh := make(chan worker.Job)
	wp := worker.New(context.TODO(), dumpPackWorkers, f, jobCh, resCh)

	go func() {
		for name := range repo.Backend().List(context.TODO(), restic.DataFile) {
			jobCh <- worker.Job{Data: name}
		}
		close(jobCh)
	}()

	for job := range resCh {
		name := job.Data.(string)

		if job.Error != nil {
			fmt.Fprintf(os.Stderr, "error for pack %v: %v\n", name, job.Error)
			continue
		}

		entries := job.Result.([]restic.Blob)
		p := Pack{
			Name:  name,
			Blobs: make([]Blob, len(entries)),
		}
		for i, blob := range entries {
			p.Blobs[i] = Blob{
				Type:   blob.Type,
				Length: blob.Length,
				ID:     blob.ID,
				Offset: blob.Offset,
			}
		}

		prettyPrintJSON(os.Stdout, p)
	}

	wp.Wait()

	return nil
}

func dumpIndexes(repo restic.Repository) error {
	for id := range repo.List(context.TODO(), restic.IndexFile) {
		fmt.Printf("index_id: %v\n", id)

		idx, err := repository.LoadIndex(context.TODO(), repo, id)
		if err != nil {
			return err
		}

		err = idx.Dump(os.Stdout)
		if err != nil {
			return err
		}
	}

	return nil
}

func runDump(gopts GlobalOptions, args []string) error {
	if len(args) != 1 {
		return errors.Fatal("type not specified")
	}

	repo, err := OpenRepository(gopts)
	if err != nil {
		return err
	}

	if !gopts.NoLock {
		lock, err := lockRepo(repo)
		defer unlockRepo(lock)
		if err != nil {
			return err
		}
	}

	err = repo.LoadIndex(context.TODO())
	if err != nil {
		return err
	}

	tpe := args[0]

	switch tpe {
	case "indexes":
		return dumpIndexes(repo)
	case "snapshots":
		return debugPrintSnapshots(repo, os.Stdout)
	case "packs":
		return printPacks(repo, os.Stdout)
	case "all":
		fmt.Printf("snapshots:\n")
		err := debugPrintSnapshots(repo, os.Stdout)
		if err != nil {
			return err
		}

		fmt.Printf("\nindexes:\n")
		err = dumpIndexes(repo)
		if err != nil {
			return err
		}

		return nil
	default:
		return errors.Fatalf("no such type %q", tpe)
	}
}
<commit_msg>dump command no longer has 'trees' option<commit_after>// xbuild debug

package main

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"os"

	"github.com/spf13/cobra"

	"github.com/restic/restic/internal/errors"
	"github.com/restic/restic/internal/pack"
	"github.com/restic/restic/internal/repository"
	"github.com/restic/restic/internal/restic"

	"github.com/restic/restic/internal/worker"
)

var cmdDump = &cobra.Command{
	Use:   "dump [indexes|snapshots|all|packs]",
	Short: "Dump data structures",
	Long: `
The "dump" command dumps data structures from the repository as JSON objects. It
is used for debugging purposes only.`,
	DisableAutoGenTag: true,
	RunE: func(cmd *cobra.Command, args []string) error {
		return runDump(globalOptions, args)
	},
}

func init() {
	cmdRoot.AddCommand(cmdDump)
}

func prettyPrintJSON(wr io.Writer, item interface{}) error {
	buf, err := json.MarshalIndent(item, "", "  ")
	if err != nil {
		return err
	}

	_, err = wr.Write(append(buf, '\n'))
	return err
}

func debugPrintSnapshots(repo *repository.Repository, wr io.Writer) error {
	for id := range repo.List(context.TODO(), restic.SnapshotFile) {
		snapshot, err := restic.LoadSnapshot(context.TODO(), repo, id)
		if err != nil {
			fmt.Fprintf(os.Stderr, "LoadSnapshot(%v): %v", id.Str(), err)
			continue
		}

		fmt.Fprintf(wr, "snapshot_id: %v\n", id)

		err = prettyPrintJSON(wr, snapshot)
		if err != nil {
			return err
		}
	}

	return nil
}

const dumpPackWorkers = 10

// Pack is the struct used in printPacks.
type Pack struct {
	Name string `json:"name"`

	Blobs []Blob `json:"blobs"`
}

// Blob is the struct used in printPacks.
type Blob struct {
	Type   restic.BlobType `json:"type"`
	Length uint            `json:"length"`
	ID     restic.ID       `json:"id"`
	Offset uint            `json:"offset"`
}

func printPacks(repo *repository.Repository, wr io.Writer) error {
	f := func(ctx context.Context, job worker.Job) (interface{}, error) {
		name := job.Data.(string)

		h := restic.Handle{Type: restic.DataFile, Name: name}

		blobInfo, err := repo.Backend().Stat(ctx, h)
		if err != nil {
			return nil, err
		}

		blobs, err := pack.List(repo.Key(), restic.ReaderAt(repo.Backend(), h), blobInfo.Size)
		if err != nil {
			return nil, err
		}

		return blobs, nil
	}

	jobCh := make(chan worker.Job)
	resCh := make(chan worker.Job)
	wp := worker.New(context.TODO(), dumpPackWorkers, f, jobCh, resCh)

	go func() {
		for name := range repo.Backend().List(context.TODO(), restic.DataFile) {
			jobCh <- worker.Job{Data: name}
		}
		close(jobCh)
	}()

	for job := range resCh {
		name := job.Data.(string)

		if job.Error != nil {
			fmt.Fprintf(os.Stderr, "error for pack %v: %v\n", name, job.Error)
			continue
		}

		entries := job.Result.([]restic.Blob)
		p := Pack{
			Name:  name,
			Blobs: make([]Blob, len(entries)),
		}
		for i, blob := range entries {
			p.Blobs[i] = Blob{
				Type:   blob.Type,
				Length: blob.Length,
				ID:     blob.ID,
				Offset: blob.Offset,
			}
		}

		prettyPrintJSON(os.Stdout, p)
	}

	wp.Wait()

	return nil
}

func dumpIndexes(repo restic.Repository) error {
	for id := range repo.List(context.TODO(), restic.IndexFile) {
		fmt.Printf("index_id: %v\n", id)

		idx, err := repository.LoadIndex(context.TODO(), repo, id)
		if err != nil {
			return err
		}

		err = idx.Dump(os.Stdout)
		if err != nil {
			return err
		}
	}

	return nil
}

func runDump(gopts GlobalOptions, args []string) error {
	if len(args) != 1 {
		return errors.Fatal("type not specified")
	}

	repo, err := OpenRepository(gopts)
	if err != nil {
		return err
	}

	if !gopts.NoLock {
		lock, err := lockRepo(repo)
		defer unlockRepo(lock)
		if err != nil {
			return err
		}
	}

	err = repo.LoadIndex(context.TODO())
	if err != nil {
		return err
	}

	tpe := args[0]

	switch tpe {
	case "indexes":
		return dumpIndexes(repo)
	case "snapshots":
		return debugPrintSnapshots(repo, os.Stdout)
	case "packs":
		return printPacks(repo, os.Stdout)
	case "all":
		fmt.Printf("snapshots:\n")
		err := debugPrintSnapshots(repo, os.Stdout)
		if err != nil {
			return err
		}

		fmt.Printf("\nindexes:\n")
		err = dumpIndexes(repo)
		if err != nil {
			return err
		}

		return nil
	default:
		return errors.Fatalf("no such type %q", tpe)
	}
}
<|endoftext|>


<commit_before>package main

// #include <string.h>
import "C"

import (
	"errors"
	"fmt"
	"unsafe"

	"github.com/ready-steady/linal/matrix"
	"github.com/ready-steady/probability/gaussian"
	"github.com/ready-steady/simulation/power"
	"github.com/ready-steady/simulation/temperature"

	"../../pkg/solver"
)

type tempTarget struct {
	problem *problem

	ic uint32 // inputs
	oc uint32 // outputs
	sc uint32 // steps

	power       *power.Power
	temperature *temperature.Solver
}

func newTempTarget(p *problem) (target, error) {
	c := &p.config

	if c.TempAnalysis.TimeStep <= 0 {
		return nil, errors.New("the time step is invalid")
	}

	power := power.New(p.platform, p.application, c.TempAnalysis.TimeStep)
	temperature, err := temperature.New(temperature.Config(c.TempAnalysis))
	if err != nil {
		return nil, err
	}

	target := &tempTarget{
		problem: p,

		ic: 1 + p.zc, // +1 for time
		oc: uint32(len(c.CoreIndex)),
		sc: uint32(p.schedule.Span / c.TempAnalysis.TimeStep),

		power:       power,
		temperature: temperature,
	}

	return target, nil
}

func (t *tempTarget) String() string {
	return fmt.Sprintf("Target{inputs: %d, outputs: %d}", t.ic, t.oc)
}

func (t *tempTarget) InputsOutputs() (uint32, uint32) {
	return t.ic, t.oc
}

func (t *tempTarget) Serve(jobs <-chan solver.Job) {
	p := t.problem
	c := &p.config

	cc, uc, zc, oc, sc := p.cc, p.uc, p.zc, t.oc, t.sc
	coreIndex := c.CoreIndex

	g := gaussian.New(0, 1)
	m := p.marginals

	P := make([]float64, cc*sc)
	S := make([]float64, t.temperature.Nodes*sc)

	z := make([]float64, zc)
	u := make([]float64, uc)
	d := make([]float64, p.tc)

	for job := range jobs {
		Q := job.Data

		if Q == nil {
			Q = make([]float64, cc*sc)

			// Independent uniform to independent Gaussian
			for i := uint32(0); i < zc; i++ {
				z[i] = g.InvCDF(job.Node[1+i]) // +1 for time
			}

			// Independent Gaussian to dependent Gaussian
			matrix.Multiply(p.transform, z, u, uc, zc, 1)

			// Dependent Gaussian to dependent uniform to dependent target
			for i, tid := range c.TaskIndex {
				d[tid] = m[i].InvCDF(g.CDF(u[i]))
			}

			// FIXME: Bad, bad, bad!
			C.memset(unsafe.Pointer(&P[0]), 0, C.size_t(8*cc*sc))

			t.power.Compute(p.time.Recompute(p.schedule, d), P, sc)
			t.temperature.ComputeTransient(P, Q, S, sc)
		}

		sid := uint32(job.Node[0] * float64(sc-1))
		for i := uint32(0); i < oc; i++ {
			job.Value[i] = Q[sid*cc+uint32(coreIndex[i])]
		}

		job.Done <- solver.Result{job.Key, Q}
	}
}
<commit_msg>Updated the usage of simulation/temperature<commit_after>package main

// #include <string.h>
import "C"

import (
	"errors"
	"fmt"
	"unsafe"

	"github.com/ready-steady/linal/matrix"
	"github.com/ready-steady/probability/gaussian"
	"github.com/ready-steady/simulation/power"
	"github.com/ready-steady/simulation/temperature"

	"../../pkg/solver"
)

type tempTarget struct {
	problem *problem

	ic uint32 // inputs
	oc uint32 // outputs
	sc uint32 // steps

	power       *power.Power
	temperature *temperature.Temperature
}

func newTempTarget(p *problem) (target, error) {
	c := &p.config

	if c.TempAnalysis.TimeStep <= 0 {
		return nil, errors.New("the time step is invalid")
	}

	power := power.New(p.platform, p.application, c.TempAnalysis.TimeStep)
	temperature, err := temperature.New(temperature.Config(c.TempAnalysis))
	if err != nil {
		return nil, err
	}

	target := &tempTarget{
		problem: p,

		ic: 1 + p.zc, // +1 for time
		oc: uint32(len(c.CoreIndex)),
		sc: uint32(p.schedule.Span / c.TempAnalysis.TimeStep),

		power:       power,
		temperature: temperature,
	}

	return target, nil
}

func (t *tempTarget) String() string {
	return fmt.Sprintf("Target{inputs: %d, outputs: %d}", t.ic, t.oc)
}

func (t *tempTarget) InputsOutputs() (uint32, uint32) {
	return t.ic, t.oc
}

func (t *tempTarget) Serve(jobs <-chan solver.Job) {
	p := t.problem
	c := &p.config

	cc, uc, zc, oc, sc := p.cc, p.uc, p.zc, t.oc, t.sc
	coreIndex := c.CoreIndex

	g := gaussian.New(0, 1)
	m := p.marginals

	P := make([]float64, cc*sc)
	S := make([]float64, t.temperature.Nodes*sc)

	z := make([]float64, zc)
	u := make([]float64, uc)
	d := make([]float64, p.tc)

	for job := range jobs {
		Q := job.Data

		if Q == nil {
			Q = make([]float64, cc*sc)

			// Independent uniform to independent Gaussian
			for i := uint32(0); i < zc; i++ {
				z[i] = g.InvCDF(job.Node[1+i]) // +1 for time
			}

			// Independent Gaussian to dependent Gaussian
			matrix.Multiply(p.transform, z, u, uc, zc, 1)

			// Dependent Gaussian to dependent uniform to dependent target
			for i, tid := range c.TaskIndex {
				d[tid] = m[i].InvCDF(g.CDF(u[i]))
			}

			// FIXME: Bad, bad, bad!
			C.memset(unsafe.Pointer(&P[0]), 0, C.size_t(8*cc*sc))

			t.power.Compute(p.time.Recompute(p.schedule, d), P, sc)
			t.temperature.ComputeTransient(P, Q, S, sc)
		}

		sid := uint32(job.Node[0] * float64(sc-1))
		for i := uint32(0); i < oc; i++ {
			job.Value[i] = Q[sid*cc+uint32(coreIndex[i])]
		}

		job.Done <- solver.Result{job.Key, Q}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"errors"
	"log"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"strings"

	"github.com/nlopes/slack"
)

const (
	iconPath = "/static/icon.png"
	memePath = "/static/spongemock.jpg"
)

var (
	atk     = os.Getenv("AUTHENTICATION_TOKEN")
	vtk     = os.Getenv("VERIFICATION_TOKEN")
	appURL  = os.Getenv("APP_URL")
	iconURL string
	memeURL string
	api     = slack.New(atk)
)

func transformText(m string) string {
	var buffer bytes.Buffer
	for i := 0; i < len(m); i++ {
		ch := m[i : i+1]
		if rand.Int()%2 == 0 {
			ch = strings.ToUpper(ch)
		}
		buffer.WriteString(ch)
	}
	return buffer.String()
}

func isValidSlackRequest(r *http.Request) bool {
	if r.Method != "POST" {
		log.Printf("want method POST, got %s\n", r.Method)
		return false
	}
	err := r.ParseForm()
	if err != nil {
		log.Printf("invalid form data: %s\n", err)
		return false
	}
	if cmd := r.PostFormValue("command"); cmd != "/spongemock" {
		log.Printf("want command /spongemock, got %s\n", cmd)
		return false
	}
	if tk := r.PostFormValue("token"); tk != vtk {
		log.Printf("received invalid token %s\n", tk)
		return false
	}
	if url := r.PostFormValue("response_url"); url == "" {
		log.Println("did not receive response url")
		return false
	}
	return true
}

func getLastSlackMessage(c string) (string, error) {
	h, err := api.GetChannelHistory(c, slack.NewHistoryParameters())
	if err != nil {
		log.Printf("history API request error: %s", err)
		return "", err
	}

	for _, msg := range h.Messages {
		if msg.SubType != "" || msg.Text == "" {
			continue
		}
		return msg.Text, nil
	}

	err = errors.New("no last message found")
	log.Println(err)
	return "", err
}

func handleSlack(w http.ResponseWriter, r *http.Request) {
	status := http.StatusOK
	defer func() {
		w.WriteHeader(status)
	}()
	if !isValidSlackRequest(r) {
		status = http.StatusBadRequest
		return
	}
	channel := r.PostFormValue("channel_id")
	lastMessage, err := getLastSlackMessage(channel)
	if err != nil {
		status = http.StatusInternalServerError
		return
	}
	params := slack.NewPostMessageParameters()
	params.Username = "Spongebob"
	params.Attachments = []slack.Attachment{{
		Text:     transformText(lastMessage),
		Fallback: "*Spongebob mocking meme*",
		ImageURL: memeURL,
	}}
	params.IconURL = iconURL
	_, _, err = api.PostMessage(channel, "", params)
	if err != nil {
		status = http.StatusInternalServerError
	}
}

func main() {
	port := os.Getenv("PORT")
	if port == "" {
		log.Fatal("$PORT must be set!")
	}
	if atk == "" {
		log.Fatal("$AUTHENTICATION_TOKEN must be set!")
	}
	if vtk == "" {
		log.Fatal("$VERIFICATION_TOKEN must be set!")
	}
	if appURL == "" {
		log.Fatal("$APP_URL must be set!")
	}
	u, err := url.Parse(appURL)
	if err != nil {
		log.Fatal("invalid $APP_URL %s", appURL)
	}
	icon, _ := url.Parse(iconPath)
	iconURL = u.ResolveReference(icon).String()
	meme, _ := url.Parse(memePath)
	memeURL = u.ResolveReference(meme).String()

	fs := http.FileServer(http.Dir("static"))
	http.Handle("/static/", http.StripPrefix("/static/", fs))
	http.HandleFunc("/slack", handleSlack)

	log.Fatal(http.ListenAndServe(":"+port, nil))
}
<commit_msg>Mock all caps people as well<commit_after>package main

import (
	"bytes"
	"errors"
	"log"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"strings"

	"github.com/nlopes/slack"
)

const (
	iconPath = "/static/icon.png"
	memePath = "/static/spongemock.jpg"
)

var (
	atk     = os.Getenv("AUTHENTICATION_TOKEN")
	vtk     = os.Getenv("VERIFICATION_TOKEN")
	appURL  = os.Getenv("APP_URL")
	iconURL string
	memeURL string
	api     = slack.New(atk)
)

func transformText(m string) string {
	var buffer bytes.Buffer
	for i := 0; i < len(m); i++ {
		ch := m[i : i+1]
		if rand.Int()%2 == 0 {
			ch = strings.ToUpper(ch)
		} else {
			ch = strings.ToLower(ch)
		}
		buffer.WriteString(ch)
	}
	return buffer.String()
}

func isValidSlackRequest(r *http.Request) bool {
	if r.Method != "POST" {
		log.Printf("want method POST, got %s\n", r.Method)
		return false
	}
	err := r.ParseForm()
	if err != nil {
		log.Printf("invalid form data: %s\n", err)
		return false
	}
	if cmd := r.PostFormValue("command"); cmd != "/spongemock" {
		log.Printf("want command /spongemock, got %s\n", cmd)
		return false
	}
	if tk := r.PostFormValue("token"); tk != vtk {
		log.Printf("received invalid token %s\n", tk)
		return false
	}
	if url := r.PostFormValue("response_url"); url == "" {
		log.Println("did not receive response url")
		return false
	}
	return true
}

func getLastSlackMessage(c string) (string, error) {
	h, err := api.GetChannelHistory(c, slack.NewHistoryParameters())
	if err != nil {
		log.Printf("history API request error: %s", err)
		return "", err
	}

	for _, msg := range h.Messages {
		if msg.SubType != "" || msg.Text == "" {
			continue
		}
		return msg.Text, nil
	}

	err = errors.New("no last message found")
	log.Println(err)
	return "", err
}

func handleSlack(w http.ResponseWriter, r *http.Request) {
	status := http.StatusOK
	defer func() {
		w.WriteHeader(status)
	}()
	if !isValidSlackRequest(r) {
		status = http.StatusBadRequest
		return
	}
	channel := r.PostFormValue("channel_id")
	lastMessage, err := getLastSlackMessage(channel)
	if err != nil {
		status = http.StatusInternalServerError
		return
	}
	params := slack.NewPostMessageParameters()
	params.Username = "Spongebob"
	params.Attachments = []slack.Attachment{{
		Text:     transformText(lastMessage),
		Fallback: "*Spongebob mocking meme*",
		ImageURL: memeURL,
	}}
	params.IconURL = iconURL
	_, _, err = api.PostMessage(channel, "", params)
	if err != nil {
		status = http.StatusInternalServerError
	}
}

func main() {
	port := os.Getenv("PORT")
	if port == "" {
		log.Fatal("$PORT must be set!")
	}
	if atk == "" {
		log.Fatal("$AUTHENTICATION_TOKEN must be set!")
	}
	if vtk == "" {
		log.Fatal("$VERIFICATION_TOKEN must be set!")
	}
	if appURL == "" {
		log.Fatal("$APP_URL must be set!")
	}
	u, err := url.Parse(appURL)
	if err != nil {
		log.Fatal("invalid $APP_URL %s", appURL)
	}
	icon, _ := url.Parse(iconPath)
	iconURL = u.ResolveReference(icon).String()
	meme, _ := url.Parse(memePath)
	memeURL = u.ResolveReference(meme).String()

	fs := http.FileServer(http.Dir("static"))
	http.Handle("/static/", http.StripPrefix("/static/", fs))
	http.HandleFunc("/slack", handleSlack)

	log.Fatal(http.ListenAndServe(":"+port, nil))
}
<|endoftext|>


<commit_before>package main

import(
	"fmt"
	"os"
	"github.com/eikeon/brikker/openjpeg"
)

// Buffer for holding all the output information as jp2s are examined
var jp2Messages chan string

// Channel for holding the files our workers need to process
var jp2Files chan string

func main() {
	// This seems like a solid value based on various bits of testing, but it may
	// make sense to do something smarter here
	maxWorkers := 8

	// Set up channels with big enough buffers that one goroutine isn't holding
	// up another
	jp2Messages = make(chan string, maxWorkers)
	jp2Files = make(chan string, maxWorkers)

	// Read JP2 file list and stuff it into the channel
	if (len(os.Args) != 2) {
		fmt.Println("You must provide a path to a file listing JP2s to verify")
		os.Exit(1)
	}
	fileCount := readJP2FileList(os.Args[1])

	// Show anything that's "info" or more serious
	openjpeg.LogLevel = 4

	fmt.Println("BEGIN")
	fmt.Println("---")

	for i := 0; i < maxWorkers; i++ {
		go createWorker()
	}

	// Read messages until we have accounted for all JP2s
	for i := 0; i < fileCount; i++ {
		message := <-jp2Messages
		fmt.Println(message)
	}

	fmt.Println("---")
	fmt.Println("COMPLETE")
}
<commit_msg>Allow parallelization<commit_after>package main

import(
	"fmt"
	"os"
	"runtime"
	"github.com/eikeon/brikker/openjpeg"
)

// Buffer for holding all the output information as jp2s are examined
var jp2Messages chan string

// Channel for holding the files our workers need to process
var jp2Files chan string

func main() {
	// This seems like a solid value based on various bits of testing, but it may
	// make sense to do something smarter here
	cpuCount := runtime.NumCPU()
	maxWorkers := 4 * cpuCount
	runtime.GOMAXPROCS(cpuCount)

	// Set up channels with big enough buffers that one goroutine isn't holding
	// up another
	jp2Messages = make(chan string, maxWorkers)
	jp2Files = make(chan string, maxWorkers)

	// Read JP2 file list and stuff it into the channel
	if (len(os.Args) != 2) {
		fmt.Println("You must provide a path to a file listing JP2s to verify")
		os.Exit(1)
	}
	fileCount := readJP2FileList(os.Args[1])

	// Show anything that's "info" or more serious
	openjpeg.LogLevel = 4

	fmt.Println("BEGIN")
	fmt.Println("---")

	for i := 0; i < maxWorkers; i++ {
		go createWorker()
	}

	// Read messages until we have accounted for all JP2s
	for i := 0; i < fileCount; i++ {
		message := <-jp2Messages
		fmt.Println(message)
	}

	fmt.Println("---")
	fmt.Println("COMPLETE")
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"github.com/Syleron/PulseHA/src/netUtils"
	"github.com/Syleron/PulseHA/src/utils"
	log "github.com/Sirupsen/logrus"
	"strconv"
)

/**
Generate a new group in memory.
 */
func GroupNew(groupName string) (string, error) {
	gconf.Lock()
	defer gconf.Unlock()
	if !gconf.ClusterCheck() {
		return "", errors.New("action can only be completed in a configured cluster")
	}
	var name string
	if groupName != "" {
		name = groupName
	} else {
		name = GenGroupName()
	}
	gconf.Groups[name] = []string{}
	return name, nil
}

/**
Remove group from memory
 */
func GroupDelete(groupName string) error {
	gconf.Lock()
	defer gconf.Unlock()
	if !gconf.ClusterCheck() {
		return errors.New("action can only be completed in a configured cluster")
	}
	if GroupExist(groupName) {
		if !NodeAssignedToInterface(groupName) {
			delete(gconf.Groups, groupName)
			return nil
		}
		return errors.New("group has network interface assignments. Please remove them and try again")
	} else {
		return errors.New("unable to delete group that doesn't exist")
	}
}

/**
Clear out all local Groups
 */
func GroupClearLocal() {
	gconf.Lock()
	defer gconf.Unlock()
	gconf.Groups = map[string][]string{}
}

/**
Add floating IP to group
 */
func GroupIpAdd(groupName string, ips []string) error {
	gconf.Lock()
	defer gconf.Unlock()
	if !gconf.ClusterCheck() {
		return errors.New("action can only be completed in a configured cluster")
	}
	_, activeMember := pulse.Server.Memberlist.getActiveMember()
	if activeMember == nil {
		return errors.New("unable to add IP(s) to group as there no active node in the cluster")
	}
	if !GroupExist(groupName) {
		return errors.New("group does not exist")
	}
	for _, ip := range ips {
		if err := utils.ValidIPAddress(ip); err == nil {
			// Check to see if the IP exists in any of the groups
			if !GroupIpExistAll(ip) {
				gconf.Groups[groupName] = append(gconf.Groups[groupName], ip)
			} else {
				return errors.New(ip + " already exists in group " + groupName + ".. skipping.")
			}
		} else {
			return err
		}
	}
	return nil
}

/**
Remove floating IP from group
 */
func GroupIpRemove(groupName string, ips []string) error {
	gconf.Lock()
	defer gconf.Unlock()
	if !gconf.ClusterCheck() {
		return errors.New("action can only be completed in a configured cluster")
	}
	if !GroupExist(groupName) {
		return errors.New("group does not exist")
	}
	for _, ip := range ips {
		if len(gconf.Groups[groupName]) > 0 {
			if exists, i := GroupIPExist(groupName, ip); exists {
				gconf.Groups[groupName] = append(gconf.Groups[groupName][:i], gconf.Groups[groupName][i+1:]...)
			} else {
				log.Warning(ip + " does not exist in group " + groupName + ".. skipping.")
			}
		}
	}
	return nil
}

/**
Assign a group to a node's interface
 */
func GroupAssign(groupName, node, iface string) error {
	gconf.Lock()
	defer gconf.Unlock()
	if !gconf.ClusterCheck() {
		return errors.New("action can only be completed in a configured cluster")
	}
	if !GroupExist(groupName) {
		return errors.New("IP group does not exist")
	}
	if netUtils.InterfaceExist(iface) {
		if exists, _ := NodeInterfaceGroupExists(node, iface, groupName); !exists {
			// Add the group
			gconf.Nodes[node].IPGroups[iface] = append(gconf.Nodes[node].IPGroups[iface], groupName)
			// make the group active
			makeGroupActive(iface, groupName)
		} else {
			log.Warning(groupName + " already exists in node " + node + ".. skipping.")
		}
		return nil
	}
	return errors.New("interface does not exist")
}

/**
Unassign a group from a node's interface
 */
func GroupUnassign(groupName, node, iface string) error {
	gconf.Lock()
	defer gconf.Unlock()
	if !gconf.ClusterCheck() {
		return errors.New("action can only be completed in a configured cluster")
	}
	if !GroupExist(groupName) {
		return errors.New("IP group does not exist")
	}
	if netUtils.InterfaceExist(iface) {
		if exists, i := NodeInterfaceGroupExists(node, iface, groupName); exists {
			// make the group passive before removing it
			makeGroupPassive(iface, groupName)
			// Remove it
			gconf.Nodes[node].IPGroups[iface] = append(gconf.Nodes[node].IPGroups[iface][:i], gconf.Nodes[node].IPGroups[iface][i+1:]...)
		} else {
			log.Warning(groupName + " does not exist in node " + node + ".. skipping.")
		}
		return nil
	}
	return errors.New("interface does not exist")
}

/**
Generates an available IP floating group name.
 */
func GenGroupName() string {
	config := gconf.GetConfig()
	totalGroups := len(config.Groups)
	for i := 1; i <= totalGroups; i++ {
		newName := "group" + strconv.Itoa(i)
		if _, ok := config.Groups[newName]; !ok {
			return newName
		}
	}
	return "group" + strconv.Itoa(totalGroups+1)
}

/**
Checks to see if a floating IP group already exists
 */
func GroupExist(name string) bool {
	config := gconf.GetConfig()
	if _, ok := config.Groups[name]; ok {
		return true
	}
	return false
}

/**
Checks to see if a floating IP already exists inside of a floating ip group
 */
func GroupIPExist(name string, ip string) (bool, int) {
	config := gconf.GetConfig()
	for index, cip := range config.Groups[name] {
		if ip == cip {
			return true, index
		}
	}
	return false, -1
}

/**
Checks to see if a floating IP already exists in any of the floating IP groups
 */
func GroupIpExistAll(ip string) (bool) {
	config := gconf.GetConfig()
	for index, cip := range config.Groups {
		log.Info(cip)
		log.Info(index)
	}
	return true
}

/**
function to get the nodes and interfaces that relate to the specified node
 */
func getGroupNodes(group string) ([]string, []string) {
	var hosts []string
	var interfaces []string
	var found = false
	config := gconf.GetConfig()
	for name, node := range config.Nodes {
		for iface, groupNameSlice := range node.IPGroups {
			for _, groupName := range groupNameSlice {
				if group == groupName {
					hosts = append(hosts, name)
					interfaces = append(interfaces, iface)
					found = true
				}
			}
		}
	}
	if found {
		return hosts, interfaces
	}
	return nil, nil
}

/**
Make a group of IPs active
 */
func makeGroupActive(iface string, groupName string) {
	//log.Infof("Bringing up IPs on Interface: %s, Group: %s", iface, groupName)
	// gconf.Reload()
	configCopy := gconf.GetConfig()
	bringUpIPs(iface, configCopy.Groups[groupName])
	//garp?
}

func makeGroupPassive(iface string, groupName string) {
	//log.Infof("Bringing down IPs on Interface: %s, Group: %s", iface, groupName)
	// gconf.Reload()
	configCopy := gconf.GetConfig()
	bringDownIPs(iface, configCopy.Groups[groupName])
	//garp?
}
<commit_msg>resolved add floating IP not working issue<commit_after>package main

import (
	"errors"
	"github.com/Syleron/PulseHA/src/netUtils"
	"github.com/Syleron/PulseHA/src/utils"
	log "github.com/Sirupsen/logrus"
	"strconv"
)

/**
Generate a new group in memory.
 */
func GroupNew(groupName string) (string, error) {
	gconf.Lock()
	defer gconf.Unlock()
	if !gconf.ClusterCheck() {
		return "", errors.New("action can only be completed in a configured cluster")
	}
	var name string
	if groupName != "" {
		name = groupName
	} else {
		name = GenGroupName()
	}
	gconf.Groups[name] = []string{}
	return name, nil
}

/**
Remove group from memory
 */
func GroupDelete(groupName string) error {
	gconf.Lock()
	defer gconf.Unlock()
	if !gconf.ClusterCheck() {
		return errors.New("action can only be completed in a configured cluster")
	}
	if GroupExist(groupName) {
		if !NodeAssignedToInterface(groupName) {
			delete(gconf.Groups, groupName)
			return nil
		}
		return errors.New("group has network interface assignments. Please remove them and try again")
	} else {
		return errors.New("unable to delete group that doesn't exist")
	}
}

/**
Clear out all local Groups
 */
func GroupClearLocal() {
	gconf.Lock()
	defer gconf.Unlock()
	gconf.Groups = map[string][]string{}
}

/**
Add floating IP to group
 */
func GroupIpAdd(groupName string, ips []string) error {
	gconf.Lock()
	defer gconf.Unlock()
	if !gconf.ClusterCheck() {
		return errors.New("action can only be completed in a configured cluster")
	}
	_, activeMember := pulse.Server.Memberlist.getActiveMember()
	if activeMember == nil {
		return errors.New("unable to add IP(s) to group as there no active node in the cluster")
	}
	if !GroupExist(groupName) {
		return errors.New("group does not exist")
	}
	for _, ip := range ips {
		if err := utils.ValidIPAddress(ip); err == nil {
			// Check to see if the IP exists in any of the groups
			if !GroupIpExistAll(ip) {
				gconf.Groups[groupName] = append(gconf.Groups[groupName], ip)
			} else {
				return errors.New(ip + " already exists in group " + groupName + ".. skipping.")
			}
		} else {
			return err
		}
	}
	return nil
}

/**
Remove floating IP from group
 */
func GroupIpRemove(groupName string, ips []string) error {
	gconf.Lock()
	defer gconf.Unlock()
	if !gconf.ClusterCheck() {
		return errors.New("action can only be completed in a configured cluster")
	}
	if !GroupExist(groupName) {
		return errors.New("group does not exist")
	}
	for _, ip := range ips {
		if len(gconf.Groups[groupName]) > 0 {
			if exists, i := GroupIPExist(groupName, ip); exists {
				gconf.Groups[groupName] = append(gconf.Groups[groupName][:i], gconf.Groups[groupName][i+1:]...)
			} else {
				log.Warning(ip + " does not exist in group " + groupName + ".. skipping.")
			}
		}
	}
	return nil
}

/**
Assign a group to a node's interface
 */
func GroupAssign(groupName, node, iface string) error {
	gconf.Lock()
	defer gconf.Unlock()
	if !gconf.ClusterCheck() {
		return errors.New("action can only be completed in a configured cluster")
	}
	if !GroupExist(groupName) {
		return errors.New("IP group does not exist")
	}
	if netUtils.InterfaceExist(iface) {
		if exists, _ := NodeInterfaceGroupExists(node, iface, groupName); !exists {
			// Add the group
			gconf.Nodes[node].IPGroups[iface] = append(gconf.Nodes[node].IPGroups[iface], groupName)
			// make the group active
			makeGroupActive(iface, groupName)
		} else {
			log.Warning(groupName + " already exists in node " + node + ".. skipping.")
		}
		return nil
	}
	return errors.New("interface does not exist")
}

/**
Unassign a group from a node's interface
 */
func GroupUnassign(groupName, node, iface string) error {
	gconf.Lock()
	defer gconf.Unlock()
	if !gconf.ClusterCheck() {
		return errors.New("action can only be completed in a configured cluster")
	}
	if !GroupExist(groupName) {
		return errors.New("IP group does not exist")
	}
	if netUtils.InterfaceExist(iface) {
		if exists, i := NodeInterfaceGroupExists(node, iface, groupName); exists {
			// make the group passive before removing it
			makeGroupPassive(iface, groupName)
			// Remove it
			gconf.Nodes[node].IPGroups[iface] = append(gconf.Nodes[node].IPGroups[iface][:i], gconf.Nodes[node].IPGroups[iface][i+1:]...)
		} else {
			log.Warning(groupName + " does not exist in node " + node + ".. skipping.")
		}
		return nil
	}
	return errors.New("interface does not exist")
}

/**
Generates an available IP floating group name.
 */
func GenGroupName() string {
	config := gconf.GetConfig()
	totalGroups := len(config.Groups)
	for i := 1; i <= totalGroups; i++ {
		newName := "group" + strconv.Itoa(i)
		if _, ok := config.Groups[newName]; !ok {
			return newName
		}
	}
	return "group" + strconv.Itoa(totalGroups+1)
}

/**
Checks to see if a floating IP group already exists
 */
func GroupExist(name string) bool {
	config := gconf.GetConfig()
	if _, ok := config.Groups[name]; ok {
		return true
	}
	return false
}

/**
Checks to see if a floating IP already exists inside of a floating ip group
 */
func GroupIPExist(name string, ip string) (bool, int) {
	config := gconf.GetConfig()
	for index, cip := range config.Groups[name] {
		if ip == cip {
			return true, index
		}
	}
	return false, -1
}

/**
Checks to see if a floating IP already exists in any of the floating IP groups
 */
func GroupIpExistAll(ip string) (bool) {
	config := gconf.GetConfig()
	for _, cip := range config.Groups {
		for _, dip := range cip {
			if ip == dip {
				return true
			}
		}
	}
	return false
}

/**
function to get the nodes and interfaces that relate to the specified node
 */
func getGroupNodes(group string) ([]string, []string) {
	var hosts []string
	var interfaces []string
	var found = false
	config := gconf.GetConfig()
	for name, node := range config.Nodes {
		for iface, groupNameSlice := range node.IPGroups {
			for _, groupName := range groupNameSlice {
				if group == groupName {
					hosts = append(hosts, name)
					interfaces = append(interfaces, iface)
					found = true
				}
			}
		}
	}
	if found {
		return hosts, interfaces
	}
	return nil, nil
}

/**
Make a group of IPs active
 */
func makeGroupActive(iface string, groupName string) {
	//log.Infof("Bringing up IPs on Interface: %s, Group: %s", iface, groupName)
	// gconf.Reload()
	configCopy := gconf.GetConfig()
	bringUpIPs(iface, configCopy.Groups[groupName])
	//garp?
}

func makeGroupPassive(iface string, groupName string) {
	//log.Infof("Bringing down IPs on Interface: %s, Group: %s", iface, groupName)
	// gconf.Reload()
	configCopy := gconf.GetConfig()
	bringDownIPs(iface, configCopy.Groups[groupName])
	//garp?
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"komorebi"
	"log"
	"net/http"
	"os"
	"time"
)

var (
	Version   string
	BuildTime string
)

func main() {

	port := flag.String("port", "8080", "Listening port")
	logfile := flag.String("logfile", "", "Logfile (default stdout)")
	dump := flag.Bool("dump", false, "Dump stories, make a snapshot")
	clear_board := flag.String("clear", "", "Clears dump from given board")
	flag.StringVar(&komorebi.PublicDir, "publicdir", "public/", "Public directory")

	flag.Parse()

	if len(*logfile) > 0 {
		f, err := os.OpenFile(*logfile, os.O_APPEND|os.O_CREATE|os.O_RDWR, 0666)
		if err != nil {
			log.Printf("error opening logfile: %v", err)
		}
		defer f.Close()
		komorebi.Logger = log.New(f, "komorebi:", log.Lmicroseconds)
	} else {
		komorebi.Logger = log.New(os.Stdout, "komorebi:", log.Lmicroseconds)
	}

	komorebi.Logger.Printf("starting ")

	db := komorebi.InitDb("komorebi.db")
	db.AddTable(komorebi.Board{}, "boards")
	db.AddTable(komorebi.Column{}, "columns")
	db.AddTable(komorebi.User{}, "users")
	db.AddTable(komorebi.Task{}, "tasks")
	db.AddTable(komorebi.Dump{}, "dumps")
	db.AddTable(komorebi.BoardUsers{}, "board_users")
	db.AddTable(komorebi.TaskUsers{}, "task_users")
	tableMap := db.AddTable(komorebi.Story{}, "stories")
	tableMap.ColMap("Desc").SetMaxSize(1024)
	tableMap.ColMap("Requirements").SetMaxSize(1024)
	db.CreateTables()

	if *dump {
		komorebi.Logger.Printf("Dump tasks")
		komorebi.DumpIt()
		os.Exit(0)
	}
	if len(*clear_board) > 0 {
		komorebi.Logger.Printf("Clear dump for board", *clear_board)
		komorebi.ClearDump(*clear_board)
		os.Exit(0)
	}

	ticker := time.NewTicker(time.Hour * 24)
	go func() {
		for _ = range ticker.C {
			komorebi.DumpIt()
		}
	}()

	router := komorebi.NewRouter()
	log.Fatal(http.ListenAndServe(":"+*port, router))
}
<commit_msg>dont dump on weekend<commit_after>package main

import (
	"flag"
	"komorebi"
	"log"
	"net/http"
	"os"
	"time"
)

var (
	Version   string
	BuildTime string
)

func main() {

	port := flag.String("port", "8080", "Listening port")
	logfile := flag.String("logfile", "", "Logfile (default stdout)")
	dump := flag.Bool("dump", false, "Dump stories, make a snapshot")
	clear_board := flag.String("clear", "", "Clears dump from given board")
	flag.StringVar(&komorebi.PublicDir, "publicdir", "public/", "Public directory")

	flag.Parse()

	if len(*logfile) > 0 {
		f, err := os.OpenFile(*logfile, os.O_APPEND|os.O_CREATE|os.O_RDWR, 0666)
		if err != nil {
			log.Printf("error opening logfile: %v", err)
		}
		defer f.Close()
		komorebi.Logger = log.New(f, "komorebi:", log.Lmicroseconds)
	} else {
		komorebi.Logger = log.New(os.Stdout, "komorebi:", log.Lmicroseconds)
	}

	komorebi.Logger.Printf("starting ")

	db := komorebi.InitDb("komorebi.db")
	db.AddTable(komorebi.Board{}, "boards")
	db.AddTable(komorebi.Column{}, "columns")
	db.AddTable(komorebi.User{}, "users")
	db.AddTable(komorebi.Task{}, "tasks")
	db.AddTable(komorebi.Dump{}, "dumps")
	db.AddTable(komorebi.BoardUsers{}, "board_users")
	db.AddTable(komorebi.TaskUsers{}, "task_users")
	tableMap := db.AddTable(komorebi.Story{}, "stories")
	tableMap.ColMap("Desc").SetMaxSize(1024)
	tableMap.ColMap("Requirements").SetMaxSize(1024)
	db.CreateTables()

	if *dump {
		komorebi.Logger.Printf("Dump tasks")
		komorebi.DumpIt()
		os.Exit(0)
	}
	if len(*clear_board) > 0 {
		komorebi.Logger.Printf("Clear dump for board", *clear_board)
		komorebi.ClearDump(*clear_board)
		os.Exit(0)
	}

	ticker := time.NewTicker(time.Hour * 24)
	go func() {
		for _ = range ticker.C {
			weekday := time.Now().Weekday().String()
			if weekday == "Sunday" || weekday == "Saturday" {
				continue
			}
			komorebi.DumpIt()
		}
	}()

	router := komorebi.NewRouter()
	log.Fatal(http.ListenAndServe(":"+*port, router))
}
<|endoftext|>


<commit_before>package udsstorage

import (
	"bufio"
	"errors"
	"fmt"
	"gopkg.in/sensorbee/sensorbee.v0/bql/udf"
	"gopkg.in/sensorbee/sensorbee.v0/core"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
)

// fsUDSStorage is a UDSStorage which store states as files on a filesystem.
// This simple storage doesn't provide any version controlling capability.
// It doesn't provide checksum, either. If such capability is required, another
// UDSStorage should be implemented.
type fsUDSStorage struct {
	dirPath     string
	tempDirPath string
}

var (
	_ udf.UDSStorage = &fsUDSStorage{}
)

func NewFS(dir, tempDir string) (udf.UDSStorage, error) {
	if err := validateDir(dir); err != nil {
		return nil, fmt.Errorf("dir (%v) isn't valid: %v", dir, err)
	}
	if tempDir == "" {
		tempDir = dir
	} else if err := validateDir(tempDir); err != nil {
		return nil, fmt.Errorf("temp_dir (%v) isn't valid: %v", tempDir, err)
	}

	return &fsUDSStorage{
		dirPath:     dir,
		tempDirPath: tempDir,
	}, nil
}

func validateDir(dir string) error {
	fi, err := os.Stat(dir)
	if err != nil {
		return err
	}
	if !fi.IsDir() {
		return errors.New("it isn't a directory")
	}
	return nil
}

func (s *fsUDSStorage) Save(topology, state, tag string) (udf.UDSStorageWriter, error) {
	if tag == "" {
		tag = "default"
	}
	if err := core.ValidateSymbol(tag); err != nil {
		return nil, err
	}

	f, err := s.stateTempFile(topology, state, tag)
	if err != nil {
		return nil, err
	}
	return &onDiskUDSStorageWriter{
		s:          s,
		f:          f,
		w:          bufio.NewWriter(f),
		targetPath: s.stateFilepath(topology, state, tag),
	}, nil
}

func (s *fsUDSStorage) stateTempFile(topology, state, tag string) (*os.File, error) {
	return ioutil.TempFile(s.tempDirPath, s.stateFilename(topology, state, tag))
}

func (s *fsUDSStorage) Load(topology, state, tag string) (io.ReadCloser, error) {
	if tag == "" {
		tag = "default"
	}
	if err := core.ValidateSymbol(tag); err != nil {
		return nil, err
	}

	f, err := os.Open(s.stateFilepath(topology, state, tag))
	if err != nil {
		return nil, err
	}
	return f, nil
}

var (
	fsUDSStorageFilePathRegexp = regexp.MustCompile(`^(.+)-(.+)-(.+).state$`)
)

func (s *fsUDSStorage) ListTopologies() ([]string, error) {
	fs, err := ioutil.ReadDir(s.dirPath)
	if err != nil {
		return nil, err
	}

	res := []string{}
	for _, f := range fs {
		m := fsUDSStorageFilePathRegexp.FindStringSubmatch(f.Name())
		if m == nil {
			continue
		}
		res = append(res, m[1])
	}
	return res, nil
}

func (s *fsUDSStorage) List(topology string) (map[string][]string, error) {
	fs, err := ioutil.ReadDir(s.dirPath)
	if err != nil {
		return nil, err
	}

	res := map[string][]string{}
	for _, f := range fs {
		m := fsUDSStorageFilePathRegexp.FindStringSubmatch(f.Name())
		if m == nil {
			continue
		}
		if m[1] != topology {
			continue
		}
		res[m[2]] = append(res[m[2]], m[3])
	}
	if len(res) == 0 {
		return nil, core.NotExistError(fmt.Errorf("a topology '%v' was not found", topology))
	}
	return res, nil
}

func (s *fsUDSStorage) stateFilename(topology, state, tag string) string {
	return fmt.Sprintf("%v-%v-%v.state", topology, state, tag)
}

func (s *fsUDSStorage) stateFilepath(topology, state, tag string) string {
	return filepath.Join(s.dirPath, s.stateFilename(topology, state, tag))
}

type onDiskUDSStorageWriter struct {
	s *fsUDSStorage
	f *os.File
	w *bufio.Writer

	targetPath string
}

func (w *onDiskUDSStorageWriter) Write(data []byte) (int, error) {
	if w.w == nil {
		return 0, errors.New("writer is already closed")
	}
	return w.w.Write(data)
}

func (w *onDiskUDSStorageWriter) Commit() (err error) {
	if w.w == nil {
		return errors.New("writer is already closed")
	}
	defer func() {
		if w.f != nil {
			if e := w.f.Close(); e != nil && err == nil {
				err = e
			}
			w.f = nil
		}
	}()

	if e := w.w.Flush(); e != nil {
		return e
	}
	w.w = nil

	f := w.f
	// TODO: Name doesn't return an absolute path, so this code might fail with
	// some scenarios.
	fn := f.Name()
	w.f = nil
	if e := f.Close(); e != nil {
		return e
	}

	// This Rename might not always be atomic and application level locking
	// might be required.
	if e := os.Rename(fn, w.targetPath); e != nil {
		return e
	}
	return nil
}

func (w *onDiskUDSStorageWriter) Abort() error {
	if w.w == nil {
		return errors.New("writer is already closed")
	}
	w.w = nil

	f := w.f
	w.f = nil
	fn := f.Name()
	if err := f.Close(); err != nil {
		return err
	}
	return os.Remove(fn)
}
<commit_msg>add tag check to fs storage<commit_after>package udsstorage

import (
	"bufio"
	"errors"
	"fmt"
	"gopkg.in/sensorbee/sensorbee.v0/bql/udf"
	"gopkg.in/sensorbee/sensorbee.v0/core"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

// fsUDSStorage is a UDSStorage which store states as files on a filesystem.
// This simple storage doesn't provide any version controlling capability.
// It doesn't provide checksum, either. If such capability is required, another
// UDSStorage should be implemented.
type fsUDSStorage struct {
	dirPath     string
	tempDirPath string
}

var (
	_ udf.UDSStorage = &fsUDSStorage{}
)

func NewFS(dir, tempDir string) (udf.UDSStorage, error) {
	if err := validateDir(dir); err != nil {
		return nil, fmt.Errorf("dir (%v) isn't valid: %v", dir, err)
	}
	if tempDir == "" {
		tempDir = dir
	} else if err := validateDir(tempDir); err != nil {
		return nil, fmt.Errorf("temp_dir (%v) isn't valid: %v", tempDir, err)
	}

	return &fsUDSStorage{
		dirPath:     dir,
		tempDirPath: tempDir,
	}, nil
}

func validateDir(dir string) error {
	fi, err := os.Stat(dir)
	if err != nil {
		return err
	}
	if !fi.IsDir() {
		return errors.New("it isn't a directory")
	}
	return nil
}

func (s *fsUDSStorage) Save(topology, state, tag string) (udf.UDSStorageWriter, error) {
	if tag == "" || strings.ToLower(tag) == "default" {
		tag = "default"
	} else if err := core.ValidateSymbol(tag); err != nil {
		return nil, err
	}

	f, err := s.stateTempFile(topology, state, tag)
	if err != nil {
		return nil, err
	}
	return &onDiskUDSStorageWriter{
		s:          s,
		f:          f,
		w:          bufio.NewWriter(f),
		targetPath: s.stateFilepath(topology, state, tag),
	}, nil
}

func (s *fsUDSStorage) stateTempFile(topology, state, tag string) (*os.File, error) {
	return ioutil.TempFile(s.tempDirPath, s.stateFilename(topology, state, tag))
}

func (s *fsUDSStorage) Load(topology, state, tag string) (io.ReadCloser, error) {
	if tag == "" || strings.ToLower(tag) == "default" {
		tag = "default"
	} else if err := core.ValidateSymbol(tag); err != nil {
		return nil, err
	}

	f, err := os.Open(s.stateFilepath(topology, state, tag))
	if err != nil {
		return nil, err
	}
	return f, nil
}

var (
	fsUDSStorageFilePathRegexp = regexp.MustCompile(`^(.+)-(.+)-(.+).state$`)
)

func (s *fsUDSStorage) ListTopologies() ([]string, error) {
	fs, err := ioutil.ReadDir(s.dirPath)
	if err != nil {
		return nil, err
	}

	res := []string{}
	for _, f := range fs {
		m := fsUDSStorageFilePathRegexp.FindStringSubmatch(f.Name())
		if m == nil {
			continue
		}
		res = append(res, m[1])
	}
	return res, nil
}

func (s *fsUDSStorage) List(topology string) (map[string][]string, error) {
	fs, err := ioutil.ReadDir(s.dirPath)
	if err != nil {
		return nil, err
	}

	res := map[string][]string{}
	for _, f := range fs {
		m := fsUDSStorageFilePathRegexp.FindStringSubmatch(f.Name())
		if m == nil {
			continue
		}
		if m[1] != topology {
			continue
		}
		res[m[2]] = append(res[m[2]], m[3])
	}
	if len(res) == 0 {
		return nil, core.NotExistError(fmt.Errorf("a topology '%v' was not found", topology))
	}
	return res, nil
}

func (s *fsUDSStorage) stateFilename(topology, state, tag string) string {
	return fmt.Sprintf("%v-%v-%v.state", topology, state, tag)
}

func (s *fsUDSStorage) stateFilepath(topology, state, tag string) string {
	return filepath.Join(s.dirPath, s.stateFilename(topology, state, tag))
}

type onDiskUDSStorageWriter struct {
	s *fsUDSStorage
	f *os.File
	w *bufio.Writer

	targetPath string
}

func (w *onDiskUDSStorageWriter) Write(data []byte) (int, error) {
	if w.w == nil {
		return 0, errors.New("writer is already closed")
	}
	return w.w.Write(data)
}

func (w *onDiskUDSStorageWriter) Commit() (err error) {
	if w.w == nil {
		return errors.New("writer is already closed")
	}
	defer func() {
		if w.f != nil {
			if e := w.f.Close(); e != nil && err == nil {
				err = e
			}
			w.f = nil
		}
	}()

	if e := w.w.Flush(); e != nil {
		return e
	}
	w.w = nil

	f := w.f
	// TODO: Name doesn't return an absolute path, so this code might fail with
	// some scenarios.
	fn := f.Name()
	w.f = nil
	if e := f.Close(); e != nil {
		return e
	}

	// This Rename might not always be atomic and application level locking
	// might be required.
	if e := os.Rename(fn, w.targetPath); e != nil {
		return e
	}
	return nil
}

func (w *onDiskUDSStorageWriter) Abort() error {
	if w.w == nil {
		return errors.New("writer is already closed")
	}
	w.w = nil

	f := w.f
	w.f = nil
	fn := f.Name()
	if err := f.Close(); err != nil {
		return err
	}
	return os.Remove(fn)
}
<|endoftext|>


<commit_before>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package reflectcodec

import (
	"errors"
	"fmt"
	"reflect"

	"github.com/ava-labs/avalanchego/codec"
	"github.com/ava-labs/avalanchego/utils/wrappers"
)

const (
	// DefaultTagName that enables serialization.
	DefaultTagName = "serialize"
)

var (
	errMarshalNil   = errors.New("can't marshal nil pointer or interface")
	errUnmarshalNil = errors.New("can't unmarshal nil")
	errNeedPointer  = errors.New("argument to unmarshal must be a pointer")
	errExtraSpace   = errors.New("trailing buffer space")
)

type TypeCodec interface {
	UnpackPrefix(*wrappers.Packer, reflect.Type) (reflect.Value, error)
	PackPrefix(*wrappers.Packer, reflect.Type) error
}

// Codec handles marshaling and unmarshaling of structs
type genericCodec struct {
	typer       TypeCodec
	maxSliceLen int
	fielder     StructFielder
}

// New returns a new, concurrency-safe codec
func New(typer TypeCodec, tagName string, maxSliceLen int) codec.Codec {
	return &genericCodec{
		typer:       typer,
		maxSliceLen: maxSliceLen,
		fielder:     NewStructFielder(tagName, maxSliceLen),
	}
}

// A few notes:
// 1) See codec_test.go for examples of usage
// 2) We use "marshal" and "serialize" interchangeably, and "unmarshal" and "deserialize" interchangeably
// 3) To include a field of a struct in the serialized form, add the tag `{tagName}:"true"` to it. `{tagName}` defaults to `serialize`.
// 4) These typed members of a struct may be serialized:
//    bool, string, uint[8,16,32,64], int[8,16,32,64],
//	  structs, slices, arrays, interface.
//	  structs, slices and arrays can only be serialized if their constituent values can be.
// 5) To marshal an interface, you must pass a pointer to the value
// 6) To unmarshal an interface,  you must call codec.RegisterType([instance of the type that fulfills the interface]).
// 7) Serialized fields must be exported
// 8) nil slices are marshaled as empty slices

// To marshal an interface, [value] must be a pointer to the interface
func (c *genericCodec) MarshalInto(value interface{}, p *wrappers.Packer) error {
	if value == nil {
		return errMarshalNil // can't marshal nil
	}

	return c.marshal(reflect.ValueOf(value), p, c.maxSliceLen)
}

// marshal writes the byte representation of [value] to [p]
// [value]'s underlying value must not be a nil pointer or interface
// c.lock should be held for the duration of this function
func (c *genericCodec) marshal(value reflect.Value, p *wrappers.Packer, maxSliceLen int) error {
	valueKind := value.Kind()
	switch valueKind {
	case reflect.Interface, reflect.Ptr, reflect.Invalid:
		if value.IsNil() { // Can't marshal nil (except nil slices)
			return errMarshalNil
		}
	}

	switch valueKind {
	case reflect.Uint8:
		p.PackByte(uint8(value.Uint()))
		return p.Err
	case reflect.Int8:
		p.PackByte(uint8(value.Int()))
		return p.Err
	case reflect.Uint16:
		p.PackShort(uint16(value.Uint()))
		return p.Err
	case reflect.Int16:
		p.PackShort(uint16(value.Int()))
		return p.Err
	case reflect.Uint32:
		p.PackInt(uint32(value.Uint()))
		return p.Err
	case reflect.Int32:
		p.PackInt(uint32(value.Int()))
		return p.Err
	case reflect.Uint64:
		p.PackLong(value.Uint())
		return p.Err
	case reflect.Int64:
		p.PackLong(uint64(value.Int()))
		return p.Err
	case reflect.String:
		p.PackStr(value.String())
		return p.Err
	case reflect.Bool:
		p.PackBool(value.Bool())
		return p.Err
	case reflect.Uintptr, reflect.Ptr:
		return c.marshal(value.Elem(), p, c.maxSliceLen)
	case reflect.Interface:
		underlyingValue := value.Interface()
		underlyingType := reflect.TypeOf(underlyingValue)
		if err := c.typer.PackPrefix(p, underlyingType); err != nil {
			return err
		}
		if err := c.marshal(value.Elem(), p, c.maxSliceLen); err != nil {
			return err
		}
		return p.Err
	case reflect.Slice:
		numElts := value.Len() // # elements in the slice/array. 0 if this slice is nil.
		if numElts > maxSliceLen {
			return fmt.Errorf("slice length, %d, exceeds maximum length, %d",
				numElts,
				maxSliceLen)
		}
		p.PackInt(uint32(numElts)) // pack # elements
		if p.Err != nil {
			return p.Err
		}
		// If this is a slice of bytes, manually pack the bytes rather
		// than calling marshal on each byte. This improves performance.
		if elemKind := value.Type().Elem().Kind(); elemKind == reflect.Uint8 {
			p.PackFixedBytes(value.Bytes())
			return p.Err
		}
		for i := 0; i < numElts; i++ { // Process each element in the slice
			if err := c.marshal(value.Index(i), p, c.maxSliceLen); err != nil {
				return err
			}
		}
		return nil
	case reflect.Array:
		numElts := value.Len()
		if elemKind := value.Type().Kind(); elemKind == reflect.Uint8 {
			sliceVal := value.Convert(reflect.TypeOf([]byte{}))
			p.PackFixedBytes(sliceVal.Bytes())
			return p.Err
		}
		if numElts > c.maxSliceLen {
			return fmt.Errorf("array length, %d, exceeds maximum length, %d", numElts, c.maxSliceLen)
		}
		for i := 0; i < numElts; i++ { // Process each element in the array
			if err := c.marshal(value.Index(i), p, c.maxSliceLen); err != nil {
				return err
			}
		}
		return nil
	case reflect.Struct:
		serializedFields, err := c.fielder.GetSerializedFields(value.Type())
		if err != nil {
			return err
		}
		for _, fieldDesc := range serializedFields { // Go through all fields of this struct that are serialized
			if err := c.marshal(value.Field(fieldDesc.Index), p, fieldDesc.MaxSliceLen); err != nil { // Serialize the field and write to byte array
				return err
			}
		}
		return nil
	default:
		return fmt.Errorf("can't marshal unknown kind %s", valueKind)
	}
}

// Unmarshal unmarshals [bytes] into [dest], where
// [dest] must be a pointer or interface
func (c *genericCodec) Unmarshal(bytes []byte, dest interface{}) error {
	if dest == nil {
		return errUnmarshalNil
	}

	p := wrappers.Packer{
		Bytes: bytes,
	}
	destPtr := reflect.ValueOf(dest)
	if destPtr.Kind() != reflect.Ptr {
		return errNeedPointer
	}
	if err := c.unmarshal(&p, destPtr.Elem(), c.maxSliceLen); err != nil {
		return err
	}
	if p.Offset != len(bytes) {
		return errExtraSpace
	}
	return nil
}

// Unmarshal from p.Bytes into [value]. [value] must be addressable.
// c.lock should be held for the duration of this function
func (c *genericCodec) unmarshal(p *wrappers.Packer, value reflect.Value, maxSliceLen int) error {
	switch value.Kind() {
	case reflect.Uint8:
		value.SetUint(uint64(p.UnpackByte()))
		if p.Err != nil {
			return fmt.Errorf("couldn't unmarshal uint8: %w", p.Err)
		}
		return nil
	case reflect.Int8:
		value.SetInt(int64(p.UnpackByte()))
		if p.Err != nil {
			return fmt.Errorf("couldn't unmarshal int8: %w", p.Err)
		}
		return nil
	case reflect.Uint16:
		value.SetUint(uint64(p.UnpackShort()))
		if p.Err != nil {
			return fmt.Errorf("couldn't unmarshal uint16: %w", p.Err)
		}
		return nil
	case reflect.Int16:
		value.SetInt(int64(p.UnpackShort()))
		if p.Err != nil {
			return fmt.Errorf("couldn't unmarshal int16: %w", p.Err)
		}
		return nil
	case reflect.Uint32:
		value.SetUint(uint64(p.UnpackInt()))
		if p.Err != nil {
			return fmt.Errorf("couldn't unmarshal uint32: %w", p.Err)
		}
		return nil
	case reflect.Int32:
		value.SetInt(int64(p.UnpackInt()))
		if p.Err != nil {
			return fmt.Errorf("couldn't unmarshal int32: %w", p.Err)
		}
		return nil
	case reflect.Uint64:
		value.SetUint(p.UnpackLong())
		if p.Err != nil {
			return fmt.Errorf("couldn't unmarshal uint64: %w", p.Err)
		}
		return nil
	case reflect.Int64:
		value.SetInt(int64(p.UnpackLong()))
		if p.Err != nil {
			return fmt.Errorf("couldn't unmarshal int64: %w", p.Err)
		}
		return nil
	case reflect.Bool:
		value.SetBool(p.UnpackBool())
		if p.Err != nil {
			return fmt.Errorf("couldn't unmarshal bool: %w", p.Err)
		}
		return nil
	case reflect.Slice:
		numElts := int(p.UnpackInt())
		if p.Err != nil {
			return fmt.Errorf("couldn't unmarshal slice: %w", p.Err)
		}
		if numElts > maxSliceLen {
			return fmt.Errorf("array length, %d, exceeds maximum length, %d",
				numElts,
				maxSliceLen)
		}
		// If this is a slice of bytes, manually unpack the bytes rather
		// than calling unmarshal on each byte. This improves performance.
		if elemKind := value.Type().Elem().Kind(); elemKind == reflect.Uint8 {
			value.SetBytes(p.UnpackFixedBytes(numElts))
			return p.Err
		}
		// set [value] to be a slice of the appropriate type/capacity (right now it is nil)
		value.Set(reflect.MakeSlice(value.Type(), numElts, numElts))
		// Unmarshal each element into the appropriate index of the slice
		for i := 0; i < numElts; i++ {
			if err := c.unmarshal(p, value.Index(i), c.maxSliceLen); err != nil {
				return fmt.Errorf("couldn't unmarshal slice element: %w", err)
			}
		}
		return nil
	case reflect.Array:
		numElts := value.Len()
		if elemKind := value.Type().Elem().Kind(); elemKind == reflect.Uint8 {
			unpackedBytes := p.UnpackFixedBytes(numElts)
			if p.Errored() {
				return p.Err
			}
			// Get a slice to the underlying array value
			underlyingSlice := value.Slice(0, numElts).Interface().([]byte)
			copy(underlyingSlice, unpackedBytes)
			return nil
		}
		for i := 0; i < numElts; i++ {
			if err := c.unmarshal(p, value.Index(i), c.maxSliceLen); err != nil {
				return fmt.Errorf("couldn't unmarshal array element: %w", err)
			}
		}
		return nil
	case reflect.String:
		value.SetString(p.UnpackStr())
		if p.Err != nil {
			return fmt.Errorf("couldn't unmarshal string: %w", p.Err)
		}
		return nil
	case reflect.Interface:
		intfImplementor, err := c.typer.UnpackPrefix(p, value.Type())
		if err != nil {
			return err
		}
		// Unmarshal into the struct
		if err := c.unmarshal(p, intfImplementor, c.maxSliceLen); err != nil {
			return fmt.Errorf("couldn't unmarshal interface: %w", err)
		}
		// And assign the filled struct to the value
		value.Set(intfImplementor)
		return nil
	case reflect.Struct:
		// Get indices of fields that will be unmarshaled into
		serializedFieldIndices, err := c.fielder.GetSerializedFields(value.Type())
		if err != nil {
			return fmt.Errorf("couldn't unmarshal struct: %w", err)
		}
		// Go through the fields and umarshal into them
		for _, fieldDesc := range serializedFieldIndices {
			if err := c.unmarshal(p, value.Field(fieldDesc.Index), fieldDesc.MaxSliceLen); err != nil {
				return fmt.Errorf("couldn't unmarshal struct: %w", err)
			}
		}
		return nil
	case reflect.Ptr:
		// Get the type this pointer points to
		t := value.Type().Elem()
		// Create a new pointer to a new value of the underlying type
		v := reflect.New(t)
		// Fill the value
		if err := c.unmarshal(p, v.Elem(), c.maxSliceLen); err != nil {
			return fmt.Errorf("couldn't unmarshal pointer: %w", err)
		}
		// Assign to the top-level struct's member
		value.Set(v)
		return nil
	case reflect.Invalid:
		return errUnmarshalNil
	default:
		return fmt.Errorf("can't unmarshal unknown type %s", value.Kind().String())
	}
}
<commit_msg>added comments from PR<commit_after>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package reflectcodec

import (
	"errors"
	"fmt"
	"reflect"

	"github.com/ava-labs/avalanchego/codec"
	"github.com/ava-labs/avalanchego/utils/wrappers"
)

const (
	// DefaultTagName that enables serialization.
	DefaultTagName = "serialize"
)

var (
	errMarshalNil   = errors.New("can't marshal nil pointer or interface")
	errUnmarshalNil = errors.New("can't unmarshal nil")
	errNeedPointer  = errors.New("argument to unmarshal must be a pointer")
	errExtraSpace   = errors.New("trailing buffer space")
)

type TypeCodec interface {
	// UnpackPrefix unpacks the prefix of an interface from the given packer.
	// The prefix specifies the concrete type that the interface should be
	// deserialized into. This function returns a new instance of that concrete
	// type. The concrete type must implement the given type.
	UnpackPrefix(*wrappers.Packer, reflect.Type) (reflect.Value, error)

	// PackPrefix packs the prefix for the given type into the given packer.
	// This identifies the bytes that follow, which are the byte representation
	// of an interface, as having the given concrete type.
	// When deserializing the bytes, the prefix specifies which concrete type
	// to deserialize into.
	PackPrefix(*wrappers.Packer, reflect.Type) error
}

// Codec handles marshaling and unmarshaling of structs
type genericCodec struct {
	typer       TypeCodec
	maxSliceLen int
	fielder     StructFielder
}

// New returns a new, concurrency-safe codec
func New(typer TypeCodec, tagName string, maxSliceLen int) codec.Codec {
	return &genericCodec{
		typer:       typer,
		maxSliceLen: maxSliceLen,
		fielder:     NewStructFielder(tagName, maxSliceLen),
	}
}

// A few notes:
// 1) See codec_test.go for examples of usage
// 2) We use "marshal" and "serialize" interchangeably, and "unmarshal" and "deserialize" interchangeably
// 3) To include a field of a struct in the serialized form, add the tag `{tagName}:"true"` to it. `{tagName}` defaults to `serialize`.
// 4) These typed members of a struct may be serialized:
//    bool, string, uint[8,16,32,64], int[8,16,32,64],
//	  structs, slices, arrays, interface.
//	  structs, slices and arrays can only be serialized if their constituent values can be.
// 5) To marshal an interface, you must pass a pointer to the value
// 6) To unmarshal an interface,  you must call codec.RegisterType([instance of the type that fulfills the interface]).
// 7) Serialized fields must be exported
// 8) nil slices are marshaled as empty slices

// To marshal an interface, [value] must be a pointer to the interface
func (c *genericCodec) MarshalInto(value interface{}, p *wrappers.Packer) error {
	if value == nil {
		return errMarshalNil // can't marshal nil
	}

	return c.marshal(reflect.ValueOf(value), p, c.maxSliceLen)
}

// marshal writes the byte representation of [value] to [p]
// [value]'s underlying value must not be a nil pointer or interface
// c.lock should be held for the duration of this function
func (c *genericCodec) marshal(value reflect.Value, p *wrappers.Packer, maxSliceLen int) error {
	valueKind := value.Kind()
	switch valueKind {
	case reflect.Interface, reflect.Ptr, reflect.Invalid:
		if value.IsNil() { // Can't marshal nil (except nil slices)
			return errMarshalNil
		}
	}

	switch valueKind {
	case reflect.Uint8:
		p.PackByte(uint8(value.Uint()))
		return p.Err
	case reflect.Int8:
		p.PackByte(uint8(value.Int()))
		return p.Err
	case reflect.Uint16:
		p.PackShort(uint16(value.Uint()))
		return p.Err
	case reflect.Int16:
		p.PackShort(uint16(value.Int()))
		return p.Err
	case reflect.Uint32:
		p.PackInt(uint32(value.Uint()))
		return p.Err
	case reflect.Int32:
		p.PackInt(uint32(value.Int()))
		return p.Err
	case reflect.Uint64:
		p.PackLong(value.Uint())
		return p.Err
	case reflect.Int64:
		p.PackLong(uint64(value.Int()))
		return p.Err
	case reflect.String:
		p.PackStr(value.String())
		return p.Err
	case reflect.Bool:
		p.PackBool(value.Bool())
		return p.Err
	case reflect.Uintptr, reflect.Ptr:
		return c.marshal(value.Elem(), p, c.maxSliceLen)
	case reflect.Interface:
		underlyingValue := value.Interface()
		underlyingType := reflect.TypeOf(underlyingValue)
		if err := c.typer.PackPrefix(p, underlyingType); err != nil {
			return err
		}
		if err := c.marshal(value.Elem(), p, c.maxSliceLen); err != nil {
			return err
		}
		return p.Err
	case reflect.Slice:
		numElts := value.Len() // # elements in the slice/array. 0 if this slice is nil.
		if numElts > maxSliceLen {
			return fmt.Errorf("slice length, %d, exceeds maximum length, %d",
				numElts,
				maxSliceLen)
		}
		p.PackInt(uint32(numElts)) // pack # elements
		if p.Err != nil {
			return p.Err
		}
		// If this is a slice of bytes, manually pack the bytes rather
		// than calling marshal on each byte. This improves performance.
		if elemKind := value.Type().Elem().Kind(); elemKind == reflect.Uint8 {
			p.PackFixedBytes(value.Bytes())
			return p.Err
		}
		for i := 0; i < numElts; i++ { // Process each element in the slice
			if err := c.marshal(value.Index(i), p, c.maxSliceLen); err != nil {
				return err
			}
		}
		return nil
	case reflect.Array:
		numElts := value.Len()
		if elemKind := value.Type().Kind(); elemKind == reflect.Uint8 {
			sliceVal := value.Convert(reflect.TypeOf([]byte{}))
			p.PackFixedBytes(sliceVal.Bytes())
			return p.Err
		}
		if numElts > c.maxSliceLen {
			return fmt.Errorf("array length, %d, exceeds maximum length, %d", numElts, c.maxSliceLen)
		}
		for i := 0; i < numElts; i++ { // Process each element in the array
			if err := c.marshal(value.Index(i), p, c.maxSliceLen); err != nil {
				return err
			}
		}
		return nil
	case reflect.Struct:
		serializedFields, err := c.fielder.GetSerializedFields(value.Type())
		if err != nil {
			return err
		}
		for _, fieldDesc := range serializedFields { // Go through all fields of this struct that are serialized
			if err := c.marshal(value.Field(fieldDesc.Index), p, fieldDesc.MaxSliceLen); err != nil { // Serialize the field and write to byte array
				return err
			}
		}
		return nil
	default:
		return fmt.Errorf("can't marshal unknown kind %s", valueKind)
	}
}

// Unmarshal unmarshals [bytes] into [dest], where
// [dest] must be a pointer or interface
func (c *genericCodec) Unmarshal(bytes []byte, dest interface{}) error {
	if dest == nil {
		return errUnmarshalNil
	}

	p := wrappers.Packer{
		Bytes: bytes,
	}
	destPtr := reflect.ValueOf(dest)
	if destPtr.Kind() != reflect.Ptr {
		return errNeedPointer
	}
	if err := c.unmarshal(&p, destPtr.Elem(), c.maxSliceLen); err != nil {
		return err
	}
	if p.Offset != len(bytes) {
		return errExtraSpace
	}
	return nil
}

// Unmarshal from p.Bytes into [value]. [value] must be addressable.
// c.lock should be held for the duration of this function
func (c *genericCodec) unmarshal(p *wrappers.Packer, value reflect.Value, maxSliceLen int) error {
	switch value.Kind() {
	case reflect.Uint8:
		value.SetUint(uint64(p.UnpackByte()))
		if p.Err != nil {
			return fmt.Errorf("couldn't unmarshal uint8: %w", p.Err)
		}
		return nil
	case reflect.Int8:
		value.SetInt(int64(p.UnpackByte()))
		if p.Err != nil {
			return fmt.Errorf("couldn't unmarshal int8: %w", p.Err)
		}
		return nil
	case reflect.Uint16:
		value.SetUint(uint64(p.UnpackShort()))
		if p.Err != nil {
			return fmt.Errorf("couldn't unmarshal uint16: %w", p.Err)
		}
		return nil
	case reflect.Int16:
		value.SetInt(int64(p.UnpackShort()))
		if p.Err != nil {
			return fmt.Errorf("couldn't unmarshal int16: %w", p.Err)
		}
		return nil
	case reflect.Uint32:
		value.SetUint(uint64(p.UnpackInt()))
		if p.Err != nil {
			return fmt.Errorf("couldn't unmarshal uint32: %w", p.Err)
		}
		return nil
	case reflect.Int32:
		value.SetInt(int64(p.UnpackInt()))
		if p.Err != nil {
			return fmt.Errorf("couldn't unmarshal int32: %w", p.Err)
		}
		return nil
	case reflect.Uint64:
		value.SetUint(p.UnpackLong())
		if p.Err != nil {
			return fmt.Errorf("couldn't unmarshal uint64: %w", p.Err)
		}
		return nil
	case reflect.Int64:
		value.SetInt(int64(p.UnpackLong()))
		if p.Err != nil {
			return fmt.Errorf("couldn't unmarshal int64: %w", p.Err)
		}
		return nil
	case reflect.Bool:
		value.SetBool(p.UnpackBool())
		if p.Err != nil {
			return fmt.Errorf("couldn't unmarshal bool: %w", p.Err)
		}
		return nil
	case reflect.Slice:
		numElts := int(p.UnpackInt())
		if p.Err != nil {
			return fmt.Errorf("couldn't unmarshal slice: %w", p.Err)
		}
		if numElts > maxSliceLen {
			return fmt.Errorf("array length, %d, exceeds maximum length, %d",
				numElts,
				maxSliceLen)
		}
		// If this is a slice of bytes, manually unpack the bytes rather
		// than calling unmarshal on each byte. This improves performance.
		if elemKind := value.Type().Elem().Kind(); elemKind == reflect.Uint8 {
			value.SetBytes(p.UnpackFixedBytes(numElts))
			return p.Err
		}
		// set [value] to be a slice of the appropriate type/capacity (right now it is nil)
		value.Set(reflect.MakeSlice(value.Type(), numElts, numElts))
		// Unmarshal each element into the appropriate index of the slice
		for i := 0; i < numElts; i++ {
			if err := c.unmarshal(p, value.Index(i), c.maxSliceLen); err != nil {
				return fmt.Errorf("couldn't unmarshal slice element: %w", err)
			}
		}
		return nil
	case reflect.Array:
		numElts := value.Len()
		if elemKind := value.Type().Elem().Kind(); elemKind == reflect.Uint8 {
			unpackedBytes := p.UnpackFixedBytes(numElts)
			if p.Errored() {
				return p.Err
			}
			// Get a slice to the underlying array value
			underlyingSlice := value.Slice(0, numElts).Interface().([]byte)
			copy(underlyingSlice, unpackedBytes)
			return nil
		}
		for i := 0; i < numElts; i++ {
			if err := c.unmarshal(p, value.Index(i), c.maxSliceLen); err != nil {
				return fmt.Errorf("couldn't unmarshal array element: %w", err)
			}
		}
		return nil
	case reflect.String:
		value.SetString(p.UnpackStr())
		if p.Err != nil {
			return fmt.Errorf("couldn't unmarshal string: %w", p.Err)
		}
		return nil
	case reflect.Interface:
		intfImplementor, err := c.typer.UnpackPrefix(p, value.Type())
		if err != nil {
			return err
		}
		// Unmarshal into the struct
		if err := c.unmarshal(p, intfImplementor, c.maxSliceLen); err != nil {
			return fmt.Errorf("couldn't unmarshal interface: %w", err)
		}
		// And assign the filled struct to the value
		value.Set(intfImplementor)
		return nil
	case reflect.Struct:
		// Get indices of fields that will be unmarshaled into
		serializedFieldIndices, err := c.fielder.GetSerializedFields(value.Type())
		if err != nil {
			return fmt.Errorf("couldn't unmarshal struct: %w", err)
		}
		// Go through the fields and umarshal into them
		for _, fieldDesc := range serializedFieldIndices {
			if err := c.unmarshal(p, value.Field(fieldDesc.Index), fieldDesc.MaxSliceLen); err != nil {
				return fmt.Errorf("couldn't unmarshal struct: %w", err)
			}
		}
		return nil
	case reflect.Ptr:
		// Get the type this pointer points to
		t := value.Type().Elem()
		// Create a new pointer to a new value of the underlying type
		v := reflect.New(t)
		// Fill the value
		if err := c.unmarshal(p, v.Elem(), c.maxSliceLen); err != nil {
			return fmt.Errorf("couldn't unmarshal pointer: %w", err)
		}
		// Assign to the top-level struct's member
		value.Set(v)
		return nil
	case reflect.Invalid:
		return errUnmarshalNil
	default:
		return fmt.Errorf("can't unmarshal unknown type %s", value.Kind().String())
	}
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/git-lfs/git-lfs/errors"
	"github.com/git-lfs/git-lfs/git/odb"
	"github.com/git-lfs/git-lfs/tools"
	"github.com/git-lfs/git-lfs/tools/humanize"
	"github.com/spf13/cobra"
)

var (
	// migrateInfoTopN is a flag given to the git-lfs-migrate(1) subcommand
	// 'info' which specifies how many info entries to show by default.
	migrateInfoTopN int

	// migrateInfoAboveFmt is a flag given to the git-lfs-migrate(1)
	// subcommand 'info' specifying a human-readable string threshold of
	// filesize before entries are counted.
	migrateInfoAboveFmt string
	// migrateInfoAbove is the number of bytes parsed from the above
	// migrateInfoAboveFmt flag.
	migrateInfoAbove uint64
)

func migrateInfoCommand(cmd *cobra.Command, args []string) {
	exts := make(map[string]*MigrateInfoEntry)

	above, err := humanize.ParseBytes(migrateInfoAboveFmt)
	if err != nil {
		ExitWithError(errors.Wrap(err, "cannot parse --above=<n>"))
	}

	migrateInfoAbove = above

	migrate(cmd, args, func(path string, b *odb.Blob) (*odb.Blob, error) {
		ext := fmt.Sprintf("*%s", filepath.Ext(path))

		if len(ext) > 1 {
			entry := exts[ext]
			if entry == nil {
				entry = &MigrateInfoEntry{Qualifier: ext}
			}

			entry.Total++
			entry.BytesTotal += b.Size

			if b.Size > int64(migrateInfoAbove) {
				entry.TotalAbove++
				entry.BytesAbove += b.Size
			}

			exts[ext] = entry
		}

		return b, nil
	})

	entries := EntriesBySize(MapToEntries(exts))
	sort.Sort(sort.Reverse(entries))

	migrateInfoTopN = tools.ClampInt(migrateInfoTopN, len(entries), 0)

	entries = entries[:tools.MaxInt(0, migrateInfoTopN)]

	entries.Print(os.Stderr)
}

// MigrateInfoEntry represents a tuple of filetype to bytes and entry count
// above and below a threshold.
type MigrateInfoEntry struct {
	// Qualifier is the filepath's extension.
	Qualifier string

	// BytesAbove is total size of all files above a given threshold.
	BytesAbove int64
	// TotalAbove is the count of all files above a given size threshold.
	TotalAbove int64
	// BytesTotal is the number of bytes of all files
	BytesTotal int64
	// Total is the count of all files.
	Total int64
}

// MapToEntries creates a set of `*MigrateInfoEntry`'s for a given map of
// filepath extensions to file size in bytes.
func MapToEntries(exts map[string]*MigrateInfoEntry) []*MigrateInfoEntry {
	entries := make([]*MigrateInfoEntry, 0, len(exts))
	for _, entry := range exts {
		entries = append(entries, entry)
	}

	return entries
}

// EntriesBySize is an implementation of sort.Interface that sorts a set of
// `*MigrateInfoEntry`'s
type EntriesBySize []*MigrateInfoEntry

// Len returns the total length of the set of `*MigrateInfoEntry`'s.
func (e EntriesBySize) Len() int { return len(e) }

// Less returns the whether or not the MigrateInfoEntry given at `i` takes up
// less total size than the MigrateInfoEntry given at `j`.
func (e EntriesBySize) Less(i, j int) bool { return e[i].BytesAbove < e[j].BytesAbove }

// Swap swaps the entries given at i, j.
func (e EntriesBySize) Swap(i, j int) { e[i], e[j] = e[j], e[i] }

// Print formats the `*MigrateInfoEntry`'s in the set and prints them to the
// given io.Writer, "to", returning "n" the number of bytes written, and any
// error, if one occurred.
func (e EntriesBySize) Print(to io.Writer) (int, error) {
	extensions := make([]string, 0, len(e))
	sizes := make([]string, 0, len(e))
	stats := make([]string, 0, len(e))
	percentages := make([]string, 0, len(e))

	for _, entry := range e {
		above := entry.TotalAbove
		total := entry.Total
		percentAbove := 100 * (float64(above) / float64(total))

		stat := fmt.Sprintf("%d/%d files(s)",
			above, total)

		percentage := fmt.Sprintf("%.0f%%", percentAbove)

		extensions = append(extensions, entry.Qualifier)
		sizes = append(sizes, humanize.FormatBytes(uint64(entry.BytesAbove)))
		stats = append(stats, stat)
		percentages = append(percentages, percentage)
	}

	extensions = tools.Ljust(extensions)
	sizes = tools.Ljust(sizes)
	stats = tools.Rjust(stats)
	percentages = tools.Rjust(percentages)

	output := make([]string, 0, len(e))
	for i := 0; i < len(e); i++ {
		extension := extensions[i]
		size := sizes[i]
		stat := stats[i]
		percentage := percentages[i]

		line := strings.Join([]string{extension, size, stat, percentage}, "\t")

		output = append(output, line)
	}

	return fmt.Fprintln(to, strings.Join(output, "\n"))
}
<commit_msg>commands/command_migrate_info: print entries to os.Stdout<commit_after>package commands

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/git-lfs/git-lfs/errors"
	"github.com/git-lfs/git-lfs/git/odb"
	"github.com/git-lfs/git-lfs/tools"
	"github.com/git-lfs/git-lfs/tools/humanize"
	"github.com/spf13/cobra"
)

var (
	// migrateInfoTopN is a flag given to the git-lfs-migrate(1) subcommand
	// 'info' which specifies how many info entries to show by default.
	migrateInfoTopN int

	// migrateInfoAboveFmt is a flag given to the git-lfs-migrate(1)
	// subcommand 'info' specifying a human-readable string threshold of
	// filesize before entries are counted.
	migrateInfoAboveFmt string
	// migrateInfoAbove is the number of bytes parsed from the above
	// migrateInfoAboveFmt flag.
	migrateInfoAbove uint64
)

func migrateInfoCommand(cmd *cobra.Command, args []string) {
	exts := make(map[string]*MigrateInfoEntry)

	above, err := humanize.ParseBytes(migrateInfoAboveFmt)
	if err != nil {
		ExitWithError(errors.Wrap(err, "cannot parse --above=<n>"))
	}

	migrateInfoAbove = above

	migrate(cmd, args, func(path string, b *odb.Blob) (*odb.Blob, error) {
		ext := fmt.Sprintf("*%s", filepath.Ext(path))

		if len(ext) > 1 {
			entry := exts[ext]
			if entry == nil {
				entry = &MigrateInfoEntry{Qualifier: ext}
			}

			entry.Total++
			entry.BytesTotal += b.Size

			if b.Size > int64(migrateInfoAbove) {
				entry.TotalAbove++
				entry.BytesAbove += b.Size
			}

			exts[ext] = entry
		}

		return b, nil
	})

	entries := EntriesBySize(MapToEntries(exts))
	sort.Sort(sort.Reverse(entries))

	migrateInfoTopN = tools.ClampInt(migrateInfoTopN, len(entries), 0)

	entries = entries[:tools.MaxInt(0, migrateInfoTopN)]

	entries.Print(os.Stdout)
}

// MigrateInfoEntry represents a tuple of filetype to bytes and entry count
// above and below a threshold.
type MigrateInfoEntry struct {
	// Qualifier is the filepath's extension.
	Qualifier string

	// BytesAbove is total size of all files above a given threshold.
	BytesAbove int64
	// TotalAbove is the count of all files above a given size threshold.
	TotalAbove int64
	// BytesTotal is the number of bytes of all files
	BytesTotal int64
	// Total is the count of all files.
	Total int64
}

// MapToEntries creates a set of `*MigrateInfoEntry`'s for a given map of
// filepath extensions to file size in bytes.
func MapToEntries(exts map[string]*MigrateInfoEntry) []*MigrateInfoEntry {
	entries := make([]*MigrateInfoEntry, 0, len(exts))
	for _, entry := range exts {
		entries = append(entries, entry)
	}

	return entries
}

// EntriesBySize is an implementation of sort.Interface that sorts a set of
// `*MigrateInfoEntry`'s
type EntriesBySize []*MigrateInfoEntry

// Len returns the total length of the set of `*MigrateInfoEntry`'s.
func (e EntriesBySize) Len() int { return len(e) }

// Less returns the whether or not the MigrateInfoEntry given at `i` takes up
// less total size than the MigrateInfoEntry given at `j`.
func (e EntriesBySize) Less(i, j int) bool { return e[i].BytesAbove < e[j].BytesAbove }

// Swap swaps the entries given at i, j.
func (e EntriesBySize) Swap(i, j int) { e[i], e[j] = e[j], e[i] }

// Print formats the `*MigrateInfoEntry`'s in the set and prints them to the
// given io.Writer, "to", returning "n" the number of bytes written, and any
// error, if one occurred.
func (e EntriesBySize) Print(to io.Writer) (int, error) {
	extensions := make([]string, 0, len(e))
	sizes := make([]string, 0, len(e))
	stats := make([]string, 0, len(e))
	percentages := make([]string, 0, len(e))

	for _, entry := range e {
		above := entry.TotalAbove
		total := entry.Total
		percentAbove := 100 * (float64(above) / float64(total))

		stat := fmt.Sprintf("%d/%d files(s)",
			above, total)

		percentage := fmt.Sprintf("%.0f%%", percentAbove)

		extensions = append(extensions, entry.Qualifier)
		sizes = append(sizes, humanize.FormatBytes(uint64(entry.BytesAbove)))
		stats = append(stats, stat)
		percentages = append(percentages, percentage)
	}

	extensions = tools.Ljust(extensions)
	sizes = tools.Ljust(sizes)
	stats = tools.Rjust(stats)
	percentages = tools.Rjust(percentages)

	output := make([]string, 0, len(e))
	for i := 0; i < len(e); i++ {
		extension := extensions[i]
		size := sizes[i]
		stat := stats[i]
		percentage := percentages[i]

		line := strings.Join([]string{extension, size, stat, percentage}, "\t")

		output = append(output, line)
	}

	return fmt.Fprintln(to, strings.Join(output, "\n"))
}
<|endoftext|>


<commit_before>package collect

import (
	"testing"
	"time"

	"github.com/v2ray/v2ray-core/testing/unit"
)

func TestTimedStringMap(t *testing.T) {
	assert := unit.Assert(t)

	nowSec := time.Now().UTC().Unix()
	m := NewTimedStringMap(2)
	m.Set("Key1", "Value1", nowSec)
	m.Set("Key2", "Value2", nowSec+5)

	v1, ok := m.Get("Key1")
	assert.Bool(ok).IsTrue()
	assert.String(v1.(string)).Equals("Value1")

	v2, ok := m.Get("Key2")
	assert.Bool(ok).IsTrue()
	assert.String(v2.(string)).Equals("Value2")

	tick := time.Tick(3 * time.Second)
	<-tick

	v1, ok = m.Get("Key1")
	assert.Bool(ok).IsFalse()

	v2, ok = m.Get("Key2")
	assert.Bool(ok).IsTrue()
	assert.String(v2.(string)).Equals("Value2")

	<-tick
	v2, ok = m.Get("Key2")
	assert.Bool(ok).IsFalse()

	<-tick
	v2, ok = m.Get("Key2")
	assert.Bool(ok).IsFalse()

	m.Set("Key1", "Value1", nowSec+10)
	v1, ok = m.Get("Key1")
	assert.Bool(ok).IsTrue()
	assert.String(v1.(string)).Equals("Value1")
}
<commit_msg>Fix time usage in timed_map_test<commit_after>package collect

import (
	"testing"
	"time"

	"github.com/v2ray/v2ray-core/testing/unit"
)

func TestTimedStringMap(t *testing.T) {
	assert := unit.Assert(t)

	nowSec := time.Now().UTC().Unix()
	m := NewTimedStringMap(2)
	m.Set("Key1", "Value1", nowSec)
	m.Set("Key2", "Value2", nowSec+5)

	v1, ok := m.Get("Key1")
	assert.Bool(ok).IsTrue()
	assert.String(v1.(string)).Equals("Value1")

	v2, ok := m.Get("Key2")
	assert.Bool(ok).IsTrue()
	assert.String(v2.(string)).Equals("Value2")

	tick := time.Tick(3 * time.Second)
	<-tick

	v1, ok = m.Get("Key1")
	assert.Bool(ok).IsFalse()

	v2, ok = m.Get("Key2")
	assert.Bool(ok).IsTrue()
	assert.String(v2.(string)).Equals("Value2")

	<-tick
	v2, ok = m.Get("Key2")
	assert.Bool(ok).IsFalse()

	<-tick
	v2, ok = m.Get("Key2")
	assert.Bool(ok).IsFalse()

	m.Set("Key1", "Value1", time.Now().UTC().Unix()+10)
	v1, ok = m.Get("Key1")
	assert.Bool(ok).IsTrue()
	assert.String(v1.(string)).Equals("Value1")
}
<|endoftext|>


<commit_before>package ssh

import (
	"bytes"
	"code.google.com/p/go.crypto/ssh"
	"errors"
	"fmt"
	"github.com/mitchellh/packer/packer"
	"io"
	"log"
	"net"
	"path/filepath"
)

type comm struct {
	client *ssh.ClientConn
	config *Config
	conn   net.Conn
}

// Config is the structure used to configure the SSH communicator.
type Config struct {
	// The configuration of the Go SSH connection
	SSHConfig *ssh.ClientConfig

	// Connection returns a new connection. The current connection
	// in use will be closed as part of the Close method, or in the
	// case an error occurs.
	Connection func() (net.Conn, error)
}

// Creates a new packer.Communicator implementation over SSH. This takes
// an already existing TCP connection and SSH configuration.
func New(config *Config) (result *comm, err error) {
	// Establish an initial connection and connect
	result = &comm{
		config: config,
	}

	if err = result.reconnect(); err != nil {
		result = nil
		return
	}

	return
}

func (c *comm) Start(cmd *packer.RemoteCmd) (err error) {
	session, err := c.newSession()
	if err != nil {
		return
	}

	// Setup our session
	session.Stdin = cmd.Stdin
	session.Stdout = cmd.Stdout
	session.Stderr = cmd.Stderr

	// Request a PTY
	termModes := ssh.TerminalModes{
		ssh.ECHO:          0,     // do not echo
		ssh.TTY_OP_ISPEED: 14400, // input speed = 14.4kbaud
		ssh.TTY_OP_OSPEED: 14400, // output speed = 14.4kbaud
	}

	if err = session.RequestPty("xterm", 80, 40, termModes); err != nil {
		return
	}

	log.Printf("starting remote command: %s", cmd.Command)
	err = session.Start(cmd.Command + "\n")
	if err != nil {
		return
	}

	// Start a goroutine to wait for the session to end and set the
	// exit boolean and status.
	go func() {
		defer session.Close()
		err := session.Wait()
		cmd.ExitStatus = 0
		if err != nil {
			exitErr, ok := err.(*ssh.ExitError)
			if ok {
				cmd.ExitStatus = exitErr.ExitStatus()
			}
		}

		cmd.Exited = true
	}()

	return
}

func (c *comm) Upload(path string, input io.Reader) error {
	session, err := c.newSession()
	if err != nil {
		return err
	}

	defer session.Close()

	// Get a pipe to stdin so that we can send data down
	w, err := session.StdinPipe()
	if err != nil {
		return err
	}

	// Set stderr/stdout to a bytes buffer
	stderr := new(bytes.Buffer)
	stdout := new(bytes.Buffer)
	session.Stderr = stderr
	session.Stdout = stdout

	// We only want to close once, so we nil w after we close it,
	// and only close in the defer if it hasn't been closed already.
	defer func() {
		if w != nil {
			w.Close()
		}
	}()

	// The target directory and file for talking the SCP protocol
	target_dir := filepath.Dir(path)
	target_file := filepath.Base(path)

	// Start the sink mode on the other side
	// TODO(mitchellh): There are probably issues with shell escaping the path
	log.Println("Starting remote scp process in sink mode")
	if err = session.Start("scp -vt " + target_dir); err != nil {
		return err
	}

	// Determine the length of the upload content by copying it
	// into an in-memory buffer. Note that this means what we upload
	// must fit into memory.
	log.Println("Copying input data into in-memory buffer so we can get the length")
	input_memory := new(bytes.Buffer)
	if _, err = io.Copy(input_memory, input); err != nil {
		return err
	}

	// Start the protocol
	log.Println("Beginning file upload...")
	fmt.Fprintln(w, "C0644", input_memory.Len(), target_file)
	io.Copy(w, input_memory)
	fmt.Fprint(w, "\x00")

	// TODO(mitchellh): Each step above results in a 0/1/2 being sent by
	// the remote side to confirm. We should check for those confirmations.

	// Close the stdin, which sends an EOF, and then set w to nil so that
	// our defer func doesn't close it again since that is unsafe with
	// the Go SSH package.
	log.Println("Upload complete, closing stdin pipe")
	w.Close()
	w = nil

	// Wait for the SCP connection to close, meaning it has consumed all
	// our data and has completed. Or has errored.
	log.Println("Waiting for SSH session to complete")
	err = session.Wait()
	if err != nil {
		if exitErr, ok := err.(*ssh.ExitError); ok {
			// Otherwise, we have an ExitErorr, meaning we can just read
			// the exit status
			log.Printf("non-zero exit status: %d", exitErr.ExitStatus())

			// If we exited with status 127, it means SCP isn't available.
			// Return a more descriptive error for that.
			if exitErr.ExitStatus() == 127 {
				return errors.New(
					"SCP failed to start. This usually means that SCP is not\n" +
						"properly installed on the remote system.")
			}
		}

		return err
	}

	log.Printf("scp stdout (length %d): %#v", stdout.Len(), stdout.Bytes())
	log.Printf("scp stderr (length %d): %s", stderr.Len(), stderr.String())

	return nil
}

func (c *comm) Download(string, io.Writer) error {
	panic("not implemented yet")
}

func (c *comm) newSession() (*ssh.Session, error) {
	log.Println("opening new ssh session")
	session, err := c.client.NewSession()
	if err != nil {
		if err := c.reconnect(); err != nil {
			return nil, err
		}

		return c.client.NewSession()
	}

	return session, nil
}

func (c *comm) reconnect() (err error) {
	if c.conn != nil {
		c.conn.Close()
	}

	log.Printf("reconnecting to TCP connection for SSH")
	c.conn, err = c.config.Connection()
	if err != nil {
		log.Printf("reconnection error: %s", err)
		return
	}

	log.Printf("handshaking with SSH")
	c.client, err = ssh.Client(c.conn, c.config.SSHConfig)
	if err != nil {
		log.Printf("handshake error: %s", err)
	}

	return
}
<commit_msg>communicator/ssh: improved logging<commit_after>package ssh

import (
	"bytes"
	"code.google.com/p/go.crypto/ssh"
	"errors"
	"fmt"
	"github.com/mitchellh/packer/packer"
	"io"
	"log"
	"net"
	"path/filepath"
)

type comm struct {
	client *ssh.ClientConn
	config *Config
	conn   net.Conn
}

// Config is the structure used to configure the SSH communicator.
type Config struct {
	// The configuration of the Go SSH connection
	SSHConfig *ssh.ClientConfig

	// Connection returns a new connection. The current connection
	// in use will be closed as part of the Close method, or in the
	// case an error occurs.
	Connection func() (net.Conn, error)
}

// Creates a new packer.Communicator implementation over SSH. This takes
// an already existing TCP connection and SSH configuration.
func New(config *Config) (result *comm, err error) {
	// Establish an initial connection and connect
	result = &comm{
		config: config,
	}

	if err = result.reconnect(); err != nil {
		result = nil
		return
	}

	return
}

func (c *comm) Start(cmd *packer.RemoteCmd) (err error) {
	session, err := c.newSession()
	if err != nil {
		return
	}

	// Setup our session
	session.Stdin = cmd.Stdin
	session.Stdout = cmd.Stdout
	session.Stderr = cmd.Stderr

	// Request a PTY
	termModes := ssh.TerminalModes{
		ssh.ECHO:          0,     // do not echo
		ssh.TTY_OP_ISPEED: 14400, // input speed = 14.4kbaud
		ssh.TTY_OP_OSPEED: 14400, // output speed = 14.4kbaud
	}

	if err = session.RequestPty("xterm", 80, 40, termModes); err != nil {
		return
	}

	log.Printf("starting remote command: %s", cmd.Command)
	err = session.Start(cmd.Command + "\n")
	if err != nil {
		return
	}

	// Start a goroutine to wait for the session to end and set the
	// exit boolean and status.
	go func() {
		defer session.Close()
		err := session.Wait()
		cmd.ExitStatus = 0
		if err != nil {
			exitErr, ok := err.(*ssh.ExitError)
			if ok {
				cmd.ExitStatus = exitErr.ExitStatus()
			}
		}

		cmd.Exited = true
	}()

	return
}

func (c *comm) Upload(path string, input io.Reader) error {
	session, err := c.newSession()
	if err != nil {
		return err
	}

	defer session.Close()

	// Get a pipe to stdin so that we can send data down
	w, err := session.StdinPipe()
	if err != nil {
		return err
	}

	// Set stderr/stdout to a bytes buffer
	stderr := new(bytes.Buffer)
	stdout := new(bytes.Buffer)
	session.Stderr = stderr
	session.Stdout = stdout

	// We only want to close once, so we nil w after we close it,
	// and only close in the defer if it hasn't been closed already.
	defer func() {
		if w != nil {
			w.Close()
		}
	}()

	// The target directory and file for talking the SCP protocol
	target_dir := filepath.Dir(path)
	target_file := filepath.Base(path)

	// Start the sink mode on the other side
	// TODO(mitchellh): There are probably issues with shell escaping the path
	log.Println("Starting remote scp process in sink mode")
	if err = session.Start("scp -vt " + target_dir); err != nil {
		return err
	}

	// Determine the length of the upload content by copying it
	// into an in-memory buffer. Note that this means what we upload
	// must fit into memory.
	log.Println("Copying input data into in-memory buffer so we can get the length")
	input_memory := new(bytes.Buffer)
	if _, err = io.Copy(input_memory, input); err != nil {
		return err
	}

	// Start the protocol
	log.Println("Beginning file upload...")
	fmt.Fprintln(w, "C0644", input_memory.Len(), target_file)
	io.Copy(w, input_memory)
	fmt.Fprint(w, "\x00")

	// TODO(mitchellh): Each step above results in a 0/1/2 being sent by
	// the remote side to confirm. We should check for those confirmations.

	// Close the stdin, which sends an EOF, and then set w to nil so that
	// our defer func doesn't close it again since that is unsafe with
	// the Go SSH package.
	log.Println("Upload complete, closing stdin pipe")
	w.Close()
	w = nil

	// Wait for the SCP connection to close, meaning it has consumed all
	// our data and has completed. Or has errored.
	log.Println("Waiting for SSH session to complete")
	err = session.Wait()
	if err != nil {
		if exitErr, ok := err.(*ssh.ExitError); ok {
			// Otherwise, we have an ExitErorr, meaning we can just read
			// the exit status
			log.Printf("non-zero exit status: %d", exitErr.ExitStatus())

			// If we exited with status 127, it means SCP isn't available.
			// Return a more descriptive error for that.
			if exitErr.ExitStatus() == 127 {
				return errors.New(
					"SCP failed to start. This usually means that SCP is not\n" +
						"properly installed on the remote system.")
			}
		}

		return err
	}

	log.Printf("scp stdout (length %d): %#v", stdout.Len(), stdout.Bytes())
	log.Printf("scp stderr (length %d): %s", stderr.Len(), stderr.String())

	return nil
}

func (c *comm) Download(string, io.Writer) error {
	panic("not implemented yet")
}

func (c *comm) newSession() (*ssh.Session, error) {
	log.Println("opening new ssh session")
	session, err := c.client.NewSession()
	if err != nil {
		log.Printf("ssh session open error: '%s', attempting reconnect", err)
		if err := c.reconnect(); err != nil {
			return nil, err
		}

		return c.client.NewSession()
	}

	return session, nil
}

func (c *comm) reconnect() (err error) {
	if c.conn != nil {
		c.conn.Close()
	}

	log.Printf("reconnecting to TCP connection for SSH")
	c.conn, err = c.config.Connection()
	if err != nil {
		log.Printf("reconnection error: %s", err)
		return
	}

	log.Printf("handshaking with SSH")
	c.client, err = ssh.Client(c.conn, c.config.SSHConfig)
	if err != nil {
		log.Printf("handshake error: %s", err)
	}

	return
}
<|endoftext|>