<commit_before>// Copyright 2018 Google LLC. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package scanner

import (
	"context"
	"sync"
	"time"

	"github.com/golang/glog"
	ct "github.com/google/certificate-transparency-go"
	"github.com/google/trillian/client/backoff"
)

// LogClient implements the subset of CT log API that the Fetcher uses.
type LogClient interface {
	BaseURI() string
	GetSTH(context.Context) (*ct.SignedTreeHead, error)
	GetRawEntries(ctx context.Context, start, end int64) (*ct.GetEntriesResponse, error)
}

// FetcherOptions holds configuration options for the Fetcher.
type FetcherOptions struct {
	// Number of entries to request in one batch from the Log.
	BatchSize int

	// Number of concurrent fetcher workers to run.
	ParallelFetch int

	// [StartIndex, EndIndex) is a log entry range to fetch. If EndIndex == 0,
	// then it gets reassigned to sth.TreeSize.
	StartIndex int64
	EndIndex   int64

	// Continuous determines whether Fetcher should run indefinitely after
	// reaching EndIndex.
	Continuous bool
}

// DefaultFetcherOptions returns new FetcherOptions with sensible defaults.
func DefaultFetcherOptions() *FetcherOptions {
	return &FetcherOptions{
		BatchSize:     1000,
		ParallelFetch: 1,
		StartIndex:    0,
		EndIndex:      0,
		Continuous:    false,
	}
}

// Fetcher is a tool that fetches entries from a CT Log.
type Fetcher struct {
	// Base URI of the CT log, for diagnostics.
	uri string
	// Client used to talk to the CT log instance.
	client LogClient
	// Configuration options for this Fetcher instance.
	opts *FetcherOptions

	// Current STH of the Log this Fetcher sends queries to.
	sth *ct.SignedTreeHead
	// The STH retrieval backoff state. Used only in Continuous fetch mode.
	sthBackoff *backoff.Backoff

	// Stops range generator, which causes the Fetcher to terminate gracefully.
	mu     sync.Mutex
	cancel context.CancelFunc
}

// EntryBatch represents a contiguous range of entries of the Log.
type EntryBatch struct {
	Start   int64          // LeafIndex of the first entry in the range.
	Entries []ct.LeafEntry // Entries of the range.
}

// fetchRange represents a range of certs to fetch from a CT log.
type fetchRange struct {
	start int64 // inclusive
	end   int64 // inclusive
}

// NewFetcher creates a Fetcher instance using client to talk to the log,
// taking configuration options from opts.
func NewFetcher(client LogClient, opts *FetcherOptions) *Fetcher {
	cancel := func() {} // Protect against calling Stop before Run.
	return &Fetcher{
		uri:    client.BaseURI(),
		client: client,
		opts:   opts,
		cancel: cancel,
	}
}

// Prepare caches the latest Log's STH if not present and returns it. It also
// adjusts the entry range to fit the size of the tree.
func (f *Fetcher) Prepare(ctx context.Context) (*ct.SignedTreeHead, error) {
	if f.sth != nil {
		return f.sth, nil
	}

	sth, err := f.client.GetSTH(ctx)
	if err != nil {
		glog.Errorf("%s: GetSTH() failed: %v", f.uri, err)
		return nil, err
	}
	glog.Infof("%s: Got STH with %d certs", f.uri, sth.TreeSize)

	if size := int64(sth.TreeSize); f.opts.EndIndex == 0 || f.opts.EndIndex > size {
		glog.Infof("%s: Reset EndIndex from %d to %d", f.uri, f.opts.EndIndex, size)
		f.opts.EndIndex = size
	}
	f.sth = sth
	return sth, nil
}

// Run performs fetching of the Log. Blocks until scanning is complete, the
// passed in context is canceled, or Stop is called (and pending work is
// finished). For each successfully fetched batch, runs the fn callback.
func (f *Fetcher) Run(ctx context.Context, fn func(EntryBatch)) error {
	glog.V(1).Infof("%s: Starting up Fetcher...", f.uri)
	if _, err := f.Prepare(ctx); err != nil {
		return err
	}

	cctx, cancel := context.WithCancel(ctx)
	defer cancel()

	f.mu.Lock()
	f.cancel = cancel
	f.mu.Unlock()

	// Use a separately-cancelable context for the range generator, so we can
	// close it down (in Stop) but still let the fetchers below run to
	// completion.
	ranges := f.genRanges(cctx)

	// Run fetcher workers.
	var wg sync.WaitGroup
	for w, cnt := 0, f.opts.ParallelFetch; w < cnt; w++ {
		wg.Add(1)
		go func(idx int) {
			defer wg.Done()
			glog.V(1).Infof("%s: Fetcher worker %d starting...", f.uri, idx)
			f.runWorker(ctx, ranges, fn)
			glog.V(1).Infof("%s: Fetcher worker %d finished", f.uri, idx)
		}(w)
	}
	wg.Wait()

	glog.V(1).Infof("%s: Fetcher terminated", f.uri)
	return nil
}

// Stop causes the Fetcher to terminate gracefully. After this call Run will
// try to finish all the started fetches, and then return. Does nothing if
// there was no preceding Run invocation.
func (f *Fetcher) Stop() {
	f.mu.Lock()
	defer f.mu.Unlock()
	f.cancel()
}

// genRanges returns a channel of ranges to fetch, and starts a goroutine that
// sends things down this channel. The goroutine terminates when all ranges
// have been generated, or if context is cancelled.
func (f *Fetcher) genRanges(ctx context.Context) <-chan fetchRange {
	batch := int64(f.opts.BatchSize)
	ranges := make(chan fetchRange)

	go func() {
		glog.V(1).Infof("%s: Range generator starting", f.uri)
		defer glog.V(1).Infof("%s: Range generator finished", f.uri)
		defer close(ranges)
		start, end := f.opts.StartIndex, f.opts.EndIndex

		for start < end || f.opts.Continuous {
			// In continuous mode wait for bigger STH every time we reach the end,
			// including, possibly, the very first iteration.
			if start == end { // Implies f.opts.Continuous == true.
				if err := f.updateSTH(ctx); err != nil {
					glog.Warningf("%s: Failed to obtain bigger STH: %v", f.uri, err)
					return
				}
				end = f.opts.EndIndex
			}

			batchEnd := start + min(end-start, batch)
			next := fetchRange{start, batchEnd - 1}
			select {
			case <-ctx.Done():
				glog.Warningf("%s: Cancelling genRanges: %v", f.uri, ctx.Err())
				return
			case ranges <- next:
			}
			start = batchEnd
		}
	}()

	return ranges
}

// updateSTH waits until a bigger STH is discovered, and updates the Fetcher
// accordingly. It is optimized for both bulk-load (new STH is way bigger then
// the last one) and keep-up (STH grows slowly) modes of operation. Waits for
// some time until the STH grows enough to request a full batch, but falls back
// to *any* STH bigger than the old one if it takes too long.
// Returns error only if the context is cancelled.
func (f *Fetcher) updateSTH(ctx context.Context) error {
	// TODO(pavelkalinnikov): Make these parameters tunable.
	const quickDur = 45 * time.Second
	if f.sthBackoff == nil {
		f.sthBackoff = &backoff.Backoff{
			Min:    1 * time.Second,
			Max:    30 * time.Second,
			Factor: 2,
			Jitter: true,
		}
	}

	lastSize := uint64(f.opts.EndIndex)
	targetSize := lastSize + uint64(f.opts.BatchSize)
	quickDeadline := time.Now().Add(quickDur)

	return f.sthBackoff.Retry(ctx, func() error {
		sth, err := f.client.GetSTH(ctx)
		if err != nil {
			return err
		}
		glog.V(2).Infof("%s: Got STH with %d certs", f.uri, sth.TreeSize)

		quick := time.Now().Before(quickDeadline)
		if sth.TreeSize <= lastSize || quick && sth.TreeSize < targetSize {
			return backoff.RetriableErrorf("wait for bigger STH than %d (last=%d, target=%d)", sth.TreeSize, lastSize, targetSize)
		}

		if quick {
			f.sthBackoff.Reset() // Growth is presumably fast, set next pause to Min.
		}
		f.sth = sth
		f.opts.EndIndex = int64(sth.TreeSize)
		return nil
	})
}

// runWorker is a worker function for handling fetcher ranges.
// Accepts cert ranges to fetch over the ranges channel, and if the fetch is
// successful sends the corresponding EntryBatch through the fn callback. Will
// retry failed attempts to retrieve ranges until the context is cancelled.
func (f *Fetcher) runWorker(ctx context.Context, ranges <-chan fetchRange, fn func(EntryBatch)) {
	for r := range ranges {
		// Logs MAY return fewer than the number of leaves requested. Only complete
		// if we actually got all the leaves we were expecting.
		for r.start <= r.end {
			// TODO(pavelkalinnikov): Make these parameters tunable.
			// This backoff will only apply to a single request and be reset for the next one.
			// This precludes reaching some kind of stability in request rate, but means that
			// an intermittent problem won't harm long-term running of the worker.
			bo := &backoff.Backoff{
				Min:    1 * time.Second,
				Max:    30 * time.Second,
				Factor: 2,
				Jitter: true,
			}

			var resp *ct.GetEntriesResponse
			// TODO(pavelkalinnikov): Report errors in a LogClient decorator on failure.
			if err := bo.Retry(ctx, func() error {
				var err error
				resp, err = f.client.GetRawEntries(ctx, r.start, r.end)
				return err
			}); err != nil {
				glog.Errorf("%s: GetRawEntries() failed: %v", f.uri, err)
				// There is no error reporting yet for this worker, so just retry again.
				continue
			}
			fn(EntryBatch{Start: r.start, Entries: resp.Entries})
			r.start += int64(len(resp.Entries))
		}
	}
}

func min(a, b int64) int64 {
	if a < b {
		return a
	}
	return b
}
<commit_msg>fetcher: Prevent spinning in workers (#837)<commit_after>// Copyright 2018 Google LLC. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package scanner

import (
	"context"
	"sync"
	"time"

	"github.com/golang/glog"
	ct "github.com/google/certificate-transparency-go"
	"github.com/google/trillian/client/backoff"
)

// LogClient implements the subset of CT log API that the Fetcher uses.
type LogClient interface {
	BaseURI() string
	GetSTH(context.Context) (*ct.SignedTreeHead, error)
	GetRawEntries(ctx context.Context, start, end int64) (*ct.GetEntriesResponse, error)
}

// FetcherOptions holds configuration options for the Fetcher.
type FetcherOptions struct {
	// Number of entries to request in one batch from the Log.
	BatchSize int

	// Number of concurrent fetcher workers to run.
	ParallelFetch int

	// [StartIndex, EndIndex) is a log entry range to fetch. If EndIndex == 0,
	// then it gets reassigned to sth.TreeSize.
	StartIndex int64
	EndIndex   int64

	// Continuous determines whether Fetcher should run indefinitely after
	// reaching EndIndex.
	Continuous bool
}

// DefaultFetcherOptions returns new FetcherOptions with sensible defaults.
func DefaultFetcherOptions() *FetcherOptions {
	return &FetcherOptions{
		BatchSize:     1000,
		ParallelFetch: 1,
		StartIndex:    0,
		EndIndex:      0,
		Continuous:    false,
	}
}

// Fetcher is a tool that fetches entries from a CT Log.
type Fetcher struct {
	// Base URI of the CT log, for diagnostics.
	uri string
	// Client used to talk to the CT log instance.
	client LogClient
	// Configuration options for this Fetcher instance.
	opts *FetcherOptions

	// Current STH of the Log this Fetcher sends queries to.
	sth *ct.SignedTreeHead
	// The STH retrieval backoff state. Used only in Continuous fetch mode.
	sthBackoff *backoff.Backoff

	// Stops range generator, which causes the Fetcher to terminate gracefully.
	mu     sync.Mutex
	cancel context.CancelFunc
}

// EntryBatch represents a contiguous range of entries of the Log.
type EntryBatch struct {
	Start   int64          // LeafIndex of the first entry in the range.
	Entries []ct.LeafEntry // Entries of the range.
}

// fetchRange represents a range of certs to fetch from a CT log.
type fetchRange struct {
	start int64 // inclusive
	end   int64 // inclusive
}

// NewFetcher creates a Fetcher instance using client to talk to the log,
// taking configuration options from opts.
func NewFetcher(client LogClient, opts *FetcherOptions) *Fetcher {
	cancel := func() {} // Protect against calling Stop before Run.
	return &Fetcher{
		uri:    client.BaseURI(),
		client: client,
		opts:   opts,
		cancel: cancel,
	}
}

// Prepare caches the latest Log's STH if not present and returns it. It also
// adjusts the entry range to fit the size of the tree.
func (f *Fetcher) Prepare(ctx context.Context) (*ct.SignedTreeHead, error) {
	if f.sth != nil {
		return f.sth, nil
	}

	sth, err := f.client.GetSTH(ctx)
	if err != nil {
		glog.Errorf("%s: GetSTH() failed: %v", f.uri, err)
		return nil, err
	}
	glog.Infof("%s: Got STH with %d certs", f.uri, sth.TreeSize)

	if size := int64(sth.TreeSize); f.opts.EndIndex == 0 || f.opts.EndIndex > size {
		glog.Infof("%s: Reset EndIndex from %d to %d", f.uri, f.opts.EndIndex, size)
		f.opts.EndIndex = size
	}
	f.sth = sth
	return sth, nil
}

// Run performs fetching of the Log. Blocks until scanning is complete, the
// passed in context is canceled, or Stop is called (and pending work is
// finished). For each successfully fetched batch, runs the fn callback.
func (f *Fetcher) Run(ctx context.Context, fn func(EntryBatch)) error {
	glog.V(1).Infof("%s: Starting up Fetcher...", f.uri)
	if _, err := f.Prepare(ctx); err != nil {
		return err
	}

	cctx, cancel := context.WithCancel(ctx)
	defer cancel()

	f.mu.Lock()
	f.cancel = cancel
	f.mu.Unlock()

	// Use a separately-cancelable context for the range generator, so we can
	// close it down (in Stop) but still let the fetchers below run to
	// completion.
	ranges := f.genRanges(cctx)

	// Run fetcher workers.
	var wg sync.WaitGroup
	for w, cnt := 0, f.opts.ParallelFetch; w < cnt; w++ {
		wg.Add(1)
		go func(idx int) {
			defer wg.Done()
			glog.V(1).Infof("%s: Fetcher worker %d starting...", f.uri, idx)
			f.runWorker(ctx, ranges, fn)
			glog.V(1).Infof("%s: Fetcher worker %d finished", f.uri, idx)
		}(w)
	}
	wg.Wait()

	glog.V(1).Infof("%s: Fetcher terminated", f.uri)
	return nil
}

// Stop causes the Fetcher to terminate gracefully. After this call Run will
// try to finish all the started fetches, and then return. Does nothing if
// there was no preceding Run invocation.
func (f *Fetcher) Stop() {
	f.mu.Lock()
	defer f.mu.Unlock()
	f.cancel()
}

// genRanges returns a channel of ranges to fetch, and starts a goroutine that
// sends things down this channel. The goroutine terminates when all ranges
// have been generated, or if context is cancelled.
func (f *Fetcher) genRanges(ctx context.Context) <-chan fetchRange {
	batch := int64(f.opts.BatchSize)
	ranges := make(chan fetchRange)

	go func() {
		glog.V(1).Infof("%s: Range generator starting", f.uri)
		defer glog.V(1).Infof("%s: Range generator finished", f.uri)
		defer close(ranges)
		start, end := f.opts.StartIndex, f.opts.EndIndex

		for start < end || f.opts.Continuous {
			// In continuous mode wait for bigger STH every time we reach the end,
			// including, possibly, the very first iteration.
			if start == end { // Implies f.opts.Continuous == true.
				if err := f.updateSTH(ctx); err != nil {
					glog.Warningf("%s: Failed to obtain bigger STH: %v", f.uri, err)
					return
				}
				end = f.opts.EndIndex
			}

			batchEnd := start + min(end-start, batch)
			next := fetchRange{start, batchEnd - 1}
			select {
			case <-ctx.Done():
				glog.Warningf("%s: Cancelling genRanges: %v", f.uri, ctx.Err())
				return
			case ranges <- next:
			}
			start = batchEnd
		}
	}()

	return ranges
}

// updateSTH waits until a bigger STH is discovered, and updates the Fetcher
// accordingly. It is optimized for both bulk-load (new STH is way bigger then
// the last one) and keep-up (STH grows slowly) modes of operation. Waits for
// some time until the STH grows enough to request a full batch, but falls back
// to *any* STH bigger than the old one if it takes too long.
// Returns error only if the context is cancelled.
func (f *Fetcher) updateSTH(ctx context.Context) error {
	// TODO(pavelkalinnikov): Make these parameters tunable.
	const quickDur = 45 * time.Second
	if f.sthBackoff == nil {
		f.sthBackoff = &backoff.Backoff{
			Min:    1 * time.Second,
			Max:    30 * time.Second,
			Factor: 2,
			Jitter: true,
		}
	}

	lastSize := uint64(f.opts.EndIndex)
	targetSize := lastSize + uint64(f.opts.BatchSize)
	quickDeadline := time.Now().Add(quickDur)

	return f.sthBackoff.Retry(ctx, func() error {
		sth, err := f.client.GetSTH(ctx)
		if err != nil {
			return err
		}
		glog.V(2).Infof("%s: Got STH with %d certs", f.uri, sth.TreeSize)

		quick := time.Now().Before(quickDeadline)
		if sth.TreeSize <= lastSize || quick && sth.TreeSize < targetSize {
			return backoff.RetriableErrorf("wait for bigger STH than %d (last=%d, target=%d)", sth.TreeSize, lastSize, targetSize)
		}

		if quick {
			f.sthBackoff.Reset() // Growth is presumably fast, set next pause to Min.
		}
		f.sth = sth
		f.opts.EndIndex = int64(sth.TreeSize)
		return nil
	})
}

// runWorker is a worker function for handling fetcher ranges.
// Accepts cert ranges to fetch over the ranges channel, and if the fetch is
// successful sends the corresponding EntryBatch through the fn callback. Will
// retry failed attempts to retrieve ranges until the context is cancelled.
func (f *Fetcher) runWorker(ctx context.Context, ranges <-chan fetchRange, fn func(EntryBatch)) {
	for r := range ranges {
		// Logs MAY return fewer than the number of leaves requested. Only complete
		// if we actually got all the leaves we were expecting.
		for r.start <= r.end {
			if ctx.Err() != nil { // Prevent spinning when context is canceled.
				return
			}
			// TODO(pavelkalinnikov): Make these parameters tunable.
			// This backoff will only apply to a single request and be reset for the next one.
			// This precludes reaching some kind of stability in request rate, but means that
			// an intermittent problem won't harm long-term running of the worker.
			bo := &backoff.Backoff{
				Min:    1 * time.Second,
				Max:    30 * time.Second,
				Factor: 2,
				Jitter: true,
			}

			var resp *ct.GetEntriesResponse
			// TODO(pavelkalinnikov): Report errors in a LogClient decorator on failure.
			if err := bo.Retry(ctx, func() error {
				var err error
				resp, err = f.client.GetRawEntries(ctx, r.start, r.end)
				return err
			}); err != nil {
				glog.Errorf("%s: GetRawEntries() failed: %v", f.uri, err)
				// There is no error reporting yet for this worker, so just retry again.
				continue
			}
			fn(EntryBatch{Start: r.start, Entries: resp.Entries})
			r.start += int64(len(resp.Entries))
		}
	}
}

func min(a, b int64) int64 {
	if a < b {
		return a
	}
	return b
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"

	log "github.com/Sirupsen/logrus"
)

func runQemu(args []string) {
	qemuFlags := flag.NewFlagSet("qemu", flag.ExitOnError)
	qemuFlags.Usage = func() {
		fmt.Printf("USAGE: %s run qemu [options] prefix\n\n", os.Args[0])
		fmt.Printf("'prefix' specifies the path to the VM image.\n")
		fmt.Printf("\n")
		fmt.Printf("Options:\n")
		qemuFlags.PrintDefaults()
	}

	// Determine Flags
	qemuGUI := qemuFlags.Bool("gui", false, "Set qemu to use video output instead of stdio")
	qemuUEFI := qemuFlags.Bool("uefi", false, "Set UEFI boot from 'prefix'-efi.iso")
	qemuIso := qemuFlags.Bool("iso", false, "Set Legacy BIOS boot from 'prefix'.iso")
	qemuKernel := qemuFlags.Bool("kernel", true, "Set boot using 'prefix'-bzImage/-initrd/-cmdline")

	// Paths and settings for Disks and UEFI firware
	qemuDiskPath := qemuFlags.String("diskpath", "", "Path to disk image to use")
	qemuDiskSize := qemuFlags.String("disksize", "", "Size of disk to create, only created if it doesn't exist")
	qemuFWPath := qemuFlags.String("fwpath", "/usr/share/ovmf/bios.bin", "Path to OVMF firmware for UEFI boot")

	// VM configuration
	qemuArch := qemuFlags.String("arch", "x86_64", "Type of architecture to use, e.g. x86_64, aarch64")
	qemuCPUs := qemuFlags.String("cpus", "1", "Number of CPUs")
	qemuMem := qemuFlags.String("mem", "1024", "Amount of memory in MB")

	if err := qemuFlags.Parse(args); err != nil {
		log.Fatal("Unable to parse args")
	}
	remArgs := qemuFlags.Args()

	if len(remArgs) == 0 {
		fmt.Println("Please specify the prefix to the image to boot")
		qemuFlags.Usage()
		os.Exit(1)
	}
	prefix := remArgs[0]

	// Print warning if conflicting UEFI and ISO flags are set
	if *qemuUEFI && *qemuIso {
		log.Warnf("Both -iso and -uefi have been used")
	}

	// Before building qemu arguments, check qemu is in the $PATH
	qemuBinPath := "qemu-system-" + *qemuArch
	qemuImgPath := "qemu-img"
	fullQemuPath, err := exec.LookPath(qemuBinPath)
	if err != nil {
		log.Fatalf("Unable to find %s within the $PATH", qemuBinPath)
	}

	// Iterate through the flags and build arguments
	var qemuArgs []string
	qemuArgs = append(qemuArgs, "-device", "virtio-rng-pci")
	qemuArgs = append(qemuArgs, "-smp", *qemuCPUs)
	qemuArgs = append(qemuArgs, "-m", *qemuMem)

	// Look for kvm device and enable for qemu if it exists
	if _, err = os.Stat("/dev/kvm"); os.IsNotExist(err) {
		qemuArgs = append(qemuArgs, "-machine", "q35")
	} else {
		qemuArgs = append(qemuArgs, "-enable-kvm")
		qemuArgs = append(qemuArgs, "-machine", "q35,accel=kvm:tcg")
	}

	if *qemuDiskPath != "" {
		// If disk doesn't exist then create one
		if _, err = os.Stat(*qemuDiskPath); os.IsNotExist(err) {
			log.Infof("Creating new qemu disk [%s]", *qemuDiskPath)
			fullQemuImgPath, err := exec.LookPath(qemuImgPath)
			if err != nil {
				log.Fatalf("Unable to find %s within the $PATH", qemuImgPath)
			}
			cmd := exec.Command(fullQemuImgPath, "create", "-f", "qcow2", *qemuDiskPath, *qemuDiskSize)
			if err = cmd.Run(); err != nil {
				log.Fatalf("Error creating disk [%s]:  %s", *qemuDiskPath, err.Error())
			}
		} else {
			log.Infof("Using existing disk [%s]", *qemuDiskPath)
		}
		qemuArgs = append(qemuArgs, "-disk", "file="+*qemuDiskPath+",format=raw")
	}

	// Check flags for iso/uefi boot and if so disable kernel boot
	if *qemuIso {
		*qemuKernel = false
		qemuIsoPath := buildPath(prefix, ".iso")
		qemuArgs = append(qemuArgs, "-cdrom", qemuIsoPath)
	}

	if *qemuUEFI {
		// Check for OVMF firmware before building paths
		_, err = os.Stat(*qemuFWPath)
		if err != nil {
			if os.IsNotExist(err) {
				log.Fatalf("File [%s] does not exist, please ensure OVMF is installed", *qemuFWPath)
			} else {
				log.Fatalf("%s", err.Error())
			}
		}

		*qemuKernel = false
		qemuIsoPath := buildPath(prefix, "-efi.iso")
		qemuArgs = append(qemuArgs, "-pflash", *qemuFWPath)
		qemuArgs = append(qemuArgs, "-cdrom", qemuIsoPath)
		qemuArgs = append(qemuArgs, "-boot", "d")
	}

	// build kernel boot config from bzImage/initrd/cmdline
	if *qemuKernel {
		qemuKernelPath := buildPath(prefix, "-bzImage")
		qemuInitrdPath := buildPath(prefix, "-initrd.img")
		qemuArgs = append(qemuArgs, "-kernel", qemuKernelPath)
		qemuArgs = append(qemuArgs, "-initrd", qemuInitrdPath)
		consoleString, err := ioutil.ReadFile(prefix + "-cmdline")
		if err != nil {
			log.Infof(" %s\n defaulting to console output", err.Error())
			qemuArgs = append(qemuArgs, "-append", "console=ttyS0 console=tty0 page_poison=1")
		} else {
			qemuArgs = append(qemuArgs, "-append", string(consoleString))
		}
	}

	if *qemuGUI != true {
		qemuArgs = append(qemuArgs, "-nographic")
	}

	// If verbosity is enabled print out the full path/arguments
	log.Debugf("%s %v\n", fullQemuPath, qemuArgs)

	cmd := exec.Command(fullQemuPath, qemuArgs...)

	// If we're not using a seperate window then link the execution to stdin/out
	if *qemuGUI != true {
		cmd.Stdin = os.Stdin
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
	}

	if err = cmd.Run(); err != nil {
		log.Fatalf("Error starting %s: %s", fullQemuPath, err.Error())
	}
}

func buildPath(prefix string, postfix string) string {
	path := prefix + postfix
	if _, err := os.Stat(path); os.IsNotExist(err) {
		log.Fatalf("File [%s] does not exist in current directory", path)
	}
	return path
}
<commit_msg>Fixed qemu argument typo and also correct use of raw disk<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"

	log "github.com/Sirupsen/logrus"
)

func runQemu(args []string) {
	qemuFlags := flag.NewFlagSet("qemu", flag.ExitOnError)
	qemuFlags.Usage = func() {
		fmt.Printf("USAGE: %s run qemu [options] prefix\n\n", os.Args[0])
		fmt.Printf("'prefix' specifies the path to the VM image.\n")
		fmt.Printf("\n")
		fmt.Printf("Options:\n")
		qemuFlags.PrintDefaults()
	}

	// Determine Flags
	qemuGUI := qemuFlags.Bool("gui", false, "Set qemu to use video output instead of stdio")
	qemuUEFI := qemuFlags.Bool("uefi", false, "Set UEFI boot from 'prefix'-efi.iso")
	qemuIso := qemuFlags.Bool("iso", false, "Set Legacy BIOS boot from 'prefix'.iso")
	qemuKernel := qemuFlags.Bool("kernel", true, "Set boot using 'prefix'-bzImage/-initrd/-cmdline")

	// Paths and settings for Disks and UEFI firware
	qemuDiskPath := qemuFlags.String("diskpath", "", "Path to disk image to use")
	qemuDiskSize := qemuFlags.String("disksize", "", "Size of disk to create, only created if it doesn't exist")
	qemuFWPath := qemuFlags.String("fwpath", "/usr/share/ovmf/bios.bin", "Path to OVMF firmware for UEFI boot")

	// VM configuration
	qemuArch := qemuFlags.String("arch", "x86_64", "Type of architecture to use, e.g. x86_64, aarch64")
	qemuCPUs := qemuFlags.String("cpus", "1", "Number of CPUs")
	qemuMem := qemuFlags.String("mem", "1024", "Amount of memory in MB")

	if err := qemuFlags.Parse(args); err != nil {
		log.Fatal("Unable to parse args")
	}
	remArgs := qemuFlags.Args()

	if len(remArgs) == 0 {
		fmt.Println("Please specify the prefix to the image to boot")
		qemuFlags.Usage()
		os.Exit(1)
	}
	prefix := remArgs[0]

	// Print warning if conflicting UEFI and ISO flags are set
	if *qemuUEFI && *qemuIso {
		log.Warnf("Both -iso and -uefi have been used")
	}

	// Before building qemu arguments, check qemu is in the $PATH
	qemuBinPath := "qemu-system-" + *qemuArch
	qemuImgPath := "qemu-img"
	fullQemuPath, err := exec.LookPath(qemuBinPath)
	if err != nil {
		log.Fatalf("Unable to find %s within the $PATH", qemuBinPath)
	}

	// Iterate through the flags and build arguments
	var qemuArgs []string
	qemuArgs = append(qemuArgs, "-device", "virtio-rng-pci")
	qemuArgs = append(qemuArgs, "-smp", *qemuCPUs)
	qemuArgs = append(qemuArgs, "-m", *qemuMem)

	// Look for kvm device and enable for qemu if it exists
	if _, err = os.Stat("/dev/kvm"); os.IsNotExist(err) {
		qemuArgs = append(qemuArgs, "-machine", "q35")
	} else {
		qemuArgs = append(qemuArgs, "-enable-kvm")
		qemuArgs = append(qemuArgs, "-machine", "q35,accel=kvm:tcg")
	}

	if *qemuDiskPath != "" {
		// If disk doesn't exist then create one
		if _, err = os.Stat(*qemuDiskPath); os.IsNotExist(err) {
			log.Infof("Creating new qemu disk [%s]", *qemuDiskPath)
			fullQemuImgPath, err := exec.LookPath(qemuImgPath)
			if err != nil {
				log.Fatalf("Unable to find %s within the $PATH", qemuImgPath)
			}
			cmd := exec.Command(fullQemuImgPath, "create", "-f", "qcow2", *qemuDiskPath, *qemuDiskSize)
			if err = cmd.Run(); err != nil {
				log.Fatalf("Error creating disk [%s]:  %s", *qemuDiskPath, err.Error())
			}
		} else {
			log.Infof("Using existing disk [%s]", *qemuDiskPath)
		}
		qemuArgs = append(qemuArgs, "-drive", "file="+*qemuDiskPath+",format=qcow2")
	}

	// Check flags for iso/uefi boot and if so disable kernel boot
	if *qemuIso {
		*qemuKernel = false
		qemuIsoPath := buildPath(prefix, ".iso")
		qemuArgs = append(qemuArgs, "-cdrom", qemuIsoPath)
	}

	if *qemuUEFI {
		// Check for OVMF firmware before building paths
		_, err = os.Stat(*qemuFWPath)
		if err != nil {
			if os.IsNotExist(err) {
				log.Fatalf("File [%s] does not exist, please ensure OVMF is installed", *qemuFWPath)
			} else {
				log.Fatalf("%s", err.Error())
			}
		}

		*qemuKernel = false
		qemuIsoPath := buildPath(prefix, "-efi.iso")
		qemuArgs = append(qemuArgs, "-pflash", *qemuFWPath)
		qemuArgs = append(qemuArgs, "-cdrom", qemuIsoPath)
		qemuArgs = append(qemuArgs, "-boot", "d")
	}

	// build kernel boot config from bzImage/initrd/cmdline
	if *qemuKernel {
		qemuKernelPath := buildPath(prefix, "-bzImage")
		qemuInitrdPath := buildPath(prefix, "-initrd.img")
		qemuArgs = append(qemuArgs, "-kernel", qemuKernelPath)
		qemuArgs = append(qemuArgs, "-initrd", qemuInitrdPath)
		consoleString, err := ioutil.ReadFile(prefix + "-cmdline")
		if err != nil {
			log.Infof(" %s\n defaulting to console output", err.Error())
			qemuArgs = append(qemuArgs, "-append", "console=ttyS0 console=tty0 page_poison=1")
		} else {
			qemuArgs = append(qemuArgs, "-append", string(consoleString))
		}
	}

	if *qemuGUI != true {
		qemuArgs = append(qemuArgs, "-nographic")
	}

	// If verbosity is enabled print out the full path/arguments
	log.Debugf("%s %v\n", fullQemuPath, qemuArgs)

	cmd := exec.Command(fullQemuPath, qemuArgs...)

	// If we're not using a seperate window then link the execution to stdin/out
	if *qemuGUI != true {
		cmd.Stdin = os.Stdin
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
	}

	if err = cmd.Run(); err != nil {
		log.Fatalf("Error starting %s: %s", fullQemuPath, err.Error())
	}
}

func buildPath(prefix string, postfix string) string {
	path := prefix + postfix
	if _, err := os.Stat(path); os.IsNotExist(err) {
		log.Fatalf("File [%s] does not exist in current directory", path)
	}
	return path
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/onsi/ginkgo/config"
	"regexp"

	"bytes"
	"flag"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"os/signal"
)

var numCPU int
var recurse bool
var runMagicI bool
var race bool
var reports []*bytes.Buffer
var executedCommands []*exec.Cmd

type suite struct {
	path     string
	isGinkgo bool
}

func init() {
	config.Flags("", false)

	flag.IntVar(&(numCPU), "nodes", 1, "The number of parallel test nodes to run")
	flag.BoolVar(&(recurse), "r", false, "Find and run test suites under the current directory recursively")
	flag.BoolVar(&(runMagicI), "i", false, "Run go test -i first, then run the test suite")
	flag.BoolVar(&(race), "race", false, "Run tests with race detection enabled")

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage of ginkgo:\n\n")
		fmt.Fprintf(os.Stderr, "ginkgo\n  Run the tests in the current directory.  The following flags are available:\n")
		flag.PrintDefaults()
		fmt.Fprintf(os.Stderr, "\n")
		fmt.Fprintf(os.Stderr, "ginkgo bootstrap\n  Bootstrap a test suite for the current package.\n\n")
		fmt.Fprintf(os.Stderr, "ginkgo generate <SUBJECT>\n  Generate a test file for SUBJECT, the file will be named SUBJECT_test.go\n  If omitted, a file named after the package will be created.\n\n")
		fmt.Fprintf(os.Stderr, "ginkgo version\n  Print ginkgo's version.\n\n")
		fmt.Fprintf(os.Stderr, "ginkgo help\n  Print this usage information.\n")
	}

	flag.Parse()
}

func main() {
	if flag.NArg() > 0 {
		handleSubcommands(flag.Args())
	}

	executedCommands = make([]*exec.Cmd, 0)
	reports = make([]*bytes.Buffer, 0)
	registerSignalHandler()

	passed := true

	suites := findSuitesInDir(".", recurse)

	for _, suite := range suites {
		passed = passed && runSuite(suite)
	}

	if passed {
		os.Exit(0)
	} else {
		os.Exit(1)
	}
}

func handleSubcommands(args []string) {
	if args[0] == "bootstrap" {
		generateBootstrap()
		os.Exit(0)
	} else if args[0] == "generate" {
		subject := ""
		if len(args) > 1 {
			subject = args[1]
		}
		generateSpec(subject)
		os.Exit(0)
	} else if args[0] == "help" {
		flag.Usage()
		os.Exit(0)
	} else if args[0] == "version" {
		fmt.Printf("Ginkgo V%s\n", config.VERSION)
		os.Exit(0)
	} else {
		fmt.Printf("Unkown command %s\n\n", args[0])
		flag.Usage()

		os.Exit(1)
	}
}

func findSuitesInDir(dir string, recurse bool) []suite {
	suites := []suite{}
	files, _ := ioutil.ReadDir(dir)
	re := regexp.MustCompile(`_test\.go$`)
	for _, file := range files {
		if !file.IsDir() && re.Match([]byte(file.Name())) {
			suites = append(suites, suite{path: dir, isGinkgo: filesHaveGinkgoSuite(dir, files)})
			break
		}
	}

	if recurse {
		re = regexp.MustCompile(`^\.`)
		for _, file := range files {
			if file.IsDir() && !re.Match([]byte(file.Name())) {
				suites = append(suites, findSuitesInDir(dir+"/"+file.Name(), recurse)...)
			}
		}
	}

	return suites
}

func filesHaveGinkgoSuite(dir string, files []os.FileInfo) bool {
	reTestFile := regexp.MustCompile(`_test\.go$`)
	reGinkgo := regexp.MustCompile(`package ginkgo|\/ginkgo"`)
	for _, file := range files {
		if !file.IsDir() && reTestFile.Match([]byte(file.Name())) {
			contents, _ := ioutil.ReadFile(dir + "/" + file.Name())
			if reGinkgo.Match(contents) {
				return true
			}
		}
	}

	return false
}

func runSuite(suite suite) bool {
	completions := make(chan bool)

	if runMagicI {
		runGoI(suite.path, race)
	}

	if suite.isGinkgo {
		for cpu := 0; cpu < numCPU; cpu++ {
			config.GinkgoConfig.ParallelNode = cpu + 1
			config.GinkgoConfig.ParallelTotal = numCPU

			args := config.BuildFlagArgs("ginkgo", config.GinkgoConfig, config.DefaultReporterConfig)
			if race {
				args = append([]string{"--race"}, args...)
			}

			var writer io.Writer
			if numCPU > 1 {
				buffer := new(bytes.Buffer)
				reports = append(reports, buffer)
				writer = buffer
			} else {
				writer = os.Stdout
			}

			go runCommand(suite.path, args, writer, completions)
		}

		passed := true

		for cpu := 0; cpu < numCPU; cpu++ {
			passed = passed && <-completions
		}

		if numCPU > 1 {
			printToScreen()
		}

		return passed
	} else {
		args := []string{}
		if race {
			args = append(args, "--race")
		}
		go runCommand(suite.path, args, os.Stdout, completions)
		return <-completions
	}
}

func printToScreen() {
	for _, report := range reports {
		fmt.Print(report.String())
	}
	os.Stdout.Sync()
}

func runGoI(path string, race bool) {
	args := []string{"test", "-i"}
	if race {
		args = append(args, "-race")
	}
	args = append(args, path)
	cmd := exec.Command("go", args...)
	output, err := cmd.CombinedOutput()
	if err != nil {
		fmt.Printf("go test -i %s failed with:\n%s", path, output)
		os.Exit(1)
	}
}

func runCommand(path string, args []string, stream io.Writer, completions chan bool) {
	args = append([]string{"test", "-v", path}, args...)

	cmd := exec.Command("go", args...)
	executedCommands = append(executedCommands, cmd)

	stdout, _ := cmd.StdoutPipe()
	stderr, _ := cmd.StderrPipe()

	go io.Copy(stream, stdout)
	go io.Copy(stream, stderr)

	err := cmd.Start()
	if err != nil {
		os.Exit(1)
	}

	err = cmd.Wait()
	completions <- (err == nil)
}

func registerSignalHandler() {
	go func() {
		c := make(chan os.Signal, 1)
		signal.Notify(c, os.Interrupt, os.Kill)

		select {
		case sig := <-c:
			for _, cmd := range executedCommands {
				cmd.Process.Signal(sig)
			}
			os.Exit(1)
		}
	}()
}
<commit_msg>ginkgo cli supports new -cover flag<commit_after>package main

import (
	"fmt"
	"github.com/onsi/ginkgo/config"
	"path/filepath"
	"regexp"

	"bytes"
	"flag"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"os/signal"
)

var numCPU int
var recurse bool
var runMagicI bool
var race bool
var cover bool
var reports []*bytes.Buffer
var executedCommands []*exec.Cmd

type suite struct {
	path        string
	packageName string
	isGinkgo    bool
}

func init() {
	config.Flags("", false)

	flag.IntVar(&(numCPU), "nodes", 1, "The number of parallel test nodes to run")
	flag.BoolVar(&(recurse), "r", false, "Find and run test suites under the current directory recursively")
	flag.BoolVar(&(runMagicI), "i", false, "Run go test -i first, then run the test suite")
	flag.BoolVar(&(race), "race", false, "Run tests with race detection enabled")
	flag.BoolVar(&(cover), "cover", false, "Run tests with coverage analysis, will generate coverage profiles with the package name in the current directory")

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage of ginkgo:\n\n")
		fmt.Fprintf(os.Stderr, "ginkgo\n  Run the tests in the current directory.  The following flags are available:\n")
		flag.PrintDefaults()
		fmt.Fprintf(os.Stderr, "\n")
		fmt.Fprintf(os.Stderr, "ginkgo bootstrap\n  Bootstrap a test suite for the current package.\n\n")
		fmt.Fprintf(os.Stderr, "ginkgo generate <SUBJECT>\n  Generate a test file for SUBJECT, the file will be named SUBJECT_test.go\n  If omitted, a file named after the package will be created.\n\n")
		fmt.Fprintf(os.Stderr, "ginkgo version\n  Print ginkgo's version.\n\n")
		fmt.Fprintf(os.Stderr, "ginkgo help\n  Print this usage information.\n")
	}

	flag.Parse()
}

func main() {
	if flag.NArg() > 0 {
		handleSubcommands(flag.Args())
	}

	executedCommands = make([]*exec.Cmd, 0)
	reports = make([]*bytes.Buffer, 0)
	registerSignalHandler()

	passed := true

	suites := findSuitesInDir(".", recurse)

	for _, suite := range suites {
		passed = passed && runSuite(suite)
	}

	if passed {
		os.Exit(0)
	} else {
		os.Exit(1)
	}
}

func handleSubcommands(args []string) {
	if args[0] == "bootstrap" {
		generateBootstrap()
		os.Exit(0)
	} else if args[0] == "generate" {
		subject := ""
		if len(args) > 1 {
			subject = args[1]
		}
		generateSpec(subject)
		os.Exit(0)
	} else if args[0] == "help" {
		flag.Usage()
		os.Exit(0)
	} else if args[0] == "version" {
		fmt.Printf("Ginkgo V%s\n", config.VERSION)
		os.Exit(0)
	} else {
		fmt.Printf("Unkown command %s\n\n", args[0])
		flag.Usage()

		os.Exit(1)
	}
}

func findSuitesInDir(dir string, recurse bool) []suite {
	suites := []suite{}
	files, _ := ioutil.ReadDir(dir)
	re := regexp.MustCompile(`_test\.go$`)
	for _, file := range files {
		if !file.IsDir() && re.Match([]byte(file.Name())) {
			suites = append(suites, suite{path: dir, packageName: packageNameForDir(dir), isGinkgo: filesHaveGinkgoSuite(dir, files)})
			break
		}
	}

	if recurse {
		re = regexp.MustCompile(`^\.`)
		for _, file := range files {
			if file.IsDir() && !re.Match([]byte(file.Name())) {
				suites = append(suites, findSuitesInDir(dir+"/"+file.Name(), recurse)...)
			}
		}
	}

	return suites
}

func packageNameForDir(dir string) string {
	path, _ := filepath.Abs(dir)
	return filepath.Base(path)
}

func filesHaveGinkgoSuite(dir string, files []os.FileInfo) bool {
	reTestFile := regexp.MustCompile(`_test\.go$`)
	reGinkgo := regexp.MustCompile(`package ginkgo|\/ginkgo"`)
	for _, file := range files {
		if !file.IsDir() && reTestFile.Match([]byte(file.Name())) {
			contents, _ := ioutil.ReadFile(dir + "/" + file.Name())
			if reGinkgo.Match(contents) {
				return true
			}
		}
	}

	return false
}

func runSuite(suite suite) bool {
	completions := make(chan bool)

	if runMagicI {
		runGoI(suite.path, race)
	}

	if suite.isGinkgo {
		for cpu := 0; cpu < numCPU; cpu++ {
			config.GinkgoConfig.ParallelNode = cpu + 1
			config.GinkgoConfig.ParallelTotal = numCPU

			args := config.BuildFlagArgs("ginkgo", config.GinkgoConfig, config.DefaultReporterConfig)
			if race {
				args = append([]string{"--race"}, args...)
			}
			if cover {
				args = append([]string{"--cover", "--coverprofile=" + suite.packageName + ".coverprofile"})
			}

			var writer io.Writer
			if numCPU > 1 {
				buffer := new(bytes.Buffer)
				reports = append(reports, buffer)
				writer = buffer
			} else {
				writer = os.Stdout
			}

			go runCommand(suite.path, args, writer, completions)
		}

		passed := true

		for cpu := 0; cpu < numCPU; cpu++ {
			passed = passed && <-completions
		}

		if numCPU > 1 {
			printToScreen()
		}

		return passed
	} else {
		args := []string{}
		if race {
			args = append(args, "--race")
		}
		if cover {
			args = append([]string{"--cover", "--coverprofile=" + suite.packageName + ".out"})
		}
		go runCommand(suite.path, args, os.Stdout, completions)
		return <-completions
	}
}

func printToScreen() {
	for _, report := range reports {
		fmt.Print(report.String())
	}
	os.Stdout.Sync()
}

func runGoI(path string, race bool) {
	args := []string{"test", "-i"}
	if race {
		args = append(args, "-race")
	}
	args = append(args, path)
	cmd := exec.Command("go", args...)
	output, err := cmd.CombinedOutput()
	if err != nil {
		fmt.Printf("go test -i %s failed with:\n%s", path, output)
		os.Exit(1)
	}
}

func runCommand(path string, args []string, stream io.Writer, completions chan bool) {
	args = append([]string{"test", "-v", path}, args...)

	cmd := exec.Command("go", args...)
	executedCommands = append(executedCommands, cmd)

	stdout, _ := cmd.StdoutPipe()
	stderr, _ := cmd.StderrPipe()

	go io.Copy(stream, stdout)
	go io.Copy(stream, stderr)

	err := cmd.Start()
	if err != nil {
		os.Exit(1)
	}

	err = cmd.Wait()
	completions <- (err == nil)
}

func registerSignalHandler() {
	go func() {
		c := make(chan os.Signal, 1)
		signal.Notify(c, os.Interrupt, os.Kill)

		select {
		case sig := <-c:
			for _, cmd := range executedCommands {
				cmd.Process.Signal(sig)
			}
			os.Exit(1)
		}
	}()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"

	"github.com/chzyer/readline"
)

func main() {
	fmt.Println("Gispy v0.0.0")
	fmt.Println("Press Ctrl+c to exit")
	rl, err := readline.New("gispy> ")
	if err != nil {
		log.Fatal(err)
	}
	defer rl.Close()
	for {
		text, err := rl.Readline()
		if nil != err {
			break
		}
		println(text)
	}
}
<commit_msg>break -> log.Fatal(err)<commit_after>package main

import (
	"fmt"
	"log"

	"github.com/chzyer/readline"
)

func main() {
	fmt.Println("Gispy v0.0.0")
	fmt.Println("Press Ctrl+c to exit")
	rl, err := readline.New("gispy> ")
	if err != nil {
		log.Fatal(err)
	}
	defer rl.Close()
	for {
		text, err := rl.Readline()
		if nil != err {
			log.Fatal(err)
		}
		println(text)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os/exec"
	"strings"
	"sync"
	"syscall"
	"time"
    "strconv"
)

const (
	SYNC_IDLE_SECS = 5
)

type GitBackend struct {
	gitPath string

	rootDir string

	syncLock      sync.Mutex
	isSyncPending bool
    isSyncActive bool   // Ignore all events during sync
    isOnline bool       // Can we talk to remote git / ssh server?

    lastEvent time.Time

	pushHook func()
}

func NewGitBackend(config *Config) *GitBackend {

	rootDir := config.syncDir

	gitPath, err := exec.LookPath("git")
	if err != nil {
		log.Fatal("Error looking for 'git' on path. ", err)
	}

	return &GitBackend{
        rootDir: rootDir,
        gitPath: gitPath,
        isOnline: true}
}

// A file or directory has been created
func (self *GitBackend) Changed(filename string) {
	if self.isGit(filename) || self.isSyncActive {
		return
	}
    self.lastEvent = time.Now()
	go self.syncLater()
}

// Run: git pull; git add --all ; git commit --all; git push
func (self *GitBackend) Sync() error {

    log.Println("* Sync start")
    self.isSyncActive = true

	var err *GitError

    self.checkGitConnection()

    if self.isOnline {
        // Pull first to ensure a fast-forward when we push
        err = self.pull()
        if err != nil {
            self.isSyncActive = false
            return err
        }
    }

	err = self.git("add", "--all")
	if err != nil {
        self.isSyncActive = false
		return err
	}

    self.displayStatus("status", "--porcelain")

	err = self.git("commit", "--all", "--message=loftus")
	if err != nil {
        // An err with status==1 means nothing to commit,
        // that counts as a clean exit
        self.isSyncActive = false
        log.Println("* Sync end")
		return err
	}

    if self.isOnline {
        err = self.push()
        if err != nil {
            self.isSyncActive = false
            return err
        }
    }

    self.isSyncActive = false
    log.Println("* Sync end")
	return nil
}

// Check whether we can talk to remote git repo, and inform
func (self *GitBackend) checkGitConnection() {

    isOnline := self.isOnlineCheck()

    if isOnline != self.isOnline {

        self.isOnline = isOnline

        if isOnline {
            log.Println("Back online")
            Info("Back online")
        } else {
            log.Println("Working offline")
            Warn("Could not connect to remote git repo. Working offline.")
        }
    }
}

//Display summary of changes
func (self *GitBackend) displayStatus(args ...string) {

    created, modified, deleted := self.status(args...)

    var msg string
    if len(created) == 1 {
        msg += "New: " + created[0]
    } else if len(created) > 1 {
        msg += "New: " + strconv.Itoa(len(created))
    }

    if len(modified) == 1 {
        msg += " Edit: " + modified[0]
    } else if len(modified) > 1 {
        msg += " Edit: " + strconv.Itoa(len(modified))
    }

    if len(deleted) == 1 {
        msg += " Del: " + deleted[0]
    } else if len(deleted) > 1 {
        msg += " Del: " + strconv.Itoa(len(deleted))
    }

    if len(msg) != 0 {
        Info(msg)
    }
}

// Register the function to be called after we push to remote
func (self *GitBackend) RegisterPushHook(callback func()) {
	self.pushHook = callback
}

// Should the inotify watch watch the given path
func (self *GitBackend) ShouldWatch(filename string) bool {
	return !self.isGit(filename)
}

// Status of directory. Returns filenames created, modified or deleted.
func (self *GitBackend) status(args ...string) (created []string, modified []string, deleted []string) {

	cmd := exec.Command(self.gitPath, args...)
	log.Println(strings.Join(cmd.Args, " "))

	cmd.Dir = self.rootDir

	output, err := cmd.CombinedOutput()
    if err != nil {
        log.Println(err)
    }
    if len(output) > 0 {
        log.Println(string(output))
    }

    for _, line := range strings.Split(string(output), "\n") {
        if len(line) == 0 {
            continue
        }

        // Replace double spaces and tabs with single space so that Split is predictable
        line = strings.Replace(line, "  ", " ", -1)
        line = strings.Replace(line, "\t", " ", -1)

        lineParts := strings.Split(line, " ")

        status := lineParts[0]
        filename := lineParts[1]

        switch status[0] {
            case 'A':
                created = append(created, filename)
            case 'M':
                modified = append(modified, filename)
            case 'R':                         // Renamed, but treat as Modified
                modified = append(modified, filename)
            case 'D':
                deleted = append(deleted, filename)
            case '?':
                log.Println("Unknown. Need git add", filename)
            default:
                log.Println("Other", status)
        }
    }

    return created, modified, deleted
}

// Is filename inside a .git directory?
func (self *GitBackend) isGit(filename string) bool {
	return strings.Contains(filename, ".git")
}

// Schedule a synchronise for in a few seconds. Run it in go routine.
func (self *GitBackend) syncLater() {

	// ensure only once per time - might be able to use sync.Once instead (?)
	self.syncLock.Lock()
	if self.isSyncPending {
		self.syncLock.Unlock()
		return
	}
	self.isSyncPending = true
	self.syncLock.Unlock()

    for time.Now().Sub(self.lastEvent) < (SYNC_IDLE_SECS * time.Second) {
        time.Sleep(time.Second)
    }

    log.Println("syncLater initiated sync")
	self.Sync()

	self.isSyncPending = false
}

// Run: git push
func (self *GitBackend) push() *GitError {
	err := self.git("push")
	if err == nil && self.pushHook != nil {
		go self.pushHook()
	}
	return err
}

// Run: git pull
func (self *GitBackend) pull() *GitError {

    var err *GitError

    err = self.git("fetch")
    if err != nil {
        return err
    }

    self.displayStatus("diff", "origin/master", "--name-status")
	err = self.git("merge", "origin/master")
	return err
}

// Run: git remote show origin
// We use this to check if we are online
func (self *GitBackend) isOnlineCheck() bool {
    return self.git("remote", "show", "origin") == nil
}

/* Runs a git command, returns nil if success, error if err
   Errors are not always bad. For example a "commit" that
   didn't have to do anything returns an error.
*/
func (self *GitBackend) git(gitCmd string, args ...string) *GitError {

	cmd := exec.Command(self.gitPath, append([]string{gitCmd}, args...)...)
	cmd.Dir = self.rootDir
	log.Println(strings.Join(cmd.Args, " "))

	output, err := cmd.CombinedOutput()
    if len(output) > 0 {
        log.Println(string(output))
    }

	if err == nil {
        return nil
    }

    exitStatus := err.(*exec.ExitError).Sys().(syscall.WaitStatus).ExitStatus()
    gitErr := &GitError{
        cmd: strings.Join(cmd.Args, " "),
        internalError: err,
        output: string(output),
        status: exitStatus}
    if exitStatus != 1 {            // 1 means command had nothing to do
        log.Println(err)
    }
    return gitErr
}

type GitError struct {
	cmd           string
	internalError error
	output        string
    status        int
}

// error implementation which displays git info
func (self *GitError) Error() string {
    msg := "git error running: " + self.cmd + "\n\n"
    msg += self.output + "\n"
	msg += self.internalError.Error()
    return msg
}
<commit_msg>Pushing if behind remote, even if no commit that time<commit_after>package main

import (
	"log"
	"os/exec"
	"strings"
	"sync"
	"syscall"
	"time"
    "strconv"
)

const (
	SYNC_IDLE_SECS = 5
)

type GitBackend struct {
	gitPath string

	rootDir string

	syncLock      sync.Mutex
	isSyncPending bool
    isSyncActive bool   // Ignore all events during sync
    isOnline bool       // Can we talk to remote git / ssh server?

    lastEvent time.Time

	pushHook func()
}

func NewGitBackend(config *Config) *GitBackend {

	rootDir := config.syncDir

	gitPath, err := exec.LookPath("git")
	if err != nil {
		log.Fatal("Error looking for 'git' on path. ", err)
	}

	return &GitBackend{
        rootDir: rootDir,
        gitPath: gitPath,
        isOnline: true}
}

// A file or directory has been created
func (self *GitBackend) Changed(filename string) {
	if self.isGit(filename) || self.isSyncActive {
		return
	}
    self.lastEvent = time.Now()
	go self.syncLater()
}

// Run: git pull; git add --all ; git commit --all; git push
func (self *GitBackend) Sync() error {

    log.Println("* Sync start")
    self.isSyncActive = true

	var err *GitError

    self.checkGitConnection()

    if self.isOnline {
        // Pull first to ensure a fast-forward when we push
        err = self.pull()
        if err != nil {
            self.isSyncActive = false
            return err
        }
    }

	err = self.git("add", "--all")
	if err != nil {
        self.isSyncActive = false
		return err
	}

    self.displayStatus("status", "--porcelain")

	err = self.git("commit", "--all", "--message=loftus")
    // An err with status==1 means nothing to commit, it's not an error
	if err != nil && err.status != 1 {
        self.isSyncActive = false
		return err
	}

    if self.isOnline && self.isBehindRemote() {
        err = self.push()
        if err != nil {
            self.isSyncActive = false
            return err
        }
    }

    self.isSyncActive = false
    log.Println("* Sync end")
	return nil
}

// Check whether we can talk to remote git repo, and inform
func (self *GitBackend) checkGitConnection() {

    isOnline := self.isOnlineCheck()

    if isOnline != self.isOnline {

        self.isOnline = isOnline

        if isOnline {
            log.Println("Back online")
            Info("Back online")
        } else {
            log.Println("Working offline")
            Warn("Could not connect to remote git repo. Working offline.")
        }
    }
}

//Display summary of changes
func (self *GitBackend) displayStatus(args ...string) {

    created, modified, deleted := self.status(args...)

    var msg string
    if len(created) == 1 {
        msg += "New: " + created[0]
    } else if len(created) > 1 {
        msg += "New: " + strconv.Itoa(len(created))
    }

    if len(modified) == 1 {
        msg += " Edit: " + modified[0]
    } else if len(modified) > 1 {
        msg += " Edit: " + strconv.Itoa(len(modified))
    }

    if len(deleted) == 1 {
        msg += " Del: " + deleted[0]
    } else if len(deleted) > 1 {
        msg += " Del: " + strconv.Itoa(len(deleted))
    }

    if len(msg) != 0 {
        Info(msg)
    }
}

// Register the function to be called after we push to remote
func (self *GitBackend) RegisterPushHook(callback func()) {
	self.pushHook = callback
}

// Should the inotify watch watch the given path
func (self *GitBackend) ShouldWatch(filename string) bool {
	return !self.isGit(filename)
}

// Status of directory. Returns filenames created, modified or deleted.
func (self *GitBackend) status(args ...string) (created []string, modified []string, deleted []string) {

	cmd := exec.Command(self.gitPath, args...)
	log.Println(strings.Join(cmd.Args, " "))

	cmd.Dir = self.rootDir

	output, err := cmd.CombinedOutput()
    if err != nil {
        log.Println(err)
    }
    if len(output) > 0 {
        log.Println(string(output))
    }

    for _, line := range strings.Split(string(output), "\n") {
        if len(line) == 0 {
            continue
        }

        // Replace double spaces and tabs with single space so that Split is predictable
        line = strings.Replace(line, "  ", " ", -1)
        line = strings.Replace(line, "\t", " ", -1)

        lineParts := strings.Split(line, " ")

        status := lineParts[0]
        filename := lineParts[1]

        switch status[0] {
            case 'A':
                created = append(created, filename)
            case 'M':
                modified = append(modified, filename)
            case 'R':                         // Renamed, but treat as Modified
                modified = append(modified, filename)
            case 'D':
                deleted = append(deleted, filename)
            case '?':
                log.Println("Unknown. Need git add", filename)
            default:
                log.Println("Other", status)
        }
    }

    return created, modified, deleted
}

// Is filename inside a .git directory?
func (self *GitBackend) isGit(filename string) bool {
	return strings.Contains(filename, ".git")
}

// Schedule a synchronise for in a few seconds. Run it in go routine.
func (self *GitBackend) syncLater() {

	// ensure only once per time - might be able to use sync.Once instead (?)
	self.syncLock.Lock()
	if self.isSyncPending {
		self.syncLock.Unlock()
		return
	}
	self.isSyncPending = true
	self.syncLock.Unlock()

    for time.Now().Sub(self.lastEvent) < (SYNC_IDLE_SECS * time.Second) {
        time.Sleep(time.Second)
    }

    log.Println("syncLater initiated sync")
	self.Sync()

	self.isSyncPending = false
}

// Run: git push
func (self *GitBackend) push() *GitError {
	err := self.git("push")
	if err == nil && self.pushHook != nil {
		go self.pushHook()
	}
	return err
}

// Run: git pull
func (self *GitBackend) pull() *GitError {

    var err *GitError

    err = self.git("fetch")
    if err != nil {
        return err
    }

    self.displayStatus("diff", "origin/master", "--name-status")
	err = self.git("merge", "origin/master")
	return err
}

// Run: git remote show origin
// We use this to check if we are online
func (self *GitBackend) isOnlineCheck() bool {
    return self.git("remote", "show", "origin") == nil
}

// Is the local repo behind the remote, i.e. is a push needed?
func (self *GitBackend) isBehindRemote() bool {

    created, modified, deleted := self.status("diff", "origin/master", "--name-status")
    return (len(created) != 0 || len(modified) != 0 || len(deleted) != 0)
}

/* Runs a git command, returns nil if success, error if err
   Errors are not always bad. For example a "commit" that
   didn't have to do anything returns an error.
*/
func (self *GitBackend) git(gitCmd string, args ...string) *GitError {

	cmd := exec.Command(self.gitPath, append([]string{gitCmd}, args...)...)
	cmd.Dir = self.rootDir
	log.Println(strings.Join(cmd.Args, " "))

	output, err := cmd.CombinedOutput()
    if len(output) > 0 {
        log.Println(string(output))
    }

	if err == nil {
        return nil
    }

    exitStatus := err.(*exec.ExitError).Sys().(syscall.WaitStatus).ExitStatus()
    gitErr := &GitError{
        cmd: strings.Join(cmd.Args, " "),
        internalError: err,
        output: string(output),
        status: exitStatus}
    if exitStatus != 1 {            // 1 means command had nothing to do
        log.Println(err)
    }
    return gitErr
}

type GitError struct {
	cmd           string
	internalError error
	output        string
    status        int
}

// error implementation which displays git info
func (self *GitError) Error() string {
    msg := "git error running: " + self.cmd + "\n\n"
    msg += self.output + "\n"
	msg += self.internalError.Error()
    return msg
}
<|endoftext|>


<commit_before>package main

import (
    "os"
    "os/exec"
    "bufio"
    "fmt"
    "flag"
    "regexp"
    "encoding/json"
    "io/ioutil"
    "github.com/speedata/gogit"
    "github.com/aws/aws-sdk-go/aws"
    "github.com/aws/aws-sdk-go/aws/session"
    "github.com/aws/aws-sdk-go/service/s3/s3manager"
    "github.com/deckarep/golang-set"
    "path/filepath"
)

const REF_S3_PUSH string = "refs/heads/s3-pushed"
const CONFIG_FILE_PATH string = ".git_s3_push"

type Repository struct {
    GitRepo         *gogit.Repository
    HeadCommit      *gogit.Commit
    LastPushCommit  *gogit.Commit
    UnpushedFiles   mapset.Set
    Config          RepoConfig
    IgnoreRegexes   []*regexp.Regexp
    S3Uploader      S3Uploader
}

type RepoConfig struct {
    S3Region        string
    S3Bucket        string
    Ignore          []string
}

func OpenRepository() (*Repository, error) {
    repo := new(Repository)
    repo.UnpushedFiles = mapset.NewSet()

    wd, err := os.Getwd()
    if err != nil {
        return nil, err
    }

    path := filepath.Join(wd, ".git")
    if _, err := os.Stat(path); os.IsNotExist(err) {
        return nil, err
    }

    gitRepo, err := gogit.OpenRepository(path)
    if err != nil {
        return nil, err
    }
    repo.GitRepo = gitRepo

    return repo, nil
}

func (repo *Repository) ReadConfigFile() error {
    file, err := ioutil.ReadFile(CONFIG_FILE_PATH)
    if err != nil {
        return err
    }

    err = json.Unmarshal(file, &repo.Config)
    if err != nil {
        return err
    }

    err = repo.CompileIgnoreRegexes()
    if err != nil {
        return err
    }

    return nil
}

func (repo *Repository) CompileIgnoreRegexes() error {
    for _, regexStr := range repo.Config.Ignore {
        regex, err := regexp.Compile(regexStr)
        if err != nil {
            return err
        }

        repo.IgnoreRegexes = append(repo.IgnoreRegexes, regex)
    }

    return nil
}

func (repo Repository) SaveConfigToFile() error {
    jsonData, err := json.Marshal(repo.Config)
    if err != nil {
        return err
    }

    err = ioutil.WriteFile(CONFIG_FILE_PATH, jsonData, 0644)
    if err != nil {
        return err
    }

    return nil
}

func (repo *Repository) FindRelevantCommits() error {
    headRef, err := repo.GitRepo.LookupReference("HEAD")
    if err != nil {
        return err
    }

    headCommit, err := repo.GitRepo.LookupCommit(headRef.Target())
    if err != nil {
        return err
    }
    repo.HeadCommit = headCommit

    lastPushRef, err := repo.GitRepo.LookupReference(REF_S3_PUSH)
    if err != nil {
        return nil
    }

    lastPushCommit, err := repo.GitRepo.LookupCommit(lastPushRef.Target())
    if err != nil {
        return nil
    }
    repo.LastPushCommit = lastPushCommit

    return nil
}

func (repo *Repository) ReadGitModifiedFiles(scanner *bufio.Scanner, stop chan bool)  {
    for scanner.Scan() {
        file := scanner.Text()

        if _, err := os.Stat(file); os.IsNotExist(err) {
            continue
        }

        matched := false
        for _, regex := range repo.IgnoreRegexes {
            if regex.Match([]byte(file)) {
                fmt.Println("Skipping file " + file + " matches ignore spec " + regex.String())
                matched = true
                break
            }
        }

        if !matched {
            repo.UnpushedFiles.Add(scanner.Text())
        }
    }

    stop <- true
}

func (repo *Repository) FindCommitModifiedFiles(commit *gogit.Commit) error {
    cmd := exec.Command("git", "show", "--name-only", "--oneline", commit.Id().String())
    out, err := cmd.StdoutPipe()
    if err != nil {
        return err
    }

    err = cmd.Start()
    if err != nil {
        return err
    }

    scanner := bufio.NewScanner(out)

    stop := make(chan bool)
    go repo.ReadGitModifiedFiles(scanner, stop)
    <-stop
    cmd.Wait()

    return nil
}

func (repo *Repository) FindUnpushedModifiedFiles() error {
    queue := []*gogit.Commit{};
    visited := mapset.NewSet();

    currentCommit := repo.HeadCommit;
    for currentCommit != nil && currentCommit.ParentCount() > 0 {
        if repo.LastPushCommit != nil && repo.LastPushCommit.Id().Equal(currentCommit.Id()) {
            break;
        }

        err := repo.FindCommitModifiedFiles(currentCommit)
        if err != nil {
            return err
        }

        for i := 0; i < currentCommit.ParentCount(); i++ {
            parentCommit := currentCommit.Parent(i)
            if !visited.Contains(parentCommit) {
                queue = append(queue, parentCommit)
            }
        }

        if len(queue) < 1 {
            break;
        }

        currentCommit = queue[0]
        queue = queue[1:]
    }

    return nil
}

func (repo Repository) UpdateGitLastPushRef() error {
    newLastPushRef := repo.HeadCommit.Id().String()
    cmd := exec.Command("git", "update-ref", REF_S3_PUSH, newLastPushRef)

    err := cmd.Start()
    if err != nil {
        return err
    }

    cmd.Wait()
    return nil
}

type S3Uploader struct {
    BucketName      *string
    S3Uploader      *s3manager.Uploader
}

func InitS3Uploader(config RepoConfig) *S3Uploader {
    uploader := new(S3Uploader)
    uploader.BucketName = aws.String(config.S3Bucket)

    s3config := aws.Config{Region: aws.String(config.S3Region)}
    s3uploader := s3manager.NewUploader(session.New(&s3config))
    uploader.S3Uploader = s3uploader

    return uploader
}

func (uploader S3Uploader) UploadFile(path string) error {
    file, err := os.Open(path)
    if err != nil {
        return err
    }

    result, err := uploader.S3Uploader.Upload(&s3manager.UploadInput{
        Body: file,
        Bucket: uploader.BucketName,
        Key: aws.String(path),
    })

    if err != nil {
        return err
    }

    fmt.Println(result.Location)
    return nil
}

func main() {
    repo, err := OpenRepository()
    if err != nil {
        fmt.Println(err)
        os.Exit(1)
    }
    repo.ReadConfigFile()

    flag.StringVar(&repo.Config.S3Bucket, "b", repo.Config.S3Bucket, "Destination S3 bucket name")
    flag.StringVar(&repo.Config.S3Region, "r", repo.Config.S3Region, "AWS region of destination bucket")
    saveConfig := flag.Bool("save", false, "Save destination region/bucket to config file")
    flag.Parse()

    if repo.Config.S3Bucket == "" {
        flag.Usage()
        os.Exit(1)
    } else if (repo.Config.S3Region == "") {
        flag.Usage()
        os.Exit(1)
    } else if (*saveConfig) {
        err := repo.SaveConfigToFile()
        if err != nil {
            fmt.Println("WARNING: Failed to save config to file: ", err)
        }
    }

    if err := repo.FindRelevantCommits(); err != nil {
        fmt.Println(err)
        os.Exit(1)
    }

    repo.FindUnpushedModifiedFiles();

    if repo.UnpushedFiles.Cardinality() == 0 {
        fmt.Println("No modified files to push")
        os.Exit(0)
    }

    uploader := InitS3Uploader(repo.Config)

    for filePath := range repo.UnpushedFiles.Iter() {
        fmt.Println("Uploading: ", filePath.(string))
        err := uploader.UploadFile(filePath.(string))
        if err != nil {
            fmt.Println(err)
            os.Exit(1)
        }
    }

    err = repo.UpdateGitLastPushRef()
    if err != nil {
        fmt.Println("Failed to update LAST_S3_PUSH ref with git: ", err)
    }
}<commit_msg>Add support for pushing files not tracked in git (ie generated files)<commit_after>package main

import (
    "os"
    "os/exec"
    "bufio"
    "fmt"
    "flag"
    "regexp"
    "encoding/json"
    "io/ioutil"
    "github.com/speedata/gogit"
    "github.com/aws/aws-sdk-go/aws"
    "github.com/aws/aws-sdk-go/aws/session"
    "github.com/aws/aws-sdk-go/service/s3/s3manager"
    "github.com/deckarep/golang-set"
    "path/filepath"
)

const REF_S3_PUSH string = "refs/heads/s3-pushed"
const CONFIG_FILE_PATH string = ".git_s3_push"

type Repository struct {
    GitRepo         *gogit.Repository
    HeadCommit      *gogit.Commit
    LastPushCommit  *gogit.Commit
    UnpushedFiles   mapset.Set
    Config          RepoConfig
    IgnoreRegexes   []*regexp.Regexp
    S3Uploader      S3Uploader
}

type RepoConfig struct {
    S3Region        string
    S3Bucket        string
    Ignore          []string
    IncludeNonGit   []string
}

func OpenRepository() (*Repository, error) {
    repo := new(Repository)
    repo.UnpushedFiles = mapset.NewSet()

    wd, err := os.Getwd()
    if err != nil {
        return nil, err
    }

    path := filepath.Join(wd, ".git")
    if _, err := os.Stat(path); os.IsNotExist(err) {
        return nil, err
    }

    gitRepo, err := gogit.OpenRepository(path)
    if err != nil {
        return nil, err
    }
    repo.GitRepo = gitRepo

    return repo, nil
}

func (repo *Repository) ReadConfigFile() error {
    file, err := ioutil.ReadFile(CONFIG_FILE_PATH)
    if err != nil {
        return err
    }

    err = json.Unmarshal(file, &repo.Config)
    if err != nil {
        return err
    }

    err = repo.CompileIgnoreRegexes()
    if err != nil {
        return err
    }

    return nil
}

func (repo *Repository) CompileIgnoreRegexes() error {
    for _, regexStr := range repo.Config.Ignore {
        regex, err := regexp.Compile(regexStr)
        if err != nil {
            return err
        }

        repo.IgnoreRegexes = append(repo.IgnoreRegexes, regex)
    }

    return nil
}

func (repo Repository) SaveConfigToFile() error {
    jsonData, err := json.Marshal(repo.Config)
    if err != nil {
        return err
    }

    err = ioutil.WriteFile(CONFIG_FILE_PATH, jsonData, 0644)
    if err != nil {
        return err
    }

    return nil
}

func (repo *Repository) FindRelevantCommits() error {
    headRef, err := repo.GitRepo.LookupReference("HEAD")
    if err != nil {
        return err
    }

    headCommit, err := repo.GitRepo.LookupCommit(headRef.Target())
    if err != nil {
        return err
    }
    repo.HeadCommit = headCommit

    lastPushRef, err := repo.GitRepo.LookupReference(REF_S3_PUSH)
    if err != nil {
        return nil
    }

    lastPushCommit, err := repo.GitRepo.LookupCommit(lastPushRef.Target())
    if err != nil {
        return nil
    }
    repo.LastPushCommit = lastPushCommit

    return nil
}

func (repo *Repository) ReadGitModifiedFiles(scanner *bufio.Scanner, stop chan bool)  {
    for scanner.Scan() {
        file := scanner.Text()

        if _, err := os.Stat(file); os.IsNotExist(err) {
            continue
        }

        matched := false
        for _, regex := range repo.IgnoreRegexes {
            if regex.Match([]byte(file)) {
                fmt.Println("Skipping file " + file + " matches ignore spec " + regex.String())
                matched = true
                break
            }
        }

        if !matched {
            repo.UnpushedFiles.Add(scanner.Text())
        }
    }

    stop <- true
}

func (repo *Repository) FindCommitModifiedFiles(commit *gogit.Commit) error {
    cmd := exec.Command("git", "show", "--name-only", "--oneline", commit.Id().String())
    out, err := cmd.StdoutPipe()
    if err != nil {
        return err
    }

    err = cmd.Start()
    if err != nil {
        return err
    }

    scanner := bufio.NewScanner(out)

    stop := make(chan bool)
    go repo.ReadGitModifiedFiles(scanner, stop)
    <-stop
    cmd.Wait()

    return nil
}

func (repo *Repository) FindUnpushedModifiedFiles() error {
    queue := []*gogit.Commit{};
    visited := mapset.NewSet();

    currentCommit := repo.HeadCommit;
    for currentCommit != nil && currentCommit.ParentCount() > 0 {
        if repo.LastPushCommit != nil && repo.LastPushCommit.Id().Equal(currentCommit.Id()) {
            break;
        }

        err := repo.FindCommitModifiedFiles(currentCommit)
        if err != nil {
            return err
        }

        for i := 0; i < currentCommit.ParentCount(); i++ {
            parentCommit := currentCommit.Parent(i)
            if !visited.Contains(parentCommit) {
                queue = append(queue, parentCommit)
            }
        }

        if len(queue) < 1 {
            break;
        }

        currentCommit = queue[0]
        queue = queue[1:]
    }

    return nil
}

func (repo Repository) UpdateGitLastPushRef() error {
    newLastPushRef := repo.HeadCommit.Id().String()
    cmd := exec.Command("git", "update-ref", REF_S3_PUSH, newLastPushRef)

    err := cmd.Start()
    if err != nil {
        return err
    }

    cmd.Wait()
    return nil
}

type S3Uploader struct {
    BucketName      *string
    S3Uploader      *s3manager.Uploader
}

func InitS3Uploader(config RepoConfig) *S3Uploader {
    uploader := new(S3Uploader)
    uploader.BucketName = aws.String(config.S3Bucket)

    s3config := aws.Config{Region: aws.String(config.S3Region)}
    s3uploader := s3manager.NewUploader(session.New(&s3config))
    uploader.S3Uploader = s3uploader

    return uploader
}

func (uploader S3Uploader) UploadFile(path string) error {
    file, err := os.Open(path)
    if err != nil {
        return err
    }

    result, err := uploader.S3Uploader.Upload(&s3manager.UploadInput{
        Body: file,
        Bucket: uploader.BucketName,
        Key: aws.String(path),
    })

    if err != nil {
        return err
    }

    fmt.Println(result.Location)
    return nil
}

func main() {
    repo, err := OpenRepository()
    if err != nil {
        fmt.Println(err)
        os.Exit(1)
    }
    repo.ReadConfigFile()

    flag.StringVar(&repo.Config.S3Bucket, "b", repo.Config.S3Bucket, "Destination S3 bucket name")
    flag.StringVar(&repo.Config.S3Region, "r", repo.Config.S3Region, "AWS region of destination bucket")
    saveConfig := flag.Bool("save", false, "Save destination region/bucket to config file")
    forceNonTracked := flag.Bool("force-external", false, "Force the upload of files not tracked in git (IncludeNonGit files in config)")
    flag.Parse()

    if repo.Config.S3Bucket == "" {
        flag.Usage()
        os.Exit(1)
    } else if (repo.Config.S3Region == "") {
        flag.Usage()
        os.Exit(1)
    } else if (*saveConfig) {
        err := repo.SaveConfigToFile()
        if err != nil {
            fmt.Println("WARNING: Failed to save config to file: ", err)
        }
    }

    if err := repo.FindRelevantCommits(); err != nil {
        fmt.Println(err)
        os.Exit(1)
    }

    repo.FindUnpushedModifiedFiles();

    if repo.UnpushedFiles.Cardinality() == 0 && !*forceNonTracked {
        fmt.Println("No modified files to push")
        os.Exit(0)
    }

    for _, includedFile := range repo.Config.IncludeNonGit {
        if _, err := os.Stat(includedFile); os.IsNotExist(err) {
            continue
        }

        repo.UnpushedFiles.Add(includedFile)
    }

    if repo.UnpushedFiles.Cardinality() == 0 {
        fmt.Println("No files to push")
        os.Exit(0)
    }

    uploader := InitS3Uploader(repo.Config)

    for filePath := range repo.UnpushedFiles.Iter() {
        fmt.Println("Uploading: ", filePath.(string))
        err := uploader.UploadFile(filePath.(string))
        if err != nil {
            fmt.Println(err)
            os.Exit(1)
        }
    }

    err = repo.UpdateGitLastPushRef()
    if err != nil {
        fmt.Println("Failed to update LAST_S3_PUSH ref with git: ", err)
    }
}<|endoftext|>


<commit_before>// Copyright 2017 Debpkg authors. All rights reserved.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package debpkg

import (
	"fmt"
	"github.com/stretchr/testify/assert"
	"github.com/xor-gate/debpkg/internal/test"
	"go/build"
	"os/exec"
	"testing"
)

// testWrite writes the deb package to a temporary file and verifies with native dpkg tool when available
func testWrite(t *testing.T, deb *DebPkg) error {
	f := test.TempFile(t)
	err := deb.Write(f)
	if err == nil {
		testReadWithNativeDpkg(t, f)
	}
	return err
}

// testReadWithNativeDpkg tests a single debian package with the dpkg tool when present
func testReadWithNativeDpkg(t *testing.T, filename string) {
	dpkgCmd, err := exec.LookPath("dpkg")
	if err != nil || dpkgCmd == "" {
		return
	}

	dpkg := func(action, filename string) error {
		return exec.Command(dpkgCmd, "--"+action, filename).Run()
	}

	// TODO test control, extract, verify...
	assert.Nil(t, dpkg("info", filename))
	assert.Nil(t, dpkg("contents", filename))
}

// TestDirectory verifies adding a single directory recursive to the package
func TestAddDirectory(t *testing.T) {
	deb := New()
	defer deb.Close()
	deb.SetName("debpkg-test-add-directory")
	deb.SetArchitecture("all")

	assert.Nil(t, deb.AddDirectory("internal"))
	assert.Nil(t, testWrite(t, deb))
}

// TestWrite verifies Write works as expected with adding just one datafile
func TestWrite(t *testing.T) {
	deb := New()
	defer deb.Close()

	deb.SetName("debpkg-test")
	deb.SetArchitecture("all")
	deb.SetVersion("0.0.1")
	deb.SetMaintainer("Foo Bar")
	deb.SetMaintainerEmail("foo@bar.com")
	deb.SetHomepage("https://foobar.com")
	deb.SetShortDescription("some awesome foobar pkg")
	deb.SetDescription("very very very very long description")

	// Set version control system info for control file
	deb.SetVcsType(VcsTypeGit)
	deb.SetVcsURL("https://github.com/xor-gate/secdl")
	deb.SetVcsBrowser("https://github.com/xor-gate/secdl")

	deb.AddFile("debpkg.go")

	assert.Nil(t, testWrite(t, deb))

	// Try to Write again on implicit closed package
	assert.Equal(t, ErrClosed, testWrite(t, deb))
}

// TestWriteError tests if the Write fails with the correct errors
func TestWriteError(t *testing.T) {
	deb := New()
	defer deb.Close()
	assert.NotNil(t, deb.Write(""), "deb.Write should return nil")

	deb.control.info.name = "pkg"
	assert.Equal(t, fmt.Errorf("empty package name"), deb.Write(""))
}

// ExampleDebPkgWrite demonstrates generating a simple package
func ExampleDebPkg_Write() {
	tempfile := TempDir() + "/foobar.deb"

	deb := New()
	defer deb.Close()

	deb.SetName("foobar")
	deb.SetVersion("1.2.3")
	deb.SetArchitecture("amd64")
	deb.SetMaintainer("Foo Bar")
	deb.SetMaintainerEmail("foo@bar.com")
	deb.SetHomepage("http://foobar.com")

	deb.SetShortDescription("Minimal foo bar package")
	deb.SetDescription("Foo bar package doesn't do anything")

	deb.AddFile("debpkg.go")
	fmt.Println(deb.Write(tempfile))

	// Do something with tempfile other than removing it...
}

// TestFilenameFromFullVersion verifies if the whole version string is correctly calculated
func TestFilenameFromFullVersion(t *testing.T) {
	deb := New()
	defer deb.Close()

	deb.SetName("foo")
	deb.SetVersion("1.33.7")
	deb.SetArchitecture("amd64")

	assert.Equal(t, "foo-1.33.7_amd64.deb", deb.GetFilename())
}

// TestGetArchitecture checks the current build.Default.GOARCH compatible debian architecture
func TestGetArchitecture(t *testing.T) {
	// On debian 386 GOARCH is presented as i386
	goarch := build.Default.GOARCH
	build.Default.GOARCH = "386"
	assert.Equal(t, "i386", GetArchitecture())
	build.Default.GOARCH = goarch

	// Check current build GOARCH
	if build.Default.GOARCH != "386" {
		assert.Equal(t, build.Default.GOARCH, GetArchitecture())
	}
}
<commit_msg>Cover more functions<commit_after>// Copyright 2017 Debpkg authors. All rights reserved.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package debpkg

import (
	"fmt"
	"go/build"
	"os/exec"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/xor-gate/debpkg/internal/test"
)

// testWrite writes the deb package to a temporary file and verifies with native dpkg tool when available
func testWrite(t *testing.T, deb *DebPkg) error {
	f := test.TempFile(t)
	err := deb.Write(f)
	if err == nil {
		testReadWithNativeDpkg(t, f)
	}
	return err
}

// testReadWithNativeDpkg tests a single debian package with the dpkg tool when present
func testReadWithNativeDpkg(t *testing.T, filename string) {
	dpkgCmd, err := exec.LookPath("dpkg")
	if err != nil || dpkgCmd == "" {
		return
	}

	dpkg := func(action, filename string) error {
		return exec.Command(dpkgCmd, "--"+action, filename).Run()
	}

	// TODO test control, extract, verify...
	assert.Nil(t, dpkg("info", filename))
	assert.Nil(t, dpkg("contents", filename))
}

// TestDirectory verifies adding a single directory recursive to the package
func TestAddDirectory(t *testing.T) {
	deb := New()
	defer deb.Close()
	deb.SetName("debpkg-test-add-directory")
	deb.SetArchitecture("all")

	assert.Nil(t, deb.AddDirectory("internal"))
	assert.Nil(t, testWrite(t, deb))
}

// TestWrite verifies Write works as expected with adding just one datafile
func TestWrite(t *testing.T) {
	deb := New()
	defer deb.Close()

	deb.SetName("debpkg-test")
	deb.SetArchitecture("all")
	deb.SetVersion("0.0.1")
	deb.SetMaintainer("Foo Bar")
	deb.SetMaintainerEmail("foo@bar.com")
	deb.SetHomepage("https://foobar.com")
	deb.SetShortDescription("some awesome foobar pkg")
	deb.SetDescription("very very very very long description")

	// Set version control system info for control file
	deb.SetVcsType(VcsTypeGit)
	deb.SetVcsURL("https://github.com/xor-gate/secdl")
	deb.SetVcsBrowser("https://github.com/xor-gate/secdl")

	deb.AddFile("debpkg.go")
	deb.AddFile("debpkg_test.go", "/foo/awesome/test.go")
	deb.AddFileString("this is a real file", "/real/file.txt")

	assert.Nil(t, testWrite(t, deb))

	// Try to Write again on implicit closed package
	assert.Equal(t, ErrClosed, testWrite(t, deb))
}

// TestWriteError tests if the Write fails with the correct errors
func TestWriteError(t *testing.T) {
	deb := New()
	defer deb.Close()
	assert.NotNil(t, deb.Write(""), "deb.Write should return nil")

	deb.control.info.name = "pkg"
	assert.Equal(t, fmt.Errorf("empty package name"), deb.Write(""))
}

// ExampleDebPkgWrite demonstrates generating a simple package
func ExampleDebPkg_Write() {
	tempfile := TempDir() + "/foobar.deb"

	deb := New()
	defer deb.Close()

	deb.SetName("foobar")
	deb.SetVersion("1.2.3")
	deb.SetArchitecture("amd64")
	deb.SetMaintainer("Foo Bar")
	deb.SetMaintainerEmail("foo@bar.com")
	deb.SetHomepage("http://foobar.com")

	deb.SetShortDescription("Minimal foo bar package")
	deb.SetDescription("Foo bar package doesn't do anything")

	deb.AddFile("debpkg.go")
	fmt.Println(deb.Write(tempfile))

	// Do something with tempfile other than removing it...
}

// TestFilenameFromFullVersion verifies if the whole version string is correctly calculated
func TestFilenameFromFullVersion(t *testing.T) {
	deb := New()
	defer deb.Close()

	deb.SetName("foo")
	deb.SetVersion("1.33.7")
	deb.SetArchitecture("amd64")

	assert.Equal(t, "foo-1.33.7_amd64.deb", deb.GetFilename())
}

// TestGetArchitecture checks the current build.Default.GOARCH compatible debian architecture
func TestGetArchitecture(t *testing.T) {
	// On debian 386 GOARCH is presented as i386
	goarch := build.Default.GOARCH
	build.Default.GOARCH = "386"
	assert.Equal(t, "i386", GetArchitecture())
	build.Default.GOARCH = goarch

	// Check current build GOARCH
	if build.Default.GOARCH != "386" {
		assert.Equal(t, build.Default.GOARCH, GetArchitecture())
	}
}
<|endoftext|>


<commit_before>/* rigo/definitions.go */
package ri

import (
	"fmt"
)

type Rter interface { /* TODO: add String() */
	Serialise() string
}

type RtName string

func (s RtName) Serialise() string {
	return string(s)
}

type RtBoolean bool

func (s RtBoolean) String() string {
	return s.Serialise()
}

func (s RtBoolean) Serialise() string {
	if bool(s) {
		return "1"
	}
	return "0"
}

type RtInt int

func (i RtInt) Serialise() string {
	return fmt.Sprintf("%d",int(i))
}

type RtIntArray []RtInt

func (a RtIntArray) Serialise() string {
	out := ""
	for i,r := range a {
		out += fmt.Sprintf("%d",int(r))
		if i < (len(a) - 1) {
			out += " "
		}
	}
	return fmt.Sprintf("[%s]",out)
}

type RtFloat float64

func (f RtFloat) String() string {
	return f.Serialise()
}

func (f RtFloat) Serialise() string {
	return fmt.Sprintf("%f",float64(f))
}

type RtFloatArray []RtFloat

func (a RtFloatArray) Serialise() string {
	out := ""
	for i,r := range a {
		out += fmt.Sprintf("%f",float64(r))
		if i < (len(a) - 1) {
			out += " "
		}
	}
	return fmt.Sprintf("[%s]",out)
}

type RtToken string

func (s RtToken) String() string {
	return s.Serialise()
}

func (s RtToken) Serialise() string {
	return fmt.Sprintf("\"%s\"",string(s))
}

type RtTokenArray []RtToken

func (a RtTokenArray) Serialise() string {
	out := ""
	for i := 0; i < len(a); i++ {
		out += a[i].Serialise()
		if i < (len(a) - 1) {
			out += " "
		}
	}
	return fmt.Sprintf("[%s]",out)
}

type RtColor []RtFloat

func (c RtColor) String() string {
	return c.Serialise()
}

func (c RtColor) Serialise() string {
	out := ""
	for i,r := range c {
		out += fmt.Sprintf("%f",float64(r))
		if i < (len(c) - 1) {
			out += " "
		}
	}
	return fmt.Sprintf("[%s]",out)
}

type RtPoint [3]RtFloat

func (p RtPoint) Serialise() string {
	return fmt.Sprintf("%f %f %f",p[0],p[1],p[2])
}

type RtPointArray []RtPoint

func (p RtPointArray) Serialise() string {
	out := ""
	for i := 0; i < len(p); i++ {
		out += p[i].Serialise()
		if i < len(p) - 1 {
			out += " "
		}
	}
	return fmt.Sprintf("[%s]",out)
}

type RtVector [3]RtFloat
type RtNormal [3]RtFloat
type RtHpoint [4]RtFloat
type RtMatrix [16]RtFloat

func (m RtMatrix) Serialise() string {
	out := ""
	for i := 0; i < 16; i++ {
		out += fmt.Sprintf("%f",m[i])
		if i < 15 {
			out += " "
		}
	}

	return fmt.Sprintf("[%s]",out)
}

type RtBasis  [16]RtFloat

func (b RtBasis) Serialise() string {
	out := ""
	for i := 0; i < 16; i++ {
		out += fmt.Sprintf("%f",b[i])
		if i < 15 {
			out += " "
		}
	}
	return fmt.Sprintf("[%s]",out)
}

type RtBound  [6]RtFloat

func (b RtBound) Serialise() string {
	return fmt.Sprintf("[%f %f %f %f %f %f]",b[0],b[1],b[2],b[3],b[4],b[5])
}

type RtString string

func (s RtString) Serialise() string {
	return fmt.Sprintf("\"%s\"",string(s))
}

type RtStringArray []RtString

func (a RtStringArray) Serialise() string {
	out := ""
	for i := 0; i < len(a); i++ {
		out += a[i].Serialise()
		if i < len(a) - 1 {
			out += " "
		}
	}
	return fmt.Sprintf("[%s]",out)
}

type RtLightHandle uint

func (l RtLightHandle) String() string {
	return l.Serialise()
}

func (l RtLightHandle) Serialise() string {
	return fmt.Sprintf("%d",uint(l))
}

type RtObjectHandle uint

func (l RtObjectHandle) String() string {
	return l.Serialise()
}

func (l RtObjectHandle) Serialise() string {
	return fmt.Sprintf("%d",uint(l))
}

type RtFilterFunc string

func (s RtFilterFunc) Serialise() string {
	return fmt.Sprintf("\"%s\"",string(s))
}

type RtProcSubdivFunc string

func (s RtProcSubdivFunc) Serialise() string {
	return fmt.Sprintf("\"%s\"",string(s))
}

type RtProcFreeFunc string

func (s RtProcFreeFunc) Serialise() string {
	return fmt.Sprintf("\"%s\"",string(s))
}

const (
	Perspective RtToken = "perspective"
	Orthographic RtToken = "orthographic"
)

type RtArchiveCallbackFunc string

func (s RtArchiveCallbackFunc) Serialise() string {
	return fmt.Sprintf("\"%s\"",string(s))
}

const (
	BoxFilter RtFilterFunc = "box"
	TriangleFilter RtFilterFunc = "triangle"
	CatmullRomFilter RtFilterFunc = "catmull-rom"
	GaussianFilter RtFilterFunc = "gaussian"
	SincFilter RtFilterFunc = "sinc"

	ReadArchiveCallback RtArchiveCallbackFunc = "ReadArchive"

	Bilinear RtToken = "bilinear"
	Bicubic RtToken = "bicubic"
	RGBA RtToken = "RGBA"
	P RtToken = "P"
	Pz RtToken = "Pz"
	Pw RtToken = "Pw"
	N RtToken = "N"
	Cs RtToken = "Cs"
	Os RtToken = "Os"

	ProcDelayedReadArchive RtProcSubdivFunc = "DelayedReadArchive"
	ProcRunProgram RtProcSubdivFunc = "RunProgram"
	ProcDynamicLoad RtProcSubdivFunc = "DynamicLoad"

	ProcFree RtProcFreeFunc = "free"

)
<commit_msg>improved definitions<commit_after>/* rigo/definitions.go */
package ri

import (
	"fmt"
)

type Rter interface {
	String() string
	Serialise() string
}

/* RtName internal use for RIB command names */
type RtName string

func (s RtName) String() string {
	return s.Serialise()
}

func (s RtName) Serialise() string {
	return string(s)
}

/* RtBoolean boolean value */
type RtBoolean bool

func (s RtBoolean) String() string {
	return s.Serialise()
}

func (s RtBoolean) Serialise() string {
	if bool(s) {
		return "1"
	}
	return "0"
}

/* RtInt integer value */
type RtInt int

func (i RtInt) String() string {
	return i.Serialise()
}

func (i RtInt) Serialise() string {
	return fmt.Sprintf("%d",int(i))
}

/* RtIntArray integer array */
type RtIntArray []RtInt

func (a RtIntArray) String() string {
	return a.Serialise()
}

func (a RtIntArray) Serialise() string {
	out := ""
	for i,r := range a {
		out += fmt.Sprintf("%d",int(r))
		if i < (len(a) - 1) {
			out += " "
		}
	}
	return fmt.Sprintf("[%s]",out)
}

/* RtFloat float64 value */
type RtFloat float64

func (f RtFloat) String() string {
	return f.Serialise()
}

func (f RtFloat) Serialise() string {
	return fmt.Sprintf("%f",float64(f))
}

/* RtFloatArray float64 array */
type RtFloatArray []RtFloat

func (a RtFloatArray) String() string {
	return a.Serialise()
}

func (a RtFloatArray) Serialise() string {
	out := ""
	for i,r := range a {
		out += fmt.Sprintf("%f",float64(r))
		if i < (len(a) - 1) {
			out += " "
		}
	}
	return fmt.Sprintf("[%s]",out)
}

/* RtToken */
type RtToken string

func (s RtToken) String() string {
	return s.Serialise()
}

func (s RtToken) Serialise() string {
	return fmt.Sprintf("\"%s\"",string(s))
}

/* RtToken array */
type RtTokenArray []RtToken

func (a RtTokenArray) String() string {
	return a.Serialise()
}

func (a RtTokenArray) Serialise() string {
	out := ""
	for i := 0; i < len(a); i++ {
		out += a[i].Serialise()
		if i < (len(a) - 1) {
			out += " "
		}
	}
	return fmt.Sprintf("[%s]",out)
}

/* RtColor implemented as an array */
type RtColor []RtFloat

func (c RtColor) String() string {
	return c.Serialise()
}

func (c RtColor) Serialise() string {
	out := ""
	for i,r := range c {
		out += fmt.Sprintf("%f",float64(r))
		if i < (len(c) - 1) {
			out += " "
		}
	}
	return fmt.Sprintf("[%s]",out)
}

/* RtPoint */
type RtPoint [3]RtFloat

func (p RtPoint) String() string {
	return p.Serialise()
}

func (p RtPoint) Serialise() string {
	return fmt.Sprintf("%f %f %f",p[0],p[1],p[2])
}

/* RtPointArray */
type RtPointArray []RtPoint

func (p RtPointArray) String() string {
	return p.Serialise()
}

func (p RtPointArray) Serialise() string {
	out := ""
	for i := 0; i < len(p); i++ {
		out += p[i].Serialise()
		if i < len(p) - 1 {
			out += " "
		}
	}
	return fmt.Sprintf("[%s]",out)
}

/* RtVector */
type RtVector [3]RtFloat

func (v RtVector) String() string {
	return v.Serialise()
}

func (v RtVector) Serialise() string {
	return fmt.Sprintf("[%f %f %f]",v[0],v[1],v[2])
}

/* RtNormal */
type RtNormal [3]RtFloat

func (n RtNormal) String() string {
	return n.Serialise()
}

func (n RtNormal) Serialise() string {
	return fmt.Sprintf("[%f %f %f]",v[0],v[1],v[2])
}

/* RtHpoint */
type RtHpoint [4]RtFloat

func (h RtHpoint) String() string {
	return h.Serialise()
}

func (h RtHpoint) Serialise() string {
	return fmt.Sprintf("[%f %f %f %f]",v[0],v[1],v[2],v[3])
}

/* RtMatrix */
type RtMatrix [16]RtFloat

func (m RtMatrix) String() string {
	return m.Serialise()
}

func (m RtMatrix) Serialise() string {
	out := ""
	for i := 0; i < 16; i++ {
		out += fmt.Sprintf("%f",m[i])
		if i < 15 {
			out += " "
		}
	}

	return fmt.Sprintf("[%s]",out)
}

/* RtBasis */
type RtBasis  [16]RtFloat

func (m RtBasis) String() string {
	return m.Serialise()
}

func (b RtBasis) Serialise() string {
	out := ""
	for i := 0; i < 16; i++ {
		out += fmt.Sprintf("%f",b[i])
		if i < 15 {
			out += " "
		}
	}
	return fmt.Sprintf("[%s]",out)
}

/* RtBound */
type RtBound  [6]RtFloat

func (b RtBound) String() string {
	return b.Serialise()
}

func (b RtBound) Serialise() string {
	return fmt.Sprintf("[%f %f %f %f %f %f]",b[0],b[1],b[2],b[3],b[4],b[5])
}

/* RtString */
type RtString string

func (s RtString) String() string {
	return s.Serialise()
}

func (s RtString) Serialise() string {
	return fmt.Sprintf("\"%s\"",string(s))
}

/* RtStringArray array of strings */
type RtStringArray []RtString

func (a RtStringArray) String() string {
	return a.Serialise()
}

func (a RtStringArray) Serialise() string {
	out := ""
	for i := 0; i < len(a); i++ {
		out += a[i].Serialise()
		if i < len(a) - 1 {
			out += " "
		}
	}
	return fmt.Sprintf("[%s]",out)
}

/* RtLightHandle */
type RtLightHandle uint

func (l RtLightHandle) String() string {
	return l.Serialise()
}

func (l RtLightHandle) Serialise() string {
	return fmt.Sprintf("%d",uint(l))
}

/* RtObjectHandle */
type RtObjectHandle uint

func (l RtObjectHandle) String() string {
	return l.Serialise()
}

func (l RtObjectHandle) Serialise() string {
	return fmt.Sprintf("%d",uint(l))
}

/* RtFilterFunc */
type RtFilterFunc string

func (s RtFilterFunc) String() string {
	return s.Serialise()
}

func (s RtFilterFunc) Serialise() string {
	return fmt.Sprintf("\"%s\"",string(s))
}

/* RtProcSubdivFunc subdivision function */
type RtProcSubdivFunc string

func (s RtProcSubdivFunc) String() string {
	return s.Serialise()
}

func (s RtProcSubdivFunc) Serialise() string {
	return fmt.Sprintf("\"%s\"",string(s))
}

/* RtProcFreeFunc */
type RtProcFreeFunc string

func (s RtProcFreeFunc) String() string {
	return s.Serialise()
}

func (s RtProcFreeFunc) Serialise() string {
	return fmt.Sprintf("\"%s\"",string(s))
}

/* RtArchiveCallbackFunc */
type RtArchiveCallbackFunc string

func (s RtArchiveCallbackFunc) String() string {
	return s.Serialise()
}

func (s RtArchiveCallbackFunc) Serialise() string {
	return fmt.Sprintf("\"%s\"",string(s))
}

const (
	BoxFilter RtFilterFunc = "box"
	TriangleFilter RtFilterFunc = "triangle"
	CatmullRomFilter RtFilterFunc = "catmull-rom"
	GaussianFilter RtFilterFunc = "gaussian"
	SincFilter RtFilterFunc = "sinc"

	ReadArchiveCallback RtArchiveCallbackFunc = "ReadArchive"

	Perspective RtToken = "perspective"
	Orthographic RtToken = "orthographic"
	Bilinear RtToken = "bilinear"
	Bicubic RtToken = "bicubic"
	RGBA RtToken = "RGBA"
	P RtToken = "P"
	Pz RtToken = "Pz"
	Pw RtToken = "Pw"
	N RtToken = "N"
	Cs RtToken = "Cs"
	Os RtToken = "Os"

	ProcDelayedReadArchive RtProcSubdivFunc = "DelayedReadArchive"
	ProcRunProgram RtProcSubdivFunc = "RunProgram"
	ProcDynamicLoad RtProcSubdivFunc = "DynamicLoad"

	ProcFree RtProcFreeFunc = "free"

)

var (
	ErrArrayTooBig = fmt.Errorf("insufficient memory to construct array")
	ErrBadArgument = fmt.Errorf("incorrect parameter value")
	ErrBadArray    = fmt.Errorf("invalid array specification")
	ErrBadBasis    = fmt.Errorf("undefined basis matrix name")
	ErrBadColor	   = fmt.Errorf("invalid color specification")
	ErrBadHandle   = fmt.Errorf("invalid light or object handle")
	ErrBadParamlist= fmt.Errorf("parameter list type mismatch")
	ErrBadRIPCode  = fmt.Errorf("invalid encoded RIB request code")
	ErrBadStringToken = fmt.Errorf("undefined encoded string token")
	ErrBadToken = fmt.Errorf("invalid binary token")
	ErrBadVersion = fmt.Errorf("protocol version number mismatch")
	ErrLimitCheck = fmt.Errorf("overflowing an internal limit")
	ErrOutOfMemory = fmt.Errorf("generic instance of insufficient memory")
	ErrProtocolBotch = fmt.Errorf("malformed binary encoding")
	ErrStringTooBig = fmt.Errorf("insufficient memory to read string")
	ErrSyntaxError = fmt.Errorf("general syntactic error")
	ErrUnregistered = fmt.Errorf("undefined RIB request")
)

<|endoftext|>


<commit_before>/* https://leetcode.com/problems/merge-k-sorted-lists/#/description
Merge k sorted linked lists and return it as one sorted list. Analyze and describe its complexity.
*/

package leetcode

import "math"

func mergeKLists(lists []*ListNode) *ListNode {
	var (
		res, tmp      *ListNode
		minIndex, min int
	)

	for cur := res; ; lists[minIndex] = lists[minIndex].Next {
		minIndex, min = -1, math.MaxInt32
		for i, list := range lists {
			if list != nil && list.Val < min {
				min = list.Val
				minIndex = i
			}
		}

		if minIndex == -1 {
			break
		}

		tmp = &ListNode{Val: min}
		if res == nil {
			res = tmp
		} else {
			cur.Next = tmp
		}
		cur = tmp
	}
	return res
}
<commit_msg>add another thinking: Divide and Conquer<commit_after>/* https://leetcode.com/problems/merge-k-sorted-lists/#/description
Merge k sorted linked lists and return it as one sorted list. Analyze and describe its complexity.
*/

package leetcode

func mergeKLists(lists []*ListNode) *ListNode {
	if len(lists) == 0 {
		return nil
	}

	var helper func(lists []*ListNode) *ListNode
	helper = func(lists []*ListNode) *ListNode {
		length := len(lists)
		half := length / 2

		if length == 1 {
			return lists[0]
		} else if length == 2 {
			var (
				c1, c2   = lists[0], lists[1]
				res, cur *ListNode
			)

			if c1 == nil {
				return c2
			}
			if c2 == nil {
				return c1
			}

			if c1.Val < c2.Val {
				res, cur, c1 = c1, c1, c1.Next
			} else {
				res, cur, c2 = c2, c2, c2.Next
			}

			for c1 != nil && c2 != nil {
				if c1.Val < c2.Val {
					cur.Next, c1 = c1, c1.Next
				} else {
					cur.Next, c2 = c2, c2.Next
				}
				cur = cur.Next
			}

			if c1 != nil {
				cur.Next = c1
			}
			if c2 != nil {
				cur.Next = c2
			}
			return res
		}
		return mergeKLists([]*ListNode{mergeKLists(lists[half:]), mergeKLists(lists[:half])})
	}

	return helper(lists)
}

/*
import "math"

func mergeKLists(lists []*ListNode) *ListNode {
	var (
		res, tmp      *ListNode
		minIndex, min int
	)

	for cur := res; ; lists[minIndex] = lists[minIndex].Next {
		minIndex, min = -1, math.MaxInt32
		for i, list := range lists {
			if list != nil && list.Val < min {
				min = list.Val
				minIndex = i
			}
		}

		if minIndex == -1 {
			break
		}

		tmp = &ListNode{Val: min}
		if res == nil {
			res = tmp
		} else {
			cur.Next = tmp
		}
		cur = tmp
	}
	return res
}
*/
<|endoftext|>