<commit_before>package main

import (
	"fmt"
	"os"

	"github.com/mitchellh/mapstructure"
	"github.com/pelletier/go-toml"
	flag "github.com/spf13/pflag"
	"log"
	"strings"
)

const defaultTomlConfig = `
title = "Tailon file viewer"
relative-root = "/"
listen-addr = ":8080"
allowed-commands = ["tail", "grep", "sed", "awk"]

[commands]

  [commands.tail]
  action = ["tail", "-n", "$lines", "-F", "$path"]

  [commands.grep]
  stdin = "tail"
  action = ["grep", "--text", "--line-buffered", "--color=never", "-e", "$script"]
  default = ".*"

  [commands.sed]
  stdin = "tail"
  action = ["sed", "-u", "-e", "$script"]
  default = "s/.*/&/"

  [commands.awk]
  stdin = "tail"
  action = ["awk", "--sandbox", "$script"]
  default = "{print $0; fflush()}"
`

const scriptDescription = `
Usage: tailon [options] -c <config file>
Usage: tailon [options] <filespec> [<filespec> ...]

Tailon is a webapp for looking at and searching through log files.
`

const scriptEpilog = `
The format of the one or more positional 'filespec' arguments is:
  [[glob|dir|file],alias=name,group=name,]<path>

Example usage:
  tailon /var/log/messages /var/log/debug /var/log/*.log
  tailon -b localhost:8080 -c config.ini
`

const configFileHelp = `
<todo>
`

type CommandSpec struct {
	Stdin   string
	Action  []string
	Default string
}

func parseTomlConfig(config string) (*toml.Tree, map[string]CommandSpec, error) {
	cfg, err := toml.Load(config)

	commands := make(map[string]CommandSpec)

	cfg_commands := cfg.Get("commands").(*toml.Tree).ToMap()
	for key, value := range cfg_commands {
		command := CommandSpec{}
		err := mapstructure.Decode(value, &command)
		if err != nil {
			log.Fatal(err)
		}
		commands[key] = command
	}

	return cfg, commands, err
}

type FileSpec struct {
	Path  string
	Type  string
	Alias string
	Group string
}

func parseFileSpec(spec string) (FileSpec, error) {
	var filespec FileSpec
	parts := strings.Split(spec, ",")

	if length := len(parts); length == 1 {
		return FileSpec{spec, "file", "", ""}, nil
	}

	path, parts := parts[len(parts)-1], parts[:len(parts)-1]

	for _, part := range parts {
		if strings.HasPrefix(part, "group=") {
			group := strings.SplitN(part, "=", 2)[1]
			group = strings.Trim(group, "'\" ")
			filespec.Group = group
		} else if strings.HasPrefix(part, "alias=") {
			filespec.Alias = strings.SplitN(part, "=", 2)[1]
		} else {
			switch part {
			case "file", "dir", "glob":
				filespec.Type = part
			}
		}

	}

	if filespec.Type == "" {
		filespec.Type = "file"
	}
	filespec.Path = path
	return filespec, nil

}

type Config struct {
	RelativeRoot        string
	BindAddr            string
	ConfigPath          string
	WrapLinesInitial    bool
	TailLinesInitial    int
	AllowedCommandNames []string

	CommandSpecs   map[string]CommandSpec
	CommandScripts map[string]string
	FileSpecs      []FileSpec
}

var config = Config{}

func main() {
	defaultConfig, commandSpecs, _ := parseTomlConfig(defaultTomlConfig)

	printHelp := flag.BoolP("help", "h", false, "Show this help message and exit")
	printConfigHelp := flag.BoolP("help-config", "e", false, "Show config file help and exit")

	flag.StringVarP(&config.BindAddr, "bind", "b", defaultConfig.Get("listen-addr").(string), "Listen on the specified address and port")
	flag.StringVarP(&config.ConfigPath, "config", "c", "", "")
	flag.StringVarP(&config.RelativeRoot, "relative-root", "r", defaultConfig.Get("relative-root").(string), "webapp relative root")

	flag.Parse()

	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, strings.TrimLeft(scriptDescription, "\n"))
		flag.PrintDefaults()
		fmt.Fprintln(os.Stderr, strings.TrimRight(scriptEpilog, "\n"))
		os.Exit(2)
	}

	if *printHelp {
		flag.Usage()
		os.Exit(0)
	}

	if *printConfigHelp {
		fmt.Fprintln(os.Stderr, strings.Trim(configFileHelp, "\n"))
		os.Exit(0)
	}

	filespecs := make([]FileSpec, len(flag.Args()))
	for _, spec := range flag.Args() {
		if filespec, err := parseFileSpec(spec); err != nil {
			fmt.Fprintf(os.Stderr, "Error parsing argument '%s': %s\n", spec, err)
			os.Exit(1)
		} else {
			filespecs = append(filespecs, filespec)
		}
	}
	config.FileSpecs = filespecs

	if len(config.FileSpecs) == 0 {
		fmt.Fprintln(os.Stderr, "No files specified on command-line or in config file")
		os.Exit(2)
	}

	config.RelativeRoot = "/" + strings.TrimLeft(config.RelativeRoot, "/")
	config.RelativeRoot = strings.TrimRight(config.RelativeRoot, "/") + "/"
	mapstructure.Decode(defaultConfig.Get("allowed-commands"), &config.AllowedCommandNames)
	config.CommandSpecs = commandSpecs

	config.CommandScripts = make(map[string]string)
	for cmd, values := range config.CommandSpecs {
		config.CommandScripts[cmd] = values.Default
	}

	logger := log.New(os.Stdout, "http: ", log.LstdFlags)
	logger.Println("Server is starting", config.RelativeRoot)

	server := SetupServer(config, logger)
	server.ListenAndServe()
}
<commit_msg>Comments and minor restructuring<commit_after>// A webapp for looking at and searching through files.
package main

import (
	"fmt"
	"os"

	"github.com/mitchellh/mapstructure"
	"github.com/pelletier/go-toml"
	flag "github.com/spf13/pflag"
	"log"
	"strings"
)

const scriptDescription = `
Usage: tailon [options] -c <config file>
Usage: tailon [options] <filespec> [<filespec> ...]

Tailon is a webapp for looking at and searching through files and streams.
`

const scriptEpilog = `
Tailon can be configured through a config file or with command-line flags.

The command-line interface expects one or more filespec arguments, which
specify the files or directories to be served. The expected format is:

  [[glob|dir|file],alias=name,group=name,]<path>

The default filespec is 'file' and points to a single, possibly non-existent
file. The file name in the UI can be overwritten with the 'alias=' specifier.

The 'glob' filespec evaluates to the list of files that match a shell file
name pattern. The pattern is evaluated each time the file list is refreshed.
The 'alias=' specifier overwrites the parent directory of each matched file.

The 'dir' specifier evaluates to all files in a directory.

The "group=" specifier sets the group in which files appear in the file
dropdown of the toolbar.

Example usage:
  tailon alias=messages,/var/log/messages "glob:/var/log/*.log"
  tailon -b localhost:8080 -c config.toml
`

const configFileHelp = `
<todo>
`

const defaultTomlConfig = `
title = "Tailon file viewer"
relative-root = "/"
listen-addr = ":8080"
allowed-commands = ["tail", "grep", "sed", "awk"]

[commands]

  [commands.tail]
  action = ["tail", "-n", "$lines", "-F", "$path"]

  [commands.grep]
  stdin = "tail"
  action = ["grep", "--text", "--line-buffered", "--color=never", "-e", "$script"]
  default = ".*"

  [commands.sed]
  stdin = "tail"
  action = ["sed", "-u", "-e", "$script"]
  default = "s/.*/&/"

  [commands.awk]
  stdin = "tail"
  action = ["awk", "--sandbox", "$script"]
  default = "{print $0; fflush()}"

`

type CommandSpec struct {
	Stdin   string
	Action  []string
	Default string
}

func parseTomlConfig(config string) (*toml.Tree, map[string]CommandSpec) {
	cfg, err := toml.Load(config)
	if err != nil {
		log.Fatal("Error parsing config: ", err)
	}

	commands := make(map[string]CommandSpec)

	cfg_commands := cfg.Get("commands").(*toml.Tree).ToMap()
	for key, value := range cfg_commands {
		command := CommandSpec{}
		err := mapstructure.Decode(value, &command)
		if err != nil {
			log.Fatal(err)
		}
		commands[key] = command
	}

	return cfg, commands
}

// FileSpec is an instance of a file to be monitored. These are mapped to
// os.Args or the [files] elements in the config file.
type FileSpec struct {
	Path  string
	Type  string
	Alias string
	Group string
}

// Parse a string into a filespec. Example inputs are:
//   file,alias=1,group=2,/var/log/messages
//   /var/log/messages
//   glob,/var/log/*
func parseFileSpec(spec string) (FileSpec, error) {
	var filespec FileSpec
	parts := strings.Split(spec, ",")

	// If no specifiers are given, default is file.
	if length := len(parts); length == 1 {
		return FileSpec{spec, "file", "", ""}, nil
	}

	// The last part is the path. We'll probably need a more robust
	// solution in the future.
	path, parts := parts[len(parts)-1], parts[:len(parts)-1]

	for _, part := range parts {
		if strings.HasPrefix(part, "group=") {
			group := strings.SplitN(part, "=", 2)[1]
			group = strings.Trim(group, "'\" ")
			filespec.Group = group
		} else if strings.HasPrefix(part, "alias=") {
			filespec.Alias = strings.SplitN(part, "=", 2)[1]
		} else {
			switch part {
			case "file", "dir", "glob":
				filespec.Type = part
			}
		}

	}

	if filespec.Type == "" {
		filespec.Type = "file"
	}
	filespec.Path = path
	return filespec, nil

}

type Config struct {
	RelativeRoot        string
	BindAddr            string
	ConfigPath          string
	WrapLinesInitial    bool
	TailLinesInitial    int
	AllowedCommandNames []string

	CommandSpecs   map[string]CommandSpec
	CommandScripts map[string]string
	FileSpecs      []FileSpec
}

var config = Config{}

func main() {
	defaultConfig, commandSpecs := parseTomlConfig(defaultTomlConfig)

	printHelp := flag.BoolP("help", "h", false, "Show this help message and exit")
	printConfigHelp := flag.BoolP("help-config", "e", false, "Show config file help and exit")

	flag.StringVarP(&config.BindAddr, "bind", "b", defaultConfig.Get("listen-addr").(string), "Listen on the specified address and port")
	flag.StringVarP(&config.ConfigPath, "config", "c", "", "")
	flag.StringVarP(&config.RelativeRoot, "relative-root", "r", defaultConfig.Get("relative-root").(string), "webapp relative root")

	flag.Parse()

	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, strings.TrimLeft(scriptDescription, "\n"))
		flag.PrintDefaults()
		fmt.Fprintln(os.Stderr, strings.TrimRight(scriptEpilog, "\n"))
		os.Exit(2)
	}

	if *printHelp {
		flag.Usage()
		os.Exit(0)
	}

	if *printConfigHelp {
		fmt.Fprintln(os.Stderr, strings.Trim(configFileHelp, "\n"))
		os.Exit(0)
	}

	filespecs := make([]FileSpec, len(flag.Args()))
	for _, spec := range flag.Args() {
		if filespec, err := parseFileSpec(spec); err != nil {
			fmt.Fprintf(os.Stderr, "Error parsing argument '%s': %s\n", spec, err)
			os.Exit(1)
		} else {
			filespecs = append(filespecs, filespec)
		}
	}
	config.FileSpecs = filespecs

	if len(config.FileSpecs) == 0 {
		fmt.Fprintln(os.Stderr, "No files specified on command-line or in config file")
		os.Exit(2)
	}

	config.RelativeRoot = "/" + strings.TrimLeft(config.RelativeRoot, "/")
	config.RelativeRoot = strings.TrimRight(config.RelativeRoot, "/") + "/"
	mapstructure.Decode(defaultConfig.Get("allowed-commands"), &config.AllowedCommandNames)
	config.CommandSpecs = commandSpecs

	config.CommandScripts = make(map[string]string)
	for cmd, values := range config.CommandSpecs {
		config.CommandScripts[cmd] = values.Default
	}

	logger := log.New(os.Stdout, "http: ", log.LstdFlags)
	logger.Println("Server is starting", config.RelativeRoot)

	server := SetupServer(config, logger)
	server.ListenAndServe()
}
<|endoftext|>


<commit_before>/*
 * Copyright 2016 Frank Wessels <fwessels@xs4all.nl>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package s3git

import (
	"errors"
	"fmt"

	"github.com/s3git/s3git-go/internal/backend"
	"github.com/s3git/s3git-go/internal/cas"
	"github.com/s3git/s3git-go/internal/core"
	"github.com/s3git/s3git-go/internal/kv"
	"github.com/s3git/s3git-go/internal/util"

	"encoding/hex"
	"github.com/dustin/go-humanize"
	"sync"
	"bytes"
)

// Perform a push to the back end for the repository
func (repo Repository) Push(hydrated bool, progress func(maxTicks int64)) error {

	list, err := kv.ListLevel1Prefixes()
	if err != nil {
		return err
	}

	return push(list, hydrated, progress)
}

// Push any new commit objects including all added objects to the back end store
func push(prefixChan <-chan []byte, hydrated bool, progress func(maxTicks int64)) error {

	client, err := backend.GetDefaultClient()
	if err != nil {
		return err
	}

	// Get map of prefixes already in store
	prefixesInBackend, err := listPrefixes(client)
	if err != nil {
		return err
	}

	prefixesToPush := []string{}

	for prefixByte := range prefixChan {

		prefix := hex.EncodeToString(prefixByte)

		_, verified := prefixesInBackend[prefix]

		// We can safely skip in case a prefix object is verified (pushed as last object)
		if !verified {

			prefixesToPush = append(prefixesToPush, prefix)
		}
	}

	if len(prefixesToPush) == 0 {
		return nil
	}

	progress(int64(len(prefixesToPush)))

	for _, prefix := range prefixesToPush {

		// Get prefix object
		po, err := core.GetPrefixObject(prefix)
		if err != nil {
			return err
		}

		// Get commit object
		co, err := core.GetCommitObject(po.S3gitFollowMe)
		if err != nil {
			return err
		}

		// Get tree object
		to, err := core.GetTreeObject(co.S3gitTree)
		if err != nil {
			return err
		}

		// first push all added blobs in this commit ...
		err = pushBlobRange(to.S3gitAdded, nil, hydrated, client)
		if err != nil {
			return err
		}

		// then push tree object
		_, err = pushBlob(co.S3gitTree, nil, client)
		if err != nil {
			return err
		}

		// then push commit object
		_, err = pushBlob(po.S3gitFollowMe, nil, client)
		if err != nil {
			return err
		}

		// ...  finally push prefix object itself
		// (if something goes in chain above, the prefix object will be missing so
		//  will be (attempted to) uploaded again during the next push)
		_, err = pushBlob(prefix, nil, client)
		if err != nil {
			return err
		}

		progress(int64(len(prefixesToPush)))
	}

	return nil
}

// Push a blob to the back end store
func pushBlob(hash string, size *uint64, client backend.Backend) (newlyUploaded bool, err error) {

	startOfLine := ""
	if size != nil {
		startOfLine = fmt.Sprintf("Uploading %s (%s)", util.FriendlyHash(hash), humanize.Bytes(*size))
	} else {
		startOfLine = fmt.Sprintf("Uploading %s", util.FriendlyHash(hash))
	}

	// TODO: Consider whether we want to verify again...
	if false {
		verified, err := client.VerifyHash(hash)
		if err != nil {
			fmt.Println(startOfLine, "verification failed", err)
			return false, err
		}

		if verified { // Resource already in back-end
			fmt.Println(startOfLine, "already in store")

			return false, nil
		}
	}

	// TODO: for back ends storing whole files: consider multipart upload?

	cr := cas.MakeReader(hash)
	if cr == nil {
		panic(errors.New("Failed to create cas reader"))
	}

	err = client.UploadWithReader(hash, cr)
	if err != nil {
		fmt.Println(startOfLine, "failed to upload to store", err)
		return false, err
	}

	// Move blob from .stage to .cache directory upon successful upload
	err = cas.MoveBlobToCache(hash)
	if err != nil {
		fmt.Println(startOfLine, "failed to move underlying blobs to cache", err)
		return false, err
	}

	//fmt.Println(startOfLine, "successfully uploaded to store")

	return true, nil
}

// Push a blob to the back end store in deduplicated format
func PushBlobDeduped(hash string, size *uint64, client backend.Backend) (newlyUploaded bool, err error) {

	// TODO: for back ends storing chunks: upload chunks in parallel

	hx, err := hex.DecodeString(hash)
	if err != nil {
		return false, err
	}

	// Get hashes for leaves
	leafHashes, _, err := kv.GetLevel1(hx)
	if err != nil {
		return false, err
	} else if len(leafHashes) == 0 {
		return false, errors.New(fmt.Sprintf("Unable to push an empty blob: %s", hash))
	}

	// Iterate over the leaves and push up to remote
	for i := 0; i < len(leafHashes); i += cas.KeySize {

		// TODO: verify whether leaf blob is already in back end, and skip if so
		err := cas.PushLeafBlob(hex.EncodeToString(leafHashes[i:i+cas.KeySize]), client)
		if err != nil {
			return false, err
		}
	}

	// Finally upload root hash
	b := bytes.NewBuffer(leafHashes)
	err = client.UploadWithReader(hash, b)
	if err != nil {
		return false, err
	}

	// TODO: Duplicate code with function above -- consider merging functions/common code
	// Move blob from .stage to .cache directory upon successful upload
	err = cas.MoveBlobToCache(hash)
	if err != nil {
		//fmt.Println(startOfLine, "failed to move underlying blobs to cache", err)
		return false, err
	}

	return true, nil
}

// Push a range of blobs to the back end store in parallel
//
// See https://github.com/adonovan/gopl.io/blob/master/ch8/thumbnail/thumbnail_test.go
//
func pushBlobRange(hashes []string, size *uint64, hydrated bool, client backend.Backend) error {

	var wg sync.WaitGroup
	var msgs = make(chan string)
	var results = make(chan error)

	for i := 0; i < min(len(hashes), 100); i++ {
		wg.Add(1)

		go func() {
			defer wg.Done()
			for hash := range msgs {

				pushHydratedToRemote := hydrated
				if !checkIfLeavesAreEqualSize(hash) {
					pushHydratedToRemote = false	// Cannot push hydrated to remote back end when eg rolling hash is used (as we do not know where the boundaries are)
				}
				var err error
				if pushHydratedToRemote {
					_, err = pushBlob(hash, size, client)
				} else {
					_, err = PushBlobDeduped(hash, size, client)
				}
				results <- err
			}
		}()
	}

	go func() {
		for _, hash := range hashes {
			msgs <- hash
		}
		close(msgs)
	}()

	go func() {
		wg.Wait()
		close(results)
	}()

	var err error
	for e := range results {
		if e != nil {
			err = e
		}
	}

	return err
}

func checkIfLeavesAreEqualSize(hash string) bool {
	// TODO: Implement: iterate over all leaves, check whether (except for last node) all sizes are equal
	return true
}

// List prefixes at back end store, doing 16 lists in parallel
func listPrefixes(client backend.Backend) (map[string]bool, error) {

	var wg sync.WaitGroup
	var results = make(chan []string)

	for i := 0x0; i <= 0xf; i++ {
		wg.Add(1)

		go func(i int) {
			defer wg.Done()
			result := make([]string, 0, 1000)

			client.List(fmt.Sprintf("%s%x", core.Prefix(), i), func(key string) {
				result = append(result, key)

				// TODO: WE NEED TO wg.Done() HERE WHEN LAST KEY HAS BEEN RECEIVED
				//       -- SEE /Users/frankw/golang/src/github.com/fwessels/listperf/listperf.go
				// IMPORTANT: WE WILL BE MISSING OBJECTS HERE
			})

			results <- result
		}(i)
	}

	go func() {
		wg.Wait()
		close(results)
	}()

	prefixHash := make(map[string]bool)
	for result := range results {
		for _, r := range result {
			prefixHash[r] = true
		}
	}

	return prefixHash, nil
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}
<commit_msg>Remove logging for upload of individual chunks/blobs<commit_after>/*
 * Copyright 2016 Frank Wessels <fwessels@xs4all.nl>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package s3git

import (
	"errors"
	"fmt"

	"github.com/s3git/s3git-go/internal/backend"
	"github.com/s3git/s3git-go/internal/cas"
	"github.com/s3git/s3git-go/internal/core"
	"github.com/s3git/s3git-go/internal/kv"

	"encoding/hex"
	"sync"
	"bytes"
)

// Perform a push to the back end for the repository
func (repo Repository) Push(hydrated bool, progress func(maxTicks int64)) error {

	list, err := kv.ListLevel1Prefixes()
	if err != nil {
		return err
	}

	return push(list, hydrated, progress)
}

// Push any new commit objects including all added objects to the back end store
func push(prefixChan <-chan []byte, hydrated bool, progress func(maxTicks int64)) error {

	client, err := backend.GetDefaultClient()
	if err != nil {
		return err
	}

	// Get map of prefixes already in store
	prefixesInBackend, err := listPrefixes(client)
	if err != nil {
		return err
	}

	prefixesToPush := []string{}

	for prefixByte := range prefixChan {

		prefix := hex.EncodeToString(prefixByte)

		_, verified := prefixesInBackend[prefix]

		// We can safely skip in case a prefix object is verified (pushed as last object)
		if !verified {

			prefixesToPush = append(prefixesToPush, prefix)
		}
	}

	if len(prefixesToPush) == 0 {
		return nil
	}

	progress(int64(len(prefixesToPush)))

	for _, prefix := range prefixesToPush {

		// Get prefix object
		po, err := core.GetPrefixObject(prefix)
		if err != nil {
			return err
		}

		// Get commit object
		co, err := core.GetCommitObject(po.S3gitFollowMe)
		if err != nil {
			return err
		}

		// Get tree object
		to, err := core.GetTreeObject(co.S3gitTree)
		if err != nil {
			return err
		}

		// first push all added blobs in this commit ...
		err = pushBlobRange(to.S3gitAdded, nil, hydrated, client)
		if err != nil {
			return err
		}

		// then push tree object
		_, err = pushBlob(co.S3gitTree, nil, client)
		if err != nil {
			return err
		}

		// then push commit object
		_, err = pushBlob(po.S3gitFollowMe, nil, client)
		if err != nil {
			return err
		}

		// ...  finally push prefix object itself
		// (if something goes in chain above, the prefix object will be missing so
		//  will be (attempted to) uploaded again during the next push)
		_, err = pushBlob(prefix, nil, client)
		if err != nil {
			return err
		}

		progress(int64(len(prefixesToPush)))
	}

	return nil
}

// Push a blob to the back end store
func pushBlob(hash string, size *uint64, client backend.Backend) (newlyUploaded bool, err error) {

	// TODO: Consider whether we want to verify again...
	if false {
		verified, err := client.VerifyHash(hash)
		if err != nil {
			return false, err
		} else if verified { // Resource already in back-end
			return false, nil
		}
	}

	// TODO: for back ends storing whole files: consider multipart upload?
	cr := cas.MakeReader(hash)
	if cr == nil {
		panic(errors.New("Failed to create cas reader"))
	}

	err = client.UploadWithReader(hash, cr)
	if err != nil {
		return false, err
	}

	// Move blob from .stage to .cache directory upon successful upload
	err = cas.MoveBlobToCache(hash)
	if err != nil {
		return false, err
	}

	return true, nil
}

// Push a blob to the back end store in deduplicated format
func PushBlobDeduped(hash string, size *uint64, client backend.Backend) (newlyUploaded bool, err error) {

	// TODO: for back ends storing chunks: upload chunks in parallel

	hx, err := hex.DecodeString(hash)
	if err != nil {
		return false, err
	}

	// Get hashes for leaves
	leafHashes, _, err := kv.GetLevel1(hx)
	if err != nil {
		return false, err
	} else if len(leafHashes) == 0 {
		return false, errors.New(fmt.Sprintf("Unable to push an empty blob: %s", hash))
	}

	// Iterate over the leaves and push up to remote
	for i := 0; i < len(leafHashes); i += cas.KeySize {

		// TODO: verify whether leaf blob is already in back end, and skip if so
		err := cas.PushLeafBlob(hex.EncodeToString(leafHashes[i:i+cas.KeySize]), client)
		if err != nil {
			return false, err
		}
	}

	// Finally upload root hash
	b := bytes.NewBuffer(leafHashes)
	err = client.UploadWithReader(hash, b)
	if err != nil {
		return false, err
	}

	// Move blob from .stage to .cache directory upon successful upload
	err = cas.MoveBlobToCache(hash)
	if err != nil {
		return false, err
	}

	return true, nil
}

// Push a range of blobs to the back end store in parallel
//
// See https://github.com/adonovan/gopl.io/blob/master/ch8/thumbnail/thumbnail_test.go
//
func pushBlobRange(hashes []string, size *uint64, hydrated bool, client backend.Backend) error {

	var wg sync.WaitGroup
	var msgs = make(chan string)
	var results = make(chan error)

	for i := 0; i < min(len(hashes), 100); i++ {
		wg.Add(1)

		go func() {
			defer wg.Done()
			for hash := range msgs {

				pushHydratedToRemote := hydrated
				if !checkIfLeavesAreEqualSize(hash) {
					pushHydratedToRemote = false	// Cannot push hydrated to remote back end when eg rolling hash is used (as we do not know where the boundaries are)
				}
				var err error
				if pushHydratedToRemote {
					_, err = pushBlob(hash, size, client)
				} else {
					_, err = PushBlobDeduped(hash, size, client)
				}
				results <- err
			}
		}()
	}

	go func() {
		for _, hash := range hashes {
			msgs <- hash
		}
		close(msgs)
	}()

	go func() {
		wg.Wait()
		close(results)
	}()

	var err error
	for e := range results {
		if e != nil {
			err = e
		}
	}

	return err
}

func checkIfLeavesAreEqualSize(hash string) bool {
	// TODO: Implement: iterate over all leaves, check whether (except for last node) all sizes are equal
	return true
}

// List prefixes at back end store, doing 16 lists in parallel
func listPrefixes(client backend.Backend) (map[string]bool, error) {

	var wg sync.WaitGroup
	var results = make(chan []string)

	for i := 0x0; i <= 0xf; i++ {
		wg.Add(1)

		go func(i int) {
			defer wg.Done()
			result := make([]string, 0, 1000)

			client.List(fmt.Sprintf("%s%x", core.Prefix(), i), func(key string) {
				result = append(result, key)

				// TODO: WE NEED TO wg.Done() HERE WHEN LAST KEY HAS BEEN RECEIVED
				//       -- SEE /Users/frankw/golang/src/github.com/fwessels/listperf/listperf.go
				// IMPORTANT: WE WILL BE MISSING OBJECTS HERE
			})

			results <- result
		}(i)
	}

	go func() {
		wg.Wait()
		close(results)
	}()

	prefixHash := make(map[string]bool)
	for result := range results {
		for _, r := range result {
			prefixHash[r] = true
		}
	}

	return prefixHash, nil
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"image"
	"image/jpeg"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strings"
	"unicode"

	"golang.org/x/text/transform"
	"golang.org/x/text/unicode/norm"

	_ "image/gif"
	_ "image/png"
)

var errHomeNotFound = errors.New("wimg: user home directory not found")

func getUserHome() string {
	home := os.Getenv("HOME")
	if home == "" {
		home = os.Getenv("USERPROFILE")
	}
	return home
}

func remove(r rune) bool {
	return unicode.Is(unicode.Mn, r)
}

func normalize(name string) (string, error) {
	t := transform.Chain(norm.NFD, transform.RemoveFunc(remove), norm.NFC)
	name = strings.TrimSpace(name)
	name, _, err := transform.String(t, name)
	if err != nil {
		return "", err
	}
	name = strings.ToLower(name)
	name = strings.Replace(name, " ", "_", -1)
	return name, nil
}

func run(src, name string, quality int) error {
	home := getUserHome()
	if home == "" {
		return errHomeNotFound
	}
	resp, err := http.Get(src)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	m, _, err := image.Decode(resp.Body)
	if err != nil {
		return err
	}
	bounds := m.Bounds()
	if name == "" {
		ext := path.Ext(src)
		name = path.Base(src)
		name = strings.TrimSuffix(name, ext)
	}
	name, err = normalize(name)
	if err != nil {
		return err
	}
	name = fmt.Sprintf("%dx%d_%s.jpg", bounds.Dx(), bounds.Dy(), name)
	filename := filepath.Join(home, "img", name)
	f, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer f.Close()
	return jpeg.Encode(f, m, nil)
}

func main() {
	var (
		name    = flag.String("name", "", "name of the image")
		quality = flag.Int("quality", 100, "jpeg quality encoding")
	)
	flag.Parse()
	args := flag.Args()
	if len(args) != 1 {
		fmt.Fprintln(os.Stderr, "wimg: invalid arguments")
		flag.PrintDefaults()
		os.Exit(1)
	}
	err := run(args[0], *name, *quality)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}
<commit_msg>Refactor default name.<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"image"
	"image/jpeg"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strings"
	"unicode"

	"golang.org/x/text/transform"
	"golang.org/x/text/unicode/norm"

	_ "image/gif"
	_ "image/png"
)

var errHomeNotFound = errors.New("wimg: user home directory not found")

func baseWithoutExt(src string) string {
	ext := path.Ext(src)
	name := path.Base(src)
	return strings.TrimSuffix(name, ext)
}

func getUserHome() string {
	home := os.Getenv("HOME")
	if home == "" {
		home = os.Getenv("USERPROFILE")
	}
	return home
}

func remove(r rune) bool {
	return unicode.Is(unicode.Mn, r)
}

func normalize(name string) (string, error) {
	t := transform.Chain(norm.NFD, transform.RemoveFunc(remove), norm.NFC)
	name = strings.TrimSpace(name)
	name, _, err := transform.String(t, name)
	if err != nil {
		return "", err
	}
	name = strings.ToLower(name)
	name = strings.Replace(name, " ", "_", -1)
	return name, nil
}

func run(src, name string, quality int) error {
	home := getUserHome()
	if home == "" {
		return errHomeNotFound
	}
	resp, err := http.Get(src)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	m, _, err := image.Decode(resp.Body)
	if err != nil {
		return err
	}
	bounds := m.Bounds()
	if name == "" {
		name = baseWithoutExt(src)
	}
	name, err = normalize(name)
	if err != nil {
		return err
	}
	name = fmt.Sprintf("%dx%d_%s.jpg", bounds.Dx(), bounds.Dy(), name)
	filename := filepath.Join(home, "img", name)
	f, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer f.Close()
	return jpeg.Encode(f, m, nil)
}

func main() {
	var (
		name    = flag.String("name", "", "name of the image")
		quality = flag.Int("quality", 100, "jpeg quality encoding")
	)
	flag.Parse()
	args := flag.Args()
	if len(args) != 1 {
		fmt.Fprintln(os.Stderr, "wimg: invalid arguments")
		flag.PrintDefaults()
		os.Exit(1)
	}
	err := run(args[0], *name, *quality)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"encoding/json"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"time"

	"cloud.google.com/go/pubsub"
	"github.com/hpcloud/tail"
	log "github.com/sirupsen/logrus"
	"google.golang.org/api/option"
	kingpin "gopkg.in/alecthomas/kingpin.v2"
)

var (
	configFile = kingpin.Flag("config", "Configuration file").Required().Short('c').String()
	version    = "master" //overridden by build system, master as default
)

/*
Todo:
- See if I can get hpcloud/tail to use logrus?
- lots of error handling
- config param defaults
- add signal handler support
*/

//FlingConfig - top level structure of json config file
type FlingConfig struct {
	Files     []FlingFile     `json:"files"`
	Rotations []FlingRotation `json:"rotations"`
	Outputs   []FlingOutput   `json:"outputs"`
}

// FlingRotation - sets of files to rotate and the commands to run afterwards
type FlingRotation struct {
	Files          []string `json:"files"`
	RotateCommand  string   `json:"rotate_command"`
	RotateInterval int      `json:"rotate_interval"`
}

// FlingOutput - A log output destination
type FlingOutput struct {
	Name           string `json:"name"`
	Type           string `json:"type"`
	PubSubProject  string `json:"pubsub_project"`
	PubSubTopic    string `json:"pubsub_topic"`
	PubSubAuthFile string `json:"pubsub_auth_file"`
}

//FlingFile - instance of a file to monitor
type FlingFile struct {
	Path       string               `json:"path"`
	IsJSON     bool                 `json:"is_json"`
	IsGlob     bool                 `json:"is_glob"`
	Outputs    []string             `json:"outputs"`
	Injections []FlingFileInjection `json:"injections"`
}

//FlingFileInjection - fields to add to the log line
type FlingFileInjection struct {
	Field    string `json:"field"`
	Value    string `json:"value"`
	ENVValue string `json:"env_value"`
	Hostname bool   `json:"hostname"`
}

func init() {
	log.SetFormatter(&log.JSONFormatter{})
	log.SetOutput(os.Stdout)
	log.SetLevel(log.InfoLevel)
}

func main() {
	log.Info("Initalizing")
	//Parse command line params
	kingpin.Version(version)
	kingpin.Parse()

	var config, err = loadConfig(*configFile)
	if err != nil {
		log.WithFields(log.Fields{
			"error": err,
		}).Fatal("Couldn't load config")
		os.Exit(-1)
	}

	//var outputs map[string]interface{}
	//start up go routines for any outputs
	outputs := handleOutputs(config.Outputs)

	//start up go routines for any rotations requested
	handleRotations(config.Rotations)

	//setup handlers for all of the files to be watched
	handleFiles(config.Files, outputs)

	select {} //Take a big nap FIXME: add signal handlers down the road
}

func loadConfig(path string) (FlingConfig, error) {
	var config FlingConfig
	configString, readError := ioutil.ReadFile(path)

	if readError != nil {
		return config, readError
	}

	parseError := json.Unmarshal(configString, &config)

	return config, parseError
}

func handleOutputs(outputs []FlingOutput) map[string]interface{} {
	var channels map[string]interface{}
	channels = make(map[string]interface{})

	for _, output := range outputs {
		channels[output.Name] = make(chan []byte, 1000)
		if output.Type == "pubsub" {
			go outputPubSubWorker(output.PubSubProject, output.PubSubTopic, output.PubSubAuthFile, channels[output.Name].(chan []byte))
		}
	}

	return channels
}

func outputPubSubWorker(project string, topicName string, authfile string, channel chan []byte) {
	ctx := context.Background()
	pubSubClient, err := pubsub.NewClient(ctx, project, option.WithServiceAccountFile(authfile))
	if err != nil {

	}

	topic := pubSubClient.Topic(topicName)
	defer topic.Stop()

	//send hello message to topic to keep track of what clients, versions etc.. are sending in data
	createPubSubInitMsg(topicName, channel)

	for {
		message := <-channel
		result := topic.Publish(ctx, &pubsub.Message{
			Data: message,
		})

		id, err := result.Get(ctx)
		if err != nil {
			log.WithFields(log.Fields{
				"error": err,
			}).Error("Failed to publish message")
		} else {
			log.WithFields(log.Fields{
				"id": id,
			}).Debug("Published Message")
		}

	}
}

func createPubSubInitMsg(topicName string, channel chan []byte) {
	var logEntry map[string]interface{}
	logEntry = make(map[string]interface{})
	hostname, _ := os.Hostname()

	logEntry["pubsub_topic"] = topicName
	logEntry["fling_version"] = version
	logEntry["hostname"] = hostname
	logEntry["@timestamp"] = get3339Time()
	logEntry["message"] = "Starting up Fling PubSub Output"

	eventJSON, marshalErr := json.Marshal(logEntry)
	if marshalErr != nil {
		log.WithFields(log.Fields{}).Error("PubSub Init message creation failed")
		return
	}

	channel <- eventJSON
	log.WithFields(log.Fields{"topic": topicName}).Info("PubSub Init message queued")
}

func handleFiles(files []FlingFile, outputs map[string]interface{}) {
	for _, file := range files {
		if file.IsGlob {
			paths, _ := filepath.Glob(file.Path)
			for _, path := range paths {
				file.Path = path
				startFileWorker(file, outputs)
			}
		} else {
			startFileWorker(file, outputs)
		}
	}
}

func startFileWorker(file FlingFile, outputs map[string]interface{}) {
	log.WithFields(log.Fields{
		"path": file.Path,
	}).Info("Adding tail for file")
	go fileWorker(file, outputs)
}

func fileWorker(file FlingFile, outputs map[string]interface{}) {
	t, tailErr := tail.TailFile(file.Path, tail.Config{Follow: true, ReOpen: true, Poll: true})

	if tailErr != nil {
		log.WithFields(log.Fields{
			"path":  file.Path,
			"error": tailErr,
		}).Error("Couldn't tail file")
	}

	for line := range t.Lines {
		var logEntry map[string]interface{}

		log.WithFields(log.Fields{
			"path": file.Path,
			"line": line.Text,
		}).Debug("Processing log line")

		if file.IsJSON {
			unmarshalErr := json.Unmarshal([]byte(line.Text), &logEntry)
			if unmarshalErr != nil {
				log.WithFields(log.Fields{
					"message": line.Text,
					"error":   unmarshalErr,
				}).Error("Couldn't parse JSON log line")

				continue
			}
		} else {
			logEntry = make(map[string]interface{})
			logEntry["message"] = line.Text
		}

		//FIXME: Inject other pertinent context info
		logEntry["fling.source"] = file.Path

		if _, ok := logEntry["@timestamp"]; !ok {
			logEntry["@timestamp"] = get3339Time()
		}

		handleInjections(&logEntry, file.Injections)

		eventJSON, marshalErr := json.Marshal(logEntry)
		if marshalErr != nil {
			log.WithFields(log.Fields{
				"error": marshalErr,
			}).Error("Couldn't create JSON")

			continue
		}

		dispatchEntry(eventJSON, file.Outputs, outputs)
	}
}

func handleInjections(logEntry *map[string]interface{}, injections []FlingFileInjection) {
	for _, injection := range injections {
		if injection.ENVValue != "" {
			(*logEntry)[injection.Field] = os.Getenv(injection.ENVValue)
		} else if injection.Value != "" {
			(*logEntry)[injection.Field] = injection.Value
		} else if injection.Hostname {
			hostname, _ := os.Hostname()
			(*logEntry)[injection.Field] = hostname
		}
	}
}

func dispatchEntry(message []byte, outputs []string, channels map[string]interface{}) {
	for _, output := range outputs {
		channels[output].(chan []byte) <- message
	}
}

func handleRotations(rotations []FlingRotation) {
	for _, rotation := range rotations {
		go rotateWorker(rotation)
	}
}

func rotateWorker(rotation FlingRotation) {
	for {
		log.WithFields(log.Fields{
			"seconds": rotation.RotateInterval,
		}).Info("Sleeping before rotate")

		time.Sleep(time.Duration(rotation.RotateInterval) * time.Second)

		rotate(rotation)
	}
}

func rotate(rotation FlingRotation) {
	//Handle the files first
	for _, path := range rotation.Files {
		//FIXME add file size check

		renameErr := os.Rename(path, path+".old")
		if renameErr != nil {
			log.WithFields(log.Fields{
				"path": path,
			}).Error("Unable to move log file in rotation")

			continue
		} else {
			log.WithFields(log.Fields{
				"path": path,
			}).Info("Moved log file")
		}
	}

	// Perform rotation command
	if rotation.RotateCommand != "" {
		_, cmdError := exec.Command("sh", "-c", rotation.RotateCommand).Output()
		if cmdError != nil {
			log.WithFields(log.Fields{
				"error":   cmdError,
				"command": rotation.RotateCommand,
			}).Error("Rotate command failed")
		} else {
			log.WithFields(log.Fields{
				"command": rotation.RotateCommand,
			}).Info("Rotation command successful")
		}
	}
}

//Get an RFC 3339 Nano Time for use in log timestamps
func get3339Time() string {
	return time.Now().UTC().Format(time.RFC3339Nano)
}
<commit_msg>Revamp of config syntax to allow for more input / output plugins<commit_after>package main

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"time"

	"cloud.google.com/go/pubsub"
	"github.com/hpcloud/tail"
	log "github.com/sirupsen/logrus"
	"google.golang.org/api/option"
	kingpin "gopkg.in/alecthomas/kingpin.v2"
)

var (
	configFile = kingpin.Flag("config", "Configuration file").Required().Short('c').String()
	version    = "master" //overridden by build system, master as default
)

/*
Todo:
- See if I can get hpcloud/tail to use logrus?
- lots of error handling
- config param defaults
- add signal handler support
*/

//FlingConfig - top level structure of json config file
type FlingConfig struct {
	Input     FlingInput      `json:"input"`
	Files     []FlingInFile   `json:"files"`
	Rotations []FlingRotation `json:"rotations"`
	Output    FlingOutput     `json:"output"`
}

//FlingInput - map of input type arrays
type FlingInput struct {
	PubSubs []FlingInPubSub `json:"pubsub"`
	Files   []FlingInFile   `json:"files"`
}

//FlingInPubSub - pub/sub input type
type FlingInPubSub struct {
	AuthFile     string           `json:"auth_file"`
	Project      string           `json:"project"`
	Subscription string           `json:"subscription"`
	Outputs      []string         `json:"outputs"`
	Injections   []FlingInjection `json:"injections"`
}

// FlingRotation - sets of files to rotate and the commands to run afterwards
type FlingRotation struct {
	Files          []string `json:"files"`
	RotateCommand  string   `json:"rotate_command"`
	RotateInterval int      `json:"rotate_interval"`
}

//FlingOutput - map of output types
type FlingOutput struct {
	PubSubs []FlingOutPubSub `json:"pubsub"`
	Loggers []FlingOutLogger `json:"logger"`
}

//FlingOutPubSub - A log output destination
type FlingOutPubSub struct {
	Name     string `json:"name"`
	Project  string `json:"project"`
	Topic    string `json:"topic"`
	AuthFile string `json:"auth_file"`
}

//FlingOutLogger - Send messages to the logger library
//FIXME add level and injections
type FlingOutLogger struct {
	Name      string `json:"name"`
	IsEnabled bool   `json:"is_enabled"`
}

//FlingInFile - instance of a file to monitor
type FlingInFile struct {
	Path       string           `json:"path"`
	IsJSON     bool             `json:"is_json"`
	IsGlob     bool             `json:"is_glob"`
	Outputs    []string         `json:"outputs"`
	Injections []FlingInjection `json:"injections"`
}

//FlingInjection - fields to add to the log line
type FlingInjection struct {
	Field    string `json:"field"`
	Value    string `json:"value"`
	ENVValue string `json:"env_value"`
	Hostname bool   `json:"hostname"`
}

func init() {
	log.SetFormatter(&log.JSONFormatter{})
	log.SetOutput(os.Stdout)
	log.SetLevel(log.DebugLevel)
}

func main() {
	log.Info("Initalizing")
	//Parse command line params
	kingpin.Version(version)
	kingpin.Parse()

	var config, err = loadConfig(*configFile)
	if err != nil {
		log.WithFields(log.Fields{
			"error": err,
		}).Fatal("Couldn't load config")
		os.Exit(-1)
	}

	//var outputs map[string]interface{}
	//start up go routines for any outputs
	outputChannels := handleOutputs(config.Output)

	//start up go routines for any rotations requested
	handleRotations(config.Rotations)

	//setup handlers for all of the files to be watched
	handleInputs(config.Input, outputChannels)

	select {} //Take a big nap FIXME: add signal handlers down the road
}

func loadConfig(path string) (FlingConfig, error) {
	var config FlingConfig
	configString, readError := ioutil.ReadFile(path)

	if readError != nil {
		return config, readError
	}

	parseError := json.Unmarshal(configString, &config)

	return config, parseError
}

func handleInputs(input FlingInput, outputChannels map[string]interface{}) {
	handleInFiles(input.Files, outputChannels)
}

func handleOutputs(outputs FlingOutput) map[string]interface{} {
	var channels map[string]interface{}
	channels = make(map[string]interface{})

	for k, v := range handleOutPubSubs(outputs.PubSubs) {
		channels[k] = v
	}

	for k, v := range handleOutLoggers(outputs.Loggers) {
		channels[k] = v
	}

	return channels
}

func handleOutLoggers(outputs []FlingOutLogger) map[string]interface{} {
	var channels map[string]interface{}
	channels = make(map[string]interface{})

	for _, output := range outputs {
		channels[output.Name] = make(chan []byte, 1000)
		go outputLoggerWorker(output.Name, output.IsEnabled, channels[output.Name].(chan []byte))
	}

	return channels
}

func outputLoggerWorker(name string, isEnabled bool, channel chan []byte) {
	for {
		message := <-channel

		if isEnabled {
			log.WithFields(log.Fields{
				"OutputName": name,
			}).Debug(fmt.Sprintf("%s", message))
		}
	}
}

func handleOutPubSubs(outputs []FlingOutPubSub) map[string]interface{} {
	var channels map[string]interface{}
	channels = make(map[string]interface{})

	for _, output := range outputs {
		channels[output.Name] = make(chan []byte, 1000)
		go outputPubSubWorker(output.Project, output.Topic, output.AuthFile, channels[output.Name].(chan []byte))
	}

	return channels
}

func outputPubSubWorker(project string, topicName string, authfile string, channel chan []byte) {
	ctx := context.Background()
	pubSubClient, err := pubsub.NewClient(ctx, project, option.WithServiceAccountFile(authfile))
	if err != nil {

	}

	topic := pubSubClient.Topic(topicName)
	defer topic.Stop()

	//send hello message to topic to keep track of what clients, versions etc.. are sending in data
	createPubSubInitMsg(topicName, channel)

	for {
		message := <-channel
		result := topic.Publish(ctx, &pubsub.Message{
			Data: message,
		})

		id, err := result.Get(ctx)
		if err != nil {
			log.WithFields(log.Fields{
				"error": err,
			}).Error("Failed to publish message")
		} else {
			log.WithFields(log.Fields{
				"id": id,
			}).Debug("Published Message")
		}

	}
}

func createPubSubInitMsg(topicName string, channel chan []byte) {
	var logEntry map[string]interface{}
	logEntry = make(map[string]interface{})
	hostname, _ := os.Hostname()

	logEntry["pubsub_topic"] = topicName
	logEntry["fling_version"] = version
	logEntry["hostname"] = hostname
	logEntry["@timestamp"] = get3339Time()
	logEntry["message"] = "Starting up Fling PubSub Output"

	eventJSON, marshalErr := json.Marshal(logEntry)
	if marshalErr != nil {
		log.WithFields(log.Fields{}).Error("PubSub Init message creation failed")
		return
	}

	channel <- eventJSON
	log.WithFields(log.Fields{"topic": topicName}).Info("PubSub Init message queued")
}

func handleInFiles(files []FlingInFile, outputs map[string]interface{}) {
	for _, file := range files {
		if file.IsGlob {
			paths, _ := filepath.Glob(file.Path)
			for _, path := range paths {
				file.Path = path
				startFileWorker(file, outputs)
			}
		} else {
			startFileWorker(file, outputs)
		}
	}
}

func startFileWorker(file FlingInFile, outputs map[string]interface{}) {
	log.WithFields(log.Fields{
		"path": file.Path,
	}).Info("Adding tail for file")
	go fileWorker(file, outputs)
}

func fileWorker(file FlingInFile, outputs map[string]interface{}) {
	t, tailErr := tail.TailFile(file.Path, tail.Config{Follow: true, ReOpen: true, Poll: true})

	if tailErr != nil {
		log.WithFields(log.Fields{
			"path":  file.Path,
			"error": tailErr,
		}).Error("Couldn't tail file")
	}

	for line := range t.Lines {
		var logEntry map[string]interface{}

		log.WithFields(log.Fields{
			"path": file.Path,
			"line": line.Text,
		}).Debug("Processing log line")

		if file.IsJSON {
			unmarshalErr := json.Unmarshal([]byte(line.Text), &logEntry)
			if unmarshalErr != nil {
				log.WithFields(log.Fields{
					"message": line.Text,
					"error":   unmarshalErr,
				}).Error("Couldn't parse JSON log line")

				continue
			}
		} else {
			logEntry = make(map[string]interface{})
			logEntry["message"] = line.Text
		}

		//FIXME: Inject other pertinent context info
		logEntry["fling.source"] = file.Path

		if _, ok := logEntry["@timestamp"]; !ok {
			logEntry["@timestamp"] = get3339Time()
		}

		handleInjections(&logEntry, file.Injections)

		eventJSON, marshalErr := json.Marshal(logEntry)
		if marshalErr != nil {
			log.WithFields(log.Fields{
				"error": marshalErr,
			}).Error("Couldn't create JSON")

			continue
		}

		dispatchEntry(eventJSON, file.Outputs, outputs)
	}
}

func handleInjections(logEntry *map[string]interface{}, injections []FlingInjection) {
	for _, injection := range injections {
		if injection.ENVValue != "" {
			(*logEntry)[injection.Field] = os.Getenv(injection.ENVValue)
		} else if injection.Value != "" {
			(*logEntry)[injection.Field] = injection.Value
		} else if injection.Hostname {
			hostname, _ := os.Hostname()
			(*logEntry)[injection.Field] = hostname
		}
	}
}

func dispatchEntry(message []byte, outputs []string, channels map[string]interface{}) {
	for _, output := range outputs {
		channels[output].(chan []byte) <- message
	}
}

func handleRotations(rotations []FlingRotation) {
	for _, rotation := range rotations {
		go rotateWorker(rotation)
	}
}

func rotateWorker(rotation FlingRotation) {
	for {
		log.WithFields(log.Fields{
			"seconds": rotation.RotateInterval,
		}).Info("Sleeping before rotate")

		time.Sleep(time.Duration(rotation.RotateInterval) * time.Second)

		rotate(rotation)
	}
}

func rotate(rotation FlingRotation) {
	//Handle the files first
	for _, path := range rotation.Files {
		//FIXME add file size check

		renameErr := os.Rename(path, path+".old")
		if renameErr != nil {
			log.WithFields(log.Fields{
				"path": path,
			}).Error("Unable to move log file in rotation")

			continue
		} else {
			log.WithFields(log.Fields{
				"path": path,
			}).Info("Moved log file")
		}
	}

	// Perform rotation command
	if rotation.RotateCommand != "" {
		_, cmdError := exec.Command("sh", "-c", rotation.RotateCommand).Output()
		if cmdError != nil {
			log.WithFields(log.Fields{
				"error":   cmdError,
				"command": rotation.RotateCommand,
			}).Error("Rotate command failed")
		} else {
			log.WithFields(log.Fields{
				"command": rotation.RotateCommand,
			}).Info("Rotation command successful")
		}
	}
}

//Get an RFC 3339 Nano Time for use in log timestamps
func get3339Time() string {
	return time.Now().UTC().Format(time.RFC3339Nano)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"database/sql"
	"flag"
	"log"
	"net/http"
	"os"
	"strings"
)

func Prompt(prompt string) (ret string) {
	input := bufio.NewReader(os.Stdin)
	os.Stdout.Write([]byte(prompt))
	for {
		line, isPrefix, err := input.ReadLine()
		if err != nil {
			return
		}
		ret += string(line)
		if !isPrefix {
			break
		}
	}
	ret = strings.TrimSpace(ret)
	return
}

func MakeAccount() {
	name := Prompt("Name: ")
	pass := Prompt("Password: ")
	displayName := Prompt("Display name: ")

	account := NewAccount()
	account.Name = name
	account.DisplayName = displayName
	account.SetPassword(pass)
	err := account.Save()
	if err != nil {
		log.Println("Error saving new account:", err.Error())
	}
}

func ServeWeb() {
	http.HandleFunc("/static/", static)
	http.HandleFunc("/rss", rss)
	http.HandleFunc("/rssCloud", rssCloud)
	http.HandleFunc("/post", post)
	//http.HandleFunc("/archive/", archive)
	http.HandleFunc("/post/", permalink)
	http.HandleFunc("/", index)

	log.Println("Ohai web servin'")
	log.Fatal(http.ListenAndServe(":8080", nil))
}

func main() {
	var makeaccount bool
	flag.BoolVar(&makeaccount, "make-account", false, "create a new account interactively")
	flag.Parse()

	var err error
	db, err = sql.Open("postgres", "host=localhost dbname=cares sslmode=disable")
	if err == nil {
		// Try a query to make sure it worked.
		_, err = db.Query("SELECT 1")
	}
	if err != nil {
		log.Println("Error connecting to db:", err.Error())
		return
	}
	err = LoadAccountForOwner()
	if err != nil {
		log.Println("Error loading site owner:", err.Error())
		return
	}

	if makeaccount {
		MakeAccount()
	} else {
		ServeWeb()
	}
}
<commit_msg>Add --init-db command for setting up database<commit_after>package main

import (
	"bufio"
	"database/sql"
	"flag"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"
)

func Prompt(prompt string) (ret string) {
	input := bufio.NewReader(os.Stdin)
	os.Stdout.Write([]byte(prompt))
	for {
		line, isPrefix, err := input.ReadLine()
		if err != nil {
			return
		}
		ret += string(line)
		if !isPrefix {
			break
		}
	}
	ret = strings.TrimSpace(ret)
	return
}

func InitializeDatabase() {
	schemaBytes, err := ioutil.ReadFile("cares.sql")
	if err != nil {
		log.Println("Error reading database schema:", err.Error())
		return
	}

	statements := strings.Split(string(schemaBytes), ";\n")
	for _, statement := range statements {
		statement = strings.TrimSpace(statement)
		if statement == "" {
			continue
		}

		_, err = db.Exec(statement)
		if err != nil {
			log.Println("Error initializing database:", err.Error())
			return
		}
	}

	// Then make the owner record too.
	MakeAccount()
}

func MakeAccount() {
	name := Prompt("Name: ")
	pass := Prompt("Password: ")
	displayName := Prompt("Display name: ")

	account := NewAccount()
	account.Name = name
	account.DisplayName = displayName
	account.SetPassword(pass)
	err := account.Save()
	if err != nil {
		log.Println("Error saving new account:", err.Error())
	}
}

func ServeWeb() {
	err := LoadAccountForOwner()
	if err != nil {
		log.Println("Error loading site owner:", err.Error())
		return
	}

	http.HandleFunc("/static/", static)
	http.HandleFunc("/rss", rss)
	http.HandleFunc("/rssCloud", rssCloud)
	http.HandleFunc("/post", post)
	//http.HandleFunc("/archive/", archive)
	http.HandleFunc("/post/", permalink)
	http.HandleFunc("/", index)

	log.Println("Ohai web servin'")
	log.Fatal(http.ListenAndServe(":8080", nil))
}

func main() {
	var dsn string
	var makeaccount bool
	var initdb bool
	flag.StringVar(&dsn, "database", "dbname=cares sslmode=disable", "database connection info")
	flag.BoolVar(&initdb, "init-db", false, "initialize the database")
	flag.BoolVar(&makeaccount, "make-account", false, "create a new account interactively")
	flag.Parse()

	var err error
	db, err = sql.Open("postgres", dsn)
	if err == nil {
		// Try a query to make sure it worked.
		_, err = db.Query("SELECT 1")
	}
	if err != nil {
		log.Println("Error connecting to db:", err.Error())
		return
	}

	if initdb {
		InitializeDatabase()
	} else if makeaccount {
		MakeAccount()
	} else {
		ServeWeb()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net"
	"os"

	"github.com/negbie/horaclifix/receiver"
)

func checkError(err error) {
	if err != nil {
		fmt.Fprintf(os.Stderr, "Fatal error: %s", err.Error())
		os.Exit(1)
	}
}

func main() {

	listener, err := net.Listen("tcp", ":4739")
	checkError(err)
	for {
		conn, err := listener.Accept()
		checkError(err)
		go receiver.SyncClient(conn)
	}
}
<commit_msg>Optional port flag<commit_after>package main

import (
	"flag"
	"fmt"
	"net"
	"os"

	"github.com/negbie/horaclifix/receiver"
)

func checkError(err error) {
	if err != nil {
		fmt.Fprintf(os.Stderr, "Fatal error: %s", err.Error())
		os.Exit(1)
	}
}

func main() {

	var (
		port = flag.String("port", ":4739", "Port to use to receive IPFIX packets")
	)

	flag.Parse()
	listener, err := net.Listen("tcp", *port)
	checkError(err)
	for {
		conn, err := listener.Accept()
		checkError(err)
		go receiver.SyncClient(conn)
	}
}
<|endoftext|>


<commit_before>// Package c2go contains the main function for running the executable.
//
// Installation
//
//     go get -u github.com/elliotchance/c2go
//
// Usage
//
//     c2go myfile.c
//
package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/elliotchance/c2go/ast"
	"github.com/elliotchance/c2go/program"
	"github.com/elliotchance/c2go/transpiler"
	"reflect"
)

// Version can be requested through the command line with:
//
//     c2go -v
//
// See https://github.com/elliotchance/c2go/wiki/Release-Process
const Version = "v0.14.1 Oganesson 2017-08-09"

var stderr io.Writer = os.Stderr

// ProgramArgs defines the options available when processing the program. There
// is no constructor since the zeroed out values are the appropriate defaults -
// you need only set the options you need.
//
// TODO: Better separation on CLI modes
// https://github.com/elliotchance/c2go/issues/134
type ProgramArgs struct {
	verbose     bool
	ast         bool
	inputFile   string
	outputFile  string
	packageName string
}

func readAST(data []byte) []string {
	uncolored := regexp.MustCompile(`\x1b\[[\d;]+m`).ReplaceAll(data, []byte{})
	return strings.Split(string(uncolored), "\n")
}

type treeNode struct {
	indent int
	node   ast.Node
}

func convertLinesToNodes(lines []string) []treeNode {
	nodes := []treeNode{}
	for _, line := range lines {
		if strings.TrimSpace(line) == "" {
			continue
		}

		// It is tempting to discard null AST nodes, but these may
		// have semantic importance: for example, they represent omitted
		// for-loop conditions, as in for(;;).
		line = strings.Replace(line, "<<<NULL>>>", "NullStmt", 1)
		trimmed := strings.TrimLeft(line, "|\\- `")
		node := ast.Parse(trimmed)
		indentLevel := (len(line) - len(trimmed)) / 2
		nodes = append(nodes, treeNode{indentLevel, node})
	}

	return nodes
}

// buildTree convert an array of nodes, each prefixed with a depth into a tree.
func buildTree(nodes []treeNode, depth int) []ast.Node {
	if len(nodes) == 0 {
		return []ast.Node{}
	}

	// Split the list into sections, treat each section as a a tree with its own root.
	sections := [][]treeNode{}
	for _, node := range nodes {
		if node.indent == depth {
			sections = append(sections, []treeNode{node})
		} else {
			sections[len(sections)-1] = append(sections[len(sections)-1], node)
		}
	}

	results := []ast.Node{}
	for _, section := range sections {
		slice := []treeNode{}
		for _, n := range section {
			if n.indent > depth {
				slice = append(slice, n)
			}
		}

		children := buildTree(slice, depth+1)
		for _, child := range children {
			section[0].node.AddChild(child)
		}
		results = append(results, section[0].node)
	}

	return results
}

func toJSON(tree []interface{}) []map[string]interface{} {
	r := make([]map[string]interface{}, len(tree))

	for j, n := range tree {
		rn := reflect.ValueOf(n).Elem()
		r[j] = make(map[string]interface{})
		r[j]["node"] = rn.Type().Name()

		for i := 0; i < rn.NumField(); i++ {
			name := strings.ToLower(rn.Type().Field(i).Name)
			value := rn.Field(i).Interface()

			if name == "children" {
				v := value.([]interface{})

				if len(v) == 0 {
					continue
				}

				value = toJSON(v)
			}

			r[j][name] = value
		}
	}

	return r
}

func check(prefix string, e error) {
	if e != nil {
		panic(prefix + e.Error())
	}
}

// Start begins transpiling an input file.
func Start(args ProgramArgs) error {
	if os.Getenv("GOPATH") == "" {
		return fmt.Errorf("The $GOPATH must be set")
	}

	// 1. Compile it first (checking for errors)
	_, err := os.Stat(args.inputFile)
	if err != nil {
		return fmt.Errorf("Input file is not found")
	}

	// 2. Preprocess
	var pp []byte
	{
		// See : https://clang.llvm.org/docs/CommandGuide/clang.html
		// clang -E <file>    Run the preprocessor stage.
		cmd := exec.Command("clang", "-E", args.inputFile)
		var out bytes.Buffer
		var stderr bytes.Buffer
		cmd.Stdout = &out
		cmd.Stderr = &stderr
		err = cmd.Run()
		if err != nil {
			return fmt.Errorf("preprocess failed: %v\nStdErr = %v", err, stderr.String())
		}
		pp = []byte(out.String())
	}

	ppFilePath := path.Join(os.TempDir(), "pp.c")
	err = ioutil.WriteFile(ppFilePath, pp, 0644)
	if err != nil {
		return fmt.Errorf("writing to /tmp/pp.c failed: %v", err)
	}

	// 3. Generate JSON from AST
	astPP, err := exec.Command("clang", "-Xclang", "-ast-dump", "-fsyntax-only", ppFilePath).Output()
	if err != nil {
		// If clang fails it still prints out the AST, so we have to run it
		// again to get the real error.
		errBody, _ := exec.Command("clang", ppFilePath).CombinedOutput()

		panic("clang failed: " + err.Error() + ":\n\n" + string(errBody))
	}

	lines := readAST(astPP)
	if args.ast {
		for _, l := range lines {
			fmt.Println(l)
		}
		fmt.Println()

		return nil
	}

	nodes := convertLinesToNodes(lines)
	tree := buildTree(nodes, 0)

	p := program.NewProgram()
	p.Verbose = args.verbose

	err = transpiler.TranspileAST(args.inputFile, args.packageName, p, tree[0].(ast.Node))
	if err != nil {
		panic(err)
	}

	outputFilePath := args.outputFile

	if outputFilePath == "" {
		cleanFileName := filepath.Clean(filepath.Base(args.inputFile))
		extension := filepath.Ext(args.inputFile)

		outputFilePath = cleanFileName[0:len(cleanFileName)-len(extension)] + ".go"
	}

	err = ioutil.WriteFile(outputFilePath, []byte(p.String()), 0755)
	if err != nil {
		return fmt.Errorf("writing C output file failed: %v", err)
	}

	return nil
}

// newTempFile - returns temp file
func newTempFile(dir, prefix, suffix string) (*os.File, error) {
	for index := 1; index < 10000; index++ {
		path := filepath.Join(dir, fmt.Sprintf("%s%03d%s", prefix, index, suffix))
		if _, err := os.Stat(path); err != nil {
			return os.Create(path)
		}
	}
	return nil, fmt.Errorf("could not create file: %s%03d%s", prefix, 1, suffix)
}

var (
	versionFlag       = flag.Bool("v", false, "print the version and exit")
	transpileCommand  = flag.NewFlagSet("transpile", flag.ContinueOnError)
	verboseFlag       = transpileCommand.Bool("V", false, "print progress as comments")
	outputFlag        = transpileCommand.String("o", "", "output Go generated code to the specified file")
	packageFlag       = transpileCommand.String("p", "main", "set the name of the generated package")
	transpileHelpFlag = transpileCommand.Bool("h", false, "print help information")
	astCommand        = flag.NewFlagSet("ast", flag.ContinueOnError)
	astHelpFlag       = astCommand.Bool("h", false, "print help information")
)

func main() {
	code := runCommand()
	if code != 0 {
		os.Exit(code)
	}
}

func runCommand() int {
	flag.Usage = func() {
		usage := "Usage: %s [-v] [<command>] [<flags>] file.c\n\n"
		usage += "Commands:\n"
		usage += "  transpile\ttranspile an input C source file to Go\n"
		usage += "  ast\t\tprint AST before translated Go code\n\n"

		usage += "Flags:\n"
		fmt.Fprintf(stderr, usage, os.Args[0])
		flag.PrintDefaults()
	}

	transpileCommand.SetOutput(stderr)
	astCommand.SetOutput(stderr)

	flag.Parse()

	if *versionFlag {
		// Simply print out the version and exit.
		fmt.Println(Version)
		return 0
	}

	if flag.NArg() < 1 {
		flag.Usage()
		return 1
	}

	args := ProgramArgs{verbose: *verboseFlag, ast: false}

	switch os.Args[1] {
	case "ast":
		err := astCommand.Parse(os.Args[2:])
		if err != nil {
			fmt.Printf("ast command cannot parse: %v", err)
			return 1
		}

		if *astHelpFlag || astCommand.NArg() == 0 {
			fmt.Fprintf(stderr, "Usage: %s ast file.c\n", os.Args[0])
			astCommand.PrintDefaults()
			return 1
		}

		args.ast = true
		args.inputFile = astCommand.Arg(0)
	case "transpile":
		err := transpileCommand.Parse(os.Args[2:])
		if err != nil {
			fmt.Printf("transpile command cannot parse: %v", err)
			return 1
		}

		if *transpileHelpFlag || transpileCommand.NArg() == 0 {
			fmt.Fprintf(stderr, "Usage: %s transpile [-V] [-o file.go] [-p package] file.c\n", os.Args[0])
			transpileCommand.PrintDefaults()
			return 1
		}

		args.inputFile = transpileCommand.Arg(0)
		args.outputFile = *outputFlag
		args.packageName = *packageFlag
	default:
		flag.Usage()
		return 1
	}

	if err := Start(args); err != nil {
		fmt.Printf("Error: %v", err)
		return 1
	}

	return 0
}
<commit_msg>Bump version<commit_after>// Package c2go contains the main function for running the executable.
//
// Installation
//
//     go get -u github.com/elliotchance/c2go
//
// Usage
//
//     c2go myfile.c
//
package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/elliotchance/c2go/ast"
	"github.com/elliotchance/c2go/program"
	"github.com/elliotchance/c2go/transpiler"
	"reflect"
)

// Version can be requested through the command line with:
//
//     c2go -v
//
// See https://github.com/elliotchance/c2go/wiki/Release-Process
const Version = "v0.14.2 Oganesson 2017-08-10"

var stderr io.Writer = os.Stderr

// ProgramArgs defines the options available when processing the program. There
// is no constructor since the zeroed out values are the appropriate defaults -
// you need only set the options you need.
//
// TODO: Better separation on CLI modes
// https://github.com/elliotchance/c2go/issues/134
type ProgramArgs struct {
	verbose     bool
	ast         bool
	inputFile   string
	outputFile  string
	packageName string
}

func readAST(data []byte) []string {
	uncolored := regexp.MustCompile(`\x1b\[[\d;]+m`).ReplaceAll(data, []byte{})
	return strings.Split(string(uncolored), "\n")
}

type treeNode struct {
	indent int
	node   ast.Node
}

func convertLinesToNodes(lines []string) []treeNode {
	nodes := []treeNode{}
	for _, line := range lines {
		if strings.TrimSpace(line) == "" {
			continue
		}

		// It is tempting to discard null AST nodes, but these may
		// have semantic importance: for example, they represent omitted
		// for-loop conditions, as in for(;;).
		line = strings.Replace(line, "<<<NULL>>>", "NullStmt", 1)
		trimmed := strings.TrimLeft(line, "|\\- `")
		node := ast.Parse(trimmed)
		indentLevel := (len(line) - len(trimmed)) / 2
		nodes = append(nodes, treeNode{indentLevel, node})
	}

	return nodes
}

// buildTree convert an array of nodes, each prefixed with a depth into a tree.
func buildTree(nodes []treeNode, depth int) []ast.Node {
	if len(nodes) == 0 {
		return []ast.Node{}
	}

	// Split the list into sections, treat each section as a a tree with its own root.
	sections := [][]treeNode{}
	for _, node := range nodes {
		if node.indent == depth {
			sections = append(sections, []treeNode{node})
		} else {
			sections[len(sections)-1] = append(sections[len(sections)-1], node)
		}
	}

	results := []ast.Node{}
	for _, section := range sections {
		slice := []treeNode{}
		for _, n := range section {
			if n.indent > depth {
				slice = append(slice, n)
			}
		}

		children := buildTree(slice, depth+1)
		for _, child := range children {
			section[0].node.AddChild(child)
		}
		results = append(results, section[0].node)
	}

	return results
}

func toJSON(tree []interface{}) []map[string]interface{} {
	r := make([]map[string]interface{}, len(tree))

	for j, n := range tree {
		rn := reflect.ValueOf(n).Elem()
		r[j] = make(map[string]interface{})
		r[j]["node"] = rn.Type().Name()

		for i := 0; i < rn.NumField(); i++ {
			name := strings.ToLower(rn.Type().Field(i).Name)
			value := rn.Field(i).Interface()

			if name == "children" {
				v := value.([]interface{})

				if len(v) == 0 {
					continue
				}

				value = toJSON(v)
			}

			r[j][name] = value
		}
	}

	return r
}

func check(prefix string, e error) {
	if e != nil {
		panic(prefix + e.Error())
	}
}

// Start begins transpiling an input file.
func Start(args ProgramArgs) error {
	if os.Getenv("GOPATH") == "" {
		return fmt.Errorf("The $GOPATH must be set")
	}

	// 1. Compile it first (checking for errors)
	_, err := os.Stat(args.inputFile)
	if err != nil {
		return fmt.Errorf("Input file is not found")
	}

	// 2. Preprocess
	var pp []byte
	{
		// See : https://clang.llvm.org/docs/CommandGuide/clang.html
		// clang -E <file>    Run the preprocessor stage.
		cmd := exec.Command("clang", "-E", args.inputFile)
		var out bytes.Buffer
		var stderr bytes.Buffer
		cmd.Stdout = &out
		cmd.Stderr = &stderr
		err = cmd.Run()
		if err != nil {
			return fmt.Errorf("preprocess failed: %v\nStdErr = %v", err, stderr.String())
		}
		pp = []byte(out.String())
	}

	ppFilePath := path.Join(os.TempDir(), "pp.c")
	err = ioutil.WriteFile(ppFilePath, pp, 0644)
	if err != nil {
		return fmt.Errorf("writing to /tmp/pp.c failed: %v", err)
	}

	// 3. Generate JSON from AST
	astPP, err := exec.Command("clang", "-Xclang", "-ast-dump", "-fsyntax-only", ppFilePath).Output()
	if err != nil {
		// If clang fails it still prints out the AST, so we have to run it
		// again to get the real error.
		errBody, _ := exec.Command("clang", ppFilePath).CombinedOutput()

		panic("clang failed: " + err.Error() + ":\n\n" + string(errBody))
	}

	lines := readAST(astPP)
	if args.ast {
		for _, l := range lines {
			fmt.Println(l)
		}
		fmt.Println()

		return nil
	}

	nodes := convertLinesToNodes(lines)
	tree := buildTree(nodes, 0)

	p := program.NewProgram()
	p.Verbose = args.verbose

	err = transpiler.TranspileAST(args.inputFile, args.packageName, p, tree[0].(ast.Node))
	if err != nil {
		panic(err)
	}

	outputFilePath := args.outputFile

	if outputFilePath == "" {
		cleanFileName := filepath.Clean(filepath.Base(args.inputFile))
		extension := filepath.Ext(args.inputFile)

		outputFilePath = cleanFileName[0:len(cleanFileName)-len(extension)] + ".go"
	}

	err = ioutil.WriteFile(outputFilePath, []byte(p.String()), 0755)
	if err != nil {
		return fmt.Errorf("writing C output file failed: %v", err)
	}

	return nil
}

// newTempFile - returns temp file
func newTempFile(dir, prefix, suffix string) (*os.File, error) {
	for index := 1; index < 10000; index++ {
		path := filepath.Join(dir, fmt.Sprintf("%s%03d%s", prefix, index, suffix))
		if _, err := os.Stat(path); err != nil {
			return os.Create(path)
		}
	}
	return nil, fmt.Errorf("could not create file: %s%03d%s", prefix, 1, suffix)
}

var (
	versionFlag       = flag.Bool("v", false, "print the version and exit")
	transpileCommand  = flag.NewFlagSet("transpile", flag.ContinueOnError)
	verboseFlag       = transpileCommand.Bool("V", false, "print progress as comments")
	outputFlag        = transpileCommand.String("o", "", "output Go generated code to the specified file")
	packageFlag       = transpileCommand.String("p", "main", "set the name of the generated package")
	transpileHelpFlag = transpileCommand.Bool("h", false, "print help information")
	astCommand        = flag.NewFlagSet("ast", flag.ContinueOnError)
	astHelpFlag       = astCommand.Bool("h", false, "print help information")
)

func main() {
	code := runCommand()
	if code != 0 {
		os.Exit(code)
	}
}

func runCommand() int {
	flag.Usage = func() {
		usage := "Usage: %s [-v] [<command>] [<flags>] file.c\n\n"
		usage += "Commands:\n"
		usage += "  transpile\ttranspile an input C source file to Go\n"
		usage += "  ast\t\tprint AST before translated Go code\n\n"

		usage += "Flags:\n"
		fmt.Fprintf(stderr, usage, os.Args[0])
		flag.PrintDefaults()
	}

	transpileCommand.SetOutput(stderr)
	astCommand.SetOutput(stderr)

	flag.Parse()

	if *versionFlag {
		// Simply print out the version and exit.
		fmt.Println(Version)
		return 0
	}

	if flag.NArg() < 1 {
		flag.Usage()
		return 1
	}

	args := ProgramArgs{verbose: *verboseFlag, ast: false}

	switch os.Args[1] {
	case "ast":
		err := astCommand.Parse(os.Args[2:])
		if err != nil {
			fmt.Printf("ast command cannot parse: %v", err)
			return 1
		}

		if *astHelpFlag || astCommand.NArg() == 0 {
			fmt.Fprintf(stderr, "Usage: %s ast file.c\n", os.Args[0])
			astCommand.PrintDefaults()
			return 1
		}

		args.ast = true
		args.inputFile = astCommand.Arg(0)
	case "transpile":
		err := transpileCommand.Parse(os.Args[2:])
		if err != nil {
			fmt.Printf("transpile command cannot parse: %v", err)
			return 1
		}

		if *transpileHelpFlag || transpileCommand.NArg() == 0 {
			fmt.Fprintf(stderr, "Usage: %s transpile [-V] [-o file.go] [-p package] file.c\n", os.Args[0])
			transpileCommand.PrintDefaults()
			return 1
		}

		args.inputFile = transpileCommand.Arg(0)
		args.outputFile = *outputFlag
		args.packageName = *packageFlag
	default:
		flag.Usage()
		return 1
	}

	if err := Start(args); err != nil {
		fmt.Printf("Error: %v", err)
		return 1
	}

	return 0
}
<|endoftext|>


<commit_before>package main

import (
  "fmt"
  "github.com/digitalocean/godo"
  "golang.org/x/oauth2"
  "log"
  "io/ioutil"
  "os/user"
  "strconv"
  "strings"
  "github.com/dropbox/godropbox/errors"
  "golang.org/x/crypto/ssh"
  "bytes"
  "flag"
)

// FIXME: be able to configure the used SSH key
func main() {
  fmt.Println("running digital ocean setup script..")

  pat, err := ReadTokenFromConfigFile()
  if err != nil {
    log.Println(err)
    return
  }

  tokenSource := &TokenSource{
    AccessToken: pat,
  }
  oauthClient := oauth2.NewClient(oauth2.NoContext, tokenSource)
  client := godo.NewClient(oauthClient)

  cmd := flag.String("c", "list", "command: list ; delete ; create ; install")

  slaveCount := flag.Int("slaves", 5, "slave count argument used upon creation")

  procType := flag.String("type", "", "which type of nodes to install (slave/master). unspecified means all")

  flag.Parse()

  droplets, err := DropletList(client)
  if err != nil {
    log.Println(err)
    return
  }

  switch (*cmd) {
  case "list":
    log.Println(droplets)
  case "delete":
    err = RemoveAllDroplets(client)
  case "create":
    err = createMasterSlaveDroplets(client, *slaveCount)
  case "install":
    runMaster := false
    runSlaves := false
    switch *procType {
    case "": runMaster = true ; runSlaves = true
    case "slave": runSlaves = true
    case "master": runMaster = true
    default:
      log.Fatal("unknown install type " + *procType)
      return
    }
    master, slaves := GetTentacularDroplets(droplets)
    err = RunTentacularOnDroplets(master, slaves, runMaster, runSlaves)
  default:
    log.Fatal("unknown command " + *cmd)
    return
  }

  if err != nil {
    log.Println(err)
    return
  }
}

func createMasterSlaveDroplets(client *godo.Client, slaveCount int) (err error) {

  sshKeys := []godo.DropletCreateSSHKey{godo.DropletCreateSSHKey{Fingerprint: "9e:6a:0b:3d:0a:d1:af:c6:7f:d3:00:aa:b3:a1:ed:dc"}}

  log.Println("creating master... ")
  _, err = createSmallDroplet(client, "master", sshKeys)
  if err != nil { return }

  for i := 0; i < slaveCount; i++ {
    slaveName := "slave" + strconv.Itoa(i)
    log.Println("creating slave with name " + slaveName)
    _, err = createSmallDroplet(client, slaveName, sshKeys)
    if err != nil { return }
  }
  return
}

func createSmallDroplet(client *godo.Client, dropletName string, sshKeys []godo.DropletCreateSSHKey) (*godo.Droplet, error) {

  // Docker 1.10.1 on 14.04 in San Francisco
  createRequest := &godo.DropletCreateRequest{
    Name:   dropletName,
    Region: "sfo1",
    Size:   "512mb",
    PrivateNetworking: true,
    SSHKeys: sshKeys,
    Image: godo.DropletCreateImage{
      Slug: "docker",
    },
  }

  newDroplet, _, err := client.Droplets.Create(createRequest)
  return newDroplet, err
}

func RemoveAllDroplets(client *godo.Client) error {
  log.Println("deleting all droplets...")
  droplets, err := DropletList(client)
  if err != nil {
    return err
  }

  for _, droplet := range droplets {
    _, err := client.Droplets.Delete(droplet.ID)

    if (err != nil) {
      return err
    }
  }
  return nil
}

type TokenSource struct {
AccessToken string
}

func DropletList(client *godo.Client) ([]godo.Droplet, error) {
  // create a list to hold our droplets
  list := []godo.Droplet{}

  // create options. initially, these will be blank
  opt := &godo.ListOptions{}
  for {
    droplets, resp, err := client.Droplets.List(opt)
    if err != nil {
      return nil, err
    }

    // append the current page's droplets to our list
    for _, d := range droplets {
      list = append(list, d)
    }

    // if we are at the last page, break out the for loop
    if resp.Links == nil || resp.Links.IsLastPage() {
      break
    }

    page, err := resp.Links.CurrentPage()
    if err != nil {
      return nil, err
    }

    // set the page we want for the next request
    opt.Page = page + 1
  }

  return list, nil
}

func ReadTokenFromConfigFile() (token string, err error) {
  usr, err := user.Current()
  if err != nil { return }

  bytes, err := ioutil.ReadFile(usr.HomeDir + "/.digitalOceanToken")
  if err != nil { return }

  return string(bytes), err
}

func (t *TokenSource) Token() (*oauth2.Token, error) {
  token := &oauth2.Token{
    AccessToken: t.AccessToken,
  }
  return token, nil
}

func GetTentacularDroplets(droplets []godo.Droplet) (master *godo.Droplet, slaves []godo.Droplet) {
  slaves = []godo.Droplet{}
  for _, droplet := range droplets {
    if IsMasterDroplet(&droplet) {
      current := droplet
      master = &current
    } else if IsSlaveDroplet(&droplet) {
      slaves = append(slaves, droplet)
    }
  }

  return master, slaves
}

func RunTentacularOnDroplets(master *godo.Droplet, slaves []godo.Droplet, runMaster, runSlaves bool) (err error) {

  nodeCount := 0
  if runMaster { nodeCount++ }
  if runSlaves { nodeCount += len(slaves) }
  doneChan := make(chan error, nodeCount)

  if runMaster {
    go func() {
      err = RunTentacularMaster(master)
      doneChan <- err
    }()
  }

  if runSlaves {
    if len(slaves) == 0 {
      return errors.New("No slave nodes available.")
    }

    masterPrivAddr, err := master.PrivateIPv4()
    if err != nil { err = errors.Wrap(err, "") ; return err }

    slaveCommand := fmt.Sprintf(RUN_PROXY_SLAVE, masterPrivAddr)

    for _, slave := range slaves {

      slaveAddr, err := slave.PublicIPv4()
      if err != nil {
        fmt.Errorf("slave address could not be obtained. ignoring")
        continue
      }

      go func() {
        log.Println("running slave proxy at " + slaveAddr)
        reString, err := RunRemoteCommand(slaveAddr, setupCmd(slaveCommand))
        log.Println("slave terminated with output " + reString)
        if err != nil {
          log.Println("slave terminated with error.")
          log.Println(err)
        }
        doneChan <- err
      }()
    }
  }

  log.Println("waiting for procs to finish..")
  for i := 0; i < nodeCount; i++ {
    <- doneChan
  }

  return nil
}

func RunTentacularMaster(master *godo.Droplet) error {
  if master == nil {
    return errors.New("Missing master node.")
  }
  masterPubAddr, err := master.PublicIPv4()
  if err != nil { err = errors.Wrap(err, "") ; return err }

  log.Println("running command")

  // adding "&" at the end of commands so that it's ok to exit

  log.Println("running master proxy at " + masterPubAddr)
  reString, err := RunRemoteCommand(masterPubAddr, setupCmd(RUN_PROXY_MASTER))
  log.Println("master terminated with output " + reString)
  if err != nil {
    log.Println("master terminated with error.")
    log.Println(err)
  }
  return nil
}

func IsMasterDroplet(droplet *godo.Droplet) bool {
  return strings.HasPrefix(droplet.Name, "master")
}

func IsSlaveDroplet(droplet *godo.Droplet) bool {
  return strings.HasPrefix(droplet.Name, "slave")
}

func setupCmd(cmd string) string {
  return "(" + cmd + ");sleep 5"
}

// on a droplet assuming root user and ssh key at id_rsa.pub deployed
func RunRemoteCommand(addr string, command string) (s string, err error) {

  usr, err := user.Current()
  if err != nil { err = errors.Wrap(err, "") ; return }

  authMethod, err := PublicKeyFile(usr.HomeDir + "/.ssh/id_rsa")
  if err != nil { err = errors.Wrap(err, "") ; return }

  sshConfig := &ssh.ClientConfig{
    User: "root",
    Auth: []ssh.AuthMethod{authMethod},
  }

  conn, err := ssh.Dial("tcp", addr + ":22", sshConfig)
  if err != nil { err = errors.Wrap(err, "ssh dial failed.") ; return }

  defer conn.Close()

  session, err := conn.NewSession()
  if err != nil { err = errors.Wrap(err, "session failed."); return }

  defer session.Close()

  var stdoutBuf bytes.Buffer
  session.Stdout = &stdoutBuf
  err = session.Run(command)
  if err != nil { err = errors.Wrap(err, "cmd failed."); return }

  return stdoutBuf.String(), nil
}

func PublicKeyFile(file string) (auth ssh.AuthMethod, err error) {
  buffer, err := ioutil.ReadFile(file)
  if err != nil { err = errors.Wrap(err, "") ; return }

  key, err := ssh.ParsePrivateKey(buffer)
  if err != nil { err = errors.Wrap(err, "") ; return }

  return ssh.PublicKeys(key), nil
}

const RUN_PROXY_MASTER = DOCKER_REMOVE_ALL_CONTAINERS + "docker run -p 8080:8080 -p 6666:6666 --name master --rm danoctavian/tentacular /go/bin/app --type=master"
const RUN_PROXY_SLAVE = DOCKER_REMOVE_ALL_CONTAINERS + "docker run -p 8080:8080 --name slave --rm danoctavian/tentacular /go/bin/app --masterurl=\"http://%s:6666\""

const DOCKER_REMOVE_ALL_CONTAINERS = "docker rm --force `docker ps -qa`;"<commit_msg>added args flag<commit_after>package main

import (
  "fmt"
  "github.com/digitalocean/godo"
  "golang.org/x/oauth2"
  "log"
  "io/ioutil"
  "os/user"
  "strconv"
  "strings"
  "github.com/dropbox/godropbox/errors"
  "golang.org/x/crypto/ssh"
  "bytes"
  "flag"
)

// FIXME: be able to configure the used SSH key
func main() {
  fmt.Println("running digital ocean setup script..")

  pat, err := ReadTokenFromConfigFile()
  if err != nil {
    log.Println(err)
    return
  }

  tokenSource := &TokenSource{
    AccessToken: pat,
  }
  oauthClient := oauth2.NewClient(oauth2.NoContext, tokenSource)
  client := godo.NewClient(oauthClient)

  cmd := flag.String("c", "list", "command: list ; delete ; create ; install")

  slaveCount := flag.Int("slaves", 5, "slave count argument used upon creation")

  procType := flag.String("type", "", "which type of nodes to install (slave/master). unspecified means all")
  procArgs := flag.String("args", "", "arguments to that specific process")

  flag.Parse()

  droplets, err := DropletList(client)
  if err != nil {
    log.Println(err)
    return
  }

  switch (*cmd) {
  case "list":
    log.Println(droplets)
  case "delete":
    err = RemoveAllDroplets(client)
  case "create":
    err = createMasterSlaveDroplets(client, *slaveCount)
  case "install":
    runMaster := false
    runSlaves := false
    switch *procType {
    case "": runMaster = true ; runSlaves = true
    case "slave": runSlaves = true
    case "master": runMaster = true
    default:
      log.Fatal("unknown install type " + *procType)
      return
    }
    master, slaves := GetTentacularDroplets(droplets)
    err = RunTentacularOnDroplets(master, slaves, runMaster, runSlaves, *procArgs)
  default:
    log.Fatal("unknown command " + *cmd)
    return
  }

  if err != nil {
    log.Println(err)
    return
  }
}

func createMasterSlaveDroplets(client *godo.Client, slaveCount int) (err error) {

  sshKeys := []godo.DropletCreateSSHKey{godo.DropletCreateSSHKey{Fingerprint: "9e:6a:0b:3d:0a:d1:af:c6:7f:d3:00:aa:b3:a1:ed:dc"}}

  log.Println("creating master... ")
  _, err = createSmallDroplet(client, "master", sshKeys)
  if err != nil { return }

  for i := 0; i < slaveCount; i++ {
    slaveName := "slave" + strconv.Itoa(i)
    log.Println("creating slave with name " + slaveName)
    _, err = createSmallDroplet(client, slaveName, sshKeys)
    if err != nil { return }
  }
  return
}

func createSmallDroplet(client *godo.Client, dropletName string, sshKeys []godo.DropletCreateSSHKey) (*godo.Droplet, error) {

  // Docker 1.10.1 on 14.04 in San Francisco
  createRequest := &godo.DropletCreateRequest{
    Name:   dropletName,
    Region: "sfo1",
    Size:   "512mb",
    PrivateNetworking: true,
    SSHKeys: sshKeys,
    Image: godo.DropletCreateImage{
      Slug: "docker",
    },
  }

  newDroplet, _, err := client.Droplets.Create(createRequest)
  return newDroplet, err
}

func RemoveAllDroplets(client *godo.Client) error {
  log.Println("deleting all droplets...")
  droplets, err := DropletList(client)
  if err != nil {
    return err
  }

  for _, droplet := range droplets {
    _, err := client.Droplets.Delete(droplet.ID)

    if (err != nil) {
      return err
    }
  }
  return nil
}

type TokenSource struct {
AccessToken string
}

func DropletList(client *godo.Client) ([]godo.Droplet, error) {
  // create a list to hold our droplets
  list := []godo.Droplet{}

  // create options. initially, these will be blank
  opt := &godo.ListOptions{}
  for {
    droplets, resp, err := client.Droplets.List(opt)
    if err != nil {
      return nil, err
    }

    // append the current page's droplets to our list
    for _, d := range droplets {
      list = append(list, d)
    }

    // if we are at the last page, break out the for loop
    if resp.Links == nil || resp.Links.IsLastPage() {
      break
    }

    page, err := resp.Links.CurrentPage()
    if err != nil {
      return nil, err
    }

    // set the page we want for the next request
    opt.Page = page + 1
  }

  return list, nil
}

func ReadTokenFromConfigFile() (token string, err error) {
  usr, err := user.Current()
  if err != nil { return }

  bytes, err := ioutil.ReadFile(usr.HomeDir + "/.digitalOceanToken")
  if err != nil { return }

  return string(bytes), err
}

func (t *TokenSource) Token() (*oauth2.Token, error) {
  token := &oauth2.Token{
    AccessToken: t.AccessToken,
  }
  return token, nil
}

func GetTentacularDroplets(droplets []godo.Droplet) (master *godo.Droplet, slaves []godo.Droplet) {
  slaves = []godo.Droplet{}
  for _, droplet := range droplets {
    if IsMasterDroplet(&droplet) {
      current := droplet
      master = &current
    } else if IsSlaveDroplet(&droplet) {
      slaves = append(slaves, droplet)
    }
  }

  return master, slaves
}

func RunTentacularOnDroplets(master *godo.Droplet, slaves []godo.Droplet,
                             runMaster, runSlaves bool, args string) (err error) {

  nodeCount := 0
  if runMaster { nodeCount++ }
  if runSlaves { nodeCount += len(slaves) }
  doneChan := make(chan error, nodeCount)

  if runMaster {
    go func() {
      err = RunTentacularMaster(master, args)
      doneChan <- err
    }()
  }

  if runSlaves {
    if len(slaves) == 0 {
      return errors.New("No slave nodes available.")
    }

    masterPrivAddr, err := master.PrivateIPv4()
    if err != nil { err = errors.Wrap(err, "") ; return err }

    slaveCommand := fmt.Sprintf(RUN_PROXY_SLAVE, masterPrivAddr)

    log.Print("Running command on slaves: " + slaveCommand)

    for _, slave := range slaves {

      slaveAddr, err := slave.PublicIPv4()
      if err != nil {
        fmt.Errorf("slave address could not be obtained. ignoring")
        continue
      }

      go func() {
        log.Println("running slave proxy at " + slaveAddr)
        reString, err := RunRemoteCommand(slaveAddr, setupCmd(slaveCommand, args))
        log.Println("slave terminated with output " + reString)
        if err != nil {
          log.Println("slave terminated with error.")
          log.Println(err)
        }
        doneChan <- err
      }()
    }
  }

  log.Println("waiting for procs to finish..")
  for i := 0; i < nodeCount; i++ {
    <- doneChan
  }

  return nil
}

func RunTentacularMaster(master *godo.Droplet, args string) error {
  if master == nil {
    return errors.New("Missing master node.")
  }
  masterPubAddr, err := master.PublicIPv4()
  if err != nil { err = errors.Wrap(err, "") ; return err }

  masterCmd := setupCmd(RUN_PROXY_MASTER, args)
  log.Print("Running command on master: " + masterCmd)
  log.Println("Running master proxy at " + masterPubAddr)

  reString, err := RunRemoteCommand(masterPubAddr, masterCmd)
  log.Println("master terminated with output " + reString)
  if err != nil {
    log.Println("master terminated with error.")
    log.Println(err)
  }
  return nil
}

func IsMasterDroplet(droplet *godo.Droplet) bool {
  return strings.HasPrefix(droplet.Name, "master")
}

func IsSlaveDroplet(droplet *godo.Droplet) bool {
  return strings.HasPrefix(droplet.Name, "slave")
}

func setupCmd(cmd string, args string) string {
  return fmt.Sprintf("(%s %s); sleep 5", cmd, args)
}

// on a droplet assuming root user and ssh key at id_rsa.pub deployed
func RunRemoteCommand(addr string, command string) (s string, err error) {

  usr, err := user.Current()
  if err != nil { err = errors.Wrap(err, "") ; return }

  authMethod, err := PublicKeyFile(usr.HomeDir + "/.ssh/id_rsa")
  if err != nil { err = errors.Wrap(err, "") ; return }

  sshConfig := &ssh.ClientConfig{
    User: "root",
    Auth: []ssh.AuthMethod{authMethod},
  }

  conn, err := ssh.Dial("tcp", addr + ":22", sshConfig)
  if err != nil { err = errors.Wrap(err, "ssh dial failed.") ; return }

  defer conn.Close()

  session, err := conn.NewSession()
  if err != nil { err = errors.Wrap(err, "session failed."); return }

  defer session.Close()

  var stdoutBuf bytes.Buffer
  session.Stdout = &stdoutBuf
  err = session.Run(command)
  if err != nil { err = errors.Wrap(err, "cmd failed."); return }

  return stdoutBuf.String(), nil
}

func PublicKeyFile(file string) (auth ssh.AuthMethod, err error) {
  buffer, err := ioutil.ReadFile(file)
  if err != nil { err = errors.Wrap(err, "") ; return }

  key, err := ssh.ParsePrivateKey(buffer)
  if err != nil { err = errors.Wrap(err, "") ; return }

  return ssh.PublicKeys(key), nil
}

const RUN_PROXY_MASTER = DOCKER_REMOVE_ALL_CONTAINERS + "docker run -p 8080:8080 -p 6666:6666 --name master --rm danoctavian/tentacular /go/bin/app --type=master"
const RUN_PROXY_SLAVE = DOCKER_REMOVE_ALL_CONTAINERS + "docker run -p 8080:8080 --name slave --rm danoctavian/tentacular /go/bin/app --masterurl=\"http://%s:6666\""

const DOCKER_REMOVE_ALL_CONTAINERS = "docker rm --force `docker ps -qa`;"<|endoftext|>