<commit_before>// gridfsd - GridFS HTTP Server
//
// Copyright (c) 2011 voxelbrain UG (haftungsbeschränkt)
// Authors:
//     Sebastian Friedel <sef@voxelbrain.com>
//     David Lehmann <dtl@voxelbrain.com>
//
// All rights reserved.
package main

import (
	"flag"
	"fmt"
	"http"
	"io"
	"launchpad.net/gobson/bson"
	"launchpad.net/mgo"
	"os"
	"regexp"
	"runtime/debug"
	"strings"
	"sort"
	"template"
)

// flags for command line configuration
var (
	httpAddr        string
	mongoAddr       string
	mongoDatabase   string
	mongoCollection string
	serveIndex      bool
)

// initializing the command line flags
func init() {
	flag.StringVar(&httpAddr, "listen", "localhost:7000", "http address")
	flag.StringVar(&mongoAddr, "dbaddr", "localhost", "database address")
	flag.StringVar(&mongoDatabase, "dbname", "", "database name")
	flag.StringVar(&mongoCollection, "dbfs", "fs", "database collection")
	flag.BoolVar(&serveIndex, "index", false, "serve directory indexes")
	flag.Parse()

	// database name is mandatory
	if mongoDatabase == "" {
		flag.Usage()
		os.Exit(1)
	}
}

// A GridFSServer implements the http.Handler interface and serves files
// from GridFS that start with a specific prefix. It handles GET requests only.
// Remember to call GridFSServer.Close to close the mgo.Mongo session.
type GridFSServer struct {
	prefix  string
	session *mgo.Session
	gfs     *mgo.GridFS
}

// Factory for GridFSServer
func NewGridFSServer(prefix, addr, database, collection string) *GridFSServer {
	session, err := mgo.Mongo(addr)
	if err != nil {
		panic(err)
	}

	db := session.DB(database)
	gfs := db.GridFS(collection)

	server := &GridFSServer{prefix, session, gfs}

	return server
}

// Closes a mgo.Mongo session.
func (srv *GridFSServer) Close() {
	srv.session.Close()
}

// Dispatch requests
func (srv *GridFSServer) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	method := req.Method
	path := req.URL.Path

	// handle errors by returning a HTTP 500 status code to the client
	defer func() {
		if e := recover(); e != nil {
			trace := debug.Stack()
			msg := fmt.Sprintf("Internal Server Error\n\n%s\n%s", trace, e)
			http.Error(w, msg, http.StatusInternalServerError)
		}
	}()

	if method != "GET" {
		http.Error(w, "Method Not Allowed "+method, http.StatusMethodNotAllowed)
		return
	}

	if strings.HasSuffix(path, "/") && serveIndex {
		srv.ServeIndex(w, req)
		return
	}

	srv.ServeFile(w, req)
}

// Serve existing files from GridFS or responde with HTTP 404 Not Found if
// no file with the requested path can be found.
func (srv *GridFSServer) ServeFile(w http.ResponseWriter, req *http.Request) {
	path := req.URL.Path[len(srv.prefix):]
	file, err := srv.gfs.Open(path)

	if err == mgo.NotFound {
		http.NotFound(w, req)
		return
	}

	if err != nil {
		panic(err)
	}

	defer file.Close()

	io.Copy(w, file)
}

// the directory listing template
var parsedIndexTemplate = template.Must(template.New("index").Parse(indexTemplate))

// needed as container for GridFS queries that are used in directory listings
type gfsFile struct {
	Filename string `bson:",omitempty"`
}

// listing context for html template rendering
type directoryIndex struct {
	Path        string
	Files       []string
	Directories sort.StringSlice
}

// Serve a simple directory listing if the `index`-flag is `true`
func (srv *GridFSServer) ServeIndex(w http.ResponseWriter, req *http.Request) {
	var (
		result gfsFile
		q      bson.M
	)

	path := req.URL.Path[len(srv.prefix):]
	ctx := &directoryIndex{req.URL.Path, make([]string, 0), make([]string, 0)}

	if path != "" {
		pattern := "^" + regexp.QuoteMeta(path)
		q = bson.M{"filename": bson.RegEx{pattern, ""}}
	}

	cur := srv.gfs.Files.Find(q).Sort(bson.M{"filename": 1})

	if path != "" {
		if count, err := cur.Count(); err != nil {
			panic(err)
		} else if count == 0 {
			http.NotFound(w, req)
			return
		}
	}

	iter := cur.Iter()

	for iter.Next(&result) {
		resultPath := result.Filename[len(path):]
		parts := strings.Split(resultPath, "/")

		if len(parts) > 1 {
			i := ctx.Directories.Search(parts[0])

			if !(i < len(ctx.Directories) && ctx.Directories[i] == parts[0]) {
				ctx.Directories = append(ctx.Directories, parts[0])
			}
			ctx.Directories.Sort()

		} else {
			ctx.Files = append(ctx.Files, resultPath)
		}
	}

	if iter.Err() != nil {
		panic(iter.Err())
	}

	if err := parsedIndexTemplate.Execute(w, ctx); err != nil {
		panic(err)
	}
}

func main() {
	prefix := "/"
	server := NewGridFSServer(prefix, mongoAddr, mongoDatabase, mongoCollection)
	defer server.Close()

	http.Handle(prefix, server)

	err := http.ListenAndServe(httpAddr, nil)
	if err != nil {
		panic(err)
	}
}
<commit_msg>Replace with newer, but still old code<commit_after>package main

import (
	"errors"
	"labix.org/v2/mgo"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strings"
)

type Dir struct {
	name  string
	fs    *mgo.GridFS
	flat  bool
	cache map[string]*File
}

func (d Dir) Open(name string) (http.File, error) {
	if d.flat && filepath.Separator != '/' && strings.IndexRune(name, filepath.Separator) >= 0 {
		return nil, errors.New("http: invalid character in file path")
	}

	filename := filepath.Join(d.name, filepath.FromSlash(path.Clean("/"+name)))

	f, err := d.fs.Open(filename)
	if err != nil {
		return nil, err
	}

	file := &File{f}

	return file, nil
}

type File struct {
	file *mgo.GridFile
}

func (f *File) Close() error {
	return f.file.Close()
}

func (f *File) Stat() (os.FileInfo, error) {
	date := f.file.UploadDate()
	info := &os.FileInfo{
		Size:     f.file.Size(),
		Ctime_ns: date,
		Mtime_ns: date,
		Atime_ns: date,
	}

	return info, nil
}

func (f *File) Readdir(count int) ([]os.FileInfo, error) {
	return nil, errors.New("http: directory listing not implemented")
}

func (f *File) Read(b []byte) (int, error) {
	return f.file.Read(b)
}

func (f *File) Seek(offset int64, whence int) (int64, error) {
	return f.file.Seek(offset, whence)
}

func main() {
	session, err := mgo.Dial("localhost")
	if err != nil {
		panic(err)
	}

	db := session.DB("sabercat")
	fs := db.GridFS("fs")
	d := &Dir{"/", fs, false, map[string]*File{}}
	http.Handle("/", http.FileServer(d))

	err = http.ListenAndServe("localhost:8080", nil)
	if err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"runtime"

	"golang.org/x/net/context"

	"bazil.org/fuse"
	"bazil.org/fuse/fs"

	skicka "github.com/pellaeon/skicka"
	"github.com/pellaeon/skicka/gdrive"
)

///////////////////////////////////////////////////////////////////////////
// Global Variables

var progName = filepath.Base(os.Args[0])

type debugging bool

var (
	gd *gdrive.GDrive

	// The key is only set if encryption is needed (i.e. if -encrypt is
	// provided for an upload, or if an encrypted file is encountered
	// during 'download' or 'cat').
	key []byte

	debug   debugging
	verbose debugging
	quiet   bool

	// Configuration read in from the skicka config file.
	config struct {
		Google struct {
			ClientId     string
			ClientSecret string
			// If set, is appended to all http requests via ?key=XXX.
			ApiKey string
		}
		Encryption struct {
			Salt             string
			Passphrase_hash  string
			Encrypted_key    string
			Encrypted_key_iv string
		}
		Upload struct {
			Ignored_Regexp         []string
			Bytes_per_second_limit int
		}
		Download struct {
			Bytes_per_second_limit int
		}
	}

	// Various statistics gathered along the way. These all should be
	// updated using atomic operations since we often have multiple threads
	// working concurrently for uploads and downloads.
	stats struct {
		DiskReadBytes     int64
		DiskWriteBytes    int64
		UploadBytes       int64
		DownloadBytes     int64
		LocalFilesUpdated int64
		DriveFilesUpdated int64
	}

	// Smaller files will be handled with multiple threads going at once;
	// doing so improves bandwidth utilization since round-trips to the
	// Drive APIs take a while.  (However, we don't want too have too many
	// workers; this would both lead to lots of 403 rate limit errors...)
	nWorkers int
)

func usage() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", progName)
	fmt.Fprintf(os.Stderr, "  %s MOUNTPOINT\n", progName)
	flag.PrintDefaults()
}

func userHomeDir() string {
	if runtime.GOOS == "windows" {
		home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
		if home == "" {
			home = os.Getenv("USERPROFILE")
		}
		return home
	}
	return os.Getenv("HOME")
}

func main() {
	log.SetFlags(0)
	log.SetPrefix(progName + ": ")

	// Initialize skicka
	home := userHomeDir()
	tokenCacheFilename := flag.String("tokencache",
		filepath.Join(home, ".skicka.tokencache.json"),
		"OAuth2 token cache file")
	configFilename := flag.String("config",
		filepath.Join(home, ".skicka.config"),
		"Configuration file")
	metadataCacheFilename := flag.String("metadata-cache-file",
		filepath.Join(home, "/.skicka.metadata.cache"),
		"Filename for local cache of Google Drive file metadata")
	nw := flag.Int("num-threads", 4, "Number of threads to use for uploads/downloads")
	vb := flag.Bool("verbose", false, "Enable verbose output")
	dbg := flag.Bool("debug", false, "Enable debugging output")
	qt := flag.Bool("quiet", false, "Suppress non-error messages")
	dumpHTTP := flag.Bool("dump-http", false, "Dump http traffic")
	flakyHTTP := flag.Bool("flaky-http", false, "Add flakiness to http traffic")
	noBrowserAuth := flag.Bool("no-browser-auth", false,
		"Don't try launching browser for authorization")
	flag.Usage = usage
	flag.Parse()

	var mountpoint string
	if flag.NArg() == 1 {
		mountpoint = flag.Arg(0)
	} else {
		mountpoint = *flag.String("mountpoint", "", "Mountpoint")
	}
	if mountpoint == "" {
		usage()
		os.Exit(2)
	}

	nWorkers = *nw

	debug = debugging(*dbg)
	verbose = debugging(*vb || bool(debug))
	quiet = *qt

	skicka.ReadConfigFile(*configFilename)

	// Set up the basic http.Transport.
	transport := http.DefaultTransport
	if tr, ok := transport.(*http.Transport); ok {
		// Increase the default number of open connections per destination host
		// to be enough for the number of goroutines we run concurrently for
		// uploads/downloads; this gives some benefit especially for uploading
		// small files.
		tr.MaxIdleConnsPerHost = 4
	} else {
		skicka.PrintErrorAndExit(fmt.Errorf("DefaultTransport not an *http.Transport?"))
	}
	if *flakyHTTP {
		transport = skicka.NewFlakyTransport(transport)
	}
	if *dumpHTTP {
		//transport = skicka.LoggingTransport{transport: transport}
		// TODO
	}

	// And now upgrade to the OAuth Transport *http.Client.
	client, err := skicka.GetOAuthClient(*tokenCacheFilename, !*noBrowserAuth,
		transport)
	if err != nil {
		skicka.PrintErrorAndExit(fmt.Errorf("error with OAuth2 Authorization: %v ", err))
	}

	// Choose the appropriate callback function for the GDrive object to
	// use for debugging output.
	var dpf func(s string, args ...interface{})
	if debug {
		dpf = skicka.DebugPrint
	} else {
		dpf = skicka.DebugNoPrint
	}

	gd, err := gdrive.New(config.Upload.Bytes_per_second_limit,
		config.Download.Bytes_per_second_limit, dpf, client,
		*metadataCacheFilename, quiet)
	if err != nil {
		log.Fatal(fmt.Errorf("error creating Google Drive "+
			"client: %v", err))
		os.Exit(3)
	}

	c, err := fuse.Mount(mountpoint)
	if err != nil {
		log.Fatal(err)
	}
	defer c.Close()

	filesys := &FS{
		gd: gd,
	}
	if err := fs.Serve(c, filesys); err != nil {
		log.Fatal(err)
	}

	// check if the mount process has an error to report
	<-c.Ready
	if err := c.MountError; err != nil {
		log.Fatal(err)
	}
}

type FS struct {
	gd *gdrive.GDrive
}

var _ fs.FS = (*FS)(nil)

func (f *FS) Root() (fs.Node, error) {
	gd_file, err := f.gd.GetFile("/")
	return Node{
		sk_file: gd_file,
	}, err
}

var _ fs.Node = (*Node)(nil)

type Node struct {
	sk_file *gdrive.File
}

func (n Node) Attr(ctx context.Context, attr *fuse.Attr) error {
	var sum uint64
	for _, c := range n.sk_file.Id {
		sum += uint64(c)
	}
	attr.Inode = sum
	attr.Size = uint64(n.sk_file.FileSize)
	attr.Blocks = uint64(n.sk_file.FileSize / 1024) // XXX: block size 1024 bytes
	attr.Atime = n.sk_file.ModTime
	attr.Mtime = n.sk_file.ModTime
	attr.Ctime = n.sk_file.ModTime
	attr.Crtime = n.sk_file.ModTime
	attr.Mode = os.ModeDir | 0755
	attr.Nlink = 0

	return nil
}
<commit_msg>Update# implement Node Lookup<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"runtime"

	"golang.org/x/net/context"

	"bazil.org/fuse"
	"bazil.org/fuse/fs"

	skicka "github.com/pellaeon/skicka"
	"github.com/pellaeon/skicka/gdrive"
)

///////////////////////////////////////////////////////////////////////////
// Global Variables

var progName = filepath.Base(os.Args[0])

type debugging bool

var (
	gd *gdrive.GDrive

	// The key is only set if encryption is needed (i.e. if -encrypt is
	// provided for an upload, or if an encrypted file is encountered
	// during 'download' or 'cat').
	key []byte

	debug   debugging
	verbose debugging
	quiet   bool

	// Configuration read in from the skicka config file.
	config struct {
		Google struct {
			ClientId     string
			ClientSecret string
			// If set, is appended to all http requests via ?key=XXX.
			ApiKey string
		}
		Encryption struct {
			Salt             string
			Passphrase_hash  string
			Encrypted_key    string
			Encrypted_key_iv string
		}
		Upload struct {
			Ignored_Regexp         []string
			Bytes_per_second_limit int
		}
		Download struct {
			Bytes_per_second_limit int
		}
	}

	// Various statistics gathered along the way. These all should be
	// updated using atomic operations since we often have multiple threads
	// working concurrently for uploads and downloads.
	stats struct {
		DiskReadBytes     int64
		DiskWriteBytes    int64
		UploadBytes       int64
		DownloadBytes     int64
		LocalFilesUpdated int64
		DriveFilesUpdated int64
	}

	// Smaller files will be handled with multiple threads going at once;
	// doing so improves bandwidth utilization since round-trips to the
	// Drive APIs take a while.  (However, we don't want too have too many
	// workers; this would both lead to lots of 403 rate limit errors...)
	nWorkers int
)

func usage() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", progName)
	fmt.Fprintf(os.Stderr, "  %s MOUNTPOINT\n", progName)
	flag.PrintDefaults()
}

func userHomeDir() string {
	if runtime.GOOS == "windows" {
		home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
		if home == "" {
			home = os.Getenv("USERPROFILE")
		}
		return home
	}
	return os.Getenv("HOME")
}

func main() {
	log.SetFlags(0)
	log.SetPrefix(progName + ": ")

	// Initialize skicka
	home := userHomeDir()
	tokenCacheFilename := flag.String("tokencache",
		filepath.Join(home, ".skicka.tokencache.json"),
		"OAuth2 token cache file")
	configFilename := flag.String("config",
		filepath.Join(home, ".skicka.config"),
		"Configuration file")
	metadataCacheFilename := flag.String("metadata-cache-file",
		filepath.Join(home, "/.skicka.metadata.cache"),
		"Filename for local cache of Google Drive file metadata")
	nw := flag.Int("num-threads", 4, "Number of threads to use for uploads/downloads")
	vb := flag.Bool("verbose", false, "Enable verbose output")
	dbg := flag.Bool("debug", false, "Enable debugging output")
	qt := flag.Bool("quiet", false, "Suppress non-error messages")
	dumpHTTP := flag.Bool("dump-http", false, "Dump http traffic")
	flakyHTTP := flag.Bool("flaky-http", false, "Add flakiness to http traffic")
	noBrowserAuth := flag.Bool("no-browser-auth", false,
		"Don't try launching browser for authorization")
	flag.Usage = usage
	flag.Parse()

	var mountpoint string
	if flag.NArg() == 1 {
		mountpoint = flag.Arg(0)
	} else {
		mountpoint = *flag.String("mountpoint", "", "Mountpoint")
	}
	if mountpoint == "" {
		usage()
		os.Exit(2)
	}

	nWorkers = *nw

	debug = debugging(*dbg)
	verbose = debugging(*vb || bool(debug))
	quiet = *qt

	skicka.ReadConfigFile(*configFilename)

	// Set up the basic http.Transport.
	transport := http.DefaultTransport
	if tr, ok := transport.(*http.Transport); ok {
		// Increase the default number of open connections per destination host
		// to be enough for the number of goroutines we run concurrently for
		// uploads/downloads; this gives some benefit especially for uploading
		// small files.
		tr.MaxIdleConnsPerHost = 4
	} else {
		skicka.PrintErrorAndExit(fmt.Errorf("DefaultTransport not an *http.Transport?"))
	}
	if *flakyHTTP {
		transport = skicka.NewFlakyTransport(transport)
	}
	if *dumpHTTP {
		//transport = skicka.LoggingTransport{transport: transport}
		// TODO
	}

	// And now upgrade to the OAuth Transport *http.Client.
	client, err := skicka.GetOAuthClient(*tokenCacheFilename, !*noBrowserAuth,
		transport)
	if err != nil {
		skicka.PrintErrorAndExit(fmt.Errorf("error with OAuth2 Authorization: %v ", err))
	}

	// Choose the appropriate callback function for the GDrive object to
	// use for debugging output.
	var dpf func(s string, args ...interface{})
	if debug {
		dpf = skicka.DebugPrint
	} else {
		dpf = skicka.DebugNoPrint
	}

	gd, err := gdrive.New(config.Upload.Bytes_per_second_limit,
		config.Download.Bytes_per_second_limit, dpf, client,
		*metadataCacheFilename, quiet)
	if err != nil {
		log.Fatal(fmt.Errorf("error creating Google Drive "+
			"client: %v", err))
		os.Exit(3)
	}

	c, err := fuse.Mount(mountpoint)
	if err != nil {
		log.Fatal(err)
	}
	defer c.Close()

	filesys := &FS{
		gd: gd,
	}
	if err := fs.Serve(c, filesys); err != nil {
		log.Fatal(err)
	}

	// check if the mount process has an error to report
	<-c.Ready
	if err := c.MountError; err != nil {
		log.Fatal(err)
	}
}

type FS struct {
	gd *gdrive.GDrive
}

var _ fs.FS = (*FS)(nil)

func (f *FS) Root() (fs.Node, error) {
	gd_file, err := f.gd.GetFile("/")
	return Node{
		sk_file: gd_file,
		fs:      f,
	}, err
}

var _ fs.Node = (*Node)(nil)

type Node struct {
	sk_file *gdrive.File
	fs      *FS
}

func (n Node) Attr(ctx context.Context, attr *fuse.Attr) error {
	var sum uint64
	for _, c := range n.sk_file.Id {
		sum += uint64(c)
	}
	attr.Inode = sum
	attr.Size = uint64(n.sk_file.FileSize)
	attr.Blocks = uint64(n.sk_file.FileSize / 1024) // XXX: block size 1024 bytes
	attr.Atime = n.sk_file.ModTime
	attr.Mtime = n.sk_file.ModTime
	attr.Ctime = n.sk_file.ModTime
	attr.Crtime = n.sk_file.ModTime
	attr.Mode = os.ModeDir | 0755
	attr.Nlink = 0

	return nil
}

var _ fs.NodeRequestLookuper = (*Node)(nil)

func (n Node) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.LookupResponse) (fs.Node, error) {
	path := req.Name
	gd_file, err := n.fs.gd.GetFile(path)
	if err != nil {
		log.Fatal(err)
		return nil, fuse.ENOENT
	}
	return Node{
		sk_file: gd_file,
		fs:      n.fs,
	}, nil
}
<|endoftext|>


<commit_before>package main

import (
    "io/ioutil"
    "os"
    "os/exec"
    "strconv"
    "strings"
    "github.com/fatih/color"
)

var (
    progPath string
    progName string
    testDir string
    ioStyle int
)

func main() {
    initVars()

    // chdir to directory containg test cases
    os.Chdir(testDir)

    setIOStyle()

    i := 1

    if ioStyle == 1 {
        for {
            if _, err := os.Stat(strconv.Itoa(i) + ".in"); os.IsNotExist(err) {
                // TODO: Add number of total test cases passed
                color.White("Finished testing")
                os.Exit(0)
            }

            copyTestData(strconv.Itoa(i) + ".in")

            runProgram()

            a := readOutput(progName + ".out")
            b := readOutput(strconv.Itoa(i) + ".out")

            if compareOutput(a, b) != -1 {
                color.Red("Wrong output for test case " + strconv.Itoa(i) + ":\nExpected:\n" + string(b[:]) + "Found:\n" + string(a[:]) + "\n")
            } else {
                color.Green("Test case " + strconv.Itoa(i) + " passed");
            }
            i++
        }
    } else if ioStyle == 2 {
        for {
            if _, err := os.Stat("I." + strconv.Itoa(i)); os.IsNotExist(err) {
                // TODO: Add number of total test cases passed
                color.White("Finished testing")
                os.Exit(0)
            }

            copyTestData("I." + strconv.Itoa(i))

            runProgram()

            a := readOutput(progName + ".out")
            b := readOutput("O." + strconv.Itoa(i))

            if compareOutput(a, b) != -1 {
                color.Red("Wrong output for test case " + strconv.Itoa(i) + ":\nExpected:\n" + string(b[:]) + "Found:\n" + string(a[:]) + "\n")
            } else {
                color.Green("Test case " + strconv.Itoa(i) + " passed");
            }
            i++
        }

    }
}

func initVars() {
    numArgs := len(os.Args)
    if numArgs >= 2 {
        progPath = os.Args[1]
    } else {
        color.Red("Please specify exectable as first argument")
        os.Exit(0)
    }

    if numArgs >= 3 {
        testDir = os.Args[2]
    } else {
        x, err := os.Getwd()
        if err != nil {
            color.Red("Error getting current working directory:\n" + err.Error())
            os.Exit(0)
        }
        testDir = x
    }

    progName = getProgName()
}

func getProgName() string {
    pos := strings.LastIndex(progPath, "/")
    if pos == -1 {
        pos = strings.LastIndex(progPath, "\\")
    }
    return progPath[pos + 1:]
}

func copyTestData(fileName string) {
    b, err := ioutil.ReadFile(fileName)
    if err != nil {
        panic(err)
    }

    err = ioutil.WriteFile(progName + ".in", b, 0644)
    if err != nil {
        panic(err)
    }
}

// if outputs are not same, return position of nonequivalence; otherwise return -1
func compareOutput(a, b []byte) int {
    fileLen := len(a)
    bLen := len(b)
    if fileLen != bLen {
        if fileLen > bLen {
            return fileLen - bLen
        }
        return bLen - fileLen
    }
    for i := 0; i < fileLen; i++ {
        if a[i] != b[i] {
            return i
        }
    }
    return -1
}

func readOutput(fileName string) []byte {
    b, err := ioutil.ReadFile(fileName)
    if err != nil {
        panic(err)
    }
    return b
}

// ioStyle 1 is of form "X.in"; ioStyle 2 is of form "I.X";
// TODO: Change function to check if list of files in "testDir" contains "X.in" or "I.X" files using regex
func setIOStyle() {
    if _, err := os.Stat("1.in"); err == nil {
        ioStyle = 1
        return
    } else if _, err = os.Stat("I.1"); err == nil {
        ioStyle = 2
        return
    }
    color.Red("Could not find test input/ouput files in " + testDir)
    os.Exit(0)
}

func runProgram() {
    cmd := exec.Command(progPath)
    err := cmd.Run()
    if err != nil {
        color.Red("Error running program:\n" + err.Error())
        os.Exit(0)
    }
}
<commit_msg>add support to execute python programs<commit_after>package main

import (
    "io/ioutil"
    "os"
    "os/exec"
    "strconv"
    "strings"
    "github.com/fatih/color"
)

var (
    progPath string
    progName string
    testDir string
    ioStyle int
    isPython bool
)

func main() {
    initVars()

    // chdir to directory containg test cases
    os.Chdir(testDir)

    setIOStyle()

    i := 1

    if ioStyle == 1 {
        for {
            if _, err := os.Stat(strconv.Itoa(i) + ".in"); os.IsNotExist(err) {
                // TODO: Add number of total test cases passed
                color.White("Finished testing")
                os.Exit(0)
            }

            copyTestData(strconv.Itoa(i) + ".in")

            runProgram()

            a := readOutput(progName + ".out")
            b := readOutput(strconv.Itoa(i) + ".out")

            if compareOutput(a, b) != -1 {
                color.Red("Wrong output for test case " + strconv.Itoa(i) + ":\nExpected:\n" + string(b[:]) + "Found:\n" + string(a[:]) + "\n")
            } else {
                color.Green("Test case " + strconv.Itoa(i) + " passed");
            }
            i++
        }
    } else if ioStyle == 2 {
        for {
            if _, err := os.Stat("I." + strconv.Itoa(i)); os.IsNotExist(err) {
                // TODO: Add number of total test cases passed
                color.White("Finished testing")
                os.Exit(0)
            }

            copyTestData("I." + strconv.Itoa(i))

            runProgram()

            a := readOutput(progName + ".out")
            b := readOutput("O." + strconv.Itoa(i))

            if compareOutput(a, b) != -1 {
                color.Red("Wrong output for test case " + strconv.Itoa(i) + ":\nExpected:\n" + string(b[:]) + "Found:\n" + string(a[:]) + "\n")
            } else {
                color.Green("Test case " + strconv.Itoa(i) + " passed");
            }
            i++
        }

    }
}

func initVars() {
    numArgs := len(os.Args)
    if numArgs >= 2 {
        progPath = os.Args[1]
    } else {
        color.Red("Please specify exectable as first argument")
        os.Exit(0)
    }

    if numArgs >= 3 {
        testDir = os.Args[2]
    } else {
        x, err := os.Getwd()
        if err != nil {
            color.Red("Error getting current working directory:\n" + err.Error())
            os.Exit(0)
        }
        testDir = x
    }

    progName = getProgName()

    if progName[len(progName) - 3:] == ".py" {
        isPython = true
        progName = progName[:len(progName) - 3]
    }
}

func getProgName() string {
    pos := strings.LastIndex(progPath, "/")
    if pos == -1 {
        pos = strings.LastIndex(progPath, "\\")
    }
    return progPath[pos + 1:]
}

func copyTestData(fileName string) {
    b, err := ioutil.ReadFile(fileName)
    if err != nil {
        panic(err)
    }

    err = ioutil.WriteFile(progName + ".in", b, 0644)
    if err != nil {
        panic(err)
    }
}

// if outputs are not same, return position of nonequivalence; otherwise return -1
func compareOutput(a, b []byte) int {
    fileLen := len(a)
    bLen := len(b)
    if fileLen != bLen {
        if fileLen > bLen {
            return fileLen - bLen
        }
        return bLen - fileLen
    }
    for i := 0; i < fileLen; i++ {
        if a[i] != b[i] {
            return i
        }
    }
    return -1
}

func readOutput(fileName string) []byte {
    b, err := ioutil.ReadFile(fileName)
    if err != nil {
        panic(err)
    }
    return b
}

// ioStyle 1 is of form "X.in"; ioStyle 2 is of form "I.X";
// TODO: Change function to check if list of files in "testDir" contains "X.in" or "I.X" files using regex
func setIOStyle() {
    if _, err := os.Stat("1.in"); err == nil {
        ioStyle = 1
        return
    } else if _, err = os.Stat("I.1"); err == nil {
        ioStyle = 2
        return
    }
    color.Red("Could not find test input/ouput files in " + testDir)
    os.Exit(0)
}

func runProgram() {
    cmd := exec.Command(progPath)
    if isPython {
        cmd = exec.Command("python", progPath)
    }
    err := cmd.Run()
    if err != nil {
        color.Red("Error running program:\n" + err.Error())
        os.Exit(0)
    }
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/fgrehm/devstep-cli/devstep"
	"os"
	"regexp"
)

var (
	project devstep.Project
	client  devstep.DockerClient
)

var commands = []cli.Command{
	buildCmd,
	hackCmd,
	runCmd,
	cleanCmd,
	infoCmd,
}

func projectRoot() string {
	pwd, err := os.Getwd()
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	return pwd
}

func homeDir() string {
	return os.Getenv("HOME")
}

func loadConfig() *devstep.ProjectConfig {
	client = devstep.NewClient("unix:///var/run/docker.sock")
	loader := devstep.NewConfigLoader(client, homeDir(), projectRoot())

	config, err := loader.Load()
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	return config
}

func newProject() devstep.Project {
	config := loadConfig()
	project, _ = devstep.NewProject(config)
	return project
}

func main() {
	app := cli.NewApp()
	app.Name = "devstep"
	app.Author = "Fábio Rehm"
	app.Email = "fgrehm@gmail.com"
	app.Usage = "development environments made easy"
	app.Version = "0.1.0"
	app.Flags = []cli.Flag{
		cli.BoolFlag{Name: "debug, d", Usage: "debug mode"},
	}
	app.Commands = commands

	app.Run(os.Args)
}

var buildCmd = cli.Command{
	Name:  "build",
	Usage: "build a docker image for the current project",
	Action: func(c *cli.Context) {
		devstep.Verbose(c.GlobalBool("debug"))

		err := newProject().Build(client)
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
	},
}

var hackCmd = cli.Command{
	Name:  "hack",
	Usage: "start a hacking session for the current project",
	Flags: []cli.Flag{
		cli.StringSliceFlag{Name: "p, publish", Value: &cli.StringSlice{}, Usage: "Publish a container's port to the host"},
		cli.StringSliceFlag{Name: "link", Value: &cli.StringSlice{}, Usage: "Add link to another container (name:alias)"},
	},
	Action: func(c *cli.Context) {
		devstep.Verbose(c.GlobalBool("debug"))

		runOpts := &devstep.DockerRunOpts{
			Publish: c.StringSlice("publish"),
			Links:   c.StringSlice("link"),
		}

		// Validate ports
		validPort := regexp.MustCompile(`^\d+:\d+$`)
		for _, port := range runOpts.Publish {
			if !validPort.MatchString(port) {
				fmt.Println("Invalid publish arg: " + port)
				os.Exit(1)
			}
		}
		// Validate links
		validLink := regexp.MustCompile(`[^:]+:[^:]+`)
		for _, link := range runOpts.Links {
			if !validLink.MatchString(link) {
				fmt.Println("Invalid link: " + link)
				os.Exit(1)
			}
		}

		err := newProject().Hack(client, runOpts)
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
	},
}

var runCmd = cli.Command{
	Name:  "run",
	Usage: "Run a one off command against the current base image",
	Flags: []cli.Flag{
		cli.StringSliceFlag{Name: "p, publish", Value: &cli.StringSlice{}, Usage: "Publish a container's port to the host"},
		cli.StringSliceFlag{Name: "link", Value: &cli.StringSlice{}, Usage: "Add link to another container (name:alias)"},
	},
	Action: func(c *cli.Context) {
		devstep.Verbose(c.GlobalBool("debug"))

		runOpts := &devstep.DockerRunOpts{
			Cmd:     c.Args(),
			Publish: c.StringSlice("publish"),
			Links:   c.StringSlice("link"),
		}

		// Validate command
		if len(runOpts.Cmd) == 0 {
			fmt.Println("No command provided to `devstep run`\n")
			cli.ShowCommandHelp(c, "run")
			os.Exit(1)
		}

		// Prepend a `--` so that it doesn't interfere with the current init
		// process args
		runOpts.Cmd = append([]string{"--"}, runOpts.Cmd...)

		// Validate ports
		validPort := regexp.MustCompile(`^\d+:\d+$`)
		for _, port := range runOpts.Publish {
			if !validPort.MatchString(port) {
				fmt.Println("Invalid publish arg: " + port)
				os.Exit(1)
			}
		}
		// Validate links
		validLink := regexp.MustCompile(`[^:]+:[^:]+`)
		for _, link := range runOpts.Links {
			if !validLink.MatchString(link) {
				fmt.Println("Invalid link: " + link)
				os.Exit(1)
			}
		}

		err := newProject().Run(client, runOpts)
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
	},
}

var cleanCmd = cli.Command{
	Name:  "clean",
	Usage: "remove previously built images for the current environment",
	Action: func(c *cli.Context) {
		devstep.Verbose(c.GlobalBool("debug"))

		err := newProject().Clean(client)
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
	},
}

var infoCmd = cli.Command{
	Name:  "info",
	Usage: "show information about the current environment",
	Action: func(c *cli.Context) {
		devstep.Verbose(c.GlobalBool("debug"))
		config := loadConfig()
		fmt.Printf("%+v\n", config)
		if config.Defaults != nil {
			fmt.Printf("%+v\n", config.Defaults)
		}
		if config.HackOpts != nil {
			fmt.Printf("%+v\n", config.HackOpts)
		}
	},
}
<commit_msg>Refactor some duplicated CLI code<commit_after>package main

import (
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/fgrehm/devstep-cli/devstep"
	"os"
	"regexp"
)

var (
	project devstep.Project
	client  devstep.DockerClient
)

var commands = []cli.Command{
	buildCmd,
	hackCmd,
	runCmd,
	cleanCmd,
	infoCmd,
}

var dockerRunFlags = []cli.Flag{
	cli.StringSliceFlag{Name: "p, publish", Value: &cli.StringSlice{}, Usage: "Publish a container's port to the host"},
	cli.StringSliceFlag{Name: "link", Value: &cli.StringSlice{}, Usage: "Add link to another container (name:alias)"},
}

func projectRoot() string {
	pwd, err := os.Getwd()
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	return pwd
}

func homeDir() string {
	return os.Getenv("HOME")
}

func loadConfig() *devstep.ProjectConfig {
	client = devstep.NewClient("unix:///var/run/docker.sock")
	loader := devstep.NewConfigLoader(client, homeDir(), projectRoot())

	config, err := loader.Load()
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	return config
}

func newProject() devstep.Project {
	config := loadConfig()
	project, _ = devstep.NewProject(config)
	return project
}

func main() {
	app := cli.NewApp()
	app.Name = "devstep"
	app.Author = "Fábio Rehm"
	app.Email = "fgrehm@gmail.com"
	app.Usage = "development environments made easy"
	app.Version = "0.1.0"
	app.Flags = []cli.Flag{
		cli.BoolFlag{Name: "debug, d", Usage: "debug mode"},
	}
	app.Commands = commands

	app.Run(os.Args)
}

var buildCmd = cli.Command{
	Name:  "build",
	Usage: "build a docker image for the current project",
	Action: func(c *cli.Context) {
		devstep.Verbose(c.GlobalBool("debug"))

		err := newProject().Build(client)
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
	},
}

var hackCmd = cli.Command{
	Name:  "hack",
	Usage: "start a hacking session for the current project",
	Flags: dockerRunFlags,
	Action: func(c *cli.Context) {
		devstep.Verbose(c.GlobalBool("debug"))

		runOpts := parseRunOpts(c)
		err := newProject().Hack(client, runOpts)
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
	},
}

var runCmd = cli.Command{
	Name:  "run",
	Usage: "Run a one off command against the current base image",
	Flags: dockerRunFlags,
	Action: func(c *cli.Context) {
		devstep.Verbose(c.GlobalBool("debug"))

		runOpts := parseRunOpts(c)
		runOpts.Cmd = c.Args()

		// Validate command
		if len(runOpts.Cmd) == 0 {
			fmt.Println("No command provided to `devstep run`\n")
			cli.ShowCommandHelp(c, "run")
			os.Exit(1)
		}

		// Prepend a `--` so that it doesn't interfere with the current init
		// process args
		runOpts.Cmd = append([]string{"--"}, runOpts.Cmd...)

		err := newProject().Run(client, runOpts)
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
	},
}

var cleanCmd = cli.Command{
	Name:  "clean",
	Usage: "remove previously built images for the current environment",
	Action: func(c *cli.Context) {
		devstep.Verbose(c.GlobalBool("debug"))

		err := newProject().Clean(client)
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
	},
}

var infoCmd = cli.Command{
	Name:  "info",
	Usage: "show information about the current environment",
	Action: func(c *cli.Context) {
		devstep.Verbose(c.GlobalBool("debug"))
		config := loadConfig()
		fmt.Printf("%+v\n", config)
		if config.Defaults != nil {
			fmt.Printf("%+v\n", config.Defaults)
		}
		if config.HackOpts != nil {
			fmt.Printf("%+v\n", config.HackOpts)
		}
	},
}

func parseRunOpts(c *cli.Context) *devstep.DockerRunOpts {
	runOpts := &devstep.DockerRunOpts{
		Publish: c.StringSlice("publish"),
		Links:   c.StringSlice("link"),
	}
	// Validate ports
	validPort := regexp.MustCompile(`^\d+:\d+$`)
	for _, port := range runOpts.Publish {
		if !validPort.MatchString(port) {
			fmt.Println("Invalid publish arg: " + port)
			os.Exit(1)
		}
	}
	// Validate links
	validLink := regexp.MustCompile(`[^:]+:[^:]+`)
	for _, link := range runOpts.Links {
		if !validLink.MatchString(link) {
			fmt.Println("Invalid link: " + link)
			os.Exit(1)
		}
	}
	return runOpts
}
<|endoftext|>


<commit_before>// This is the boot2docker management utilty.
package main

import (
	"os"

	// keep 3rd-party imports separate from stdlib with an empty line
	flag "github.com/ogier/pflag"
)

// The following vars will be injected during the build process.
var (
	Version string
	GitSHA  string
)

func main() {
	// os.Exit will terminate the program at the place of call without running
	// any deferred cleanup statements. It might cause unintended effects. To
	// be safe, we wrap the program in run() and only os.Exit() outside the
	// wrapper. Be careful not to indirectly trigger os.Exit() in the program,
	// notably via log.Fatal().
	os.Exit(run())
}

// Run the program and return exit code.
func run() int {
	flag.Usage = usageLong // make "-h" work similarly to "help"

	if err := config(); err != nil {
		errf("%s\n", err)
		return 1
	}

	switch cmd := flag.Arg(0); cmd {
	case "download":
		return cmdDownload()
	case "init":
		return cmdInit()
	case "up", "start", "boot", "resume":
		return cmdUp()
	case "save", "suspend":
		return cmdSave()
	case "down", "halt", "stop":
		return cmdStop()
	case "poweroff":
		return cmdPoweroff()
	case "restart":
		return cmdRestart()
	case "reset":
		return cmdReset()
	case "delete":
		return cmdDelete()
	case "info":
		return cmdInfo()
	case "status":
		return cmdStatus()
	case "ssh":
		return cmdSSH()
	case "version":
		outf("Client version: %s\nGit commit: %s\n", Version, GitSHA)
		return 0
	case "help":
		flag.Usage()
		return 0
	case "":
		usageShort()
		return 0
	default:
		errf("Unknown command %q\n", cmd)
		usageShort()
		return 1
	}
}

func usageShort() {
	errf("Usage: %s [<options>] {help|init|up|ssh|save|down|poweroff|reset|restart|status|info|delete|download|version} [<args>]\n", os.Args[0])

}

func usageLong() {
	// NOTE: the help message uses spaces, not tabs for indentation!
	errf(`boot2docker management utility.

Usage: %s [<options>] <command> [<args>]

Commands:
    init [<vm>]             Create a new boot2docker VM.
    up|start|boot [<vm>]    Start VM from any states.
    ssh                     Login to VM via SSH.
    save|suspend [<vm>]     Suspend VM and save state to disk.
    down|stop|halt [<vm>]   Gracefully shutdown the VM.
    restart [<vm>]          Gracefully reboot the VM.
    poweroff [<vm>]         Forcefully power off the VM (might corrupt disk image).
    reset [<vm>]            Forcefully power cycle the VM (might corrupt disk image).
    delete [<vm>]           Delete boot2docker VM and its disk image.
    info [<vm>]             Display detailed information of VM.
    status [<vm>]           Display current state of VM.
    download                Download boot2docker ISO image.
    version                 Display version information.

Options:
`, os.Args[0])
	flag.PrintDefaults()
}
<commit_msg>Fixed 3rd-party import url<commit_after>// This is the boot2docker management utilty.
package main

import (
	"os"

	// keep 3rd-party imports separate from stdlib with an empty line
	flag "github.com/riobard/pflag"
)

// The following vars will be injected during the build process.
var (
	Version string
	GitSHA  string
)

func main() {
	// os.Exit will terminate the program at the place of call without running
	// any deferred cleanup statements. It might cause unintended effects. To
	// be safe, we wrap the program in run() and only os.Exit() outside the
	// wrapper. Be careful not to indirectly trigger os.Exit() in the program,
	// notably via log.Fatal().
	os.Exit(run())
}

// Run the program and return exit code.
func run() int {
	flag.Usage = usageLong // make "-h" work similarly to "help"

	if err := config(); err != nil {
		errf("%s\n", err)
		return 1
	}

	switch cmd := flag.Arg(0); cmd {
	case "download":
		return cmdDownload()
	case "init":
		return cmdInit()
	case "up", "start", "boot", "resume":
		return cmdUp()
	case "save", "suspend":
		return cmdSave()
	case "down", "halt", "stop":
		return cmdStop()
	case "poweroff":
		return cmdPoweroff()
	case "restart":
		return cmdRestart()
	case "reset":
		return cmdReset()
	case "delete":
		return cmdDelete()
	case "info":
		return cmdInfo()
	case "status":
		return cmdStatus()
	case "ssh":
		return cmdSSH()
	case "version":
		outf("Client version: %s\nGit commit: %s\n", Version, GitSHA)
		return 0
	case "help":
		flag.Usage()
		return 0
	case "":
		usageShort()
		return 0
	default:
		errf("Unknown command %q\n", cmd)
		usageShort()
		return 1
	}
}

func usageShort() {
	errf("Usage: %s [<options>] {help|init|up|ssh|save|down|poweroff|reset|restart|status|info|delete|download|version} [<args>]\n", os.Args[0])

}

func usageLong() {
	// NOTE: the help message uses spaces, not tabs for indentation!
	errf(`boot2docker management utility.

Usage: %s [<options>] <command> [<args>]

Commands:
    init [<vm>]             Create a new boot2docker VM.
    up|start|boot [<vm>]    Start VM from any states.
    ssh                     Login to VM via SSH.
    save|suspend [<vm>]     Suspend VM and save state to disk.
    down|stop|halt [<vm>]   Gracefully shutdown the VM.
    restart [<vm>]          Gracefully reboot the VM.
    poweroff [<vm>]         Forcefully power off the VM (might corrupt disk image).
    reset [<vm>]            Forcefully power cycle the VM (might corrupt disk image).
    delete [<vm>]           Delete boot2docker VM and its disk image.
    info [<vm>]             Display detailed information of VM.
    status [<vm>]           Display current state of VM.
    download                Download boot2docker ISO image.
    version                 Display version information.

Options:
`, os.Args[0])
	flag.PrintDefaults()
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"sync"

	"github.com/docker/docker/client"
	"github.com/komuw/meli/api"
	"github.com/komuw/meli/cli"

	"gopkg.in/yaml.v2"
)

/* DOCS:
1. https://godoc.org/github.com/moby/moby/client
2. https://docs.docker.com/engine/api/v1.31/
*/

var version = "master"

func main() {
	followLogs, dockerComposeFile := cli.Cli()

	data, err := ioutil.ReadFile(dockerComposeFile)
	if err != nil {
		log.Fatal(err, " :unable to read docker-compose file")
	}

	var dockerCyaml api.DockerComposeConfig
	err = yaml.Unmarshal([]byte(data), &dockerCyaml)
	if err != nil {
		log.Fatal(err, " :unable to parse docker-compose file contents")
	}

	ctx := context.Background()
	cli, err := client.NewEnvClient()
	if err != nil {
		log.Fatal(err, " :unable to intialize docker client")
	}
	defer cli.Close()
	curentDir, err := os.Getwd()
	if err != nil {
		log.Fatal(err, " :unable to get the current working directory")
	}
	networkName := "meli_network_" + api.GetCwdName(curentDir)
	networkID, err := api.GetNetwork(ctx, networkName, cli)
	if err != nil {
		log.Fatal(err, " :unable to create/get network")
	}
	api.GetAuth()

	// Create top level volumes, if any
	if len(dockerCyaml.Volumes) > 0 {
		for k := range dockerCyaml.Volumes {
			// TODO we need to synchronise here else we'll get a race
			// but I think we can get away for now because:
			// 1. there are on average a lot more containers in a compose file
			// than volumes, so the sync in the for loop for containers is enough
			// 2. since we intend to stream logs as containers run(see; issues/24);
			// then meli will be up long enough for the volume creation goroutines to have finished.
			go api.CreateDockerVolume(ctx, "meli_"+k, "local", cli)
		}
	}

	var wg sync.WaitGroup
	for k, v := range dockerCyaml.Services {
		wg.Add(1)
		v.Labels = append(v.Labels, fmt.Sprintf("meli_service=meli_%s", k))
		//go fakestartContainers(ctx, k, v, networkID, networkName, &wg, followLogs,  dockerComposeFile)
		go startContainers(
			ctx,
			k,
			v,
			networkID,
			networkName,
			&wg,
			followLogs,
			dockerComposeFile,
			cli)
	}
	wg.Wait()
}

func fakestartContainers(
	ctx context.Context,
	k string,
	s api.ServiceConfig,
	networkName, networkID string,
	wg *sync.WaitGroup,
	followLogs bool,
	dockerComposeFile string) {
	defer wg.Done()
}

func startContainers(
	ctx context.Context,
	k string,
	s api.ServiceConfig,
	networkID, networkName string,
	wg *sync.WaitGroup,
	followLogs bool,
	dockerComposeFile string,
	cli *client.Client) {
	defer wg.Done()

	/*
		1. Pull Image
		2. Create a container
		3. Connect container to network
		4. Start container
		5. Stream container logs
	*/

	formattedContainerName := api.FormatContainerName(k)
	if len(s.Image) > 0 {
		err := api.PullDockerImage(ctx, s.Image, cli)
		if err != nil {
			// clean exit since we want other goroutines for fetching other images
			// to continue running
			log.Printf("\n\t service=%s error=%s", k, err)
			return
		}
	}
	alreadyCreated, containerID, err := api.CreateContainer(
		ctx,
		s,
		networkName,
		formattedContainerName,
		dockerComposeFile,
		cli)
	if err != nil {
		// clean exit since we want other goroutines for fetching other images
		// to continue running
		log.Printf("\n\t service=%s error=%s", k, err)
		return
	}

	if !alreadyCreated {
		err = api.ConnectNetwork(
			ctx,
			networkID,
			containerID,
			cli)
		if err != nil {
			// create whitespace so that error is visible to human
			log.Printf("\n\t service=%s error=%s", k, err)
			return
		}
	}

	err = api.ContainerStart(
		ctx,
		containerID,
		cli)
	if err != nil {
		log.Printf("\n\t service=%s error=%s", k, err)
		return
	}

	err = api.ContainerLogs(
		ctx,
		containerID,
		followLogs,
		cli)
	if err != nil {
		log.Printf("\n\t service=%s error=%s", k, err)
		return
	}
}
<commit_msg>rm unused func<commit_after>package main

import (
	"context"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"sync"

	"github.com/docker/docker/client"
	"github.com/komuw/meli/api"
	"github.com/komuw/meli/cli"

	"gopkg.in/yaml.v2"
)

/* DOCS:
1. https://godoc.org/github.com/moby/moby/client
2. https://docs.docker.com/engine/api/v1.31/
*/

var version = "master"

func main() {
	followLogs, dockerComposeFile := cli.Cli()

	data, err := ioutil.ReadFile(dockerComposeFile)
	if err != nil {
		log.Fatal(err, " :unable to read docker-compose file")
	}

	var dockerCyaml api.DockerComposeConfig
	err = yaml.Unmarshal([]byte(data), &dockerCyaml)
	if err != nil {
		log.Fatal(err, " :unable to parse docker-compose file contents")
	}

	ctx := context.Background()
	cli, err := client.NewEnvClient()
	if err != nil {
		log.Fatal(err, " :unable to intialize docker client")
	}
	defer cli.Close()
	curentDir, err := os.Getwd()
	if err != nil {
		log.Fatal(err, " :unable to get the current working directory")
	}
	networkName := "meli_network_" + api.GetCwdName(curentDir)
	networkID, err := api.GetNetwork(ctx, networkName, cli)
	if err != nil {
		log.Fatal(err, " :unable to create/get network")
	}
	api.GetAuth()

	// Create top level volumes, if any
	if len(dockerCyaml.Volumes) > 0 {
		for k := range dockerCyaml.Volumes {
			// TODO we need to synchronise here else we'll get a race
			// but I think we can get away for now because:
			// 1. there are on average a lot more containers in a compose file
			// than volumes, so the sync in the for loop for containers is enough
			// 2. since we intend to stream logs as containers run(see; issues/24);
			// then meli will be up long enough for the volume creation goroutines to have finished.
			go api.CreateDockerVolume(ctx, "meli_"+k, "local", cli)
		}
	}

	var wg sync.WaitGroup
	for k, v := range dockerCyaml.Services {
		wg.Add(1)
		v.Labels = append(v.Labels, fmt.Sprintf("meli_service=meli_%s", k))

		go startContainers(
			ctx,
			k,
			v,
			networkID,
			networkName,
			&wg,
			followLogs,
			dockerComposeFile,
			cli)
	}
	wg.Wait()
}

func startContainers(
	ctx context.Context,
	k string,
	s api.ServiceConfig,
	networkID, networkName string,
	wg *sync.WaitGroup,
	followLogs bool,
	dockerComposeFile string,
	cli *client.Client) {
	defer wg.Done()

	/*
		1. Pull Image
		2. Create a container
		3. Connect container to network
		4. Start container
		5. Stream container logs
	*/

	formattedContainerName := api.FormatContainerName(k)
	if len(s.Image) > 0 {
		err := api.PullDockerImage(ctx, s.Image, cli)
		if err != nil {
			// clean exit since we want other goroutines for fetching other images
			// to continue running
			log.Printf("\n\t service=%s error=%s", k, err)
			return
		}
	}
	alreadyCreated, containerID, err := api.CreateContainer(
		ctx,
		s,
		networkName,
		formattedContainerName,
		dockerComposeFile,
		cli)
	if err != nil {
		// clean exit since we want other goroutines for fetching other images
		// to continue running
		log.Printf("\n\t service=%s error=%s", k, err)
		return
	}

	if !alreadyCreated {
		err = api.ConnectNetwork(
			ctx,
			networkID,
			containerID,
			cli)
		if err != nil {
			// create whitespace so that error is visible to human
			log.Printf("\n\t service=%s error=%s", k, err)
			return
		}
	}

	err = api.ContainerStart(
		ctx,
		containerID,
		cli)
	if err != nil {
		log.Printf("\n\t service=%s error=%s", k, err)
		return
	}

	err = api.ContainerLogs(
		ctx,
		containerID,
		followLogs,
		cli)
	if err != nil {
		log.Printf("\n\t service=%s error=%s", k, err)
		return
	}
}
<|endoftext|>


<commit_before>package main

import "log"

func main() {
	log.Println("Hello world!")
}
<commit_msg>added example sdl2 code<commit_after>package main

import (
	"log"

	"github.com/veandco/go-sdl2/sdl"
)

func main() {
	log.Println("Hello world!")

	sdl.Init(sdl.INIT_EVERYTHING)

	window, err := sdl.CreateWindow("test", sdl.WINDOWPOS_UNDEFINED, sdl.WINDOWPOS_UNDEFINED,
		800, 600, sdl.WINDOW_SHOWN)
	if err != nil {
		panic(err)
	}
	defer window.Destroy()

	surface, err := window.GetSurface()
	if err != nil {
		panic(err)
	}

	rect := sdl.Rect{0, 0, 200, 200}
	surface.FillRect(&rect, 0xffff0000)
	window.UpdateSurface()

	sdl.Delay(1000)
	sdl.Quit()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"
	"strings"

	"github.com/codegangsta/negroni"
	"github.com/unrolled/render"
	"github.com/xyproto/permissions"
	"github.com/xyproto/webhandle"
)

const (
	Version = "0.1"
)

func main() {
	fmt.Println("dashboard ", Version)

	r := render.New(render.Options{})

	n := negroni.Classic()

	perm := permissions.New()
	userstate := perm.UserState()

	mux := http.NewServeMux()

	mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		data := map[string]string{
			"version_string": "0.1",
			"server":         "server",
			"status":         "up",
			"shortname":      "dx",
			"statuscolor":    "#a0a0ff",
		}

		// !! Reload template !!
		r = render.New(render.Options{})

		// Render and return
		r.HTML(w, http.StatusOK, "index", data)
	})

	mux.HandleFunc("/status", func(w http.ResponseWriter, req *http.Request) {
		fmt.Fprintf(w, "Has user bob: %v\nLogged in on server: %v\n", userstate.HasUser("bob"), userstate.IsLoggedIn("bob"))
		fmt.Fprintf(w, "Is confirmed: %v\n", userstate.IsConfirmed("bob"))
		fmt.Fprintf(w, "Current user is logged in, has a valid cookie and *user rights*: %v\n", userstate.UserRights(req))
		fmt.Fprintf(w, "Current user is logged in, has a valid cookie and *admin rights*: %v\n", userstate.AdminRights(req))
		fmt.Fprintf(w, "Username stored in cookies (or blank): %v\n", userstate.GetUsername(req))
		fmt.Fprint(w, "\nTry: /register, /confirm, /remove, /login, /logout, /makeadmin and /admin")
	})

	mux.HandleFunc("/hello/", func(w http.ResponseWriter, req *http.Request) {
		fmt.Fprintf(w, "Hello, %v", webhandle.GetLast(req.URL))
	})

	mux.HandleFunc("/register", func(w http.ResponseWriter, req *http.Request) {
		userstate.AddUser("bob", "hunter1", "bob@zombo.com")
		fmt.Fprintf(w, "User bob was created: %v\n", userstate.HasUser("bob"))
	})

	mux.HandleFunc("/confirm", func(w http.ResponseWriter, req *http.Request) {
		userstate.MarkConfirmed("bob")
		fmt.Fprintf(w, "User bob was confirmed: %v\n", userstate.IsConfirmed("bob"))
	})

	mux.HandleFunc("/remove", func(w http.ResponseWriter, req *http.Request) {
		userstate.RemoveUser("bob")
		fmt.Fprintf(w, "User bob was removed: %v\n", !userstate.HasUser("bob"))
	})

	mux.HandleFunc("/login", func(w http.ResponseWriter, req *http.Request) {
		userstate.Login(w, "bob")
		fmt.Fprintf(w, "bob is now logged in: %v\n", userstate.IsLoggedIn("bob"))
	})

	mux.HandleFunc("/logout", func(w http.ResponseWriter, req *http.Request) {
		userstate.Logout("bob")
		fmt.Fprintf(w, "bob is now logged out: %v\n", !userstate.IsLoggedIn("bob"))
	})

	mux.HandleFunc("/makeadmin", func(w http.ResponseWriter, req *http.Request) {
		userstate.SetAdminStatus("bob")
		fmt.Fprintf(w, "bob is now administrator: %v\n", userstate.IsAdmin("bob"))
	})

	mux.HandleFunc("/data", func(w http.ResponseWriter, req *http.Request) {
		fmt.Fprintf(w, "user page that only logged in users must see!")
	})

	mux.HandleFunc("/admin", func(w http.ResponseWriter, req *http.Request) {
		fmt.Fprintf(w, "super secret information that only logged in administrators must see!\n\n")
		usernames, err := userstate.GetAllUsernames()
		if err == nil {
			fmt.Fprintf(w, "list of all users: "+strings.Join(usernames, ", "))
		}
	})

	// Publish the generated SVG as "/circles.svg"
	svgurl := "/circles.svg"
	mux.HandleFunc(svgurl, func(w http.ResponseWriter, req *http.Request) {
		w.Header().Add("Content-Type", "image/svg+xml")
		fmt.Fprint(w, svgPage().String())
	})

	// Generate a Page that includes the svg image
	page := indexPage(svgurl)
	// Publish the generated Page in a way that connects the HTML and CSS
	page.Publish(mux, "/html", "/style.css", false)

	// Permission middleware

	perm.SetDenyFunction(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "Permission denied!")
	})

	n.Use(perm)

	// Share the files in static
	n.Use(negroni.NewStatic(http.Dir("static")))

	n.UseHandler(mux)

	n.Run(":3003")
}
<commit_msg>Commented out code that is not needed for negroni Classic<commit_after>package main

import (
	"fmt"
	"net/http"
	"strings"

	"github.com/codegangsta/negroni"
	"github.com/unrolled/render"
	"github.com/xyproto/permissions"
	"github.com/xyproto/webhandle"
)

const (
	Version = "0.1"
)

func main() {
	fmt.Println("dashboard ", Version)

	r := render.New(render.Options{})

	n := negroni.Classic()

	perm := permissions.New()
	userstate := perm.UserState()

	mux := http.NewServeMux()

	mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		data := map[string]string{
			"version_string": "0.1",
			"server":         "server",
			"status":         "up",
			"shortname":      "dx",
			"statuscolor":    "#a0a0ff",
		}

		// !! Reload template !!
		r = render.New(render.Options{})

		// Render and return
		r.HTML(w, http.StatusOK, "index", data)
	})

	mux.HandleFunc("/status", func(w http.ResponseWriter, req *http.Request) {
		fmt.Fprintf(w, "Has user bob: %v\nLogged in on server: %v\n", userstate.HasUser("bob"), userstate.IsLoggedIn("bob"))
		fmt.Fprintf(w, "Is confirmed: %v\n", userstate.IsConfirmed("bob"))
		fmt.Fprintf(w, "Current user is logged in, has a valid cookie and *user rights*: %v\n", userstate.UserRights(req))
		fmt.Fprintf(w, "Current user is logged in, has a valid cookie and *admin rights*: %v\n", userstate.AdminRights(req))
		fmt.Fprintf(w, "Username stored in cookies (or blank): %v\n", userstate.GetUsername(req))
		fmt.Fprint(w, "\nTry: /register, /confirm, /remove, /login, /logout, /makeadmin and /admin")
	})

	mux.HandleFunc("/hello/", func(w http.ResponseWriter, req *http.Request) {
		fmt.Fprintf(w, "Hello, %v", webhandle.GetLast(req.URL))
	})

	mux.HandleFunc("/register", func(w http.ResponseWriter, req *http.Request) {
		userstate.AddUser("bob", "hunter1", "bob@zombo.com")
		fmt.Fprintf(w, "User bob was created: %v\n", userstate.HasUser("bob"))
	})

	mux.HandleFunc("/confirm", func(w http.ResponseWriter, req *http.Request) {
		userstate.MarkConfirmed("bob")
		fmt.Fprintf(w, "User bob was confirmed: %v\n", userstate.IsConfirmed("bob"))
	})

	mux.HandleFunc("/remove", func(w http.ResponseWriter, req *http.Request) {
		userstate.RemoveUser("bob")
		fmt.Fprintf(w, "User bob was removed: %v\n", !userstate.HasUser("bob"))
	})

	mux.HandleFunc("/login", func(w http.ResponseWriter, req *http.Request) {
		userstate.Login(w, "bob")
		fmt.Fprintf(w, "bob is now logged in: %v\n", userstate.IsLoggedIn("bob"))
	})

	mux.HandleFunc("/logout", func(w http.ResponseWriter, req *http.Request) {
		userstate.Logout("bob")
		fmt.Fprintf(w, "bob is now logged out: %v\n", !userstate.IsLoggedIn("bob"))
	})

	mux.HandleFunc("/makeadmin", func(w http.ResponseWriter, req *http.Request) {
		userstate.SetAdminStatus("bob")
		fmt.Fprintf(w, "bob is now administrator: %v\n", userstate.IsAdmin("bob"))
	})

	mux.HandleFunc("/data", func(w http.ResponseWriter, req *http.Request) {
		fmt.Fprintf(w, "user page that only logged in users must see!")
	})

	mux.HandleFunc("/admin", func(w http.ResponseWriter, req *http.Request) {
		fmt.Fprintf(w, "super secret information that only logged in administrators must see!\n\n")
		usernames, err := userstate.GetAllUsernames()
		if err == nil {
			fmt.Fprintf(w, "list of all users: "+strings.Join(usernames, ", "))
		}
	})

	// Publish the generated SVG as "/circles.svg"
	svgurl := "/circles.svg"
	mux.HandleFunc(svgurl, func(w http.ResponseWriter, req *http.Request) {
		w.Header().Add("Content-Type", "image/svg+xml")
		fmt.Fprint(w, svgPage().String())
	})

	// Generate a Page that includes the svg image
	page := indexPage(svgurl)
	// Publish the generated Page in a way that connects the HTML and CSS
	page.Publish(mux, "/html", "/style.css", false)

	// Permission middleware

	perm.SetDenyFunction(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "Permission denied!")
	})

	// Activate the permission middleware

	n.Use(perm)

	// Already included in negroni.Classic:
	//// Share the files in static
	//n.Use(negroni.NewStatic(http.Dir("static")))

	n.UseHandler(mux)

	n.Run(":3000")
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"crypto/x509"
	"encoding/hex"
	"encoding/pem"
	"fmt"
	"strings"
	"io/ioutil"
	"golang.org/x/crypto/pkcs12"
)

func main() {
	file := os.Args[1]

	certs := getCerts(file)
	for i := range certs {
		displayCert(certs[i])
	}
}

func getCerts(file string) []*x509.Certificate {
	var certs []*x509.Certificate
	data, _ := ioutil.ReadFile(file)
	if strings.HasSuffix(file, "pem") {
		block, data := pem.Decode(data)
		for block != nil {
			cert, _ := x509.ParseCertificates(block.Bytes)
			certs = append(certs, cert[0])
			block, data = pem.Decode(data)
		}
	} else if strings.HasSuffix(file, "p12") {
		_, cert, _ := pkcs12.Decode(data, "password")
		certs = append(certs, cert)
	}
	return certs
}

func displayCert(cert *x509.Certificate) {

	//Expiry Date
	fmt.Println("Expiry Date:", cert.NotAfter)

	//Algorithm
	fmt.Println("Algorithm Type:", cert.SignatureAlgorithm)

	//Subject Info
	fmt.Println("Subject Info:")
	fmt.Println("	CommonName:", cert.Subject.CommonName)
	fmt.Println("	Organization:", cert.Subject.Organization)
	fmt.Println("	OrganizationalUnit:", cert.Subject.OrganizationalUnit)
	fmt.Println("	Country:", cert.Subject.Country)
	fmt.Println("	Locality:", cert.Subject.Locality)

	//Issuer Info
	fmt.Println("Issuer Info:")
	fmt.Println("	CommonName:", cert.Issuer.CommonName)
	fmt.Println("	Organization:", cert.Issuer.Organization)
	fmt.Println("	OrganizationalUnit:", cert.Issuer.OrganizationalUnit)
	fmt.Println("	Country:", cert.Issuer.Country)
	fmt.Println("	Locality:", cert.Issuer.Locality)

	//Subject Key ID
	fmt.Print("Subject Key ID: ")
	for i := 0; i < len(cert.SubjectKeyId); i++ {
		fmt.Print(hex.EncodeToString(cert.SubjectKeyId[i : i+1]))
		if i < len(cert.SubjectKeyId)-1 {
			fmt.Print(":")
		}
	}

	//Authority Key ID
	fmt.Print("\nAuthority Key ID: ")
	for i := 0; i < len(cert.AuthorityKeyId); i++ {
		fmt.Print(hex.EncodeToString(cert.AuthorityKeyId[i : i+1]))
		if i < len(cert.AuthorityKeyId)-1 {
			fmt.Print(":")
		}
	}

	//SANs, (alternate DNS Names)
	fmt.Println("\nAlternate DNS Names:", cert.DNSNames)

	//Serial Number
	fmt.Println("Serial Number:", cert.SerialNumber)
	fmt.Println("\n")
}
<commit_msg>main.go wasn't added to the previous commit? So here's that.<commit_after>package main

import (
	"crypto/x509"
	"encoding/hex"
	"encoding/pem"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"golang.org/x/crypto/pkcs12"
)

func main() {
	file := os.Args[1]

	certs := getCerts(file)
	for i := range certs {
		displayCert(certs[i])
	}
}

func getCerts(file string) []*x509.Certificate {
	var certs []*x509.Certificate
	data, _ := ioutil.ReadFile(file)
	if strings.HasSuffix(file, "pem") {
		block, data := pem.Decode(data)
		for block != nil {
			cert, _ := x509.ParseCertificates(block.Bytes)
			certs = append(certs, cert[0])
			block, data = pem.Decode(data)
		}
	} else if strings.HasSuffix(file, "p12") {
		_, cert, _ := pkcs12.Decode(data, "password")
		certs = append(certs, cert)
	}
	return certs
}

func displayCert(cert *x509.Certificate) {

	//Expiry Date
	fmt.Println("Expiry Date:", cert.NotAfter)

	//Algorithm
	fmt.Println("Algorithm Type:", cert.SignatureAlgorithm)

	//Subject Info
	fmt.Println("Subject Info:")
	fmt.Println("	CommonName:", cert.Subject.CommonName)
	fmt.Println("	Organization:", cert.Subject.Organization)
	fmt.Println("	OrganizationalUnit:", cert.Subject.OrganizationalUnit)
	fmt.Println("	Country:", cert.Subject.Country)
	fmt.Println("	Locality:", cert.Subject.Locality)

	//Issuer Info
	fmt.Println("Issuer Info:")
	fmt.Println("	CommonName:", cert.Issuer.CommonName)
	fmt.Println("	Organization:", cert.Issuer.Organization)
	fmt.Println("	OrganizationalUnit:", cert.Issuer.OrganizationalUnit)
	fmt.Println("	Country:", cert.Issuer.Country)
	fmt.Println("	Locality:", cert.Issuer.Locality)

	//Subject Key ID
	fmt.Print("Subject Key ID: ")
	for i := 0; i < len(cert.SubjectKeyId); i++ {
		fmt.Print(hex.EncodeToString(cert.SubjectKeyId[i : i+1]))
		if i < len(cert.SubjectKeyId)-1 {
			fmt.Print(":")
		}
	}

	//Authority Key ID
	fmt.Print("\nAuthority Key ID: ")
	for i := 0; i < len(cert.AuthorityKeyId); i++ {
		fmt.Print(hex.EncodeToString(cert.AuthorityKeyId[i : i+1]))
		if i < len(cert.AuthorityKeyId)-1 {
			fmt.Print(":")
		}
	}

	//SANs, (alternate DNS Names)
	fmt.Println("\nAlternate DNS Names:", cert.DNSNames)

	//Serial Number
	fmt.Println("Serial Number:", cert.SerialNumber)
	fmt.Println("\n")
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	client "github.com/influxdata/influxdb/client/v2"
	"github.com/kardianos/osext"
	snmp "github.com/paulstuart/snmputil"
	"gopkg.in/gcfg.v1"
)

const layout = "2006-01-02 15:04:05"

// SnmpConfig specifies the snmp device to probe
type SnmpConfig struct {
	Host      string `gcfg:"host"`
	Community string `gcfg:"community"`
	Version   string `gcfg:"version"`
	Port      int    `gcfg:"port"`
	Retries   int    `gcfg:"retries"`
	Timeout   int    `gcfg:"timeout"`
	Freq      int    `gcfg:"freq"`
	Aliases   string `gcfg:"aliases"`
	Config    string `gcfg:"config"`
	Mibs      string `gcfg:"mibs"`
	Tags      string `gcfg:"tags"`
}

// CommonConfig specifies general parameters
type CommonConfig struct {
	HTTPPort int    `gcfg:"httpPort"`
	LogDir   string `gcfg:"logDir"`
	OidFile  string `gcfg:"oidFile"`
	Tags     string `gcfg:"tags"`
	Mibs     string `gcfg:"mibs"`
	MibFile  string `gcfg:"mibfile"`
}

// MibConfig specifies what OIDs to query
type MibConfig struct {
	Scalers bool     `gcfg:"scalers"`
	Name    string   `gcfg:"name"`
	Index   string   `gcfg:"index"`
	Columns []string `gcfg:"column"`
	Regexps []string `gcfg:"regexp"`
	Keep    bool     `gcfg:"keep"`
}

// InfluxConfig defines connection requirements
type InfluxConfig struct {
	Host        string `gcfg:"host"`
	Port        int    `gcfg:"port"`
	Database    string `gcfg:"database"`
	Username    string `gcfg:"username"`
	Password    string `gcfg:"password"`
	Retention   string `gcfg:"retention"`
	Consistency string `gcfg:"consistency"`
	SkipVerify  bool   `gcfg:"skip_verify"`
	Timeout     int    `gcfg:"timeout"`
	BatchSize   int    `gcfg:"batchSize"`
	QueueSize   int    `gcfg:"queueSize"`
	Flush       int    `gcfg:"flush"`
}

type snmpStats struct {
	GetCnt    int
	ErrCnt    int
	LastError error
	LastTime  time.Time
}

type statsFn func() snmpStats

// SystemStatus provides operating statistics
type SystemStatus struct {
	Period, Started string
	Uptime          string
	DB, LogFile     string
	SNMP            map[string]*SnmpConfig
	Influx          map[string]*InfluxConfig
	SnmpStats       map[string]snmpStats
}

var (
	wg            sync.WaitGroup
	quit          = make(chan struct{})
	verbose       bool
	startTime     = time.Now()
	snmpNames     bool
	sample        bool
	dump          bool
	httpPort      = 8080
	appdir, _     = osext.ExecutableFolder()
	logDir        = filepath.Join(appdir, "log")
	oidFile       = filepath.Join(appdir, "oids.json")
	configFile    = filepath.Join(appdir, "config.gcfg")
	errorLog      *os.File
	errorDuration = time.Duration(10 * time.Minute)
	errorPeriod   = errorDuration.String()
	errorMax      = 100
	errorName     string
	mibs          string
	senders       = map[string]Sender{}
	statsMap      = make(map[string]statsFn)
	logger        *log.Logger
	commonTags    map[string]string

	cfg = struct {
		Snmp   map[string]*SnmpConfig
		Mibs   map[string]*MibConfig
		Influx map[string]*InfluxConfig
		Common CommonConfig
	}{}
)

func getTags(list string) map[string]string {
	t := make(map[string]string)
	for _, item := range strings.Fields(list) {
		if pair := strings.Split(item, "="); len(pair) == 2 {
			t[pair[0]] = pair[1]
		}
	}
	return t
}

func status() SystemStatus {
	stats := map[string]snmpStats{}
	for name, fn := range statsMap {
		stats[name] = fn()
	}
	return SystemStatus{
		LogFile:   errorName,
		Started:   startTime.Format(layout),
		Uptime:    time.Now().Sub(startTime).String(),
		Period:    errorPeriod,
		SNMP:      cfg.Snmp,
		Influx:    cfg.Influx,
		SnmpStats: stats,
	}
}

func flags() *flag.FlagSet {
	var f flag.FlagSet
	f.BoolVar(&snmpNames, "names", snmpNames, "print column names and exit")
	f.BoolVar(&sample, "sample", sample, "print a sample of collected values and exit")
	f.BoolVar(&dump, "dump", dump, "print output of parsed mibs and exit")
	f.StringVar(&configFile, "config", configFile, "config file")
	f.BoolVar(&verbose, "verbose", verbose, "verbose mode")
	f.IntVar(&httpPort, "http", httpPort, "http port")
	f.StringVar(&logDir, "logs", logDir, "log directory")
	f.StringVar(&oidFile, "oids", oidFile, "OIDs file")
	f.StringVar(&mibs, "mibs", mibs, "mibs to use")
	f.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
		f.VisitAll(func(flag *flag.Flag) {
			format := "%10s: %s\n"
			fmt.Fprintf(os.Stderr, format, "-"+flag.Name, flag.Usage)
		})
		fmt.Fprintf(os.Stderr, "\nAll settings can be set in config file: %s\n", configFile)
		os.Exit(1)

	}
	return &f
}

func init() {
	log.SetOutput(os.Stderr)

	// parse first time to see if config file is being specified
	f := flags()
	f.Parse(os.Args[1:])

	// now load up config settings
	if _, err := os.Stat(configFile); err != nil {
		log.Fatal(err)
	}
	data, err := ioutil.ReadFile(configFile)
	if err != nil {
		log.Fatal(err)
	}
	err = gcfg.ReadStringInto(&cfg, string(data))
	if err != nil {
		log.Fatalf("Failed to parse gcfg data: %s", err)
	}
	httpPort = cfg.Common.HTTPPort

	if len(cfg.Common.LogDir) > 0 {
		logDir = cfg.Common.LogDir
	}
	if len(cfg.Common.OidFile) > 0 {
		oidFile = cfg.Common.OidFile
	}

	commonTags = getTags(cfg.Common.Tags)

	// re-read cmd line args to override as indicated
	f = flags()
	f.Parse(os.Args[1:])
	os.Mkdir(logDir, 0755)

	// Load or generate mib data
	if len(cfg.Common.MibFile) > 0 {
		if err := snmp.CachedMibInfo(cfg.Common.MibFile, cfg.Common.Mibs); err != nil {
			panic(err)
		}
	} else {
		if err := snmp.LoadMibs(cfg.Common.Mibs); err != nil {
			panic(err)
		}
	}

	for name, c := range cfg.Influx {
		sender, err := makeSender(c)
		if err != nil {
			panic(err)
		}
		senders[name] = sender
	}
	if verbose {
		logger = log.New(os.Stderr, "", 0)
	}

	var ferr error
	errorName = fmt.Sprintf("error.%d.log", httpPort)
	errorPath := filepath.Join(logDir, errorName)
	errorLog, ferr = os.OpenFile(errorPath, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0664)
	if ferr != nil {
		log.Fatal("Can't open error log:", ferr)
	}
}

func errFn(err error) {
	log.Println(err)
}

func makeSender(cfg *InfluxConfig) (Sender, error) {
	conf := client.HTTPConfig{
		Addr:               fmt.Sprintf("http://%s:%d", cfg.Host, cfg.Port),
		Username:           cfg.Username,
		Password:           cfg.Password,
		Timeout:            (time.Duration(cfg.Timeout) * time.Second),
		InsecureSkipVerify: cfg.SkipVerify,
	}
	batch := client.BatchPointsConfig{
		Precision:        "s",
		Database:         cfg.Database,
		RetentionPolicy:  cfg.Retention,
		WriteConsistency: cfg.Consistency,
	}

	return NewSender(conf, batch, cfg.BatchSize, cfg.QueueSize, cfg.Flush, errFn)
}

func pairs(s string) map[string]string {
	m := make(map[string]string)
	for _, p := range strings.Fields(s) {
		b := strings.Split(p, "=")
		if len(b) == 2 {
			m[b[0]] = b[1]
		}
	}
	return m
}

func gather(send Sender, c *SnmpConfig, mib *MibConfig) {
	if c.Freq < 1 {
		panic("invalid polling frequency for: " + c.Host)
	}

	regexps := make([]string, 0, len(mib.Regexps))
	for _, r := range mib.Regexps {
		for _, x := range strings.Fields(r) {
			regexps = append(regexps, x)
		}
	}

	crit := snmp.Criteria{
		OID:     mib.Name,
		Index:   mib.Index,
		Regexps: regexps,
		Keep:    mib.Keep,
		Tags:    getTags(c.Tags),
		Freq:    c.Freq,
		Aliases: pairs(c.Aliases),
	}

	for k, v := range commonTags {
		crit.Tags[k] = v
	}

	p := snmp.Profile{
		Host:      c.Host,
		Community: c.Community,
		Version:   c.Version,
		Port:      c.Port,
		Retries:   c.Retries,
		Timeout:   c.Timeout,
	}

	if dump {
		if err := snmp.OIDList(mibs, os.Stdout); err != nil {
			panic(err)
		}
		return
	}
	if sample {
		sender, _ := snmp.DebugSender(nil, nil)
		wg.Add(1)
		go func() {
			if err := snmp.Sampler(p, crit, sender); err != nil {
				panic(err)
			}
			wg.Done()
		}()
		return
	}

	sender := func(name string, tags map[string]string, value interface{}, when time.Time) error {
		values := map[string]interface{}{"value": value}
		return send(name, tags, values, when)
	}

	// influxdb saves uint64 as a string
	// so this is a workaround for now
	sender = snmp.IntegerSender(sender)

	var stats snmpStats
	var m sync.Mutex

	errFn := func(err error) {
		m.Lock()
		if err == nil {
			stats.GetCnt++
		} else {
			stats.ErrCnt++
			stats.LastError = err
			stats.LastTime = time.Now()
		}
		m.Unlock()
	}
	if err := snmp.Bulkwalker(p, crit, sender, errFn, logger); err != nil {
		panic(err)
	}

	name := fmt.Sprintf("%s/%s", c.Host, mib.Name)
	statsMap[name] = func() snmpStats {
		m.Lock()
		defer m.Unlock()
		return stats
	}
}

func main() {
	for name, c := range cfg.Snmp {
		send, ok := senders[name]
		if !ok {
			send, ok = senders["*"]
			if !ok {
				panic("No sender for: " + name)
			}
		}
		if len(c.Mibs) > 0 {
			for _, m := range strings.Fields(c.Mibs) {
				mib, ok := cfg.Mibs[m]
				if !ok {
					panic("no mib config found for: " + m)
				}
				gather(send, c, mib)
			}
			continue
		}
		mib, ok := cfg.Mibs[name]
		if !ok {
			if mib, ok = cfg.Mibs["*"]; !ok {
				panic("no mib found for: " + name)
			}
		}
		gather(send, c, mib)
	}
	if sample {
		wg.Wait()
		return
	}
	if httpPort > 0 {
		webServer(httpPort)
	} else {
		<-quit
	}
	errorLog.Close()
}
<commit_msg>allow creating filtered mib data files, clean up code<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	client "github.com/influxdata/influxdb/client/v2"
	"github.com/kardianos/osext"
	snmp "github.com/paulstuart/snmputil"
	"gopkg.in/gcfg.v1"
)

const layout = "2006-01-02 15:04:05"

// SnmpConfig specifies the snmp device to probe
type SnmpConfig struct {
	Host      string `gcfg:"host"`
	Community string `gcfg:"community"`
	Version   string `gcfg:"version"`
	Port      int    `gcfg:"port"`
	Retries   int    `gcfg:"retries"`
	Timeout   int    `gcfg:"timeout"`
	Freq      int    `gcfg:"freq"`
	Aliases   string `gcfg:"aliases"`
	Config    string `gcfg:"config"`
	Mibs      string `gcfg:"mibs"`
	Tags      string `gcfg:"tags"`
}

// CommonConfig specifies general parameters
type CommonConfig struct {
	HTTPPort int    `gcfg:"httpPort"`
	LogDir   string `gcfg:"logDir"`
	OidFile  string `gcfg:"oidFile"`
	Tags     string `gcfg:"tags"`
	Mibs     string `gcfg:"mibs"`
	MibFile  string `gcfg:"mibfile"`
}

// MibConfig specifies what OIDs to query
type MibConfig struct {
	Name    string   `gcfg:"name"`
	Index   string   `gcfg:"index"`
	Regexps []string `gcfg:"regexp"`
	Keep    bool     `gcfg:"keep"`
}

// InfluxConfig defines connection requirements
type InfluxConfig struct {
	Host        string `gcfg:"host"`
	Port        int    `gcfg:"port"`
	Database    string `gcfg:"database"`
	Username    string `gcfg:"username"`
	Password    string `gcfg:"password"`
	Retention   string `gcfg:"retention"`
	Consistency string `gcfg:"consistency"`
	SkipVerify  bool   `gcfg:"skip_verify"`
	Timeout     int    `gcfg:"timeout"`
	BatchSize   int    `gcfg:"batchSize"`
	QueueSize   int    `gcfg:"queueSize"`
	Flush       int    `gcfg:"flush"`
}

type snmpStats struct {
	GetCnt    int
	ErrCnt    int
	LastError error
	LastTime  time.Time
}

type statsFn func() snmpStats

type SnmpInfo struct {
	Name   string
	Config *SnmpConfig
	MIB    *MibConfig
}

// SystemStatus provides operating statistics
type SystemStatus struct {
	Period, Started string
	Uptime          string
	DB, LogFile     string
	SNMP            map[string]*SnmpConfig
	Influx          map[string]*InfluxConfig
	SnmpStats       map[string]snmpStats
}

type TimeStamp snmp.TimeStamp

var (
	quit          = make(chan struct{})
	verbose       bool
	startTime     = time.Now()
	snmpNames     bool
	sample        bool
	dump          bool
	filter        bool
	httpPort      = 8080
	appdir, _     = osext.ExecutableFolder()
	logDir        = filepath.Join(appdir, "log")
	oidFile       = filepath.Join(appdir, "oids.json")
	configFile    = filepath.Join(appdir, "config.gcfg")
	errorLog      *os.File
	errorDuration = time.Duration(10 * time.Minute)
	errorPeriod   = errorDuration.String()
	errorMax      = 100
	errorName     string
	mibs          string
	statsMap      = make(map[string]statsFn)
	logger        *log.Logger
	commonTags    map[string]string

	cfg = struct {
		Snmp   map[string]*SnmpConfig
		Mibs   map[string]*MibConfig
		Influx map[string]*InfluxConfig
		Common CommonConfig
	}{}
)

func Senders() map[string]Sender {
	s := map[string]Sender{}
	for name, c := range cfg.Influx {
		sender, err := makeSender(c)
		if err != nil {
			panic(err)
		}
		s[name] = sender
	}
	return s
}

func (c *SnmpConfig) Profile() snmp.Profile {
	return snmp.Profile{
		Host:      c.Host,
		Community: c.Community,
		Version:   c.Version,
		Port:      c.Port,
		Retries:   c.Retries,
		Timeout:   c.Timeout,
	}
}

func getTags(list string) map[string]string {
	t := make(map[string]string)
	for _, item := range strings.Fields(list) {
		if pair := strings.Split(item, "="); len(pair) == 2 {
			t[pair[0]] = pair[1]
		}
	}
	return t
}

func status() SystemStatus {
	stats := map[string]snmpStats{}
	for name, fn := range statsMap {
		stats[name] = fn()
	}
	return SystemStatus{
		LogFile:   errorName,
		Started:   startTime.Format(layout),
		Uptime:    time.Now().Sub(startTime).String(),
		Period:    errorPeriod,
		SNMP:      cfg.Snmp,
		Influx:    cfg.Influx,
		SnmpStats: stats,
	}
}

func flags() *flag.FlagSet {
	var f flag.FlagSet
	f.BoolVar(&snmpNames, "names", snmpNames, "print column names and exit")
	f.BoolVar(&sample, "sample", sample, "print a sample of collected values and exit")
	f.BoolVar(&dump, "dump", dump, "print output of parsed mibs and exit")
	f.BoolVar(&filter, "filter", filter, "print (filtered by used OIDs) output of parsed mibs and exit")
	f.StringVar(&configFile, "config", configFile, "config file")
	f.BoolVar(&verbose, "verbose", verbose, "verbose mode")
	f.IntVar(&httpPort, "http", httpPort, "http port")
	f.StringVar(&logDir, "logs", logDir, "log directory")
	f.StringVar(&oidFile, "oids", oidFile, "OIDs file")
	f.StringVar(&mibs, "mibs", mibs, "mibs to use")
	f.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
		f.VisitAll(func(flag *flag.Flag) {
			format := "%10s: %s\n"
			fmt.Fprintf(os.Stderr, format, "-"+flag.Name, flag.Usage)
		})
		fmt.Fprintf(os.Stderr, "\nAll settings can be set in config file: %s\n", configFile)
		os.Exit(1)

	}
	return &f
}

func init() {
	log.SetOutput(os.Stderr)

	// parse first time to see if config file is being specified
	f := flags()
	f.Parse(os.Args[1:])

	// now load up config settings
	if _, err := os.Stat(configFile); err != nil {
		log.Fatal(err)
	}
	data, err := ioutil.ReadFile(configFile)
	if err != nil {
		log.Fatal(err)
	}
	err = gcfg.ReadStringInto(&cfg, string(data))
	if err != nil {
		log.Fatalf("Failed to parse gcfg data: %s", err)
	}
	httpPort = cfg.Common.HTTPPort

	if len(cfg.Common.LogDir) > 0 {
		logDir = cfg.Common.LogDir
	}
	if len(cfg.Common.OidFile) > 0 {
		oidFile = cfg.Common.OidFile
	}

	commonTags = getTags(cfg.Common.Tags)

	// re-read cmd line args to override as indicated
	f = flags()
	f.Parse(os.Args[1:])
	os.Mkdir(logDir, 0755)

	// Load or generate mib data
	if len(cfg.Common.MibFile) > 0 {
		if err := snmp.CachedMibInfo(cfg.Common.MibFile, cfg.Common.Mibs); err != nil {
			panic(err)
		}
	} else {
		if err := snmp.LoadMibs(cfg.Common.Mibs); err != nil {
			panic(err)
		}
	}
	if len(mibs) == 0 {
		mibs = cfg.Common.Mibs
	}

	if verbose {
		logger = log.New(os.Stderr, "", 0)
	}

	var ferr error
	errorName = fmt.Sprintf("error.%d.log", httpPort)
	errorPath := filepath.Join(logDir, errorName)
	errorLog, ferr = os.OpenFile(errorPath, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0664)
	if ferr != nil {
		log.Fatal("Can't open error log:", ferr)
	}
}

func errFn(err error) {
	log.Println(err)
}

func makeSender(cfg *InfluxConfig) (Sender, error) {
	conf := client.HTTPConfig{
		Addr:               fmt.Sprintf("http://%s:%d", cfg.Host, cfg.Port),
		Username:           cfg.Username,
		Password:           cfg.Password,
		Timeout:            (time.Duration(cfg.Timeout) * time.Second),
		InsecureSkipVerify: cfg.SkipVerify,
	}
	batch := client.BatchPointsConfig{
		Precision:        "s",
		Database:         cfg.Database,
		RetentionPolicy:  cfg.Retention,
		WriteConsistency: cfg.Consistency,
	}

	return NewSender(conf, batch, cfg.BatchSize, cfg.QueueSize, cfg.Flush, errFn)
}

func pairs(s string) map[string]string {
	m := make(map[string]string)
	for _, p := range strings.Fields(s) {
		b := strings.Split(p, "=")
		if len(b) == 2 {
			m[b[0]] = b[1]
		}
	}
	return m
}

func prep(c *SnmpConfig, mib *MibConfig) (snmp.Profile, snmp.Criteria) {
	regexps := make([]string, 0, len(mib.Regexps))
	for _, r := range mib.Regexps {
		for _, x := range strings.Fields(r) {
			regexps = append(regexps, x)
		}
	}

	crit := snmp.Criteria{
		OID:     mib.Name,
		Index:   mib.Index,
		Regexps: regexps,
		Keep:    mib.Keep,
		Tags:    getTags(c.Tags),
		Freq:    c.Freq,
		Aliases: pairs(c.Aliases),
	}

	for k, v := range commonTags {
		crit.Tags[k] = v
	}

	return c.Profile(), crit
}

func gather(send Sender, c *SnmpConfig, mib *MibConfig) {
	if c.Freq < 1 {
		panic("invalid polling frequency for: " + c.Host)
	}

	p, crit := prep(c, mib)

	sender := func(name string, tags map[string]string, value interface{}, ts snmp.TimeStamp) error {
		values := map[string]interface{}{"value": value, "elapsed": ts.Stop.Sub(ts.Start)}
		return send(name, tags, values, ts.Stop)
	}

	// influxdb saves uint64 as a string
	// so this is a workaround for now
	sender = snmp.IntegerSender(sender)

	var stats snmpStats
	var m sync.Mutex

	errFn := func(err error) {
		m.Lock()
		if err == nil {
			stats.GetCnt++
		} else {
			stats.ErrCnt++
			stats.LastError = err
			stats.LastTime = time.Now()
		}
		m.Unlock()
	}
	if err := snmp.Bulkwalker(p, crit, sender, errFn, logger); err != nil {
		panic(err)
	}

	name := fmt.Sprintf("%s/%s", c.Host, mib.Name)
	statsMap[name] = func() snmpStats {
		m.Lock()
		defer m.Unlock()
		return stats
	}
}

// agentList returns an array of snmp hosts and their associated mib info
func agentList() ([]SnmpInfo, error) {
	info := make([]SnmpInfo, 0, 32)
	for name, c := range cfg.Snmp {
		if len(c.Mibs) > 0 {
			for _, m := range strings.Fields(c.Mibs) {
				mib, ok := cfg.Mibs[m]
				if !ok {
					return info, fmt.Errorf("no mib config found for:%s", m)
				}
				info = append(info, SnmpInfo{name, c, mib})
			}
			continue
		}
		mib, ok := cfg.Mibs[name]
		if !ok {
			if mib, ok = cfg.Mibs["*"]; !ok {
				return info, fmt.Errorf("no mib config found for:%s", name)
			}
		}
		info = append(info, SnmpInfo{name, c, mib})
	}
	return info, nil
}

// filtered returns a list of all OIDs encountered by
// polling the specified devices and their respective OIDs
func filtered(a []SnmpInfo) []string {
	lookup, err := snmp.OIDNames(mibs)
	if err != nil {
		panic(err)
	}

	var w1, w2 sync.WaitGroup
	valid := snmp.RootOID(lookup)
	c := make(chan string, 1024)
	used := make(map[string]struct{})

	// queued access to updating used
	w1.Add(1)
	go func() {
		for oid := range c {
			root := valid(oid)
			if len(root) == 0 {
				panic("no root found for OID:" + oid)
			}
			used[root] = struct{}{}
		}
		w1.Done()
	}()

	poll := func(cfg *SnmpConfig, oid string) {
		client, err := snmp.NewClient(cfg.Profile())
		if err != nil {
			panic(err)
		}
		snmp.BulkWalkAll(client, oid, snmp.OIDCollector(c))
		w2.Done()
		client.Conn.Close()
	}

	for _, s := range a {
		for _, name := range strings.Fields(s.MIB.Name) {
			oid, ok := lookup[name]
			if !ok {
				panic("No OID for name:" + name)
			}
			c <- oid
			w2.Add(1)
			go poll(s.Config, oid)
		}
	}
	w2.Wait()
	close(c)
	w1.Wait()

	list := make([]string, 0, len(used))
	for k, _ := range used {
		list = append(list, k)
	}
	return list
}

// sampler dumps a single fetch of data from each snmp host/mib
func sampler(agents []SnmpInfo) error {
	var wg sync.WaitGroup
	e := make(chan error)
	cnt := 0
	sender, _ := snmp.DebugSender(nil, nil)
	for _, a := range agents {
		wg.Add(1)
		cnt++
		go func(c int, host string, config *SnmpConfig, mib *MibConfig) {
			p, crit := prep(config, mib)
			if err := snmp.Sampler(p, crit, sender); err != nil {
				e <- err
			}
			wg.Done()
		}(cnt, a.Config.Host, a.Config, a.MIB)
	}
	wg.Wait()
	select {
	case err := <-e:
		return err
	default:
	}
	return nil
}

func dumper(agents []SnmpInfo) error {
	if len(mibs) == 0 {
		return fmt.Errorf("error: no MIBs specified")
	}
	var oids []string
	if filter {
		oids = filtered(agents)
	}
	return snmp.OIDList(mibs, oids, os.Stdout)
}

func main() {
	agents, err := agentList()
	if err != nil {
		panic(err)
	}

	if dump {
		if err := dumper(agents); err != nil {
			panic(err)
		}
		return
	}

	if sample {
		if err := sampler(agents); err != nil {
			panic(err)
		}
		return
	}

	senders := Senders()
	for _, a := range agents {
		send, ok := senders[a.Name]
		if !ok {
			send, ok = senders["*"]
			if !ok {
				panic("No sender for: " + a.Name)
			}
		}
		go gather(send, a.Config, a.MIB)
	}

	if httpPort > 0 {
		webServer(httpPort)
	} else {
		<-quit
	}
	errorLog.Close()
}
<|endoftext|>


<commit_before>// Copyright 2015 CodeIgnition. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"crypto/rand"
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"os/user"
	"path/filepath"
	"time"

	"github.com/codeignition/recon/internal/fileutil"
)

const (
	metricsPath = "/metrics.json" // metrics path in the master server
	agentsPath  = "/agents.json"  // agents path in the master server
)

// config file path in the local machine
var configPath string

func init() {
	usr, err := user.Current()
	if err != nil {
		log.Fatal(err)
	}
	configPath = filepath.Join(usr.HomeDir, ".recon")
	if fileutil.Exists(configPath) {
		err := os.Remove(configPath)
		if err != nil {
			log.Fatalln(err)
		}
	}
}

func main() {
	log.SetPrefix("recon: ")

	var masterAddr = flag.String("masterAddr", "http://localhost:3000", "address of the recon-master server (along with protocol)")
	flag.Parse()

	uid, err := generateUID()
	if err != nil {
		log.Fatalln(err)
	}

	f, err := os.Create(configPath)
	if err != nil {
		log.Fatalln(err)
	}
	defer f.Close()

	_, err = f.WriteString("uid: " + uid + "\n")
	if err != nil {
		log.Fatalln(err)
	}

	if err := registerAgent(*masterAddr, uid); err != nil {
		log.Fatalln(err)
	}

	c := time.Tick(5 * time.Second)
	for now := range c {
		log.Println("Update sent at", now)
		if err := update(*masterAddr); err != nil {
			log.Println(err)
		}
	}
}

func registerAgent(addr, uid string) error {
	var buf bytes.Buffer
	d := map[string]interface{}{
		"agent": map[string]string{
			"uid": uid,
		},
	}
	if err := json.NewEncoder(&buf).Encode(&d); err != nil {
		return err
	}
	resp, err := http.Post(addr+agentsPath, "application/json", &buf)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	// TODO: Don't print the response, but store the messaging server URL and subscribe to it.
	fmt.Printf("%s\n", resp.Body)
	return nil
}

func generateUID() (string, error) {
	b := make([]byte, 4)
	_, err := rand.Read(b)
	if err != nil {
		return "", err
	}
	uid := fmt.Sprintf("%x", b)
	return uid, nil
}

func update(addr string) error {
	var buf bytes.Buffer
	d := map[string]interface{}{
		"metric": accumulateData(),
	}
	if err := json.NewEncoder(&buf).Encode(&d); err != nil {
		return err
	}
	resp, err := http.Post(addr+metricsPath, "application/json", &buf)
	if err != nil {
		return err
	}
	if resp.StatusCode != http.StatusCreated {
		return fmt.Errorf("response status code not %d; response is %v\n", http.StatusCreated, resp)
	}
	return nil
}
<commit_msg>use url.Parse for resiliency<commit_after>// Copyright 2015 CodeIgnition. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"crypto/rand"
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"os"
	"os/user"
	"path/filepath"
	"time"

	"github.com/codeignition/recon/internal/fileutil"
)

const (
	metricsPath = "/metrics.json" // metrics path in the master server
	agentsPath  = "/agents.json"  // agents path in the master server
)

// config file path in the local machine
var configPath string

func init() {
	usr, err := user.Current()
	if err != nil {
		log.Fatal(err)
	}
	configPath = filepath.Join(usr.HomeDir, ".recon")
	if fileutil.Exists(configPath) {
		err := os.Remove(configPath)
		if err != nil {
			log.Fatalln(err)
		}
	}
}

func main() {
	log.SetPrefix("recon: ")

	var masterAddr = flag.String("masterAddr", "http://localhost:3000", "address of the recon-master server (along with protocol)")
	flag.Parse()

	uid, err := generateUID()
	if err != nil {
		log.Fatalln(err)
	}

	f, err := os.Create(configPath)
	if err != nil {
		log.Fatalln(err)
	}
	defer f.Close()

	_, err = f.WriteString("uid: " + uid + "\n")
	if err != nil {
		log.Fatalln(err)
	}

	if err := registerAgent(*masterAddr, uid); err != nil {
		log.Fatalln(err)
	}

	c := time.Tick(5 * time.Second)
	for now := range c {
		log.Println("Update sent at", now)
		if err := update(*masterAddr); err != nil {
			log.Println(err)
		}
	}
}

func registerAgent(addr, uid string) error {
	var buf bytes.Buffer
	d := map[string]interface{}{
		"agent": map[string]string{
			"uid": uid,
		},
	}
	if err := json.NewEncoder(&buf).Encode(&d); err != nil {
		return err
	}

	// url.Parse instead of just appending will inform
	// about errors if the code changes and the url is malformed.
	l, err := url.Parse(addr + agentsPath)
	if err != nil {
		return err
	}
	resp, err := http.Post(l.String(), "application/json", &buf)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	// TODO: Don't print the response, but store the messaging server URL and subscribe to it.
	fmt.Printf("%s\n", resp.Body)
	return nil
}

func generateUID() (string, error) {
	b := make([]byte, 4)
	_, err := rand.Read(b)
	if err != nil {
		return "", err
	}
	uid := fmt.Sprintf("%x", b)
	return uid, nil
}

func update(addr string) error {
	var buf bytes.Buffer
	d := map[string]interface{}{
		"metric": accumulateData(),
	}
	if err := json.NewEncoder(&buf).Encode(&d); err != nil {
		return err
	}

	l, err := url.Parse(addr + metricsPath)
	if err != nil {
		return err
	}
	resp, err := http.Post(l.String(), "application/json", &buf)
	if err != nil {
		return err
	}
	if resp.StatusCode != http.StatusCreated {
		return fmt.Errorf("response status code not %d; response is %v\n", http.StatusCreated, resp)
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"gopkg.in/tylerb/graceful.v1"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
	"time"
)

var (
	githubUsername = ""
	githubPassword = ""
	githubAddr     = ""
	Addr           = ""
)

type GitHubRateLimit struct {
	Resources struct {
		Core struct {
			Limit     int `json:"limit"`
			Remaining int `json:"remaining"`
			Reset     int `json:"reset"`
		} `json:"core"`
		Search struct {
			Limit     int `json:"limit"`
			Remaining int `json:"remaining"`
			Reset     int `json:"reset"`
		} `json:"search"`
		Graphql struct {
			Limit     int `json:"limit"`
			Remaining int `json:"remaining"`
			Reset     int `json:"reset"`
		} `json:"graphql"`
	} `json:"resources"`
	Rate struct {
		Limit     int `json:"limit"`
		Remaining int `json:"remaining"`
		Reset     int `json:"reset"`
	} `json:"rate"`
}

func (g *GitHubRateLimit) WriteTo(w io.Writer) {
	buf := &bytes.Buffer{}

	// GitHub Rate Limit: Resources
	buf.WriteString(fmt.Sprintf("# HELP %s %s\n", "github_ratelimit_resources_limit", "GitHub Rate Limit: Resources"))
	buf.WriteString(fmt.Sprintf("# TYPE %s %s\n", "github_ratelimit_resources_limit", "gauge"))

	buf.WriteString(fmt.Sprintf("%s{type=\"%s\"} %d\n", "github_ratelimit_resources_limit", "core", g.Resources.Core.Limit))
	buf.WriteString(fmt.Sprintf("%s{type=\"%s\"} %d\n", "github_ratelimit_resources_limit", "search", g.Resources.Search.Limit))
	buf.WriteString(fmt.Sprintf("%s{type=\"%s\"} %d\n", "github_ratelimit_resources_limit", "graphql", g.Resources.Graphql.Limit))

	// GitHub Rate Remaining: Resources
	buf.WriteString(fmt.Sprintf("# HELP %s %s\n", "github_ratelimit_resources_remaining", "GitHub Rate Remaining: Resources"))
	buf.WriteString(fmt.Sprintf("# TYPE %s %s\n", "github_ratelimit_resources_remaining", "gauge"))

	buf.WriteString(fmt.Sprintf("%s{type=\"%s\"} %d\n", "github_ratelimit_resources_remaining", "core", g.Resources.Core.Remaining))
	buf.WriteString(fmt.Sprintf("%s{type=\"%s\"} %d\n", "github_ratelimit_resources_remaining", "search", g.Resources.Search.Remaining))
	buf.WriteString(fmt.Sprintf("%s{type=\"%s\"} %d\n", "github_ratelimit_resources_remaining", "graphql", g.Resources.Graphql.Remaining))

	// GitHub Rate Reset: Resources
	buf.WriteString(fmt.Sprintf("# HELP %s %s\n", "github_ratelimit_resources_reset", "GitHub Rate Reset: Resources"))
	buf.WriteString(fmt.Sprintf("# TYPE %s %s\n", "github_ratelimit_resources_reset", "counter"))

	buf.WriteString(fmt.Sprintf("%s{type=\"%s\"} %d\n", "github_ratelimit_resources_reset", "core", g.Resources.Core.Reset))
	buf.WriteString(fmt.Sprintf("%s{type=\"%s\"} %d\n", "github_ratelimit_resources_reset", "search", g.Resources.Search.Reset))
	buf.WriteString(fmt.Sprintf("%s{type=\"%s\"} %d\n", "github_ratelimit_resources_reset", "graphql", g.Resources.Graphql.Reset))

	// GitHub Rate Rate
	buf.WriteString(fmt.Sprintf("# HELP %s %s\n", "github_ratelimit_rate_limit", "GitHub Rate Limit"))
	buf.WriteString(fmt.Sprintf("# TYPE %s %s\n", "github_ratelimit_rate_limit", "gauge"))
	buf.WriteString(fmt.Sprintf("%s %d\n", "github_ratelimit_rate_limit", g.Rate.Limit))

	buf.WriteString(fmt.Sprintf("# HELP %s %s\n", "github_ratelimit_rate_remaining", "GitHub Rate Remaining"))
	buf.WriteString(fmt.Sprintf("# TYPE %s %s\n", "github_ratelimit_rate_remaining", "gauge"))
	buf.WriteString(fmt.Sprintf("%s %d\n", "github_ratelimit_rate_remaining", g.Rate.Remaining))

	buf.WriteString(fmt.Sprintf("# HELP %s %s\n", "github_ratelimit_rate_reset", "GitHub Rate Reset"))
	buf.WriteString(fmt.Sprintf("# TYPE %s %s\n", "github_ratelimit_rate_reset", "counter"))
	buf.WriteString(fmt.Sprintf("%s %d\n", "github_ratelimit_rate_reset", g.Rate.Reset))

	io.Copy(w, buf)
}

func main() {
	addr := flag.String("addr", "0.0.0.0:8080", "HTTP Server address")
	url := flag.String("url", "https://api.github.com", "Github API address")
	username := flag.String("username", "", "GitHub username")
	password := flag.String("password", "", "GitHub password")
	flag.Parse()

	Addr = *addr
	githubAddr = *url + "/rate_limit"
	githubUsername = *username
	githubPassword = *password

	log.Println("Starting GitHub exporter.")
	log.Println("Listening on:", "'"+Addr+"'")
	log.Println("Scrapping:", "'"+githubAddr+"'", "with username", "'"+githubUsername+"'", "and password", "'"+strings.Repeat("*", len(githubPassword))+"'", ".")

	server := &graceful.Server{
		Timeout: 10 * time.Second,
		Server: &http.Server{
			Addr:        Addr,
			ReadTimeout: time.Duration(5) * time.Second,
			Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				req, err := http.NewRequest("GET", githubAddr, nil)
				if err != nil {
					log.Println(err)
				}

				if (githubUsername != "") && (githubPassword != "") {
					req.SetBasicAuth(githubUsername, githubPassword)
				}

				transport := http.Transport{}
				resp, err := transport.RoundTrip(req)
				if err != nil {
					log.Println(err)
				}

				body, err := ioutil.ReadAll(resp.Body)
				if err != nil {
					log.Println(err)
				}

				resp.Body.Close()

				res := GitHubRateLimit{}
				json.Unmarshal([]byte(body), &res)

				res.WriteTo(w)
			}),
		},
	}

	if err := server.ListenAndServe(); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Golint, Goimports<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
	"time"

	graceful "gopkg.in/tylerb/graceful.v1"
)

var (
	address        = ""
	githubUsername = ""
	githubPassword = ""
	githubAddr     = ""
)

type GitHubRateLimit struct {
	Resources struct {
		Core struct {
			Limit     int `json:"limit"`
			Remaining int `json:"remaining"`
			Reset     int `json:"reset"`
		} `json:"core"`
		Search struct {
			Limit     int `json:"limit"`
			Remaining int `json:"remaining"`
			Reset     int `json:"reset"`
		} `json:"search"`
		Graphql struct {
			Limit     int `json:"limit"`
			Remaining int `json:"remaining"`
			Reset     int `json:"reset"`
		} `json:"graphql"`
	} `json:"resources"`
	Rate struct {
		Limit     int `json:"limit"`
		Remaining int `json:"remaining"`
		Reset     int `json:"reset"`
	} `json:"rate"`
}

func (g *GitHubRateLimit) WriteTo(w io.Writer) {
	buf := &bytes.Buffer{}

	// GitHub Rate Limit: Resources
	buf.WriteString(fmt.Sprintf("# HELP %s %s\n", "github_ratelimit_resources_limit", "GitHub Rate Limit: Resources"))
	buf.WriteString(fmt.Sprintf("# TYPE %s %s\n", "github_ratelimit_resources_limit", "gauge"))

	buf.WriteString(fmt.Sprintf("%s{type=\"%s\"} %d\n", "github_ratelimit_resources_limit", "core", g.Resources.Core.Limit))
	buf.WriteString(fmt.Sprintf("%s{type=\"%s\"} %d\n", "github_ratelimit_resources_limit", "search", g.Resources.Search.Limit))
	buf.WriteString(fmt.Sprintf("%s{type=\"%s\"} %d\n", "github_ratelimit_resources_limit", "graphql", g.Resources.Graphql.Limit))

	// GitHub Rate Remaining: Resources
	buf.WriteString(fmt.Sprintf("# HELP %s %s\n", "github_ratelimit_resources_remaining", "GitHub Rate Remaining: Resources"))
	buf.WriteString(fmt.Sprintf("# TYPE %s %s\n", "github_ratelimit_resources_remaining", "gauge"))

	buf.WriteString(fmt.Sprintf("%s{type=\"%s\"} %d\n", "github_ratelimit_resources_remaining", "core", g.Resources.Core.Remaining))
	buf.WriteString(fmt.Sprintf("%s{type=\"%s\"} %d\n", "github_ratelimit_resources_remaining", "search", g.Resources.Search.Remaining))
	buf.WriteString(fmt.Sprintf("%s{type=\"%s\"} %d\n", "github_ratelimit_resources_remaining", "graphql", g.Resources.Graphql.Remaining))

	// GitHub Rate Reset: Resources
	buf.WriteString(fmt.Sprintf("# HELP %s %s\n", "github_ratelimit_resources_reset", "GitHub Rate Reset: Resources"))
	buf.WriteString(fmt.Sprintf("# TYPE %s %s\n", "github_ratelimit_resources_reset", "counter"))

	buf.WriteString(fmt.Sprintf("%s{type=\"%s\"} %d\n", "github_ratelimit_resources_reset", "core", g.Resources.Core.Reset))
	buf.WriteString(fmt.Sprintf("%s{type=\"%s\"} %d\n", "github_ratelimit_resources_reset", "search", g.Resources.Search.Reset))
	buf.WriteString(fmt.Sprintf("%s{type=\"%s\"} %d\n", "github_ratelimit_resources_reset", "graphql", g.Resources.Graphql.Reset))

	// GitHub Rate Rate
	buf.WriteString(fmt.Sprintf("# HELP %s %s\n", "github_ratelimit_rate_limit", "GitHub Rate Limit"))
	buf.WriteString(fmt.Sprintf("# TYPE %s %s\n", "github_ratelimit_rate_limit", "gauge"))
	buf.WriteString(fmt.Sprintf("%s %d\n", "github_ratelimit_rate_limit", g.Rate.Limit))

	buf.WriteString(fmt.Sprintf("# HELP %s %s\n", "github_ratelimit_rate_remaining", "GitHub Rate Remaining"))
	buf.WriteString(fmt.Sprintf("# TYPE %s %s\n", "github_ratelimit_rate_remaining", "gauge"))
	buf.WriteString(fmt.Sprintf("%s %d\n", "github_ratelimit_rate_remaining", g.Rate.Remaining))

	buf.WriteString(fmt.Sprintf("# HELP %s %s\n", "github_ratelimit_rate_reset", "GitHub Rate Reset"))
	buf.WriteString(fmt.Sprintf("# TYPE %s %s\n", "github_ratelimit_rate_reset", "counter"))
	buf.WriteString(fmt.Sprintf("%s %d\n", "github_ratelimit_rate_reset", g.Rate.Reset))

	io.Copy(w, buf)
}

func main() {
	addr := flag.String("addr", "0.0.0.0:8080", "HTTP Server address")
	url := flag.String("url", "https://api.github.com", "Github API address")
	username := flag.String("username", "", "GitHub username")
	password := flag.String("password", "", "GitHub password")
	flag.Parse()

	address = *addr
	githubAddr = *url + "/rate_limit"
	githubUsername = *username
	githubPassword = *password

	log.Println("Starting GitHub exporter.")
	log.Println("Listening on:", "'"+address+"'")
	log.Println("Scrapping:", "'"+githubAddr+"'", "with username", "'"+githubUsername+"'", "and password", "'"+strings.Repeat("*", len(githubPassword))+"'", ".")

	server := &graceful.Server{
		Timeout: 10 * time.Second,
		Server: &http.Server{
			Addr:        address,
			ReadTimeout: time.Duration(5) * time.Second,
			Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				req, err := http.NewRequest("GET", githubAddr, nil)
				if err != nil {
					log.Println(err)
				}

				if (githubUsername != "") && (githubPassword != "") {
					req.SetBasicAuth(githubUsername, githubPassword)
				}

				transport := http.Transport{}
				resp, err := transport.RoundTrip(req)
				if err != nil {
					log.Println(err)
				}

				body, err := ioutil.ReadAll(resp.Body)
				if err != nil {
					log.Println(err)
				}

				resp.Body.Close()

				res := GitHubRateLimit{}
				json.Unmarshal([]byte(body), &res)

				res.WriteTo(w)
			}),
		},
	}

	if err := server.ListenAndServe(); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>