<commit_before>// +build freebsd
// tun_freebsd.go -- tun interface with cgo for linux / bsd
//

package samtun

/*

#include <string.h>
#include <unistd.h>
#include <fcntl.h>
#include <netinet/in.h>
#include <netinet/ip.h>
#include <arpa/inet.h>
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <net/if.h>
#include <net/if_tun.h>
#include <stdio.h>

int tundev_open(char * ifname) {
  if (strlen(ifname) > IFNAMSIZ) {
    return -1;
  }
  char name[IFNAMSIZ];
  sprintf(name, "/dev/%s", ifname);
  int fd = open(name, O_RDWR);
  if (fd > 0) {
    int i = 0;
    ioctl(fd, TUNSIFHEAD, &i);
  }
  return fd;
}

int tundev_up(char * ifname, char * addr, char * netmask, int mtu) {

  struct ifreq ifr;
  memset(&ifr, 0, sizeof(struct ifreq));
  strncpy(ifr.ifr_name, ifname, IFNAMSIZ);
  int fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_IP);
  if ( fd > 0 ) {
    ifr.ifr_mtu = mtu;
    if ( ioctl(fd, SIOCSIFMTU, (void*) &ifr) < 0) {
      close(fd);
      perror("SIOCSIFMTU");
      return -1;
    }

    struct sockaddr_in src;
    memset(&src, 0, sizeof(struct sockaddr_in));
    src.sin_family = AF_INET;
    if ( ! inet_aton(addr, &src.sin_addr) ) {
      printf("invalid srcaddr %s\n", addr);
      close(fd);
      return -1;
    }

    memset(&ifr, 0, sizeof(struct ifreq));
    strncpy(ifr.ifr_name, ifname, IFNAMSIZ);
    memcpy(&ifr.ifr_addr, &src, sizeof(struct sockaddr_in));
    if ( ioctl(fd, SIOCSIFADDR, (void*)&ifr) < 0 ) {
      close(fd);
      perror("SIOCSIFADDR");
     return -1;
    }

    memset(&ifr, 0, sizeof(struct ifreq));
    strncpy(ifr.ifr_name, ifname, IFNAMSIZ);
    if ( ioctl(fd, SIOCGIFFLAGS, (void*)&ifr) < 0 ) {
      close(fd);
      perror("SIOCGIFFLAGS");
      return -1;
    }
    ifr.ifr_flags |= IFF_UP ;
    if ( ioctl(fd, SIOCSIFFLAGS, (void*)&ifr) < 0 ) {
      perror("SIOCSIFFLAGS");
      close(fd);
      return -1;
    }

    close(fd);
    return 0;
  }
  return -1;
}

void tundev_close(int fd) {
  close(fd);
}

*/
import "C"

import (
  "errors"
)

type tunDev struct {
  fd C.int
}

func newTun(ifname, addr, dstaddr string, mtu int) (t tunDev, err error) {
  fd := C.tundev_open(C.CString(ifname))

  if fd == -1 {
    err = errors.New("cannot open tun interface")
  } else {
    if C.tundev_up(C.CString(ifname), C.CString(addr), C.CString(dstaddr), C.int(mtu)) < C.int(0) {
      err = errors.New("cannot put up interface")
    } else {
      t = tunDev{fd}
    }
  }
  return
}

// read from the tun device
func (t *tunDev) Read(d []byte) (n int, err error) {
  return fdRead(C.int(t.fd), d)
}

func (t *tunDev) Write(d []byte) (n int, err error) {
  return fdWrite(C.int(t.fd), d)
}

func (t *tunDev) Close() {
  C.tundev_close(C.int(t.fd))
}
<commit_msg>check for invalid file descriptor on freebsd correctly<commit_after>// +build freebsd
// tun_freebsd.go -- tun interface with cgo for linux / bsd
//

package samtun

/*

#include <string.h>
#include <unistd.h>
#include <fcntl.h>
#include <netinet/in.h>
#include <netinet/ip.h>
#include <arpa/inet.h>
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <net/if.h>
#include <net/if_tun.h>
#include <stdio.h>

int tundev_open(char * ifname) {
  if (strlen(ifname) > IFNAMSIZ) {
    return -1;
  }
  char name[IFNAMSIZ];
  sprintf(name, "/dev/%s", ifname);
  int fd = open(name, O_RDWR);
  if (fd > 0) {
    int i = 0;
    ioctl(fd, TUNSIFHEAD, &i);
  }
  return fd;
}

int tundev_up(char * ifname, char * addr, char * netmask, int mtu) {

  struct ifreq ifr;
  memset(&ifr, 0, sizeof(struct ifreq));
  strncpy(ifr.ifr_name, ifname, IFNAMSIZ);
  int fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_IP);
  if ( fd > 0 ) {
    ifr.ifr_mtu = mtu;
    if ( ioctl(fd, SIOCSIFMTU, (void*) &ifr) < 0) {
      close(fd);
      perror("SIOCSIFMTU");
      return -1;
    }

    struct sockaddr_in src;
    memset(&src, 0, sizeof(struct sockaddr_in));
    src.sin_family = AF_INET;
    if ( ! inet_aton(addr, &src.sin_addr) ) {
      printf("invalid srcaddr %s\n", addr);
      close(fd);
      return -1;
    }

    memset(&ifr, 0, sizeof(struct ifreq));
    strncpy(ifr.ifr_name, ifname, IFNAMSIZ);
    memcpy(&ifr.ifr_addr, &src, sizeof(struct sockaddr_in));
    if ( ioctl(fd, SIOCSIFADDR, (void*)&ifr) < 0 ) {
      close(fd);
      perror("SIOCSIFADDR");
     return -1;
    }

    memset(&ifr, 0, sizeof(struct ifreq));
    strncpy(ifr.ifr_name, ifname, IFNAMSIZ);
    if ( ioctl(fd, SIOCGIFFLAGS, (void*)&ifr) < 0 ) {
      close(fd);
      perror("SIOCGIFFLAGS");
      return -1;
    }
    ifr.ifr_flags |= IFF_UP ;
    if ( ioctl(fd, SIOCSIFFLAGS, (void*)&ifr) < 0 ) {
      perror("SIOCSIFFLAGS");
      close(fd);
      return -1;
    }

    close(fd);
    return 0;
  }
  return -1;
}

void tundev_close(int fd) {
  close(fd);
}

*/
import "C"

import (
  "errors"
)

type tunDev struct {
  fd C.int
}

func newTun(ifname, addr, dstaddr string, mtu int) (t tunDev, err error) {
  fd := C.tundev_open(C.CString(ifname))

  if fd == C.int(-1) {
    err = errors.New("cannot open tun interface")
  } else {
    if C.tundev_up(C.CString(ifname), C.CString(addr), C.CString(dstaddr), C.int(mtu)) < C.int(0) {
      err = errors.New("cannot put up interface")
    } else {
      t = tunDev{fd}
    }
  }
  return
}

// read from the tun device
func (t *tunDev) Read(d []byte) (n int, err error) {
  return fdRead(C.int(t.fd), d)
}

func (t *tunDev) Write(d []byte) (n int, err error) {
  return fdWrite(C.int(t.fd), d)
}

func (t *tunDev) Close() {
  C.tundev_close(C.int(t.fd))
}
<|endoftext|>


<commit_before>package mongo

import (
	"fmt"
	"os"
	"os/exec"
	"path"
	"path/filepath"

	"github.com/juju/loggo"

	"launchpad.net/juju-core/upstart"
	"launchpad.net/juju-core/utils"
)

const (
	maxFiles = 65000
	maxProcs = 20000
)

var (
	logger = loggo.GetLogger("juju.agent.mongo")

	oldMongoServiceName = "juju-db"

	// JujuMongodPath holds the default path to the juju-specific mongod.
	JujuMongodPath = "/usr/lib/juju/bin/mongod"
)

// MongoPath returns the executable path to be used to run mongod on this
// machine. If the juju-bundled version of mongo exists, it will return that
// path, otherwise it will return the command to run mongod from the path.
func MongodPath() (string, error) {
	if _, err := os.Stat(JujuMongodPath); err == nil {
		return JujuMongodPath, nil
	}

	path, err := exec.LookPath("mongod")
	if err != nil {
		return "", err
	}
	return path, nil
}

// EnsureMongoServer ensures that the correct mongo upstart script is installed
// and running.
//
// This method will remove old versions of the mongo upstart script as necessary
// before installing the new version.
func EnsureMongoServer(dir string, port int) error {
	name := makeServiceName(mongoScriptVersion)
	service, err := MongoUpstartService(name, dir, port)
	if err != nil {
		return err
	}
	if service.Installed() {
		return nil
	}

	if err := removeOldMongoServices(mongoScriptVersion); err != nil {
		return err
	}

	if err := makeJournalDirs(dir); err != nil {
		return err
	}

	if err := service.Install(); err != nil {
		return fmt.Errorf("failed to install mongo service %q: %v", service.Name, err)
	}
	return service.Start()
}

func makeJournalDirs(dir string) error {
	journalDir := path.Join(dir, "journal")

	if err := os.MkdirAll(journalDir, 0700); err != nil {
		logger.Errorf("failed to make mongo journal dir %s: %v", journalDir, err)
		return err
	}

	// manually create the prealloc files, since otherwise they get created as 100M files.
	zeroes := make([]byte, 64*1024) // should be enough for anyone
	for x := 0; x < 3; x++ {
		name := fmt.Sprintf("prealloc.%d", x)
		filename := filepath.Join(journalDir, name)
		f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0700)
		if err != nil {
			return fmt.Errorf("failed to open mongo prealloc file %q: %v", filename, err)
		}
		defer f.Close()
		for total := 0; total < 1024*1024; {
			n, err := f.Write(zeroes)
			if err != nil {
				return fmt.Errorf("failed to write to mongo prealloc file %q: %v", filename, err)
			}
			total += n
		}
	}
	return nil
}

// removeOldMongoServices looks for any old juju mongo upstart scripts and
// removes them.
func removeOldMongoServices(curVersion int) error {
	old := upstart.NewService(oldMongoServiceName)
	if err := old.StopAndRemove(); err != nil {
		logger.Errorf("Failed to remove old mongo upstart service %q: %v", old.Name, err)
		return err
	}

	// the new formatting for the script name started at version 2
	for x := 2; x < curVersion; x++ {
		old := upstart.NewService(makeServiceName(x))
		if err := old.StopAndRemove(); err != nil {
			logger.Errorf("Failed to remove old mongo upstart service %q: %v", old.Name, err)
			return err
		}
	}
	return nil
}

func makeServiceName(version int) string {
	return fmt.Sprintf("juju-db-v%d", version)
}

// mongoScriptVersion keeps track of changes to the mongo upstart script.
// Update this version when you update the script that gets installed from
// MongoUpstartService.
const mongoScriptVersion = 2

// MongoUpstartService returns the upstart config for the mongo state service.
//
// This method assumes there is a server.pem keyfile in dataDir.
func MongoUpstartService(name, dataDir string, port int) (*upstart.Conf, error) {

	keyFile := path.Join(dataDir, "server.pem")
	svc := upstart.NewService(name)

	dbDir := path.Join(dataDir, "db")

	mongodpath, err := MongodPath()
	if err != nil {
		return nil, err
	}

	conf := &upstart.Conf{
		Service: *svc,
		Desc:    "juju state database",
		Limit: map[string]string{
			"nofile": fmt.Sprintf("%d %d", maxFiles, maxFiles),
			"nproc":  fmt.Sprintf("%d %d", maxProcs, maxProcs),
		},
		Cmd: mongodpath +
			" --auth" +
			" --dbpath=" + dbDir +
			" --sslOnNormalPorts" +
			" --sslPEMKeyFile " + utils.ShQuote(keyFile) +
			" --sslPEMKeyPassword ignored" +
			" --bind_ip 0.0.0.0" +
			" --port " + fmt.Sprint(port) +
			" --noprealloc" +
			" --syslog" +
			" --smallfiles",
		// TODO(Nate): uncomment when we commit HA stuff
		// +
		//	" --replSet juju",
	}
	return conf, nil
}
<commit_msg>Partial revert of r2347 to go back to always using default mongodb in upstart script<commit_after>package mongo

import (
	"fmt"
	"os"
	"os/exec"
	"path"
	"path/filepath"

	"github.com/juju/loggo"

	"launchpad.net/juju-core/upstart"
	"launchpad.net/juju-core/utils"
)

const (
	maxFiles = 65000
	maxProcs = 20000
)

var (
	logger = loggo.GetLogger("juju.agent.mongo")

	oldMongoServiceName = "juju-db"

	// JujuMongodPath holds the default path to the juju-specific mongod.
	JujuMongodPath = "/usr/lib/juju/bin/mongod"
)

// MongoPath returns the executable path to be used to run mongod on this
// machine. If the juju-bundled version of mongo exists, it will return that
// path, otherwise it will return the command to run mongod from the path.
func MongodPath() (string, error) {
	if _, err := os.Stat(JujuMongodPath); err == nil {
		return JujuMongodPath, nil
	}

	path, err := exec.LookPath("mongod")
	if err != nil {
		return "", err
	}
	return path, nil
}

// EnsureMongoServer ensures that the correct mongo upstart script is installed
// and running.
//
// This method will remove old versions of the mongo upstart script as necessary
// before installing the new version.
func EnsureMongoServer(dir string, port int) error {
	name := makeServiceName(mongoScriptVersion)
	service, err := MongoUpstartService(name, dir, port)
	if err != nil {
		return err
	}
	if service.Installed() {
		return nil
	}

	if err := removeOldMongoServices(mongoScriptVersion); err != nil {
		return err
	}

	if err := makeJournalDirs(dir); err != nil {
		return err
	}

	if err := service.Install(); err != nil {
		return fmt.Errorf("failed to install mongo service %q: %v", service.Name, err)
	}
	return service.Start()
}

func makeJournalDirs(dir string) error {
	journalDir := path.Join(dir, "journal")

	if err := os.MkdirAll(journalDir, 0700); err != nil {
		logger.Errorf("failed to make mongo journal dir %s: %v", journalDir, err)
		return err
	}

	// manually create the prealloc files, since otherwise they get created as 100M files.
	zeroes := make([]byte, 64*1024) // should be enough for anyone
	for x := 0; x < 3; x++ {
		name := fmt.Sprintf("prealloc.%d", x)
		filename := filepath.Join(journalDir, name)
		f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0700)
		if err != nil {
			return fmt.Errorf("failed to open mongo prealloc file %q: %v", filename, err)
		}
		defer f.Close()
		for total := 0; total < 1024*1024; {
			n, err := f.Write(zeroes)
			if err != nil {
				return fmt.Errorf("failed to write to mongo prealloc file %q: %v", filename, err)
			}
			total += n
		}
	}
	return nil
}

// removeOldMongoServices looks for any old juju mongo upstart scripts and
// removes them.
func removeOldMongoServices(curVersion int) error {
	old := upstart.NewService(oldMongoServiceName)
	if err := old.StopAndRemove(); err != nil {
		logger.Errorf("Failed to remove old mongo upstart service %q: %v", old.Name, err)
		return err
	}

	// the new formatting for the script name started at version 2
	for x := 2; x < curVersion; x++ {
		old := upstart.NewService(makeServiceName(x))
		if err := old.StopAndRemove(); err != nil {
			logger.Errorf("Failed to remove old mongo upstart service %q: %v", old.Name, err)
			return err
		}
	}
	return nil
}

func makeServiceName(version int) string {
	return fmt.Sprintf("juju-db-v%d", version)
}

// mongoScriptVersion keeps track of changes to the mongo upstart script.
// Update this version when you update the script that gets installed from
// MongoUpstartService.
const mongoScriptVersion = 2

// MongoUpstartService returns the upstart config for the mongo state service.
//
// This method assumes there is a server.pem keyfile in dataDir.
func MongoUpstartService(name, dataDir string, port int) (*upstart.Conf, error) {

	keyFile := path.Join(dataDir, "server.pem")
	svc := upstart.NewService(name)

	dbDir := path.Join(dataDir, "db")

	conf := &upstart.Conf{
		Service: *svc,
		Desc:    "juju state database",
		Limit: map[string]string{
			"nofile": fmt.Sprintf("%d %d", maxFiles, maxFiles),
			"nproc":  fmt.Sprintf("%d %d", maxProcs, maxProcs),
		},
		Cmd: "/usr/bin/mongod" +
			" --auth" +
			" --dbpath=" + dbDir +
			" --sslOnNormalPorts" +
			" --sslPEMKeyFile " + utils.ShQuote(keyFile) +
			" --sslPEMKeyPassword ignored" +
			" --bind_ip 0.0.0.0" +
			" --port " + fmt.Sprint(port) +
			" --noprealloc" +
			" --syslog" +
			" --smallfiles",
		// TODO(Nate): uncomment when we commit HA stuff
		// +
		//	" --replSet juju",
	}
	return conf, nil
}
<|endoftext|>


<commit_before>package form

import (
	"net/url"
)

// Errors is an error type that is a map of other errors
type Errors map[string]error

func (Errors) Error() string {
	return "errors encountered"
}

// Parser is an interface used to to parse a specfic type
type Parser interface {
	Parse([]string) error
}

// ParserList is a simple implementation of a parserLister that simple returns
// itself
type ParserList map[string]Parser

// ParserList is an implementation of parserLister
func (p ParserList) ParserList() ParserList {
	return p
}

type parserLister interface {
	ParserList() ParserList
}

// ParseValue parses a single values
func ParseValue(name string, value Parser, data url.Values) error {
	return Parse(ParserList{name: value}, data)
}

// Parse parses the given url.Values into the type given
func Parse(p parserLister, data url.Values) error {
	errs := make(Errors)
	for k, v := range p.ParserList() {
		if d, ok := data[k]; ok {
			if err := v.Parse(d); err != nil {
				errs[k] = err
			}
		}
	}
	if len(errs) > 0 {
		return errs
	}
	return nil
}
<commit_msg>Made ParserLister exported<commit_after>package form

import (
	"net/url"
)

// Errors is an error type that is a map of other errors
type Errors map[string]error

func (Errors) Error() string {
	return "errors encountered"
}

// Parser is an interface used to to parse a specfic type
type Parser interface {
	Parse([]string) error
}

// ParserList is a simple implementation of a parserLister that simple returns
// itself
type ParserList map[string]Parser

// ParserList is an implementation of parserLister
func (p ParserList) ParserList() ParserList {
	return p
}

// ParserLister is the main interface for this package. The single method
// ParserList returns a ParserList map of field names to Parser's
type ParserLister interface {
	ParserList() ParserList
}

// ParseValue parses a single values
func ParseValue(name string, value Parser, data url.Values) error {
	return Parse(ParserList{name: value}, data)
}

// Parse parses the given url.Values into the type given
func Parse(p parserLister, data url.Values) error {
	errs := make(Errors)
	for k, v := range p.ParserList() {
		if d, ok := data[k]; ok {
			if err := v.Parse(d); err != nil {
				errs[k] = err
			}
		}
	}
	if len(errs) > 0 {
		return errs
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2016 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package main

import (
	"archive/tar"
	"compress/gzip"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"cloud.google.com/go/storage"
	"github.com/google/syzkaller/config"
	"github.com/google/syzkaller/gce"
	. "github.com/google/syzkaller/log"
	"golang.org/x/net/context"
)

var (
	flagConfig    = flag.String("config", "", "config file")
	flagNoRebuild = flag.Bool("norebuild", false, "don't download/create image, update/rebuild syzkaller (for testing)")

	cfg           *Config
	ctx           context.Context
	storageClient *storage.Client
	GCE           *gce.Context
)

type Config struct {
	Image_Archive     string
	Image_Path        string
	Image_Name        string
	Http_Port         int
	Manager_Http_Port int
	Machine_Type      string
	Machine_Count     int
	Sandbox           string
	Procs             int
}

func main() {
	flag.Parse()
	cfg = readConfig(*flagConfig)
	EnableLogCaching(1000, 1<<20)
	initHttp(fmt.Sprintf(":%v", cfg.Http_Port))

	gopath, err := filepath.Abs("gopath")
	if err != nil {
		Fatalf("failed to get absolute path: %v", err)
	}
	os.Setenv("GOPATH", gopath)

	ctx = context.Background()
	storageClient, err = storage.NewClient(ctx)
	if err != nil {
		Fatalf("failed to create cloud storage client: %v", err)
	}

	GCE, err = gce.NewContext()
	if err != nil {
		Fatalf("failed to init gce: %v", err)
	}
	Logf(0, "gce initialized: running on %v, internal IP, %v project %v, zone %v", GCE.Instance, GCE.InternalIP, GCE.ProjectID, GCE.ZoneID)

	if !*flagNoRebuild {
		Logf(0, "downloading image archive...")
		archive, updated, err := openFile(cfg.Image_Archive)
		if err != nil {
			Fatalf("%v", err)
		}
		_ = updated
		if err := os.RemoveAll("image"); err != nil {
			Fatalf("failed to remove image dir: %v", err)
		}
		if err := downloadAndExtract(archive, "image"); err != nil {
			Fatalf("failed to download and extract %v: %v", cfg.Image_Archive, err)
		}

		Logf(0, "uploading image...")
		if err := uploadFile("image/disk.tar.gz", cfg.Image_Path); err != nil {
			Fatalf("failed to upload image: %v", err)
		}

		Logf(0, "creating gce image...")
		if err := GCE.DeleteImage(cfg.Image_Name); err != nil {
			Fatalf("failed to delete GCE image: %v", err)
		}
		if err := GCE.CreateImage(cfg.Image_Name, cfg.Image_Path); err != nil {
			Fatalf("failed to create GCE image: %v", err)
		}

		Logf(0, "building syzkaller...")
		syzBin, err := updateSyzkallerBuild()
		if err != nil {
			Fatalf("failed to update/build syzkaller: %v", err)
		}
		_ = syzBin
	}

	Logf(0, "starting syzkaller...")
	if err := writeManagerConfig("manager.cfg"); err != nil {
		Fatalf("failed to write manager config: %v", err)
	}

	manager := exec.Command("gopath/src/github.com/google/syzkaller/bin/syz-manager", "-config=manager.cfg")
	manager.Stdout = os.Stdout
	manager.Stderr = os.Stderr
	if err := manager.Start(); err != nil {
		Fatalf("failed to start syz-manager: %v", err)
	}
	err = manager.Wait()
	Fatalf("syz-manager exited with: %v", err)
}

func readConfig(filename string) *Config {
	if filename == "" {
		Fatalf("supply config in -config flag")
	}
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		Fatalf("failed to read config file: %v", err)
	}
	cfg := new(Config)
	if err := json.Unmarshal(data, cfg); err != nil {
		Fatalf("failed to parse config file: %v", err)
	}
	return cfg
}

func writeManagerConfig(file string) error {
	managerCfg := &config.Config{
		Http:         fmt.Sprintf(":%v", cfg.Manager_Http_Port),
		Rpc:          ":0",
		Workdir:      "workdir",
		Vmlinux:      "image/obj/vmlinux",
		Syzkaller:    "gopath/src/github.com/google/syzkaller",
		Type:         "gce",
		Machine_Type: cfg.Machine_Type,
		Count:        cfg.Machine_Count,
		Image:        cfg.Image_Name,
		Sshkey:       "image/key",
		Sandbox:      cfg.Sandbox,
		Procs:        cfg.Procs,
		Cover:        true,
	}
	data, err := json.MarshalIndent(managerCfg, "", "\t")
	if err != nil {
		return err
	}
	if err := ioutil.WriteFile(file, data, 0600); err != nil {
		return err
	}
	return nil
}

func openFile(file string) (*storage.ObjectHandle, time.Time, error) {
	pos := strings.IndexByte(file, '/')
	if pos == -1 {
		return nil, time.Time{}, fmt.Errorf("invalid GCS file name: %v", file)
	}
	bkt := storageClient.Bucket(file[:pos])
	f := bkt.Object(file[pos+1:])
	attrs, err := f.Attrs(ctx)
	if err != nil {
		return nil, time.Time{}, fmt.Errorf("failed to read %v attributes: %v", file, err)
	}
	if !attrs.Deleted.IsZero() {
		return nil, time.Time{}, fmt.Errorf("file %v is deleted", file)
	}
	f = f.WithConditions(
		storage.IfGenerationMatch(attrs.Generation),
		storage.IfMetaGenerationMatch(attrs.MetaGeneration),
	)
	return f, attrs.Updated, nil
}

func downloadAndExtract(f *storage.ObjectHandle, dir string) error {
	r, err := f.NewReader(ctx)
	if err != nil {
		return err
	}
	defer r.Close()
	gz, err := gzip.NewReader(r)
	if err != nil {
		return err
	}
	ar := tar.NewReader(gz)
	for {
		hdr, err := ar.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}
		Logf(0, "extracting file: %v (%v bytes)", hdr.Name, hdr.Size)
		if len(hdr.Name) == 0 || hdr.Name[len(hdr.Name)-1] == '/' {
			continue
		}
		base, file := filepath.Split(hdr.Name)
		if err := os.MkdirAll(filepath.Join(dir, base), 0700); err != nil {
			return err
		}
		dst, err := os.OpenFile(filepath.Join(dir, base, file), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
		if err != nil {
			return err
		}
		_, err = io.Copy(dst, ar)
		dst.Close()
		if err != nil {
			return err
		}
	}
	return nil
}

func uploadFile(localFile string, gcsFile string) error {
	local, err := os.Open(localFile)
	if err != nil {
		return err
	}
	defer local.Close()
	pos := strings.IndexByte(gcsFile, '/')
	if pos == -1 {
		return fmt.Errorf("invalid GCS file name: %v", gcsFile)
	}
	bkt := storageClient.Bucket(gcsFile[:pos])
	f := bkt.Object(gcsFile[pos+1:])
	w := f.NewWriter(ctx)
	defer w.Close()
	io.Copy(w, local)
	return nil
}

func updateSyzkallerBuild() (string, error) {
	goGet := exec.Command("go", "get", "-u", "-d", "github.com/google/syzkaller/syz-manager", "github.com/google/syzkaller/syz-gce")
	if output, err := goGet.CombinedOutput(); err != nil {
		return "", fmt.Errorf("%v\n%s", err, output)
	}
	makeCmd := exec.Command("make")
	makeCmd.Dir = "gopath/src/github.com/google/syzkaller"
	if output, err := makeCmd.CombinedOutput(); err != nil {
		return "", fmt.Errorf("%v\n%s", err, output)
	}
	return "gopath/src/github.com/google/syzkaller/bin", nil
}
<commit_msg>syz-gce: allow to not recreate image, rebuild syzkaller<commit_after>// Copyright 2016 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package main

import (
	"archive/tar"
	"compress/gzip"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"cloud.google.com/go/storage"
	"github.com/google/syzkaller/config"
	"github.com/google/syzkaller/gce"
	. "github.com/google/syzkaller/log"
	"golang.org/x/net/context"
)

var (
	flagConfig        = flag.String("config", "", "config file")
	flagNoImageCreate = flag.Bool("noimagecreate", false, "don't download/create image (for testing)")
	flagNoRebuild     = flag.Bool("norebuild", false, "don't update/rebuild syzkaller (for testing)")

	cfg           *Config
	ctx           context.Context
	storageClient *storage.Client
	GCE           *gce.Context
)

type Config struct {
	Image_Archive     string
	Image_Path        string
	Image_Name        string
	Http_Port         int
	Manager_Http_Port int
	Machine_Type      string
	Machine_Count     int
	Sandbox           string
	Procs             int
}

func main() {
	flag.Parse()
	cfg = readConfig(*flagConfig)
	EnableLogCaching(1000, 1<<20)
	initHttp(fmt.Sprintf(":%v", cfg.Http_Port))

	gopath, err := filepath.Abs("gopath")
	if err != nil {
		Fatalf("failed to get absolute path: %v", err)
	}
	os.Setenv("GOPATH", gopath)

	ctx = context.Background()
	storageClient, err = storage.NewClient(ctx)
	if err != nil {
		Fatalf("failed to create cloud storage client: %v", err)
	}

	GCE, err = gce.NewContext()
	if err != nil {
		Fatalf("failed to init gce: %v", err)
	}
	Logf(0, "gce initialized: running on %v, internal IP, %v project %v, zone %v", GCE.Instance, GCE.InternalIP, GCE.ProjectID, GCE.ZoneID)

	if !*flagNoImageCreate {
		Logf(0, "downloading image archive...")
		archive, updated, err := openFile(cfg.Image_Archive)
		if err != nil {
			Fatalf("%v", err)
		}
		_ = updated
		if err := os.RemoveAll("image"); err != nil {
			Fatalf("failed to remove image dir: %v", err)
		}
		if err := downloadAndExtract(archive, "image"); err != nil {
			Fatalf("failed to download and extract %v: %v", cfg.Image_Archive, err)
		}

		Logf(0, "uploading image...")
		if err := uploadFile("image/disk.tar.gz", cfg.Image_Path); err != nil {
			Fatalf("failed to upload image: %v", err)
		}

		Logf(0, "creating gce image...")
		if err := GCE.DeleteImage(cfg.Image_Name); err != nil {
			Fatalf("failed to delete GCE image: %v", err)
		}
		if err := GCE.CreateImage(cfg.Image_Name, cfg.Image_Path); err != nil {
			Fatalf("failed to create GCE image: %v", err)
		}
	}

	if !*flagNoRebuild {
		Logf(0, "building syzkaller...")
		syzBin, err := updateSyzkallerBuild()
		if err != nil {
			Fatalf("failed to update/build syzkaller: %v", err)
		}
		_ = syzBin
	}

	Logf(0, "starting syzkaller...")
	if err := writeManagerConfig("manager.cfg"); err != nil {
		Fatalf("failed to write manager config: %v", err)
	}

	manager := exec.Command("gopath/src/github.com/google/syzkaller/bin/syz-manager", "-config=manager.cfg")
	manager.Stdout = os.Stdout
	manager.Stderr = os.Stderr
	if err := manager.Start(); err != nil {
		Fatalf("failed to start syz-manager: %v", err)
	}
	err = manager.Wait()
	Fatalf("syz-manager exited with: %v", err)
}

func readConfig(filename string) *Config {
	if filename == "" {
		Fatalf("supply config in -config flag")
	}
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		Fatalf("failed to read config file: %v", err)
	}
	cfg := new(Config)
	if err := json.Unmarshal(data, cfg); err != nil {
		Fatalf("failed to parse config file: %v", err)
	}
	return cfg
}

func writeManagerConfig(file string) error {
	managerCfg := &config.Config{
		Http:         fmt.Sprintf(":%v", cfg.Manager_Http_Port),
		Rpc:          ":0",
		Workdir:      "workdir",
		Vmlinux:      "image/obj/vmlinux",
		Syzkaller:    "gopath/src/github.com/google/syzkaller",
		Type:         "gce",
		Machine_Type: cfg.Machine_Type,
		Count:        cfg.Machine_Count,
		Image:        cfg.Image_Name,
		Sshkey:       "image/key",
		Sandbox:      cfg.Sandbox,
		Procs:        cfg.Procs,
		Cover:        true,
	}
	data, err := json.MarshalIndent(managerCfg, "", "\t")
	if err != nil {
		return err
	}
	if err := ioutil.WriteFile(file, data, 0600); err != nil {
		return err
	}
	return nil
}

func openFile(file string) (*storage.ObjectHandle, time.Time, error) {
	pos := strings.IndexByte(file, '/')
	if pos == -1 {
		return nil, time.Time{}, fmt.Errorf("invalid GCS file name: %v", file)
	}
	bkt := storageClient.Bucket(file[:pos])
	f := bkt.Object(file[pos+1:])
	attrs, err := f.Attrs(ctx)
	if err != nil {
		return nil, time.Time{}, fmt.Errorf("failed to read %v attributes: %v", file, err)
	}
	if !attrs.Deleted.IsZero() {
		return nil, time.Time{}, fmt.Errorf("file %v is deleted", file)
	}
	f = f.WithConditions(
		storage.IfGenerationMatch(attrs.Generation),
		storage.IfMetaGenerationMatch(attrs.MetaGeneration),
	)
	return f, attrs.Updated, nil
}

func downloadAndExtract(f *storage.ObjectHandle, dir string) error {
	r, err := f.NewReader(ctx)
	if err != nil {
		return err
	}
	defer r.Close()
	gz, err := gzip.NewReader(r)
	if err != nil {
		return err
	}
	ar := tar.NewReader(gz)
	for {
		hdr, err := ar.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}
		Logf(0, "extracting file: %v (%v bytes)", hdr.Name, hdr.Size)
		if len(hdr.Name) == 0 || hdr.Name[len(hdr.Name)-1] == '/' {
			continue
		}
		base, file := filepath.Split(hdr.Name)
		if err := os.MkdirAll(filepath.Join(dir, base), 0700); err != nil {
			return err
		}
		dst, err := os.OpenFile(filepath.Join(dir, base, file), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
		if err != nil {
			return err
		}
		_, err = io.Copy(dst, ar)
		dst.Close()
		if err != nil {
			return err
		}
	}
	return nil
}

func uploadFile(localFile string, gcsFile string) error {
	local, err := os.Open(localFile)
	if err != nil {
		return err
	}
	defer local.Close()
	pos := strings.IndexByte(gcsFile, '/')
	if pos == -1 {
		return fmt.Errorf("invalid GCS file name: %v", gcsFile)
	}
	bkt := storageClient.Bucket(gcsFile[:pos])
	f := bkt.Object(gcsFile[pos+1:])
	w := f.NewWriter(ctx)
	defer w.Close()
	io.Copy(w, local)
	return nil
}

func updateSyzkallerBuild() (string, error) {
	goGet := exec.Command("go", "get", "-u", "-d", "github.com/google/syzkaller/syz-manager", "github.com/google/syzkaller/syz-gce")
	if output, err := goGet.CombinedOutput(); err != nil {
		return "", fmt.Errorf("%v\n%s", err, output)
	}
	makeCmd := exec.Command("make")
	makeCmd.Dir = "gopath/src/github.com/google/syzkaller"
	if output, err := makeCmd.CombinedOutput(); err != nil {
		return "", fmt.Errorf("%v\n%s", err, output)
	}
	return "gopath/src/github.com/google/syzkaller/bin", nil
}
<|endoftext|>


<commit_before>/*
Package ftcp implements a basic framed messaging protocol over TCP/TLS, based on
github.com/oxtoacart/framed.

ftcp can work with both plain text connections (see Dial()) and TLS connections
(see DialTLS()).

Example:

	package main

	import (
		"github.com/oxtoacart/ftcp"
	)

	func main() {
		// Replace host:port with an actual TCP server, for example the echo service
		if conn, err := ftcp.Dial("host:port"); err == nil {
			framedConn.Write([]byte("Hello World"))
			msg := framedConn.Read()
		}
	}

TODO: add auto-reconnect functionality
*/
package ftcp

import (
	"crypto/tls"
	"github.com/oxtoacart/framed"
	"net"
)

/*
Message encapsulates a message received from an ftcp connection, including both
the data (payload) of the message and, if using TLS, the tls.ConnectionState.
*/
type Message struct {
	data     []byte
	TLSState tls.ConnectionState
}

/*
Conn is an ftcp connection to which one can write []byte frames using Write()
and from which one can receive Messages using Read().

Multiple goroutines may invoke methods on a Conn simultaneously.
*/
type Conn struct {
	stream   framed.Framed
	orig     interface{}
	writeCh  chan []byte
	readCh   chan []byte
	messages chan Message
	errors   chan error
	closed   bool
}

/*
Listener is a thin wrapper around net.Listener that allows accepting new
connections using Accept().
*/
type Listener struct {
	net.Listener
}

/*
Dial opens a tcp connection to the given address, similarly to net.Dial.
*/
func Dial(addr string) (conn Conn, err error) {
	var orig net.Conn
	if orig, err = net.Dial("tcp", addr); err == nil {
		conn = newConn(orig)
	}
	return
}

/*
Dial opens a TLS connection to the given address with the given (optional)
tls.Config, similarly to tls.Dial.
*/
func DialTLS(addr string, config *tls.Config) (conn Conn, err error) {
	var orig *tls.Conn
	if orig, err = tls.Dial("tcp", addr, config); err == nil {
		conn = newConn(orig)
	}
	return
}

/*
Listen listens on a TCP socket at the given listen address, similarly to
net.Listen.
*/
func Listen(laddr string) (listener Listener, err error) {
	if orig, err := net.Listen("tcp", laddr); err == nil {
		listener = Listener{orig}
	}
	return
}

/*
ListenTLS listens on a TLS socket at the given listen address with the given
(optional) tls.Config, similarly to tls.Listen.
*/
func ListenTLS(laddr string, config *tls.Config) (listener Listener, err error) {
	if orig, err := tls.Listen("tcp", laddr, config); err == nil {
		listener = Listener{orig}
	}
	return
}

/*
Accept accepts a new connection on, similarly to net.Listener.Accept.
*/
func (listener *Listener) Accept() (conn Conn, err error) {
	var orig net.Conn
	if orig, err = listener.Listener.Accept(); err == nil {
		conn = newConn(orig)
	}
	return
}

/*
Write requests a write of the given message frame to the connection.
*/
func (conn Conn) Write(msg []byte) {
	conn.writeCh <- msg
}

/*
Read reads the next message to arrive on the connection.
*/
func (conn Conn) Read() (msg Message, err error) {
	select {
	case msg = <-conn.messages:
		return
	case err = <-conn.errors:
		return
	}
}

/*
Close closes the connection.
*/
func (conn Conn) Close() (err error) {
	switch orig := conn.orig.(type) {
	case *net.Conn:
		err = (*orig).Close()
	case *tls.Conn:
		err = orig.Close()
	}
	conn.closed = true
	return
}

/*
newConn creates a new connection and starts reading/writing to it.
*/
func newConn(orig net.Conn) (conn Conn) {
	conn = Conn{
		stream:   framed.Framed{orig},
		orig:     &orig,
		writeCh:  make(chan []byte),
		readCh:   make(chan []byte),
		messages: make(chan Message),
		errors:   make(chan error),
		closed:   false,
	}

	go conn.read()
	go conn.process()

	return
}

/*
Read on goroutine.  Doing our reads on a single goroutine ensures that length
prefixes and their corresponding frames are read in the correct order.
*/
func (conn Conn) read() {
	for conn.closed == false {
		if frame, err := conn.stream.ReadFrame(); err != nil {
			// TODO: catch EOF and try reconnecting
			conn.errors <- err
		} else {
			conn.readCh <- frame
		}
	}
}

/*
Process requests to write and manage connection on a single goroutine.
*/
func (conn Conn) process() {
	for conn.closed == false {
		select {
		case frame := <-conn.writeCh:
			if err := conn.stream.WriteFrame(frame); err != nil {
				// TODO: catch EOF and try reconnecting
				conn.errors <- err
			}
		case frame := <-conn.readCh:
			var connectionState tls.ConnectionState
			switch orig := conn.orig.(type) {
			case *tls.Conn:
				connectionState = orig.ConnectionState()
			}
			conn.messages <- Message{frame, connectionState}
		}
	}
}
<commit_msg>Godoc update<commit_after>/*
Package ftcp implements a basic framed messaging protocol over TCP/TLS, based on
github.com/oxtoacart/framed.

ftcp can work with both plain text connections (see Dial()) and TLS connections
(see DialTLS()).

Example:

	package main

	import (
		"github.com/oxtoacart/ftcp"
		"log"
	)

	func main() {
		// Replace host:port with an actual TCP server, for example the echo service
		if conn, err := ftcp.Dial("host:port"); err == nil {
			if err := framedConn.Write([]byte("Hello World")); err == nil {
				if msg, err := framedConn.Read(); err == nil {
					log.Println("We're done!")
				}
			}
		}
	}

TODO: add auto-reconnect functionality
*/
package ftcp

import (
	"crypto/tls"
	"github.com/oxtoacart/framed"
	"net"
)

/*
Message encapsulates a message received from an ftcp connection, including both
the data (payload) of the message and, if using TLS, the tls.ConnectionState.
*/
type Message struct {
	data     []byte
	TLSState tls.ConnectionState
}

/*
Conn is an ftcp connection to which one can write []byte frames using Write()
and from which one can receive Messages using Read().

Multiple goroutines may invoke methods on a Conn simultaneously.
*/
type Conn struct {
	stream   framed.Framed
	orig     interface{}
	writeCh  chan []byte
	readCh   chan []byte
	messages chan Message
	errors   chan error
	closed   bool
}

/*
Listener is a thin wrapper around net.Listener that allows accepting new
connections using Accept().
*/
type Listener struct {
	net.Listener
}

/*
Dial opens a tcp connection to the given address, similarly to net.Dial.
*/
func Dial(addr string) (conn Conn, err error) {
	var orig net.Conn
	if orig, err = net.Dial("tcp", addr); err == nil {
		conn = newConn(orig)
	}
	return
}

/*
Dial opens a TLS connection to the given address with the given (optional)
tls.Config, similarly to tls.Dial.
*/
func DialTLS(addr string, config *tls.Config) (conn Conn, err error) {
	var orig *tls.Conn
	if orig, err = tls.Dial("tcp", addr, config); err == nil {
		conn = newConn(orig)
	}
	return
}

/*
Listen listens on a TCP socket at the given listen address, similarly to
net.Listen.
*/
func Listen(laddr string) (listener Listener, err error) {
	if orig, err := net.Listen("tcp", laddr); err == nil {
		listener = Listener{orig}
	}
	return
}

/*
ListenTLS listens on a TLS socket at the given listen address with the given
(optional) tls.Config, similarly to tls.Listen.
*/
func ListenTLS(laddr string, config *tls.Config) (listener Listener, err error) {
	if orig, err := tls.Listen("tcp", laddr, config); err == nil {
		listener = Listener{orig}
	}
	return
}

/*
Accept accepts a new connection on, similarly to net.Listener.Accept.
*/
func (listener *Listener) Accept() (conn Conn, err error) {
	var orig net.Conn
	if orig, err = listener.Listener.Accept(); err == nil {
		conn = newConn(orig)
	}
	return
}

/*
Write requests a write of the given message frame to the connection.
*/
func (conn Conn) Write(msg []byte) {
	conn.writeCh <- msg
}

/*
Read reads the next message to arrive on the connection.
*/
func (conn Conn) Read() (msg Message, err error) {
	select {
	case msg = <-conn.messages:
		return
	case err = <-conn.errors:
		return
	}
}

/*
Close closes the connection.
*/
func (conn Conn) Close() (err error) {
	switch orig := conn.orig.(type) {
	case *net.Conn:
		err = (*orig).Close()
	case *tls.Conn:
		err = orig.Close()
	}
	conn.closed = true
	return
}

/*
newConn creates a new connection and starts reading/writing to it.
*/
func newConn(orig net.Conn) (conn Conn) {
	conn = Conn{
		stream:   framed.Framed{orig},
		orig:     &orig,
		writeCh:  make(chan []byte),
		readCh:   make(chan []byte),
		messages: make(chan Message),
		errors:   make(chan error),
		closed:   false,
	}

	go conn.read()
	go conn.process()

	return
}

/*
Read on goroutine.  Doing our reads on a single goroutine ensures that length
prefixes and their corresponding frames are read in the correct order.
*/
func (conn Conn) read() {
	for conn.closed == false {
		if frame, err := conn.stream.ReadFrame(); err != nil {
			// TODO: catch EOF and try reconnecting
			conn.errors <- err
		} else {
			conn.readCh <- frame
		}
	}
}

/*
Process requests to write and manage connection on a single goroutine.
*/
func (conn Conn) process() {
	for conn.closed == false {
		select {
		case frame := <-conn.writeCh:
			if err := conn.stream.WriteFrame(frame); err != nil {
				// TODO: catch EOF and try reconnecting
				conn.errors <- err
			}
		case frame := <-conn.readCh:
			var connectionState tls.ConnectionState
			switch orig := conn.orig.(type) {
			case *tls.Conn:
				connectionState = orig.ConnectionState()
			}
			conn.messages <- Message{frame, connectionState}
		}
	}
}
<|endoftext|>


<commit_before>package aws

import (
	"context"
	"errors"
	"fmt"
	"net/http"
	"time"

	cloudTypes "github.com/atlassian/gostatsd/cloudprovider/types"
	"github.com/atlassian/gostatsd/types"

	log "github.com/Sirupsen/logrus"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds"
	"github.com/aws/aws-sdk-go/aws/ec2metadata"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/spf13/viper"
)

const (
	// ProviderName is the name of AWS cloud provider.
	ProviderName = "aws"
)

const sampleConfig = `
[aws]
	# maximum number of retries in case of retriable errors
	max_retries = 5 # optional, default to 3
`

// Provider represents an AWS provider.
type Provider struct {
	Metadata *ec2metadata.EC2Metadata
	Ec2      *ec2.EC2
}

func newEc2Filter(name string, value string) *ec2.Filter {
	return &ec2.Filter{
		Name: aws.String(name),
		Values: []*string{
			aws.String(value),
		},
	}
}

// Instance returns the instance details from aws.
func (p *Provider) Instance(ctx context.Context, IP types.IP) (*cloudTypes.Instance, error) {
	req, _ := p.Ec2.DescribeInstancesRequest(&ec2.DescribeInstancesInput{
		Filters: []*ec2.Filter{
			newEc2Filter("private-ip-address", string(IP)),
		},
	})
	req.HTTPRequest = req.HTTPRequest.WithContext(ctx)
	var inst *ec2.Instance
	err := req.EachPage(func(data interface{}, isLastPage bool) bool {
		for _, reservation := range data.(*ec2.DescribeInstancesOutput).Reservations {
			for _, instance := range reservation.Instances {
				inst = instance
				return false
			}
		}
		return true
	})
	if err != nil {
		return nil, fmt.Errorf("error listing AWS instances: %v", err)
	}
	if inst == nil {
		return nil, errors.New("no instances found")
	}
	region, err := azToRegion(aws.StringValue(inst.Placement.AvailabilityZone))
	if err != nil {
		log.Errorf("Error getting instance region: %v", err)
	}
	tags := make(types.Tags, len(inst.Tags))
	for idx, tag := range inst.Tags {
		tags[idx] = fmt.Sprintf("%s:%s",
			types.NormalizeTagKey(aws.StringValue(tag.Key)),
			aws.StringValue(tag.Value))
	}
	instance := &cloudTypes.Instance{
		ID:     aws.StringValue(inst.InstanceId),
		Region: region,
		Tags:   tags,
	}
	return instance, nil
}

// ProviderName returns the name of the provider.
func (p *Provider) ProviderName() string {
	return ProviderName
}

// SampleConfig returns the sample config for the datadog backend.
func (p *Provider) SampleConfig() string {
	return sampleConfig
}

// SelfIP returns host's IPv4 address.
func (p *Provider) SelfIP() (types.IP, error) {
	ip, err := p.Metadata.GetMetadata("local-ipv4")
	return types.IP(ip), err
}

// Derives the region from a valid az name.
// Returns an error if the az is known invalid (empty).
func azToRegion(az string) (string, error) {
	if az == "" {
		return "", errors.New("invalid (empty) AZ")
	}
	region := az[:len(az)-1]
	return region, nil
}

// NewProviderFromViper returns a new aws provider.
func NewProviderFromViper(v *viper.Viper) (cloudTypes.Interface, error) {
	a := getSubViper(v, "aws")
	a.SetDefault("max_retries", 3)
	a.SetDefault("http_timeout", 3*time.Second)
	httpTimeout := a.GetDuration("http_timeout")
	if httpTimeout <= 0 {
		return nil, errors.New("http client timeout must be positive")
	}

	// This is the main config without credentials.
	config := &aws.Config{
		MaxRetries: aws.Int(a.GetInt("max_retries")),
		HTTPClient: &http.Client{
			Timeout: httpTimeout,
		},
	}
	metadata := ec2metadata.New(session.New(config))
	az, err := metadata.GetMetadata("placement/availability-zone")
	if err != nil {
		return nil, fmt.Errorf("error getting availability zone: %v", err)
	}
	region, err := azToRegion(az)
	if err != nil {
		return nil, fmt.Errorf("error getting aws region: %v", err)
	}
	return &Provider{
		Metadata: metadata,
		Ec2: ec2.New(session.New(config.Copy(&aws.Config{
			Credentials: credentials.NewChainCredentials(
				[]credentials.Provider{
					&credentials.EnvProvider{},
					&ec2rolecreds.EC2RoleProvider{
						Client: metadata,
					},
					&credentials.SharedCredentialsProvider{},
				}),
			Region: aws.String(region),
		}))),
	}, nil
}

func getSubViper(v *viper.Viper, key string) *viper.Viper {
	n := v.Sub(key)
	if n == nil {
		n = viper.New()
	}
	return n
}
<commit_msg>TLS and timeouts config, add proxy support<commit_after>package aws

import (
	"context"
	"crypto/tls"
	"errors"
	"fmt"
	"net"
	"net/http"
	"time"

	cloudTypes "github.com/atlassian/gostatsd/cloudprovider/types"
	"github.com/atlassian/gostatsd/types"

	log "github.com/Sirupsen/logrus"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds"
	"github.com/aws/aws-sdk-go/aws/ec2metadata"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/spf13/viper"
)

const (
	// ProviderName is the name of AWS cloud provider.
	ProviderName = "aws"
)

const sampleConfig = `
[aws]
	# maximum number of retries in case of retriable errors
	max_retries = 5 # optional, default to 3
`

// Provider represents an AWS provider.
type Provider struct {
	Metadata *ec2metadata.EC2Metadata
	Ec2      *ec2.EC2
}

func newEc2Filter(name string, value string) *ec2.Filter {
	return &ec2.Filter{
		Name: aws.String(name),
		Values: []*string{
			aws.String(value),
		},
	}
}

// Instance returns the instance details from aws.
func (p *Provider) Instance(ctx context.Context, IP types.IP) (*cloudTypes.Instance, error) {
	req, _ := p.Ec2.DescribeInstancesRequest(&ec2.DescribeInstancesInput{
		Filters: []*ec2.Filter{
			newEc2Filter("private-ip-address", string(IP)),
		},
	})
	req.HTTPRequest = req.HTTPRequest.WithContext(ctx)
	var inst *ec2.Instance
	err := req.EachPage(func(data interface{}, isLastPage bool) bool {
		for _, reservation := range data.(*ec2.DescribeInstancesOutput).Reservations {
			for _, instance := range reservation.Instances {
				inst = instance
				return false
			}
		}
		return true
	})
	if err != nil {
		return nil, fmt.Errorf("error listing AWS instances: %v", err)
	}
	if inst == nil {
		return nil, errors.New("no instances found")
	}
	region, err := azToRegion(aws.StringValue(inst.Placement.AvailabilityZone))
	if err != nil {
		log.Errorf("Error getting instance region: %v", err)
	}
	tags := make(types.Tags, len(inst.Tags))
	for idx, tag := range inst.Tags {
		tags[idx] = fmt.Sprintf("%s:%s",
			types.NormalizeTagKey(aws.StringValue(tag.Key)),
			aws.StringValue(tag.Value))
	}
	instance := &cloudTypes.Instance{
		ID:     aws.StringValue(inst.InstanceId),
		Region: region,
		Tags:   tags,
	}
	return instance, nil
}

// ProviderName returns the name of the provider.
func (p *Provider) ProviderName() string {
	return ProviderName
}

// SampleConfig returns the sample config for the datadog backend.
func (p *Provider) SampleConfig() string {
	return sampleConfig
}

// SelfIP returns host's IPv4 address.
func (p *Provider) SelfIP() (types.IP, error) {
	ip, err := p.Metadata.GetMetadata("local-ipv4")
	return types.IP(ip), err
}

// Derives the region from a valid az name.
// Returns an error if the az is known invalid (empty).
func azToRegion(az string) (string, error) {
	if az == "" {
		return "", errors.New("invalid (empty) AZ")
	}
	region := az[:len(az)-1]
	return region, nil
}

// NewProviderFromViper returns a new aws provider.
func NewProviderFromViper(v *viper.Viper) (cloudTypes.Interface, error) {
	a := getSubViper(v, "aws")
	a.SetDefault("max_retries", 3)
	a.SetDefault("http_timeout", 3*time.Second)
	httpTimeout := a.GetDuration("http_timeout")
	if httpTimeout <= 0 {
		return nil, errors.New("http client timeout must be positive")
	}

	// This is the main config without credentials.
	config := &aws.Config{
		MaxRetries: aws.Int(a.GetInt("max_retries")),
		HTTPClient: &http.Client{
			Transport: &http.Transport{
				Proxy:               http.ProxyFromEnvironment,
				TLSHandshakeTimeout: 5 * time.Second,
				TLSClientConfig: &tls.Config{
					// Can't use SSLv3 because of POODLE and BEAST
					// Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher
					// Can't use TLSv1.1 because of RC4 cipher usage
					MinVersion: tls.VersionTLS12,
				},
				DialContext: (&net.Dialer{
					Timeout:   5 * time.Second,
					KeepAlive: 30 * time.Second,
				}).DialContext,
			},
			Timeout: httpTimeout,
		},
	}
	metadata := ec2metadata.New(session.New(config))
	az, err := metadata.GetMetadata("placement/availability-zone")
	if err != nil {
		return nil, fmt.Errorf("error getting availability zone: %v", err)
	}
	region, err := azToRegion(az)
	if err != nil {
		return nil, fmt.Errorf("error getting aws region: %v", err)
	}
	return &Provider{
		Metadata: metadata,
		Ec2: ec2.New(session.New(config.Copy(&aws.Config{
			Credentials: credentials.NewChainCredentials(
				[]credentials.Provider{
					&credentials.EnvProvider{},
					&ec2rolecreds.EC2RoleProvider{
						Client: metadata,
					},
					&credentials.SharedCredentialsProvider{},
				}),
			Region: aws.String(region),
		}))),
	}, nil
}

func getSubViper(v *viper.Viper, key string) *viper.Viper {
	n := v.Sub(key)
	if n == nil {
		n = viper.New()
	}
	return n
}
<|endoftext|>


<commit_before>package scanner

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"unicode"

	"github.com/FarmRadioHangar/fessboxconfig/ast"
)

const eof = rune(-1)

// Scanner is a lexical scanner for scanning configuration files.
// This works only on UTF-& text.
type Scanner struct {
	r       *bufio.Reader
	txt     *bytes.Buffer
	currPos int
	line    int
	err     error
	column  int
}

// NewScanner takes src and returns a new Scanner.
func NewScanner(src io.Reader) *Scanner {
	return &Scanner{
		r:   bufio.NewReader(src),
		txt: &bytes.Buffer{},
	}
}

//Scan returns a new token for every call by advancing on the consumed UTF-8
//encoded input text.
//
// Anything after ; is considered a comment. White space is preserved together
// with  new lines. New lines and spaces are interpreted differently.
func (s *Scanner) Scan() (*ast.Token, error) {
	ch := s.peek()
	if isIdent(ch) {
		return s.scanIdent()
	}
	switch ch {
	case ';':
		return s.scanComment()
	case ' ', '\t':
		return s.scanWhitespace()
	case '\n', '\r':
		return s.scanNewline()
	case '=':
		return s.scanRune(ast.Assign)
	case '[':
		return s.scanRune(ast.LBrace)
	case ']':
		return s.scanRune(ast.RBrace)
	case '(':
		return s.scanRune(ast.LBracket)
	case ')':
		return s.scanRune(ast.RBracket)
	case '!':
		return s.scanRune(ast.Exclam)
	case eof:
		return nil, io.EOF
	}
	return nil, errors.New("unrecognized token " + string(ch))
}

//scanComment scans the input for Comments, only single line comments are
//supported.
//
// A comment is all the text that is after a comment identifier, This does not
// enforce the identifier, so it is up to the caller to decide where the comment
// starts, this will read all the text up to the end of the line and return it
// as a single comment token.
//
// TODO(gernest) accept the comment identifier, or check whether the first
// rune is the supported token identifier.
func (s *Scanner) scanComment() (*ast.Token, error) {
	tok := &ast.Token{}
	buf := &bytes.Buffer{}
	isBlock := false
	for _ = range make([]struct{}, 4) {
		ch, _, err := s.r.ReadRune()
		if err != nil {
			if err.Error() == io.EOF.Error() {
				goto final
			}
			return nil, err
		}
		buf.WriteRune(ch)
	}

	if buf.String() == ";-- " {
		isBlock = true
	}
END:
	for {
	begin:
		ch, _, err := s.r.ReadRune()
		if err != nil {
			if err.Error() == io.EOF.Error() {
				fmt.Println("END")
				break END
			}
			return nil, err
		}
		switch ch {
		case '\n', '\r':
			if isBlock {
				buf.WriteRune(ch)
				goto begin
			}
			_ = s.r.UnreadRune()
			break END
		case '-':
			if isBlock {
				var str string
				for _ = range make([]struct{}, 2) {
					ch, _, err = s.r.ReadRune()
					if err != nil {
						if err.Error() == io.EOF.Error() {

							goto final
						}
						return nil, err
					}
					str += string(ch)
				}
				buf.WriteString(str)
				if str == "-;" {
					break END
				}
			}
			fallthrough
		default:
			_, _ = buf.WriteRune(ch)
		}
	}
final:
	s.column++
	//fmt.Printf(" HERE  %d %d \n", s.currPos, buf.Len())
	tok.Begin = s.currPos
	s.currPos += buf.Len() // advance the current position
	tok.End = s.currPos
	tok.Column = s.column
	tok.Type = ast.Comment
	tok.Text = buf.String()
	tok.Line = s.line
	return tok, nil
}

//scanWhitespace scans all utf-8 white space characters until it hits a non
//whitespace character.
//
// Tabs ('\t') and space(' ') all represent white space.
func (s *Scanner) scanWhitespace() (*ast.Token, error) {
	tok := &ast.Token{}

	// There can be arbitrary spaces so we need to bugger them up.
	buf := &bytes.Buffer{}
END:
	for {
		ch, _, err := s.r.ReadRune()
		if err != nil {
			if err.Error() == io.EOF.Error() {
				break END
			}
			return nil, err
		}
		switch ch {
		case ' ', '\t':
			_, _ = buf.WriteRune(ch)
		default:
			// Stop after hitting non whitespace character
			// Reseting the buffer is necessary so that the scanned character can be
			// accessed for the next call to Scan method.
			_ = s.r.UnreadRune()
			break END
		}
	}
	tok.Column = s.column
	tok.Begin = s.currPos
	s.currPos += buf.Len()
	tok.End = s.currPos
	tok.Type = ast.WhiteSpace
	tok.Text = buf.String()
	tok.Line = s.line
	return tok, nil
}

//scanNewline returns a token of type NewLine. It is necessary to separate
//newlines from normal spaces because many configuration files formats make use
//of new lines.
//
// A new line can either be a carriage return( '\r') or a new line
// character('\n')
//
// TODO(gernest) accept a new line character as input.
func (s *Scanner) scanNewline() (*ast.Token, error) {
	ch, size, err := s.r.ReadRune()
	if err != nil {
		return nil, err
	}
	tok := &ast.Token{}
	tok.Type = ast.NLine
	tok.Text = string(ch)
	tok.Begin = s.currPos
	s.currPos += size
	tok.End = s.currPos
	s.column = 0
	s.line++
	tok.Column = s.column
	tok.Line = s.line
	return tok, nil
}

//isIdent returns true if ch is a valid identifier
// valid identifiers are
//	underscore _
//	dash -
//	plus +
//	a unicode letter a-zA-Z
//	a unicode digit 0-9
func isIdent(ch rune) bool {
	return ch == '_' || ch == '-' || ch == '+' || unicode.IsLetter(ch) || unicode.IsDigit(ch)
}

//scanIdent returns the current character in the input source as an Ident Token
//
// TODO(gernest) Accept the character as input argument.
func (s *Scanner) scanIdent() (*ast.Token, error) {
	return s.scanRune(ast.Ident)
}

// scanRune scans the current rune and returns a token of type typ, whose Text
// is the scanned character
//
// Use this for single character tokens
func (s *Scanner) scanRune(typ ast.TokenType) (*ast.Token, error) {
	ch, size, err := s.r.ReadRune()
	if err != nil {
		return nil, err
	}
	tok := &ast.Token{}
	tok.Type = typ
	tok.Text = string(ch)
	tok.Begin = s.currPos
	s.currPos += size
	tok.End = s.currPos
	s.column++
	tok.Column = s.column
	tok.Line = s.line
	return tok, nil
}

// peek returns the next rune in the input buffer but does not advance the
// position of the current buffer.
//
// This is a safe way to peek at the next  rune character without actually
// reading it.
func (s *Scanner) peek() rune {
	ch, _, err := s.r.ReadRune()
	if err != nil {
		if err.Error() == io.EOF.Error() {
			return eof
		}
		panic(err)
	}
	_ = s.r.UnreadRune()
	return ch
}
<commit_msg>Fix scanning for block comments<commit_after>package scanner

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"unicode"

	"github.com/FarmRadioHangar/fessboxconfig/ast"
)

const eof = rune(-1)

// Scanner is a lexical scanner for scanning configuration files.
// This works only on UTF-& text.
type Scanner struct {
	r       *bufio.Reader
	txt     *bytes.Buffer
	currPos int
	line    int
	err     error
	column  int
}

// NewScanner takes src and returns a new Scanner.
func NewScanner(src io.Reader) *Scanner {
	return &Scanner{
		r:   bufio.NewReader(src),
		txt: &bytes.Buffer{},
	}
}

//Scan returns a new token for every call by advancing on the consumed UTF-8
//encoded input text.
//
// Anything after ; is considered a comment. White space is preserved together
// with  new lines. New lines and spaces are interpreted differently.
func (s *Scanner) Scan() (*ast.Token, error) {
	ch := s.peek()
	if isIdent(ch) {
		return s.scanIdent()
	}
	switch ch {
	case ';':
		return s.scanComment()
	case ' ', '\t':
		return s.scanWhitespace()
	case '\n', '\r':
		return s.scanNewline()
	case '=':
		return s.scanRune(ast.Assign)
	case '[':
		return s.scanRune(ast.LBrace)
	case ']':
		return s.scanRune(ast.RBrace)
	case '(':
		return s.scanRune(ast.LBracket)
	case ')':
		return s.scanRune(ast.RBracket)
	case '!':
		return s.scanRune(ast.Exclam)
	case eof:
		return nil, io.EOF
	}
	return nil, errors.New("unrecognized token " + string(ch))
}

//scanComment scans the input for Comments, only single line comments are
//supported.
//
// A comment is all the text that is after a comment identifier, This does not
// enforce the identifier, so it is up to the caller to decide where the comment
// starts, this will read all the text up to the end of the line and return it
// as a single comment token.
//
// TODO(gernest) accept the comment identifier, or check whether the first
// rune is the supported token identifier.
func (s *Scanner) scanComment() (*ast.Token, error) {
	tok := &ast.Token{}
	buf := &bytes.Buffer{}
	isBlock := false
	for _ = range make([]struct{}, 4) {
		ch, _, err := s.r.ReadRune()
		if err != nil {
			if err.Error() == io.EOF.Error() {
				goto final
			}
			return nil, err
		}
		buf.WriteRune(ch)
	}

	if buf.String() == ";-- " {
		isBlock = true
	}
END:
	for {
	begin:
		ch, _, err := s.r.ReadRune()
		if err != nil {
			if err.Error() == io.EOF.Error() {
				fmt.Println("END")
				break END
			}
			return nil, err
		}
		switch ch {
		case '\n', '\r':
			if isBlock {
				buf.WriteRune(ch)
				goto begin
			}
			_ = s.r.UnreadRune()
			break END
		case '-':
			buf.WriteRune(ch)
			if isBlock {
				var str string
				for _ = range make([]struct{}, 2) {
					ch, _, err = s.r.ReadRune()
					if err != nil {
						if err.Error() == io.EOF.Error() {
							goto final
						}
						return nil, err
					}
					str += string(ch)
				}
				buf.WriteString(str)
				if str == "-;" {
					break END
				}
			}
		default:
			_, _ = buf.WriteRune(ch)
		}
	}
final:
	s.column++
	tok.Begin = s.currPos
	s.currPos += buf.Len() // advance the current position
	tok.End = s.currPos
	tok.Column = s.column
	tok.Type = ast.Comment
	tok.Text = buf.String()
	tok.Line = s.line
	return tok, nil
}

//scanWhitespace scans all utf-8 white space characters until it hits a non
//whitespace character.
//
// Tabs ('\t') and space(' ') all represent white space.
func (s *Scanner) scanWhitespace() (*ast.Token, error) {
	tok := &ast.Token{}

	// There can be arbitrary spaces so we need to bugger them up.
	buf := &bytes.Buffer{}
END:
	for {
		ch, _, err := s.r.ReadRune()
		if err != nil {
			if err.Error() == io.EOF.Error() {
				break END
			}
			return nil, err
		}
		switch ch {
		case ' ', '\t':
			_, _ = buf.WriteRune(ch)
		default:
			// Stop after hitting non whitespace character
			// Reseting the buffer is necessary so that the scanned character can be
			// accessed for the next call to Scan method.
			_ = s.r.UnreadRune()
			break END
		}
	}
	tok.Column = s.column
	tok.Begin = s.currPos
	s.currPos += buf.Len()
	tok.End = s.currPos
	tok.Type = ast.WhiteSpace
	tok.Text = buf.String()
	tok.Line = s.line
	return tok, nil
}

//scanNewline returns a token of type NewLine. It is necessary to separate
//newlines from normal spaces because many configuration files formats make use
//of new lines.
//
// A new line can either be a carriage return( '\r') or a new line
// character('\n')
//
// TODO(gernest) accept a new line character as input.
func (s *Scanner) scanNewline() (*ast.Token, error) {
	ch, size, err := s.r.ReadRune()
	if err != nil {
		return nil, err
	}
	tok := &ast.Token{}
	tok.Type = ast.NLine
	tok.Text = string(ch)
	tok.Begin = s.currPos
	s.currPos += size
	tok.End = s.currPos
	s.column = 0
	s.line++
	tok.Column = s.column
	tok.Line = s.line
	return tok, nil
}

//isIdent returns true if ch is a valid identifier
// valid identifiers are
//	underscore _
//	dash -
//	plus +
//	a unicode letter a-zA-Z
//	a unicode digit 0-9
func isIdent(ch rune) bool {
	return ch == '_' || ch == '-' || ch == '+' || unicode.IsLetter(ch) || unicode.IsDigit(ch)
}

//scanIdent returns the current character in the input source as an Ident Token
//
// TODO(gernest) Accept the character as input argument.
func (s *Scanner) scanIdent() (*ast.Token, error) {
	return s.scanRune(ast.Ident)
}

// scanRune scans the current rune and returns a token of type typ, whose Text
// is the scanned character
//
// Use this for single character tokens
func (s *Scanner) scanRune(typ ast.TokenType) (*ast.Token, error) {
	ch, size, err := s.r.ReadRune()
	if err != nil {
		return nil, err
	}
	tok := &ast.Token{}
	tok.Type = typ
	tok.Text = string(ch)
	tok.Begin = s.currPos
	s.currPos += size
	tok.End = s.currPos
	s.column++
	tok.Column = s.column
	tok.Line = s.line
	return tok, nil
}

// peek returns the next rune in the input buffer but does not advance the
// position of the current buffer.
//
// This is a safe way to peek at the next  rune character without actually
// reading it.
func (s *Scanner) peek() rune {
	ch, _, err := s.r.ReadRune()
	if err != nil {
		if err.Error() == io.EOF.Error() {
			return eof
		}
		panic(err)
	}
	_ = s.r.UnreadRune()
	return ch
}
<|endoftext|>


<commit_before>// 30 december 2012
package main

import (
	"os"
	"github.com/hanwen/go-fuse/fuse"
	"path/filepath"
	"errors"
"fmt"
)

var errNoSuchGame = errors.New("no such game")
var errGameNotFound = errors.New("game not found")

// note to self: this needs to be an embed in a struct as DefaultFileSystem will implement the  methods I don't override here and have them return fuse.ENOSYS
type mamefuse struct {
	fuse.DefaultFileSystem
}

func getgame(gamename string) (*Game, error) {
	g, ok := games[gamename]
	if !ok {				// not a valid game
		return nil, errNoSuchGame
	}
//	ret := make(chan string)
//	zipRequests <- zipRequest{
//		Game:	gamename,
//		Return:	ret,
//	}
//	zipname := <-ret
//	close(ret)
//	if zipname == "" {		// none given
	good, err := g.Find()
	if err != nil {
		return nil, err
	}
	if !good {
		return nil, errGameNotFound
	}
	return g, nil
}

func getgame_fuseerr(gamename string) (*Game, fuse.Status) {
	g, err := getgame(gamename)
	if err == errNoSuchGame {
		return nil, fuse.EINVAL
	} else if err == errGameNotFound {
		return nil, fuse.ENOENT
	} else if err != nil {
		// TODO report error
		return nil, fuse.EIO
	}
	return g, fuse.OK
}

func getloopbackfile(filename string) (*fuse.LoopbackFile, error) {
	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	// according to the go-fuse source (fuse/file.go), fuse.LoopbackFile will take ownership of our *os.FIle, calling Close() on it itself
	return &fuse.LoopbackFile{
		File:	f,
	}, nil
}

func getloopbackfile_fuseerr(filename string) (*fuse.LoopbackFile, fuse.Status) {
	loopfile, err := getloopbackfile(filename)
	if err != nil {
		// TODO report error
		return nil, fuse.EIO
	}
	return loopfile, fuse.OK
}

func getattr(filename string) (*fuse.Attr, fuse.Status) {
	stat, err := os.Stat(filename)
	if err != nil {
		// TODO report error
		return nil, fuse.EIO
	}
	return fuse.ToAttr(stat), fuse.OK
}

func getchdparts(name string) (gamename string, chdname string) {
fmt.Print(name, " ")
	gamename, chdname = filepath.Split(name)	// split out CHD filename
	_, gamename = filepath.Split(gamename)		// and game name
	chdname = chdname[:len(chdname) - 4]		// strip extension
	return
}

func (fs *mamefuse) GetAttr(name string, context *fuse.Context) (*fuse.Attr, fuse.Status) {
	basename := filepath.Base(name)
	switch filepath.Ext(basename) {
	case ".zip":				// ROM set
		gamename := basename[:len(basename) - 4]
		g, err := getgame_fuseerr(gamename)
		if err != fuse.OK {
			return nil, err
		}
//		return getattr(g.ROMLoc)
		// TODO merely returning getattr() always results in
		// 2012/12/31 12:13:27 writer: Write/Writev failed, err: 22=invalid argument. opcode: LOOKUP
		// but this works
		a, err := getattr(g.ROMLoc)
		return a, err
	case ".chd":
		gamename, chdname := getchdparts(name)
		if gamename == "" {		// we need a game name to disambiguate
			return nil, fuse.ENOENT
		}
		g, err := getgame_fuseerr(gamename)
		if err != fuse.OK {
			return nil, err
		}
//		return getattr(g.ROMLoc)
		// TODO merely returning getattr() always results in
		// 2012/12/31 12:13:27 writer: Write/Writev failed, err: 22=invalid argument. opcode: LOOKUP
		// but this works
		a, err := getattr(g.CHDLoc[chdname])
		return a, err
	}
	return nil, fuse.ENOENT		// any other file is invalid
}

func (fs *mamefuse) Open(name string, flags uint32, context *fuse.Context) (file fuse.File, code fuse.Status) {
	basename := filepath.Base(name)
	switch filepath.Ext(basename) {
	case ".zip":				// ROM set
		gamename := basename[:len(basename) - 4]
		g, err := getgame_fuseerr(gamename)
		if err != fuse.OK {
			return nil, err
		}
		// TODO worry about closing the file?
		return getloopbackfile_fuseerr(g.ROMLoc)
	case ".chd":				// CHD
		gamename, chdname := getchdparts(name)
		if gamename == "" {		// we need a game name to disambiguate
			return nil, fuse.ENOENT
		}
		g, err := getgame_fuseerr(gamename)
		if err != fuse.OK {
			return nil, err
		}
		// TODO worry about closing the file?
		return getloopbackfile_fuseerr(g.CHDLoc[chdname])
	case "":					// folder
		// ...
	// TODO root directory?
	}
	return nil, fuse.ENOENT		// otherwise 404
}

func (fs *mamefuse) OpenDir(name string, context *fuse.Context) (c []fuse.DirEntry, code fuse.Status) {
	// TODO
	return nil, fuse.ENOENT
}<commit_msg>Fixed MAME not looking for the CHD file in the subdirectory by telling MAME the subdirectory exists, oops =P Still not loading the CHD...<commit_after>// 30 december 2012
package main

import (
	"os"
	"github.com/hanwen/go-fuse/fuse"
	"path/filepath"
	"errors"
"fmt"
)

var errNoSuchGame = errors.New("no such game")
var errGameNotFound = errors.New("game not found")

// note to self: this needs to be an embed in a struct as DefaultFileSystem will implement the  methods I don't override here and have them return fuse.ENOSYS
type mamefuse struct {
	fuse.DefaultFileSystem
}

func getgame(gamename string) (*Game, error) {
	g, ok := games[gamename]
	if !ok {				// not a valid game
		return nil, errNoSuchGame
	}
//	ret := make(chan string)
//	zipRequests <- zipRequest{
//		Game:	gamename,
//		Return:	ret,
//	}
//	zipname := <-ret
//	close(ret)
//	if zipname == "" {		// none given
	good, err := g.Find()
	if err != nil {
		return nil, err
	}
	if !good {
		return nil, errGameNotFound
	}
	return g, nil
}

func getgame_fuseerr(gamename string) (*Game, fuse.Status) {
	g, err := getgame(gamename)
	if err == errNoSuchGame {
		return nil, fuse.EINVAL
	} else if err == errGameNotFound {
		return nil, fuse.ENOENT
	} else if err != nil {
		// TODO report error
		return nil, fuse.EIO
	}
	return g, fuse.OK
}

func getloopbackfile(filename string) (*fuse.LoopbackFile, error) {
	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	// according to the go-fuse source (fuse/file.go), fuse.LoopbackFile will take ownership of our *os.FIle, calling Close() on it itself
	return &fuse.LoopbackFile{
		File:	f,
	}, nil
}

func getloopbackfile_fuseerr(filename string) (*fuse.LoopbackFile, fuse.Status) {
	loopfile, err := getloopbackfile(filename)
	if err != nil {
		// TODO report error
		return nil, fuse.EIO
	}
	return loopfile, fuse.OK
}

func getattr(filename string) (*fuse.Attr, fuse.Status) {
	stat, err := os.Stat(filename)
	if err != nil {
		// TODO report error
		return nil, fuse.EIO
	}
	return fuse.ToAttr(stat), fuse.OK
}

func getchdparts(name string) (gamename string, chdname string) {
fmt.Print(name, " ")
	gamename, chdname = filepath.Split(name)	// split out CHD filename
	_, gamename = filepath.Split(gamename)		// and game name
	chdname = chdname[:len(chdname) - 4]		// strip extension
	return
}

func (fs *mamefuse) GetAttr(name string, context *fuse.Context) (*fuse.Attr, fuse.Status) {
	basename := filepath.Base(name)
	switch filepath.Ext(basename) {
	case ".zip":				// ROM set
		gamename := basename[:len(basename) - 4]
		g, err := getgame_fuseerr(gamename)
		if err != fuse.OK {
			return nil, err
		}
//		return getattr(g.ROMLoc)
		// TODO merely returning getattr() always results in
		// 2012/12/31 12:13:27 writer: Write/Writev failed, err: 22=invalid argument. opcode: LOOKUP
		// but this works
		a, err := getattr(g.ROMLoc)
		return a, err
	case ".chd":
		gamename, chdname := getchdparts(name)
		if gamename == "" {		// we need a game name to disambiguate
			return nil, fuse.ENOENT
		}
		g, err := getgame_fuseerr(gamename)
		if err != fuse.OK {
			return nil, err
		}
//		return getattr(g.ROMLoc)
		// TODO merely returning getattr() always results in
		// 2012/12/31 12:13:27 writer: Write/Writev failed, err: 22=invalid argument. opcode: LOOKUP
		// but this works
		a, err := getattr(g.CHDLoc[chdname])
		return a, err
	default:
		// is it a folder that stores CHDs?
		if _, ok := games[basename]; ok {		// yes
			return &fuse.Attr{
				Mode:	fuse.S_IFDIR | 0755,
			}, fuse.OK
		}
		// no; fall out
	}
	return nil, fuse.ENOENT		// any other file is invalid
}

func (fs *mamefuse) Open(name string, flags uint32, context *fuse.Context) (file fuse.File, code fuse.Status) {
	basename := filepath.Base(name)
	switch filepath.Ext(basename) {
	case ".zip":				// ROM set
		gamename := basename[:len(basename) - 4]
		g, err := getgame_fuseerr(gamename)
		if err != fuse.OK {
			return nil, err
		}
		// TODO worry about closing the file?
		return getloopbackfile_fuseerr(g.ROMLoc)
	case ".chd":				// CHD
		gamename, chdname := getchdparts(name)
		if gamename == "" {		// we need a game name to disambiguate
			return nil, fuse.ENOENT
		}
		g, err := getgame_fuseerr(gamename)
		if err != fuse.OK {
			return nil, err
		}
		// TODO worry about closing the file?
		return getloopbackfile_fuseerr(g.CHDLoc[chdname])
	case "":					// folder
		// ...
	// TODO root directory?
	}
	return nil, fuse.ENOENT		// otherwise 404
}

func (fs *mamefuse) OpenDir(name string, context *fuse.Context) (c []fuse.DirEntry, code fuse.Status) {
	// TODO
	return nil, fuse.ENOENT
}<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"sync"
	"time"
)

var (
	debug                        bool
	verbose                      bool
	info                         bool
	quiet                        bool
	force                        bool
	usemove                      bool
	usecacheFallback             bool
	retryGitCommands             bool
	pfMode                       bool
	pfLocation                   string
	dryRun                       bool
	check4update                 bool
	checkSum                     bool
	gitObjectSyntaxNotSupported  bool
	moduleDirParam               string
	cacheDirParam                string
	branchParam                  string
	tags                         bool
	outputNameParam              string
	moduleParam                  string
	configFile                   string
	config                       ConfigSettings
	mutex                        sync.Mutex
	empty                        struct{}
	syncGitCount                 int
	syncForgeCount               int
	needSyncGitCount             int
	needSyncForgeCount           int
	needSyncDirs                 []string
	needSyncEnvs                 map[string]struct{}
	syncGitTime                  float64
	syncForgeTime                float64
	ioGitTime                    float64
	ioForgeTime                  float64
	forgeJSONParseTime           float64
	metadataJSONParseTime        float64
	gmetadataJSONParseTime       float64
	buildtime                    string
	uniqueForgeModules           map[string]ForgeModule
	latestForgeModules           LatestForgeModules
	maxworker                    int
	maxExtractworker             int
	forgeModuleDeprecationNotice string
)

// LatestForgeModules contains a map of unique Forge modules
// that should be the latest versions of them
type LatestForgeModules struct {
	sync.RWMutex
	m map[string]string
}

// ConfigSettings contains the key value pairs from the g10k config file
type ConfigSettings struct {
	CacheDir                    string `yaml:"cachedir"`
	ForgeCacheDir               string
	ModulesCacheDir             string
	EnvCacheDir                 string
	Git                         Git
	Forge                       Forge
	Sources                     map[string]Source
	Timeout                     int      `yaml:"timeout"`
	IgnoreUnreachableModules    bool     `yaml:"ignore_unreachable_modules"`
	Maxworker                   int      `yaml:"maxworker"`
	MaxExtractworker            int      `yaml:"maxextractworker"`
	UseCacheFallback            bool     `yaml:"use_cache_fallback"`
	RetryGitCommands            bool     `yaml:"retry_git_commands"`
	GitObjectSyntaxNotSupported bool     `yaml:"git_object_syntax_not_supported"`
	PostRunCommand              []string `yaml:"postrun"`
}

// Forge is a simple struct that contains the base URL of
// the Forge that g10k should use. Defaults to: https://forgeapi.puppetlabs.com
type Forge struct {
	Baseurl string `yaml:"baseurl"`
}

// Git is a simple struct that contains the optional SSH private key to
// use for authentication
type Git struct {
	privateKey string `yaml:"private_key"`
}

// Source contains basic information about a Puppet environment repository
type Source struct {
	Remote                      string
	Basedir                     string
	Prefix                      string
	PrivateKey                  string `yaml:"private_key"`
	ForceForgeVersions          bool   `yaml:"force_forge_versions"`
	WarnMissingBranch           bool   `yaml:"warn_if_branch_is_missing"`
	ExitIfUnreachable           bool   `yaml:"exit_if_unreachable"`
	AutoCorrectEnvironmentNames string `yaml:"invalid_branches"`
}

// Puppetfile contains the key value pairs from the Puppetfile
type Puppetfile struct {
	forgeBaseURL      string
	forgeCacheTTL     time.Duration
	forgeModules      map[string]ForgeModule
	gitModules        map[string]GitModule
	privateKey        string
	source            string
	workDir           string
	moduleDirs        []string
	controlRepoBranch string
}

// ForgeModule contains information (Version, Name, Author, md5 checksum, file size of the tar.gz archive, Forge BaseURL if custom) about a Puppetlabs Forge module
type ForgeModule struct {
	version   string
	name      string
	author    string
	md5sum    string
	fileSize  int64
	baseURL   string
	cacheTTL  time.Duration
	sha256sum string
	moduleDir string
}

// GitModule contains information about a Git Puppet module
type GitModule struct {
	privateKey        string
	git               string
	branch            string
	tag               string
	commit            string
	ref               string
	link              bool
	ignoreUnreachable bool
	fallback          []string
	installPath       string
	local             bool
	moduleDir         string
}

// ForgeResult is returned by queryForgeAPI and contains if and which version of the Puppetlabs Forge module needs to be downloaded
type ForgeResult struct {
	needToGet     bool
	versionNumber string
	md5sum        string
	fileSize      int64
}

// ExecResult contains the exit code and output of an external command (e.g. git)
type ExecResult struct {
	returnCode int
	output     string
}

func init() {
	// initialize global maps
	needSyncEnvs = make(map[string]struct{})
	uniqueForgeModules = make(map[string]ForgeModule)
}

func main() {

	var (
		configFileFlag = flag.String("config", "", "which config file to use")
		versionFlag    = flag.Bool("version", false, "show build time and version number")
	)
	flag.StringVar(&branchParam, "branch", "", "which git branch of the Puppet environment to update, e.g. core_foobar")
	flag.BoolVar(&tags, "tags", false, "to pull tags as well as branches")
	flag.StringVar(&outputNameParam, "outputname", "", "overwrite the environment name if -branch is specified")
	flag.StringVar(&moduleParam, "module", "", "which module of the Puppet environment to update, e.g. stdlib")
	flag.StringVar(&moduleDirParam, "moduledir", "", "allows overriding of Puppetfile specific moduledir setting, the folder in which Puppet modules will be extracted")
	flag.StringVar(&cacheDirParam, "cachedir", "", "allows overriding of the g10k config file cachedir setting, the folder in which g10k will download git repositories and Forge modules")
	flag.IntVar(&maxworker, "maxworker", 50, "how many Goroutines are allowed to run in parallel for Git and Forge module resolving")
	flag.IntVar(&maxExtractworker, "maxextractworker", 20, "how many Goroutines are allowed to run in parallel for local Git and Forge module extracting processes (git clone, untar and gunzip)")
	flag.BoolVar(&pfMode, "puppetfile", false, "install all modules from Puppetfile in cwd")
	flag.StringVar(&pfLocation, "puppetfilelocation", "./Puppetfile", "which Puppetfile to use in -puppetfile mode")
	flag.BoolVar(&force, "force", false, "purge the Puppet environment directory and do a full sync")
	flag.BoolVar(&dryRun, "dryrun", false, "do not modify anything, just print what would be changed")
	flag.BoolVar(&usemove, "usemove", false, "do not use hardlinks to populate your Puppet environments with Puppetlabs Forge modules. Instead uses simple move commands and purges the Forge cache directory after each run! (Useful for g10k runs inside a Docker container)")
	flag.BoolVar(&check4update, "check4update", false, "only check if the is newer version of the Puppet module avaialable. Does implicitly set dryrun to true")
	flag.BoolVar(&checkSum, "checksum", false, "get the md5 check sum for each Puppetlabs Forge module and verify the integrity of the downloaded archive. Increases g10k run time!")
	flag.BoolVar(&debug, "debug", false, "log debug output, defaults to false")
	flag.BoolVar(&verbose, "verbose", false, "log verbose output, defaults to false")
	flag.BoolVar(&info, "info", false, "log info output, defaults to false")
	flag.BoolVar(&quiet, "quiet", false, "no output, defaults to false")
	flag.BoolVar(&usecacheFallback, "usecachefallback", false, "if g10k should try to use its cache for sources and modules instead of failing")
	flag.BoolVar(&retryGitCommands, "retrygitcommands", false, "if g10k should purge the local repository and retry a failed git command (clone or remote update) instead of failing")
	flag.BoolVar(&gitObjectSyntaxNotSupported, "gitobjectsyntaxnotsupported", false, "if your git version is too old to support reference syntax like master^{object} use this setting to revert to the older syntax")
	flag.Parse()

	configFile = *configFileFlag
	version := *versionFlag

	if version {
		fmt.Println("g10k version 0.5.4 Build time:", buildtime, "UTC")
		os.Exit(0)
	}

	if check4update {
		dryRun = true
	}

	// check for git executable dependency
	if _, err := exec.LookPath("git"); err != nil {
		Fatalf("Error: could not find 'git' executable in PATH")
	}

	target := ""
	before := time.Now()
	if len(configFile) > 0 {
		if usemove {
			Fatalf("Error: -usemove parameter is only allowed in -puppetfile mode!")
		}
		if pfMode {
			Fatalf("Error: -puppetfile parameter is not allowed with -config parameter!")
		}
		if (len(outputNameParam) > 0) && (len(branchParam) == 0) {
			Fatalf("Error: -outputname specified without -branch!")
		}
		if usecacheFallback {
			config.UseCacheFallback = true
		}
		Debugf("Using as config file: " + configFile)
		config = readConfigfile(configFile)
		target = configFile
		if len(branchParam) > 0 {
			resolvePuppetEnvironment(branchParam, tags, outputNameParam)
			target += " with branch " + branchParam
		} else {
			resolvePuppetEnvironment("", tags, "")
		}
	} else {
		if pfMode {
			Debugf("Trying to use as Puppetfile: " + pfLocation)
			sm := make(map[string]Source)
			sm["cmdlineparam"] = Source{Basedir: "./"}
			cachedir := "/tmp/g10k"
			if len(os.Getenv("g10k_cachedir")) > 0 {
				cachedir = os.Getenv("g10k_cachedir")
				cachedir = checkDirAndCreate(cachedir, "cachedir environment variable g10k_cachedir")
				Debugf("Found environment variable g10k_cachedir set to: " + cachedir)
			} else if len(cacheDirParam) > 0 {
				Debugf("Using -cachedir parameter set to : " + cacheDirParam)
				cachedir = checkDirAndCreate(cacheDirParam, "cachedir CLI param")
			} else {
				cachedir = checkDirAndCreate(cachedir, "cachedir default value")
			}
			forgeDefaultSettings := Forge{Baseurl: "https://forgeapi.puppetlabs.com"}
			config = ConfigSettings{CacheDir: cachedir, ForgeCacheDir: cachedir, ModulesCacheDir: cachedir, EnvCacheDir: cachedir, Sources: sm, Forge: forgeDefaultSettings, Maxworker: maxworker, UseCacheFallback: usecacheFallback, MaxExtractworker: maxExtractworker, RetryGitCommands: retryGitCommands, GitObjectSyntaxNotSupported: gitObjectSyntaxNotSupported}
			target = pfLocation
			puppetfile := readPuppetfile(target, "", "cmdlineparam", false, false)
			puppetfile.workDir = "./"
			pfm := make(map[string]Puppetfile)
			pfm["cmdlineparam"] = puppetfile
			resolvePuppetfile(pfm)
		} else {
			Fatalf("Error: you need to specify at least a config file or use the Puppetfile mode\nExample call: " + os.Args[0] + " -config test.yaml or " + os.Args[0] + " -puppetfile\n")
		}
	}

	if usemove {
		// we can not reuse the Forge cache at all when -usemove gets used, because we can not delete the -latest link for some reason
		defer purgeDir(config.ForgeCacheDir, "main() -puppetfile mode with -usemove parameter")
	}

	Debugf("Forge response JSON parsing took " + strconv.FormatFloat(forgeJSONParseTime, 'f', 4, 64) + " seconds")
	Debugf("Forge modules metadata.json parsing took " + strconv.FormatFloat(metadataJSONParseTime, 'f', 4, 64) + " seconds")

	if !check4update && !quiet {
		if len(forgeModuleDeprecationNotice) > 0 {
			Warnf(strings.TrimSuffix(forgeModuleDeprecationNotice, "\n"))
		}
		fmt.Println("Synced", target, "with", syncGitCount, "git repositories and", syncForgeCount, "Forge modules in "+strconv.FormatFloat(time.Since(before).Seconds(), 'f', 1, 64)+"s with git ("+strconv.FormatFloat(syncGitTime, 'f', 1, 64)+"s sync, I/O", strconv.FormatFloat(ioGitTime, 'f', 1, 64)+"s) and Forge ("+strconv.FormatFloat(syncForgeTime, 'f', 1, 64)+"s query+download, I/O", strconv.FormatFloat(ioForgeTime, 'f', 1, 64)+"s) using", strconv.Itoa(config.Maxworker), "resolv and", strconv.Itoa(config.MaxExtractworker), "extract workers")
	}
	if dryRun && (needSyncForgeCount > 0 || needSyncGitCount > 0) {
		os.Exit(1)
	}

	checkForAndExecutePostrunCommand()
}
<commit_msg>bump version to v0.5.5<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"sync"
	"time"
)

var (
	debug                        bool
	verbose                      bool
	info                         bool
	quiet                        bool
	force                        bool
	usemove                      bool
	usecacheFallback             bool
	retryGitCommands             bool
	pfMode                       bool
	pfLocation                   string
	dryRun                       bool
	check4update                 bool
	checkSum                     bool
	gitObjectSyntaxNotSupported  bool
	moduleDirParam               string
	cacheDirParam                string
	branchParam                  string
	tags                         bool
	outputNameParam              string
	moduleParam                  string
	configFile                   string
	config                       ConfigSettings
	mutex                        sync.Mutex
	empty                        struct{}
	syncGitCount                 int
	syncForgeCount               int
	needSyncGitCount             int
	needSyncForgeCount           int
	needSyncDirs                 []string
	needSyncEnvs                 map[string]struct{}
	syncGitTime                  float64
	syncForgeTime                float64
	ioGitTime                    float64
	ioForgeTime                  float64
	forgeJSONParseTime           float64
	metadataJSONParseTime        float64
	gmetadataJSONParseTime       float64
	buildtime                    string
	uniqueForgeModules           map[string]ForgeModule
	latestForgeModules           LatestForgeModules
	maxworker                    int
	maxExtractworker             int
	forgeModuleDeprecationNotice string
)

// LatestForgeModules contains a map of unique Forge modules
// that should be the latest versions of them
type LatestForgeModules struct {
	sync.RWMutex
	m map[string]string
}

// ConfigSettings contains the key value pairs from the g10k config file
type ConfigSettings struct {
	CacheDir                    string `yaml:"cachedir"`
	ForgeCacheDir               string
	ModulesCacheDir             string
	EnvCacheDir                 string
	Git                         Git
	Forge                       Forge
	Sources                     map[string]Source
	Timeout                     int      `yaml:"timeout"`
	IgnoreUnreachableModules    bool     `yaml:"ignore_unreachable_modules"`
	Maxworker                   int      `yaml:"maxworker"`
	MaxExtractworker            int      `yaml:"maxextractworker"`
	UseCacheFallback            bool     `yaml:"use_cache_fallback"`
	RetryGitCommands            bool     `yaml:"retry_git_commands"`
	GitObjectSyntaxNotSupported bool     `yaml:"git_object_syntax_not_supported"`
	PostRunCommand              []string `yaml:"postrun"`
}

// Forge is a simple struct that contains the base URL of
// the Forge that g10k should use. Defaults to: https://forgeapi.puppetlabs.com
type Forge struct {
	Baseurl string `yaml:"baseurl"`
}

// Git is a simple struct that contains the optional SSH private key to
// use for authentication
type Git struct {
	privateKey string `yaml:"private_key"`
}

// Source contains basic information about a Puppet environment repository
type Source struct {
	Remote                      string
	Basedir                     string
	Prefix                      string
	PrivateKey                  string `yaml:"private_key"`
	ForceForgeVersions          bool   `yaml:"force_forge_versions"`
	WarnMissingBranch           bool   `yaml:"warn_if_branch_is_missing"`
	ExitIfUnreachable           bool   `yaml:"exit_if_unreachable"`
	AutoCorrectEnvironmentNames string `yaml:"invalid_branches"`
}

// Puppetfile contains the key value pairs from the Puppetfile
type Puppetfile struct {
	forgeBaseURL      string
	forgeCacheTTL     time.Duration
	forgeModules      map[string]ForgeModule
	gitModules        map[string]GitModule
	privateKey        string
	source            string
	workDir           string
	moduleDirs        []string
	controlRepoBranch string
}

// ForgeModule contains information (Version, Name, Author, md5 checksum, file size of the tar.gz archive, Forge BaseURL if custom) about a Puppetlabs Forge module
type ForgeModule struct {
	version   string
	name      string
	author    string
	md5sum    string
	fileSize  int64
	baseURL   string
	cacheTTL  time.Duration
	sha256sum string
	moduleDir string
}

// GitModule contains information about a Git Puppet module
type GitModule struct {
	privateKey        string
	git               string
	branch            string
	tag               string
	commit            string
	ref               string
	link              bool
	ignoreUnreachable bool
	fallback          []string
	installPath       string
	local             bool
	moduleDir         string
}

// ForgeResult is returned by queryForgeAPI and contains if and which version of the Puppetlabs Forge module needs to be downloaded
type ForgeResult struct {
	needToGet     bool
	versionNumber string
	md5sum        string
	fileSize      int64
}

// ExecResult contains the exit code and output of an external command (e.g. git)
type ExecResult struct {
	returnCode int
	output     string
}

func init() {
	// initialize global maps
	needSyncEnvs = make(map[string]struct{})
	uniqueForgeModules = make(map[string]ForgeModule)
}

func main() {

	var (
		configFileFlag = flag.String("config", "", "which config file to use")
		versionFlag    = flag.Bool("version", false, "show build time and version number")
	)
	flag.StringVar(&branchParam, "branch", "", "which git branch of the Puppet environment to update, e.g. core_foobar")
	flag.BoolVar(&tags, "tags", false, "to pull tags as well as branches")
	flag.StringVar(&outputNameParam, "outputname", "", "overwrite the environment name if -branch is specified")
	flag.StringVar(&moduleParam, "module", "", "which module of the Puppet environment to update, e.g. stdlib")
	flag.StringVar(&moduleDirParam, "moduledir", "", "allows overriding of Puppetfile specific moduledir setting, the folder in which Puppet modules will be extracted")
	flag.StringVar(&cacheDirParam, "cachedir", "", "allows overriding of the g10k config file cachedir setting, the folder in which g10k will download git repositories and Forge modules")
	flag.IntVar(&maxworker, "maxworker", 50, "how many Goroutines are allowed to run in parallel for Git and Forge module resolving")
	flag.IntVar(&maxExtractworker, "maxextractworker", 20, "how many Goroutines are allowed to run in parallel for local Git and Forge module extracting processes (git clone, untar and gunzip)")
	flag.BoolVar(&pfMode, "puppetfile", false, "install all modules from Puppetfile in cwd")
	flag.StringVar(&pfLocation, "puppetfilelocation", "./Puppetfile", "which Puppetfile to use in -puppetfile mode")
	flag.BoolVar(&force, "force", false, "purge the Puppet environment directory and do a full sync")
	flag.BoolVar(&dryRun, "dryrun", false, "do not modify anything, just print what would be changed")
	flag.BoolVar(&usemove, "usemove", false, "do not use hardlinks to populate your Puppet environments with Puppetlabs Forge modules. Instead uses simple move commands and purges the Forge cache directory after each run! (Useful for g10k runs inside a Docker container)")
	flag.BoolVar(&check4update, "check4update", false, "only check if the is newer version of the Puppet module avaialable. Does implicitly set dryrun to true")
	flag.BoolVar(&checkSum, "checksum", false, "get the md5 check sum for each Puppetlabs Forge module and verify the integrity of the downloaded archive. Increases g10k run time!")
	flag.BoolVar(&debug, "debug", false, "log debug output, defaults to false")
	flag.BoolVar(&verbose, "verbose", false, "log verbose output, defaults to false")
	flag.BoolVar(&info, "info", false, "log info output, defaults to false")
	flag.BoolVar(&quiet, "quiet", false, "no output, defaults to false")
	flag.BoolVar(&usecacheFallback, "usecachefallback", false, "if g10k should try to use its cache for sources and modules instead of failing")
	flag.BoolVar(&retryGitCommands, "retrygitcommands", false, "if g10k should purge the local repository and retry a failed git command (clone or remote update) instead of failing")
	flag.BoolVar(&gitObjectSyntaxNotSupported, "gitobjectsyntaxnotsupported", false, "if your git version is too old to support reference syntax like master^{object} use this setting to revert to the older syntax")
	flag.Parse()

	configFile = *configFileFlag
	version := *versionFlag

	if version {
		fmt.Println("g10k version 0.5.5 Build time:", buildtime, "UTC")
		os.Exit(0)
	}

	if check4update {
		dryRun = true
	}

	// check for git executable dependency
	if _, err := exec.LookPath("git"); err != nil {
		Fatalf("Error: could not find 'git' executable in PATH")
	}

	target := ""
	before := time.Now()
	if len(configFile) > 0 {
		if usemove {
			Fatalf("Error: -usemove parameter is only allowed in -puppetfile mode!")
		}
		if pfMode {
			Fatalf("Error: -puppetfile parameter is not allowed with -config parameter!")
		}
		if (len(outputNameParam) > 0) && (len(branchParam) == 0) {
			Fatalf("Error: -outputname specified without -branch!")
		}
		if usecacheFallback {
			config.UseCacheFallback = true
		}
		Debugf("Using as config file: " + configFile)
		config = readConfigfile(configFile)
		target = configFile
		if len(branchParam) > 0 {
			resolvePuppetEnvironment(branchParam, tags, outputNameParam)
			target += " with branch " + branchParam
		} else {
			resolvePuppetEnvironment("", tags, "")
		}
	} else {
		if pfMode {
			Debugf("Trying to use as Puppetfile: " + pfLocation)
			sm := make(map[string]Source)
			sm["cmdlineparam"] = Source{Basedir: "./"}
			cachedir := "/tmp/g10k"
			if len(os.Getenv("g10k_cachedir")) > 0 {
				cachedir = os.Getenv("g10k_cachedir")
				cachedir = checkDirAndCreate(cachedir, "cachedir environment variable g10k_cachedir")
				Debugf("Found environment variable g10k_cachedir set to: " + cachedir)
			} else if len(cacheDirParam) > 0 {
				Debugf("Using -cachedir parameter set to : " + cacheDirParam)
				cachedir = checkDirAndCreate(cacheDirParam, "cachedir CLI param")
			} else {
				cachedir = checkDirAndCreate(cachedir, "cachedir default value")
			}
			forgeDefaultSettings := Forge{Baseurl: "https://forgeapi.puppetlabs.com"}
			config = ConfigSettings{CacheDir: cachedir, ForgeCacheDir: cachedir, ModulesCacheDir: cachedir, EnvCacheDir: cachedir, Sources: sm, Forge: forgeDefaultSettings, Maxworker: maxworker, UseCacheFallback: usecacheFallback, MaxExtractworker: maxExtractworker, RetryGitCommands: retryGitCommands, GitObjectSyntaxNotSupported: gitObjectSyntaxNotSupported}
			target = pfLocation
			puppetfile := readPuppetfile(target, "", "cmdlineparam", false, false)
			puppetfile.workDir = "./"
			pfm := make(map[string]Puppetfile)
			pfm["cmdlineparam"] = puppetfile
			resolvePuppetfile(pfm)
		} else {
			Fatalf("Error: you need to specify at least a config file or use the Puppetfile mode\nExample call: " + os.Args[0] + " -config test.yaml or " + os.Args[0] + " -puppetfile\n")
		}
	}

	if usemove {
		// we can not reuse the Forge cache at all when -usemove gets used, because we can not delete the -latest link for some reason
		defer purgeDir(config.ForgeCacheDir, "main() -puppetfile mode with -usemove parameter")
	}

	Debugf("Forge response JSON parsing took " + strconv.FormatFloat(forgeJSONParseTime, 'f', 4, 64) + " seconds")
	Debugf("Forge modules metadata.json parsing took " + strconv.FormatFloat(metadataJSONParseTime, 'f', 4, 64) + " seconds")

	if !check4update && !quiet {
		if len(forgeModuleDeprecationNotice) > 0 {
			Warnf(strings.TrimSuffix(forgeModuleDeprecationNotice, "\n"))
		}
		fmt.Println("Synced", target, "with", syncGitCount, "git repositories and", syncForgeCount, "Forge modules in "+strconv.FormatFloat(time.Since(before).Seconds(), 'f', 1, 64)+"s with git ("+strconv.FormatFloat(syncGitTime, 'f', 1, 64)+"s sync, I/O", strconv.FormatFloat(ioGitTime, 'f', 1, 64)+"s) and Forge ("+strconv.FormatFloat(syncForgeTime, 'f', 1, 64)+"s query+download, I/O", strconv.FormatFloat(ioForgeTime, 'f', 1, 64)+"s) using", strconv.Itoa(config.Maxworker), "resolv and", strconv.Itoa(config.MaxExtractworker), "extract workers")
	}
	if dryRun && (needSyncForgeCount > 0 || needSyncGitCount > 0) {
		os.Exit(1)
	}

	checkForAndExecutePostrunCommand()
}
<|endoftext|>