<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"log"
	"net"
	"net/url"
	"os"
	"strings"
	"sync"
	"time"

	"koding/kite-handler/command"
	"koding/kite-handler/fs"
	"koding/kite-handler/terminal"
	"koding/kites/klient/collaboration"
	"koding/kites/klient/protocol"
	"koding/kites/klient/usage"

	"github.com/koding/kite"
	"github.com/koding/kite/config"
)

var (
	flagIP          = flag.String("ip", "", "Change public ip")
	flagPort        = flag.Int("port", 56789, "Change running port")
	flagVersion     = flag.Bool("version", false, "Show version and exit")
	flagProxy       = flag.Bool("proxy", false, "Start klient behind a proxy")
	flagEnvironment = flag.String("env", protocol.Environment, "Change environment")
	flagRegion      = flag.String("region", protocol.Region, "Change region")
	flagRegisterURL = flag.String("register-url", "", "Change register URL to kontrol")

	// update paramters
	flagUpdateInterval = flag.Duration("update-interval", time.Minute*5,
		"Change interval for checking for new updates")
	flagUpdateURL = flag.String("update-url",
		"https://s3.amazonaws.com/koding-klient/"+protocol.Environment+"/latest-version.txt",
		"Change update endpoint for latest version")

	VERSION = protocol.Version
	NAME    = protocol.Name

	// this is our main reference to count and measure metrics for the klient
	usg  = usage.NewUsage()
	klog kite.Logger

	// this is used to allow other users to call any klient method.
	collab = collaboration.New()

	// we also could use an atomic boolean this is simple for now.
	updating   = false
	updatingMu sync.Mutex // protects updating
)

func main() {
	flag.Parse()
	if *flagVersion {
		fmt.Println(VERSION)
		os.Exit(0)
	}

	// Close the klient.db in any case. Corrupt db would be catastrophic
	defer collab.Close()

	k := kite.New(NAME, VERSION)
	conf := config.MustGet()
	k.Config = conf
	k.Config.Port = *flagPort
	k.Config.Environment = *flagEnvironment
	k.Config.Region = *flagRegion

	// FIXME: It's ugly I know. It's a fix for Koding local development and is
	// needed
	if !strings.Contains(k.Config.KontrolURL, "ngrok") {
		// override current kontrolURL so it talks to port 3000, this is needed
		// because ELB can forward requests based on ports. The port 80 and 443 are
		// HTTP/HTTPS only so our kite can't connect it (we use websocket). However
		// We have a TCP proxy at 3000 which allows us to connect via WebSocket.
		u, _ := url.Parse(k.Config.KontrolURL)

		host := u.Host
		if HasPort(u.Host) {
			host, _, _ = net.SplitHostPort(u.Host)
		}

		u.Host = AddPort(host, "3000")
		u.Scheme = "http"
		k.Config.KontrolURL = u.String()
	}

	klog = k.Log

	if *flagUpdateInterval < time.Minute {
		klog.Warning("Update interval can't be less than one minute. Setting to one minute.")
		*flagUpdateInterval = time.Minute
	}

	updater := &Updater{
		Endpoint: *flagUpdateURL,
		Interval: *flagUpdateInterval,
	}

	// before we register check for latest update and re-update itself before
	// we continue
	k.Log.Info("Checking for new updates")
	if err := updater.checkAndUpdate(); err != nil {
		klog.Warning("Self-update: %s", err)
	}

	go updater.Run()

	// always boot up with the same id in the kite.key
	k.Id = conf.Id

	userIn := func(user string, users ...string) bool {
		for _, u := range users {
			if u == user {
				return true
			}
		}
		return false
	}

	// don't pass any request if the caller is outside of our scope.
	// don't allow anyone to call a method if we are during an update.
	k.PreHandleFunc(func(r *kite.Request) (interface{}, error) {
		// only authenticated methods have correct username. For example
		// kite.ping has authentication disabled so username can be empty.
		if r.Auth != nil {
			k.Log.Info("Kite '%s/%s/%s' called method: '%s'",
				r.Username, r.Client.Environment, r.Client.Name, r.Method)

			// Allow these users by default
			allowedUsers := []string{k.Config.Username, "koding", "unknown"}

			// Allow collaboration users as well
			sharedUsers, err := collab.GetAll()
			if err != nil {
				return nil, fmt.Errorf("Can't read shared users from the storage. Err: %v", err)
			}
			allowedUsers = append(allowedUsers, sharedUsers...)

			if !userIn(r.Username, allowedUsers...) {
				return nil, fmt.Errorf("User '%s' is not allowed to make a call to us.", r.Username)
			}
		}

		updatingMu.Lock()
		defer updatingMu.Unlock()

		if updating {
			return nil, errors.New("Updating klient. Can't accept any method.")
		}

		return true, nil
	})

	// count only those methods, please add/remove methods here that will reset
	// the timer of a klient.
	usg.CountedMethods = map[string]bool{
		"fs.readDirectory":    true,
		"fs.glob":             true,
		"fs.readFile":         true,
		"fs.writeFile":        true,
		"fs.uniquePath":       true,
		"fs.getInfo":          true,
		"fs.setPermissions":   true,
		"fs.remove":           true,
		"fs.rename":           true,
		"fs.createDirectory":  true,
		"fs.move":             true,
		"fs.copy":             true,
		"webterm.getSessions": true,
		"webterm.connect":     true,
		"webterm.killSession": true,
		"exec":                true,
	}

	// we measure every incoming request
	k.PreHandleFunc(usg.Counter)

	// this provides us to get the current usage whenever we want
	k.HandleFunc("klient.usage", usg.Current)

	// also invoke updating
	k.Handle("klient.update", updater)

	// Collaboration
	k.HandleFunc("klient.share", collab.Share)
	k.HandleFunc("klient.unshare", collab.Unshare)
	k.HandleFunc("klient.shared", collab.Shared)

	k.HandleFunc("fs.readDirectory", fs.ReadDirectory)
	k.HandleFunc("fs.glob", fs.Glob)
	k.HandleFunc("fs.readFile", fs.ReadFile)
	k.HandleFunc("fs.writeFile", fs.WriteFile)
	k.HandleFunc("fs.uniquePath", fs.UniquePath)
	k.HandleFunc("fs.getInfo", fs.GetInfo)
	k.HandleFunc("fs.setPermissions", fs.SetPermissions)
	k.HandleFunc("fs.remove", fs.Remove)
	k.HandleFunc("fs.rename", fs.Rename)
	k.HandleFunc("fs.createDirectory", fs.CreateDirectory)
	k.HandleFunc("fs.move", fs.Move)
	k.HandleFunc("fs.copy", fs.Copy)

	terminal.ResetFunc = usg.Reset

	k.HandleFunc("webterm.getSessions", terminal.GetSessions)
	k.HandleFunc("webterm.connect", terminal.Connect)

	k.HandleFunc("webterm.killSession", terminal.KillSession)
	k.HandleFunc("exec", command.Exec)

	registerURL, err := getRegisterURL(k)
	if err != nil {
		log.Panic("could not get public ip" + err.Error())
	}

	if *flagRegisterURL != "" {
		u, err := url.Parse(*flagRegisterURL)
		if err != nil {
			k.Log.Fatal("Couldn't parse register url: %s", err)
		}

		registerURL = u
	}

	if registerURL == nil {
		log.Panic("register url is nil")
	}

	k.Log.Info("Going to register to kontrol with URL: %s", registerURL)
	if err := k.RegisterForever(registerURL); err != nil {
		log.Panic(err)
	}

	k.Log.Info("Running as version %s", VERSION)

	k.Run()
}

// Given a string of the form "host", "host:port", or "[ipv6::address]:port",
// return true if the string includes a port.
func HasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") }

// Given a string of the form "host", "port", returns "host:port"
func AddPort(host, port string) string {
	if ok := HasPort(host); ok {
		return host
	}

	return host + ":" + port
}
<commit_msg>klient: Remove the debugging artifact.<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"log"
	"net"
	"net/url"
	"os"
	"strings"
	"sync"
	"time"

	"koding/kite-handler/command"
	"koding/kite-handler/fs"
	"koding/kite-handler/terminal"
	"koding/kites/klient/collaboration"
	"koding/kites/klient/protocol"
	"koding/kites/klient/usage"

	"github.com/koding/kite"
	"github.com/koding/kite/config"
)

var (
	flagIP          = flag.String("ip", "", "Change public ip")
	flagPort        = flag.Int("port", 56789, "Change running port")
	flagVersion     = flag.Bool("version", false, "Show version and exit")
	flagProxy       = flag.Bool("proxy", false, "Start klient behind a proxy")
	flagEnvironment = flag.String("env", protocol.Environment, "Change environment")
	flagRegion      = flag.String("region", protocol.Region, "Change region")
	flagRegisterURL = flag.String("register-url", "", "Change register URL to kontrol")

	// update paramters
	flagUpdateInterval = flag.Duration("update-interval", time.Minute*5,
		"Change interval for checking for new updates")
	flagUpdateURL = flag.String("update-url",
		"https://s3.amazonaws.com/koding-klient/"+protocol.Environment+"/latest-version.txt",
		"Change update endpoint for latest version")

	VERSION = protocol.Version
	NAME    = protocol.Name

	// this is our main reference to count and measure metrics for the klient
	usg  = usage.NewUsage()
	klog kite.Logger

	// this is used to allow other users to call any klient method.
	collab = collaboration.New()

	// we also could use an atomic boolean this is simple for now.
	updating   = false
	updatingMu sync.Mutex // protects updating
)

func main() {
	flag.Parse()
	if *flagVersion {
		fmt.Println(VERSION)
		os.Exit(0)
	}

	// Close the klient.db in any case. Corrupt db would be catastrophic
	defer collab.Close()

	k := kite.New(NAME, VERSION)
	conf := config.MustGet()
	k.Config = conf
	k.Config.Port = *flagPort
	k.Config.Environment = *flagEnvironment
	k.Config.Region = *flagRegion

	// FIXME: It's ugly I know. It's a fix for Koding local development and is
	// needed
	if !strings.Contains(k.Config.KontrolURL, "ngrok") {
		// override current kontrolURL so it talks to port 3000, this is needed
		// because ELB can forward requests based on ports. The port 80 and 443 are
		// HTTP/HTTPS only so our kite can't connect it (we use websocket). However
		// We have a TCP proxy at 3000 which allows us to connect via WebSocket.
		u, _ := url.Parse(k.Config.KontrolURL)

		host := u.Host
		if HasPort(u.Host) {
			host, _, _ = net.SplitHostPort(u.Host)
		}

		u.Host = AddPort(host, "3000")
		u.Scheme = "http"
		k.Config.KontrolURL = u.String()
	}

	klog = k.Log

	if *flagUpdateInterval < time.Minute {
		klog.Warning("Update interval can't be less than one minute. Setting to one minute.")
		*flagUpdateInterval = time.Minute
	}

	updater := &Updater{
		Endpoint: *flagUpdateURL,
		Interval: *flagUpdateInterval,
	}

	// before we register check for latest update and re-update itself before
	// we continue
	k.Log.Info("Checking for new updates")
	if err := updater.checkAndUpdate(); err != nil {
		klog.Warning("Self-update: %s", err)
	}

	go updater.Run()

	// always boot up with the same id in the kite.key
	k.Id = conf.Id

	userIn := func(user string, users ...string) bool {
		for _, u := range users {
			if u == user {
				return true
			}
		}
		return false
	}

	// don't pass any request if the caller is outside of our scope.
	// don't allow anyone to call a method if we are during an update.
	k.PreHandleFunc(func(r *kite.Request) (interface{}, error) {
		// only authenticated methods have correct username. For example
		// kite.ping has authentication disabled so username can be empty.
		if r.Auth != nil {
			k.Log.Info("Kite '%s/%s/%s' called method: '%s'",
				r.Username, r.Client.Environment, r.Client.Name, r.Method)

			// Allow these users by default
			allowedUsers := []string{k.Config.Username, "koding"}

			// Allow collaboration users as well
			sharedUsers, err := collab.GetAll()
			if err != nil {
				return nil, fmt.Errorf("Can't read shared users from the storage. Err: %v", err)
			}
			allowedUsers = append(allowedUsers, sharedUsers...)

			if !userIn(r.Username, allowedUsers...) {
				return nil, fmt.Errorf("User '%s' is not allowed to make a call to us.", r.Username)
			}
		}

		updatingMu.Lock()
		defer updatingMu.Unlock()

		if updating {
			return nil, errors.New("Updating klient. Can't accept any method.")
		}

		return true, nil
	})

	// count only those methods, please add/remove methods here that will reset
	// the timer of a klient.
	usg.CountedMethods = map[string]bool{
		"fs.readDirectory":    true,
		"fs.glob":             true,
		"fs.readFile":         true,
		"fs.writeFile":        true,
		"fs.uniquePath":       true,
		"fs.getInfo":          true,
		"fs.setPermissions":   true,
		"fs.remove":           true,
		"fs.rename":           true,
		"fs.createDirectory":  true,
		"fs.move":             true,
		"fs.copy":             true,
		"webterm.getSessions": true,
		"webterm.connect":     true,
		"webterm.killSession": true,
		"exec":                true,
	}

	// we measure every incoming request
	k.PreHandleFunc(usg.Counter)

	// this provides us to get the current usage whenever we want
	k.HandleFunc("klient.usage", usg.Current)

	// also invoke updating
	k.Handle("klient.update", updater)

	// Collaboration
	k.HandleFunc("klient.share", collab.Share)
	k.HandleFunc("klient.unshare", collab.Unshare)
	k.HandleFunc("klient.shared", collab.Shared)

	k.HandleFunc("fs.readDirectory", fs.ReadDirectory)
	k.HandleFunc("fs.glob", fs.Glob)
	k.HandleFunc("fs.readFile", fs.ReadFile)
	k.HandleFunc("fs.writeFile", fs.WriteFile)
	k.HandleFunc("fs.uniquePath", fs.UniquePath)
	k.HandleFunc("fs.getInfo", fs.GetInfo)
	k.HandleFunc("fs.setPermissions", fs.SetPermissions)
	k.HandleFunc("fs.remove", fs.Remove)
	k.HandleFunc("fs.rename", fs.Rename)
	k.HandleFunc("fs.createDirectory", fs.CreateDirectory)
	k.HandleFunc("fs.move", fs.Move)
	k.HandleFunc("fs.copy", fs.Copy)

	terminal.ResetFunc = usg.Reset

	k.HandleFunc("webterm.getSessions", terminal.GetSessions)
	k.HandleFunc("webterm.connect", terminal.Connect)

	k.HandleFunc("webterm.killSession", terminal.KillSession)
	k.HandleFunc("exec", command.Exec)

	registerURL, err := getRegisterURL(k)
	if err != nil {
		log.Panic("could not get public ip" + err.Error())
	}

	if *flagRegisterURL != "" {
		u, err := url.Parse(*flagRegisterURL)
		if err != nil {
			k.Log.Fatal("Couldn't parse register url: %s", err)
		}

		registerURL = u
	}

	if registerURL == nil {
		log.Panic("register url is nil")
	}

	k.Log.Info("Going to register to kontrol with URL: %s", registerURL)
	if err := k.RegisterForever(registerURL); err != nil {
		log.Panic(err)
	}

	k.Log.Info("Running as version %s", VERSION)

	k.Run()
}

// Given a string of the form "host", "host:port", or "[ipv6::address]:port",
// return true if the string includes a port.
func HasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") }

// Given a string of the form "host", "port", returns "host:port"
func AddPort(host, port string) string {
	if ok := HasPort(host); ok {
		return host
	}

	return host + ":" + port
}
<|endoftext|>


<commit_before>package rest

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"socialapi/workers/common/response"
	"socialapi/workers/integration/webhook/api"
	"strconv"
)

const (
	IntegrationEndPoint = "http://localhost:7300"
	MiddlewareEndPoint  = "http://localhost:7350"
)

var ErrTypecastError = errors.New("typecast error")

func DoGithubPush(data string, token string) error {
	url := fmt.Sprintf("%s/github/%s", MiddlewareEndPoint, token)

	reader := bytes.NewReader([]byte(data))
	req, err := http.NewRequest("POST", url, reader)
	if err != nil {
		return nil
	}

	req.Header.Set("content-type", "application/json")
	req.Header.Set("User-Agent", "GitHub-Hookshot/aef1442")
	req.Header.Set("X-GitHub-Delivery", "9c072a80-19f7-11e5-8043-a9077ed0d1e6")
	req.Header.Set("X-GitHub-Event", "push")

	req.Header.Set("X-Hub-Signature", "sha1=151b614b891bec1d49f86bee527d841c8eec9abd")

	client := http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return err
	}

	if resp.StatusCode != 200 {
		return errors.New(resp.Status)
	}

	return nil
}

func DoPushRequest(data *api.PushRequest, token string) error {
	url := fmt.Sprintf("%s/push/%s", IntegrationEndPoint, token)
	_, err := sendModel("POST", url, data)

	return err
}

func DoBotChannelRequest(token string) (int64, error) {
	req := new(response.SuccessResponse)
	url := fmt.Sprintf("%s/botchannel", IntegrationEndPoint)

	resp, err := marshallAndSendRequestWithAuth("GET", url, req, token)
	if err != nil {
		return 0, err
	}

	err = json.Unmarshal(resp, req)
	if err != nil {
		return 0, err
	}
	res, ok := req.Data.(map[string]interface{})
	if !ok {
		return 0, ErrTypecastError
	}
	channelResponse, channelFound := res["channel"]
	if !channelFound {
		return 0, fmt.Errorf("channel field does not exit")
	}

	cr, channelResponseOk := channelResponse.(map[string]interface{})
	if !channelResponseOk {
		return 0, ErrTypecastError
	}

	channelIdResponse, channelIdFound := cr["id"]
	if !channelIdFound {
		return 0, fmt.Errorf("channel.id field does not exist")
	}

	channelId, channelIdOk := channelIdResponse.(string)
	if !channelIdOk {
		return 0, ErrTypecastError
	}

	return strconv.ParseInt(channelId, 10, 64)
}
<commit_msg>pivotal: rest function is written for tests<commit_after>package rest

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"socialapi/workers/common/response"
	"socialapi/workers/integration/webhook/api"
	"strconv"
)

const (
	IntegrationEndPoint = "http://localhost:7300"
	MiddlewareEndPoint  = "http://localhost:7350"
)

var ErrTypecastError = errors.New("typecast error")

func DoGithubPush(data string, token string) error {
	url := fmt.Sprintf("%s/github/%s", MiddlewareEndPoint, token)

	reader := bytes.NewReader([]byte(data))
	req, err := http.NewRequest("POST", url, reader)
	if err != nil {
		return nil
	}

	req.Header.Set("content-type", "application/json")
	req.Header.Set("User-Agent", "GitHub-Hookshot/aef1442")
	req.Header.Set("X-GitHub-Delivery", "9c072a80-19f7-11e5-8043-a9077ed0d1e6")
	req.Header.Set("X-GitHub-Event", "push")

	req.Header.Set("X-Hub-Signature", "sha1=151b614b891bec1d49f86bee527d841c8eec9abd")

	client := http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return err
	}

	if resp.StatusCode != 200 {
		return errors.New(resp.Status)
	}

	return nil
}

func DoPivotalPush(method string, data string, token string) error {
	url := fmt.Sprintf("%s/pivotal/%s", MiddlewareEndPoint, token)

	reader := bytes.NewReader([]byte(data))
	req, err := http.NewRequest(method, url, reader)
	if err != nil {
		return nil
	}

	req.Header.Set("Content-type", "application/json")
	req.Header.Set("X-TrackerToken", token)

	client := http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return err
	}

	if resp.StatusCode != 200 {
		return errors.New(resp.Status)
	}

	return nil
}

func DoPushRequest(data *api.PushRequest, token string) error {
	url := fmt.Sprintf("%s/push/%s", IntegrationEndPoint, token)
	_, err := sendModel("POST", url, data)

	return err
}

func DoBotChannelRequest(token string) (int64, error) {
	req := new(response.SuccessResponse)
	url := fmt.Sprintf("%s/botchannel", IntegrationEndPoint)

	resp, err := marshallAndSendRequestWithAuth("GET", url, req, token)
	if err != nil {
		return 0, err
	}

	err = json.Unmarshal(resp, req)
	if err != nil {
		return 0, err
	}
	res, ok := req.Data.(map[string]interface{})
	if !ok {
		return 0, ErrTypecastError
	}
	channelResponse, channelFound := res["channel"]
	if !channelFound {
		return 0, fmt.Errorf("channel field does not exit")
	}

	cr, channelResponseOk := channelResponse.(map[string]interface{})
	if !channelResponseOk {
		return 0, ErrTypecastError
	}

	channelIdResponse, channelIdFound := cr["id"]
	if !channelIdFound {
		return 0, fmt.Errorf("channel.id field does not exist")
	}

	channelId, channelIdOk := channelIdResponse.(string)
	if !channelIdOk {
		return 0, ErrTypecastError
	}

	return strconv.ParseInt(channelId, 10, 64)
}
<|endoftext|>


<commit_before>
package main

import (
  "flag"
  "bufio"
  "io"
  "os"
  "os/exec"
  "path/filepath"
  "fmt"
  "strconv"
  "strings"
  "time"
)

var resource = "-l fat,gpu=K20"

func GetFiles(inputdir string) []string {
  files, _ := filepath.Glob(inputdir + "/*.gexf")
  return files
}

func GetOutName (outdir, file string, iterations int) string {
  outname := strings.TrimSuffix(file, filepath.Ext(file))
  absout, _ := filepath.Abs(outdir)
  return filepath.Join(absout, outname) + "-" + strconv.Itoa(iterations)
}

func ReserveNode() int {
  //"-native", resource,
  command := exec.Command("preserve","-native", resource,
  "-t", "30:00", "-#", "1")
  fmt.Println(command.Args)
  cmdout, _ := command.Output()

  fmt.Println("Reserve output:", string(cmdout[:]))
  time.Sleep(2 * time.Second)

  getId := exec.Command("preserve", "-list")
  out, _ := getId.Output()
  outputString := string(out[:])
  fmt.Println(outputString)
  lines := strings.Split(outputString, "\n")
  for _, line := range lines {
    if strings.Contains(line, "jdonkerv") {
      res, _ := strconv.Atoi(strings.Fields(line)[0])
      return res
    }
  }
  return -1
}

func CleanNode(nodeid int) {
  command := exec.Command("preserve", "-c", strconv.Itoa(nodeid))
  command.Run()
}

func main () {
  inputDirPtr := flag.String("indir", "", "The directory with the gexf files to run")
  outputDirPtr := flag.String("outdir", "", "The directory where the run time files will be placed.")

  flag.Parse()

  files := GetFiles(*inputDirPtr)

  for _, fin := range files {
    iterations := 100
    outfilename := GetOutName(*outputDirPtr, filepath.Base(fin), iterations)
    fmt.Println(outfilename)

    nodeid := ReserveNode()

    fmt.Println("Using node ", nodeid)
    command := exec.Command("prun", "-no-panda", "-reserve", strconv.Itoa(nodeid),
    "-native", resource, "./ap", "1", "-i", fin,
    "-n", strconv.Itoa(iterations))

    grep := exec.Command("grep", "time", "-A", "1")
    commandOut, _ := command.StdoutPipe()
    grep.Stdin = commandOut

    outfile, _ := os.Create(outfilename)
    defer outfile.Close()

    writer := bufio.NewWriter(outfile)
    defer writer.Flush()

    grepOut, _ := grep.StdoutPipe()
    grep.Start()
    command.Start()
    io.Copy(writer, grepOut)
    grep.Wait()
    command.Wait()

    CleanNode(nodeid)
  }
}
<commit_msg>reduced number of iterations. Changed card.<commit_after>
package main

import (
  "flag"
  "bufio"
  "io"
  "os"
  "os/exec"
  "path/filepath"
  "fmt"
  "strconv"
  "strings"
  "time"
)

var resource = "-l fat,gpu=gtx680"

func GetFiles(inputdir string) []string {
  files, _ := filepath.Glob(inputdir + "/*.gexf")
  return files
}

func GetOutName (outdir, file string, iterations int) string {
  outname := strings.TrimSuffix(file, filepath.Ext(file))
  absout, _ := filepath.Abs(outdir)
  return filepath.Join(absout, outname) + "-" + strconv.Itoa(iterations)
}

func ReserveNode() int {
  //"-native", resource,
  command := exec.Command("preserve","-native", resource,
  "-t", "30:00", "-#", "1")
  fmt.Println(command.Args)
  cmdout, _ := command.Output()

  fmt.Println("Reserve output:", string(cmdout[:]))
  time.Sleep(2 * time.Second)

  getId := exec.Command("preserve", "-list")
  out, _ := getId.Output()
  outputString := string(out[:])
  fmt.Println(outputString)
  lines := strings.Split(outputString, "\n")
  for _, line := range lines {
    if strings.Contains(line, "jdonkerv") {
      res, _ := strconv.Atoi(strings.Fields(line)[0])
      return res
    }
  }
  return -1
}

func CleanNode(nodeid int) {
  command := exec.Command("preserve", "-c", strconv.Itoa(nodeid))
  command.Run()
}

func main () {
  inputDirPtr := flag.String("indir", "", "The directory with the gexf files to run")
  outputDirPtr := flag.String("outdir", "", "The directory where the run time files will be placed.")

  flag.Parse()

  files := GetFiles(*inputDirPtr)

  for _, fin := range files {
    iterations := 10
    outfilename := GetOutName(*outputDirPtr, filepath.Base(fin), iterations)
    fmt.Println(outfilename)

    nodeid := ReserveNode()

    fmt.Println("Using node ", nodeid)
    command := exec.Command("prun", "-no-panda", "-reserve", strconv.Itoa(nodeid),
    "-native", resource, "./ap", "1", "-i", fin,
    "-n", strconv.Itoa(iterations))

    grep := exec.Command("grep", "time", "-A", "1")
    commandOut, _ := command.StdoutPipe()
    grep.Stdin = commandOut

    outfile, _ := os.Create(outfilename)
    defer outfile.Close()

    writer := bufio.NewWriter(outfile)
    defer writer.Flush()

    grepOut, _ := grep.StdoutPipe()
    grep.Start()
    command.Start()
    io.Copy(writer, grepOut)
    grep.Wait()
    command.Wait()

    CleanNode(nodeid)
  }
}
<|endoftext|>


<commit_before>package app

import (
	"github.com/labstack/echo"
	"github.com/prometheus/client_golang/prometheus"
	"strconv"
	"time"
)

// HTTPMetrics is the middleware function that logs duration of responses.
func HTTPMetrics(appName string) echo.MiddlewareFunc {
	labels := []string{"method", "uri", "code"}

	echoRequests := prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: appName,
		Subsystem: "http",
		Name:      "requests_count",
		Help:      "Requests count by method/path/status.",
	}, labels)

	echoDurations := prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Namespace: appName,
		Subsystem: "http",
		Name:      "responses_histogram_seconds",
		Help:      "Response time by method/path/status.",
	}, labels)

	prometheus.MustRegister(echoRequests, echoDurations)

	return func(next echo.HandlerFunc) echo.HandlerFunc {
		return func(c echo.Context) error {
			start := time.Now()
			if err := next(c); err != nil {
				c.Error(err)
			}

			metrics := []string{c.Request().Method, c.Path(), strconv.Itoa(c.Response().Status)}

			echoDurations.WithLabelValues(metrics...).Observe(time.Since(start).Seconds())
			echoRequests.WithLabelValues(metrics...).Inc()

			return nil
		}
	}
}
<commit_msg>Change echoDurations from Histogram to Summary<commit_after>package app

import (
	"github.com/labstack/echo"
	"github.com/prometheus/client_golang/prometheus"
	"strconv"
	"time"
)

// HTTPMetrics is the middleware function that logs duration of responses.
func HTTPMetrics(appName string) echo.MiddlewareFunc {
	labels := []string{"method", "uri", "code"}

	echoRequests := prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: appName,
		Subsystem: "http",
		Name:      "requests_count",
		Help:      "Requests count by method/path/status.",
	}, labels)

	echoDurations := prometheus.NewSummaryVec(prometheus.SummaryOpts{
		Namespace: appName,
		Subsystem: "http",
		Name:      "responses_duration_seconds",
		Help:      "Response time by method/path/status.",
	}, labels)

	prometheus.MustRegister(echoRequests, echoDurations)

	return func(next echo.HandlerFunc) echo.HandlerFunc {
		return func(c echo.Context) error {
			start := time.Now()
			if err := next(c); err != nil {
				c.Error(err)
			}

			metrics := []string{c.Request().Method, c.Path(), strconv.Itoa(c.Response().Status)}

			echoDurations.WithLabelValues(metrics...).Observe(time.Since(start).Seconds())
			echoRequests.WithLabelValues(metrics...).Inc()

			return nil
		}
	}
}
<|endoftext|>


<commit_before>package app

import ()

const (
	Unsupported Format = iota // not supported
	LZ4                       // LZ4 compression
)

type Format int

func (f Format) String() string {
	switch f {
	case LZ4:
		return "lz4"
	}
	return "unsupported"
}

// Magic numbers for supported formats
var (
	headerLZ4 = []byte{0x18, 0x4d, 0x22, 0x04} // first 4 bytes
)

// getFileFormat tries to match up the data in the Reader to a supported
// magic number, if a match isn't found, UnsupportedFmt is returned
func getFileFormat(r io.ReaderAt) Format {
	h := make([]byte, 8, 8) // 8 is minimum cap of a byte slice so...
	// Reat the first 8 bytes since that's where most magic numbers are
	r.ReadAt(h, 0)
	// for the byte comparison, use only the parts we need
	if bytes.Equal(headerLZ4, h[0:4]) {
		return LZ4
	}
	return Unsupported
}
<commit_msg>import the referenced packages, duh!<commit_after>package app

import (
	"bytes"
	"io"
)

const (
	Unsupported Format = iota // not supported
	LZ4                       // LZ4 compression
)

type Format int

func (f Format) String() string {
	switch f {
	case LZ4:
		return "lz4"
	}
	return "unsupported"
}

// Magic numbers for supported formats
var (
	headerLZ4 = []byte{0x18, 0x4d, 0x22, 0x04} // first 4 bytes
)

// getFileFormat tries to match up the data in the Reader to a supported
// magic number, if a match isn't found, UnsupportedFmt is returned
func getFileFormat(r io.ReaderAt) Format {
	h := make([]byte, 8, 8) // 8 is minimum cap of a byte slice so...
	// Reat the first 8 bytes since that's where most magic numbers are
	r.ReadAt(h, 0)
	// for the byte comparison, use only the parts we need
	if bytes.Equal(headerLZ4, h[0:4]) {
		return LZ4
	}
	return Unsupported
}
<|endoftext|>


<commit_before>/*
 * (c) 2014, Tonnerre Lombard <tonnerre@ancient-solutions.com>,
 *	     Starship Factory. All rights reserved.
 *
 * Redistribution and use in source  and binary forms, with or without
 * modification, are permitted  provided that the following conditions
 * are met:
 *
 * * Redistributions of  source code  must retain the  above copyright
 *   notice, this list of conditions and the following disclaimer.
 * * Redistributions in binary form must reproduce the above copyright
 *   notice, this  list of conditions and the  following disclaimer in
 *   the  documentation  and/or  other  materials  provided  with  the
 *   distribution.
 * * Neither  the name  of the Starship Factory  nor the  name  of its
 *   contributors may  be used to endorse or  promote products derived
 *   from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS"  AND ANY EXPRESS  OR IMPLIED WARRANTIES  OF MERCHANTABILITY
 * AND FITNESS  FOR A PARTICULAR  PURPOSE ARE DISCLAIMED. IN  NO EVENT
 * SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
 * INDIRECT, INCIDENTAL, SPECIAL,  EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED  TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE,  DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT  LIABILITY,  OR  TORT  (INCLUDING NEGLIGENCE  OR  OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
 * OF THE POSSIBILITY OF SUCH DAMAGE.
 */

package main

import (
	"flag"
	"github.com/tonnerre/x509keyserver"
	"html/template"
	"log"
	"net/http"
	"net/rpc"
)

func main() {
	var tmpl *template.Template
	var ks *x509keyserver.X509KeyServer
	var kdb *x509keyserver.X509KeyDB
	var bind string
	var tmpl_path string
	var dbserver, keyspace string
	var err error

	flag.StringVar(&bind, "bind", "[::]:8080",
		"host:port pair to bind the HTTP/RPC server to")
	flag.StringVar(&tmpl_path, "template", "keylist.html",
		"Path to the template file for displaying")

	flag.StringVar(&dbserver, "cassandra-server", "localhost:9160",
		"host:port pair of the Cassandra database server")
	flag.StringVar(&keyspace, "cassandra-keyspace", "x509certs",
		"Cassandra keyspace in which the relevant column families are stored")
	flag.Parse()

	// Set up the connection to the key database.
	kdb, err = x509keyserver.NewX509KeyDB(dbserver, keyspace)
	if err != nil {
		log.Fatal("Error connecting to key database: ", err)
	}
	ks = &x509keyserver.X509KeyServer{
		Db: kdb,
	}

	// Register the RPC service.
	err = rpc.Register(ks)
	if err != nil {
		log.Fatal("Error registering RPC handler for the key server: ", err)
	}

	tmpl, err = template.ParseFiles(tmpl_path)
	if err != nil {
		log.Fatal("Error parsing template ", tmpl_path, ": ", err)
	}

	// Tell the HTTP server to handle RPCs.
	rpc.HandleHTTP()

	err = http.ListenAndServe(bind, &x509keyserver.HTTPKeyService{
		Db:   kdb,
		Tmpl: tmpl,
	})
	if err != nil {
		log.Fatal("Error binding to ", bind, ": ", err)
	}
}
<commit_msg>Export /css/ and /js/ static to allow styling.<commit_after>/*
 * (c) 2014, Tonnerre Lombard <tonnerre@ancient-solutions.com>,
 *	     Starship Factory. All rights reserved.
 *
 * Redistribution and use in source  and binary forms, with or without
 * modification, are permitted  provided that the following conditions
 * are met:
 *
 * * Redistributions of  source code  must retain the  above copyright
 *   notice, this list of conditions and the following disclaimer.
 * * Redistributions in binary form must reproduce the above copyright
 *   notice, this  list of conditions and the  following disclaimer in
 *   the  documentation  and/or  other  materials  provided  with  the
 *   distribution.
 * * Neither  the name  of the Starship Factory  nor the  name  of its
 *   contributors may  be used to endorse or  promote products derived
 *   from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS"  AND ANY EXPRESS  OR IMPLIED WARRANTIES  OF MERCHANTABILITY
 * AND FITNESS  FOR A PARTICULAR  PURPOSE ARE DISCLAIMED. IN  NO EVENT
 * SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
 * INDIRECT, INCIDENTAL, SPECIAL,  EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED  TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE,  DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT  LIABILITY,  OR  TORT  (INCLUDING NEGLIGENCE  OR  OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
 * OF THE POSSIBILITY OF SUCH DAMAGE.
 */

package main

import (
	"flag"
	"github.com/tonnerre/x509keyserver"
	"html/template"
	"log"
	"net/http"
	"net/rpc"
)

func main() {
	var tmpl *template.Template
	var ks *x509keyserver.X509KeyServer
	var kdb *x509keyserver.X509KeyDB
	var bind string
	var tmpl_path, static_path string
	var dbserver, keyspace string
	var err error

	flag.StringVar(&bind, "bind", "[::]:8080",
		"host:port pair to bind the HTTP/RPC server to")
	flag.StringVar(&static_path, "static-path", ".",
		"Path to the required static files for the web interface")
	flag.StringVar(&tmpl_path, "template", "keylist.html",
		"Path to the template file for displaying")

	flag.StringVar(&dbserver, "cassandra-server", "localhost:9160",
		"host:port pair of the Cassandra database server")
	flag.StringVar(&keyspace, "cassandra-keyspace", "x509certs",
		"Cassandra keyspace in which the relevant column families are stored")
	flag.Parse()

	// Set up the connection to the key database.
	kdb, err = x509keyserver.NewX509KeyDB(dbserver, keyspace)
	if err != nil {
		log.Fatal("Error connecting to key database: ", err)
	}
	ks = &x509keyserver.X509KeyServer{
		Db: kdb,
	}

	// Register the RPC service.
	err = rpc.Register(ks)
	if err != nil {
		log.Fatal("Error registering RPC handler for the key server: ", err)
	}

	tmpl, err = template.ParseFiles(tmpl_path)
	if err != nil {
		log.Fatal("Error parsing template ", tmpl_path, ": ", err)
	}

	// Tell the HTTP server to handle RPCs.
	rpc.HandleHTTP()

	http.Handle("/", &x509keyserver.HTTPKeyService{
		Db:   kdb,
		Tmpl: tmpl,
	})
	http.Handle("/css/", http.FileServer(http.Dir(static_path)))
	http.Handle("/js/", http.FileServer(http.Dir(static_path)))

	err = http.ListenAndServe(bind, nil)
	if err != nil {
		log.Fatal("Error binding to ", bind, ": ", err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package models

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/Unknwon/com"
)

var (
	sshOpLocker = sync.Mutex{}
	//publicKeyRootPath string
	sshPath string
	appPath string
	// "### autogenerated by gitgos, DO NOT EDIT\n"
	tmplPublicKey = "command=\"%s serv key-%d\",no-port-forwarding," +
		"no-X11-forwarding,no-agent-forwarding,no-pty %s\n"
)

func exePath() (string, error) {
	file, err := exec.LookPath(os.Args[0])
	if err != nil {
		return "", err
	}
	return filepath.Abs(file)
}

func homeDir() string {
	home, err := com.HomeDir()
	if err != nil {
		return "/"
	}
	return home
}

func init() {
	var err error
	appPath, err = exePath()
	if err != nil {
		println(err.Error())
		os.Exit(2)
	}

	sshPath = filepath.Join(homeDir(), ".ssh")
}

type PublicKey struct {
	Id      int64
	OwnerId int64     `xorm:"index"`
	Name    string    `xorm:"unique not null"`
	Content string    `xorm:"text not null"`
	Created time.Time `xorm:"created"`
	Updated time.Time `xorm:"updated"`
}

func GenAuthorizedKey(keyId int64, key string) string {
	return fmt.Sprintf(tmplPublicKey, appPath, keyId, key)
}

func AddPublicKey(key *PublicKey) error {
	_, err := orm.Insert(key)
	if err != nil {
		return err
	}

	err = SaveAuthorizedKeyFile(key)
	if err != nil {
		_, err2 := orm.Delete(key)
		if err2 != nil {
			// TODO: log the error
		}
		return err
	}

	return nil
}

// DeletePublicKey deletes SSH key information both in database and authorized_keys file.
func DeletePublicKey(key *PublicKey) (err error) {
	if _, err = orm.Delete(key); err != nil {
		return err
	}

	sshOpLocker.Lock()
	defer sshOpLocker.Unlock()

	p := filepath.Join(sshPath, "authorized_keys")
	tmpP := filepath.Join(sshPath, "authorized_keys.tmp")
	fr, err := os.Open(p)
	if err != nil {
		return err
	}
	defer fr.Close()

	fw, err := os.Create(tmpP)
	if err != nil {
		return err
	}
	defer fw.Close()

	buf := bufio.NewReader(fr)
	for {
		line, errRead := buf.ReadString('\n')
		line = strings.TrimSpace(line)

		if errRead != nil {
			if errRead != io.EOF {
				return errRead
			}

			// Reached end of file, if nothing to read then break,
			// otherwise handle the last line.
			if len(line) == 0 {
				break
			}
		}

		// Found the line and copy rest of file.
		if strings.Contains(line, key.Content) {
			if _, err = io.Copy(fw, fr); err != nil {
				return err
			}
			break
		}

		// Still finding the line, copy the line that currently read.
		if _, err = fw.WriteString(line + "\n"); err != nil {
			return err
		}

		if errRead == io.EOF {
			break
		}
	}

	if err = os.Remove(p); err != nil {
		return err
	}

	return os.Rename(tmpP, p)
}

func ListPublicKey(userId int64) ([]PublicKey, error) {
	keys := make([]PublicKey, 0)
	err := orm.Find(&keys, &PublicKey{OwnerId: userId})
	return keys, err
}

func SaveAuthorizedKeyFile(key *PublicKey) error {
	sshOpLocker.Lock()
	defer sshOpLocker.Unlock()

	p := filepath.Join(sshPath, "authorized_keys")
	f, err := os.OpenFile(p, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600)
	if err != nil {
		return err
	}
	defer f.Close()

	//os.Chmod(p, 0600)
	_, err = f.WriteString(GenAuthorizedKey(key.Id, key.Content))
	return err
}
<commit_msg>Bug fix<commit_after>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package models

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/Unknwon/com"
)

var (
	sshOpLocker = sync.Mutex{}
	//publicKeyRootPath string
	sshPath string
	appPath string
	// "### autogenerated by gitgos, DO NOT EDIT\n"
	tmplPublicKey = "command=\"%s serv key-%d\",no-port-forwarding," +
		"no-X11-forwarding,no-agent-forwarding,no-pty %s\n"
)

func exePath() (string, error) {
	file, err := exec.LookPath(os.Args[0])
	if err != nil {
		return "", err
	}
	return filepath.Abs(file)
}

func homeDir() string {
	home, err := com.HomeDir()
	if err != nil {
		return "/"
	}
	return home
}

func init() {
	var err error
	appPath, err = exePath()
	if err != nil {
		println(err.Error())
		os.Exit(2)
	}

	sshPath = filepath.Join(homeDir(), ".ssh")
}

type PublicKey struct {
	Id      int64
	OwnerId int64     `xorm:"index"`
	Name    string    `xorm:"unique not null"`
	Content string    `xorm:"text not null"`
	Created time.Time `xorm:"created"`
	Updated time.Time `xorm:"updated"`
}

func GenAuthorizedKey(keyId int64, key string) string {
	return fmt.Sprintf(tmplPublicKey, appPath, keyId, key)
}

func AddPublicKey(key *PublicKey) error {
	_, err := orm.Insert(key)
	if err != nil {
		return err
	}

	err = SaveAuthorizedKeyFile(key)
	if err != nil {
		_, err2 := orm.Delete(key)
		if err2 != nil {
			// TODO: log the error
		}
		return err
	}

	return nil
}

// DeletePublicKey deletes SSH key information both in database and authorized_keys file.
func DeletePublicKey(key *PublicKey) (err error) {
	has, err := orm.Id(key.Id).Get(key)
	if err != nil {
		return err
	} else if !has {
		return errors.New("Public key does not exist")
	}
	if _, err = orm.Delete(key); err != nil {
		return err
	}

	sshOpLocker.Lock()
	defer sshOpLocker.Unlock()

	p := filepath.Join(sshPath, "authorized_keys")
	tmpP := filepath.Join(sshPath, "authorized_keys.tmp")
	fr, err := os.Open(p)
	if err != nil {
		return err
	}
	defer fr.Close()

	fw, err := os.Create(tmpP)
	if err != nil {
		return err
	}
	defer fw.Close()

	buf := bufio.NewReader(fr)
	for {
		line, errRead := buf.ReadString('\n')
		line = strings.TrimSpace(line)

		if errRead != nil {
			if errRead != io.EOF {
				return errRead
			}

			// Reached end of file, if nothing to read then break,
			// otherwise handle the last line.
			if len(line) == 0 {
				break
			}
		}

		// Found the line and copy rest of file.
		if strings.Contains(line, key.Content) {
			continue
		}
		// Still finding the line, copy the line that currently read.
		if _, err = fw.WriteString(line + "\n"); err != nil {
			return err
		}

		if errRead == io.EOF {
			break
		}
	}

	if err = os.Remove(p); err != nil {
		return err
	}

	return os.Rename(tmpP, p)
}

func ListPublicKey(userId int64) ([]PublicKey, error) {
	keys := make([]PublicKey, 0)
	err := orm.Find(&keys, &PublicKey{OwnerId: userId})
	return keys, err
}

func SaveAuthorizedKeyFile(key *PublicKey) error {
	sshOpLocker.Lock()
	defer sshOpLocker.Unlock()

	p := filepath.Join(sshPath, "authorized_keys")
	f, err := os.OpenFile(p, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600)
	if err != nil {
		return err
	}
	defer f.Close()

	//os.Chmod(p, 0600)
	_, err = f.WriteString(GenAuthorizedKey(key.Id, key.Content))
	return err
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Run "make install" to build package.

package main

import (
	"bytes"
	"os"
	"path/filepath"
	"template"
)

// domake builds the package in dir.
// If local is false, the package was copied from an external system.
// For non-local packages or packages without Makefiles,
// domake generates a standard Makefile and passes it
// to make on standard input.
func domake(dir, pkg string, root *pkgroot, local, isCmd bool) (err os.Error) {
	needMakefile := true
	if local {
		_, err := os.Stat(dir + "/Makefile")
		if err == nil {
			needMakefile = false
		}
	}
	cmd := []string{"gomake"}
	var makefile []byte
	if needMakefile {
		if makefile, err = makeMakefile(dir, pkg, root, isCmd); err != nil {
			return err
		}
		cmd = append(cmd, "-f-")
	}
	if *clean {
		cmd = append(cmd, "clean")
	}
	cmd = append(cmd, "install")
	return run(dir, makefile, cmd...)
}

// makeMakefile computes the standard Makefile for the directory dir
// installing as package pkg.  It includes all *.go files in the directory
// except those in package main and those ending in _test.go.
func makeMakefile(dir, pkg string, root *pkgroot, isCmd bool) ([]byte, os.Error) {
	if !safeName(pkg) {
		return nil, os.ErrorString("unsafe name: " + pkg)
	}
	targ := pkg
	targDir := root.pkgDir()
	if isCmd {
		// use the last part of the package name only
		_, targ = filepath.Split(pkg)
		// if building the working dir use the directory name
		if targ == "." {
			d, err := filepath.Abs(dir)
			if err != nil {
				return nil, os.NewError("finding path: " + err.String())
			}
			_, targ = filepath.Split(d)
		}
		targDir = root.binDir()
	}
	dirInfo, err := scanDir(dir, isCmd)
	if err != nil {
		return nil, err
	}

	cgoFiles := dirInfo.cgoFiles
	isCgo := make(map[string]bool, len(cgoFiles))
	for _, file := range cgoFiles {
		if !safeName(file) {
			return nil, os.ErrorString("bad name: " + file)
		}
		isCgo[file] = true
	}

	goFiles := make([]string, 0, len(dirInfo.goFiles))
	for _, file := range dirInfo.goFiles {
		if !safeName(file) {
			return nil, os.ErrorString("unsafe name: " + file)
		}
		if !isCgo[file] {
			goFiles = append(goFiles, file)
		}
	}

	oFiles := make([]string, 0, len(dirInfo.cFiles)+len(dirInfo.sFiles))
	cgoOFiles := make([]string, 0, len(dirInfo.cFiles))
	for _, file := range dirInfo.cFiles {
		if !safeName(file) {
			return nil, os.ErrorString("unsafe name: " + file)
		}
		// When cgo is in use, C files are compiled with gcc,
		// otherwise they're compiled with gc.
		if len(cgoFiles) > 0 {
			cgoOFiles = append(cgoOFiles, file[:len(file)-2]+".o")
		} else {
			oFiles = append(oFiles, file[:len(file)-2]+".$O")
		}
	}

	for _, file := range dirInfo.sFiles {
		if !safeName(file) {
			return nil, os.ErrorString("unsafe name: " + file)
		}
		oFiles = append(oFiles, file[:len(file)-2]+".$O")
	}

	var buf bytes.Buffer
	md := makedata{targ, targDir, "pkg", goFiles, oFiles, cgoFiles, cgoOFiles, imports}
	if isCmd {
		md.Type = "cmd"
	}
	if err := makefileTemplate.Execute(&buf, &md); err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

var safeBytes = []byte("+-./0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz")

func safeName(s string) bool {
	if s == "" {
		return false
	}
	for i := 0; i < len(s); i++ {
		if c := s[i]; c < 0x80 && bytes.IndexByte(safeBytes, c) < 0 {
			return false
		}
	}
	return true
}

// makedata is the data type for the makefileTemplate.
type makedata struct {
	Targ      string   // build target
	TargDir   string   // build target directory
	Type      string   // build type: "pkg" or "cmd"
	GoFiles   []string // list of non-cgo .go files
	OFiles    []string // list of .$O files
	CgoFiles  []string // list of cgo .go files
	CgoOFiles []string // list of cgo .o files, without extension
	Imports   []string // gc/ld import paths
}

var makefileTemplate = template.MustParse(`
include $(GOROOT)/src/Make.inc

TARG={Targ}
TARGDIR={TargDir}

{.section GoFiles}
GOFILES=\
{.repeated section GoFiles}
	{@}\
{.end}

{.end}
{.section OFiles}
OFILES=\
{.repeated section OFiles}
	{@}\
{.end}

{.end}
{.section CgoFiles}
CGOFILES=\
{.repeated section CgoFiles}
	{@}\
{.end}

{.end}
{.section CgoOFiles}
CGO_OFILES=\
{.repeated section CgoOFiles}
	{@}\
{.end}

{.end}
GCIMPORTS={.repeated section Imports}-I "{@}" {.end}
LDIMPORTS={.repeated section Imports}-L "{@}" {.end}

include $(GOROOT)/src/Make.{Type}
`,
	nil)
<commit_msg>goinstall: use bash to execute gomake<commit_after>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Run "make install" to build package.

package main

import (
	"bytes"
	"os"
	"path/filepath"
	"template"
)

// domake builds the package in dir.
// If local is false, the package was copied from an external system.
// For non-local packages or packages without Makefiles,
// domake generates a standard Makefile and passes it
// to make on standard input.
func domake(dir, pkg string, root *pkgroot, local, isCmd bool) (err os.Error) {
	needMakefile := true
	if local {
		_, err := os.Stat(filepath.Join(dir, "Makefile"))
		if err == nil {
			needMakefile = false
		}
	}
	cmd := []string{"bash", "gomake"}
	var makefile []byte
	if needMakefile {
		if makefile, err = makeMakefile(dir, pkg, root, isCmd); err != nil {
			return err
		}
		cmd = append(cmd, "-f-")
	}
	if *clean {
		cmd = append(cmd, "clean")
	}
	cmd = append(cmd, "install")
	return run(dir, makefile, cmd...)
}

// makeMakefile computes the standard Makefile for the directory dir
// installing as package pkg.  It includes all *.go files in the directory
// except those in package main and those ending in _test.go.
func makeMakefile(dir, pkg string, root *pkgroot, isCmd bool) ([]byte, os.Error) {
	if !safeName(pkg) {
		return nil, os.ErrorString("unsafe name: " + pkg)
	}
	targ := pkg
	targDir := root.pkgDir()
	if isCmd {
		// use the last part of the package name only
		_, targ = filepath.Split(pkg)
		// if building the working dir use the directory name
		if targ == "." {
			d, err := filepath.Abs(dir)
			if err != nil {
				return nil, os.NewError("finding path: " + err.String())
			}
			_, targ = filepath.Split(d)
		}
		targDir = root.binDir()
	}
	dirInfo, err := scanDir(dir, isCmd)
	if err != nil {
		return nil, err
	}

	cgoFiles := dirInfo.cgoFiles
	isCgo := make(map[string]bool, len(cgoFiles))
	for _, file := range cgoFiles {
		if !safeName(file) {
			return nil, os.ErrorString("bad name: " + file)
		}
		isCgo[file] = true
	}

	goFiles := make([]string, 0, len(dirInfo.goFiles))
	for _, file := range dirInfo.goFiles {
		if !safeName(file) {
			return nil, os.ErrorString("unsafe name: " + file)
		}
		if !isCgo[file] {
			goFiles = append(goFiles, file)
		}
	}

	oFiles := make([]string, 0, len(dirInfo.cFiles)+len(dirInfo.sFiles))
	cgoOFiles := make([]string, 0, len(dirInfo.cFiles))
	for _, file := range dirInfo.cFiles {
		if !safeName(file) {
			return nil, os.ErrorString("unsafe name: " + file)
		}
		// When cgo is in use, C files are compiled with gcc,
		// otherwise they're compiled with gc.
		if len(cgoFiles) > 0 {
			cgoOFiles = append(cgoOFiles, file[:len(file)-2]+".o")
		} else {
			oFiles = append(oFiles, file[:len(file)-2]+".$O")
		}
	}

	for _, file := range dirInfo.sFiles {
		if !safeName(file) {
			return nil, os.ErrorString("unsafe name: " + file)
		}
		oFiles = append(oFiles, file[:len(file)-2]+".$O")
	}

	var buf bytes.Buffer
	md := makedata{targ, targDir, "pkg", goFiles, oFiles, cgoFiles, cgoOFiles, imports}
	if isCmd {
		md.Type = "cmd"
	}
	if err := makefileTemplate.Execute(&buf, &md); err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

var safeBytes = []byte("+-./0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz")

func safeName(s string) bool {
	if s == "" {
		return false
	}
	for i := 0; i < len(s); i++ {
		if c := s[i]; c < 0x80 && bytes.IndexByte(safeBytes, c) < 0 {
			return false
		}
	}
	return true
}

// makedata is the data type for the makefileTemplate.
type makedata struct {
	Targ      string   // build target
	TargDir   string   // build target directory
	Type      string   // build type: "pkg" or "cmd"
	GoFiles   []string // list of non-cgo .go files
	OFiles    []string // list of .$O files
	CgoFiles  []string // list of cgo .go files
	CgoOFiles []string // list of cgo .o files, without extension
	Imports   []string // gc/ld import paths
}

var makefileTemplate = template.MustParse(`
include $(GOROOT)/src/Make.inc

TARG={Targ}
TARGDIR={TargDir}

{.section GoFiles}
GOFILES=\
{.repeated section GoFiles}
	{@}\
{.end}

{.end}
{.section OFiles}
OFILES=\
{.repeated section OFiles}
	{@}\
{.end}

{.end}
{.section CgoFiles}
CGOFILES=\
{.repeated section CgoFiles}
	{@}\
{.end}

{.end}
{.section CgoOFiles}
CGO_OFILES=\
{.repeated section CgoOFiles}
	{@}\
{.end}

{.end}
GCIMPORTS={.repeated section Imports}-I "{@}" {.end}
LDIMPORTS={.repeated section Imports}-L "{@}" {.end}

include $(GOROOT)/src/Make.{Type}
`,
	nil)
<|endoftext|>


<commit_before>package idea

import (
	"bufio"
	"bytes"
	"fmt"
	"github.com/ghthor/gospec"
	. "github.com/ghthor/gospec"
	"github.com/ghthor/journal/git"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
)

func DescribeIdeaStore(c gospec.Context) {
	c.Specify("a directory store", func() {
		makeEmptyDirectory := func(prefix string) string {
			d, err := ioutil.TempDir("_test", prefix+"_")
			c.Assume(err, IsNil)
			return d
		}

		makeDirectoryStore := func(prefix string) (*DirectoryStore, string) {
			d := makeEmptyDirectory(prefix)

			// Verify the directory isn't an DirectoryStore
			_, err := NewDirectoryStore(d)
			c.Assume(IsInvalidDirectoryStoreError(err), IsTrue)

			// Initialize the directory
			id, _, err := InitDirectoryStore(d)
			c.Assume(err, IsNil)
			c.Assume(id, Not(IsNil))

			// Verify the directory has been initialized
			id, err = NewDirectoryStore(d)
			c.Assume(err, IsNil)
			c.Assume(id, Not(IsNil))

			return id, d
		}

		c.Specify("can be initialized", func() {
			d := makeEmptyDirectory("directory_store_init")

			id, commitable, err := InitDirectoryStore(d)
			c.Assume(err, IsNil)
			c.Expect(id, Not(IsNil))

			c.Expect(id.root, Equals, d)

			c.Specify("only once", func() {
				_, _, err = InitDirectoryStore(d)
				c.Expect(err, Equals, ErrInitOnExistingDirectoryStore)
			})

			c.Specify("and the modifications made during initialization are commitable", func() {
				c.Expect(commitable, Not(IsNil))
				c.Expect(commitable.WorkingDirectory(), Equals, d)
				c.Expect(commitable.Changes(), ContainsAll, []git.ChangedFile{
					git.ChangedFile("nextid"),
					git.ChangedFile("active"),
				})
				c.Expect(commitable.CommitMsg(), Equals, "directory store initialized")

				// Initialize and empty repo
				c.Assume(git.Init(d), IsNil)
				// Commit the directory store initialization
				c.Expect(git.Commit(commitable), IsNil)

				o, err := git.Command(d, "show", "--no-color", "--pretty=format:\"%s%b\"").Output()
				c.Assume(err, IsNil)
				c.Expect(string(o), Equals,
					`"directory store initialized"
diff --git a/active b/active
new file mode 100644
index 0000000..e69de29
diff --git a/nextid b/nextid
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/nextid
@@ -0,0 +1 @@
+1
`)
			})
		})

		c.Specify("contains an index of the next available id", func() {
			_, d := makeDirectoryStore("directory_store_spec")

			data, err := ioutil.ReadFile(filepath.Join(d, "nextid"))
			c.Expect(err, IsNil)
			c.Expect(string(data), Equals, "1\n")
		})

		c.Specify("contains an index of active ideas", func() {
			_, d := makeDirectoryStore("directory_store_spec")

			_, err := os.Stat(filepath.Join(d, "active"))
			c.Expect(err, IsNil)
		})

		c.Specify("contains ideas stored in a files", func() {
			c.Specify("with the id as the filename", func() {
			})
		})

		c.Specify("can create a new idea", func() {
			id, d := makeDirectoryStore("directory_store_create")

			type newIdea struct {
				changes git.Commitable
				idea    *Idea
			}

			newIdeas := []*newIdea{{
				idea: &Idea{
					IS_Active,
					0,
					"A New Idea",
					"Some body text\n",
				},
			}, {
				idea: &Idea{
					IS_Inactive,
					0,
					"Another New Idea",
					"That isn't active\n",
				},
			}, {
				idea: &Idea{
					IS_Active,
					0,
					"Another Active New Idea",
					"That should be active\n",
				},
			}}

			for _, ni := range newIdeas {
				changes, err := id.SaveNewIdea(ni.idea)
				c.Assume(err, IsNil)
				c.Assume(changes, Not(IsNil))
				ni.changes = changes
			}

			c.Specify("by assigning the next available id to the idea", func() {
				c.Expect(newIdeas[0].idea.Id, Equals, uint(1))
				c.Expect(newIdeas[1].idea.Id, Equals, uint(2))
			})

			c.Specify("by incrementing the next available id", func() {
				data, err := ioutil.ReadFile(filepath.Join(d, "nextid"))
				c.Assume(err, IsNil)

				var nextId uint
				n, err := fmt.Fscan(bytes.NewReader(data), &nextId)
				c.Expect(err, IsNil)
				c.Expect(n, Equals, 1)
				c.Expect(nextId, Equals, uint(len(newIdeas)+1))

				c.Specify("and will return a commitable change for modifying the next available id", func() {
					for _, ni := range newIdeas {
						c.Expect(ni.changes.Changes(), Contains, git.ChangedFile("nextid"))
					}
				})
			})

			c.Specify("by writing the idea to a file", func() {
				pathTo := func(idea *Idea) string {
					return filepath.Join(d, fmt.Sprintf("%d", idea.Id))
				}

				c.Specify("with the id as the filename", func() {

					for _, ni := range newIdeas {
						_, err := os.Stat(pathTo(ni.idea))
						c.Expect(!os.IsNotExist(err), IsTrue)
					}
				})

				for _, ni := range newIdeas {
					actualData, err := ioutil.ReadFile(pathTo(ni.idea))
					c.Assume(err, IsNil)

					r, err := NewIdeaReader(*ni.idea)
					c.Assume(err, IsNil)
					expectedData, err := ioutil.ReadAll(r)
					c.Assume(err, IsNil)

					c.Expect(string(actualData), Equals, string(expectedData))
				}

				c.Specify("and return a commitable change for the new idea file", func() {
					for _, ni := range newIdeas {
						c.Expect(ni.changes.Changes(), Contains, git.ChangedFile(fmt.Sprint(ni.idea.Id)))
					}
				})
			})

			c.Specify("and if the idea's status is active", func() {
				activeIdeas := make([]*Idea, 0, 2)
				for _, ni := range newIdeas {
					if ni.idea.Status == IS_Active {
						activeIdeas = append(activeIdeas, ni.idea)
					}
				}
				c.Assume(len(activeIdeas), Equals, 2)

				c.Specify("will add the idea's id to the active index", func() {
					data, err := ioutil.ReadFile(filepath.Join(d, "active"))
					c.Assume(err, IsNil)

					r := bytes.NewReader(data)

					var id uint
					activeIdeaIds := make([]uint, 0, len(activeIdeas))

					// Can just use fmt.Fscan because we know how many lines there are
					for i := 0; i < len(activeIdeas); i++ {
						_, err := fmt.Fscan(r, &id)
						c.Assume(err, IsNil)

						activeIdeaIds = append(activeIdeaIds, id)
					}

					_, err = fmt.Fscan(r, &id)
					c.Assume(err, Equals, io.EOF)

					c.Specify("and will return a commitable change for modifying the index", func() {
						for _, ni := range newIdeas {
							if ni.idea.Status == IS_Active {
								c.Expect(ni.changes.Changes(), Contains, git.ChangedFile("active"))
							} else {
								c.Expect(ni.changes.Changes(), Not(Contains), git.ChangedFile("active"))
							}
						}
					})
				})
			})

			c.Specify("and if the idea's status isn't active", func() {
				notActiveIdeas := make([]*Idea, 0, 1)
				for _, ni := range newIdeas {
					if ni.idea.Status != IS_Active {
						notActiveIdeas = append(notActiveIdeas, ni.idea)
					}
				}
				c.Assume(len(notActiveIdeas), Equals, 1)

				c.Specify("will not add the idea's id to the active index", func() {
					// Collect the id's from the index file
					data, err := ioutil.ReadFile(filepath.Join(d, "active"))
					c.Assume(err, IsNil)

					// Using a scanner because I don't know how many there are
					scanner := bufio.NewScanner(bytes.NewReader(data))
					activeIds := make([]uint, 0, len(newIdeas))

					for scanner.Scan() {
						var id uint
						_, err := fmt.Fscan(bytes.NewReader(scanner.Bytes()), &id)
						c.Assume(err, IsNil)
						activeIds = append(activeIds, id)
					}
					c.Assume(len(activeIds), Equals, 2)

					for _, idea := range notActiveIdeas {
						c.Expect(activeIds, Not(Contains), idea.Id)
					}
				})
			})

			c.Specify("and returns a commitable change", func() {
				for _, ni := range newIdeas {
					c.Expect(ni.changes.CommitMsg(), Equals, fmt.Sprintf("IDEA - %d - Created", ni.idea.Id))
				}
			})
		})

		c.Specify("can update an existing idea", func() {
			c.Specify("by writing the idea to the file", func() {
				c.Specify("with the id as the filename", func() {
				})

				c.Specify("and will return a commitable change for the modified idea file", func() {
				})
			})

			c.Specify("and if the idea's status is active", func() {
				c.Specify("will add the idea's id to the active index", func() {
					c.Specify("and will return a commitable change for modifying the index", func() {
					})
				})
			})

			c.Specify("and if the idea's status isn't active", func() {
				c.Specify("will not add the idea's id to the active index", func() {
				})
			})
		})
	})
}
<commit_msg>Fixed spec for expected contents of active index<commit_after>package idea

import (
	"bufio"
	"bytes"
	"fmt"
	"github.com/ghthor/gospec"
	. "github.com/ghthor/gospec"
	"github.com/ghthor/journal/git"
	"io/ioutil"
	"os"
	"path/filepath"
)

func DescribeIdeaStore(c gospec.Context) {
	c.Specify("a directory store", func() {
		makeEmptyDirectory := func(prefix string) string {
			d, err := ioutil.TempDir("_test", prefix+"_")
			c.Assume(err, IsNil)
			return d
		}

		makeDirectoryStore := func(prefix string) (*DirectoryStore, string) {
			d := makeEmptyDirectory(prefix)

			// Verify the directory isn't an DirectoryStore
			_, err := NewDirectoryStore(d)
			c.Assume(IsInvalidDirectoryStoreError(err), IsTrue)

			// Initialize the directory
			id, _, err := InitDirectoryStore(d)
			c.Assume(err, IsNil)
			c.Assume(id, Not(IsNil))

			// Verify the directory has been initialized
			id, err = NewDirectoryStore(d)
			c.Assume(err, IsNil)
			c.Assume(id, Not(IsNil))

			return id, d
		}

		c.Specify("can be initialized", func() {
			d := makeEmptyDirectory("directory_store_init")

			id, commitable, err := InitDirectoryStore(d)
			c.Assume(err, IsNil)
			c.Expect(id, Not(IsNil))

			c.Expect(id.root, Equals, d)

			c.Specify("only once", func() {
				_, _, err = InitDirectoryStore(d)
				c.Expect(err, Equals, ErrInitOnExistingDirectoryStore)
			})

			c.Specify("and the modifications made during initialization are commitable", func() {
				c.Expect(commitable, Not(IsNil))
				c.Expect(commitable.WorkingDirectory(), Equals, d)
				c.Expect(commitable.Changes(), ContainsAll, []git.ChangedFile{
					git.ChangedFile("nextid"),
					git.ChangedFile("active"),
				})
				c.Expect(commitable.CommitMsg(), Equals, "directory store initialized")

				// Initialize and empty repo
				c.Assume(git.Init(d), IsNil)
				// Commit the directory store initialization
				c.Expect(git.Commit(commitable), IsNil)

				o, err := git.Command(d, "show", "--no-color", "--pretty=format:\"%s%b\"").Output()
				c.Assume(err, IsNil)
				c.Expect(string(o), Equals,
					`"directory store initialized"
diff --git a/active b/active
new file mode 100644
index 0000000..e69de29
diff --git a/nextid b/nextid
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/nextid
@@ -0,0 +1 @@
+1
`)
			})
		})

		c.Specify("contains an index of the next available id", func() {
			_, d := makeDirectoryStore("directory_store_spec")

			data, err := ioutil.ReadFile(filepath.Join(d, "nextid"))
			c.Expect(err, IsNil)
			c.Expect(string(data), Equals, "1\n")
		})

		c.Specify("contains an index of active ideas", func() {
			_, d := makeDirectoryStore("directory_store_spec")

			_, err := os.Stat(filepath.Join(d, "active"))
			c.Expect(err, IsNil)
		})

		c.Specify("contains ideas stored in a files", func() {
			c.Specify("with the id as the filename", func() {
			})
		})

		c.Specify("can create a new idea", func() {
			id, d := makeDirectoryStore("directory_store_create")

			type newIdea struct {
				changes git.Commitable
				idea    *Idea
			}

			newIdeas := []*newIdea{{
				idea: &Idea{
					IS_Active,
					0,
					"A New Idea",
					"Some body text\n",
				},
			}, {
				idea: &Idea{
					IS_Inactive,
					0,
					"Another New Idea",
					"That isn't active\n",
				},
			}, {
				idea: &Idea{
					IS_Active,
					0,
					"Another Active New Idea",
					"That should be active\n",
				},
			}}

			activeIdeas := make([]*newIdea, 0, 2)

			for _, ni := range newIdeas {
				// Build a parallel slice of active ideas
				if ni.idea.Status == IS_Active {
					activeIdeas = append(activeIdeas, ni)
				}

				changes, err := id.SaveNewIdea(ni.idea)
				c.Assume(err, IsNil)
				c.Assume(changes, Not(IsNil))
				ni.changes = changes
			}

			c.Assume(len(activeIdeas), Equals, 2)

			c.Specify("by assigning the next available id to the idea", func() {
				c.Expect(newIdeas[0].idea.Id, Equals, uint(1))
				c.Expect(newIdeas[1].idea.Id, Equals, uint(2))
			})

			c.Specify("by incrementing the next available id", func() {
				data, err := ioutil.ReadFile(filepath.Join(d, "nextid"))
				c.Assume(err, IsNil)

				var nextId uint
				n, err := fmt.Fscan(bytes.NewReader(data), &nextId)
				c.Expect(err, IsNil)
				c.Expect(n, Equals, 1)
				c.Expect(nextId, Equals, uint(len(newIdeas)+1))

				c.Specify("and will return a commitable change for modifying the next available id", func() {
					for _, ni := range newIdeas {
						c.Expect(ni.changes.Changes(), Contains, git.ChangedFile("nextid"))
					}
				})
			})

			c.Specify("by writing the idea to a file", func() {
				pathTo := func(idea *Idea) string {
					return filepath.Join(d, fmt.Sprintf("%d", idea.Id))
				}

				c.Specify("with the id as the filename", func() {

					for _, ni := range newIdeas {
						_, err := os.Stat(pathTo(ni.idea))
						c.Expect(!os.IsNotExist(err), IsTrue)
					}
				})

				for _, ni := range newIdeas {
					actualData, err := ioutil.ReadFile(pathTo(ni.idea))
					c.Assume(err, IsNil)

					r, err := NewIdeaReader(*ni.idea)
					c.Assume(err, IsNil)
					expectedData, err := ioutil.ReadAll(r)
					c.Assume(err, IsNil)

					c.Expect(string(actualData), Equals, string(expectedData))
				}

				c.Specify("and return a commitable change for the new idea file", func() {
					for _, ni := range newIdeas {
						c.Expect(ni.changes.Changes(), Contains, git.ChangedFile(fmt.Sprint(ni.idea.Id)))
					}
				})
			})

			c.Specify("and if the idea's status is active", func() {
				c.Specify("will add the idea's id to the active index", func() {
					data, err := ioutil.ReadFile(filepath.Join(d, "active"))
					c.Assume(err, IsNil)

					actualActiveIds := make([]uint, 0, len(activeIdeas))
					scanner := bufio.NewScanner(bytes.NewReader(data))

					for scanner.Scan() {
						var id uint
						_, err := fmt.Fscan(bytes.NewReader(scanner.Bytes()), &id)
						c.Assume(err, IsNil)

						actualActiveIds = append(actualActiveIds, id)
					}

					expectedActiveIds := make([]uint, 0, len(activeIdeas))
					for _, ni := range activeIdeas {
						expectedActiveIds = append(expectedActiveIds, ni.idea.Id)
					}

					c.Expect(actualActiveIds, ContainsExactly, expectedActiveIds)

					c.Specify("and will return a commitable change for modifying the index", func() {
						for _, ni := range newIdeas {
							if ni.idea.Status == IS_Active {
								c.Expect(ni.changes.Changes(), Contains, git.ChangedFile("active"))
							} else {
								c.Expect(ni.changes.Changes(), Not(Contains), git.ChangedFile("active"))
							}
						}
					})
				})
			})

			c.Specify("and if the idea's status isn't active", func() {
				notActiveIdeas := make([]*Idea, 0, 1)
				for _, ni := range newIdeas {
					if ni.idea.Status != IS_Active {
						notActiveIdeas = append(notActiveIdeas, ni.idea)
					}
				}
				c.Assume(len(notActiveIdeas), Equals, 1)

				c.Specify("will not add the idea's id to the active index", func() {
					// Collect the id's from the index file
					data, err := ioutil.ReadFile(filepath.Join(d, "active"))
					c.Assume(err, IsNil)

					// Using a scanner because I don't know how many there are
					scanner := bufio.NewScanner(bytes.NewReader(data))
					activeIds := make([]uint, 0, len(newIdeas))

					for scanner.Scan() {
						var id uint
						_, err := fmt.Fscan(bytes.NewReader(scanner.Bytes()), &id)
						c.Assume(err, IsNil)
						activeIds = append(activeIds, id)
					}
					c.Assume(len(activeIds), Equals, 2)

					for _, idea := range notActiveIdeas {
						c.Expect(activeIds, Not(Contains), idea.Id)
					}
				})
			})

			c.Specify("and returns a commitable change", func() {
				for _, ni := range newIdeas {
					c.Expect(ni.changes.CommitMsg(), Equals, fmt.Sprintf("IDEA - %d - Created", ni.idea.Id))
				}
			})
		})

		c.Specify("can update an existing idea", func() {
			c.Specify("by writing the idea to the file", func() {
				c.Specify("with the id as the filename", func() {
				})

				c.Specify("and will return a commitable change for the modified idea file", func() {
				})
			})

			c.Specify("and if the idea's status is active", func() {
				c.Specify("will add the idea's id to the active index", func() {
					c.Specify("and will return a commitable change for modifying the index", func() {
					})
				})
			})

			c.Specify("and if the idea's status isn't active", func() {
				c.Specify("will not add the idea's id to the active index", func() {
				})
			})
		})
	})
}
<|endoftext|>


<commit_before>// Package configurator implements simple configuration built on top of Viper.
//
// It aims to make application configuration as easy as possible.
// This is accomplish by allowing you to annotate your struct with env, file,
// flag, default annotations that will tell Viper where to where to look for
// configuration values. Since configurator is built on top of Viper the
// same source precedence are applicable:
//
// The priority of config sources is the following:
// 1. Overrides, or setting the config struct field directly.\n
// 2. Flags - note that github.com/spf13/pflag is used
// 3. Environment Variables
// 4. Configuration file values
// 5. Default values
//
// NOTE: Viper key/value store and/or watching config sources is not yet supported.
//
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.
package configurator

import (
	"errors"
	"fmt"
	"go/ast"
	"reflect"
	"strconv"

	"github.com/spf13/pflag"
	"github.com/spf13/viper"
)

const (
	tagEnv     = "env"
	tagFlag    = "flag"
	tagFile    = "file"
	tagDefault = "default"
)

var (
	// ErrValueNotStruct is returned when value passed to config.Load() is not a struct.
	ErrValueNotStruct = errors.New("Value does not appear to be a struct!")

	// ErrValueNotStructPointer is returned when value passed to config.Load() is not a pointer to a struct.
	ErrValueNotStructPointer = errors.New("Value passed was not a struct pointer!")
)

var c *Config

func init() {
	c = &Config{
		FileName:  "config",
		FilePaths: []string{"."},
	}
}

// Config is a convenience configuration struct built on top of Viper. You
// use Config by annotating your configuration struct with env, flag, file,
// and default tags which will be parsed by Config. You can either
// embed Configurator.Config in your struct or reference configurator.Load()
// directly. The priority of the sources is the same Viper:
// 1. overrides
// 2. flags
// 3. env. variables
// 4. config file
// 5. defaults
//
// For example, if you embedded configurator.Config in your struct and
// configured it like so:
//
//  type AppConfig struct {
//	configurator.Config
//	Secret      string `file:"secret" env:"APP_SECRET" flag:"secret" default:"abc123xyz"`
//	User        string `file:"user" env:"APP_USER" flag:"user" default:"root"`
//	Environment string `file:"env" env:"APP_ENV" flag:"env" default:"dev"`
//   }
//
// Assuming your source values were the following:
//  File : {
// 	"user": "test_user"
//	"secret": "defaultsecret"
//  }
//  Env : {
//  	"APP_SECRET": "somesecretkey"
//  }
//
// This is how you would load the configuration:
//
//  func loadConfig() {
// 	config := AppConfig{}
// 	err := config.Load(&config)
//
// 	if err != nil {
// 		// Always handle your errors
// 		log.Fatalf("Unable to load application configuration! Error: %s", err.Error())
// 	}
//
// 	fmt.Println("config.Secret =", config.Secret)           // somesecretkey, from env
// 	fmt.Println("config.User =", config.User)               // test_user, from file
// 	fmt.Println("config.Environment =", config.Environment) // dev, from defaults
//  }
//
type Config struct {
	// FileName is the name of the configuration file without any extensions.
	FileName string

	// FilePaths is an array of configuration file paths to search for the configuration file.
	FilePaths []string

	externalConfig *interface{}
	viper          *viper.Viper
}

// Load attempts to populate the struct with configuration values.
// The value passed to load must be a struct reference or an error
// will be returned.
func Load(structRef interface{}) error {
	return c.Load(structRef)
}

// Load attempts to populate the struct with configuration values.
// The value passed to load must be a struct reference or an error
// will be returned.
func (c *Config) Load(structRef interface{}) error {
	c.viper = viper.New()

	canLoadErr := c.canLoad(structRef)
	if canLoadErr != nil {
		return canLoadErr
	}

	ptrRef := reflect.ValueOf(structRef)
	ref := ptrRef.Elem()

	return c.parseStructConfigValues(ref, structRef)
}

func (c *Config) canLoad(structRef interface{}) error {
	ptrRef := reflect.ValueOf(structRef)
	if ptrRef.Kind() != reflect.Ptr {
		return ErrValueNotStructPointer
	}
	elemRef := ptrRef.Elem()
	if elemRef.Kind() != reflect.Struct {
		return ErrValueNotStruct
	}

	return nil
}

/////////////
// Parsing //
/////////////

type parsedValue struct {
	tagValue  string
	fieldType reflect.Type
}

func (c *Config) parseStructConfigValues(structRef reflect.Value, val interface{}) error {
	// Parse configurator values on our struct
	defaultValues := parseDefaultValues(structRef)
	envValues := parseEnvValues(structRef)
	flagValues := parseFlagValues(structRef)
	configValues := parseConfigFileValues(structRef)

	c.populateDefaults(defaultValues)
	c.bindEnvValues(envValues)
	c.bindFlagValues(flagValues)
	c.bindConfigFileValues(configValues)

	err := c.populateConfigStruct(structRef)

	return err
}

func parseDefaultValues(structRef reflect.Value) map[string]parsedValue {
	values := parseValuesForTag(structRef, tagDefault)
	return values
}

func parseEnvValues(structRef reflect.Value) map[string]parsedValue {
	values := parseValuesForTag(structRef, tagEnv)
	return values
}

func parseFlagValues(structRef reflect.Value) map[string]parsedValue {
	values := parseValuesForTag(structRef, tagFlag)
	return values
}

func parseConfigFileValues(structRef reflect.Value) map[string]parsedValue {
	values := parseValuesForTag(structRef, tagFile)
	return values
}

func parseValuesForTag(structRef reflect.Value, tagName string) map[string]parsedValue {
	values := map[string]parsedValue{}

	structType := structRef.Type()
	for i := 0; i < structType.NumField(); i++ {
		structField := structType.Field(i)
		tag := structField.Tag
		tagValue := tag.Get(tagName)

		if tagValue != "" && ast.IsExported(structField.Name) {
			values[structField.Name] = parsedValue{tagValue, structField.Type}
		}
	}

	return values
}

/////////////
// Binding //
/////////////

func (c *Config) bindEnvValues(envValues map[string]parsedValue) {
	for k, v := range envValues {
		c.viper.BindEnv(k, v.tagValue)
	}
}

func (c *Config) bindFlagValues(flagValues map[string]parsedValue) *pflag.FlagSet {
	flagSet := pflag.NewFlagSet("configurator", pflag.PanicOnError)

	for k, v := range flagValues {
		pflag.String(v.tagValue, "", "")
		flag := pflag.Lookup(v.tagValue)

		c.viper.BindPFlag(k, flag)
		flagSet.AddFlag(flag)
	}

	return flagSet
}

func (c *Config) bindConfigFileValues(configValues map[string]parsedValue) {
	c.viper.SetConfigName(c.FileName)

	for _, filePath := range c.FilePaths {
		c.viper.AddConfigPath(filePath)
	}

	// Map the config file keys to our variable
	for k, v := range configValues {
		c.viper.RegisterAlias(k, v.tagValue)
	}
}

//////////////
// Populate //
//////////////

func (c *Config) populateDefaults(defaultValues map[string]parsedValue) {
	for k, v := range defaultValues {
		c.viper.SetDefault(k, v.tagValue)
	}
}

func (c *Config) populateConfigStruct(structRef reflect.Value) error {
	c.viper.ReadInConfig()

	structType := structRef.Type()
	for i := 0; i < structType.NumField(); i++ {
		structField := structType.Field(i)
		configValue := c.viper.Get(structField.Name)
		if configValue != nil {
			err := populateStructField(structField, structRef.Field(i), fmt.Sprintf("%v", configValue))

			if err != nil {
				return err
			}
		}
	}

	return nil
}

func populateStructField(field reflect.StructField, fieldValue reflect.Value, value string) error {
	switch fieldValue.Kind() {
	case reflect.String:
		if isZeroOfUnderlyingType(fieldValue.Interface()) {
			fieldValue.SetString(value)
		}

	case reflect.Bool:
		bvalue, err := strconv.ParseBool(value)
		if err != nil {
			return fmt.Errorf("Unable to convert value (%s) for to bool for field: %s! Error: %s", value, field.Name, err.Error())
		}

		if isZeroOfUnderlyingType(fieldValue.Interface()) {
			fieldValue.SetBool(bvalue)
		}

	case reflect.Float32, reflect.Float64:
		floatValue, err := strconv.ParseFloat(value, 64)
		if err != nil {
			return fmt.Errorf("Unable to convert value (%s) for to float for field: %s! Error: %s", value, field.Name, err.Error())
		}

		if isZeroOfUnderlyingType(fieldValue.Interface()) {
			fieldValue.SetFloat(floatValue)
		}

	case reflect.Int, reflect.Int8, reflect.Int32, reflect.Int64:
		intValue, err := strconv.ParseInt(value, 10, 64)
		if err != nil {
			return fmt.Errorf("Unable to convert value (%s) for to int for field: %s! Error: %s", value, field.Name, err.Error())
		}

		if isZeroOfUnderlyingType(fieldValue.Interface()) {
			fieldValue.SetInt(intValue)
		}
	case reflect.Uint, reflect.Uint8, reflect.Uint32, reflect.Uint64:
		intValue, err := strconv.ParseUint(value, 10, 64)
		if err != nil {
			return fmt.Errorf("Unable to convert value (%s) for to unsigned int for field: %s! Error: %s", value, field.Name, err.Error())
		}

		if isZeroOfUnderlyingType(fieldValue.Interface()) {
			fieldValue.SetUint(intValue)
		}
	}
	return nil
}

/////////////
// Utility //
/////////////

func isZeroOfUnderlyingType(x interface{}) bool {
	// Source: http://stackoverflow.com/questions/13901819/quick-way-to-detect-empty-values-via-reflection-in-go
	return x == reflect.Zero(reflect.TypeOf(x)).Interface()
}
<commit_msg>Tweaked documentation<commit_after>// Package configurator implements simple configuration built on top of Viper.
//
// It aims to make application configuration as easy as possible.
// This is accomplish by allowing you to annotate your struct with env, file,
// flag, default annotations that will tell Viper where to where to look for
// configuration values. Since configurator is built on top of Viper the
// same source precedence is applicable.
//
// The priority of config sources is the following:
// 1. Overrides, or setting the config struct field directly.
// 2. Flags - note that github.com/spf13/pflag is used  in lieu of the standard flag package.
// 3. Environment variables.
// 4. Configuration file values.
// 5. Default values.
//
// NOTE: Viper key/value store and/or watching config sources is not yet supported.
//
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.
package configurator

import (
	"errors"
	"fmt"
	"go/ast"
	"reflect"
	"strconv"

	"github.com/spf13/pflag"
	"github.com/spf13/viper"
)

const (
	tagEnv     = "env"
	tagFlag    = "flag"
	tagFile    = "file"
	tagDefault = "default"
)

var (
	// ErrValueNotStruct is returned when value passed to config.Load() is not a struct.
	ErrValueNotStruct = errors.New("Value does not appear to be a struct!")

	// ErrValueNotStructPointer is returned when value passed to config.Load() is not a pointer to a struct.
	ErrValueNotStructPointer = errors.New("Value passed was not a struct pointer!")
)

var c *Config

func init() {
	c = &Config{
		FileName:  "config",
		FilePaths: []string{"."},
	}
}

// Config is a convenience configuration struct built on top of Viper. You
// use Config by annotating your configuration struct with env, flag, file,
// and default tags which will be parsed by Config. You can either
// embed Configurator.Config in your struct or reference configurator.Load()
// directly. The priority of the sources is the same Viper:
// 1. overrides
// 2. flags
// 3. env. variables
// 4. config file
// 5. defaults
//
// For example, if you embedded configurator.Config in your struct and
// configured it like so:
//
//  type AppConfig struct {
//	configurator.Config
//	Secret      string `file:"secret" env:"APP_SECRET" flag:"secret" default:"abc123xyz"`
//	User        string `file:"user" env:"APP_USER" flag:"user" default:"root"`
//	Environment string `file:"env" env:"APP_ENV" flag:"env" default:"dev"`
//   }
//
// Assuming your source values were the following:
//  File : {
// 	"user": "test_user"
//	"secret": "defaultsecret"
//  }
//  Env : {
//  	"APP_SECRET": "somesecretkey"
//  }
//
// This is how you would load the configuration:
//
//  func loadConfig() {
// 	config := AppConfig{}
// 	err := config.Load(&config)
//
// 	if err != nil {
// 		// Always handle your errors
// 		log.Fatalf("Unable to load application configuration! Error: %s", err.Error())
// 	}
//
// 	fmt.Println("config.Secret =", config.Secret)           // somesecretkey, from env
// 	fmt.Println("config.User =", config.User)               // test_user, from file
// 	fmt.Println("config.Environment =", config.Environment) // dev, from defaults
//  }
//
type Config struct {
	// FileName is the name of the configuration file without any extensions.
	FileName string

	// FilePaths is an array of configuration file paths to search for the configuration file.
	FilePaths []string

	externalConfig *interface{}
	viper          *viper.Viper
}

// Load attempts to populate the struct with configuration values.
// The value passed to load must be a struct reference or an error
// will be returned.
func Load(structRef interface{}) error {
	return c.Load(structRef)
}

// Load attempts to populate the struct with configuration values.
// The value passed to load must be a struct reference or an error
// will be returned.
func (c *Config) Load(structRef interface{}) error {
	c.viper = viper.New()

	canLoadErr := c.canLoad(structRef)
	if canLoadErr != nil {
		return canLoadErr
	}

	ptrRef := reflect.ValueOf(structRef)
	ref := ptrRef.Elem()

	return c.parseStructConfigValues(ref, structRef)
}

func (c *Config) canLoad(structRef interface{}) error {
	ptrRef := reflect.ValueOf(structRef)
	if ptrRef.Kind() != reflect.Ptr {
		return ErrValueNotStructPointer
	}
	elemRef := ptrRef.Elem()
	if elemRef.Kind() != reflect.Struct {
		return ErrValueNotStruct
	}

	return nil
}

/////////////
// Parsing //
/////////////

type parsedValue struct {
	tagValue  string
	fieldType reflect.Type
}

func (c *Config) parseStructConfigValues(structRef reflect.Value, val interface{}) error {
	// Parse configurator values on our struct
	defaultValues := parseDefaultValues(structRef)
	envValues := parseEnvValues(structRef)
	flagValues := parseFlagValues(structRef)
	configValues := parseConfigFileValues(structRef)

	c.populateDefaults(defaultValues)
	c.bindEnvValues(envValues)
	c.bindFlagValues(flagValues)
	c.bindConfigFileValues(configValues)

	err := c.populateConfigStruct(structRef)

	return err
}

func parseDefaultValues(structRef reflect.Value) map[string]parsedValue {
	values := parseValuesForTag(structRef, tagDefault)
	return values
}

func parseEnvValues(structRef reflect.Value) map[string]parsedValue {
	values := parseValuesForTag(structRef, tagEnv)
	return values
}

func parseFlagValues(structRef reflect.Value) map[string]parsedValue {
	values := parseValuesForTag(structRef, tagFlag)
	return values
}

func parseConfigFileValues(structRef reflect.Value) map[string]parsedValue {
	values := parseValuesForTag(structRef, tagFile)
	return values
}

func parseValuesForTag(structRef reflect.Value, tagName string) map[string]parsedValue {
	values := map[string]parsedValue{}

	structType := structRef.Type()
	for i := 0; i < structType.NumField(); i++ {
		structField := structType.Field(i)
		tag := structField.Tag
		tagValue := tag.Get(tagName)

		if tagValue != "" && ast.IsExported(structField.Name) {
			values[structField.Name] = parsedValue{tagValue, structField.Type}
		}
	}

	return values
}

/////////////
// Binding //
/////////////

func (c *Config) bindEnvValues(envValues map[string]parsedValue) {
	for k, v := range envValues {
		c.viper.BindEnv(k, v.tagValue)
	}
}

func (c *Config) bindFlagValues(flagValues map[string]parsedValue) *pflag.FlagSet {
	flagSet := pflag.NewFlagSet("configurator", pflag.PanicOnError)

	for k, v := range flagValues {
		pflag.String(v.tagValue, "", "")
		flag := pflag.Lookup(v.tagValue)

		c.viper.BindPFlag(k, flag)
		flagSet.AddFlag(flag)
	}

	return flagSet
}

func (c *Config) bindConfigFileValues(configValues map[string]parsedValue) {
	c.viper.SetConfigName(c.FileName)

	for _, filePath := range c.FilePaths {
		c.viper.AddConfigPath(filePath)
	}

	// Map the config file keys to our variable
	for k, v := range configValues {
		c.viper.RegisterAlias(k, v.tagValue)
	}
}

//////////////
// Populate //
//////////////

func (c *Config) populateDefaults(defaultValues map[string]parsedValue) {
	for k, v := range defaultValues {
		c.viper.SetDefault(k, v.tagValue)
	}
}

func (c *Config) populateConfigStruct(structRef reflect.Value) error {
	c.viper.ReadInConfig()

	structType := structRef.Type()
	for i := 0; i < structType.NumField(); i++ {
		structField := structType.Field(i)
		configValue := c.viper.Get(structField.Name)
		if configValue != nil {
			err := populateStructField(structField, structRef.Field(i), fmt.Sprintf("%v", configValue))

			if err != nil {
				return err
			}
		}
	}

	return nil
}

func populateStructField(field reflect.StructField, fieldValue reflect.Value, value string) error {
	switch fieldValue.Kind() {
	case reflect.String:
		if isZeroOfUnderlyingType(fieldValue.Interface()) {
			fieldValue.SetString(value)
		}

	case reflect.Bool:
		bvalue, err := strconv.ParseBool(value)
		if err != nil {
			return fmt.Errorf("Unable to convert value (%s) for to bool for field: %s! Error: %s", value, field.Name, err.Error())
		}

		if isZeroOfUnderlyingType(fieldValue.Interface()) {
			fieldValue.SetBool(bvalue)
		}

	case reflect.Float32, reflect.Float64:
		floatValue, err := strconv.ParseFloat(value, 64)
		if err != nil {
			return fmt.Errorf("Unable to convert value (%s) for to float for field: %s! Error: %s", value, field.Name, err.Error())
		}

		if isZeroOfUnderlyingType(fieldValue.Interface()) {
			fieldValue.SetFloat(floatValue)
		}

	case reflect.Int, reflect.Int8, reflect.Int32, reflect.Int64:
		intValue, err := strconv.ParseInt(value, 10, 64)
		if err != nil {
			return fmt.Errorf("Unable to convert value (%s) for to int for field: %s! Error: %s", value, field.Name, err.Error())
		}

		if isZeroOfUnderlyingType(fieldValue.Interface()) {
			fieldValue.SetInt(intValue)
		}
	case reflect.Uint, reflect.Uint8, reflect.Uint32, reflect.Uint64:
		intValue, err := strconv.ParseUint(value, 10, 64)
		if err != nil {
			return fmt.Errorf("Unable to convert value (%s) for to unsigned int for field: %s! Error: %s", value, field.Name, err.Error())
		}

		if isZeroOfUnderlyingType(fieldValue.Interface()) {
			fieldValue.SetUint(intValue)
		}
	}
	return nil
}

/////////////
// Utility //
/////////////

func isZeroOfUnderlyingType(x interface{}) bool {
	// Source: http://stackoverflow.com/questions/13901819/quick-way-to-detect-empty-values-via-reflection-in-go
	return x == reflect.Zero(reflect.TypeOf(x)).Interface()
}
<|endoftext|>