<commit_before>package pagerank

import (
	"fmt"
	"math"
	"math/rand"
	"runtime"
	"testing"
)

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())
}

func round(f float64) float64 {
	return math.Floor(f*10+0.5) / 10
}

func toPercentage(f float64) float64 {
	tenPow3 := math.Pow(10, 3)
	return round(100 * (f * tenPow3) / tenPow3)
}

func assertRank(t *testing.T, pageRank Interface, expected map[int]float64) {
	const tolerance = 0.0001
	pageRank.Rank(0.85, tolerance, func(label int, rank float64) {
		rankAsPercentage := toPercentage(rank)
		if math.Abs(rankAsPercentage - expected[label]) > tolerance {
			t.Error("Rank for", label, "should be", expected[label], "but was", rankAsPercentage)
		}
	})
}

func assertEqual(t *testing.T, actual, expected interface{}) {
	if actual != expected {
		t.Error("Should be", expected, "but was", actual)
	}
}

func assert(t *testing.T, actual bool) {
	if !actual {
		t.Error("Should be true")
	}
}

func TestRound(t *testing.T) {
	assertEqual(t, round(0.6666666), 0.7)
}

func TestRankToPercentage(t *testing.T) {
	assertEqual(t, toPercentage(0.6666666), 66.7)
}

func TestShouldEnterTheBlock(t *testing.T) {
	pageRank := New()
	pageRank.Link(0, 1)

	entered := false
	pageRank.Rank(0.85, 0.0001, func(_ int, _ float64) {
		entered = true
	})

	assert(t, entered)
}

func TestShouldBePossibleToRecalculateTheRanksAfterANewLinkIsAdded(t *testing.T) {
	pageRank := New()
	pageRank.Link(0, 1)
	assertRank(t, pageRank, map[int]float64{0: 35.1, 1: 64.9})
	pageRank.Link(1, 2)
	assertRank(t, pageRank, map[int]float64{0: 18.4, 1: 34.1, 2: 47.4})
}

func TestShouldBePossibleToClearTheGraph(t *testing.T) {
	pageRank := New()
	pageRank.Link(0, 1)
	pageRank.Link(1, 2)
	pageRank.Clear()
	pageRank.Link(0, 1)
	assertRank(t, pageRank, map[int]float64{0: 35.1, 1: 64.9})
}

func TestShouldNotFailWhenCalculatingTheRankOfAnEmptyGraph(t *testing.T) {
	pageRank := New()
	pageRank.Rank(0.85, 0.0001, func(label int, rank float64) {
		t.Error("This should not be seen")
	})
}

func TestShouldReturnCorrectResultsWhenHavingADanglingNode(t *testing.T) {
	pageRank := New()
	//node 2 is a dangling node because it has no outbound links
	pageRank.Link(0, 2)
	pageRank.Link(1, 2)

	expectedRank := map[int]float64{
		0: 21.3,
		1: 21.3,
		2: 57.4,
	}

	assertRank(t, pageRank, expectedRank)
}

func TestShouldNotChangeTheGraphWhenAddingTheSameLinkManyTimes(t *testing.T) {
	pageRank := New()
	pageRank.Link(0, 2)
	pageRank.Link(0, 2)
	pageRank.Link(0, 2)
	pageRank.Link(1, 2)
	pageRank.Link(1, 2)

	expectedRank := map[int]float64{
		0: 21.3,
		1: 21.3,
		2: 57.4,
	}

	assertRank(t, pageRank, expectedRank)
}

func TestShouldReturnCorrectResultsForAStarGraph(t *testing.T) {
	pageRank := New()
	pageRank.Link(0, 2)
	pageRank.Link(1, 2)
	pageRank.Link(2, 2)

	expectedRank := map[int]float64{
		0: 5,
		1: 5,
		2: 90,
	}

	assertRank(t, pageRank, expectedRank)
}

func TestShouldBeUniformForACircularGraph(t *testing.T) {
	pageRank := New()
	pageRank.Link(0, 1)
	pageRank.Link(1, 2)
	pageRank.Link(2, 3)
	pageRank.Link(3, 4)
	pageRank.Link(4, 0)

	expectedRank := map[int]float64{
		0: 20,
		1: 20,
		2: 20,
		3: 20,
		4: 20,
	}

	assertRank(t, pageRank, expectedRank)
}

func TestShouldReturnCorrectResultsForAConvergingGraph(t *testing.T) {
	pageRank := New()
	pageRank.Link(0, 1)
	pageRank.Link(0, 2)
	pageRank.Link(1, 2)
	pageRank.Link(2, 2)

	expectedRank := map[int]float64{
		0: 5,
		1: 7.1,
		2: 87.9,
	}

	assertRank(t, pageRank, expectedRank)
}

func TestShouldCorrectlyReproduceTheWikipediaExample(t *testing.T) {
	//http://en.wikipedia.org/wiki/File:PageRanks-Example.svg
	pageRank := New()
	pageRank.Link(1, 2)
	pageRank.Link(2, 1)
	pageRank.Link(3, 0)
	pageRank.Link(3, 1)
	pageRank.Link(4, 3)
	pageRank.Link(4, 1)
	pageRank.Link(4, 5)
	pageRank.Link(5, 4)
	pageRank.Link(5, 1)
	pageRank.Link(6, 1)
	pageRank.Link(6, 4)
	pageRank.Link(7, 1)
	pageRank.Link(7, 4)
	pageRank.Link(8, 1)
	pageRank.Link(8, 4)
	pageRank.Link(9, 4)
	pageRank.Link(10, 4)

	expectedRank := map[int]float64{
		0:  3.3,  //a
		1:  38.4, //b
		2:  34.3, //c
		3:  3.9,  //d
		4:  8.1,  //e
		5:  3.9,  //f
		6:  1.6,  //g
		7:  1.6,  //h
		8:  1.6,  //i
		9:  1.6,  //j
		10: 1.6,  //k
	}

	assertRank(t, pageRank, expectedRank)
}

func BenchmarkOneMillion(b *testing.B) {
	n := 1000000

	pageRank := New()

	rand.Seed(5)

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		for from := 0; from < n; from++ {
			for j := 0; j < rand.Intn(60); j++ {
				too := rand.Intn(n)

				to := too
				if too > 800000 {
					to = rand.Intn(3)
				}

				pageRank.Link(from, to)
			}
		}
	}

	result := make([]float64, n)
	pageRank.Rank(0.85, 0.001, func(key int, val float64) {
		result[key] = val
	})

	fmt.Println("5 first values are", result[0], ",", result[1], ",", result[2], ",", result[3], ",", result[4])
	pageRank.Clear()
}
<commit_msg>More links for the benchmark graph<commit_after>package pagerank

import (
	"fmt"
	"math"
	"math/rand"
	"runtime"
	"testing"
)

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())
}

func round(f float64) float64 {
	return math.Floor(f*10+0.5) / 10
}

func toPercentage(f float64) float64 {
	tenPow3 := math.Pow(10, 3)
	return round(100 * (f * tenPow3) / tenPow3)
}

func assertRank(t *testing.T, pageRank Interface, expected map[int]float64) {
	const tolerance = 0.0001
	pageRank.Rank(0.85, tolerance, func(label int, rank float64) {
		rankAsPercentage := toPercentage(rank)
		if math.Abs(rankAsPercentage - expected[label]) > tolerance {
			t.Error("Rank for", label, "should be", expected[label], "but was", rankAsPercentage)
		}
	})
}

func assertEqual(t *testing.T, actual, expected interface{}) {
	if actual != expected {
		t.Error("Should be", expected, "but was", actual)
	}
}

func assert(t *testing.T, actual bool) {
	if !actual {
		t.Error("Should be true")
	}
}

func TestRound(t *testing.T) {
	assertEqual(t, round(0.6666666), 0.7)
}

func TestRankToPercentage(t *testing.T) {
	assertEqual(t, toPercentage(0.6666666), 66.7)
}

func TestShouldEnterTheBlock(t *testing.T) {
	pageRank := New()
	pageRank.Link(0, 1)

	entered := false
	pageRank.Rank(0.85, 0.0001, func(_ int, _ float64) {
		entered = true
	})

	assert(t, entered)
}

func TestShouldBePossibleToRecalculateTheRanksAfterANewLinkIsAdded(t *testing.T) {
	pageRank := New()
	pageRank.Link(0, 1)
	assertRank(t, pageRank, map[int]float64{0: 35.1, 1: 64.9})
	pageRank.Link(1, 2)
	assertRank(t, pageRank, map[int]float64{0: 18.4, 1: 34.1, 2: 47.4})
}

func TestShouldBePossibleToClearTheGraph(t *testing.T) {
	pageRank := New()
	pageRank.Link(0, 1)
	pageRank.Link(1, 2)
	pageRank.Clear()
	pageRank.Link(0, 1)
	assertRank(t, pageRank, map[int]float64{0: 35.1, 1: 64.9})
}

func TestShouldNotFailWhenCalculatingTheRankOfAnEmptyGraph(t *testing.T) {
	pageRank := New()
	pageRank.Rank(0.85, 0.0001, func(label int, rank float64) {
		t.Error("This should not be seen")
	})
}

func TestShouldReturnCorrectResultsWhenHavingADanglingNode(t *testing.T) {
	pageRank := New()
	//node 2 is a dangling node because it has no outbound links
	pageRank.Link(0, 2)
	pageRank.Link(1, 2)

	expectedRank := map[int]float64{
		0: 21.3,
		1: 21.3,
		2: 57.4,
	}

	assertRank(t, pageRank, expectedRank)
}

func TestShouldNotChangeTheGraphWhenAddingTheSameLinkManyTimes(t *testing.T) {
	pageRank := New()
	pageRank.Link(0, 2)
	pageRank.Link(0, 2)
	pageRank.Link(0, 2)
	pageRank.Link(1, 2)
	pageRank.Link(1, 2)

	expectedRank := map[int]float64{
		0: 21.3,
		1: 21.3,
		2: 57.4,
	}

	assertRank(t, pageRank, expectedRank)
}

func TestShouldReturnCorrectResultsForAStarGraph(t *testing.T) {
	pageRank := New()
	pageRank.Link(0, 2)
	pageRank.Link(1, 2)
	pageRank.Link(2, 2)

	expectedRank := map[int]float64{
		0: 5,
		1: 5,
		2: 90,
	}

	assertRank(t, pageRank, expectedRank)
}

func TestShouldBeUniformForACircularGraph(t *testing.T) {
	pageRank := New()
	pageRank.Link(0, 1)
	pageRank.Link(1, 2)
	pageRank.Link(2, 3)
	pageRank.Link(3, 4)
	pageRank.Link(4, 0)

	expectedRank := map[int]float64{
		0: 20,
		1: 20,
		2: 20,
		3: 20,
		4: 20,
	}

	assertRank(t, pageRank, expectedRank)
}

func TestShouldReturnCorrectResultsForAConvergingGraph(t *testing.T) {
	pageRank := New()
	pageRank.Link(0, 1)
	pageRank.Link(0, 2)
	pageRank.Link(1, 2)
	pageRank.Link(2, 2)

	expectedRank := map[int]float64{
		0: 5,
		1: 7.1,
		2: 87.9,
	}

	assertRank(t, pageRank, expectedRank)
}

func TestShouldCorrectlyReproduceTheWikipediaExample(t *testing.T) {
	//http://en.wikipedia.org/wiki/File:PageRanks-Example.svg
	pageRank := New()
	pageRank.Link(1, 2)
	pageRank.Link(2, 1)
	pageRank.Link(3, 0)
	pageRank.Link(3, 1)
	pageRank.Link(4, 3)
	pageRank.Link(4, 1)
	pageRank.Link(4, 5)
	pageRank.Link(5, 4)
	pageRank.Link(5, 1)
	pageRank.Link(6, 1)
	pageRank.Link(6, 4)
	pageRank.Link(7, 1)
	pageRank.Link(7, 4)
	pageRank.Link(8, 1)
	pageRank.Link(8, 4)
	pageRank.Link(9, 4)
	pageRank.Link(10, 4)

	expectedRank := map[int]float64{
		0:  3.3,  //a
		1:  38.4, //b
		2:  34.3, //c
		3:  3.9,  //d
		4:  8.1,  //e
		5:  3.9,  //f
		6:  1.6,  //g
		7:  1.6,  //h
		8:  1.6,  //i
		9:  1.6,  //j
		10: 1.6,  //k
	}

	assertRank(t, pageRank, expectedRank)
}

func BenchmarkOneMillion(b *testing.B) {
	n := 1000000

	pageRank := New()

	rand.Seed(5)

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		for from := 0; from < n; from++ {
			for j := 0; j < rand.Intn(400); j++ {
				too := rand.Intn(n)

				to := too
				if too > 800000 {
					to = rand.Intn(3)
				}

				pageRank.Link(from, to)
			}
		}
	}

	result := make([]float64, n)
	pageRank.Rank(0.85, 0.001, func(key int, val float64) {
		result[key] = val
	})

	fmt.Println("5 first values are", result[0], ",", result[1], ",", result[2], ",", result[3], ",", result[4])
	pageRank.Clear()
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"
)

var (
	addr       = flag.String("http", ":8080", "HTTP address to serve on")
	tlsAddr    = flag.String("https", "", "HTTPS address to serve on")
	pemFile    = flag.String("pem-file", "", "HTTPS / SSL certificate .pem file")
	keyFile    = flag.String("key-file", "", "HTTPS / SSL certificate .key file")
	update     = flag.Bool("update", true, "update via Git and shutdown server after pull")
	updateRate = flag.String("update-rate", "60s", "rate at which to check for updates via Git")
	dev        = flag.Bool("dev", false, "reload all templates on each request")

	staticDirPrefix = "/static/" // Path prefix to strip to get static dir root.
	staticDir       = "static/"  // Directory to serve for static files.

	errorTemplate = "error"      // Template to use for errors.
	templateGlob  = "**/*.tmpl"  // Glob to match all template files.
	templateDir   = "templates/" // Relative directory that templates reside in.

	src = &GitUpdater{
		Dir: ".",
	}
	lastUpdate  = time.Now()
	tmpls       *template.Template
	updateRateT time.Duration
)

func reloadTemplates() error {
	matches, err := filepath.Glob(templateGlob)
	if err != nil {
		return err
	}
	tmpls = template.New("")
	for _, m := range matches {
		name := strings.TrimPrefix(m, templateDir)
		name = strings.TrimSuffix(name, filepath.Ext(name))
		if _, err := tmpls.New(name).Parse(m); err != nil {
			return err
		}
	}
	return nil
}

type TemplateData struct {
	Request *http.Request
	Error   string
}

func handler(w http.ResponseWriter, r *http.Request) error {
	if *dev {
		if err := reloadTemplates(); err != nil {
			return err
		}
	}

	var data = &TemplateData{
		Request: r,
	}

	// Determine template name.
	tmplName := r.URL.Path[1:]
	if stat, err := os.Stat(filepath.Join(templateDir, tmplName)); err != nil {
		return err
	} else {
		if stat.IsDir() {
			tmplName = "index"
		}
	}

	// Find the requested page.
	root := tmpls.Lookup(tmplName)
	if root == nil {
		// Couldn't find the page, render the error page.
		w.WriteHeader(http.StatusNotFound)
		return fmt.Errorf("%v - %v", http.StatusNotFound, http.StatusText(http.StatusNotFound))
	}

	// Execute the requested template page.
	return root.Execute(w, data)
}

func logHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		log.Printf("%v %v\n", r.Method, r.URL)
		h.ServeHTTP(w, r)
	})
}

func errorHandler(h func(w http.ResponseWriter, r *http.Request) error) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		err := h(w, r)
		if err == nil {
			log.Printf("%v %v\n", r.Method, r.URL)
			return
		}
		log.Printf("[error] %v %v - %v\n", r.Method, r.URL, err)

		// Render the error template.
		data := &TemplateData{
			Request: r,
			Error:   err.Error(),
		}
		if err := tmpls.ExecuteTemplate(w, errorTemplate, data); err != nil {
			panic(err)
		}
	})
}

func checkForUpdates() {
	// If not enough time has elapsed, skip the update.
	if time.Since(lastUpdate) < updateRateT {
		return
	}
	lastUpdate = time.Now()

	// Check for updates.
	updated, err := src.Update()
	if err != nil {
		log.Println("Update error:", err)
		return
	}
	if updated {
		log.Println("Updated source code. Exiting server..")
		os.Exit(0)
	}
	log.Println("checked for updates (none available)")
}

func init() {
	if err := reloadTemplates(); err != nil {
		log.Fatal(err)
	}

	if *update {
		go func() {
			for {
				time.Sleep(updateRateT)
				checkForUpdates()
			}
		}()
	}
}

func main() {
	flag.Parse()

	// Parse update rate flag.
	var err error
	updateRateT, err = time.ParseDuration(*updateRate)
	if err != nil {
		log.Fatal(err)
	}

	// Static file hosting
	http.Handle("/static/", logHandler(http.StripPrefix(staticDirPrefix, http.FileServer(http.Dir(staticDir)))))

	// App handler.
	http.Handle("/", errorHandler(handler))

	// Start HTTPS server:
	if *tlsAddr != "" {
		go func() {
			if *pemFile == "" {
				log.Fatal("expected -pem-file flag")
			}
			if *keyFile == "" {
				log.Fatal("expected -key-file flag")
			}
			log.Println("Serving on", *tlsAddr)
			log.Fatal(http.ListenAndServeTLS(*tlsAddr, *pemFile, *keyFile, nil))
		}()
	}

	// Start HTTP server:
	log.Println("Serving on", *addr)
	log.Fatal(http.ListenAndServe(*addr, nil))
}
<commit_msg>cmd/fiz3d-org: fix template name resolution for index files in subdirectories<commit_after>package main

import (
	"flag"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"
)

var (
	addr       = flag.String("http", ":8080", "HTTP address to serve on")
	tlsAddr    = flag.String("https", "", "HTTPS address to serve on")
	pemFile    = flag.String("pem-file", "", "HTTPS / SSL certificate .pem file")
	keyFile    = flag.String("key-file", "", "HTTPS / SSL certificate .key file")
	update     = flag.Bool("update", true, "update via Git and shutdown server after pull")
	updateRate = flag.String("update-rate", "60s", "rate at which to check for updates via Git")
	dev        = flag.Bool("dev", false, "reload all templates on each request")

	staticDirPrefix = "/static/" // Path prefix to strip to get static dir root.
	staticDir       = "static/"  // Directory to serve for static files.

	errorTemplate = "error"      // Template to use for errors.
	templateGlob  = "**/*.tmpl"  // Glob to match all template files.
	templateDir   = "templates/" // Relative directory that templates reside in.

	src = &GitUpdater{
		Dir: ".",
	}
	lastUpdate  = time.Now()
	tmpls       *template.Template
	updateRateT time.Duration
)

func reloadTemplates() error {
	matches, err := filepath.Glob(templateGlob)
	if err != nil {
		return err
	}
	tmpls = template.New("")
	for _, m := range matches {
		name := strings.TrimPrefix(m, templateDir)
		name = strings.TrimSuffix(name, filepath.Ext(name))
		if _, err := tmpls.New(name).Parse(m); err != nil {
			return err
		}
	}
	return nil
}

type TemplateData struct {
	Request *http.Request
	Error   string
}

func handler(w http.ResponseWriter, r *http.Request) error {
	if *dev {
		if err := reloadTemplates(); err != nil {
			return err
		}
	}

	var data = &TemplateData{
		Request: r,
	}

	// Determine template name.
	tmplName := r.URL.Path[1:]
	if stat, err := os.Stat(filepath.Join(templateDir, tmplName)); err != nil {
		return err
	} else {
		if stat.IsDir() {
			tmplName = path.Join(tmplName, "index")
		}
	}

	// Find the requested page.
	root := tmpls.Lookup(tmplName)
	if root == nil {
		// Couldn't find the page, render the error page.
		w.WriteHeader(http.StatusNotFound)
		return fmt.Errorf("%v - %v", http.StatusNotFound, http.StatusText(http.StatusNotFound))
	}

	// Execute the requested template page.
	return root.Execute(w, data)
}

func logHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		log.Printf("%v %v\n", r.Method, r.URL)
		h.ServeHTTP(w, r)
	})
}

func errorHandler(h func(w http.ResponseWriter, r *http.Request) error) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		err := h(w, r)
		if err == nil {
			log.Printf("%v %v\n", r.Method, r.URL)
			return
		}
		log.Printf("[error] %v %v - %v\n", r.Method, r.URL, err)

		// Render the error template.
		data := &TemplateData{
			Request: r,
			Error:   err.Error(),
		}
		if err := tmpls.ExecuteTemplate(w, errorTemplate, data); err != nil {
			panic(err)
		}
	})
}

func checkForUpdates() {
	// If not enough time has elapsed, skip the update.
	if time.Since(lastUpdate) < updateRateT {
		return
	}
	lastUpdate = time.Now()

	// Check for updates.
	updated, err := src.Update()
	if err != nil {
		log.Println("Update error:", err)
		return
	}
	if updated {
		log.Println("Updated source code. Exiting server..")
		os.Exit(0)
	}
	log.Println("checked for updates (none available)")
}

func init() {
	if err := reloadTemplates(); err != nil {
		log.Fatal(err)
	}

	if *update {
		go func() {
			for {
				time.Sleep(updateRateT)
				checkForUpdates()
			}
		}()
	}
}

func main() {
	flag.Parse()

	// Parse update rate flag.
	var err error
	updateRateT, err = time.ParseDuration(*updateRate)
	if err != nil {
		log.Fatal(err)
	}

	// Static file hosting
	http.Handle("/static/", logHandler(http.StripPrefix(staticDirPrefix, http.FileServer(http.Dir(staticDir)))))

	// App handler.
	http.Handle("/", errorHandler(handler))

	// Start HTTPS server:
	if *tlsAddr != "" {
		go func() {
			if *pemFile == "" {
				log.Fatal("expected -pem-file flag")
			}
			if *keyFile == "" {
				log.Fatal("expected -key-file flag")
			}
			log.Println("Serving on", *tlsAddr)
			log.Fatal(http.ListenAndServeTLS(*tlsAddr, *pemFile, *keyFile, nil))
		}()
	}

	// Start HTTP server:
	log.Println("Serving on", *addr)
	log.Fatal(http.ListenAndServe(*addr, nil))
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"launchpad.net/gnuflag"
	"launchpad.net/juju-core/juju/cmd"
	"launchpad.net/juju-core/juju/log"
	"launchpad.net/tomb"
	"launchpad.net/juju-core/juju/container"
	"launchpad.net/juju-core/juju/state"
)

// MachineAgent is a cmd.Command responsible for running a machine agent.
type MachineAgent struct {
	Conf      AgentConf
	MachineId int
}

// Info returns usage information for the command.
func (a *MachineAgent) Info() *cmd.Info {
	return &cmd.Info{"machine", "", "run a juju machine agent", ""}
}

// Init initializes the command for running.
func (a *MachineAgent) Init(f *gnuflag.FlagSet, args []string) error {
	a.Conf.addFlags(f)
	f.IntVar(&a.MachineId, "machine-id", -1, "id of the machine to run")
	if err := f.Parse(true, args); err != nil {
		return err
	}
	if a.MachineId < 0 {
		return fmt.Errorf("--machine-id option must be set, and expects a non-negative integer")
	}
	return a.Conf.checkArgs(f.Args())
}

// Run runs a machine agent.
func (a *MachineAgent) Run(_ *cmd.Context) error {
	// TODO reconnect when the agent fails.
	m, err := NewMachiner(&a.Conf.StateInfo, a.MachineId)
	if err != nil {
		return err
	}
	go m.loop()
	return m.Wait()
}

func NewMachiner(info *state.Info, machineId int) (*Machiner, error) {
	m := new(Machiner)
	var err error
	m.st, err = state.Open(info)
	if err != nil {
		return nil, err
	}
	m.machine, err = m.st.Machine(machineId)
	if err != nil {
		return nil, err
	}
	go m.loop()
	return m, nil
}

type Machiner struct {
	tomb tomb.Tomb
	st *state.State
	machine *state.Machine
}

func (m *Machiner) loop() {
	defer m.tomb.Done()
	defer m.st.Close()

	watcher := m.machine.WatchUnits()
	// TODO read initial units, check if they're running
	// and restart them if not.
	for {
		select {
		case <-m.tomb.Dying():
			return
		case change := <-watcher.Changes():
			for _, u := range change.Deleted {
				if u.IsPrincipal() {
					if err := container.Simple(u).Destroy(); err != nil {
						log.Printf("cannot destroy unit %s: %v", u.Name(), err)
					}
				}
			}
			for _, u := range change.Added {
				if u.IsPrincipal() {
					if err := container.Simple(u).Deploy(); err != nil {
						log.Printf("cannot deploy unit %s: %v", u.Name(), err)
					}
				}
			}
		}
	}
}

func (m *Machiner) Wait() error {
	return m.tomb.Wait()
}

func (m *Machiner) Stop() {
	m.tomb.Kill(nil)
	return m.tomb.Wait()
}
<commit_msg>cmd/jujud: machine agent compiles<commit_after>package main

import (
	"fmt"
	"launchpad.net/gnuflag"
	"launchpad.net/juju-core/juju/cmd"
	"launchpad.net/juju-core/juju/log"
	"launchpad.net/tomb"
	"launchpad.net/juju-core/juju/container"
	"launchpad.net/juju-core/juju/state"
)

// MachineAgent is a cmd.Command responsible for running a machine agent.
type MachineAgent struct {
	Conf      AgentConf
	MachineId int
}

// Info returns usage information for the command.
func (a *MachineAgent) Info() *cmd.Info {
	return &cmd.Info{"machine", "", "run a juju machine agent", ""}
}

// Init initializes the command for running.
func (a *MachineAgent) Init(f *gnuflag.FlagSet, args []string) error {
	a.Conf.addFlags(f)
	f.IntVar(&a.MachineId, "machine-id", -1, "id of the machine to run")
	if err := f.Parse(true, args); err != nil {
		return err
	}
	if a.MachineId < 0 {
		return fmt.Errorf("--machine-id option must be set, and expects a non-negative integer")
	}
	return a.Conf.checkArgs(f.Args())
}

// Run runs a machine agent.
func (a *MachineAgent) Run(_ *cmd.Context) error {
	// TODO reconnect when the agent fails.
	m, err := NewMachiner(&a.Conf.StateInfo, a.MachineId)
	if err != nil {
		return err
	}
	go m.loop()
	return m.Wait()
}

func NewMachiner(info *state.Info, machineId int) (*Machiner, error) {
	m := new(Machiner)
	var err error
	m.st, err = state.Open(info)
	if err != nil {
		return nil, err
	}
	m.machine, err = m.st.Machine(machineId)
	if err != nil {
		return nil, err
	}
	go m.loop()
	return m, nil
}

type Machiner struct {
	tomb tomb.Tomb
	st *state.State
	machine *state.Machine
}

func (m *Machiner) loop() {
	defer m.tomb.Done()
	defer m.st.Close()

	watcher := m.machine.WatchUnits()
	// TODO read initial units, check if they're running
	// and restart them if not.
	for {
		select {
		case <-m.tomb.Dying():
			return
		case change := <-watcher.Changes():
			for _, u := range change.Deleted {
				if u.IsPrincipal() {
					if err := container.Simple(u).Destroy(); err != nil {
						log.Printf("cannot destroy unit %s: %v", u.Name(), err)
					}
				}
			}
			for _, u := range change.Added {
				if u.IsPrincipal() {
					if err := container.Simple(u).Deploy(); err != nil {
						log.Printf("cannot deploy unit %s: %v", u.Name(), err)
					}
				}
			}
		}
	}
}

func (m *Machiner) Wait() error {
	return m.tomb.Wait()
}

func (m *Machiner) Stop() error {
	m.tomb.Kill(nil)
	return m.tomb.Wait()
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"os"
	"os/exec"
	"os/signal"
	"regexp"
	"strings"
)

// RunShellCommand executes a shell command and returns the output/error
func RunShellCommand(input string) (string, error) {
	inputCmd := strings.Split(input, " ")[0]
	args := strings.Split(input, " ")[1:]

	cmd := exec.Command(inputCmd, args...)
	outputBytes := &bytes.Buffer{}
	cmd.Stdout = outputBytes
	cmd.Stderr = outputBytes
	cmd.Start()
	err := cmd.Wait() // wait for command to finish
	outstring := outputBytes.String()
	return outstring, err
}

// HandleShellCommand runs the shell command
// The openTerm argument specifies whether a terminal should be opened (for viewing output
// or interacting with stdin)
func HandleShellCommand(input string, openTerm bool) {
	inputCmd := strings.Split(input, " ")[0]
	if !openTerm {
		// Simply run the command in the background and notify the user when it's done
		messenger.Message("Running...")
		go func() {
			output, err := RunShellCommand(input)
			totalLines := strings.Split(output, "\n")

			if len(totalLines) < 3 {
				if err == nil {
					messenger.Message(inputCmd, " exited without error")
				} else {
					messenger.Message(inputCmd, " exited with error: ", err, ": ", output)
				}
			} else {
				messenger.Message(output)
			}
			// We have to make sure to redraw
			RedrawAll()
		}()
	} else {
		// Shut down the screen because we're going to interact directly with the shell
		screen.Fini()
		screen = nil

		args := strings.Split(input, " ")[1:]

		// Set up everything for the command
		cmd := exec.Command(inputCmd, args...)
		cmd.Stdin = os.Stdin
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr

		// This is a trap for Ctrl-C so that it doesn't kill micro
		// Instead we trap Ctrl-C to kill the program we're running
		c := make(chan os.Signal, 1)
		signal.Notify(c, os.Interrupt)
		go func() {
			for range c {
				cmd.Process.Kill()
			}
		}()

		// Start the command
		cmd.Start()
		cmd.Wait()

		// This is just so we don't return right away and let the user press enter to return
		TermMessage("")

		// Start the screen back up
		InitScreen()
	}
}

// HandleCommand handles input from the user
func HandleCommand(input string) {
	inputCmd := strings.Split(input, " ")[0]
	args := strings.Split(input, " ")[1:]

	switch inputCmd {
	case "set":
		// Set an option and we have to set it for every view
		for _, view := range views {
			SetOption(view, args)
		}
	case "run":
		// Run a shell command in the background (openTerm is false)
		HandleShellCommand(strings.Join(args, " "), false)
	case "quit":
		// This is a bit weird because micro only has one view for now so there is no way to close
		// a single view
		// Currently if multiple views were open, it would close all of them, and not check the non-mainviews
		// for unsaved changes. This, and the behavior of Ctrl-Q need to be changed when splits are implemented
		if views[mainView].CanClose("Quit anyway? (yes, no, save) ") {
			screen.Fini()
			os.Exit(0)
		}
	case "save":
		// Save the main view
		views[mainView].Save()
	case "replace":
		// This is a regex to parse the replace expression
		// We allow no quotes if there are no spaces, but if you want to search
		// for or replace an expression with spaces, you can add double quotes
		r := regexp.MustCompile(`"[^"\\]*(?:\\.[^"\\]*)*"|[^\s]*`)
		replaceCmd := r.FindAllString(strings.Join(args, " "), -1)
		if len(replaceCmd) < 2 {
			// We need to find both a search and replace expression
			messenger.Error("Invalid replace statement: " + strings.Join(args, " "))
			return
		}

		var flags string
		if len(replaceCmd) == 3 {
			// The user included some flags
			flags = replaceCmd[2]
		}

		search := string(replaceCmd[0])
		replace := string(replaceCmd[1])

		// If the search and replace expressions have quotes, we need to remove those
		if strings.HasPrefix(search, `"`) && strings.HasSuffix(search, `"`) {
			search = search[1 : len(search)-1]
		}
		if strings.HasPrefix(replace, `"`) && strings.HasSuffix(replace, `"`) {
			replace = replace[1 : len(replace)-1]
		}

		// We replace all escaped double quotes to real double quotes
		search = strings.Replace(search, `\"`, `"`, -1)
		replace = strings.Replace(replace, `\"`, `"`, -1)

		regex, err := regexp.Compile(search)
		if err != nil {
			// There was an error with the user's regex
			messenger.Error(err.Error())
			return
		}

		view := views[mainView]

		found := false
		for {
			match := regex.FindStringIndex(view.Buf.String())
			if match == nil {
				break
			}
			found = true
			if strings.Contains(flags, "c") {
				// The 'check' flag was used
				Search(search, view, true)
				view.Relocate()
				if settings["syntax"].(bool) {
					view.matches = Match(view)
				}
				RedrawAll()
				choice, canceled := messenger.YesNoPrompt("Perform replacement? (y,n)")
				if canceled {
					if view.Cursor.HasSelection() {
						view.Cursor.SetLoc(view.Cursor.curSelection[0])
						view.Cursor.ResetSelection()
					}
					messenger.Reset()
					return
				}
				if choice {
					view.Cursor.DeleteSelection()
					view.Buf.Insert(match[0], replace)
					view.Cursor.ResetSelection()
					messenger.Reset()
				} else {
					if view.Cursor.HasSelection() {
						searchStart = view.Cursor.curSelection[1]
					} else {
						searchStart = ToCharPos(view.Cursor.x, view.Cursor.y, view.Buf)
					}
					continue
				}
			} else {
				view.Buf.Replace(match[0], match[1], replace)
			}
		}
		if !found {
			messenger.Message("Nothing matched " + search)
		}
	default:
		messenger.Error("Unknown command: " + inputCmd)
	}
}
<commit_msg>Allow users to insert \t and \n in replace commands<commit_after>package main

import (
	"bytes"
	"os"
	"os/exec"
	"os/signal"
	"regexp"
	"strings"
)

// RunShellCommand executes a shell command and returns the output/error
func RunShellCommand(input string) (string, error) {
	inputCmd := strings.Split(input, " ")[0]
	args := strings.Split(input, " ")[1:]

	cmd := exec.Command(inputCmd, args...)
	outputBytes := &bytes.Buffer{}
	cmd.Stdout = outputBytes
	cmd.Stderr = outputBytes
	cmd.Start()
	err := cmd.Wait() // wait for command to finish
	outstring := outputBytes.String()
	return outstring, err
}

// HandleShellCommand runs the shell command
// The openTerm argument specifies whether a terminal should be opened (for viewing output
// or interacting with stdin)
func HandleShellCommand(input string, openTerm bool) {
	inputCmd := strings.Split(input, " ")[0]
	if !openTerm {
		// Simply run the command in the background and notify the user when it's done
		messenger.Message("Running...")
		go func() {
			output, err := RunShellCommand(input)
			totalLines := strings.Split(output, "\n")

			if len(totalLines) < 3 {
				if err == nil {
					messenger.Message(inputCmd, " exited without error")
				} else {
					messenger.Message(inputCmd, " exited with error: ", err, ": ", output)
				}
			} else {
				messenger.Message(output)
			}
			// We have to make sure to redraw
			RedrawAll()
		}()
	} else {
		// Shut down the screen because we're going to interact directly with the shell
		screen.Fini()
		screen = nil

		args := strings.Split(input, " ")[1:]

		// Set up everything for the command
		cmd := exec.Command(inputCmd, args...)
		cmd.Stdin = os.Stdin
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr

		// This is a trap for Ctrl-C so that it doesn't kill micro
		// Instead we trap Ctrl-C to kill the program we're running
		c := make(chan os.Signal, 1)
		signal.Notify(c, os.Interrupt)
		go func() {
			for range c {
				cmd.Process.Kill()
			}
		}()

		// Start the command
		cmd.Start()
		cmd.Wait()

		// This is just so we don't return right away and let the user press enter to return
		TermMessage("")

		// Start the screen back up
		InitScreen()
	}
}

// HandleCommand handles input from the user
func HandleCommand(input string) {
	inputCmd := strings.Split(input, " ")[0]
	args := strings.Split(input, " ")[1:]

	switch inputCmd {
	case "set":
		// Set an option and we have to set it for every view
		for _, view := range views {
			SetOption(view, args)
		}
	case "run":
		// Run a shell command in the background (openTerm is false)
		HandleShellCommand(strings.Join(args, " "), false)
	case "quit":
		// This is a bit weird because micro only has one view for now so there is no way to close
		// a single view
		// Currently if multiple views were open, it would close all of them, and not check the non-mainviews
		// for unsaved changes. This, and the behavior of Ctrl-Q need to be changed when splits are implemented
		if views[mainView].CanClose("Quit anyway? (yes, no, save) ") {
			screen.Fini()
			os.Exit(0)
		}
	case "save":
		// Save the main view
		views[mainView].Save()
	case "replace":
		// This is a regex to parse the replace expression
		// We allow no quotes if there are no spaces, but if you want to search
		// for or replace an expression with spaces, you can add double quotes
		r := regexp.MustCompile(`"[^"\\]*(?:\\.[^"\\]*)*"|[^\s]*`)
		replaceCmd := r.FindAllString(strings.Join(args, " "), -1)
		if len(replaceCmd) < 2 {
			// We need to find both a search and replace expression
			messenger.Error("Invalid replace statement: " + strings.Join(args, " "))
			return
		}

		var flags string
		if len(replaceCmd) == 3 {
			// The user included some flags
			flags = replaceCmd[2]
		}

		search := string(replaceCmd[0])
		replace := string(replaceCmd[1])

		// If the search and replace expressions have quotes, we need to remove those
		if strings.HasPrefix(search, `"`) && strings.HasSuffix(search, `"`) {
			search = search[1 : len(search)-1]
		}
		if strings.HasPrefix(replace, `"`) && strings.HasSuffix(replace, `"`) {
			replace = replace[1 : len(replace)-1]
		}

		// We replace all escaped double quotes to real double quotes
		search = strings.Replace(search, `\"`, `"`, -1)
		replace = strings.Replace(replace, `\"`, `"`, -1)
		// Replace some things so users can actually insert newlines and tabs in replacements
		replace = strings.Replace(replace, "\\n", "\n", -1)
		replace = strings.Replace(replace, "\\t", "\t", -1)

		regex, err := regexp.Compile(search)
		if err != nil {
			// There was an error with the user's regex
			messenger.Error(err.Error())
			return
		}

		view := views[mainView]

		found := false
		for {
			match := regex.FindStringIndex(view.Buf.String())
			if match == nil {
				break
			}
			found = true
			if strings.Contains(flags, "c") {
				// The 'check' flag was used
				Search(search, view, true)
				view.Relocate()
				if settings["syntax"].(bool) {
					view.matches = Match(view)
				}
				RedrawAll()
				choice, canceled := messenger.YesNoPrompt("Perform replacement? (y,n)")
				if canceled {
					if view.Cursor.HasSelection() {
						view.Cursor.SetLoc(view.Cursor.curSelection[0])
						view.Cursor.ResetSelection()
					}
					messenger.Reset()
					return
				}
				if choice {
					view.Cursor.DeleteSelection()
					view.Buf.Insert(match[0], replace)
					view.Cursor.ResetSelection()
					messenger.Reset()
				} else {
					if view.Cursor.HasSelection() {
						searchStart = view.Cursor.curSelection[1]
					} else {
						searchStart = ToCharPos(view.Cursor.x, view.Cursor.y, view.Buf)
					}
					continue
				}
			} else {
				view.Buf.Replace(match[0], match[1], replace)
			}
		}
		if !found {
			messenger.Message("Nothing matched " + search)
		}
	default:
		messenger.Error("Unknown command: " + inputCmd)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"os"

	"github.com/osrg/gobgp/packet/bgp"
	"github.com/osrg/gobgp/packet/mrt"
	log "github.com/sirupsen/logrus"
	"github.com/zmap/zannotate/zmrt"
)

type MRT2JsonGlobalConf struct {
	InputFilePath  string
	OutputFilePath string
	LogFilePath    string
	Verbosity      int
}

type RawPeer struct {
	Type      string `json:"type"`
	BgpId     string `json:"bgp_id"`
	IpAddress string `json:"ip_address"`
	AS        uint32 `json:"as"`
}

type RawPeerIndexTableJSON struct {
	CollectorBgpId string     `json:"collector_bgp_id"`
	ViewName       string     `json:"view_name"`
	Peers          []*RawPeer `json:"peers"`
	Type           string     `json:"type"`
}

type RawRibEntry struct {
	PeerIndex      uint16                       `json:"peer_index"`
	OriginatedTime uint32                       `json:"orginated_time"`
	PathIdentifier uint32                       `json:"path_identifier"`
	PathAttributes []bgp.PathAttributeInterface `json:"path_attributes"`
	Type           string                       `json:"type"`
}

type RawRib struct {
	SubType        string                  `json:"sub_type"`
	SequenceNumber uint32                  `json:"sequence_number"`
	Prefix         bgp.AddrPrefixInterface `json:"prefix"`
	Entries        []*RawRibEntry          `json:"entries"`
	RouteFamily    bgp.RouteFamily         `json:"route_family"`
}

func raw(conf *MRT2JsonGlobalConf, f *os.File) {
	f, err := os.Open(conf.InputFilePath)
	if err != nil {
		log.Fatal(err)
	}
	zmrt.MrtRawIterate(f, func(msg *mrt.MRTMessage) error {
		if msg.Header.Type != mrt.TABLE_DUMPv2 {
			log.Fatal("not an MRT TABLE_DUMPv2")
		}
		switch mrt.MRTSubTypeTableDumpv2(msg.Header.SubType) {
		case mrt.PEER_INDEX_TABLE:
			peerIndexTable := msg.Body.(*mrt.PeerIndexTable)
			var out RawPeerIndexTableJSON
			out.CollectorBgpId = peerIndexTable.CollectorBgpId.String()
			out.ViewName = peerIndexTable.ViewName
			out.Type = "peer_index_table"
			for _, peer := range peerIndexTable.Peers {
				var outPeer RawPeer
				//outPeer.Type = peer.Type
				outPeer.BgpId = peer.BgpId.String()
				outPeer.IpAddress = peer.IpAddress.String()
				outPeer.AS = peer.AS
				out.Peers = append(out.Peers, &outPeer)
			}
			json, err := json.Marshal(out)
			if err != nil {
				log.Fatal("unable to json marshal peer table")
			}
			f.WriteString(string(json))
			f.WriteString("\n")
		case mrt.RIB_IPV4_UNICAST, mrt.RIB_IPV6_UNICAST,
			mrt.RIB_IPV4_MULTICAST, mrt.RIB_IPV6_MULTICAST, mrt.RIB_GENERIC:
			//
			rib := msg.Body.(*mrt.Rib)
			var out RawRib
			out.SequenceNumber = rib.SequenceNumber
			out.Prefix = rib.Prefix
			out.RouteFamily = rib.RouteFamily
			for _, entry := range rib.Entries {
				var ribOut RawRibEntry
				ribOut.PeerIndex = entry.PeerIndex
				ribOut.OriginatedTime = entry.OriginatedTime
				ribOut.PathIdentifier = entry.PathIdentifier
				ribOut.PathAttributes = entry.PathAttributes
				out.Entries = append(out.Entries, &ribOut)
			}
			out.SubType = zmrt.MrtSubTypeToName(msg.Header.SubType)
			json, err := json.Marshal(out)
			if err != nil {
				log.Fatal("unable to json marshal peer table")
			}
			f.WriteString(string(json))
			f.WriteString("\n")
		case mrt.GEO_PEER_TABLE:
			//geopeers := msg.Body.(*mrt.GeoPeerTable)
		default:
			log.Fatalf("unsupported subType: %v", msg.Header.SubType)
		}
		return nil
	})
}

func paths(conf *MRT2JsonGlobalConf, f *os.File) {
	f, err := os.Open(conf.InputFilePath)
	if err != nil {
		log.Fatal(err)
	}
	zmrt.MrtPathIterate(f, func(msg *zmrt.RIBEntry) {
		json, _ := json.Marshal(msg)
		f.WriteString(string(json))
		f.WriteString("\n")
	})
}

func main() {

	var conf MRT2JsonGlobalConf
	flags := flag.NewFlagSet("flags", flag.ExitOnError)
	flags.StringVar(&conf.InputFilePath, "input-file", "", "ip addresses to read")
	flags.StringVar(&conf.OutputFilePath, "output-file", "-", "where should JSON output be saved")
	flags.StringVar(&conf.LogFilePath, "log-file", "", "where should JSON output be saved")
	flags.IntVar(&conf.Verbosity, "verbosity", 3, "where should JSON output be saved")
	flags.Parse(os.Args[2:])

	if conf.LogFilePath != "" {
		f, err := os.OpenFile(conf.LogFilePath, os.O_WRONLY|os.O_CREATE, 0666)
		if err != nil {
			log.Fatalf("Unable to open log file (%s): %s", conf.LogFilePath, err.Error())
		}
		log.SetOutput(f)
	}
	if conf.InputFilePath == "" {
		log.Fatal("no input path provided")
	}
	// Translate the assigned verbosity level to a logrus log level.
	switch conf.Verbosity {
	case 1: // Fatal
		log.SetLevel(log.FatalLevel)
	case 2: // Error
		log.SetLevel(log.ErrorLevel)
	case 3: // Warnings  (default)
		log.SetLevel(log.WarnLevel)
	case 4: // Information
		log.SetLevel(log.InfoLevel)
	case 5: // Debugging
		log.SetLevel(log.DebugLevel)
	default:
		log.Fatal("Unknown verbosity level specified. Must be between 1 (lowest)--5 (highest)")
	}
	var f *os.File
	if conf.OutputFilePath == "-" {
		f = os.Stdout
	} else {
		var err error
		f, err = os.OpenFile(conf.OutputFilePath, os.O_WRONLY|os.O_CREATE, 0666)
		if err != nil {
			log.Fatal("unable to open metadata file:", err.Error())
		}
		defer f.Close()
	}
	if os.Args[1] == "raw" {
		raw(&conf, f)
	} else if os.Args[1] == "entries" {
		paths(&conf, f)
	} else {
		log.Fatal("invalid command")
	}
}
<commit_msg>mrt2json works now<commit_after>package main

import (
	"encoding/json"
	"flag"
	"os"

	"github.com/osrg/gobgp/packet/bgp"
	"github.com/osrg/gobgp/packet/mrt"
	log "github.com/sirupsen/logrus"
	"github.com/zmap/zannotate/zmrt"
)

type MRT2JsonGlobalConf struct {
	InputFilePath  string
	OutputFilePath string
	LogFilePath    string
	Verbosity      int
}

type RawPeer struct {
	Type      string `json:"type"`
	BgpId     string `json:"bgp_id"`
	IpAddress string `json:"ip_address"`
	AS        uint32 `json:"as"`
}

type RawPeerIndexTableJSON struct {
	CollectorBgpId string     `json:"collector_bgp_id"`
	ViewName       string     `json:"view_name"`
	Peers          []*RawPeer `json:"peers"`
	Type           string     `json:"type"`
}

type RawRibEntry struct {
	PeerIndex      uint16                       `json:"peer_index"`
	OriginatedTime uint32                       `json:"orginated_time"`
	PathIdentifier uint32                       `json:"path_identifier"`
	PathAttributes []bgp.PathAttributeInterface `json:"path_attributes"`
	Type           string                       `json:"type"`
}

type RawRib struct {
	SubType        string                  `json:"sub_type"`
	SequenceNumber uint32                  `json:"sequence_number"`
	Prefix         bgp.AddrPrefixInterface `json:"prefix"`
	Entries        []*RawRibEntry          `json:"entries"`
	RouteFamily    bgp.RouteFamily         `json:"route_family"`
}

func raw(conf *MRT2JsonGlobalConf, f *os.File) {
	inputFile, err := os.Open(conf.InputFilePath)
	if err != nil {
		log.Fatal(err)
	}
	zmrt.MrtRawIterate(inputFile, func(msg *mrt.MRTMessage) error {
		if msg.Header.Type != mrt.TABLE_DUMPv2 {
			log.Fatal("not an MRT TABLE_DUMPv2")
		}
		switch mrt.MRTSubTypeTableDumpv2(msg.Header.SubType) {
		case mrt.PEER_INDEX_TABLE:
			peerIndexTable := msg.Body.(*mrt.PeerIndexTable)
			var out RawPeerIndexTableJSON
			out.CollectorBgpId = peerIndexTable.CollectorBgpId.String()
			out.ViewName = peerIndexTable.ViewName
			out.Type = "peer_index_table"
			for _, peer := range peerIndexTable.Peers {
				var outPeer RawPeer
				//outPeer.Type = peer.Type
				outPeer.BgpId = peer.BgpId.String()
				outPeer.IpAddress = peer.IpAddress.String()
				outPeer.AS = peer.AS
				out.Peers = append(out.Peers, &outPeer)
			}
			json, err := json.Marshal(out)
			if err != nil {
				log.Fatal("unable to json marshal peer table")
			}
			f.WriteString(string(json))
			f.WriteString("\n")
		case mrt.RIB_IPV4_UNICAST, mrt.RIB_IPV6_UNICAST,
			mrt.RIB_IPV4_MULTICAST, mrt.RIB_IPV6_MULTICAST, mrt.RIB_GENERIC:
			//
			rib := msg.Body.(*mrt.Rib)
			var out RawRib
			out.SequenceNumber = rib.SequenceNumber
			out.Prefix = rib.Prefix
			out.RouteFamily = rib.RouteFamily
			for _, entry := range rib.Entries {
				var ribOut RawRibEntry
				ribOut.PeerIndex = entry.PeerIndex
				ribOut.OriginatedTime = entry.OriginatedTime
				ribOut.PathIdentifier = entry.PathIdentifier
				ribOut.PathAttributes = entry.PathAttributes
				out.Entries = append(out.Entries, &ribOut)
			}
			out.SubType = zmrt.MrtSubTypeToName(msg.Header.SubType)
			json, err := json.Marshal(out)
			if err != nil {
				log.Fatal("unable to json marshal peer table")
			}
			f.WriteString(string(json))
			f.WriteString("\n")
		case mrt.GEO_PEER_TABLE:
		//geopeers := msg.Body.(*mrt.GeoPeerTable)
		default:
			log.Fatalf("unsupported subType: %v", msg.Header.SubType)
		}
		return nil
	})
}

func paths(conf *MRT2JsonGlobalConf, f *os.File) {
	inputFile, err := os.Open(conf.InputFilePath)
	if err != nil {
		log.Fatal(err)
	}
	zmrt.MrtPathIterate(inputFile, func(msg *zmrt.RIBEntry) {
		json, _ := json.Marshal(msg)
		f.WriteString(string(json))
		f.WriteString("\n")
	})
}

func main() {

	var conf MRT2JsonGlobalConf
	flags := flag.NewFlagSet("flags", flag.ExitOnError)
	flags.StringVar(&conf.InputFilePath, "input-file", "", "path to MRT file")
	flags.StringVar(&conf.OutputFilePath, "output-file", "-", "where should JSON output be saved")
	flags.StringVar(&conf.LogFilePath, "log-file", "", "where should JSON output be saved")
	flags.IntVar(&conf.Verbosity, "verbosity", 3, "where should JSON output be saved")

	if len(os.Args) < 2 {
		log.Fatalf("No command provided. Must choose raw or entries")
	}
	flags.Parse(os.Args[2:])

	if conf.LogFilePath != "" {
		f, err := os.OpenFile(conf.LogFilePath, os.O_WRONLY|os.O_CREATE, 0666)
		if err != nil {
			log.Fatalf("Unable to open log file (%s): %s", conf.LogFilePath, err.Error())
		}
		log.SetOutput(f)
	}
	if conf.InputFilePath == "" {
		log.Fatal("no input path provided")
	}
	// Translate the assigned verbosity level to a logrus log level.
	switch conf.Verbosity {
	case 1: // Fatal
		log.SetLevel(log.FatalLevel)
	case 2: // Error
		log.SetLevel(log.ErrorLevel)
	case 3: // Warnings  (default)
		log.SetLevel(log.WarnLevel)
	case 4: // Information
		log.SetLevel(log.InfoLevel)
	case 5: // Debugging
		log.SetLevel(log.DebugLevel)
	default:
		log.Fatal("Unknown verbosity level specified. Must be between 1 (lowest)--5 (highest)")
	}
	var f *os.File
	if conf.OutputFilePath == "-" {
		f = os.Stdout
	} else {
		var err error
		f, err = os.OpenFile(conf.OutputFilePath, os.O_WRONLY|os.O_CREATE, 0666)
		if err != nil {
			log.Fatal("unable to open output file: ", err.Error())
		}
		defer f.Close()
	}
	if os.Args[1] == "raw" {
		raw(&conf, f)
	} else if os.Args[1] == "entries" {
		paths(&conf, f)
	} else {
		log.Fatal("invalid command")
	}

}
<|endoftext|>


<commit_before>// Use of this source code is governed by a BSD-style license that can be found
// in the LICENSE file.

package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/google/puffs/lang/parse"
	"github.com/google/puffs/lang/render"
	"github.com/google/puffs/lang/token"
)

var (
	lFlag = flag.Bool("l", false, "list files whose formatting differs from puffsfmt's")
	wFlag = flag.Bool("w", false, "write result to (source) file instead of stdout")
)

func usage() {
	fmt.Fprintf(os.Stderr, "usage: puffsfmt [flags] [path ...]\n")
	flag.PrintDefaults()
}

func main() {
	if err := main1(); err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
	}
}

func main1() error {
	flag.Usage = usage
	flag.Parse()

	if flag.NArg() == 0 {
		if *lFlag {
			return errors.New("cannot use -l with standard input")
		}
		if *wFlag {
			return errors.New("cannot use -w with standard input")
		}
		return do(os.Stdin, "<standard input>")
	}

	if !*lFlag && !*wFlag {
		return errors.New("must use -l or -w if paths are given")
	}

	for i := 0; i < flag.NArg(); i++ {
		arg := flag.Arg(i)
		switch dir, err := os.Stat(arg); {
		case err != nil:
			return err
		case dir.IsDir():
			return filepath.Walk(arg, walk)
		default:
			if err := do(nil, arg); err != nil {
				return err
			}
		}
	}

	return nil
}

func isPuffsFile(info os.FileInfo) bool {
	name := info.Name()
	return !info.IsDir() && !strings.HasPrefix(name, ".") && strings.HasSuffix(name, ".puffs")
}

func walk(filename string, info os.FileInfo, err error) error {
	if err == nil && isPuffsFile(info) {
		err = do(nil, filename)
	}
	// Don't complain if a file was deleted in the meantime (i.e. the directory
	// changed concurrently while running puffsfmt).
	if err != nil && !os.IsNotExist(err) {
		return err
	}
	return nil
}

func do(r io.Reader, filename string) error {
	src, err := []byte(nil), error(nil)
	if r != nil {
		src, err = ioutil.ReadAll(r)
	} else {
		src, err = ioutil.ReadFile(filename)
	}
	if err != nil {
		return err
	}

	idMap := token.IDMap{}
	tokens, comments, err := token.Tokenize(src, &idMap, filename)
	if err != nil {
		return err
	}
	// We don't need the AST node to pretty-print, but it's worth puffsfmt
	// rejecting syntax errors. This is just a parse, not a full type check.
	if _, err := parse.ParseFile(tokens, &idMap, filename); err != nil {
		return err
	}
	buf := &bytes.Buffer{}
	if err := render.RenderFile(buf, tokens, comments, &idMap); err != nil {
		return err
	}
	dst := buf.Bytes()

	if r != nil {
		if _, err := os.Stdout.Write(dst); err != nil {
			return err
		}
	} else if !bytes.Equal(dst, src) {
		if *lFlag {
			fmt.Println(filename)
		}
		if *wFlag {
			if err := writeFile(filename, dst); err != nil {
				return err
			}
		}
	}

	return nil
}

const chmodSupported = runtime.GOOS != "windows"

func writeFile(filename string, b []byte) error {
	info, err := os.Stat(filename)
	if err != nil {
		return err
	}
	f, err := ioutil.TempFile(filepath.Dir(filename), filepath.Base(filename))
	if err != nil {
		return err
	}
	if chmodSupported {
		f.Chmod(info.Mode().Perm())
	}
	_, werr := f.Write(b)
	cerr := f.Close()
	if werr != nil {
		os.Remove(f.Name())
		return werr
	}
	if cerr != nil {
		os.Remove(f.Name())
		return cerr
	}
	return os.Rename(f.Name(), filename)
}
<commit_msg>Add a puffsfmt doc comment.<commit_after>// Use of this source code is governed by a BSD-style license that can be found
// in the LICENSE file.

// puffsfmt formats Puffs programs.
//
// Without explicit paths, it rewrites the standard input to standard output.
// Otherwise, the -l or -w or both flags must be given. Given a file path, it
// operates on that file; given a directory path, it operates on all .puffs
// files in that directory, recursively. Files starting with a period are
// ignored.
package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/google/puffs/lang/parse"
	"github.com/google/puffs/lang/render"
	"github.com/google/puffs/lang/token"
)

var (
	lFlag = flag.Bool("l", false, "list files whose formatting differs from puffsfmt's")
	wFlag = flag.Bool("w", false, "write result to (source) file instead of stdout")
)

func usage() {
	fmt.Fprintf(os.Stderr, "usage: puffsfmt [flags] [path ...]\n")
	flag.PrintDefaults()
}

func main() {
	if err := main1(); err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
	}
}

func main1() error {
	flag.Usage = usage
	flag.Parse()

	if flag.NArg() == 0 {
		if *lFlag {
			return errors.New("cannot use -l with standard input")
		}
		if *wFlag {
			return errors.New("cannot use -w with standard input")
		}
		return do(os.Stdin, "<standard input>")
	}

	if !*lFlag && !*wFlag {
		return errors.New("must use -l or -w if paths are given")
	}

	for i := 0; i < flag.NArg(); i++ {
		arg := flag.Arg(i)
		switch dir, err := os.Stat(arg); {
		case err != nil:
			return err
		case dir.IsDir():
			return filepath.Walk(arg, walk)
		default:
			if err := do(nil, arg); err != nil {
				return err
			}
		}
	}

	return nil
}

func isPuffsFile(info os.FileInfo) bool {
	name := info.Name()
	return !info.IsDir() && !strings.HasPrefix(name, ".") && strings.HasSuffix(name, ".puffs")
}

func walk(filename string, info os.FileInfo, err error) error {
	if err == nil && isPuffsFile(info) {
		err = do(nil, filename)
	}
	// Don't complain if a file was deleted in the meantime (i.e. the directory
	// changed concurrently while running puffsfmt).
	if err != nil && !os.IsNotExist(err) {
		return err
	}
	return nil
}

func do(r io.Reader, filename string) error {
	src, err := []byte(nil), error(nil)
	if r != nil {
		src, err = ioutil.ReadAll(r)
	} else {
		src, err = ioutil.ReadFile(filename)
	}
	if err != nil {
		return err
	}

	idMap := token.IDMap{}
	tokens, comments, err := token.Tokenize(src, &idMap, filename)
	if err != nil {
		return err
	}
	// We don't need the AST node to pretty-print, but it's worth puffsfmt
	// rejecting syntax errors. This is just a parse, not a full type check.
	if _, err := parse.ParseFile(tokens, &idMap, filename); err != nil {
		return err
	}
	buf := &bytes.Buffer{}
	if err := render.RenderFile(buf, tokens, comments, &idMap); err != nil {
		return err
	}
	dst := buf.Bytes()

	if r != nil {
		if _, err := os.Stdout.Write(dst); err != nil {
			return err
		}
	} else if !bytes.Equal(dst, src) {
		if *lFlag {
			fmt.Println(filename)
		}
		if *wFlag {
			if err := writeFile(filename, dst); err != nil {
				return err
			}
		}
	}

	return nil
}

const chmodSupported = runtime.GOOS != "windows"

func writeFile(filename string, b []byte) error {
	info, err := os.Stat(filename)
	if err != nil {
		return err
	}
	f, err := ioutil.TempFile(filepath.Dir(filename), filepath.Base(filename))
	if err != nil {
		return err
	}
	if chmodSupported {
		f.Chmod(info.Mode().Perm())
	}
	_, werr := f.Write(b)
	cerr := f.Close()
	if werr != nil {
		os.Remove(f.Name())
		return werr
	}
	if cerr != nil {
		os.Remove(f.Name())
		return cerr
	}
	return os.Rename(f.Name(), filename)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"os"
	"strings"

	"github.com/proidiot/gone/log"
	"github.com/stuphlabs/pullcord/authentication"
	"github.com/stuphlabs/pullcord/config"
	"github.com/stuphlabs/pullcord/monitor"
	pcnet "github.com/stuphlabs/pullcord/net"
	"github.com/stuphlabs/pullcord/proxy"
	"github.com/stuphlabs/pullcord/trigger"
	"github.com/stuphlabs/pullcord/util"
)

const defaultConfigFilePath = "/etc/pullcord.json"
const defaultConfig = `{
	"resources": {
		"hresource": {
			"type": "exactpathrouter",
			"data": {
				"routes": {
					"/favicon.ico": {
						"type": "standardresponse",
						"data": 404
					}
				},
				"default": {
					"type": "landingfaulter",
					"data": {}
				}
			}
		},
		"lresource": {
			"type": "basiclistener",
			"data": {
				"proto": "tcp",
				"laddr": ":8080"
			}
		}
	},
	"listener": "lresource",
	"handler": "hresource"
}`

func main() {
	var inlineCfg string
	var cfgPath string
	var cfgFallback bool

	flag.StringVar(
		&inlineCfg,
		"inline-config",
		"",
		"Inline pullcord config instead of using a config file",
	)

	flag.StringVar(
		&cfgPath,
		"config",
		defaultConfigFilePath,
		"Path to pullcord config file",
	)

	flag.BoolVar(
		&cfgFallback,
		"config-fallback",
		true,
		"Fallback to basic config if unable to find the config file",
	)

	flag.Parse()

	var err error
	var cfgReader io.Reader
	if inlineCfg != "" {
		cfgReader = strings.NewReader(inlineCfg)
	}

	if cfgReader == nil {
		cfgReader, err = os.Open(cfgPath)
		if err != nil {
			log.Error(
				fmt.Sprintf(
					"Unable to open specified config file"+
						" %s: %s",
					cfgPath,
					err.Error(),
				),
			)
		} else {
			log.Info(
				fmt.Sprintf(
					"Reading config from file: %s",
					cfgPath,
				),
			)
		}
	}

	if cfgReader == nil {
		if !cfgFallback {
			log.Crit(
				"No config defined and not falling back to"+
					" default, aborting.",
			)
			os.Exit(1)
		} else {
			cfgReader = strings.NewReader(defaultConfig)
		}
	}

	authentication.LoadPlugin()
	monitor.LoadPlugin()
	pcnet.LoadPlugin()
	proxy.LoadPlugin()
	trigger.LoadPlugin()
	util.LoadPlugin()

	log.Debug("Plugins loaded")

	server, err := config.ServerFromReader(cfgReader)
	if err != nil {
		log.Debug(err)
		critErr := fmt.Errorf(
			"Error while parsing server config: %s",
			err.Error(),
		)
		log.Crit(critErr)
		panic(critErr)
	}

	log.Notice(
		fmt.Sprintf(
			"Starting server at %s...",
			server.Listener.Addr(),
		),
	)

	err = server.Serve()
	if err != nil {
		log.Debug(err)
		critErr := fmt.Errorf(
			"Error while running server: %s",
			err.Error(),
		)
		log.Crit(critErr)
		panic(critErr)
	}
}
<commit_msg>Making gofmt happy.<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"os"
	"strings"

	"github.com/proidiot/gone/log"
	"github.com/stuphlabs/pullcord/authentication"
	"github.com/stuphlabs/pullcord/config"
	"github.com/stuphlabs/pullcord/monitor"
	pcnet "github.com/stuphlabs/pullcord/net"
	"github.com/stuphlabs/pullcord/proxy"
	"github.com/stuphlabs/pullcord/trigger"
	"github.com/stuphlabs/pullcord/util"
)

const defaultConfigFilePath = "/etc/pullcord.json"
const defaultConfig = `{
	"resources": {
		"hresource": {
			"type": "exactpathrouter",
			"data": {
				"routes": {
					"/favicon.ico": {
						"type": "standardresponse",
						"data": 404
					}
				},
				"default": {
					"type": "landingfaulter",
					"data": {}
				}
			}
		},
		"lresource": {
			"type": "basiclistener",
			"data": {
				"proto": "tcp",
				"laddr": ":8080"
			}
		}
	},
	"listener": "lresource",
	"handler": "hresource"
}`

func main() {
	var inlineCfg string
	var cfgPath string
	var cfgFallback bool

	flag.StringVar(
		&inlineCfg,
		"inline-config",
		"",
		"Inline pullcord config instead of using a config file",
	)

	flag.StringVar(
		&cfgPath,
		"config",
		defaultConfigFilePath,
		"Path to pullcord config file",
	)

	flag.BoolVar(
		&cfgFallback,
		"config-fallback",
		true,
		"Fallback to basic config if unable to find the config file",
	)

	flag.Parse()

	var err error
	var cfgReader io.Reader
	if inlineCfg != "" {
		cfgReader = strings.NewReader(inlineCfg)
	}

	if cfgReader == nil {
		cfgReader, err = os.Open(cfgPath)
		if err != nil {
			log.Error(
				fmt.Sprintf(
					"Unable to open specified config file"+
						" %s: %s",
					cfgPath,
					err.Error(),
				),
			)
		} else {
			log.Info(
				fmt.Sprintf(
					"Reading config from file: %s",
					cfgPath,
				),
			)
		}
	}

	if cfgReader == nil {
		if !cfgFallback {
			log.Crit(
				"No config defined and not falling back to" +
					" default, aborting.",
			)
			os.Exit(1)
		} else {
			cfgReader = strings.NewReader(defaultConfig)
		}
	}

	authentication.LoadPlugin()
	monitor.LoadPlugin()
	pcnet.LoadPlugin()
	proxy.LoadPlugin()
	trigger.LoadPlugin()
	util.LoadPlugin()

	log.Debug("Plugins loaded")

	server, err := config.ServerFromReader(cfgReader)
	if err != nil {
		log.Debug(err)
		critErr := fmt.Errorf(
			"Error while parsing server config: %s",
			err.Error(),
		)
		log.Crit(critErr)
		panic(critErr)
	}

	log.Notice(
		fmt.Sprintf(
			"Starting server at %s...",
			server.Listener.Addr(),
		),
	)

	err = server.Serve()
	if err != nil {
		log.Debug(err)
		critErr := fmt.Errorf(
			"Error while running server: %s",
			err.Error(),
		)
		log.Crit(critErr)
		panic(critErr)
	}
}
<|endoftext|>


<commit_before>// Package main defines a command line tool for submitting date
// ranges for reprocessing
package main

// TODO - note about setting up batch table and giving permission.

/*
Strategies...
  1. Work from a prefix, or range of prefixes.
  2. Work from a date range
  3. Work from a month prefix, but explicitly iterate over days.
      maybe use a separate goroutine for each date?

Usage:

*/

import (
	"bytes"
	"context"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"sync"
	"time"

	"google.golang.org/api/iterator"

	"cloud.google.com/go/storage"
)

type httpClientIntf interface {
	Get(url string) (resp *http.Response, err error)
}

// This is used to intercept Get requests to the queue_pusher when invoked
// with -dry_run.
type dryRunHTTP struct{}

func (dr *dryRunHTTP) Get(url string) (resp *http.Response, err error) {
	resp = &http.Response{}
	resp.Body = ioutil.NopCloser(bytes.NewReader([]byte{}))
	resp.Status = "200 OK"
	resp.StatusCode = 200
	return
}

var (
	fProject = flag.String("project", "", "Project containing queues.")
	fQueue   = flag.String("queue", "etl-ndt-batch-", "Base of queue name.")
	// TODO implement listing queues to determine number of queue, and change this to 0
	fNumQueues = flag.Int("num_queues", 8, "Number of queues.  Normally determined by listing queues.")
	fBucket    = flag.String("bucket", "archive-mlab-oti", "Source bucket.")
	fExper     = flag.String("experiment", "ndt", "Experiment prefix, trailing slash optional")
	fMonth     = flag.String("month", "", "Single month spec, as YYYY/MM")
	fDay       = flag.String("day", "", "Single day spec, as YYYY/MM/DD")
	fDryRun    = flag.Bool("dry_run", false, "Prevents all output to queue_pusher.")

	bucket *storage.BucketHandle
	// Use an interface to allow test fake.
	httpClient httpClientIntf = http.DefaultClient
)

func init() {
	// Always prepend the filename and line number.
	log.SetFlags(log.LstdFlags | log.Lshortfile)
}

// postOne sends a single https request to the queue pusher to add a task.
// Iff dryRun is true, this does nothing.
func postOne(queue string, bucket string, fn string) error {
	reqStr := fmt.Sprintf("https://queue-pusher-dot-%s.appspot.com/receiver?queue=%s&filename=gs://%s/%s", *fProject, queue, bucket, fn)

	resp, err := httpClient.Get(reqStr)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return errors.New("http error: " + resp.Status)
	}

	return nil
}

// Post all items in an ObjectIterator into specific
// queue.
func postDay(wg *sync.WaitGroup, queue string, it *storage.ObjectIterator) {
	qpErrCount := 0
	gcsErrCount := 0
	fileCount := 0
	if wg != nil {
		defer wg.Done()
	}
	for o, err := it.Next(); err != iterator.Done; o, err = it.Next() {
		if err != nil {
			// TODO - should this retry?
			log.Println(err)
			if gcsErrCount > 3 {
				log.Printf("Failed after %d files to %s.\n", fileCount, queue)
				return
			}
			continue
		}

		err = postOne(queue, *fBucket, o.Name)
		if err != nil {
			// TODO - should this retry?
			log.Println(err)
			if qpErrCount > 3 {
				log.Printf("Failed after %d files to %s.\n", fileCount, queue)
				return
			}
		} else {
			fileCount++
		}
	}
	log.Println("Added ", fileCount, " tasks to ", queue)
}

// Initially this used a hash, but using day ordinal is better
// as it distributes across the queues more evenly.
func queueFor(date time.Time) string {
	day := date.Unix() / (24 * 60 * 60)
	return fmt.Sprintf("%s%d", *fQueue, int(day)%*fNumQueues)
}

// day fetches an iterator over the objects with ndt/YYYY/MM/DD prefix,
// and passes the iterator to postDay with appropriate queue.
// Iff wq is not nil, postDay will call done on wg when finished
// posting.
func day(wg *sync.WaitGroup, prefix string) {
	date, err := time.Parse("ndt/2006/01/02/", prefix)
	if err != nil {
		log.Println("Failed parsing date from ", prefix)
		log.Println(err)
		if wg != nil {
			wg.Done()
		}
		return
	}
	queue := queueFor(date)
	log.Println("Adding ", prefix, " to ", queue)
	q := storage.Query{
		Delimiter: "/",
		Prefix:    prefix,
	}
	// TODO - can this error?
	it := bucket.Objects(context.Background(), &q)
	if wg != nil {
		go postDay(wg, queue, it)
	} else {
		postDay(nil, queue, it)
	}
}

// month adds all of the files from dates within a specified month.
// It is difficult to test without hitting GCS.  8-(
func month(prefix string) {
	q := storage.Query{
		Delimiter: "/",
		// TODO - validate.
		Prefix: prefix,
	}
	it := bucket.Objects(context.Background(), &q)

	var wg sync.WaitGroup
	for o, err := it.Next(); err != iterator.Done; o, err = it.Next() {
		if err != nil {
			log.Println(err)
		} else if o.Prefix != "" {
			wg.Add(1)
			day(&wg, o.Prefix)
		} else {
			log.Println("Skipping: ", o.Name)
		}
	}
	wg.Wait()
}

func setup(fakeHTTP httpClientIntf) {
	if *fDryRun {
		httpClient = &dryRunHTTP{}
	}
	if fakeHTTP != nil {
		httpClient = fakeHTTP
	}

	storageClient, err := storage.NewClient(context.Background())
	if err != nil {
		log.Println(err)
		panic(err)
	}

	bucket = storageClient.Bucket(*fBucket)
	_, err = bucket.Attrs(context.Background())
	if err != nil {
		log.Println(err)
		panic(err)
	}
}

func run() {
	if *fMonth != "" {
		month(*fExper + "/" + *fMonth + "/")
	} else if *fDay != "" {
		day(nil, *fExper+"/"+*fDay+"/")
	}
}

func main() {
	flag.Parse()
	if *fProject == "" && !*fDryRun {
		log.Println("Must specify project (or --dry_run)")
		flag.PrintDefaults()
		return
	}

	setup(nil)
	run()
}
<commit_msg>Update header comment<commit_after>// Package main defines a command line tool for submitting date
// ranges for reprocessing
package main

/*
Strategies...
  1. Work from a month prefix, but explicitly iterate over days.
      maybe use a separate goroutine for each date? (DONE)
  2. Work from a prefix, or range of prefixes.
  3. Work from a date range

*/

import (
	"bytes"
	"context"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"sync"
	"time"

	"google.golang.org/api/iterator"

	"cloud.google.com/go/storage"
)

type httpClientIntf interface {
	Get(url string) (resp *http.Response, err error)
}

// This is used to intercept Get requests to the queue_pusher when invoked
// with -dry_run.
type dryRunHTTP struct{}

func (dr *dryRunHTTP) Get(url string) (resp *http.Response, err error) {
	resp = &http.Response{}
	resp.Body = ioutil.NopCloser(bytes.NewReader([]byte{}))
	resp.Status = "200 OK"
	resp.StatusCode = 200
	return
}

var (
	fProject = flag.String("project", "", "Project containing queues.")
	fQueue   = flag.String("queue", "etl-ndt-batch-", "Base of queue name.")
	// TODO implement listing queues to determine number of queue, and change this to 0
	fNumQueues = flag.Int("num_queues", 8, "Number of queues.  Normally determined by listing queues.")
	fBucket    = flag.String("bucket", "archive-mlab-oti", "Source bucket.")
	fExper     = flag.String("experiment", "ndt", "Experiment prefix, trailing slash optional")
	fMonth     = flag.String("month", "", "Single month spec, as YYYY/MM")
	fDay       = flag.String("day", "", "Single day spec, as YYYY/MM/DD")
	fDryRun    = flag.Bool("dry_run", false, "Prevents all output to queue_pusher.")

	bucket *storage.BucketHandle
	// Use an interface to allow test fake.
	httpClient httpClientIntf = http.DefaultClient
)

func init() {
	// Always prepend the filename and line number.
	log.SetFlags(log.LstdFlags | log.Lshortfile)
}

// postOne sends a single https request to the queue pusher to add a task.
// Iff dryRun is true, this does nothing.
func postOne(queue string, bucket string, fn string) error {
	reqStr := fmt.Sprintf("https://queue-pusher-dot-%s.appspot.com/receiver?queue=%s&filename=gs://%s/%s", *fProject, queue, bucket, fn)

	resp, err := httpClient.Get(reqStr)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return errors.New("http error: " + resp.Status)
	}

	return nil
}

// Post all items in an ObjectIterator into specific
// queue.
func postDay(wg *sync.WaitGroup, queue string, it *storage.ObjectIterator) {
	qpErrCount := 0
	gcsErrCount := 0
	fileCount := 0
	if wg != nil {
		defer wg.Done()
	}
	for o, err := it.Next(); err != iterator.Done; o, err = it.Next() {
		if err != nil {
			// TODO - should this retry?
			log.Println(err)
			if gcsErrCount > 3 {
				log.Printf("Failed after %d files to %s.\n", fileCount, queue)
				return
			}
			continue
		}

		err = postOne(queue, *fBucket, o.Name)
		if err != nil {
			// TODO - should this retry?
			log.Println(err)
			if qpErrCount > 3 {
				log.Printf("Failed after %d files to %s.\n", fileCount, queue)
				return
			}
		} else {
			fileCount++
		}
	}
	log.Println("Added ", fileCount, " tasks to ", queue)
}

// Initially this used a hash, but using day ordinal is better
// as it distributes across the queues more evenly.
func queueFor(date time.Time) string {
	day := date.Unix() / (24 * 60 * 60)
	return fmt.Sprintf("%s%d", *fQueue, int(day)%*fNumQueues)
}

// day fetches an iterator over the objects with ndt/YYYY/MM/DD prefix,
// and passes the iterator to postDay with appropriate queue.
// Iff wq is not nil, postDay will call done on wg when finished
// posting.
func day(wg *sync.WaitGroup, prefix string) {
	date, err := time.Parse("ndt/2006/01/02/", prefix)
	if err != nil {
		log.Println("Failed parsing date from ", prefix)
		log.Println(err)
		if wg != nil {
			wg.Done()
		}
		return
	}
	queue := queueFor(date)
	log.Println("Adding ", prefix, " to ", queue)
	q := storage.Query{
		Delimiter: "/",
		Prefix:    prefix,
	}
	// TODO - can this error?  Or do errors only occur on iterator ops?
	it := bucket.Objects(context.Background(), &q)
	if wg != nil {
		go postDay(wg, queue, it)
	} else {
		postDay(nil, queue, it)
	}
}

// month adds all of the files from dates within a specified month.
// It is difficult to test without hitting GCS.  8-(
func month(prefix string) {
	q := storage.Query{
		Delimiter: "/",
		// TODO - validate.
		Prefix: prefix,
	}
	it := bucket.Objects(context.Background(), &q)

	var wg sync.WaitGroup
	for o, err := it.Next(); err != iterator.Done; o, err = it.Next() {
		if err != nil {
			log.Println(err)
		} else if o.Prefix != "" {
			wg.Add(1)
			day(&wg, o.Prefix)
		} else {
			log.Println("Skipping: ", o.Name)
		}
	}
	wg.Wait()
}

func setup(fakeHTTP httpClientIntf) {
	if *fDryRun {
		httpClient = &dryRunHTTP{}
	}
	if fakeHTTP != nil {
		httpClient = fakeHTTP
	}

	storageClient, err := storage.NewClient(context.Background())
	if err != nil {
		log.Println(err)
		panic(err)
	}

	bucket = storageClient.Bucket(*fBucket)
	_, err = bucket.Attrs(context.Background())
	if err != nil {
		log.Println(err)
		panic(err)
	}
}

func run() {
	if *fMonth != "" {
		month(*fExper + "/" + *fMonth + "/")
	} else if *fDay != "" {
		day(nil, *fExper+"/"+*fDay+"/")
	}
}

func main() {
	flag.Parse()
	if *fProject == "" && !*fDryRun {
		log.Println("Must specify project (or --dry_run)")
		flag.PrintDefaults()
		return
	}

	setup(nil)
	run()
}
<|endoftext|>


<commit_before>// tinystat is used to compare two or more sets of measurements (e.g., runs of a
// multiple runs of benchmarks of two possible implementations) and determine if
// they are statistically different.
//
// Imagine we have the results of different animals' SAT scores. Each animal
// took the SATs multiple times, and we're assuming that differences between
// each animal's attempts are measurement error (i.e., normally distributed). We
// can test for differences as follows:
//
//     $ tinystat examples/iguana examples/chameleon examples/leopard
//
//     Filename            N  Min         Max         Median      Mean        StdDev
//     examples/iguana     7  50.000000   750.000000  200.000000  300.000000  238.047614
//     examples/chameleon  5  150.000000  930.000000  500.000000  540.000000  299.081929
//                         No difference proven at 95% confidence.
//     examples/leopard    6  700.000000  700.000000  700.000000  700.000000  0.000000
//                         Difference at 95% confidence!
//                            400.000000 +/- 215.283224
//                            57.142857% +/- 30.754746%
//                            (Student's t, pooled s = 175.809815)
//
// As you can see, despite the superficial differences between the iguana's
// scores and the chameleon's scores, there is no statistically significant
// difference between the two at a 95% confidence level. The leopard, on the
// other hand, has statistically significantly different scores.
package main

import (
	"encoding/csv"
	"fmt"
	"os"
	"strconv"
	"text/tabwriter"

	"github.com/codahale/tinystat"
	"github.com/docopt/docopt-go"
)

func main() {
	usage := `tinystat helps you conduct experiments.

Usage:
  tinystat [options] <control> [<experiment>...]

Arguments:
  <control>     Input file containing control measurements.
  <experiment>  Input file containing experimental measurements.

Options:
  -h --help               Display this usage information.
  -c --confidence=<pct>   Specify confidence level for analysis. [default: 95]
  -C --column=<num>       The column to analyze. [default: 0]
  -d --delimiter=(t|s|c)  Tab, space, or comma delimited data. [default: t]
`

	args, err := docopt.Parse(usage, nil, true, "", true)
	if err != nil {
		panic(err)
	}

	confidence, err := strconv.ParseFloat(args["--confidence"].(string), 64)
	if err != nil {
		panic(err)
	}

	column, err := strconv.Atoi(args["--column"].(string))
	if err != nil {
		panic(err)
	}

	var delimiter rune
	switch args["--delimiter"] {
	case "t":
		delimiter = '\t'
	case "c":
		delimiter = ','
	case "s":
		delimiter = ' '
	default:
		panic("bad delimiter")
	}

	controlFilename := args["<control>"].(string)
	experimentFilenames := args["<experiment>"].([]string)

	w := new(tabwriter.Writer)
	w.Init(os.Stdout, 2, 0, 2, ' ', 0)
	fmt.Fprintln(w, "Filename\tN\tMin\tMax\tMedian\tMean\tStdDev\t")

	control, err := readFile(controlFilename, column, delimiter)
	if err != nil {
		panic(err)
	}
	fmt.Fprintf(w,
		"%s\t%.0f\t%f\t%f\t%f\t%f\t%f\t\n",
		controlFilename,
		control.N,
		control.Min,
		control.Max,
		control.Median,
		control.Mean,
		control.StdDev,
	)

	for _, filename := range experimentFilenames {
		experimental, err := readFile(filename, column, delimiter)
		if err != nil {
			panic(err)
		}

		fmt.Fprintf(w,
			"%s\t%.0f\t%f\t%f\t%f\t%f\t%f\t\n",
			filename,
			experimental.N,
			experimental.Min,
			experimental.Max,
			experimental.Median,
			experimental.Mean,
			experimental.StdDev,
		)

		d := tinystat.Compare(control, experimental, confidence)
		if d.Significant() {
			fmt.Fprintf(w, "\tDifference at %v%% confidence!\n", confidence)
			fmt.Fprintf(w, "\t\t%10f +/- %10f\n", d.Delta, d.Error)
			fmt.Fprintf(w, "\t\t%9f%% +/- %9f%%\n", d.PctDelta, d.PctError)
			fmt.Fprintf(w, "\t\t(Student's t, pooled s = %f)\n", d.PooledStdDev)
		} else {
			fmt.Fprintf(w, "\tNo difference proven at %v%% confidence.\n", confidence)
		}
	}
	_ = w.Flush()
}

func readFile(filename string, col int, del rune) (tinystat.Summary, error) {
	f, err := os.Open(filename)
	if err != nil {
		return tinystat.Summary{}, err
	}
	defer f.Close()

	r := csv.NewReader(f)
	r.Comma = del
	records, err := r.ReadAll()
	if err != nil {
		return tinystat.Summary{}, err
	}

	data := make([]float64, len(records))
	for i, s := range records {
		n, err := strconv.ParseFloat(s[col], 64)
		if err != nil {
			return tinystat.Summary{}, err
		}
		data[i] = n
	}

	return tinystat.Summarize(data), nil
}
<commit_msg>Nicer error message.<commit_after>// tinystat is used to compare two or more sets of measurements (e.g., runs of a
// multiple runs of benchmarks of two possible implementations) and determine if
// they are statistically different.
//
// Imagine we have the results of different animals' SAT scores. Each animal
// took the SATs multiple times, and we're assuming that differences between
// each animal's attempts are measurement error (i.e., normally distributed). We
// can test for differences as follows:
//
//     $ tinystat examples/iguana examples/chameleon examples/leopard
//
//     Filename            N  Min         Max         Median      Mean        StdDev
//     examples/iguana     7  50.000000   750.000000  200.000000  300.000000  238.047614
//     examples/chameleon  5  150.000000  930.000000  500.000000  540.000000  299.081929
//                         No difference proven at 95% confidence.
//     examples/leopard    6  700.000000  700.000000  700.000000  700.000000  0.000000
//                         Difference at 95% confidence!
//                            400.000000 +/- 215.283224
//                            57.142857% +/- 30.754746%
//                            (Student's t, pooled s = 175.809815)
//
// As you can see, despite the superficial differences between the iguana's
// scores and the chameleon's scores, there is no statistically significant
// difference between the two at a 95% confidence level. The leopard, on the
// other hand, has statistically significantly different scores.
package main

import (
	"encoding/csv"
	"fmt"
	"os"
	"strconv"
	"text/tabwriter"

	"github.com/codahale/tinystat"
	"github.com/docopt/docopt-go"
)

func main() {
	usage := `tinystat helps you conduct experiments.

Usage:
  tinystat [options] <control> [<experiment>...]

Arguments:
  <control>     Input file containing control measurements.
  <experiment>  Input file containing experimental measurements.

Options:
  -h --help               Display this usage information.
  -c --confidence=<pct>   Specify confidence level for analysis. [default: 95]
  -C --column=<num>       The column to analyze. [default: 0]
  -d --delimiter=(t|s|c)  Tab, space, or comma delimited data. [default: t]
`

	args, err := docopt.Parse(usage, nil, true, "", true)
	if err != nil {
		panic(err)
	}

	confidence, err := strconv.ParseFloat(args["--confidence"].(string), 64)
	if err != nil {
		panic(err)
	}

	column, err := strconv.Atoi(args["--column"].(string))
	if err != nil {
		panic(err)
	}

	var delimiter rune
	switch d := args["--delimiter"]; d {
	case "t":
		delimiter = '\t'
	case "c":
		delimiter = ','
	case "s":
		delimiter = ' '
	default:
		panic(fmt.Errorf("bad delimiter: %#v", d))
	}

	controlFilename := args["<control>"].(string)
	experimentFilenames := args["<experiment>"].([]string)

	w := new(tabwriter.Writer)
	w.Init(os.Stdout, 2, 0, 2, ' ', 0)
	fmt.Fprintln(w, "Filename\tN\tMin\tMax\tMedian\tMean\tStdDev\t")

	control, err := readFile(controlFilename, column, delimiter)
	if err != nil {
		panic(err)
	}
	fmt.Fprintf(w,
		"%s\t%.0f\t%f\t%f\t%f\t%f\t%f\t\n",
		controlFilename,
		control.N,
		control.Min,
		control.Max,
		control.Median,
		control.Mean,
		control.StdDev,
	)

	for _, filename := range experimentFilenames {
		experimental, err := readFile(filename, column, delimiter)
		if err != nil {
			panic(err)
		}

		fmt.Fprintf(w,
			"%s\t%.0f\t%f\t%f\t%f\t%f\t%f\t\n",
			filename,
			experimental.N,
			experimental.Min,
			experimental.Max,
			experimental.Median,
			experimental.Mean,
			experimental.StdDev,
		)

		d := tinystat.Compare(control, experimental, confidence)
		if d.Significant() {
			fmt.Fprintf(w, "\tDifference at %v%% confidence!\n", confidence)
			fmt.Fprintf(w, "\t\t%10f +/- %10f\n", d.Delta, d.Error)
			fmt.Fprintf(w, "\t\t%9f%% +/- %9f%%\n", d.PctDelta, d.PctError)
			fmt.Fprintf(w, "\t\t(Student's t, pooled s = %f)\n", d.PooledStdDev)
		} else {
			fmt.Fprintf(w, "\tNo difference proven at %v%% confidence.\n", confidence)
		}
	}
	_ = w.Flush()
}

func readFile(filename string, col int, del rune) (tinystat.Summary, error) {
	f, err := os.Open(filename)
	if err != nil {
		return tinystat.Summary{}, err
	}
	defer f.Close()

	r := csv.NewReader(f)
	r.Comma = del
	records, err := r.ReadAll()
	if err != nil {
		return tinystat.Summary{}, err
	}

	data := make([]float64, len(records))
	for i, s := range records {
		n, err := strconv.ParseFloat(s[col], 64)
		if err != nil {
			return tinystat.Summary{}, err
		}
		data[i] = n
	}

	return tinystat.Summarize(data), nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"

	difflib "github.com/kylelemons/godebug/diff"
	"github.com/nochso/tocenize"
)

var VERSION = "?"

func main() {
	log.SetFlags(0)
	flag.Usage = func() {
		fmt.Println("tocenize [options] FILE...")
		fmt.Println()
		flag.PrintDefaults()
	}
	job := tocenize.Job{}
	flag.IntVar(&job.MinDepth, "min", 1, "minimum depth")
	flag.IntVar(&job.MaxDepth, "max", 99, "maximum depth")
	flag.StringVar(&tocenize.Indent, "indent", "\t", "string used for nesting")
	doDiff := flag.Bool("d", false, "print full diff to stdout")
	doPrint := flag.Bool("p", false, "print full result to stdout")
	flag.BoolVar(&job.ExistingOnly, "e", false, "update only existing TOC (no insert)")
	showVersion := flag.Bool("v", false, "print version")
	flag.Parse()

	if *showVersion {
		fmt.Println(VERSION)
		os.Exit(0)
	}
	if flag.NArg() == 0 {
		log.Println("too few arguments")
		flag.Usage()
		os.Exit(2)
	}

	action := update
	if *doDiff {
		action = diff
	}
	if *doPrint {
		action = print
	}

	for _, arg := range flag.Args() {
		paths, err := filepath.Glob(arg)
		if err != nil {
			log.Println(err)
			continue
		}

		for _, path := range paths {
			log.SetPrefix(path + ": ")

			doc, err := tocenize.NewDocument(path)
			if err != nil {
				log.Println(err)
				continue
			}
			toc := tocenize.NewTOC(doc, job)
			newDoc, err := doc.Update(toc, job.ExistingOnly)
			if err != nil {
				log.Println(err)
				continue
			}
			action(job, doc, newDoc)
			if err != nil {
				log.Println(err)
			}
		}
		log.SetPrefix("")
	}
}

func diff(job tocenize.Job, a, b tocenize.Document) error {
	log.Println()
	d := difflib.Diff(a.String(), b.String())
	if d != "" {
		fmt.Println(d)
	}
	return nil
}

func print(job tocenize.Job, a, b tocenize.Document) error {
	fmt.Println(b.String())
	return nil
}

func update(job tocenize.Job, a, b tocenize.Document) error {
	return ioutil.WriteFile(b.Path, []byte(b.String()), 0644)
}
<commit_msg>Extract actionFunc and runAction<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"

	difflib "github.com/kylelemons/godebug/diff"
	"github.com/nochso/tocenize"
)

var VERSION = "?"

func main() {
	log.SetFlags(0)
	flag.Usage = func() {
		fmt.Println("tocenize [options] FILE...")
		fmt.Println()
		flag.PrintDefaults()
	}
	job := tocenize.Job{}
	flag.IntVar(&job.MinDepth, "min", 1, "minimum depth")
	flag.IntVar(&job.MaxDepth, "max", 99, "maximum depth")
	flag.StringVar(&tocenize.Indent, "indent", "\t", "string used for nesting")
	doDiff := flag.Bool("d", false, "print full diff to stdout")
	doPrint := flag.Bool("p", false, "print full result to stdout")
	flag.BoolVar(&job.ExistingOnly, "e", false, "update only existing TOC (no insert)")
	showVersion := flag.Bool("v", false, "print version")
	flag.Parse()

	if *showVersion {
		fmt.Println(VERSION)
		os.Exit(0)
	}
	if flag.NArg() == 0 {
		log.Println("too few arguments")
		flag.Usage()
		os.Exit(2)
	}

	action := update
	if *doDiff {
		action = diff
	}
	if *doPrint {
		action = print
	}

	for _, arg := range flag.Args() {
		paths, err := filepath.Glob(arg)
		if err != nil {
			log.Println(err)
			continue
		}

		for _, path := range paths {
			log.SetPrefix(path + ": ")
			err = runAction(path, job, action)
			if err != nil {
				log.Println(err)
			}
		}
		log.SetPrefix("")
	}
}

type actionFunc func(job tocenize.Job, a, b tocenize.Document) error

func runAction(path string, job tocenize.Job, action actionFunc) error {
	doc, err := tocenize.NewDocument(path)
	if err != nil {
		return err
	}
	toc := tocenize.NewTOC(doc, job)
	newDoc, err := doc.Update(toc, job.ExistingOnly)
	if err != nil {
		return err
	}
	return action(job, doc, newDoc)
}

func diff(job tocenize.Job, a, b tocenize.Document) error {
	log.Println()
	d := difflib.Diff(a.String(), b.String())
	if d != "" {
		fmt.Println(d)
	}
	return nil
}

func print(job tocenize.Job, a, b tocenize.Document) error {
	fmt.Println(b.String())
	return nil
}

func update(job tocenize.Job, a, b tocenize.Document) error {
	return ioutil.WriteFile(b.Path, []byte(b.String()), 0644)
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"net/http"

	"os"
	"os/signal"
	"syscall"

	"github.com/antihax/evedata/internal/redigohelper"
	"github.com/antihax/evedata/internal/sqlhelper"
	"github.com/antihax/evedata/services/vanguard"
	"github.com/antihax/evedata/services/vanguard/models"
	_ "github.com/antihax/evedata/services/vanguard/views"
	"github.com/garyburd/redigo/redis"
	"github.com/gorilla/context"
)

func main() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)
	log.SetPrefix("evedata vanguard: ")

	r := redigohelper.ConnectRedisProdPool()
	db := sqlhelper.NewDatabase()

	// Make a new service and send it into the background.
	vanguard := vanguard.NewVanguard(r, db)
	log.Printf("Setup Router\n")
	rtr := vanguard.NewRouter()
	defer vanguard.Close()

	// Handle command line arguments
	if len(os.Args) > 1 {
		if os.Args[1] == "dumpdb" {
			// Dump the database to sql file.
			log.Printf("Dumping Database to evedata.sql\n")
			err := models.DumpDatabase("./sql/evedata.sql", "evedata")
			if err != nil {
				log.Fatalln(err)
			}
		} else if os.Args[1] == "flushcache" {
			// Erase http cache in redis
			log.Printf("Flushing Redis\n")
			conn := r.Get()
			defer conn.Close()
			keys, err := redis.Strings(conn.Do("KEYS", "*rediscache*"))
			if err != nil {
				log.Println(err)
			} else {
				for _, key := range keys {
					conn.Do("DEL", key)
					log.Printf("Deleting %s\n", key)
				}
			}
		} else if os.Args[1] == "flushstructures" {
			// Erase http cache in redis
			log.Printf("Flushing structures\n")
			conn := r.Get()
			defer conn.Close()
			keys, err := redis.Strings(conn.Do("KEYS", "*structure_failure*"))
			if err != nil {
				log.Println(err)
			} else {
				for _, key := range keys {
					conn.Do("DEL", key)
					log.Printf("Deleting %s\n", key)
				}
			}

		} else if os.Args[1] == "flushredis" {
			// Erase everything in redis for modified deployments
			log.Printf("Flushing Redis\n")
			conn := r.Get()
			defer conn.Close()
			conn.Do("FLUSHALL")
		}
	}

	log.Printf("Start Listening\n")
	go log.Fatalln(http.ListenAndServe(":3000", context.ClearHandler(rtr)))

	log.Printf("In production\n")
	// Handle SIGINT and SIGTERM.
	ch := make(chan os.Signal)
	signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
	log.Println(<-ch)
}
<commit_msg>add flush kills<commit_after>package main

import (
	"log"
	"net/http"

	"os"
	"os/signal"
	"syscall"

	"github.com/antihax/evedata/internal/redigohelper"
	"github.com/antihax/evedata/internal/sqlhelper"
	"github.com/antihax/evedata/services/vanguard"
	"github.com/antihax/evedata/services/vanguard/models"
	_ "github.com/antihax/evedata/services/vanguard/views"
	"github.com/garyburd/redigo/redis"
	"github.com/gorilla/context"
)

func main() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)
	log.SetPrefix("evedata vanguard: ")

	r := redigohelper.ConnectRedisProdPool()
	db := sqlhelper.NewDatabase()

	// Make a new service and send it into the background.
	vanguard := vanguard.NewVanguard(r, db)
	log.Printf("Setup Router\n")
	rtr := vanguard.NewRouter()
	defer vanguard.Close()

	// Handle command line arguments
	if len(os.Args) > 1 {
		if os.Args[1] == "dumpdb" {
			// Dump the database to sql file.
			log.Printf("Dumping Database to evedata.sql\n")
			err := models.DumpDatabase("./sql/evedata.sql", "evedata")
			if err != nil {
				log.Fatalln(err)
			}
		} else if os.Args[1] == "flushcache" {
			// Erase http cache in redis
			log.Printf("Flushing Redis\n")
			conn := r.Get()
			defer conn.Close()
			keys, err := redis.Strings(conn.Do("KEYS", "*rediscache*"))
			if err != nil {
				log.Println(err)
			} else {
				for _, key := range keys {
					conn.Do("DEL", key)
					log.Printf("Deleting %s\n", key)
				}
			}
		} else if os.Args[1] == "flushstructures" {
			// Erase http cache in redis
			log.Printf("Flushing structures\n")
			conn := r.Get()
			defer conn.Close()
			keys, err := redis.Strings(conn.Do("KEYS", "*structure_failure*"))
			if err != nil {
				log.Println(err)
			} else {
				for _, key := range keys {
					conn.Do("DEL", key)
					log.Printf("Deleting %s\n", key)
				}
			}
		} else if os.Args[1] == "flushkills" {
			// Erase http cache in redis
			log.Printf("Flushing killmails\n")
			conn := r.Get()
			defer conn.Close()
			i, err := redis.Int64(conn.Do("DEL", "evedata_known_kills"))
			log.Printf("%d %s\n", i, err)
		} else if os.Args[1] == "flushredis" {
			// Erase everything in redis for modified deployments
			log.Printf("Flushing Redis\n")
			conn := r.Get()
			defer conn.Close()
			conn.Do("FLUSHALL")
		}
	}

	log.Printf("Start Listening\n")
	go log.Fatalln(http.ListenAndServe(":3000", context.ClearHandler(rtr)))

	log.Printf("In production\n")
	// Handle SIGINT and SIGTERM.
	ch := make(chan os.Signal)
	signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
	log.Println(<-ch)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
)

// Client for the deployment service
type Client struct {
	token   string
	baseURL string
}

// NewClient makes a new Client
func NewClient(token, baseURL string) Client {
	return Client{
		token:   token,
		baseURL: baseURL,
	}
}

func (c Client) newRequest(method, path string, body io.Reader) (*http.Request, error) {
	req, err := http.NewRequest(method, c.baseURL+path, body)
	if err != nil {
		return nil, err
	}
	req.Header.Add("Authorization", fmt.Sprintf("Scope-Probe token=%s", c.token))
	return req, nil
}

// Deploy notifies the deployment service about a new deployment
func (c Client) Deploy(deployment Deployment) error {
	var buf bytes.Buffer
	if err := json.NewEncoder(&buf).Encode(deployment); err != nil {
		return err
	}
	req, err := c.newRequest("POST", "/api/deploy/deploy", &buf)
	if err != nil {
		return err
	}
	res, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	if res.StatusCode != 204 {
		return fmt.Errorf("Error making request: %s", res.Status)
	}
	return nil
}

// GetDeployments returns a list of deployments
func (c Client) GetDeployments(from, through int64) ([]Deployment, error) {
	req, err := c.newRequest("GET", fmt.Sprintf("/api/deploy/deploy?from=%d&through=%d", from, through), nil)
	if err != nil {
		return nil, err
	}
	res, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	if res.StatusCode != 200 {
		return nil, fmt.Errorf("Error making request: %s", res.Status)
	}
	var response struct {
		Deployments []Deployment `json:"deployments"`
	}
	if err := json.NewDecoder(res.Body).Decode(&response); err != nil {
		return nil, err
	}
	return response.Deployments, nil
}

// GetEvents returns the raw events.
func (c Client) GetEvents(from, through int64) ([]byte, error) {
	req, err := c.newRequest("GET", fmt.Sprintf("/api/deploy/event?from=%d&through=%d", from, through), nil)
	if err != nil {
		return nil, err
	}
	res, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	if res.StatusCode != 200 {
		return nil, fmt.Errorf("Error making request: %s", res.Status)
	}
	return ioutil.ReadAll(res.Body)
}

// GetConfig returns the current Config
func (c Client) GetConfig() (*Config, error) {
	req, err := c.newRequest("GET", "/api/config/deploy", nil)
	if err != nil {
		return nil, err
	}
	res, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	if res.StatusCode == 404 {
		return nil, fmt.Errorf("No configuration uploaded yet.")
	}
	if res.StatusCode != 200 {
		return nil, fmt.Errorf("Error making request: %s", res.Status)
	}
	var config Config
	if err := json.NewDecoder(res.Body).Decode(&config); err != nil {
		return nil, err
	}
	return &config, nil
}

// SetConfig sets the current Config
func (c Client) SetConfig(config *Config) error {
	var buf bytes.Buffer
	if err := json.NewEncoder(&buf).Encode(config); err != nil {
		return err
	}
	req, err := c.newRequest("POST", "/api/config/deploy", &buf)
	if err != nil {
		return err
	}
	res, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	if res.StatusCode != 204 {
		return fmt.Errorf("Error making request: %s", res.Status)
	}
	return nil
}

// GetLogs returns the logs for a given deployment.
func (c Client) GetLogs(deployID string) ([]byte, error) {
	req, err := c.newRequest("GET", fmt.Sprintf("/api/deploy/%s/log", deployID), nil)
	if err != nil {
		return nil, err
	}
	res, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	if res.StatusCode != 200 {
		return nil, fmt.Errorf("Error making request: %s", res.Status)
	}
	return ioutil.ReadAll(res.Body)
}
<commit_msg>Fix logs path<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
)

// Client for the deployment service
type Client struct {
	token   string
	baseURL string
}

// NewClient makes a new Client
func NewClient(token, baseURL string) Client {
	return Client{
		token:   token,
		baseURL: baseURL,
	}
}

func (c Client) newRequest(method, path string, body io.Reader) (*http.Request, error) {
	req, err := http.NewRequest(method, c.baseURL+path, body)
	if err != nil {
		return nil, err
	}
	req.Header.Add("Authorization", fmt.Sprintf("Scope-Probe token=%s", c.token))
	return req, nil
}

// Deploy notifies the deployment service about a new deployment
func (c Client) Deploy(deployment Deployment) error {
	var buf bytes.Buffer
	if err := json.NewEncoder(&buf).Encode(deployment); err != nil {
		return err
	}
	req, err := c.newRequest("POST", "/api/deploy/deploy", &buf)
	if err != nil {
		return err
	}
	res, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	if res.StatusCode != 204 {
		return fmt.Errorf("Error making request: %s", res.Status)
	}
	return nil
}

// GetDeployments returns a list of deployments
func (c Client) GetDeployments(from, through int64) ([]Deployment, error) {
	req, err := c.newRequest("GET", fmt.Sprintf("/api/deploy/deploy?from=%d&through=%d", from, through), nil)
	if err != nil {
		return nil, err
	}
	res, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	if res.StatusCode != 200 {
		return nil, fmt.Errorf("Error making request: %s", res.Status)
	}
	var response struct {
		Deployments []Deployment `json:"deployments"`
	}
	if err := json.NewDecoder(res.Body).Decode(&response); err != nil {
		return nil, err
	}
	return response.Deployments, nil
}

// GetEvents returns the raw events.
func (c Client) GetEvents(from, through int64) ([]byte, error) {
	req, err := c.newRequest("GET", fmt.Sprintf("/api/deploy/event?from=%d&through=%d", from, through), nil)
	if err != nil {
		return nil, err
	}
	res, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	if res.StatusCode != 200 {
		return nil, fmt.Errorf("Error making request: %s", res.Status)
	}
	return ioutil.ReadAll(res.Body)
}

// GetConfig returns the current Config
func (c Client) GetConfig() (*Config, error) {
	req, err := c.newRequest("GET", "/api/config/deploy", nil)
	if err != nil {
		return nil, err
	}
	res, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	if res.StatusCode == 404 {
		return nil, fmt.Errorf("No configuration uploaded yet.")
	}
	if res.StatusCode != 200 {
		return nil, fmt.Errorf("Error making request: %s", res.Status)
	}
	var config Config
	if err := json.NewDecoder(res.Body).Decode(&config); err != nil {
		return nil, err
	}
	return &config, nil
}

// SetConfig sets the current Config
func (c Client) SetConfig(config *Config) error {
	var buf bytes.Buffer
	if err := json.NewEncoder(&buf).Encode(config); err != nil {
		return err
	}
	req, err := c.newRequest("POST", "/api/config/deploy", &buf)
	if err != nil {
		return err
	}
	res, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	if res.StatusCode != 204 {
		return fmt.Errorf("Error making request: %s", res.Status)
	}
	return nil
}

// GetLogs returns the logs for a given deployment.
func (c Client) GetLogs(deployID string) ([]byte, error) {
	req, err := c.newRequest("GET", fmt.Sprintf("/api/deploy/deploy/%s/log", deployID), nil)
	if err != nil {
		return nil, err
	}
	res, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	if res.StatusCode != 200 {
		return nil, fmt.Errorf("Error making request: %s", res.Status)
	}
	return ioutil.ReadAll(res.Body)
}
<|endoftext|>


<commit_before>package cmd

import (
	"github.com/qiniu/qshell/v2/cmd_test/test"
	"strings"
	"testing"
)

func TestPFop(t *testing.T) {
	result, errs := test.RunCmdWithError("pfop", "qshell-na0", "test_mv.mp4", "avthumb/mp4")
	if len(errs) > 0{
		t.Fail()
	}
	result = strings.ReplaceAll(result, "\n", "")
	result, errs = test.RunCmdWithError("prefop", result)
	if len(errs) > 0{
		t.Fail()
	}
}
<commit_msg>add fop test case<commit_after>package cmd

import (
	"github.com/qiniu/qshell/v2/cmd_test/test"
	"strings"
	"testing"
)

func TestFop(t *testing.T) {
	result, errs := test.RunCmdWithError("pfop", "qshell-na0", "test_mv.mp4", "avthumb/mp4")
	if len(errs) > 0{
		t.Fail()
	}

	result = strings.ReplaceAll(result, "\n", "")
	result, errs = test.RunCmdWithError("prefop", result)
	if len(errs) > 0{
		t.Fail()
	}
}
<|endoftext|>