<commit_before>package main

import (
	"encoding/json"
	"log"
	"net/http"
	"os"
	"strconv"
	"time"

	"fmt"

	"github.com/danielfireman/acessoatodos/maps"
	"github.com/danielfireman/acessoatodos/placesdb"
	"github.com/julienschmidt/httprouter"
	"github.com/newrelic/go-agent"
)

const (
	maxRPS                   = 10
	nearbySearchRadiusMeters = 100
	opsTimeout               = 50 * time.Second
	limit                    = 200
)

type PostAccessibilityRequest struct {
	Accessibility []string `json:"accessibility"`
}

type NearbySearchResponse struct {
	Results []NearbySearchResult `json:"results"`
}

type GetPlaceResult struct {
	Name              string   `json:"name"`
	Location          LatLng   `json:"loc"`
	Accessibility     []string `json:"accessibility"`
	GoogleMapsPlaceID string   `json:"gmplaceid"`
}

type NearbySearchResult struct {
	Name              string   `json:"name"`
	Location          LatLng   `json:"loc"`
	Accessibility     []string `json:"accessibility"`
	GoogleMapsPlaceID string   `json:"gmplaceid"`
}

type LatLng struct {
	Lat float64 `json:"lat"`
	Lng float64 `json:"lng"`
}

func main() {
	// Environment validation. We would like to fail fast.
	port := os.Getenv("PORT")
	if port == "" {
		log.Fatal("$PORT env var is mandatory")
	}

	// Google Maps initialization.
	gMapsClient, err := maps.DialGoogle(os.Getenv("GOOGLE_MAPS_KEY"), opsTimeout, maxRPS, limit)
	if err != nil {
		log.Fatal(err)
	}

	// DB initialization.
	db, err := placesdb.Dial(os.Getenv("MONGODB_URI"), opsTimeout, maxRPS, limit)
	if err != nil {
		log.Fatal(err)
	}

	// New relic initialization.
	nrLicence := os.Getenv("NEW_RELIC_LICENSE_KEY")
	if nrLicence == "" {
		log.Fatal("$NEW_RELIC_LICENSE_KEY must be set")
	}
	config := newrelic.NewConfig("acessoatodos", nrLicence)
	app, err := newrelic.NewApplication(config)
	if err != nil {
		log.Fatal(err)
	}
	log.Println("NewRelic monitoring successfully initiated..")

	router := httprouter.New()
	router.GET("/", func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
		http.Redirect(w, r, "//github.com/danielfireman/acessoatodos", http.StatusTemporaryRedirect)
	})

	router.GET("/api/v1/nearby", func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
		txn := app.StartTransaction("NearbySerch", w, r)
		defer txn.End()

		query := r.URL.Query()
		lat, err := strconv.ParseFloat(query.Get("lat"), 64)
		if err != nil {
			log.Printf("Error processing request: %q", err)
			http.Error(w, "Invalid lat param", http.StatusBadRequest)
			return
		}
		lng, err := strconv.ParseFloat(query.Get("lng"), 64)
		if err != nil {
			log.Printf("Error processing request: %q", err)
			http.Error(w, "Invalid lng param", http.StatusBadRequest)
			return
		}

		// Asynchronously sends a request to google maps.
		gMapsChan := gMapsClient.NearbySearch(txn, lat, lng, nearbySearchRadiusMeters)

		// Placing db results in a map to ease merging results.
		dbRes, err := db.NearbySearch(txn, lat, lng, nearbySearchRadiusMeters)
		if err != nil {
			// NOTE: Deliberately let the request advance even when we don't had an error trying to fetch
			// accessibility.
			// TODO(danielfireman): Log request.
			log.Printf("Error fetching data from database: %q", err)
		}

		dbResMap := make(map[string]*placesdb.Place, len(dbRes))
		for _, r := range dbRes {
			dbResMap[r.GoogleMapsPlaceID] = &r
		}

		gMapsResp := <-gMapsChan
		if gMapsResp.Error != nil {
			// TODO(danielfireman): Log request
			log.Printf("Error fetching data from GMaps: %q", err)
			http.Error(w, "Invalid lng param", http.StatusInternalServerError)
			return
		}
		var results []NearbySearchResult
		for _, r := range gMapsResp.Results {
			result := NearbySearchResult{
				GoogleMapsPlaceID: r.ID,
				Location: LatLng{
					Lat: r.Lat,
					Lng: r.Lng,
				},
				Name: r.Name,
			}
			dbEntry, ok := dbResMap[result.GoogleMapsPlaceID]
			if ok {
				result.Accessibility = dbEntry.Accessibility
			}
			results = append(results, result)
		}

		w.Header().Set("Content-Type", "application/json")
		if err := json.NewEncoder(w).Encode(NearbySearchResponse{Results: results}); err != nil {
			log.Printf("Error marshaling response: %q", err)
			http.Error(w, "Problems marshaling response.", http.StatusInternalServerError)
		}
	})

	router.PUT("/api/v1/place/:id", func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
		txn := app.StartTransaction("PutPlace", w, r)
		defer txn.End()

		placeID, err := placeID(ps.ByName("id"))
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}

		// Triggering Google Maps request asynchronously.
		gMapsChan := gMapsClient.Get(txn, placeID)

		// Decode request body.
		decoder := json.NewDecoder(r.Body)
		var p PostAccessibilityRequest
		if err := decoder.Decode(&p); err != nil {
			log.Printf("Error decoding request body: %q", err)
			http.Error(w, "Invalid input.", http.StatusBadRequest)
			return
		}
		defer r.Body.Close()

		// Block waiting for Google Maps data.
		gMapsResp := <-gMapsChan
		if gMapsResp.Error != nil {
			log.Printf("Error fetching place details from google maps: %v", err)
			http.Error(w, "Error fetching place details from Google Maps.", http.StatusInternalServerError)
			return
		}
		if len(gMapsResp.Results) == 0 {
			http.Error(w, "Place not found", http.StatusNotFound)
			return
		}
		// Updating database object with information from google maps and from request.
		placeDetails := gMapsResp.Results[0]
		lat := placeDetails.Lat
		lng := placeDetails.Lng
		if err := db.Put(txn, placeID, p.Accessibility, lat, lng); err != nil {
			log.Printf("Error insertind record on database: %q", err)
			http.Error(w, "Problems inserting into database response.", http.StatusInternalServerError)
			return
		}

		w.Header().Set("Content-Type", "application/json")
		w.WriteHeader(http.StatusCreated)
		if err := json.NewEncoder(w).Encode(p); err != nil {
			log.Printf("Error marshaling response: %q", err)
			http.Error(w, "Problems marshaling response.", http.StatusInternalServerError)
		}
	})

	router.GET("/api/v1/place/:id", func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
		txn := app.StartTransaction("GetPlace", w, r)
		defer txn.End()

		placeID, err := placeID(ps.ByName("id"))
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}

		// Triggering Google Maps request asynchronously.
		gMapsChan := gMapsClient.Get(txn, placeID)

		// Fetches data from DB.
		dbPlace, err := db.Get(txn, placeID)
		if err != nil {
			// NOTE: Deliberately let the request advance even when we don't had an error trying to fetch
			// accessibility.
			log.Printf("Error fetching place (placeID:%s) details from database: %v", placeID, err)
		}

		// Block waiting for Google Maps data.
		gMapsResp := <-gMapsChan
		if gMapsResp.Error != nil {
			log.Printf("Error fetching place details from google maps: %v", err)
			http.Error(w, "Error fetching place details from Google Maps.", http.StatusServiceUnavailable)
			return
		}
		if len(gMapsResp.Results) == 0 {
			http.Error(w, "Place not found", http.StatusNotFound)
			return
		}
		placeDetails := gMapsResp.Results[0]
		result := GetPlaceResult{
			GoogleMapsPlaceID: placeDetails.ID,
			Location: LatLng{
				Lat: placeDetails.Lat,
				Lng: placeDetails.Lng,
			},
			Name:          placeDetails.Name,
			Accessibility: dbPlace.Accessibility,
		}
		w.Header().Set("Content-Type", "application/json")
		if err := json.NewEncoder(w).Encode(result); err != nil {
			log.Printf("Error marshaling response: %q", err)
			http.Error(w, "Problems marshaling response.", http.StatusInternalServerError)
		}

	})
	log.Println("Service listening at port ", port)
	log.Fatal(http.ListenAndServe(":"+port, router))
}

func placeID(placeID string) (string, error) {
	if placeID == "" || len(placeID) <= 3 {
		return "", fmt.Errorf("Invalid PlaceID:%s", placeID)
	}
	return placeID[4:], nil
}
<commit_msg>Removing blank line. Triggering travis CI<commit_after>package main

import (
	"encoding/json"
	"log"
	"net/http"
	"os"
	"strconv"
	"time"
	"fmt"

	"github.com/danielfireman/acessoatodos/maps"
	"github.com/danielfireman/acessoatodos/placesdb"
	"github.com/julienschmidt/httprouter"
	"github.com/newrelic/go-agent"
)

const (
	maxRPS                   = 10
	nearbySearchRadiusMeters = 100
	opsTimeout               = 50 * time.Second
	limit                    = 200
)

type PostAccessibilityRequest struct {
	Accessibility []string `json:"accessibility"`
}

type NearbySearchResponse struct {
	Results []NearbySearchResult `json:"results"`
}

type GetPlaceResult struct {
	Name              string   `json:"name"`
	Location          LatLng   `json:"loc"`
	Accessibility     []string `json:"accessibility"`
	GoogleMapsPlaceID string   `json:"gmplaceid"`
}

type NearbySearchResult struct {
	Name              string   `json:"name"`
	Location          LatLng   `json:"loc"`
	Accessibility     []string `json:"accessibility"`
	GoogleMapsPlaceID string   `json:"gmplaceid"`
}

type LatLng struct {
	Lat float64 `json:"lat"`
	Lng float64 `json:"lng"`
}

func main() {
	// Environment validation. We would like to fail fast.
	port := os.Getenv("PORT")
	if port == "" {
		log.Fatal("$PORT env var is mandatory")
	}

	// Google Maps initialization.
	gMapsClient, err := maps.DialGoogle(os.Getenv("GOOGLE_MAPS_KEY"), opsTimeout, maxRPS, limit)
	if err != nil {
		log.Fatal(err)
	}

	// DB initialization.
	db, err := placesdb.Dial(os.Getenv("MONGODB_URI"), opsTimeout, maxRPS, limit)
	if err != nil {
		log.Fatal(err)
	}

	// New relic initialization.
	nrLicence := os.Getenv("NEW_RELIC_LICENSE_KEY")
	if nrLicence == "" {
		log.Fatal("$NEW_RELIC_LICENSE_KEY must be set")
	}
	config := newrelic.NewConfig("acessoatodos", nrLicence)
	app, err := newrelic.NewApplication(config)
	if err != nil {
		log.Fatal(err)
	}
	log.Println("NewRelic monitoring successfully initiated..")

	router := httprouter.New()
	router.GET("/", func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
		http.Redirect(w, r, "//github.com/danielfireman/acessoatodos", http.StatusTemporaryRedirect)
	})

	router.GET("/api/v1/nearby", func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
		txn := app.StartTransaction("NearbySerch", w, r)
		defer txn.End()

		query := r.URL.Query()
		lat, err := strconv.ParseFloat(query.Get("lat"), 64)
		if err != nil {
			log.Printf("Error processing request: %q", err)
			http.Error(w, "Invalid lat param", http.StatusBadRequest)
			return
		}
		lng, err := strconv.ParseFloat(query.Get("lng"), 64)
		if err != nil {
			log.Printf("Error processing request: %q", err)
			http.Error(w, "Invalid lng param", http.StatusBadRequest)
			return
		}

		// Asynchronously sends a request to google maps.
		gMapsChan := gMapsClient.NearbySearch(txn, lat, lng, nearbySearchRadiusMeters)

		// Placing db results in a map to ease merging results.
		dbRes, err := db.NearbySearch(txn, lat, lng, nearbySearchRadiusMeters)
		if err != nil {
			// NOTE: Deliberately let the request advance even when we don't had an error trying to fetch
			// accessibility.
			// TODO(danielfireman): Log request.
			log.Printf("Error fetching data from database: %q", err)
		}

		dbResMap := make(map[string]*placesdb.Place, len(dbRes))
		for _, r := range dbRes {
			dbResMap[r.GoogleMapsPlaceID] = &r
		}

		gMapsResp := <-gMapsChan
		if gMapsResp.Error != nil {
			// TODO(danielfireman): Log request
			log.Printf("Error fetching data from GMaps: %q", err)
			http.Error(w, "Invalid lng param", http.StatusInternalServerError)
			return
		}
		var results []NearbySearchResult
		for _, r := range gMapsResp.Results {
			result := NearbySearchResult{
				GoogleMapsPlaceID: r.ID,
				Location: LatLng{
					Lat: r.Lat,
					Lng: r.Lng,
				},
				Name: r.Name,
			}
			dbEntry, ok := dbResMap[result.GoogleMapsPlaceID]
			if ok {
				result.Accessibility = dbEntry.Accessibility
			}
			results = append(results, result)
		}

		w.Header().Set("Content-Type", "application/json")
		if err := json.NewEncoder(w).Encode(NearbySearchResponse{Results: results}); err != nil {
			log.Printf("Error marshaling response: %q", err)
			http.Error(w, "Problems marshaling response.", http.StatusInternalServerError)
		}
	})

	router.PUT("/api/v1/place/:id", func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
		txn := app.StartTransaction("PutPlace", w, r)
		defer txn.End()

		placeID, err := placeID(ps.ByName("id"))
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}

		// Triggering Google Maps request asynchronously.
		gMapsChan := gMapsClient.Get(txn, placeID)

		// Decode request body.
		decoder := json.NewDecoder(r.Body)
		var p PostAccessibilityRequest
		if err := decoder.Decode(&p); err != nil {
			log.Printf("Error decoding request body: %q", err)
			http.Error(w, "Invalid input.", http.StatusBadRequest)
			return
		}
		defer r.Body.Close()

		// Block waiting for Google Maps data.
		gMapsResp := <-gMapsChan
		if gMapsResp.Error != nil {
			log.Printf("Error fetching place details from google maps: %v", err)
			http.Error(w, "Error fetching place details from Google Maps.", http.StatusInternalServerError)
			return
		}
		if len(gMapsResp.Results) == 0 {
			http.Error(w, "Place not found", http.StatusNotFound)
			return
		}
		// Updating database object with information from google maps and from request.
		placeDetails := gMapsResp.Results[0]
		lat := placeDetails.Lat
		lng := placeDetails.Lng
		if err := db.Put(txn, placeID, p.Accessibility, lat, lng); err != nil {
			log.Printf("Error insertind record on database: %q", err)
			http.Error(w, "Problems inserting into database response.", http.StatusInternalServerError)
			return
		}

		w.Header().Set("Content-Type", "application/json")
		w.WriteHeader(http.StatusCreated)
		if err := json.NewEncoder(w).Encode(p); err != nil {
			log.Printf("Error marshaling response: %q", err)
			http.Error(w, "Problems marshaling response.", http.StatusInternalServerError)
		}
	})

	router.GET("/api/v1/place/:id", func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
		txn := app.StartTransaction("GetPlace", w, r)
		defer txn.End()

		placeID, err := placeID(ps.ByName("id"))
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}

		// Triggering Google Maps request asynchronously.
		gMapsChan := gMapsClient.Get(txn, placeID)

		// Fetches data from DB.
		dbPlace, err := db.Get(txn, placeID)
		if err != nil {
			// NOTE: Deliberately let the request advance even when we don't had an error trying to fetch
			// accessibility.
			log.Printf("Error fetching place (placeID:%s) details from database: %v", placeID, err)
		}

		// Block waiting for Google Maps data.
		gMapsResp := <-gMapsChan
		if gMapsResp.Error != nil {
			log.Printf("Error fetching place details from google maps: %v", err)
			http.Error(w, "Error fetching place details from Google Maps.", http.StatusServiceUnavailable)
			return
		}
		if len(gMapsResp.Results) == 0 {
			http.Error(w, "Place not found", http.StatusNotFound)
			return
		}
		placeDetails := gMapsResp.Results[0]
		result := GetPlaceResult{
			GoogleMapsPlaceID: placeDetails.ID,
			Location: LatLng{
				Lat: placeDetails.Lat,
				Lng: placeDetails.Lng,
			},
			Name:          placeDetails.Name,
			Accessibility: dbPlace.Accessibility,
		}
		w.Header().Set("Content-Type", "application/json")
		if err := json.NewEncoder(w).Encode(result); err != nil {
			log.Printf("Error marshaling response: %q", err)
			http.Error(w, "Problems marshaling response.", http.StatusInternalServerError)
		}

	})
	log.Println("Service listening at port ", port)
	log.Fatal(http.ListenAndServe(":"+port, router))
}

func placeID(placeID string) (string, error) {
	if placeID == "" || len(placeID) <= 3 {
		return "", fmt.Errorf("Invalid PlaceID:%s", placeID)
	}
	return placeID[4:], nil
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/howeyc/fsnotify"
)

const (
	watchFlags = fsnotify.FSN_CREATE |
		fsnotify.FSN_DELETE |
		fsnotify.FSN_MODIFY
)

var (
	watcher  *fsnotify.Watcher
	watched  = make(map[string]struct{})
	exitCode = make(chan int)
	rootPath string

	emptyStruct = struct{}{}
	hasSuffix   = strings.HasSuffix

	buildQueued = false
)

func panicIf(err error) {
	if err != nil {
		panic(err)
	}
}

func build() {
	{
		log.Println("glitch: building")
		cmd := exec.Command("go", "build")
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		cmd.Dir = rootPath
		if err := cmd.Run(); err != nil {
			return
		}
	}

	{
		log.Println("glitch: build OK - vetting")
		cmd := exec.Command("go", "vet", "./...")
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		cmd.Dir = rootPath
		if err := cmd.Run(); err != nil {
			return
		}
	}

	{
		log.Println("glitch: vet OK - testing")
		cmd := exec.Command("go", "test", "./...")
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		cmd.Dir = rootPath
		if err := cmd.Run(); err != nil {
			return
		}
	}

	{
		log.Println("glitch: test OK - installing")
		cmd := exec.Command("go", "install")
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		cmd.Dir = rootPath
		if err := cmd.Run(); err != nil {
			return
		}
	}

	log.Println("glitch: install OK - wating for next build event")
}

func maybeQueueBuild(path string) {
	buildQueued = hasSuffix(path, ".go")
}

func handleCreate(path string) {
	watch(path)
	maybeQueueBuild(path)
}

func handleDelete(path string) {
	if _, watching := watched[path]; watching {
		panicIf(watcher.RemoveWatch(path))
		delete(watched, path)
	}
	maybeQueueBuild(path)
}

func handleModify(path string) {
	maybeQueueBuild(path)
}

func handleEvent(ev *fsnotify.FileEvent) {
	if len(ev.Name) > 0 {
		switch {
		case ev.IsCreate():
			handleCreate(ev.Name)
		case ev.IsDelete():
			handleDelete(ev.Name)
		case ev.IsModify():
			handleModify(ev.Name)
		}
	}
}

func watch(dir string) {
	if _, watching := watched[dir]; watching {
		return
	}

	walker := func(path string, fileInfo os.FileInfo, err error) error {
		if err == nil && fileInfo.IsDir() {
			if err = watcher.WatchFlags(path, watchFlags); err == nil {
				watched[path] = emptyStruct
			}
		}
		return err
	}

	panicIf(filepath.Walk(dir, walker))
}

func periodicallyLogWatchedPaths() {
	logWatchedPaths := func() {
		log.Printf("glitch: watching: %v paths", len(watched))
		for path, _ := range watched {
			log.Println("glitch: watching:", path)
		}
	}

	for _ = range time.Tick(5 * time.Second) {
		logWatchedPaths()
	}
}

func periodicallyLogBuildStatus() {
	logBuildStatus := func() {
		log.Println("glitch: buildQueued:", buildQueued)
	}

	for _ = range time.Tick(300 * time.Millisecond) {
		logBuildStatus()
	}
}

func runEventLoop() {
	for {
		select {
		case ev := <-watcher.Event:
			handleEvent(ev)
		case err := <-watcher.Error:
			panicIf(err)
		}
	}
}

func runBuildLoop() {
	consumeBuildQueue := func() {
		if buildQueued {
			buildQueued = false
			build()
		}
	}

	for _ = range time.Tick(1 * time.Second) {
		go consumeBuildQueue()
	}
}

func main() {
	wd, err := os.Getwd()
	panicIf(err)
	rootPath = wd

	w, err := fsnotify.NewWatcher()
	panicIf(err)
	watcher = w
	defer watcher.Close()

	//go periodicallyLogWatchedPaths()
	//go periodicallyLogBuildStatus()
	go runEventLoop()
	go runBuildLoop()

	watch(rootPath)
	os.Exit(<-exitCode)
}
<commit_msg>ignore errors while walking<commit_after>package main

import (
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/howeyc/fsnotify"
)

const (
	watchFlags = fsnotify.FSN_CREATE |
		fsnotify.FSN_DELETE |
		fsnotify.FSN_MODIFY
)

var (
	watcher  *fsnotify.Watcher
	watched  = make(map[string]struct{})
	exitCode = make(chan int)
	rootPath string

	emptyStruct = struct{}{}
	hasSuffix   = strings.HasSuffix

	buildQueued = false
)

func panicIf(err error) {
	if err != nil {
		panic(err)
	}
}

func build() {
	{
		log.Println("glitch: building")
		cmd := exec.Command("go", "build")
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		cmd.Dir = rootPath
		if err := cmd.Run(); err != nil {
			return
		}
	}

	{
		log.Println("glitch: build OK - vetting")
		cmd := exec.Command("go", "vet", "./...")
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		cmd.Dir = rootPath
		if err := cmd.Run(); err != nil {
			return
		}
	}

	{
		log.Println("glitch: vet OK - testing")
		cmd := exec.Command("go", "test", "./...")
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		cmd.Dir = rootPath
		if err := cmd.Run(); err != nil {
			return
		}
	}

	{
		log.Println("glitch: test OK - installing")
		cmd := exec.Command("go", "install")
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		cmd.Dir = rootPath
		if err := cmd.Run(); err != nil {
			return
		}
	}

	log.Println("glitch: install OK - wating for next build event")
}

func maybeQueueBuild(path string) {
	buildQueued = hasSuffix(path, ".go")
}

func handleCreate(path string) {
	watch(path)
	maybeQueueBuild(path)
}

func handleDelete(path string) {
	if _, watching := watched[path]; watching {
		panicIf(watcher.RemoveWatch(path))
		delete(watched, path)
	}
	maybeQueueBuild(path)
}

func handleModify(path string) {
	maybeQueueBuild(path)
}

func handleEvent(ev *fsnotify.FileEvent) {
	if len(ev.Name) > 0 {
		switch {
		case ev.IsCreate():
			handleCreate(ev.Name)
		case ev.IsDelete():
			handleDelete(ev.Name)
		case ev.IsModify():
			handleModify(ev.Name)
		}
	}
}

func watch(dir string) {
	if _, watching := watched[dir]; watching {
		return
	}

	walker := func(path string, fileInfo os.FileInfo, err error) error {
		if err != nil {
			return nil
		}

		if fileInfo.IsDir() {
			if err = watcher.WatchFlags(path, watchFlags); err == nil {
				watched[path] = emptyStruct
			}
		}

		return err
	}

	panicIf(filepath.Walk(dir, walker))
}

func periodicallyLogWatchedPaths() {
	logWatchedPaths := func() {
		log.Printf("glitch: watching: %v paths", len(watched))
		for path, _ := range watched {
			log.Println("glitch: watching:", path)
		}
	}

	for _ = range time.Tick(5 * time.Second) {
		logWatchedPaths()
	}
}

func periodicallyLogBuildStatus() {
	logBuildStatus := func() {
		log.Println("glitch: buildQueued:", buildQueued)
	}

	for _ = range time.Tick(300 * time.Millisecond) {
		logBuildStatus()
	}
}

func runEventLoop() {
	for {
		select {
		case ev := <-watcher.Event:
			handleEvent(ev)
		case err := <-watcher.Error:
			panicIf(err)
		}
	}
}

func runBuildLoop() {
	consumeBuildQueue := func() {
		if buildQueued {
			buildQueued = false
			build()
		}
	}

	for _ = range time.Tick(1 * time.Second) {
		go consumeBuildQueue()
	}
}

func main() {
	wd, err := os.Getwd()
	panicIf(err)
	rootPath = wd

	w, err := fsnotify.NewWatcher()
	panicIf(err)
	watcher = w
	defer watcher.Close()

	//go periodicallyLogWatchedPaths()
	//go periodicallyLogBuildStatus()
	go runEventLoop()
	go runBuildLoop()

	watch(rootPath)
	os.Exit(<-exitCode)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"errors"
	"os"
	"strings"

	"gopkg.in/kothar/go-backblaze.v0"
)

var (
	in     = bufio.NewReader(os.Stdin)
	out    = bufio.NewWriter(os.Stdout)
	bucket *backblaze.Bucket
	prefix string
)

func getConfig(name string) string {
	out.WriteString("GETCONFIG ")
	out.WriteString(name)
	out.WriteString("\n")
	out.Flush()

	line, err := in.ReadString('\n')
	if err != nil {
		log.Fatalf("Couldn't get config variable %s: %v", name, err)
	}
	line = strings.TrimSuffix(line, "\n")

	if strings.HasPrefix(line, "VALUE ") {
		return strings.TrimPrefix(line, "VALUE ")
	}

	return ""
}

func prepare(mode string) {
	err := prepareInternal()
	if err != nil {
		out.WriteString(mode + "-FAILURE " + err.Error() + "\n")
		return
	}

	out.WriteString(mode + "-SUCCESS\n")
}

func prepareInternal() error {
	accountID := getConfig("accountid")
	if accountID == "" {
		accountID = os.Getenv("B2_ACCOUNT_ID")
	}
	if accountID == "" {
		return errors.New("You must set accountid to the backblaze account id")
	}

	appKey := getConfig("appkey")
	if appKey == "" {
		appKey = os.Getenv("B2_APP_KEY")
	}
	if appKey == "" {
		return errors.New("You must set appkey to the backblaze application key")
	}

	bucketName := getConfig("bucket")
	if bucketName == "" {
		return errors.New("You must set bucket to the bucket name")
	}

	prefix = getConfig("prefix")
	// prefix == "" is ok.
	if prefix != "" && !strings.HasSuffix(prefix, "/") {
		prefix = prefix + "/"
	}

	b2, err := backblaze.NewB2(backblaze.Credentials{
		AccountID: accountID,
		ApplicationKey: appKey,
	})
	if err != nil {
		return fmt.Errorf("Couldn't authorize: %v", err)
	}

	bucket, err = b2.Bucket(bucketName)
	if err != nil {
		return fmt.Errorf("Couldn't open bucket %#v: %v", bucketName, err)
	}

	if bucket == nil {
		fmt.Fprintf(os.Stderr, "Creating private B2 bucket %#v\n", bucketName)
		bucket, err = b2.CreateBucket(bucketName, backblaze.AllPrivate)
		if err != nil {
			return fmt.Errorf("Couldn't create bucket %#v: %v", bucketName, err)
		}
	}

	return nil
}

func store(key, file string) {
	ok := false
	defer func() {
		if ok {
			out.WriteString("TRANSFER-SUCCESS STORE ")
		} else {
			out.WriteString("TRANSFER-FAILURE STORE ")
		}
		out.WriteString(key)
		out.WriteString("\n")
	}()

	fh, err := os.Open(file)
	if err != nil {
		log.Printf("Couldn't open %v for reading: %v", file, err)
		return
	}
	defer fh.Close()

	res, err := bucket.ListFileNames(prefix + key, 1)
	if err != nil {
		log.Printf("Couldn't list filenames: %v", err)
		return
	}

	if len(res.Files) > 0 && res.Files[0].Name == prefix + key {
		// file already stored
		ok = true
		return
	}

	_, err = bucket.UploadFile(prefix + key, nil, fh)
	if err != nil {
		log.Printf("Couldn't upload file: %v", err)
		return
	}

	ok = true
}

func retrieve(key, file string) {
	ok := false
	defer func() {
		if ok {
			out.WriteString("TRANSFER-SUCCESS RETRIEVE ")
		} else {
			out.WriteString("TRANSFER-FAILURE RETRIEVE ")
		}
		out.WriteString(key)
		out.WriteString("\n")
	}()

	_, rc, err := bucket.DownloadFileByName(prefix + key)
	if rc != nil {
		defer rc.Close()
	}
	if err != nil {
		log.Printf("Couldn't download file: %v", err)
		return
	}

	fh, err := os.Create(file)
	if err != nil {
		log.Printf("Couldn't open %v for writing: %v", file, err)
		return
	}
	defer fh.Close()

	_, err = io.Copy(fh, rc)
	if err != nil {
		log.Printf("Couldn't download file: %v", err)
		return
	}

	ok = true
}

func checkPresent(key string) {
	ret := "UNKNOWN"
	defer func() {
		out.WriteString("CHECKPRESENT-" + ret + " " + key + "\n")
	}()

	res, err := bucket.ListFileNames(prefix + key, 1)
	if err != nil {
		log.Printf("Couldn't list filenames: %v", err)
		return
	}

	if len(res.Files) == 0 || res.Files[0].Name != prefix + key {
		ret = "FAILURE"
	} else {
		ret = "SUCCESS"
	}
}

func remove(key string) {
	ret := "FAILURE"
	defer func() {
		out.WriteString("REMOVE-" + ret + " " + key + "\n")
	}()

	res, err := bucket.ListFileNames(prefix + key, 1)
	if err != nil {
		log.Printf("Couldn't list filenames: %v", err)
		return
	}

	if len(res.Files) == 0 || res.Files[0].Name != prefix + key {
		// File already non-existent
		ret = "SUCCESS"
		return
	}

	_, err = bucket.DeleteFileVersion(res.Files[0].Name, res.Files[0].ID)
	if err != nil {
		log.Printf("Couldn't delete file version: %v", err)
		return
	}

	ret = "SUCCESS"
}

func main() {
	out.Write([]byte("VERSION 1\n"))

	for {
		out.Flush()

		line, err := in.ReadString('\n')
		if err != nil {
			if err == io.EOF {
				break
			}

			log.Fatal(err)
		}
		line = strings.TrimSuffix(line, "\n")

		switch {
		case strings.HasPrefix(line, "INITREMOTE"):
			prepare("INITREMOTE")

		case strings.HasPrefix(line, "PREPARE"):
			prepare("PREPARE")

		case strings.HasPrefix(line, "TRANSFER "):
			line = strings.TrimPrefix(line, "TRANSFER ")

			fields := strings.SplitN(line, " ", 3)
			for len(fields) < 3 {
				fields = append(fields, "")
			}

			switch fields[0] {
			case "STORE":
				store(fields[1], fields[2])
			case "RETRIEVE":
				retrieve(fields[1], fields[2])
			default:
				out.WriteString("UNSUPPORTED-REQUEST\n")
			}

		case strings.HasPrefix(line, "CHECKPRESENT "):
			key := strings.TrimPrefix(line, "CHECKPRESENT ")
			checkPresent(key)

		case strings.HasPrefix(line, "REMOVE "):
			key := strings.TrimPrefix(line, "REMOVE ")
			remove(key)

		default:
			out.WriteString("UNSUPPORTED-REQUEST\n")
		}
	}
}
<commit_msg>go fmt<commit_after>package main

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"strings"

	"gopkg.in/kothar/go-backblaze.v0"
)

var (
	in     = bufio.NewReader(os.Stdin)
	out    = bufio.NewWriter(os.Stdout)
	bucket *backblaze.Bucket
	prefix string
)

func getConfig(name string) string {
	out.WriteString("GETCONFIG ")
	out.WriteString(name)
	out.WriteString("\n")
	out.Flush()

	line, err := in.ReadString('\n')
	if err != nil {
		log.Fatalf("Couldn't get config variable %s: %v", name, err)
	}
	line = strings.TrimSuffix(line, "\n")

	if strings.HasPrefix(line, "VALUE ") {
		return strings.TrimPrefix(line, "VALUE ")
	}

	return ""
}

func prepare(mode string) {
	err := prepareInternal()
	if err != nil {
		out.WriteString(mode + "-FAILURE " + err.Error() + "\n")
		return
	}

	out.WriteString(mode + "-SUCCESS\n")
}

func prepareInternal() error {
	accountID := getConfig("accountid")
	if accountID == "" {
		accountID = os.Getenv("B2_ACCOUNT_ID")
	}
	if accountID == "" {
		return errors.New("You must set accountid to the backblaze account id")
	}

	appKey := getConfig("appkey")
	if appKey == "" {
		appKey = os.Getenv("B2_APP_KEY")
	}
	if appKey == "" {
		return errors.New("You must set appkey to the backblaze application key")
	}

	bucketName := getConfig("bucket")
	if bucketName == "" {
		return errors.New("You must set bucket to the bucket name")
	}

	prefix = getConfig("prefix")
	// prefix == "" is ok.
	if prefix != "" && !strings.HasSuffix(prefix, "/") {
		prefix = prefix + "/"
	}

	b2, err := backblaze.NewB2(backblaze.Credentials{
		AccountID:      accountID,
		ApplicationKey: appKey,
	})
	if err != nil {
		return fmt.Errorf("Couldn't authorize: %v", err)
	}

	bucket, err = b2.Bucket(bucketName)
	if err != nil {
		return fmt.Errorf("Couldn't open bucket %#v: %v", bucketName, err)
	}

	if bucket == nil {
		fmt.Fprintf(os.Stderr, "Creating private B2 bucket %#v\n", bucketName)
		bucket, err = b2.CreateBucket(bucketName, backblaze.AllPrivate)
		if err != nil {
			return fmt.Errorf("Couldn't create bucket %#v: %v", bucketName, err)
		}
	}

	return nil
}

func store(key, file string) {
	ok := false
	defer func() {
		if ok {
			out.WriteString("TRANSFER-SUCCESS STORE ")
		} else {
			out.WriteString("TRANSFER-FAILURE STORE ")
		}
		out.WriteString(key)
		out.WriteString("\n")
	}()

	fh, err := os.Open(file)
	if err != nil {
		log.Printf("Couldn't open %v for reading: %v", file, err)
		return
	}
	defer fh.Close()

	res, err := bucket.ListFileNames(prefix+key, 1)
	if err != nil {
		log.Printf("Couldn't list filenames: %v", err)
		return
	}

	if len(res.Files) > 0 && res.Files[0].Name == prefix+key {
		// file already stored
		ok = true
		return
	}

	_, err = bucket.UploadFile(prefix+key, nil, fh)
	if err != nil {
		log.Printf("Couldn't upload file: %v", err)
		return
	}

	ok = true
}

func retrieve(key, file string) {
	ok := false
	defer func() {
		if ok {
			out.WriteString("TRANSFER-SUCCESS RETRIEVE ")
		} else {
			out.WriteString("TRANSFER-FAILURE RETRIEVE ")
		}
		out.WriteString(key)
		out.WriteString("\n")
	}()

	_, rc, err := bucket.DownloadFileByName(prefix + key)
	if rc != nil {
		defer rc.Close()
	}
	if err != nil {
		log.Printf("Couldn't download file: %v", err)
		return
	}

	fh, err := os.Create(file)
	if err != nil {
		log.Printf("Couldn't open %v for writing: %v", file, err)
		return
	}
	defer fh.Close()

	_, err = io.Copy(fh, rc)
	if err != nil {
		log.Printf("Couldn't download file: %v", err)
		return
	}

	ok = true
}

func checkPresent(key string) {
	ret := "UNKNOWN"
	defer func() {
		out.WriteString("CHECKPRESENT-" + ret + " " + key + "\n")
	}()

	res, err := bucket.ListFileNames(prefix+key, 1)
	if err != nil {
		log.Printf("Couldn't list filenames: %v", err)
		return
	}

	if len(res.Files) == 0 || res.Files[0].Name != prefix+key {
		ret = "FAILURE"
	} else {
		ret = "SUCCESS"
	}
}

func remove(key string) {
	ret := "FAILURE"
	defer func() {
		out.WriteString("REMOVE-" + ret + " " + key + "\n")
	}()

	res, err := bucket.ListFileNames(prefix+key, 1)
	if err != nil {
		log.Printf("Couldn't list filenames: %v", err)
		return
	}

	if len(res.Files) == 0 || res.Files[0].Name != prefix+key {
		// File already non-existent
		ret = "SUCCESS"
		return
	}

	_, err = bucket.DeleteFileVersion(res.Files[0].Name, res.Files[0].ID)
	if err != nil {
		log.Printf("Couldn't delete file version: %v", err)
		return
	}

	ret = "SUCCESS"
}

func main() {
	out.Write([]byte("VERSION 1\n"))

	for {
		out.Flush()

		line, err := in.ReadString('\n')
		if err != nil {
			if err == io.EOF {
				break
			}

			log.Fatal(err)
		}
		line = strings.TrimSuffix(line, "\n")

		switch {
		case strings.HasPrefix(line, "INITREMOTE"):
			prepare("INITREMOTE")

		case strings.HasPrefix(line, "PREPARE"):
			prepare("PREPARE")

		case strings.HasPrefix(line, "TRANSFER "):
			line = strings.TrimPrefix(line, "TRANSFER ")

			fields := strings.SplitN(line, " ", 3)
			for len(fields) < 3 {
				fields = append(fields, "")
			}

			switch fields[0] {
			case "STORE":
				store(fields[1], fields[2])
			case "RETRIEVE":
				retrieve(fields[1], fields[2])
			default:
				out.WriteString("UNSUPPORTED-REQUEST\n")
			}

		case strings.HasPrefix(line, "CHECKPRESENT "):
			key := strings.TrimPrefix(line, "CHECKPRESENT ")
			checkPresent(key)

		case strings.HasPrefix(line, "REMOVE "):
			key := strings.TrimPrefix(line, "REMOVE ")
			remove(key)

		default:
			out.WriteString("UNSUPPORTED-REQUEST\n")
		}
	}
}
<|endoftext|>


<commit_before>package main

// The import section defines libraries that we are going to use in our program.
import (
	"fmt"
	"log"
	"net/http"

	"github.com/ory-am/common/env"
	"github.com/gorilla/mux"
	"encoding/json"
)

// In a 12 factor app, we must obey the environment variables.
var envHost = env.Getenv("HOST", "")
var envPort = env.Getenv("PORT", "5678")

// Contact defines the structure of a contact which including name, department and company.
type Contact struct {
	ID string `json:"id"`

	// Name is the contact's full name.
	Name string `json:"name"`

	// Department is the contact's department in a company.
	Department string `json:"department"`

	// Company is the name of the company the contact works for.
	Company string `json:"company"`
}

// Contacts is a list of contact structs.
type Contacts []Contact

// myContacts is an exemplary
var MyContacts = Contacts{
	{
		ID:         "john-bravo",
		Name:       "John Bravo",
		Department: "IT",
		Company:    "ACME Inc",
	},
	{
		ID:         "cathrine-mueller",
		Name:       "Cathrine Müller",
		Department: "HR",
		Company:    "Grove AG",
	},
	{
		ID:         "john-bravo",
		Name:       "Maximilian Schmidt",
		Department: "PR",
		Company:    "Titanpad AG",
	},
}

// The main routine is going the "entry" point.
func main() {
	// Create a new router.
	router := mux.NewRouter()

	// Requests to "/" are by method listContacts.
	router.HandleFunc("/", ListContacts(&MyContacts)).Methods("GET")
	router.HandleFunc("/", AddContact(&MyContacts)).Methods("POST")

	// Print some information.
	fmt.Printf("Listening on %s\n", "http://localhost:5678")

	// Start up the server and check for errors.
	listenOn := fmt.Sprintf("%s:%s", envHost, envPort)
	err := http.ListenAndServe(listenOn, router)
	if err != nil {
		log.Fatalf("Could not set up server because %s", err)
	}
}

// listContacts takes a contact list and outputs it.
func ListContacts(contacts *Contacts) func(rw http.ResponseWriter, r *http.Request) {
	return func(rw http.ResponseWriter, r *http.Request) {
		output, err := json.MarshalIndent(contacts, "", "\t")

		// If an error occurs, handle it.
		if err != nil {
			http.Error(rw, fmt.Sprintf("Could not read vcards because %s", err), http.StatusInternalServerError)
			return
		}

		rw.Write(output)
	}
}

// addContact will add a contact to the list
func AddContact(contacts *Contacts) func(rw http.ResponseWriter, r *http.Request) {
	return func(rw http.ResponseWriter, r *http.Request) {

		// newContact poses as a placeholder for the contact that the request is going to add.
		var newContact Contact

		// We decode the information and "inject" it to newContact.
		err := json.NewDecoder(r.Body).Decode(&newContact)

		// If an error occurs while decoding, handle it.
		if err != nil {
			http.Error(rw, fmt.Sprintf("Could not read vcards because %s", err), http.StatusInternalServerError)
			return
		}

		// Save newContact to the list of available contacts.
		*contacts = append(*contacts, newContact)

		// Per specification, RESTful may return an empty response when a POST request was successful
		rw.WriteHeader(http.StatusNoContent)
	}
}
<commit_msg>add cors<commit_after>package main

// The import section defines libraries that we are going to use in our program.
import (
	"fmt"
	"log"
	"net/http"

	"encoding/json"
	"github.com/gorilla/mux"
	"github.com/ory-am/common/env"
	"github.com/rs/cors"
)

// In a 12 factor app, we must obey the environment variables.
var envHost = env.Getenv("HOST", "")
var envPort = env.Getenv("PORT", "5678")

// Contact defines the structure of a contact which including name, department and company.
type Contact struct {
	ID string `json:"id"`

	// Name is the contact's full name.
	Name string `json:"name"`

	// Department is the contact's department in a company.
	Department string `json:"department"`

	// Company is the name of the company the contact works for.
	Company string `json:"company"`
}

// Contacts is a list of contact structs.
type Contacts []Contact

// myContacts is an exemplary
var MyContacts = Contacts{
	{
		ID:         "john-bravo",
		Name:       "John Bravo",
		Department: "IT",
		Company:    "ACME Inc",
	},
	{
		ID:         "cathrine-mueller",
		Name:       "Cathrine Müller",
		Department: "HR",
		Company:    "Grove AG",
	},
	{
		ID:         "john-bravo",
		Name:       "Maximilian Schmidt",
		Department: "PR",
		Company:    "Titanpad AG",
	},
}

// The main routine is going the "entry" point.
func main() {
	// Create a new router.
	router := mux.NewRouter()

	// Requests to "/" are by method listContacts.
	router.HandleFunc("/", ListContacts(&MyContacts)).Methods("GET")
	router.HandleFunc("/", AddContact(&MyContacts)).Methods("POST")

	// Print some information.
	fmt.Printf("Listening on %s\n", "http://localhost:5678")

	// Cross origin resource requests
	c := cors.New(cors.Options{AllowedOrigins: []string{"*"}})

	// Start up the server and check for errors.
	listenOn := fmt.Sprintf("%s:%s", envHost, envPort)
	err := http.ListenAndServe(listenOn, c.Handler(router))
	if err != nil {
		log.Fatalf("Could not set up server because %s", err)
	}
}

// listContacts takes a contact list and outputs it.
func ListContacts(contacts *Contacts) func(rw http.ResponseWriter, r *http.Request) {
	return func(rw http.ResponseWriter, r *http.Request) {
		output, err := json.MarshalIndent(contacts, "", "\t")

		// If an error occurs, handle it.
		if err != nil {
			http.Error(rw, fmt.Sprintf("Could not read vcards because %s", err), http.StatusInternalServerError)
			return
		}

		rw.Write(output)
	}
}

// addContact will add a contact to the list
func AddContact(contacts *Contacts) func(rw http.ResponseWriter, r *http.Request) {
	return func(rw http.ResponseWriter, r *http.Request) {

		// newContact poses as a placeholder for the contact that the request is going to add.
		var newContact Contact

		// We decode the information and "inject" it to newContact.
		err := json.NewDecoder(r.Body).Decode(&newContact)

		// If an error occurs while decoding, handle it.
		if err != nil {
			http.Error(rw, fmt.Sprintf("Could not read vcards because %s", err), http.StatusInternalServerError)
			return
		}

		// Save newContact to the list of available contacts.
		*contacts = append(*contacts, newContact)

		// Per specification, RESTful may return an empty response when a POST request was successful
		rw.WriteHeader(http.StatusNoContent)
	}
}
<|endoftext|>


<commit_before>// Hellofs implements a simple "hello world" file system.
package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"time"

	"bytes"

	"bazil.org/fuse"
	"bazil.org/fuse/fs"
	_ "bazil.org/fuse/fs/fstestutil"
	"github.com/tsileo/blobsnap/clientutil"
	"github.com/tsileo/blobstash/client"
	"golang.org/x/net/context"
)

var Usage = func() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "  %s MOUNTPOINT\n", os.Args[0])
	flag.PrintDefaults()
}

func main() {
	flag.Usage = Usage
	flag.Parse()

	if flag.NArg() != 1 {
		Usage()
		os.Exit(2)
	}
	mountpoint := flag.Arg(0)
	bs := client.NewBlobStore("")
	kvs := client.NewKvStore("")
	c, err := fuse.Mount(
		mountpoint,
		fuse.FSName("blobfs"),
		fuse.Subtype("blobfs"),
		fuse.LocalVolume(),
		fuse.VolumeName("BlobFS"),
	)
	if err != nil {
		log.Fatal(err)
	}
	defer c.Close()

	err = fs.Serve(c, &FS{bs: bs, kvs: kvs, uploader: clientutil.NewUploader(bs, kvs)})
	if err != nil {
		log.Fatal(err)
	}

	// check if the mount process has an error to report
	<-c.Ready
	if err := c.MountError; err != nil {
		log.Fatal(err)
	}
}

// FS implements the hello world file system.
type FS struct {
	kvs      *client.KvStore
	bs       *client.BlobStore
	uploader *clientutil.Uploader
}

func (f *FS) Root() (fs.Node, error) {
	// FIXME load/store the root from kvs
	root := &Dir{fs: f, Name: "_root", Children: map[string]*clientutil.Meta{}}
	if err := root.Save(); err != nil {
		return nil, err
	}
	return root, nil
}

// Dir implements both Node and Handle for the root directory.
type Dir struct {
	fs   *FS
	meta *clientutil.Meta

	Name     string
	Hash     string
	Children map[string]*clientutil.Meta
}

func NewDir(fs *FS, meta *clientutil.Meta) *Dir {
	d := &Dir{
		fs:       fs,
		meta:     meta,
		Name:     meta.Name,
		Hash:     meta.Hash,
		Children: map[string]*clientutil.Meta{},
	}
	for _, ref := range d.meta.Refs {
		m, err := clientutil.NewMetaFromBlobStore(fs.bs, ref.(string))
		if err != nil {
			panic(err)
		}
		d.Children[m.Hash] = m
	}
	return d
}

func (d *Dir) Attr(ctx context.Context, a *fuse.Attr) error {
	a.Inode = 1
	a.Mode = os.ModeDir | 0555
	return nil
}

func (d *Dir) Lookup(ctx context.Context, name string) (fs.Node, error) {
	if c, ok := d.Children[name]; ok {
		if c.IsFile() {
			return NewFile(d.fs, c), nil
		} else {
			return NewDir(d.fs, c), nil
		}
	}
	//if name == "hello" {
	//	return File{}, nil
	//}
	return nil, fuse.ENOENT
}

var dirDirs = []fuse.Dirent{
	{Inode: 2, Name: "hello", Type: fuse.DT_File},
}

func (d *Dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
	dirs := []fuse.Dirent{}
	for _, c := range d.Children {
		if c.IsDir() {
			dirs = append(dirs, fuse.Dirent{
				Inode: 1,
				Name:  c.Name,
				Type:  fuse.DT_Dir,
			})
		} else {
			dirs = append(dirs, fuse.Dirent{
				Inode: 2,
				Name:  c.Name,
				Type:  fuse.DT_File,
			})
		}
	}
	return dirs, nil
}

func (d *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) {
	newdir := &Dir{Name: req.Name, fs: d.fs, Children: map[string]*clientutil.Meta{}}
	if err := newdir.Save(); err != nil {
		log.Printf("newdir err: %v", err)
		return nil, err
	}
	d.Children[newdir.meta.Hash] = newdir.meta
	log.Printf("%+v %+v", d, newdir)
	d.meta.AddRef(newdir.Hash)
	if err := d.Save(); err != nil {
		return nil, err
	}
	return newdir, nil
}

func (d *Dir) Save() error {
	log.Printf("Dir.Save %+v", d)
	m := clientutil.NewMeta()
	m.Type = "dir"
	m.Name = d.Name
	m.Mode = uint32(os.ModeDir | 0555)
	m.ModTime = time.Now().Format(time.RFC3339)
	mhash, mjs := m.Json()
	m.Hash = mhash
	d.meta = m
	mexists, err := d.fs.bs.Stat(mhash)
	if err != nil {
		log.Printf("stats failed %v", err)
		return err
	}
	if !mexists {
		if err := d.fs.bs.Put(mhash, mjs); err != nil {
			log.Printf("put failed %v", err)
			return err
		}
	}
	d.Hash = mhash
	log.Printf("Dir.Save end %+v", d)
	return nil
}
func (d *Dir) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) {
	m := clientutil.NewMeta()
	m.Type = "file"
	m.Name = req.Name
	m.ModTime = time.Now().Format(time.RFC3339)
	mhash, mjs := m.Json()
	m.Hash = mhash
	mexists, err := d.fs.bs.Stat(mhash)
	if err != nil {
		return nil, nil, err
	}
	if !mexists {
		if err := d.fs.bs.Put(mhash, mjs); err != nil {
			return nil, nil, err
		}
	}
	d.meta = m
	d.Hash = mhash

	time.Sleep(100 * time.Millisecond)
	f := NewFile(d.fs, m)
	d.Children[m.Hash] = m
	d.meta.AddRef(m.Hash)
	if err := d.Save(); err != nil {
		return nil, nil, err
	}
	return f, f, nil
}

type File struct {
	fs       *FS
	buf      *bytes.Buffer
	Meta     *clientutil.Meta
	FakeFile *clientutil.FakeFile
}

func NewFile(fs *FS, meta *clientutil.Meta) *File {
	meta, err := clientutil.NewMetaFromBlobStore(fs.bs, meta.Hash)
	if err != nil {
		panic(err)
	}
	return &File{
		fs:   fs,
		Meta: meta,
		buf:  &bytes.Buffer{},
	}
}
func (f *File) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.WriteResponse) error {
	log.Printf("Write %v %v", req.Offset, len(req.Data))
	if req.Offset != int64(f.buf.Len()) {
		return fmt.Errorf("failed")
	}
	n, err := f.buf.Write(req.Data)
	if err != nil {
		return err
	}
	resp.Size = n
	return nil
}

type ClosingBuffer struct {
	*bytes.Buffer
}

func (*ClosingBuffer) Close() error {
	return nil
}
func (f *File) Flush(ctx context.Context, req *fuse.FlushRequest) error {
	log.Printf("FLUSH")
	m2, _, err := f.fs.uploader.PutReader(f.Meta.Name, &ClosingBuffer{f.buf})
	f.buf.Reset()
	if err != nil {
		return err
	}
	f.Meta = m2
	return nil
}

func (f *File) Attr(ctx context.Context, a *fuse.Attr) error {
	a.Inode = 2
	a.Mode = os.FileMode(f.Meta.Mode)
	a.Size = uint64(f.Meta.Size)
	if f.Meta.ModTime != "" {
		t, err := time.Parse(time.RFC3339, f.Meta.ModTime)
		if err != nil {
			panic(fmt.Errorf("error parsing mtime for %v: %v", f, err))
		}
		a.Mtime = t
	}
	return nil
}

func (f *File) Open(ctx context.Context, req *fuse.OpenRequest, res *fuse.OpenResponse) (fs.Handle, error) {
	log.Printf("OPEN")
	f.FakeFile = clientutil.NewFakeFile(f.fs.bs, f.Meta)
	return f, nil
}

func (f *File) Release(ctx context.Context, req *fuse.ReleaseRequest) error {
	if f.FakeFile != nil {
		f.FakeFile.Close()
		f.FakeFile = nil
	}
	return nil
}
func (f *File) Fsync(ctx context.Context, req *fuse.FsyncRequest) error {
	return nil
}
func (f *File) Read(ctx context.Context, req *fuse.ReadRequest, res *fuse.ReadResponse) error {
	log.Printf("Read %+v", f)
	if req.Offset >= int64(f.Meta.Size) {
		return nil
	}
	buf := make([]byte, req.Size)
	n, err := f.FakeFile.ReadAt(buf, req.Offset)
	if err == io.EOF {
		err = nil
	}
	if err != nil {
		log.Printf("Error reading FakeFile %+v on %v at %d: %v", f, f.Meta.Hash, req.Offset, err)
		return fuse.EIO
	}
	res.Data = buf[:n]
	return nil
}
<commit_msg>fs: better write support and added rm support<commit_after>// Hellofs implements a simple "hello world" file system.
package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"syscall"
	"time"

	"bytes"

	"bazil.org/fuse"
	"bazil.org/fuse/fs"
	_ "bazil.org/fuse/fs/fstestutil"
	"bazil.org/fuse/fuseutil"
	"github.com/tsileo/blobsnap/clientutil"
	"github.com/tsileo/blobstash/client"
	"golang.org/x/net/context"
)

const maxInt = int(^uint(0) >> 1)

var Usage = func() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "  %s MOUNTPOINT\n", os.Args[0])
	flag.PrintDefaults()
}

func main() {
	flag.Usage = Usage
	flag.Parse()

	if flag.NArg() != 1 {
		Usage()
		os.Exit(2)
	}
	mountpoint := flag.Arg(0)
	bs := client.NewBlobStore("")
	kvs := client.NewKvStore("")
	c, err := fuse.Mount(
		mountpoint,
		fuse.FSName("blobfs"),
		fuse.Subtype("blobfs"),
		fuse.LocalVolume(),
		fuse.VolumeName("BlobFS"),
	)
	if err != nil {
		log.Fatal(err)
	}
	defer c.Close()

	err = fs.Serve(c, &FS{bs: bs, kvs: kvs, uploader: clientutil.NewUploader(bs, kvs)})
	if err != nil {
		log.Fatal(err)
	}

	// check if the mount process has an error to report
	<-c.Ready
	if err := c.MountError; err != nil {
		log.Fatal(err)
	}
}

// FS implements the hello world file system.
type FS struct {
	kvs      *client.KvStore
	bs       *client.BlobStore
	uploader *clientutil.Uploader
}

func (f *FS) Root() (fs.Node, error) {
	// FIXME load/store the root from kvs
	root := &Dir{fs: f, Name: "_root", Children: map[string]*clientutil.Meta{}}
	if err := root.Save(); err != nil {
		return nil, err
	}
	return root, nil
}

// Dir implements both Node and Handle for the root directory.
type Dir struct {
	fs       *FS
	meta     *clientutil.Meta
	parent   *Dir
	Name     string
	Hash     string
	Children map[string]*clientutil.Meta
}

func NewDir(fs *FS, meta *clientutil.Meta, parent *Dir) *Dir {
	d := &Dir{
		fs:       fs,
		meta:     meta,
		Name:     meta.Name,
		Hash:     meta.Hash,
		parent:   parent,
		Children: map[string]*clientutil.Meta{},
	}
	for _, ref := range d.meta.Refs {
		m, err := clientutil.NewMetaFromBlobStore(fs.bs, ref.(string))
		if err != nil {
			panic(err)
		}
		d.Children[m.Name] = m
	}
	return d
}

func (d *Dir) Attr(ctx context.Context, a *fuse.Attr) error {
	a.Inode = 1
	a.Mode = os.ModeDir | 0555
	return nil
}

func (d *Dir) Lookup(ctx context.Context, name string) (fs.Node, error) {
	if c, ok := d.Children[name]; ok {
		if c.IsFile() {
			return NewFile(d.fs, c, d), nil
		} else {
			return NewDir(d.fs, c, d), nil
		}
	}
	//if name == "hello" {
	//	return File{}, nil
	//}
	return nil, fuse.ENOENT
}

var dirDirs = []fuse.Dirent{
	{Inode: 2, Name: "hello", Type: fuse.DT_File},
}

func (d *Dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
	dirs := []fuse.Dirent{}
	for _, c := range d.Children {
		if c.IsDir() {
			dirs = append(dirs, fuse.Dirent{
				Inode: 1,
				Name:  c.Name,
				Type:  fuse.DT_Dir,
			})
		} else {
			dirs = append(dirs, fuse.Dirent{
				Inode: 2,
				Name:  c.Name,
				Type:  fuse.DT_File,
			})
		}
	}
	return dirs, nil
}

func (d *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) {
	newdir := &Dir{Name: req.Name, fs: d.fs, Children: map[string]*clientutil.Meta{}}
	if err := newdir.Save(); err != nil {
		log.Printf("newdir err: %v", err)
		return nil, err
	}
	d.Children[newdir.meta.Name] = newdir.meta
	log.Printf("%+v %+v", d, newdir)
	if err := d.Save(); err != nil {
		return nil, err
	}
	return newdir, nil
}
func (d *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error {
	delete(d.Children, req.Name)
	if err := d.Save(); err != nil {
		return err
	}
	if d.parent != nil {
		d.parent.Children[d.Name] = d.meta
	}
	return nil
}

func (d *Dir) Save() error {
	log.Printf("Dir.Save %+v", d)
	m := clientutil.NewMeta()
	m.Type = "dir"
	m.Name = d.Name
	m.Mode = uint32(os.ModeDir | 0555)
	m.ModTime = time.Now().Format(time.RFC3339)
	for _, c := range d.Children {
		m.AddRef(c.Hash)
	}
	mhash, mjs := m.Json()
	m.Hash = mhash
	d.meta = m
	mexists, err := d.fs.bs.Stat(mhash)
	if err != nil {
		log.Printf("stats failed %v", err)
		return err
	}
	if !mexists {
		if err := d.fs.bs.Put(mhash, mjs); err != nil {
			log.Printf("put failed %v", err)
			return err
		}
	}
	if d.parent != nil {
		d.parent.Children[d.Name] = m
		if err := d.parent.Save(); err != nil {
			return err
		}
	}
	d.Hash = mhash
	log.Printf("Dir.Save end %+v", d)
	return nil
}
func (d *Dir) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) {
	log.Printf("Dir %+v Create %v", d, req.Name)
	m := clientutil.NewMeta()
	m.Type = "file"
	m.Name = req.Name
	m.ModTime = time.Now().Format(time.RFC3339)
	mhash, mjs := m.Json()
	m.Hash = mhash
	mexists, err := d.fs.bs.Stat(mhash)
	if err != nil {
		return nil, nil, err
	}
	if !mexists {
		if err := d.fs.bs.Put(mhash, mjs); err != nil {
			return nil, nil, err
		}
	}
	d.meta = m
	d.Hash = mhash

	time.Sleep(100 * time.Millisecond)
	f := NewFile(d.fs, m, d)
	d.Children[m.Name] = m
	if err := d.Save(); err != nil {
		return nil, nil, err
	}
	return f, f, nil
}

type File struct {
	fs       *FS
	buf      *bytes.Buffer
	ReadOnly bool
	Data     []byte
	Meta     *clientutil.Meta
	FakeFile *clientutil.FakeFile
	parent   *Dir
	Wrote    bool
}

func NewFile(fs *FS, meta *clientutil.Meta, parent *Dir) *File {
	meta, err := clientutil.NewMetaFromBlobStore(fs.bs, meta.Hash)
	if err != nil {
		panic(err)
	}
	return &File{
		parent: parent,
		fs:     fs,
		Meta:   meta,
		buf:    &bytes.Buffer{},
	}
}
func (f *File) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.WriteResponse) error {
	log.Printf("Write %v %v %v", f.Meta.Name, req.Offset, len(req.Data))
	newLen := req.Offset + int64(len(req.Data))
	if newLen > int64(maxInt) {
		return fuse.Errno(syscall.EFBIG)
	}

	n := copy(f.Data[req.Offset:], req.Data)
	if n < len(req.Data) {
		f.Data = append(f.Data, req.Data[n:]...)
	}

	resp.Size = len(req.Data)
	//	if req.Offset != int64(f.buf.Len()) {
	//		return fmt.Errorf("failed")
	//	}
	//	n, err := f.buf.Write(req.Data)
	//	if err != nil {
	//		return err
	//	}
	//	resp.Size = n
	f.Wrote = true
	return nil
}

type ClosingBuffer struct {
	*bytes.Buffer
}

func (*ClosingBuffer) Close() error {
	return nil
}
func (f *File) Flush(ctx context.Context, req *fuse.FlushRequest) error {
	log.Printf("FLUSH %v %v", f.Meta.Name, f.Meta.Hash)
	if f.Wrote {
		buf := bytes.NewBuffer(f.Data)
		m2, _, err := f.fs.uploader.PutReader(f.Meta.Name, &ClosingBuffer{buf})
		f.buf.Reset()
		if err != nil {
			return err
		}
		f.parent.Children[m2.Name] = m2
		f.Meta = m2
		log.Printf("FLUSH END %v %+v", f.Meta.Name, f.Meta)
	}
	return nil
}

func (f *File) Attr(ctx context.Context, a *fuse.Attr) error {
	a.Inode = 2
	a.Mode = os.FileMode(f.Meta.Mode)
	a.Size = uint64(f.Meta.Size)
	if f.Meta.ModTime != "" {
		t, err := time.Parse(time.RFC3339, f.Meta.ModTime)
		if err != nil {
			panic(fmt.Errorf("error parsing mtime for %v: %v", f, err))
		}
		a.Mtime = t
	}
	return nil
}

func (f *File) Open(ctx context.Context, req *fuse.OpenRequest, res *fuse.OpenResponse) (fs.Handle, error) {
	log.Printf("OPEN %+v %+v [ro:%v]", f.Meta.Name, f.Meta, f.ReadOnly)
	if req.Flags.IsReadOnly() || req.Flags.IsReadWrite() {
		log.Printf("Open with fakefile")
		f.ReadOnly = true
		f.FakeFile = clientutil.NewFakeFile(f.fs.bs, f.Meta)
	}
	return f, nil
}

func (f *File) Release(ctx context.Context, req *fuse.ReleaseRequest) error {
	log.Printf("Release %+v", f.Meta.Name)
	if f.FakeFile != nil {
		f.FakeFile.Close()
		f.FakeFile = nil
	}
	f.ReadOnly = false
	f.Data = nil
	f.Wrote = false
	return nil
}
func (f *File) Fsync(ctx context.Context, req *fuse.FsyncRequest) error {
	log.Printf("fsync %+v", f)
	return nil
}
func (f *File) Read(ctx context.Context, req *fuse.ReadRequest, res *fuse.ReadResponse) error {
	log.Printf("Read %+v %+v", f, req)
	if !f.Wrote && f.FakeFile != nil {
		log.Printf("Read using FakeFile at %v %v", req.Offset, req.Size)
		if req.Offset >= int64(f.Meta.Size) {
			return nil
		}
		buf := make([]byte, req.Size)
		n, err := f.FakeFile.ReadAt(buf, req.Offset)
		if err == io.EOF {
			err = nil
		}
		if err != nil {
			log.Printf("Error reading FakeFile %+v on %v at %d: %v", f, f.Meta.Hash, req.Offset, err)
			return fuse.EIO
		}
		res.Data = buf[:n]
		log.Printf("Read res=%s", res.Data)
	} else {
		fuseutil.HandleRead(req, res, f.Data)
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"html/template"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"time"

	"github.com/aws/aws-sdk-go/aws/session"
	"gopkg.in/alecthomas/kingpin.v2"
)

type container struct {
	Host    string
	Port    string
	Address string
}

type metadata struct {
	Cluster              string
	ContainerInstanceArn string
	Version              string
}

var (
	errClusterNotActive = errors.New("ecs-nginx-proxy: cluster is not active")
)

var (
	app = kingpin.New("ecs-gen", "docker-gen for AWS ECS.")

	region       = app.Flag("region", "AWS region.").Short('r').Default("ap-northeast-1").String()
	cluster      = app.Flag("cluster", "ECS cluster name.").Short('c').String()
	templateFile = app.Flag("template", "Path to template file.").Short('t').Required().ExistingFile()
	outputFile   = app.Flag("output", "Path to output file.").Short('o').Required().String()
	taskName     = app.Flag("task", "Name of ECS task containing nginx.").Default("ecs-nginx-proxy").String()

	signal = app.Flag("signal", "Command to run to signal change.").Short('s').Default("nginx -s reload").String()

	freq = app.Flag("frequency", "Time in seconds between polling. Must be >0.").Short('f').Default("30").Int()
	once = app.Flag("once", "Only execute the template once and exit.").Bool()
)

var version = ""

func main() {
	app.Version(version)
	app.DefaultEnvars()
	kingpin.MustParse(app.Parse(os.Args[1:]))
	if *cluster == "" {
		var err error
		cluster, err = findClusterName()
		if *cluster == "" || err != nil {
			panic("could not determine cluster name. please define using --cluster / -c.")
		}
		log.Println("found cluster name to be:", *cluster)
	}
	sess, err := session.NewSession()
	if err != nil {
		panic(err)
	}
	ec2 := newEC2(*region, sess)
	ecs := newECS(*region, sess)
	if *once {
		updateAndWrite(ec2, ecs)
		return
	}
	execute(ec2, ecs)
	for range time.Tick(time.Second * time.Duration(*freq)) {
		execute(ec2, ecs)
	}
}

func execute(ec2 *ec2Client, ecs *ecsClient) {
	updateAndWrite(ec2, ecs)
	var err error
	err = runSignal()
	if err != nil {
		log.Println("failed to run signal command")
		log.Println("error: ", err)
		switch err := err.(type) {
		case *exec.ExitError:
			log.Println(err.Stderr)
		}
		os.Exit(1)
	}
}

func updateAndWrite(ec2 *ec2Client, ecs *ecsClient) {
	containers, err := newScanner(*cluster, ec2, ecs).scan()
	if err != nil {
		log.Println(err)
	}
	err = writeConfig(containers)
	if err != nil {
		log.Println(err)
	}
}

func runSignal() error {
	log.Println("running signal command")
	output, err := exec.Command("/bin/sh", "-c", *signal).CombinedOutput()
	log.Println("===== output start =====")
	log.Println(string(output))
	log.Println("===== output end =====")
	return err
}

func writeConfig(params []*container) error {
	var containers map[string][]*container
	containers = make(map[string][]*container);
	// Remap the containers as a 2D array with the domain as the index
	for _, v := range params {
		if _, ok := containers[v.Host]; !ok {
			containers[v.Host] = make([]*container, 0)
		}
		containers[v.Host] = append(containers[v.Host], v)
	}
	tmpl, err := template.ParseFiles(*templateFile)
	if err != nil {
		return err
	}
	f, err := os.Create(*outputFile)
	if err != nil {
		return err
	}
	defer f.Close()
	return tmpl.Execute(f, containers)
}

func findClusterName() (*string, error) {
	ip, err := findHostIP()
	if err != nil {
		return nil, err
	}
	meta, err := fetchMetadata(ip)
	if err != nil {
		return nil, err
	}
	return &meta.Cluster, nil
}

func findHostIP() (string, error) {
	result, err := sendHTTRequest("http://169.254.169.254/latest/meta-data/local-ipv4")
	return string(result), err
}

func fetchMetadata(host string) (*metadata, error) {
	result, err := sendHTTRequest("http://" + host + ":51678/v1/metadata")
	var meta metadata
	err = json.Unmarshal(result, &meta)
	if err != nil {
		return nil, err
	}
	return &meta, nil
}

func sendHTTRequest(url string) ([]byte, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	return body, nil
}
<commit_msg>Fix dereferencing null pointer<commit_after>package main

import (
	"encoding/json"
	"errors"
	"html/template"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"time"

	"github.com/aws/aws-sdk-go/aws/session"
	"gopkg.in/alecthomas/kingpin.v2"
)

type container struct {
	Host    string
	Port    string
	Address string
}

type metadata struct {
	Cluster              string
	ContainerInstanceArn string
	Version              string
}

var (
	errClusterNotActive = errors.New("ecs-nginx-proxy: cluster is not active")
)

var (
	app = kingpin.New("ecs-gen", "docker-gen for AWS ECS.")

	region       = app.Flag("region", "AWS region.").Short('r').Default("ap-northeast-1").String()
	cluster      = app.Flag("cluster", "ECS cluster name.").Short('c').String()
	templateFile = app.Flag("template", "Path to template file.").Short('t').Required().ExistingFile()
	outputFile   = app.Flag("output", "Path to output file.").Short('o').Required().String()
	taskName     = app.Flag("task", "Name of ECS task containing nginx.").Default("ecs-nginx-proxy").String()

	signal = app.Flag("signal", "Command to run to signal change.").Short('s').Default("nginx -s reload").String()

	freq = app.Flag("frequency", "Time in seconds between polling. Must be >0.").Short('f').Default("30").Int()
	once = app.Flag("once", "Only execute the template once and exit.").Bool()
)

var version = ""

func main() {
	app.Version(version)
	app.DefaultEnvars()
	kingpin.MustParse(app.Parse(os.Args[1:]))
	if *cluster == "" {
		var err error
		cluster, err = findClusterName()
		if err != nil || *cluster == "" {
			panic("could not determine cluster name. please define using --cluster / -c.")
		}
		log.Println("found cluster name to be:", *cluster)
	}
	sess, err := session.NewSession()
	if err != nil {
		panic(err)
	}
	ec2 := newEC2(*region, sess)
	ecs := newECS(*region, sess)
	if *once {
		updateAndWrite(ec2, ecs)
		return
	}
	execute(ec2, ecs)
	for range time.Tick(time.Second * time.Duration(*freq)) {
		execute(ec2, ecs)
	}
}

func execute(ec2 *ec2Client, ecs *ecsClient) {
	updateAndWrite(ec2, ecs)
	var err error
	err = runSignal()
	if err != nil {
		log.Println("failed to run signal command")
		log.Println("error: ", err)
		switch err := err.(type) {
		case *exec.ExitError:
			log.Println(err.Stderr)
		}
		os.Exit(1)
	}
}

func updateAndWrite(ec2 *ec2Client, ecs *ecsClient) {
	containers, err := newScanner(*cluster, ec2, ecs).scan()
	if err != nil {
		log.Println(err)
	}
	err = writeConfig(containers)
	if err != nil {
		log.Println(err)
	}
}

func runSignal() error {
	log.Println("running signal command")
	output, err := exec.Command("/bin/sh", "-c", *signal).CombinedOutput()
	log.Println("===== output start =====")
	log.Println(string(output))
	log.Println("===== output end =====")
	return err
}

func writeConfig(params []*container) error {
	var containers map[string][]*container
	containers = make(map[string][]*container);
	// Remap the containers as a 2D array with the domain as the index
	for _, v := range params {
		if _, ok := containers[v.Host]; !ok {
			containers[v.Host] = make([]*container, 0)
		}
		containers[v.Host] = append(containers[v.Host], v)
	}
	tmpl, err := template.ParseFiles(*templateFile)
	if err != nil {
		return err
	}
	f, err := os.Create(*outputFile)
	if err != nil {
		return err
	}
	defer f.Close()
	return tmpl.Execute(f, containers)
}

func findClusterName() (*string, error) {
	ip, err := findHostIP()
	if err != nil {
		return nil, err
	}
	meta, err := fetchMetadata(ip)
	if err != nil {
		return nil, err
	}
	return &meta.Cluster, nil
}

func findHostIP() (string, error) {
	result, err := sendHTTRequest("http://169.254.169.254/latest/meta-data/local-ipv4")
	return string(result), err
}

func fetchMetadata(host string) (*metadata, error) {
	result, err := sendHTTRequest("http://" + host + ":51678/v1/metadata")
	var meta metadata
	err = json.Unmarshal(result, &meta)
	if err != nil {
		return nil, err
	}
	return &meta, nil
}

func sendHTTRequest(url string) ([]byte, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	return body, nil
}
<|endoftext|>


<commit_before>package main

import (
	_ "github.com/mattn/go-sqlite3"
	"github.com/zuzuleinen/dave/database"
	"github.com/zuzuleinen/dave/prompt"
)

const DB_PATH = "davedb.db"

func main() {
	db := database.Connect(DB_PATH)
	defer db.Close()

	prompt.SayHello()
	command := prompt.AskForCommand()
	prompt.ObeyCommand(command, db)
}

<commit_msg>improved db path to fetch from the same location<commit_after>package main

import (
	_ "github.com/mattn/go-sqlite3"
	"github.com/zuzuleinen/dave/database"
	"github.com/zuzuleinen/dave/prompt"
	"os"
	"github.com/mitchellh/go-homedir"
)

const DB_FILE = "davedb.db"

func main() {
	db := database.Connect(dbPath())
	defer db.Close()

	prompt.SayHello()
	command := prompt.AskForCommand()
	prompt.ObeyCommand(command, db)
}

func dbPath() string {
	homeDir, _ := homedir.Dir()
	return homeDir + string(os.PathSeparator) + DB_FILE;
}

<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/tajtiattila/vjoy"
	"github.com/tajtiattila/xinput"
	"math"
	"os"
	"time"
)

func main() {
	cfg := NewConfig()
	defcfg := flag.String("defcfg", "", "save default configfile into file specified")
	fn := flag.String("cfg", "", "configfile")
	flag.Parse()

	if *defcfg != "" {
		err := cfg.Save(*defcfg)
		if err != nil {
			abort(err)
		}
		return
	}

	if *fn != "" {
		err := cfg.Load(*fn)
		if err != nil {
			abort(err)
		}
	}

	fmt.Println("vJoy version:", vjoy.Version())
	fmt.Println("  Product:       ", vjoy.ProductString())
	fmt.Println("  Manufacturer:  ", vjoy.ManufacturerString())
	fmt.Println("  Serial number: ", vjoy.SerialNumberString())
	d, err := vjoy.Acquire(1)
	if err != nil {
		abort(err)
	}
	defer d.Relinquish()

	var xs xinput.State
	for {
		xinput.GetState(0, &xs)
		update(cfg, &xs.Gamepad, d)
		time.Sleep(time.Duration(cfg.UpdateMicros) * time.Microsecond)
	}
}

func abort(err error) {
	fmt.Println(err)
	os.Exit(1)
}

func update(c *Config, gp *xinput.Gamepad, d *vjoy.Device) {
	updateAxes(c, gp, d)
	updateButtons(c, gp, d)
	d.Update()
}

func updateAxes(c *Config, gp *xinput.Gamepad, d *vjoy.Device) {
	var lt, rt thumbStick
	lt.set(gp.ThumbLX, gp.ThumbLY)
	rt.set(gp.ThumbRX, gp.ThumbRY)
	if c.ThumbCircle {
		lt.circularize()
		rt.circularize()
	}
	d.Axis(vjoy.AxisX).Setf(axismap(c.ThumbLX, lt.xv, lt.xs))
	d.Axis(vjoy.AxisY).Setf(axismap(c.ThumbLY, lt.yv, lt.ys))
	d.Axis(vjoy.AxisRX).Setf(axismap(c.ThumbRX, rt.xv, rt.xs))
	d.Axis(vjoy.AxisRY).Setf(axismap(c.ThumbRY, rt.yv, rt.ys))
	if c.LeftTrigger.Axis {
		d.Axis(vjoy.AxisZ).Setuf(float32(gp.LeftTrigger) / 255)
	}
	if c.RightTrigger.Axis {
		d.Axis(vjoy.AxisRZ).Setuf(float32(gp.RightTrigger) / 255)
	}
}

func updateButtons(c *Config, gp *xinput.Gamepad, d *vjoy.Device) {
	btns := uint32(gp.Buttons)
	if c.LeftTrigger.touch <= uint16(gp.LeftTrigger) {
		if c.LeftTrigger.pull <= uint16(gp.LeftTrigger) {
			btns |= 1 << 16
		} else {
			btns |= 1 << 17
		}
	}
	if c.RightTrigger.touch <= uint16(gp.RightTrigger) {
		if c.RightTrigger.pull <= uint16(gp.RightTrigger) {
			btns |= 1 << 18
		} else {
			btns |= 1 << 19
		}
	}
	for _, bc := range c.Buttons {
		bc.handler.Handle(d, bc, (btns&bc.fmask) == bc.imask)
	}
}

type thumbStick struct {
	xv float64
	xs float32
	yv float64
	ys float32
}

func (t *thumbStick) set(xi, yi int16) {
	t.xv, t.xs = float64FromInt16(xi)
	t.yv, t.ys = float64FromInt16(yi)
}

func (t *thumbStick) circularize() {
	if t.xv*t.xv+t.yv*t.yv < 1e-3 {
		return
	}

	var u float64
	if t.xv > t.yv {
		u = t.yv / t.xv // yu/xu = yv/xv
	} else {
		u = t.xv / t.yv // xu/yu = xv/yv
	}
	m := math.Sqrt(1 + u*u)
	t.xv *= m
	t.yv *= m
}

func axismap(c *AxisConfig, vabs float64, sign float32) float32 {
	if vabs <= c.Min {
		return 0
	}
	if c.Max <= vabs {
		return sign
	}
	vabs = (vabs - c.Min) / (c.Max - c.Min)
	return sign * float32(math.Pow(vabs, c.Pow))
}

func float64FromInt16(v int16) (abs float64, sign float32) {
	if v < 0 {
		return float64(-int(v)) / 0x8000, -1
	} else {
		return float64(v) / 0x7fff, 1
	}
	return 0, 0 // not reached
}
<commit_msg>main: try to load "joyster.cfg" if no filename given<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/tajtiattila/vjoy"
	"github.com/tajtiattila/xinput"
	"math"
	"os"
	"time"
)

func main() {
	cfg := NewConfig()
	defcfg := flag.String("defcfg", "", "save default configfile into file specified")
	fn := flag.String("cfg", "", "configfile")
	flag.Parse()

	if *defcfg != "" {
		err := cfg.Save(*defcfg)
		if err != nil {
			abort(err)
		}
		return
	}

	if *fn != "" {
		err := cfg.Load(*fn)
		if err != nil {
			abort(err)
		}
	} else {
		if cfg.Load("joyster.cfg") != nil {
			// no error, but reset to default
			cfg = NewConfig()
		}
	}

	fmt.Println("vJoy version:", vjoy.Version())
	fmt.Println("  Product:       ", vjoy.ProductString())
	fmt.Println("  Manufacturer:  ", vjoy.ManufacturerString())
	fmt.Println("  Serial number: ", vjoy.SerialNumberString())
	d, err := vjoy.Acquire(1)
	if err != nil {
		abort(err)
	}
	defer d.Relinquish()

	var xs xinput.State
	for {
		xinput.GetState(0, &xs)
		update(cfg, &xs.Gamepad, d)
		time.Sleep(time.Duration(cfg.UpdateMicros) * time.Microsecond)
	}
}

func abort(err error) {
	fmt.Println(err)
	os.Exit(1)
}

func update(c *Config, gp *xinput.Gamepad, d *vjoy.Device) {
	updateAxes(c, gp, d)
	updateButtons(c, gp, d)
	d.Update()
}

func updateAxes(c *Config, gp *xinput.Gamepad, d *vjoy.Device) {
	var lt, rt thumbStick
	lt.set(gp.ThumbLX, gp.ThumbLY)
	rt.set(gp.ThumbRX, gp.ThumbRY)
	if c.ThumbCircle {
		lt.circularize()
		rt.circularize()
	}
	d.Axis(vjoy.AxisX).Setf(axismap(c.ThumbLX, lt.xv, lt.xs))
	d.Axis(vjoy.AxisY).Setf(axismap(c.ThumbLY, lt.yv, lt.ys))
	d.Axis(vjoy.AxisRX).Setf(axismap(c.ThumbRX, rt.xv, rt.xs))
	d.Axis(vjoy.AxisRY).Setf(axismap(c.ThumbRY, rt.yv, rt.ys))
	if c.LeftTrigger.Axis {
		d.Axis(vjoy.AxisZ).Setuf(float32(gp.LeftTrigger) / 255)
	}
	if c.RightTrigger.Axis {
		d.Axis(vjoy.AxisRZ).Setuf(float32(gp.RightTrigger) / 255)
	}
}

func updateButtons(c *Config, gp *xinput.Gamepad, d *vjoy.Device) {
	btns := uint32(gp.Buttons)
	if c.LeftTrigger.touch <= uint16(gp.LeftTrigger) {
		if c.LeftTrigger.pull <= uint16(gp.LeftTrigger) {
			btns |= 1 << 16
		} else {
			btns |= 1 << 17
		}
	}
	if c.RightTrigger.touch <= uint16(gp.RightTrigger) {
		if c.RightTrigger.pull <= uint16(gp.RightTrigger) {
			btns |= 1 << 18
		} else {
			btns |= 1 << 19
		}
	}
	for _, bc := range c.Buttons {
		bc.handler.Handle(d, bc, (btns&bc.fmask) == bc.imask)
	}
}

type thumbStick struct {
	xv float64
	xs float32
	yv float64
	ys float32
}

func (t *thumbStick) set(xi, yi int16) {
	t.xv, t.xs = float64FromInt16(xi)
	t.yv, t.ys = float64FromInt16(yi)
}

func (t *thumbStick) circularize() {
	if t.xv*t.xv+t.yv*t.yv < 1e-3 {
		return
	}

	var u float64
	if t.xv > t.yv {
		u = t.yv / t.xv // yu/xu = yv/xv
	} else {
		u = t.xv / t.yv // xu/yu = xv/yv
	}
	m := math.Sqrt(1 + u*u)
	t.xv *= m
	t.yv *= m
}

func axismap(c *AxisConfig, vabs float64, sign float32) float32 {
	if vabs <= c.Min {
		return 0
	}
	if c.Max <= vabs {
		return sign
	}
	vabs = (vabs - c.Min) / (c.Max - c.Min)
	return sign * float32(math.Pow(vabs, c.Pow))
}

func float64FromInt16(v int16) (abs float64, sign float32) {
	if v < 0 {
		return float64(-int(v)) / 0x8000, -1
	} else {
		return float64(v) / 0x7fff, 1
	}
	return 0, 0 // not reached
}
<|endoftext|>


<commit_before>package main

import (
	"archive/zip"
	"io"
	"log"
	"net/http"
	"os"
	"time"

	_ "github.com/heroku/x/hmetrics"
)

type remote_file struct {
	path     string
	response *http.Response
}

func main() {
	port := os.Getenv("PORT")
	if port == "" {
		port = "7777"
	}
	log.Println("Listening on port " + port)
	http.ListenAndServe(":"+port, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		r.ParseForm()
		fileName := r.FormValue("filename")
		if fileName == "" {
			fileName = "download.zip"
		}
		w.Header().Add("Content-Disposition", "attachment; filename=\""+fileName+"\"")
		log.Println("Generating zip for " + fileName)

		archive := zip.NewWriter(w)

		receiver := make(chan remote_file, 10)
		done := make(chan error)

		go func() {
			for rf := range receiver {
				log.Println("Received response for " + rf.path)
				entryHeader := &zip.FileHeader{
					Name:   rf.path,
					Method: zip.Deflate,
				}
				entryHeader.SetModTime(time.Now())
				entryHeader.SetMode(os.ModePerm)
				entry, err := archive.CreateHeader(entryHeader)
				if err != nil {
					done <- err
					return
				}

				modifiedTime, err := time.Parse("Mon, 2 Jan 2006 15:04:05 MST", rf.response.Header.Get("Last-Modified"))
				if err != nil {
					entryHeader.SetModTime(modifiedTime)
				}
				io.Copy(entry, rf.response.Body)
				rf.response.Body.Close()
				log.Println("Zipped and closed " + rf.path)
			}
			log.Println("All zipping complete, finishing up.")
			done <- nil
		}()

		for path, urls := range r.PostForm {
			log.Println("Downloading key: " + path + " URL: " + urls[0])
			download, err := http.Get(urls[0])
			if err != nil {
				handleError(err, w)
				return
			}
			receiver <- remote_file{path: path, response: download}
		}
		log.Println("All downloads finished, closing receiver.")
		close(receiver)

		err := <-done
		if err != nil {
			handleError(err, w)
			return
		}
		err = archive.Close()
		if err != nil {
			handleError(err, w)
			return
		}
	}))

}

func handleError(err error, w http.ResponseWriter) {
	http.Error(w, "error", http.StatusBadRequest)
	log.Println(err)
	return
}
<commit_msg>Parallelized downloads correctly while keeping the zipping serial<commit_after>package main

import (
	"archive/zip"
	"io"
	"log"
	"net/http"
	"os"
	"time"

	"sync"

	"strconv"

	_ "github.com/heroku/x/hmetrics"
)

type fileDef struct {
	path string
	url  string
}
type remoteFile struct {
	path     string
	response *http.Response
}

func main() {
	port := os.Getenv("PORT")
	if port == "" {
		port = "7777"
	}
	log.Println("Listening on port " + port)
	http.ListenAndServe(":"+port, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		r.ParseForm()
		fileName := r.FormValue("filename")
		if fileName == "" {
			fileName = "download.zip"
		}
		w.Header().Add("Content-Disposition", "attachment; filename=\""+fileName+"\"")
		log.Println("Generating zip for " + fileName)

		archive := zip.NewWriter(w)

		downloadQueue := make(chan fileDef)
		archiveQueue := make(chan remoteFile)
		downloaderWaitGroup := sync.WaitGroup{}
		finished := make(chan error)

		// Run archiver
		go func() {
			for rf := range archiveQueue {
				log.Println("Received data for " + rf.path)
				entryHeader := &zip.FileHeader{
					Name:   rf.path,
					Method: zip.Deflate,
				}
				entryHeader.SetModTime(time.Now())
				entryHeader.SetMode(os.ModePerm)
				entry, err := archive.CreateHeader(entryHeader)
				if err != nil {
					finished <- err
					return
				}
				log.Println(rf.response.Header.Get("Last-Modified"))
				modifiedTime, err := time.Parse("Mon, 2 Jan 2006 15:04:05 MST", rf.response.Header.Get("Last-Modified"))
				if err != nil {
					entryHeader.SetModTime(modifiedTime)
				}

				io.Copy(entry, rf.response.Body)
				rf.response.Body.Close()
				log.Println("Zipped and closed " + rf.path)
			}
			log.Println("All zipping complete, closing up.")
			err := archive.Close()
			if err != nil {
				finished <- err
				return
			}
			log.Println("Acrhive closed.")
			finished <- nil
		}()

		// Run downloaders
		for w := 1; w <= 5; w++ {
			go func(w int) {
				log.Println("Downloader Ready: " + strconv.Itoa(w))
				for df := range downloadQueue {
					log.Println("Downloading[" + strconv.Itoa(w) + "] path: " + df.path + " URL: " + df.url)
					download, err := http.Get(df.url)
					if err != nil {
						downloaderWaitGroup.Done()
						finished <- err
						return
					}
					archiveQueue <- remoteFile{path: df.path, response: download}
					downloaderWaitGroup.Done()
				}
				log.Println("Downloader Shutting Down: " + strconv.Itoa(w))
			}(w)
		}

		for path, urls := range r.PostForm {
			downloaderWaitGroup.Add(1)
			downloadQueue <- fileDef{path: path, url: urls[0]}
		}
		close(downloadQueue)
		downloaderWaitGroup.Wait()
		close(archiveQueue)
		err := <-finished
		if err != nil {
			handleError(err, w)
			return
		}
	}))
}

func handleError(err error, w http.ResponseWriter) {
	http.Error(w, "error", http.StatusBadRequest)
	log.Println(err)
	return
}
<|endoftext|>


<commit_before>// Copyright (C) 2015 Foursquare Labs Inc.

package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"runtime"
	"time"

	_ "expvar"
	_ "net/http/pprof"

	"github.com/foursquare/fsgo/adminz"
	"github.com/foursquare/fsgo/report"
	"github.com/foursquare/quiver/hfile"
)

type SettingDefs struct {
	port  int
	debug bool
	mlock bool

	configJsonUrl string

	cachePath string

	zk            string
	discoveryPath string
}

var Settings SettingDefs

func readSettings() []string {
	s := SettingDefs{}
	flag.IntVar(&s.port, "port", 9999, "listen port")

	flag.BoolVar(&s.debug, "debug", false, "print debug output")

	flag.BoolVar(&s.mlock, "mlock", false, "mlock mapped files in memory rather than copy to heap.")

	flag.StringVar(&s.configJsonUrl, "config-json", "", "URL of collection configuration json")

	flag.StringVar(&s.cachePath, "cache", os.TempDir(), "local path to write files fetched (*not* cleaned up automatically)")

	flag.StringVar(&s.zk, "zookeeper", "", "zookeeper")
	flag.StringVar(&s.discoveryPath, "discovery", "", "service discovery base path")

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr,
			`
Usage: %s [options] col1=path1 col2=path2 ...
`, os.Args[0])
		flag.PrintDefaults()
	}

	flag.Parse()
	Settings = s

	if (len(flag.Args()) > 0) == (Settings.configJsonUrl != "") {
		log.Println("Collections must be specified OR URL to configuration json.")
		flag.Usage()
		os.Exit(-1)
	}

	return flag.Args()
}

func main() {
	fmt.Println("max procs:", runtime.GOMAXPROCS(-1))

	graphite := report.Flag()
	args := readSettings()

	stats := report.NewRecorder().
		EnableGCInfoCollection().
		MaybeReportTo(graphite).
		SetAsDefault()

	hostname, err := os.Hostname()
	if err != nil {
		hostname = "localhost"
	}

	registrations := new(Registrations)
	if Settings.discoveryPath != "" {
		registrations.Connect()
		defer registrations.Close()
	}

	configs := getCollectionConfig(args)

	log.Printf("Loading collections (debug %v)...\n", Settings.debug)

	t := time.Now()
	cs, err := hfile.LoadCollections(configs, Settings.cachePath)
	if err != nil {
		log.Fatal(err)
	}
	stats.TimeSince("load", t)

	log.Printf("Serving on http://%s:%d/ \n", hostname, Settings.port)

	http.Handle("/rpc/HFileService", NewHttpRpcHandler(cs, stats))
	http.Handle("/", &DebugHandler{cs})

	admin := adminz.New()
	admin.KillfilePaths(adminz.Killfiles(Settings.port))

	admin.OnPause(registrations.Leave)
	admin.OnResume(func() {
		if Settings.discoveryPath != "" {
			registrations.Join(hostname, Settings.discoveryPath, configs, 0)
		}
	})

	admin.Start()

	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", Settings.port), nil))
}
<commit_msg>enable http stats export<commit_after>// Copyright (C) 2015 Foursquare Labs Inc.

package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"runtime"
	"time"

	_ "expvar"
	_ "net/http/pprof"

	"github.com/foursquare/fsgo/adminz"
	"github.com/foursquare/fsgo/report"
	"github.com/foursquare/quiver/hfile"
)

type SettingDefs struct {
	port  int
	debug bool
	mlock bool

	configJsonUrl string

	cachePath string

	zk            string
	discoveryPath string
}

var Settings SettingDefs

func readSettings() []string {
	s := SettingDefs{}
	flag.IntVar(&s.port, "port", 9999, "listen port")

	flag.BoolVar(&s.debug, "debug", false, "print debug output")

	flag.BoolVar(&s.mlock, "mlock", false, "mlock mapped files in memory rather than copy to heap.")

	flag.StringVar(&s.configJsonUrl, "config-json", "", "URL of collection configuration json")

	flag.StringVar(&s.cachePath, "cache", os.TempDir(), "local path to write files fetched (*not* cleaned up automatically)")

	flag.StringVar(&s.zk, "zookeeper", "", "zookeeper")
	flag.StringVar(&s.discoveryPath, "discovery", "", "service discovery base path")

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr,
			`
Usage: %s [options] col1=path1 col2=path2 ...
`, os.Args[0])
		flag.PrintDefaults()
	}

	flag.Parse()
	Settings = s

	if (len(flag.Args()) > 0) == (Settings.configJsonUrl != "") {
		log.Println("Collections must be specified OR URL to configuration json.")
		flag.Usage()
		os.Exit(-1)
	}

	return flag.Args()
}

func main() {
	fmt.Println("max procs:", runtime.GOMAXPROCS(-1))

	graphite := report.Flag()
	args := readSettings()

	stats := report.NewRecorder().
		EnableGCInfoCollection().
		MaybeReportTo(graphite).
		RegisterHttp().
		SetAsDefault()

	hostname, err := os.Hostname()
	if err != nil {
		hostname = "localhost"
	}

	registrations := new(Registrations)
	if Settings.discoveryPath != "" {
		registrations.Connect()
		defer registrations.Close()
	}

	configs := getCollectionConfig(args)

	log.Printf("Loading collections (debug %v)...\n", Settings.debug)

	t := time.Now()
	cs, err := hfile.LoadCollections(configs, Settings.cachePath)
	if err != nil {
		log.Fatal(err)
	}
	stats.TimeSince("load", t)

	log.Printf("Serving on http://%s:%d/ \n", hostname, Settings.port)

	http.Handle("/rpc/HFileService", NewHttpRpcHandler(cs, stats))
	http.Handle("/", &DebugHandler{cs})

	admin := adminz.New()
	admin.KillfilePaths(adminz.Killfiles(Settings.port))

	admin.OnPause(registrations.Leave)
	admin.OnResume(func() {
		if Settings.discoveryPath != "" {
			registrations.Join(hostname, Settings.discoveryPath, configs, 0)
		}
	})

	admin.Start()

	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", Settings.port), nil))
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"
	"os/signal"
	"sync"
	"syscall"

	"github.com/Shopify/sarama"
)

func main() {

	config := InitConfig()

	config.GetConfig()

	consumerBrokers := config.srcBrokers
	producerBrokers := config.dstBrokers

	logFile, err := os.OpenFile(config.firehoseLog, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
	if err != nil {
		log.Fatalln("Failed to open log file:", err)
	}
	defer logFile.Close()

	log.SetOutput(logFile)

	metricsFile, err := os.OpenFile(config.metricsLog, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
	if err != nil {
		log.Fatalln("Failed to open log file:", err)
	}
	defer metricsFile.Close()

	log.Println("Getting the Kafka consumer")
	consumer, err := GetKafkaConsumer(consumerBrokers, metricsFile)
	if err != nil {
		log.Fatalln("Unable to create consumer", err)
	}

	log.Println("Getting the Kafka producer")
	producer, err := GetKafkaProducer(producerBrokers, metricsFile)
	if err != nil {
		log.Fatalln("Unable to create producer", err)
	}

	defer CloseConsumer(&consumer)
	defer CloseProducer(&producer)

	signals := make(chan os.Signal, 1)
	signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT, syscall.SIGQUIT)

	var wg sync.WaitGroup

	configClient := NewClient(config)
	defer configClient.Close()

	transferChan := make(chan sarama.ProducerMessage, 100000)

	log.Println(configClient.GetNumPartitions())
	for partition := 0; partition < configClient.GetNumPartitions(); partition++ {

		syncChan := make(chan int64, 1)
		offset := sarama.OffsetNewest
		finalOffset := int64(-1)

		if config.historical {
			c := NewClient(config)
			offset, finalOffset = c.GetCustomOffset(lastFourHours)
			err := c.Close()
			if err != nil {
				log.Fatalln("Unable to close client. ", err)
			}

		}

		log.Println("Using offset ", offset, " for partition ", partition)
		partitionConsumer, err := consumer.ConsumePartition(config.topic, int32(partition), offset)
		if err != nil {
			log.Fatalln("Unable to create partition consumer", err)
		}

		wg.Add(1)

		go PullFromTopic(partitionConsumer, transferChan, signals, finalOffset, syncChan, &wg)
		log.Println("Started consumer for partition ", partition)

		wg.Add(2)
		go PushToTopic(producer, transferChan, signals, syncChan, &wg)
		log.Println("Started producer")
		go PushToTopic(producer, transferChan, signals, syncChan, &wg)
		log.Println("Started producer")
	}

	go MonitorChan(transferChan, signals, &wg)

	wg.Wait()

	log.Println("All threads done, closing clients and ending")
}
<commit_msg>Attempt to recalculate offset if the first one fails<commit_after>package main

import (
	"log"
	"os"
	"os/signal"
	"sync"
	"syscall"

	"github.com/Shopify/sarama"
)

func main() {

	config := InitConfig()

	config.GetConfig()

	consumerBrokers := config.srcBrokers
	producerBrokers := config.dstBrokers

	logFile, err := os.OpenFile(config.firehoseLog, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
	if err != nil {
		log.Fatalln("Failed to open log file:", err)
	}
	defer logFile.Close()

	log.SetOutput(logFile)

	metricsFile, err := os.OpenFile(config.metricsLog, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
	if err != nil {
		log.Fatalln("Failed to open log file:", err)
	}
	defer metricsFile.Close()

	log.Println("Getting the Kafka consumer")
	consumer, err := GetKafkaConsumer(consumerBrokers, metricsFile)
	if err != nil {
		log.Fatalln("Unable to create consumer", err)
	}

	log.Println("Getting the Kafka producer")
	producer, err := GetKafkaProducer(producerBrokers, metricsFile)
	if err != nil {
		log.Fatalln("Unable to create producer", err)
	}

	defer CloseConsumer(&consumer)
	defer CloseProducer(&producer)

	signals := make(chan os.Signal, 1)
	signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT, syscall.SIGQUIT)

	var wg sync.WaitGroup

	configClient := NewClient(config)
	defer configClient.Close()

	transferChan := make(chan sarama.ProducerMessage, 100000)

	log.Println(configClient.GetNumPartitions())
	for partition := 0; partition < configClient.GetNumPartitions(); partition++ {

		syncChan := make(chan int64, 1)
		offset := sarama.OffsetNewest
		finalOffset := int64(-1)

		if config.historical {
			c := NewClient(config)
			offset, finalOffset = c.GetCustomOffset(lastFourHours)
			err := c.Close()
			if err != nil {
				log.Fatalln("Unable to close client. ", err)
			}

		}

		log.Println("Using offset ", offset, " for partition ", partition)
		partitionConsumer, err := consumer.ConsumePartition(config.topic, int32(partition), offset)
		if err != nil {
			log.Println("Unable to create partition consumer", err)
			c := NewClient(config)
			offset, finalOffset = c.GetCustomOffset(lastFourHours)
			err := c.Close()
			if err != nil {
				log.Fatalln("Unable to close client. ", err)
			}
			partitionConsumer, err = consumer.ConsumePartition(config.topic, int32(partition), offset)
			if err != nil {
				log.Println("Unable to create partition consumer", err)
				return
			}
		}

		wg.Add(1)

		go PullFromTopic(partitionConsumer, transferChan, signals, finalOffset, syncChan, &wg)
		log.Println("Started consumer for partition ", partition)

		wg.Add(2)
		go PushToTopic(producer, transferChan, signals, syncChan, &wg)
		log.Println("Started producer")
		go PushToTopic(producer, transferChan, signals, syncChan, &wg)
		log.Println("Started producer")
	}

	go MonitorChan(transferChan, signals, &wg)

	wg.Wait()

	log.Println("All threads done, closing clients and ending")
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"net"
	"os"
	"path/filepath"
	"runtime"
	"runtime/pprof"
	"strconv"
)

var (
	envPath  = os.Getenv("PATH")
	envLevel = os.Getenv("LF_LEVEL")
)

var (
	gClientID      int
	gHostname      string
	gLastDirPath   string
	gSelectionPath string
	gSocketProt    string
	gSocketPath    string
	gLogPath       string
	gServerLogPath string
	gSelect        string
	gCommand       string
	gVersion       string
)

func init() {
	h, err := os.Hostname()
	if err != nil {
		log.Printf("hostname: %s", err)
	}
	gHostname = h

	if envLevel == "" {
		envLevel = "0"
	}
}

func exportEnvVars() {
	os.Setenv("id", strconv.Itoa(gClientID))

	os.Setenv("OPENER", envOpener)
	os.Setenv("EDITOR", envEditor)
	os.Setenv("PAGER", envPager)
	os.Setenv("SHELL", envShell)

	level, err := strconv.Atoi(envLevel)
	if err != nil {
		log.Printf("reading lf level: %s", err)
	}

	level++

	os.Setenv("LF_LEVEL", strconv.Itoa(level))
}

func startServer() {
	cmd := detachedCommand(os.Args[0], "-server")
	if err := cmd.Start(); err != nil {
		log.Printf("starting server: %s", err)
	}
}

func checkServer() {
	if gSocketProt == "unix" {
		if _, err := os.Stat(gSocketPath); os.IsNotExist(err) {
			startServer()
		} else if _, err := net.Dial(gSocketProt, gSocketPath); err != nil {
			os.Remove(gSocketPath)
			startServer()
		}
	} else {
		if _, err := net.Dial(gSocketProt, gSocketPath); err != nil {
			startServer()
		}
	}
}

func main() {
	showDoc := flag.Bool(
		"doc",
		false,
		"show documentation")

	showVersion := flag.Bool(
		"version",
		false,
		"show version")

	remoteCmd := flag.String(
		"remote",
		"",
		"send remote command to server")

	serverMode := flag.Bool(
		"server",
		false,
		"start server (automatic)")

	cpuprofile := flag.String(
		"cpuprofile",
		"",
		"path to the file to write the CPU profile")

	memprofile := flag.String(
		"memprofile",
		"",
		"path to the file to write the memory profile")

	flag.StringVar(&gLastDirPath,
		"last-dir-path",
		"",
		"path to the file to write the last dir on exit (to use for cd)")

	flag.StringVar(&gSelectionPath,
		"selection-path",
		"",
		"path to the file to write selected files on open (to use as open file dialog)")

	flag.StringVar(&gCommand,
		"command",
		"",
		"command to execute on client initialization")

	flag.Parse()

	gSocketProt = gDefaultSocketProt
	gSocketPath = gDefaultSocketPath

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatalf("could not create CPU profile: %s", err)
		}
		if err := pprof.StartCPUProfile(f); err != nil {
			log.Fatalf("could not start CPU profile: %s", err)
		}
		defer pprof.StopCPUProfile()
	}

	switch {
	case *showDoc:
		fmt.Print(genDocString)
	case *showVersion:
		fmt.Println(gVersion)
	case *remoteCmd != "":
		if err := remote(*remoteCmd); err != nil {
			log.Fatalf("remote command: %s", err)
		}
	case *serverMode:
		gServerLogPath = filepath.Join(os.TempDir(), fmt.Sprintf("lf.%s.server.log", gUser.Username))
		serve()
	default:
		checkServer()

		gClientID = 1000
		gLogPath = filepath.Join(os.TempDir(), fmt.Sprintf("lf.%s.%d.log", gUser.Username, gClientID))
		for _, err := os.Stat(gLogPath); !os.IsNotExist(err); _, err = os.Stat(gLogPath) {
			gClientID++
			gLogPath = filepath.Join(os.TempDir(), fmt.Sprintf("lf.%s.%d.log", gUser.Username, gClientID))
		}

		switch flag.NArg() {
		case 0:
			_, err := os.Getwd()
			if err != nil {
				fmt.Fprintf(os.Stderr, "%s\n", err)
				os.Exit(2)
			}
		case 1:
			gSelect = flag.Arg(0)
		default:
			fmt.Fprintf(os.Stderr, "only single file or directory is allowed\n")
			os.Exit(2)
		}

		exportEnvVars()

		run()
	}

	if *memprofile != "" {
		f, err := os.Create(*memprofile)
		if err != nil {
			log.Fatal("could not create memory profile: ", err)
		}
		runtime.GC()
		if err := pprof.WriteHeapProfile(f); err != nil {
			log.Fatal("could not write memory profile: ", err)
		}
		f.Close()
	}
}
<commit_msg>start server process in home directory<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"net"
	"os"
	"path/filepath"
	"runtime"
	"runtime/pprof"
	"strconv"
)

var (
	envPath  = os.Getenv("PATH")
	envLevel = os.Getenv("LF_LEVEL")
)

var (
	gClientID      int
	gHostname      string
	gLastDirPath   string
	gSelectionPath string
	gSocketProt    string
	gSocketPath    string
	gLogPath       string
	gServerLogPath string
	gSelect        string
	gCommand       string
	gVersion       string
)

func init() {
	h, err := os.Hostname()
	if err != nil {
		log.Printf("hostname: %s", err)
	}
	gHostname = h

	if envLevel == "" {
		envLevel = "0"
	}
}

func exportEnvVars() {
	os.Setenv("id", strconv.Itoa(gClientID))

	os.Setenv("OPENER", envOpener)
	os.Setenv("EDITOR", envEditor)
	os.Setenv("PAGER", envPager)
	os.Setenv("SHELL", envShell)

	level, err := strconv.Atoi(envLevel)
	if err != nil {
		log.Printf("reading lf level: %s", err)
	}

	level++

	os.Setenv("LF_LEVEL", strconv.Itoa(level))
}

func startServer() {
	cmd := detachedCommand(os.Args[0], "-server")
	cmd.Dir = gUser.HomeDir
	if err := cmd.Start(); err != nil {
		log.Printf("starting server: %s", err)
	}
}

func checkServer() {
	if gSocketProt == "unix" {
		if _, err := os.Stat(gSocketPath); os.IsNotExist(err) {
			startServer()
		} else if _, err := net.Dial(gSocketProt, gSocketPath); err != nil {
			os.Remove(gSocketPath)
			startServer()
		}
	} else {
		if _, err := net.Dial(gSocketProt, gSocketPath); err != nil {
			startServer()
		}
	}
}

func main() {
	showDoc := flag.Bool(
		"doc",
		false,
		"show documentation")

	showVersion := flag.Bool(
		"version",
		false,
		"show version")

	remoteCmd := flag.String(
		"remote",
		"",
		"send remote command to server")

	serverMode := flag.Bool(
		"server",
		false,
		"start server (automatic)")

	cpuprofile := flag.String(
		"cpuprofile",
		"",
		"path to the file to write the CPU profile")

	memprofile := flag.String(
		"memprofile",
		"",
		"path to the file to write the memory profile")

	flag.StringVar(&gLastDirPath,
		"last-dir-path",
		"",
		"path to the file to write the last dir on exit (to use for cd)")

	flag.StringVar(&gSelectionPath,
		"selection-path",
		"",
		"path to the file to write selected files on open (to use as open file dialog)")

	flag.StringVar(&gCommand,
		"command",
		"",
		"command to execute on client initialization")

	flag.Parse()

	gSocketProt = gDefaultSocketProt
	gSocketPath = gDefaultSocketPath

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatalf("could not create CPU profile: %s", err)
		}
		if err := pprof.StartCPUProfile(f); err != nil {
			log.Fatalf("could not start CPU profile: %s", err)
		}
		defer pprof.StopCPUProfile()
	}

	switch {
	case *showDoc:
		fmt.Print(genDocString)
	case *showVersion:
		fmt.Println(gVersion)
	case *remoteCmd != "":
		if err := remote(*remoteCmd); err != nil {
			log.Fatalf("remote command: %s", err)
		}
	case *serverMode:
		gServerLogPath = filepath.Join(os.TempDir(), fmt.Sprintf("lf.%s.server.log", gUser.Username))
		serve()
	default:
		checkServer()

		gClientID = 1000
		gLogPath = filepath.Join(os.TempDir(), fmt.Sprintf("lf.%s.%d.log", gUser.Username, gClientID))
		for _, err := os.Stat(gLogPath); !os.IsNotExist(err); _, err = os.Stat(gLogPath) {
			gClientID++
			gLogPath = filepath.Join(os.TempDir(), fmt.Sprintf("lf.%s.%d.log", gUser.Username, gClientID))
		}

		switch flag.NArg() {
		case 0:
			_, err := os.Getwd()
			if err != nil {
				fmt.Fprintf(os.Stderr, "%s\n", err)
				os.Exit(2)
			}
		case 1:
			gSelect = flag.Arg(0)
		default:
			fmt.Fprintf(os.Stderr, "only single file or directory is allowed\n")
			os.Exit(2)
		}

		exportEnvVars()

		run()
	}

	if *memprofile != "" {
		f, err := os.Create(*memprofile)
		if err != nil {
			log.Fatal("could not create memory profile: ", err)
		}
		runtime.GC()
		if err := pprof.WriteHeapProfile(f); err != nil {
			log.Fatal("could not write memory profile: ", err)
		}
		f.Close()
	}
}
<|endoftext|>