<commit_before>package main

import (
	"fmt"
	"github.com/jideji/servicelauncher/config"
	"github.com/jideji/servicelauncher/service"
	"github.com/jideji/servicelauncher/web"
	"net/http"
	"os"
	"sort"
)

func main() {
	if len(os.Args) < 2 {
		showHelp()
	}

	action := os.Args[1]

	services := config.LoadServices()

	if action == "server" {
		h := web.WebHandler(services)
		http.Handle("/", h)
		fmt.Println("Listening on port :8080")
		err := http.ListenAndServe(":8080", nil)
		if err != nil {
			panic(err)
		}
		return
	}

	selected := resolveServices(services, os.Args[2:]...)

	lastErrCode := 0
	for _, service := range selected {
		err := doAction(service, action)
		if err != nil {
			if c, ok := err.(CmdError); ok {
				println(c.Msg)
				lastErrCode = c.Code
			} else {
				println(err.Error())
			}
		}
	}

	os.Exit(lastErrCode)
}

func resolveServices(services map[string]service.Service, names ...string) []service.Service {
	var selected []service.Service
	if len(names) > 0 {
		for _, name := range names {
			service, ok := services[name]
			if !ok {
				println(fmt.Sprintf("No service named '%s' found.", name))
				os.Exit(10)
			}
			selected = append(selected, service)
		}
	} else {
		for _, service := range services {
			selected = append(selected, service)
		}
	}
	sort.Sort(byName(selected))

	return selected
}

func doAction(srv service.Service, action string) error {
	running, err := srv.IsRunning()
	if err != nil {
		return err
	}

	if action == "status" {
		if running {
			pid, _ := srv.Pid()
			fmt.Printf("(\033[32mx\033[0m) Service '%s' is running with pid %d.\n", srv.Name(), pid)
		} else {
			fmt.Printf("( ) Service '%s' is not running.\n", srv.Name())
		}
		return nil
	}

	if action == "stop" || action == "restart" {
		if running {
			pid, err := srv.Pid()
			if err != nil {
				return err
			}
			fmt.Printf("Killing service '%s' (process %d).\n", srv.Name(), pid)
			srv.Stop()
			running = false
		} else {
			fmt.Printf("(\033[33m!\033[0m) Service '%s' not running.\n", srv.Name())
		}
	}

	if action == "start" || action == "restart" {
		if running {
			return cmdError(fmt.Sprintf("(\033[31m!\033[0m) Service '%s' already running. Try restart.", srv.Name()), 11)
		}
		err := srv.Start()
		if err != nil {
			return err
		}
		pid, err := srv.Pid()
		if err != nil {
			return err
		}
		fmt.Printf("Service '%s' started with pid %d.\n", srv.Name(), pid)
	}
	return nil
}

func showHelp() {
	fmt.Fprint(os.Stderr, "SYNTAX:\n")
	fmt.Fprintf(os.Stderr, "\t%s <action> [<service name>]\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "\t%s server\n", os.Args[0])
	fmt.Fprint(os.Stderr, "Actions:\n")
	fmt.Fprint(os.Stderr, "\tstart, stop, restart, status\n")
	fmt.Fprint(os.Stderr, "Examples:\n")
	fmt.Fprintf(os.Stderr, "\t%s start httpserver\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "\t%s status\n", os.Args[0])
	os.Exit(1)
}

func cmdError(msg string, code int) error {
	return CmdError{Msg: msg, Code: code}
}

type CmdError struct {
	Msg  string
	Code int
}

func (c CmdError) Error() string {
	return c.Msg
}

// ByName implements sort.Interface for []ServiceStatus based on
// the Name field.
type byName []service.Service

func (a byName) Len() int           { return len(a) }
func (a byName) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a byName) Less(i, j int) bool { return a[i].Name() < a[j].Name() }
<commit_msg>Added list command.<commit_after>package main

import (
	"fmt"
	"github.com/jideji/servicelauncher/config"
	"github.com/jideji/servicelauncher/service"
	"github.com/jideji/servicelauncher/web"
	"net/http"
	"os"
	"sort"
)

func main() {
	if len(os.Args) < 2 {
		showHelp()
	}

	action := os.Args[1]

	services := config.LoadServices()

	if action == "server" {
		h := web.WebHandler(services)
		http.Handle("/", h)
		fmt.Println("Listening on port :8080")
		err := http.ListenAndServe(":8080", nil)
		if err != nil {
			panic(err)
		}
		return
	}

	selected := resolveServices(services, os.Args[2:]...)

	lastErrCode := 0
	for _, service := range selected {
		err := doAction(service, action)
		if err != nil {
			if c, ok := err.(CmdError); ok {
				println(c.Msg)
				lastErrCode = c.Code
			} else {
				println(err.Error())
			}
		}
	}

	os.Exit(lastErrCode)
}

func resolveServices(services map[string]service.Service, names ...string) []service.Service {
	var selected []service.Service
	if len(names) > 0 {
		for _, name := range names {
			service, ok := services[name]
			if !ok {
				println(fmt.Sprintf("No service named '%s' found.", name))
				os.Exit(10)
			}
			selected = append(selected, service)
		}
	} else {
		for _, service := range services {
			selected = append(selected, service)
		}
	}
	sort.Sort(byName(selected))

	return selected
}

func doAction(srv service.Service, action string) error {
	if action == "list" {
		fmt.Println(srv.Name())
		return nil
	}

	running, err := srv.IsRunning()
	if err != nil {
		return err
	}

	if action == "status" {
		if running {
			pid, _ := srv.Pid()
			fmt.Printf("(\033[32mx\033[0m) Service '%s' is running with pid %d.\n", srv.Name(), pid)
		} else {
			fmt.Printf("( ) Service '%s' is not running.\n", srv.Name())
		}
		return nil
	}

	if action == "stop" || action == "restart" {
		if running {
			pid, err := srv.Pid()
			if err != nil {
				return err
			}
			fmt.Printf("Killing service '%s' (process %d).\n", srv.Name(), pid)
			srv.Stop()
			running = false
		} else {
			fmt.Printf("(\033[33m!\033[0m) Service '%s' not running.\n", srv.Name())
		}
	}

	if action == "start" || action == "restart" {
		if running {
			return cmdError(fmt.Sprintf("(\033[31m!\033[0m) Service '%s' already running. Try restart.", srv.Name()), 11)
		}
		err := srv.Start()
		if err != nil {
			return err
		}
		pid, err := srv.Pid()
		if err != nil {
			return err
		}
		fmt.Printf("Service '%s' started with pid %d.\n", srv.Name(), pid)
	}
	return nil
}

func showHelp() {
	fmt.Fprint(os.Stderr, "SYNTAX:\n")
	fmt.Fprintf(os.Stderr, "\t%s <action> [<service name>]\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "\t%s server\n", os.Args[0])
	fmt.Fprint(os.Stderr, "Actions:\n")
	fmt.Fprint(os.Stderr, "\tstart, stop, restart, status\n")
	fmt.Fprint(os.Stderr, "Examples:\n")
	fmt.Fprintf(os.Stderr, "\t%s start httpserver\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "\t%s status\n", os.Args[0])
	os.Exit(1)
}

func cmdError(msg string, code int) error {
	return CmdError{Msg: msg, Code: code}
}

type CmdError struct {
	Msg  string
	Code int
}

func (c CmdError) Error() string {
	return c.Msg
}

// ByName implements sort.Interface for []ServiceStatus based on
// the Name field.
type byName []service.Service

func (a byName) Len() int           { return len(a) }
func (a byName) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a byName) Less(i, j int) bool { return a[i].Name() < a[j].Name() }
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	cb "github.com/clearblade/Go-SDK"
	"github.com/clearblade/cblib"
	"os"
	"strings"
)

func init() {
	flag.StringVar(&cblib.URL, "url", "", "Platform URL")
	flag.BoolVar(&cblib.Help, "help", false, "Print help message")
	flag.StringVar(&cblib.Email, "email", "", "Developer username")
	flag.StringVar(&cblib.Password, "password", "", "Developer password")
}

func main() {
	flag.Usage = func() {
		fmt.Printf("DA USAGE\n")
	}
	flag.Parse()
	if cblib.URL != "" {
		cb.CB_ADDR = cblib.URL
	} else {
		cblib.URL = cb.CB_ADDR
	}
	client, err := doAuth()
	if err != nil {
		fmt.Println(err.Error())
		os.Exit(1)
	}
	cmd := strings.ToLower(flag.Arg(0))
	subCommand, err := cblib.GetCommand(cmd)
	if err != nil {
		fmt.Println(err.Error())
		os.Exit(1)
	}
	subArgs := flag.Args()
	err = subCommand.Execute(client, subArgs[1:])
	if err != nil {
		fmt.Printf("Aborting: %s\n", err.Error())
	}
}

func doAuth() (*cb.DevClient, error) {
	if cblib.Email != "" && cblib.Password != "" {
		return cblib.AuthUserPass(cblib.Email, cblib.Password)
	} else {
		return cblib.Auth("")
	}
}
<commit_msg>fixed auth model<commit_after>package main

import (
	"flag"
	"fmt"
	cb "github.com/clearblade/Go-SDK"
	"github.com/clearblade/cblib"
	"os"
	"strings"
)

func init() {
	flag.StringVar(&cblib.URL, "url", "", "Platform URL")
	flag.BoolVar(&cblib.Help, "help", false, "Print help message")
	flag.StringVar(&cblib.Email, "email", "", "Developer username")
	flag.StringVar(&cblib.Password, "password", "", "Developer password")
}

func main() {
	flag.Usage = func() {
		fmt.Printf("DA USAGE\n")
	}
	if err := setupFromOptsAndCBMeta(); err != nil {
		fmt.Println(err.Error())
		os.Exit(1)
	}
	client, err := doAuth()
	if err != nil {
		fmt.Println(err.Error())
		os.Exit(1)
	}
	cmd := strings.ToLower(flag.Arg(0))
	subCommand, err := cblib.GetCommand(cmd)
	if err != nil {
		fmt.Println(err.Error())
		os.Exit(1)
	}
	subArgs := flag.Args()
	err = subCommand.Execute(client, subArgs[1:])
	if err != nil {
		fmt.Printf("Aborting: %s\n", err.Error())
	}
}

func setupFromOptsAndCBMeta() error {
	flag.Parse()
	cblib.DevToken, _ = cblib.Load_auth_info(cblib.AuthInfoFile)
	err := cblib.GoToRepoRootDir()
	if err != nil && err.Error() != cblib.SpecialNoCBMetaError {
		return err
	}
	if cblib.URL != "" {
		cb.CB_ADDR = cblib.URL
	} else if url, ok := cblib.MetaInfo["platformURL"]; ok {
		cblib.URL = url.(string)
		cb.CB_ADDR = url.(string)
	} else {
		cblib.URL = cb.CB_ADDR
	}

	fmt.Printf("Using system at '%s'\n", cb.CB_ADDR)

	if cblib.Email == "" {
		if email, ok := cblib.MetaInfo["developerEmail"]; ok {
			cblib.Email = email.(string)
		}
	} else {
		cblib.CommandLineEmail = true
	}
	return nil
}

func doAuth() (*cb.DevClient, error) {
	if cblib.Email != "" && cblib.Password != "" {
		return cblib.AuthUserPass(cblib.Email, cblib.Password)
	} else if cblib.CommandLineEmail {
		return cblib.AuthPromptPass(cblib.Email)
	} else {
		return cblib.Auth("")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"os/signal"
	"strconv"
	"strings"

	"./schemas"
	"github.com/Shopify/sarama"
	"github.com/companieshouse/chs.go/avro"
	"github.com/companieshouse/chs.go/avro/schema"
	"github.com/companieshouse/chs.go/kafka/producer"
)

// Assigns all flags to variables
var (
	brokerPtr         = flag.String("broker", "", "Broker address")
	topicPtr          = flag.String("topic", "", "Topic name")
	schemaPtr         = flag.String("schema", "", "Schema name")
	schemaRegistryPtr = flag.String("schema-registry", "", "Schema Registry")
	partitionPtr      = flag.Int64("partition", 0, "Partition (default: 0)")
	offsetPtr         = flag.String("offset", "", "Offset number")
	jsonOutPtr        = flag.Int64("json-out", 0, "Print deserialized JSON message (default: 0)")
)

func main() {
	flag.Parse()

	// check if all mandatory flags have been provided
	validateFlags()

	// Print flags and parameters of the tool
	printFlags()

	// create offset array
	offsetArray := createOffsetArray(*offsetPtr)

	// create default config for sarama
	config := sarama.NewConfig()

	//create consumer
	consumer, err := sarama.NewConsumer([]string{*brokerPtr}, config)
	if err != nil {
		panic(err)
	}

	// create messages chan
	messages := make(chan *sarama.ConsumerMessage)
	go consumePartition(consumer, *topicPtr, int32(*partitionPtr), messages, offsetArray)

	// create signals chan
	signals := make(chan os.Signal, 1)
	signal.Notify(signals, os.Interrupt)

	consumed := 0
ConsumerLoop:
	for {
		select {
		case msg := <-messages:

			schemaStruct := schemas.IdentifySchema(*topicPtr)
			var err error

			// Get schema from schema registry and use it to create avro consumer
			schema, err := schema.Get(*schemaRegistryPtr, *schemaPtr)
			consumerAvro := &avro.Schema{
				Definition: schema,
			}

			// Unmarshal message value and assign it to schemaStruct
			if err = consumerAvro.Unmarshal(msg.Value, schemaStruct); err != nil {
				fmt.Errorf("error unmarshalling avro: %v", err)
			}

			// create avro producer
			producerAvro := &avro.Schema{
				Definition: schema,
			}

			// Marshall schemaStruct that contains message value ready for republishing
			messageBytes, err := producerAvro.Marshal(schemaStruct)
			if err != nil {
				fmt.Errorf("error marshalling avro: %v", err)
			}

			// create producer message
			producerMessage := &producer.Message{
				Value: messageBytes,
				Topic: *topicPtr,
			}

			//create new producer
			p, err := producer.New(&producer.Config{Acks: &producer.WaitForAll, BrokerAddrs: []string{*brokerPtr}})
			if err != nil {
				fmt.Errorf("error creating producer: %v", err)
				os.Exit(1)
			}

			// republish message
			partition, offset, err := p.Send(producerMessage)
			if err != nil {
				fmt.Errorf("error republishing message to topic: %v", err)
			}

			fmt.Printf("Message republished to topic: %v using partition: %v offset: %v \n", *topicPtr, partition, offset)

			consumed++
		case <-signals:
			fmt.Println("break")
			break ConsumerLoop
		}
	}
}

// Validates the flags to make sure all mandatory flags have been supplied
// throw an error if not the case
func validateFlags() {
	flag.VisitAll(func(f *flag.Flag) {
		if string(f.Value.String()) == "" {
			fmt.Printf("Value not supplied for: %v \n", f.Name)
			os.Exit(1)
		}
	})
}

// Prints the flags and parameters of the tool
func printFlags() {
	fmt.Println("---------------------")
	fmt.Println("Parameters:")
	flag.VisitAll(func(f *flag.Flag) {
		fmt.Printf("%v: %v\n", f.Name, f.Value)
	})
	fmt.Println("---------------------")
}

// The offset slice is passed into method and is iterated over and each message
// related to the offset in the topic/partition is outputted into the 'out' chan
func consumePartition(consumer sarama.Consumer, topic string, partition int32, out chan *sarama.ConsumerMessage, offsetArray []int64) {
	for offset := range offsetArray {
		partitionConsumer, err := consumer.ConsumePartition(topic, partition, int64(offset))
		if err != nil {
			panic(err)
		}

		select {
		case msg := <-partitionConsumer.Messages():
			partitionConsumer.Close()
			out <- msg
		}
	}
}

// This creates the offset array depending on whether a range or a single value
// is entered as an argument to the tool
func createOffsetArray(offset string) []int64 {
	arraySize := make([]int64, 0)
	if strings.ContainsAny(offset, "-") {
		slice := strings.Split(offset, "-")
		minRange, err := strconv.ParseInt(slice[0], 10, 64)
		maxRange, err := strconv.ParseInt(slice[1], 10, 64)
		if err != nil {
			panic(err)
		}

		index := 0
		for value := minRange; value <= maxRange; value++ {
			arraySize = append(arraySize, value)
			index++
		}
	} else {
		value, err := strconv.ParseInt(offset, 10, 64)
		if err != nil {
			panic(err)
		}
		arraySize = append(arraySize, value)
	}
	return arraySize
}
<commit_msg>Add exit statement if error is thrown<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"os/signal"
	"strconv"
	"strings"

	"./schemas"
	"github.com/Shopify/sarama"
	"github.com/companieshouse/chs.go/avro"
	"github.com/companieshouse/chs.go/avro/schema"
	"github.com/companieshouse/chs.go/kafka/producer"
)

// Assigns all flags to variables
var (
	brokerPtr         = flag.String("broker", "", "Broker address")
	topicPtr          = flag.String("topic", "", "Topic name")
	schemaPtr         = flag.String("schema", "", "Schema name")
	schemaRegistryPtr = flag.String("schema-registry", "", "Schema Registry")
	partitionPtr      = flag.Int64("partition", 0, "Partition (default: 0)")
	offsetPtr         = flag.String("offset", "", "Offset number")
	jsonOutPtr        = flag.Int64("json-out", 0, "Print deserialized JSON message (default: 0)")
)

func main() {
	flag.Parse()

	// check if all mandatory flags have been provided
	validateFlags()

	// Print flags and parameters of the tool
	printFlags()

	// create offset array
	offsetArray := createOffsetArray(*offsetPtr)

	// create default config for sarama
	config := sarama.NewConfig()

	//create consumer
	consumer, err := sarama.NewConsumer([]string{*brokerPtr}, config)
	if err != nil {
		panic(err)
	}

	// create messages chan
	messages := make(chan *sarama.ConsumerMessage)
	go consumePartition(consumer, *topicPtr, int32(*partitionPtr), messages, offsetArray)

	// create signals chan
	signals := make(chan os.Signal, 1)
	signal.Notify(signals, os.Interrupt)

	consumed := 0
ConsumerLoop:
	for {
		select {
		case msg := <-messages:

			schemaStruct := schemas.IdentifySchema(*topicPtr)
			var err error

			// Get schema from schema registry and use it to create avro consumer
			schema, err := schema.Get(*schemaRegistryPtr, *schemaPtr)
			consumerAvro := &avro.Schema{
				Definition: schema,
			}

			// Unmarshal message value and assign it to schemaStruct
			if err = consumerAvro.Unmarshal(msg.Value, schemaStruct); err != nil {
				fmt.Errorf("error unmarshalling avro: %v", err)
				os.Exit(1)
			}

			// create avro producer
			producerAvro := &avro.Schema{
				Definition: schema,
			}

			// Marshall schemaStruct that contains message value ready for republishing
			messageBytes, err := producerAvro.Marshal(schemaStruct)
			if err != nil {
				fmt.Errorf("error marshalling avro: %v", err)
				os.Exit(1)
			}

			// create producer message
			producerMessage := &producer.Message{
				Value: messageBytes,
				Topic: *topicPtr,
			}

			//create new producer
			p, err := producer.New(&producer.Config{Acks: &producer.WaitForAll, BrokerAddrs: []string{*brokerPtr}})
			if err != nil {
				fmt.Errorf("error creating producer: %v", err)
				os.Exit(1)
			}

			// republish message
			partition, offset, err := p.Send(producerMessage)
			if err != nil {
				fmt.Errorf("error republishing message to topic: %v", err)
				os.Exit(1)
			}

			fmt.Printf("Message republished to topic: %v using partition: %v offset: %v \n", *topicPtr, partition, offset)

			consumed++
		case <-signals:
			fmt.Println("break")
			break ConsumerLoop
		}
	}
}

// Validates the flags to make sure all mandatory flags have been supplied
// throw an error if not the case
func validateFlags() {
	flag.VisitAll(func(f *flag.Flag) {
		if string(f.Value.String()) == "" {
			fmt.Printf("Value not supplied for: %v \n", f.Name)
			os.Exit(1)
		}
	})
}

// Prints the flags and parameters of the tool
func printFlags() {
	fmt.Println("---------------------")
	fmt.Println("Parameters:")
	flag.VisitAll(func(f *flag.Flag) {
		fmt.Printf("%v: %v\n", f.Name, f.Value)
	})
	fmt.Println("---------------------")
}

// The offset slice is passed into method and is iterated over and each message
// related to the offset in the topic/partition is outputted into the 'out' chan
func consumePartition(consumer sarama.Consumer, topic string, partition int32, out chan *sarama.ConsumerMessage, offsetArray []int64) {
	for offset := range offsetArray {
		partitionConsumer, err := consumer.ConsumePartition(topic, partition, int64(offset))
		if err != nil {
			panic(err)
		}

		select {
		case msg := <-partitionConsumer.Messages():
			partitionConsumer.Close()
			out <- msg
		}
	}
}

// This creates the offset array depending on whether a range or a single value
// is entered as an argument to the tool
func createOffsetArray(offset string) []int64 {
	arraySize := make([]int64, 0)
	if strings.ContainsAny(offset, "-") {
		slice := strings.Split(offset, "-")
		minRange, err := strconv.ParseInt(slice[0], 10, 64)
		maxRange, err := strconv.ParseInt(slice[1], 10, 64)
		if err != nil {
			panic(err)
		}

		index := 0
		for value := minRange; value <= maxRange; value++ {
			arraySize = append(arraySize, value)
			index++
		}
	} else {
		value, err := strconv.ParseInt(offset, 10, 64)
		if err != nil {
			panic(err)
		}
		arraySize = append(arraySize, value)
	}
	return arraySize
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"github.com/zenazn/goji/bind"
	"github.com/zenazn/goji/web"
	"github.com/zenazn/goji/web/middleware"
	"log"
	"net/http"
	"net/http/pprof"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"time"
)

var clusterNames = flag.String("cluster-name", "default", "The user-visible names for the clusters")
var resourceManagerURL = flag.String("resource-manager-url", "http://localhost:8088", "The HTTP URL to access the resource manager.")
var historyServerURL = flag.String("history-server-url", "http://localhost:19888", "The HTTP URL to access the history server.")
var proxyServerURL = flag.String("proxy-server-url", "", "The HTTP URL to access the proxy server, if separate from the resource manager.")
var namenodeAddress = flag.String("namenode-address", "localhost:9000", "The host:port to access the Namenode metadata service.")
var yarnLogDir = flag.String("yarn-logs-dir", "/tmp/logs", "The HDFS path where YARN stores logs. This is the controlled by the hadoop property yarn.nodemanager.remote-app-log-dir.")
var yarnHistoryDir = flag.String("yarn-history-dir", "/tmp/staging/history/done", "The HDFS path where YARN stores finished job history files. This is the controlled by the hadoop property mapreduce.jobhistory.done-dir.")
var httpTimeout = flag.Duration("http-timeout", time.Second*2, "The timeout used for connecting to YARN API. Pass values like: 2s")
var pollInterval = flag.Duration("poll-interval", time.Second*5, "How often should we poll the job APIs. Pass values like: 2s")
var enableDebug = flag.Bool("pprof", false, "Enable pprof debugging tools at /debug.")
var s3BucketName = flag.String("s3-bucket", "", "S3 bucket to fetch old jobs from")
var s3Region = flag.String("s3-region", "", "AWS region for the job storage S3 bucket")
var s3JobsPrefix = flag.String("s3-jobs-prefix", "", "S3 key prefix (\"folder\") where jobs are stored")
var s3FlowPrefix = flag.String("s3-flow-prefix", "", "S3 key prefix (\"folder\") where cascading flows are stored")

var jts map[string]*jobTracker
var persistedJobClient PersistedJobClient

var rootPath, staticPath string

var mux *web.Mux

func init() {
	bind.WithFlag()
	mux = web.New()
	mux.Use(middleware.RequestID)
	mux.Use(middleware.Logger)
	mux.Use(middleware.Recoverer)
	mux.Use(middleware.AutomaticOptions)
}

func index(c web.C, w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, filepath.Join(rootPath, "index.html"))
}

func getJobs(c web.C, w http.ResponseWriter, r *http.Request) {
	// We only need the details for listing pages.
	var jobs []*job
	for clusterName, tracker := range jts {
		for _, j := range tracker.jobs {
			jobs = append(jobs, &job{
				Cluster: tracker.clusterName,
				Details: j.Details,
				conf: conf{
					Input:         j.conf.Input,
					Output:        j.conf.Output,
					ScaldingSteps: j.conf.ScaldingSteps,
					name:          j.conf.name,
				},
			})
		}
		log.Printf("Appending %d jobs for Cluster %s: %s %s\n", len(jobs), clusterName, tracker.hs, tracker.rm)
	}

	jsonBytes, err := json.Marshal(jobs)
	if err != nil {
		log.Println("getJobs error:", err)
		w.WriteHeader(500)
		return
	}

	w.Write(jsonBytes)
}

func getNumClusters(c web.C, w http.ResponseWriter, r *http.Request) {
	jsonBytes, err := json.Marshal(len(jts))
	if err != nil {
		log.Println("getNumbClusters error:", err)
		w.WriteHeader(500)
		return
	}
	w.Write(jsonBytes)
}

func getConf(c web.C, w http.ResponseWriter, r *http.Request) {
	id := c.URLParams["id"]
	log.Printf("Getting job conf for %s", id)

	job := getJob(id)
	if job == nil {
		w.WriteHeader(404)
		return
	}

	jsonBytes, err := json.Marshal(jobConf{
		Conf: job.conf,
		ID:   job.Details.ID,
		Name: job.Details.Name,
	})
	if err != nil {
		log.Println("could not marshal:", err)
		w.WriteHeader(500)
		return
	}

	w.Write(jsonBytes)
}

func getJob(rawJobID string) *job {
	// check if we have it in memory
	for clusterName, jt := range jts {
		if _, ok := jt.jobs[jobID(rawJobID)]; ok {
			job := jt.reifyJob(rawJobID)
			job.Cluster = clusterName
			return job
		}
	}

	// check if we have it in long-term storage
	job, _ := persistedJobClient.FetchJob(rawJobID)

	return job
}

func getJobAPIHandler(c web.C, w http.ResponseWriter, r *http.Request) {
	job := getJob(c.URLParams["id"])

	if job == nil {
		w.WriteHeader(404)
		return
	}

	jsonBytes, err := json.Marshal(job)
	if err != nil {
		log.Println("error serializing job:", err)
		w.WriteHeader(500)
		return
	}

	w.Write(jsonBytes)
}

func getJobIdsAPIHandler(c web.C, w http.ResponseWriter, r *http.Request) {
	jobIds, err := persistedJobClient.FetchFlowJobIds(c.URLParams["flowID"])
	if err != nil {
		log.Println("FetchFlowJobIds error:", err)
		w.WriteHeader(500)
		return
	}

	jsonBytes, err := json.Marshal(jobIds)
	if err != nil {
		log.Println("JSON marshal error:", err)
		w.WriteHeader(500)
		return
	}

	w.Write(jsonBytes)
}

func killJob(c web.C, w http.ResponseWriter, r *http.Request) {
	id := c.URLParams["id"]
	jobID := jobID(id)

	for _, jt := range jts {
		if _, ok := jt.jobs[jobID]; ok {
			err := jt.killJob(id, jt.jobs[jobID].Details.User)
			if err != nil {
				log.Println("killJob error: ", err)
				w.WriteHeader(500)
				return
			}
			w.WriteHeader(204)
			return
		}
	}

	w.WriteHeader(404)
}

func init() {
	binPath, err := filepath.Abs(filepath.Dir(os.Args[0]))
	if err != nil {
		log.Fatal(err)
	}

	rootPath = filepath.Join(binPath, "..")
	staticPath = filepath.Join(rootPath, "static")
}

func main() {
	flag.Parse()

	var clusterNames = strings.Split(*clusterNames, ",")
	var resourceManagerURLs = strings.Split(*resourceManagerURL, ",")
	var historyServerURLs = strings.Split(*historyServerURL, ",")
	var proxyServerURLs = strings.Split(*proxyServerURL, ",")
	var namenodeAddresses = strings.Split(*namenodeAddress, ",")

	if len(resourceManagerURLs) != len(historyServerURLs) {
		log.Fatal("resource-manager-url and history-server-url are not 1:1")
	}
	if !reflect.DeepEqual(proxyServerURLs, []string{""}) && len(proxyServerURLs) != len(resourceManagerURLs) {
		log.Fatal("proxy-server-url exists and is not 1:1 with resource-manager-url")
	}
	if len(resourceManagerURLs) != len(namenodeAddresses) {
		log.Fatal("resource-manager-url and namenode-address are not 1:1")
	}
	if len(resourceManagerURLs) != len(clusterNames) {
		log.Fatal("cluster-names and resource-manager-url are not 1:1")
	}

	persistedJobClient = NewS3JobClient(*s3Region, *s3BucketName, *s3JobsPrefix, *s3FlowPrefix)
	jts = make(map[string]*jobTracker)
	for i := range resourceManagerURLs {
		var proxyServerURL string
		if reflect.DeepEqual(proxyServerURLs, []string{""}) {
			proxyServerURL = resourceManagerURLs[i]
		} else {
			proxyServerURL = proxyServerURLs[i]
		}
		log.Printf("Creating new JT [%d]: %s %s %s\n", i, resourceManagerURLs[i], historyServerURLs[i], proxyServerURL)
		jts[clusterNames[i]] = newJobTracker(
			clusterNames[i],
			newRecentJobClient(
				resourceManagerURLs[i],
				historyServerURLs[i],
				proxyServerURL,
				namenodeAddresses[i],
			),
		)
	}

	log.Println("initiating JT loop")

	for clusterName, jt := range jts {
		go jt.Loop()
		if err := jt.testLogsDir(); err != nil {
			log.Printf("WARNING: Could not read yarn logs directory for cluster %s. Error message: `%s`\n", clusterName, err)
			log.Println("\tYou can change the path with --yarn-logs-dir=HDFS_PATH.")
			log.Println("\tTo talk to HDFS, Timberlake needs to be able to access the namenode (--namenode-address) and datanodes.")
		}
	}

	sse := newSSE()
	go sse.Loop()

	static := http.StripPrefix("/static/", http.FileServer(http.Dir(staticPath)))
	log.Println("serving static files from", staticPath)

	mux.Get("/static/*", static)
	mux.Get("/", index)
	mux.Get("/jobs/", getJobs)
	mux.Get("/numClusters/", getNumClusters)
	mux.Get("/sse", sse)
	mux.Get("/jobIds/:flowID", getJobIdsAPIHandler)
	mux.Get("/jobs/:id", getJobAPIHandler)
	mux.Get("/jobs/:id/conf", getConf)
	mux.Post("/jobs/:id/kill", killJob)

	if *enableDebug {
		mux.Get("/debug/pprof/*", pprof.Index)
		mux.Get("/debug/pprof/cmdline", pprof.Cmdline)
		mux.Get("/debug/pprof/profile", pprof.Profile)
		mux.Get("/debug/pprof/symbol", pprof.Symbol)
		mux.Get("/debug/pprof/trace", pprof.Trace)
	}

	for _, jt := range jts {
		go jt.sendUpdates(sse)
	}

	http.Serve(bind.Default(), mux)
}
<commit_msg>app id<commit_after>package main

import (
	"encoding/json"
	"flag"
	"github.com/zenazn/goji/bind"
	"github.com/zenazn/goji/web"
	"github.com/zenazn/goji/web/middleware"
	"log"
	"net/http"
	"net/http/pprof"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"time"
)

var clusterNames = flag.String("cluster-name", "default", "The user-visible names for the clusters")
var resourceManagerURL = flag.String("resource-manager-url", "http://localhost:8088", "The HTTP URL to access the resource manager.")
var historyServerURL = flag.String("history-server-url", "http://localhost:19888", "The HTTP URL to access the history server.")
var proxyServerURL = flag.String("proxy-server-url", "", "The HTTP URL to access the proxy server, if separate from the resource manager.")
var namenodeAddress = flag.String("namenode-address", "localhost:9000", "The host:port to access the Namenode metadata service.")
var yarnLogDir = flag.String("yarn-logs-dir", "/tmp/logs", "The HDFS path where YARN stores logs. This is the controlled by the hadoop property yarn.nodemanager.remote-app-log-dir.")
var yarnHistoryDir = flag.String("yarn-history-dir", "/tmp/staging/history/done", "The HDFS path where YARN stores finished job history files. This is the controlled by the hadoop property mapreduce.jobhistory.done-dir.")
var httpTimeout = flag.Duration("http-timeout", time.Second*2, "The timeout used for connecting to YARN API. Pass values like: 2s")
var pollInterval = flag.Duration("poll-interval", time.Second*5, "How often should we poll the job APIs. Pass values like: 2s")
var enableDebug = flag.Bool("pprof", false, "Enable pprof debugging tools at /debug.")
var s3BucketName = flag.String("s3-bucket", "", "S3 bucket to fetch old jobs from")
var s3Region = flag.String("s3-region", "", "AWS region for the job storage S3 bucket")
var s3JobsPrefix = flag.String("s3-jobs-prefix", "", "S3 key prefix (\"folder\") where jobs are stored")
var s3FlowPrefix = flag.String("s3-flow-prefix", "", "S3 key prefix (\"folder\") where cascading flows are stored")

var jts map[string]*jobTracker
var persistedJobClient PersistedJobClient

var rootPath, staticPath string

var mux *web.Mux

func init() {
	bind.WithFlag()
	mux = web.New()
	mux.Use(middleware.RequestID)
	mux.Use(middleware.Logger)
	mux.Use(middleware.Recoverer)
	mux.Use(middleware.AutomaticOptions)
}

func index(c web.C, w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, filepath.Join(rootPath, "index.html"))
}

func getJobs(c web.C, w http.ResponseWriter, r *http.Request) {
	// We only need the details for listing pages.
	var jobs []*job
	for clusterName, tracker := range jts {
		for _, j := range tracker.jobs {
			jobs = append(jobs, &job{
				Cluster: tracker.clusterName,
				Details: j.Details,
				conf: conf{
					Input:         j.conf.Input,
					Output:        j.conf.Output,
					ScaldingSteps: j.conf.ScaldingSteps,
					name:          j.conf.name,
				},
			})
		}
		log.Printf("Appending %d jobs for Cluster %s: %s %s\n", len(jobs), clusterName, tracker.hs, tracker.rm)
	}

	jsonBytes, err := json.Marshal(jobs)
	if err != nil {
		log.Println("getJobs error:", err)
		w.WriteHeader(500)
		return
	}

	w.Write(jsonBytes)
}

func getNumClusters(c web.C, w http.ResponseWriter, r *http.Request) {
	jsonBytes, err := json.Marshal(len(jts))
	if err != nil {
		log.Println("getNumbClusters error:", err)
		w.WriteHeader(500)
		return
	}
	w.Write(jsonBytes)
}

func getConf(c web.C, w http.ResponseWriter, r *http.Request) {
	id := c.URLParams["id"]
	log.Printf("Getting job conf for %s", id)

	job := getJob(id)
	if job == nil {
		w.WriteHeader(404)
		return
	}

	jsonBytes, err := json.Marshal(jobConf{
		Conf: job.conf,
		ID:   job.Details.ID,
		Name: job.Details.Name,
	})
	if err != nil {
		log.Println("could not marshal:", err)
		w.WriteHeader(500)
		return
	}

	w.Write(jsonBytes)
}

func getJob(rawJobID string) *job {
	// check if we have it in memory
	for clusterName, jt := range jts {
		if _, ok := jt.jobs[jobID(rawJobID)]; ok {
			job := jt.reifyJob(rawJobID)
			job.Cluster = clusterName
			return job
		}
	}

	// check if we have it in long-term storage
	job, _ := persistedJobClient.FetchJob(rawJobID)

	return job
}

func getJobAPIHandler(c web.C, w http.ResponseWriter, r *http.Request) {
	job := getJob(c.URLParams["id"])

	if job == nil {
		w.WriteHeader(404)
		return
	}

	jsonBytes, err := json.Marshal(job)
	if err != nil {
		log.Println("error serializing job:", err)
		w.WriteHeader(500)
		return
	}

	w.Write(jsonBytes)
}

func getJobIdsAPIHandler(c web.C, w http.ResponseWriter, r *http.Request) {
	jobIds, err := persistedJobClient.FetchFlowJobIds(c.URLParams["flowID"])
	if err != nil {
		log.Println("FetchFlowJobIds error:", err)
		w.WriteHeader(500)
		return
	}

	jsonBytes, err := json.Marshal(jobIds)
	if err != nil {
		log.Println("JSON marshal error:", err)
		w.WriteHeader(500)
		return
	}

	w.Write(jsonBytes)
}

func killJob(c web.C, w http.ResponseWriter, r *http.Request) {
	id := c.URLParams["id"]
	jobID := jobID(id)

	for _, jt := range jts {
		if _, ok := jt.jobs[jobID]; ok {
			app, _ := hadoopIDs(id)
			err := jt.killJob(app, jt.jobs[jobID].Details.User)
			if err != nil {
				log.Println("killJob error: ", err)
				w.WriteHeader(500)
				return
			}
			w.WriteHeader(204)
			return
		}
	}

	w.WriteHeader(404)
}

func init() {
	binPath, err := filepath.Abs(filepath.Dir(os.Args[0]))
	if err != nil {
		log.Fatal(err)
	}

	rootPath = filepath.Join(binPath, "..")
	staticPath = filepath.Join(rootPath, "static")
}

func main() {
	flag.Parse()

	var clusterNames = strings.Split(*clusterNames, ",")
	var resourceManagerURLs = strings.Split(*resourceManagerURL, ",")
	var historyServerURLs = strings.Split(*historyServerURL, ",")
	var proxyServerURLs = strings.Split(*proxyServerURL, ",")
	var namenodeAddresses = strings.Split(*namenodeAddress, ",")

	if len(resourceManagerURLs) != len(historyServerURLs) {
		log.Fatal("resource-manager-url and history-server-url are not 1:1")
	}
	if !reflect.DeepEqual(proxyServerURLs, []string{""}) && len(proxyServerURLs) != len(resourceManagerURLs) {
		log.Fatal("proxy-server-url exists and is not 1:1 with resource-manager-url")
	}
	if len(resourceManagerURLs) != len(namenodeAddresses) {
		log.Fatal("resource-manager-url and namenode-address are not 1:1")
	}
	if len(resourceManagerURLs) != len(clusterNames) {
		log.Fatal("cluster-names and resource-manager-url are not 1:1")
	}

	persistedJobClient = NewS3JobClient(*s3Region, *s3BucketName, *s3JobsPrefix, *s3FlowPrefix)
	jts = make(map[string]*jobTracker)
	for i := range resourceManagerURLs {
		var proxyServerURL string
		if reflect.DeepEqual(proxyServerURLs, []string{""}) {
			proxyServerURL = resourceManagerURLs[i]
		} else {
			proxyServerURL = proxyServerURLs[i]
		}
		log.Printf("Creating new JT [%d]: %s %s %s\n", i, resourceManagerURLs[i], historyServerURLs[i], proxyServerURL)
		jts[clusterNames[i]] = newJobTracker(
			clusterNames[i],
			newRecentJobClient(
				resourceManagerURLs[i],
				historyServerURLs[i],
				proxyServerURL,
				namenodeAddresses[i],
			),
		)
	}

	log.Println("initiating JT loop")

	for clusterName, jt := range jts {
		go jt.Loop()
		if err := jt.testLogsDir(); err != nil {
			log.Printf("WARNING: Could not read yarn logs directory for cluster %s. Error message: `%s`\n", clusterName, err)
			log.Println("\tYou can change the path with --yarn-logs-dir=HDFS_PATH.")
			log.Println("\tTo talk to HDFS, Timberlake needs to be able to access the namenode (--namenode-address) and datanodes.")
		}
	}

	sse := newSSE()
	go sse.Loop()

	static := http.StripPrefix("/static/", http.FileServer(http.Dir(staticPath)))
	log.Println("serving static files from", staticPath)

	mux.Get("/static/*", static)
	mux.Get("/", index)
	mux.Get("/jobs/", getJobs)
	mux.Get("/numClusters/", getNumClusters)
	mux.Get("/sse", sse)
	mux.Get("/jobIds/:flowID", getJobIdsAPIHandler)
	mux.Get("/jobs/:id", getJobAPIHandler)
	mux.Get("/jobs/:id/conf", getConf)
	mux.Post("/jobs/:id/kill", killJob)

	if *enableDebug {
		mux.Get("/debug/pprof/*", pprof.Index)
		mux.Get("/debug/pprof/cmdline", pprof.Cmdline)
		mux.Get("/debug/pprof/profile", pprof.Profile)
		mux.Get("/debug/pprof/symbol", pprof.Symbol)
		mux.Get("/debug/pprof/trace", pprof.Trace)
	}

	for _, jt := range jts {
		go jt.sendUpdates(sse)
	}

	http.Serve(bind.Default(), mux)
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"
	"syscall"
)

var versions = map[string]string {
	"2": "/usr/bin/python2",
	"3": "/usr/bin/python3",
	"":  "/usr/bin/python",
}

func main() {
	python := os.Getenv("PYTHON")
	path, _ := versions[python]

	if _, err := os.Stat(path); err != nil {
		log.Fatalf("unknown python version '%s'\n", python)
	}

	args := os.Args
	args[0] = path

	p, err := os.StartProcess(path, args, &os.ProcAttr{
		Files: []*os.File{os.Stdin, os.Stdout, os.Stderr},
	})
	if err != nil {
		log.Fatalf("failed to start python: %v\n", err)
	}

	s, err := p.Wait()
	if err != nil {
		log.Fatalf("failed to wait for python: %v\n", err)
	}

	if !s.Success() {
		// try to exit with the right exit code
		if ws, ok := s.Sys().(syscall.WaitStatus); ok {
			os.Exit(ws.ExitStatus())
		}

		// not on a Posix system, so we don't know the exit code.  1's as
		// good as any other
		os.Exit(1)
	}
}
<commit_msg>handle $PYTHON being a filename; not that this allows stupid things like 'PYTHON=/usr/bin/echo python'<commit_after>package main

import (
	"log"
	"os"
	"syscall"
)

var versions = map[string]string {
	"2": "/usr/bin/python2",
	"3": "/usr/bin/python3",
	"":  "/usr/bin/python",
}

func main() {
	var path string
	python := os.Getenv("PYTHON")

	// if $PYTHON is a path to an executable, use it; if not, try to expand the
	// shortcuts listed above
	if _, err := os.Stat(python); err == nil {
		path = python
	} else {
		path, _ = versions[python]
	}

	// make sure we have something to exec
	if _, err := os.Stat(path); err != nil {
		log.Fatalf("unknown python version '%s'\n", python)
	}

	// give python our arguments but the path to the python executable
	os.Args[0] = path

	// start python connected to our fds 0, 1, and 2
	p, err := os.StartProcess(path, os.Args, &os.ProcAttr{
		Files: []*os.File{os.Stdin, os.Stdout, os.Stderr},
	})
	if err != nil {
		log.Fatalf("failed to start python: %v\n", err)
	}

	// wait for it to execute, then clean up
	s, err := p.Wait()
	if err != nil {
		log.Fatalf("failed to wait for python: %v\n", err)
	}

	if !s.Success() {
		// try to exit with the right exit code
		if ws, ok := s.Sys().(syscall.WaitStatus); ok {
			os.Exit(ws.ExitStatus())
		}

		// not on a Posix system, so we don't know the exit code.  1's as
		// good as any other
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/danielsoro/qrcode-generate/handlers"
	"github.com/gin-gonic/gin"
)

func main() {
	r := gin.Default()
	r.POST("/qrcode", handlers.QrcodeHandler)
	r.Run()
}
<commit_msg>Simple refactore for main<commit_after>package main

import (
	"github.com/danielsoro/qrcode-generate/handlers"
	"github.com/gin-gonic/gin"
)

func main() {
	startGin()
}

func startGin() {
	gin.SetMode(gin.DebugMode)

	r := gin.New()
	r.Use(gin.Recovery())

	r.POST("/qrcode", handlers.QrcodeHandler)
	r.Run()
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"flag"
	"os"
	"time"

	"github.com/cloudfoundry-incubator/uaago"
	"github.com/cloudfoundry-incubator/varz-firehose-nozzle/config"
	"github.com/cloudfoundry-incubator/varz-firehose-nozzle/emitter"
	"github.com/cloudfoundry-incubator/varz-firehose-nozzle/server"
	"github.com/cloudfoundry/gosteno"
	"github.com/cloudfoundry/loggregatorlib/cfcomponent"
	"github.com/cloudfoundry/loggregatorlib/cfcomponent/instrumentation"
	"github.com/cloudfoundry/loggregatorlib/cfcomponent/registrars/collectorregistrar"
	"github.com/cloudfoundry/noaa"
	//	"github.com/cloudfoundry/sonde-go/events"
	"github.com/cloudfoundry-incubator/varz-firehose-nozzle/varzfirehosenozzle"
)

type varzHealthMonitor struct{}

func (*varzHealthMonitor) Ok() bool {
	return true
}

func main() {
	var (
		configFilePath = flag.String("config", "config/varz_firehose_nozzle.json", "Location of the nozzle config json file")
	)
	flag.Parse()

	logger := initLogger()

	config, err := config.ParseConfig(*configFilePath)
	if err != nil {
		logger.Fatal(err.Error())
		return
	}

	uaaClient, err := uaago.NewClient(config.UAAURL)
	if err != nil {
		logger.Fatalf("Error creating uaa client: %s", err.Error())
		return
	}

	authToken, err := uaaClient.GetAuthToken(config.UAAUser, config.UAAPass, config.InsecureSSLSkipVerify)
	if err != nil {
		logger.Fatalf("Error getting oauth token: %s. Please check your username and password.", err.Error())
		return
	}

	consumer := noaa.NewConsumer(
		config.TrafficControllerURL,
		&tls.Config{InsecureSkipVerify: config.InsecureSSLSkipVerify},
		nil)

	component, registrar, err := initRegistrar(config, logger)
	if err != nil {
		logger.Fatal(err.Error())
		return
	}

	varzEmitter := emitter.New("varz-nozzle")

	varzServer := server.New(varzEmitter, int(component.StatusPort), component.StatusCredentials[0], component.StatusCredentials[1])
	go varzServer.Start()
	go registrar.Run()

	logger.Info("Started listening for messages")
	defer logger.Info("Exiting")

	firehoseNozzle := varzfirehosenozzle.NewVarzFirehoseNozzle(config.FirehoseSubscriptionID, authToken, consumer, varzEmitter, logger)
	firehoseNozzle.Run()
}

func initLogger() *gosteno.Logger {
	c := &gosteno.Config{
		Sinks: []gosteno.Sink{
			gosteno.NewIOSink(os.Stdout),
		},
		Level:     gosteno.LOG_INFO,
		Codec:     gosteno.NewJsonCodec(),
		EnableLOC: true,
	}
	gosteno.Init(c)

	return gosteno.NewLogger("Varz Firehose Nozzle")
}

func initRegistrar(config *config.VarzConfig, logger *gosteno.Logger) (*cfcomponent.Component, *collectorregistrar.CollectorRegistrar, error) {
	interval := time.Duration(config.CollectorRegistrarIntervalMilliseconds) * time.Millisecond
	instrumentables := []instrumentation.Instrumentable{}
	component, err := cfcomponent.NewComponent(logger, config.NatsType, config.Index, &varzHealthMonitor{}, config.VarzPort, []string{config.VarzUser, config.VarzPass}, instrumentables)

	if err != nil {
		return nil, nil, err
	}

	registrar := collectorregistrar.NewCollectorRegistrar(cfcomponent.DefaultYagnatsClientProvider, component, interval, &config.Config)

	return &component, registrar, nil
}
<commit_msg>Dumps goroutine stack when SIGUSR1 signal is sent<commit_after>package main

import (
	"crypto/tls"
	"flag"
	"os"
	"time"

	"github.com/cloudfoundry-incubator/uaago"
	"github.com/cloudfoundry-incubator/varz-firehose-nozzle/config"
	"github.com/cloudfoundry-incubator/varz-firehose-nozzle/emitter"
	"github.com/cloudfoundry-incubator/varz-firehose-nozzle/server"
	"github.com/cloudfoundry-incubator/varz-firehose-nozzle/varzfirehosenozzle"
	"github.com/cloudfoundry/gosteno"
	"github.com/cloudfoundry/loggregatorlib/cfcomponent"
	"github.com/cloudfoundry/loggregatorlib/cfcomponent/instrumentation"
	"github.com/cloudfoundry/loggregatorlib/cfcomponent/registrars/collectorregistrar"
	"github.com/cloudfoundry/noaa"
	"os/signal"
	"runtime/pprof"
	"syscall"
)

type varzHealthMonitor struct{}

func (*varzHealthMonitor) Ok() bool {
	return true
}

func main() {
	var (
		configFilePath = flag.String("config", "config/varz_firehose_nozzle.json", "Location of the nozzle config json file")
	)
	flag.Parse()

	logger := initLogger()

	config, err := config.ParseConfig(*configFilePath)
	if err != nil {
		logger.Fatal(err.Error())
		return
	}

	uaaClient, err := uaago.NewClient(config.UAAURL)
	if err != nil {
		logger.Fatalf("Error creating uaa client: %s", err.Error())
		return
	}

	authToken, err := uaaClient.GetAuthToken(config.UAAUser, config.UAAPass, config.InsecureSSLSkipVerify)
	if err != nil {
		logger.Fatalf("Error getting oauth token: %s. Please check your username and password.", err.Error())
		return
	}

	consumer := noaa.NewConsumer(
		config.TrafficControllerURL,
		&tls.Config{InsecureSkipVerify: config.InsecureSSLSkipVerify},
		nil)

	component, registrar, err := initRegistrar(config, logger)
	if err != nil {
		logger.Fatal(err.Error())
		return
	}

	threadDumpChan := registerGoRoutineDumpSignalChannel()
	defer close(threadDumpChan)
	go dumpGoRoutine(threadDumpChan)

	varzEmitter := emitter.New("varz-nozzle")

	varzServer := server.New(varzEmitter, int(component.StatusPort), component.StatusCredentials[0], component.StatusCredentials[1])
	go varzServer.Start()
	go registrar.Run()

	logger.Info("Started listening for messages")
	defer logger.Info("Exiting")

	firehoseNozzle := varzfirehosenozzle.NewVarzFirehoseNozzle(config.FirehoseSubscriptionID, authToken, consumer, varzEmitter, logger)
	firehoseNozzle.Run()
}

func initLogger() *gosteno.Logger {
	c := &gosteno.Config{
		Sinks: []gosteno.Sink{
			gosteno.NewIOSink(os.Stdout),
		},
		Level:     gosteno.LOG_INFO,
		Codec:     gosteno.NewJsonCodec(),
		EnableLOC: true,
	}
	gosteno.Init(c)

	return gosteno.NewLogger("Varz Firehose Nozzle")
}

func initRegistrar(config *config.VarzConfig, logger *gosteno.Logger) (*cfcomponent.Component, *collectorregistrar.CollectorRegistrar, error) {
	interval := time.Duration(config.CollectorRegistrarIntervalMilliseconds) * time.Millisecond
	instrumentables := []instrumentation.Instrumentable{}
	component, err := cfcomponent.NewComponent(logger, config.NatsType, config.Index, &varzHealthMonitor{}, config.VarzPort, []string{config.VarzUser, config.VarzPass}, instrumentables)

	if err != nil {
		return nil, nil, err
	}

	registrar := collectorregistrar.NewCollectorRegistrar(cfcomponent.DefaultYagnatsClientProvider, component, interval, &config.Config)

	return &component, registrar, nil
}

func registerGoRoutineDumpSignalChannel() chan os.Signal {
	threadDumpChan := make(chan os.Signal, 1)
	signal.Notify(threadDumpChan, syscall.SIGUSR1)

	return threadDumpChan
}

func dumpGoRoutine(dumpChan chan os.Signal) {
	for range dumpChan {
		goRoutineProfiles := pprof.Lookup("goroutine")
		if goRoutineProfiles != nil {
			goRoutineProfiles.WriteTo(os.Stdout, 2)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"math"
	"os"
	"sort"
	"sync"
	"time"

	"github.com/alexaandru/utils"
	"github.com/mitchellh/goamz/aws"
	"github.com/mitchellh/goamz/s3"
)

// Exit codes
const (
	Success = iota
	S3AuthError
	CmdLineOptionError
)

// max number of attempts to retry a failed upload.
const maxTries = 10

type uploader func(*sourceFile) error

// filesLists returns both the current files list as well as the difference from the old (cached) files list.
func filesLists() (current utils.FileHashes, diff []string) {
	current = utils.FileHashesNew(opts.source)
	old := utils.FileHashes{}
	old.Load(opts.cacheFile)
	diff = current.Diff(old)

	return
}

// upload fetches sourceFiles from uploads chan, attempts to upload them and enqueue the results to
// completed list. On failure it attempts to retry, up to maxTries per source file.
func upload(id string, fn uploader, uploads chan *sourceFile, rejected *syncedlist, wgUploads, wgWorkers *sync.WaitGroup) {
	defer wgWorkers.Done()

	for src := range uploads {
		src := src

		if opts.dryRun {
			fmt.Print(msg("Pretending to upload "+src.fname, "."))
			wgUploads.Done()
			continue
		}

		err := fn(src)
		if err == nil {
			wgUploads.Done()
			fmt.Print(msg("Uploaded "+src.fname, "."))
			continue
		}

		src.recordAttempt()
		if !src.retriable() || !isRecoverable(err) {
			rejected.add(src.fname)
			fmt.Print(msg("Failed to upload "+src.fname, "f"))
			wgUploads.Done()
			continue
		}

		go func() {
			wait := time.Duration(100.0*math.Pow(2, float64(src.attempts))) * time.Millisecond
			if appEnv == "test" {
				wait = time.Nanosecond
			}
			<-time.After(wait)
			uploads <- src
		}()
	}
}

// FIXME: For some (all?) errors, we should re-initialize the bucket before retrying.
func main() {
	validateCmdLineFlags(opts)
	goto Setup

Setup:
	// AWS setup
	auth, err := aws.EnvAuth()
	if err != nil {
		fmt.Println("S3 Error:", err)
		os.Exit(S3AuthError)
	}
	newBucket := func() *s3.Bucket { return s3.New(auth, aws.EUWest).Bucket(opts.bucketName) }
	bucket := newBucket()
	s3put := func(src *sourceFile) (err error) {
		var body []byte
		body, err = src.body()
		if err != nil {
			// If we can't read the file, there's no point to retry.
			src.attempts = maxTries
			return
		}

		return bucket.PutHeader(src.fname, body, src.hdrs, s3.PublicRead)
	}

	// Sync setup
	uploads, rejected := make(chan *sourceFile), &syncedlist{}
	wgUploads, wgWorkers := new(sync.WaitGroup), new(sync.WaitGroup)

	// Current list of files, and diff to be uploaded
	current, diff := filesLists()
	if len(diff) == 0 {
		fmt.Print(msg("Nothing to upload."))
		goto Finish
	}
	fmt.Print(msg(fmt.Sprintf("There are %d files to be uploaded to '%s'", len(diff), opts.bucketName), "Uploading "))
	goto Upload

Upload:
	if !opts.doUpload {
		fmt.Print(msg("Skipping upload", ""))
		goto Cache
	}

	// We need to wait until ALL files are processed, one way or another.
	wgUploads.Add(len(diff))
	wgWorkers.Add(opts.workersCount)
	for i := 0; i < opts.workersCount; i++ {
		go upload(fmt.Sprintf("%d", i), s3put, uploads, rejected, wgUploads, wgWorkers)
	}

	sort.Strings(diff)
	for _, fname := range diff {
		uploads <- newSourceFile(fname)
	}

	wgUploads.Wait()
	close(uploads)
	wgWorkers.Wait()
	fmt.Print(msg("Done uploading files.", ""))

Cache:
	if !opts.doCache {
		fmt.Print(msg("Skipping cache.", ""))
		goto Finish
	}

	if opts.dryRun {
		fmt.Print(msg("Pretending to update cache.\nUpdated cache.", ""))
		goto Finish
	}

	// Only retain files actually completed.
	current = current.Reject(rejected.list)
	if err := current.Dump(opts.cacheFile); err != nil {
		fmt.Println("Caching failed: ", err)
		goto Finish
	}
	fmt.Print(msg("Done updating cache.", ""))

Finish:
	fmt.Print(msg("All done!", " done!\n"))
}
<commit_msg>Added message for retries<commit_after>package main

import (
	"fmt"
	"math"
	"os"
	"sort"
	"sync"
	"time"

	"github.com/alexaandru/utils"
	"github.com/mitchellh/goamz/aws"
	"github.com/mitchellh/goamz/s3"
)

// Exit codes
const (
	Success = iota
	S3AuthError
	CmdLineOptionError
)

// max number of attempts to retry a failed upload.
const maxTries = 10

type uploader func(*sourceFile) error

// filesLists returns both the current files list as well as the difference from the old (cached) files list.
func filesLists() (current utils.FileHashes, diff []string) {
	current = utils.FileHashesNew(opts.source)
	old := utils.FileHashes{}
	old.Load(opts.cacheFile)
	diff = current.Diff(old)

	return
}

// upload fetches sourceFiles from uploads chan, attempts to upload them and enqueue the results to
// completed list. On failure it attempts to retry, up to maxTries per source file.
func upload(id string, fn uploader, uploads chan *sourceFile, rejected *syncedlist, wgUploads, wgWorkers *sync.WaitGroup) {
	defer wgWorkers.Done()

	for src := range uploads {
		src := src

		if opts.dryRun {
			fmt.Print(msg("Pretending to upload "+src.fname, "."))
			wgUploads.Done()
			continue
		}

		err := fn(src)
		if err == nil {
			wgUploads.Done()
			fmt.Print(msg("Uploaded "+src.fname, "."))
			continue
		}

		src.recordAttempt()
		if !src.retriable() || !isRecoverable(err) {
			rejected.add(src.fname)
			fmt.Print(msg("Failed to upload "+src.fname, "f"))
			wgUploads.Done()
			continue
		}

		go func() {
			fmt.Print(msg("Retrying "+src.fname, "r"))
			wait := time.Duration(100.0*math.Pow(2, float64(src.attempts))) * time.Millisecond
			if appEnv == "test" {
				wait = time.Nanosecond
			}
			<-time.After(wait)
			uploads <- src
		}()
	}
}

// FIXME: For some (all?) errors, we should re-initialize the bucket before retrying.
func main() {
	validateCmdLineFlags(opts)
	goto Setup

Setup:
	// AWS setup
	auth, err := aws.EnvAuth()
	if err != nil {
		fmt.Println("S3 Error:", err)
		os.Exit(S3AuthError)
	}
	newBucket := func() *s3.Bucket { return s3.New(auth, aws.EUWest).Bucket(opts.bucketName) }
	bucket := newBucket()
	s3put := func(src *sourceFile) (err error) {
		var body []byte
		body, err = src.body()
		if err != nil {
			// If we can't read the file, there's no point to retry.
			src.attempts = maxTries
			return
		}

		return bucket.PutHeader(src.fname, body, src.hdrs, s3.PublicRead)
	}

	// Sync setup
	uploads, rejected := make(chan *sourceFile), &syncedlist{}
	wgUploads, wgWorkers := new(sync.WaitGroup), new(sync.WaitGroup)

	// Current list of files, and diff to be uploaded
	current, diff := filesLists()
	if len(diff) == 0 {
		fmt.Print(msg("Nothing to upload."))
		goto Finish
	}
	fmt.Print(msg(fmt.Sprintf("There are %d files to be uploaded to '%s'", len(diff), opts.bucketName), "Uploading "))
	goto Upload

Upload:
	if !opts.doUpload {
		fmt.Print(msg("Skipping upload", ""))
		goto Cache
	}

	// We need to wait until ALL files are processed, one way or another.
	wgUploads.Add(len(diff))
	wgWorkers.Add(opts.workersCount)
	for i := 0; i < opts.workersCount; i++ {
		go upload(fmt.Sprintf("%d", i), s3put, uploads, rejected, wgUploads, wgWorkers)
	}

	sort.Strings(diff)
	for _, fname := range diff {
		uploads <- newSourceFile(fname)
	}

	wgUploads.Wait()
	close(uploads)
	wgWorkers.Wait()
	fmt.Print(msg("Done uploading files.", ""))

Cache:
	if !opts.doCache {
		fmt.Print(msg("Skipping cache.", ""))
		goto Finish
	}

	if opts.dryRun {
		fmt.Print(msg("Pretending to update cache.\nUpdated cache.", ""))
		goto Finish
	}

	// Only retain files actually completed.
	current = current.Reject(rejected.list)
	if err := current.Dump(opts.cacheFile); err != nil {
		fmt.Println("Caching failed: ", err)
		goto Finish
	}
	fmt.Print(msg("Done updating cache.", ""))

Finish:
	fmt.Print(msg("All done!", " done!\n"))
}
<|endoftext|>


<commit_before>//
// Simple daemon to grab keys and then send them via websocket to the
// keysocket Chrome plugin.
//
// Written by Daniel Hobe but based off of many code examples found online.
//
// Websocket server: garyburd
// https://gist.github.com/1316852
//
// keybinding example: BurtSushi
// https://github.com/BurntSushi/xgbutil/blob/master/examples/simple-keybinding/main.go

//
// Future features:
// - config file
// - connect to Gnome setting daemon over dbus

package main

import (
	"code.google.com/p/go.net/websocket"
	"flag"
	"github.com/BurntSushi/xgbutil"
	"github.com/BurntSushi/xgbutil/keybind"
	"github.com/BurntSushi/xgbutil/xevent"
	"log"
	"net/http"
)

type websocket_connections struct {
	connections map[*connection]bool
	broadcast   chan string
	register    chan *connection
	unregister  chan *connection
}

var h = websocket_connections{
	broadcast:   make(chan string),
	register:    make(chan *connection),
	unregister:  make(chan *connection),
	connections: make(map[*connection]bool),
}

func (h *websocket_connections) run() {
	for {
		select {
		case c := <-h.register:
			h.connections[c] = true
		case c := <-h.unregister:
			delete(h.connections, c)
			close(c.send)
		case m := <-h.broadcast:
			log.Printf("Sending %q\n", m)
			for c := range h.connections {
				select {
				case c.send <- m:
				default:
					delete(h.connections, c)
					close(c.send)
					go c.ws.Close()
				}
			}
		}
	}
}

type connection struct {
	// The websocket connection.
	ws *websocket.Conn

	// Buffered channel of outbound messages.
	send chan string
}

func (c *connection) reader() {
	for {
		var message string
		err := websocket.Message.Receive(c.ws, &message)
		if err != nil {
			break
		}
		if message == "Ping" {
			err := websocket.Message.Send(c.ws, "Pong")
			if err != nil {
				break
			}
		}
	}
	c.ws.Close()
}

func (c *connection) writer() {
	for message := range c.send {
		err := websocket.Message.Send(c.ws, message)
		if err != nil {
			break
		}
	}
	c.ws.Close()
}

func wsHandler(ws *websocket.Conn) {
	c := &connection{send: make(chan string, 2), ws: ws}
	h.register <- c
	defer func() { h.unregister <- c }()
	go c.writer()
	c.reader()
}

func BindKeys(keymap map[string]string) {
	X, err := xgbutil.NewConn()
	if err != nil {
		log.Fatal(err)
	}
	keybind.Initialize(X)
	for k, v := range keymap {
		v := v
		err = keybind.KeyPressFun(func(X *xgbutil.XUtil, e xevent.KeyPressEvent) {
			h.broadcast <- v
		}).Connect(X, X.RootWin(), k, true)
		if err != nil {
			log.Fatal(err)
		}
	}
	log.Println("Program initialized. Start pressing keys!")
	xevent.Main(X)
}

var addr = flag.String("addr", "localhost:1337", "http service address")
var next_key = flag.String("next_key", "XF86AudioNext", "Key to skip to next track.")
var play_key = flag.String("play_key", "XF86AudioPlay", "Key to play and pause.")
var prev_key = flag.String("pause_key", "XF86AudioPrev", "Key to skip to previous track.")

func main() {
	flag.Parse()
  // Maybe make this flag or config file settable.
  var keymap = map[string]string{
  	*next_key: "19", // next
  	*prev_key: "20", // prev
  	*play_key: "16", // play/pause
  }

	go h.run()
	go BindKeys(keymap)
	http.Handle("/", websocket.Handler(wsHandler))
	if err := http.ListenAndServe(*addr, nil); err != nil {
		log.Fatal("ListenAndServe:", err)
	}
}
<commit_msg>go fmt<commit_after>//
// Simple daemon to grab keys and then send them via websocket to the
// keysocket Chrome plugin.
//
// Written by Daniel Hobe but based off of many code examples found online.
//
// Websocket server: garyburd
// https://gist.github.com/1316852
//
// keybinding example: BurtSushi
// https://github.com/BurntSushi/xgbutil/blob/master/examples/simple-keybinding/main.go

//
// Future features:
// - config file
// - connect to Gnome setting daemon over dbus

package main

import (
	"flag"
	"log"
	"net/http"

	"code.google.com/p/go.net/websocket"
	"github.com/BurntSushi/xgbutil"
	"github.com/BurntSushi/xgbutil/keybind"
	"github.com/BurntSushi/xgbutil/xevent"
)

type websocket_connections struct {
	connections map[*connection]bool
	broadcast   chan string
	register    chan *connection
	unregister  chan *connection
}

var h = websocket_connections{
	broadcast:   make(chan string),
	register:    make(chan *connection),
	unregister:  make(chan *connection),
	connections: make(map[*connection]bool),
}

func (h *websocket_connections) run() {
	for {
		select {
		case c := <-h.register:
			h.connections[c] = true
		case c := <-h.unregister:
			delete(h.connections, c)
			close(c.send)
		case m := <-h.broadcast:
			log.Printf("Sending %q\n", m)
			for c := range h.connections {
				select {
				case c.send <- m:
				default:
					delete(h.connections, c)
					close(c.send)
					go c.ws.Close()
				}
			}
		}
	}
}

type connection struct {
	// The websocket connection.
	ws *websocket.Conn

	// Buffered channel of outbound messages.
	send chan string
}

func (c *connection) reader() {
	for {
		var message string
		err := websocket.Message.Receive(c.ws, &message)
		if err != nil {
			break
		}
		if message == "Ping" {
			err := websocket.Message.Send(c.ws, "Pong")
			if err != nil {
				break
			}
		}
	}
	c.ws.Close()
}

func (c *connection) writer() {
	for message := range c.send {
		err := websocket.Message.Send(c.ws, message)
		if err != nil {
			break
		}
	}
	c.ws.Close()
}

func wsHandler(ws *websocket.Conn) {
	c := &connection{send: make(chan string, 2), ws: ws}
	h.register <- c
	defer func() { h.unregister <- c }()
	go c.writer()
	c.reader()
}

func BindKeys(keymap map[string]string) {
	X, err := xgbutil.NewConn()
	if err != nil {
		log.Fatal(err)
	}
	keybind.Initialize(X)
	for k, v := range keymap {
		v := v
		err = keybind.KeyPressFun(func(X *xgbutil.XUtil, e xevent.KeyPressEvent) {
			h.broadcast <- v
		}).Connect(X, X.RootWin(), k, true)
		if err != nil {
			log.Fatal(err)
		}
	}
	log.Println("Program initialized. Start pressing keys!")
	xevent.Main(X)
}

var addr = flag.String("addr", "localhost:1337", "http service address")
var next_key = flag.String("next_key", "XF86AudioNext", "Key to skip to next track.")
var play_key = flag.String("play_key", "XF86AudioPlay", "Key to play and pause.")
var prev_key = flag.String("pause_key", "XF86AudioPrev", "Key to skip to previous track.")

func main() {
	flag.Parse()
	// Maybe make this flag or config file settable.
	var keymap = map[string]string{
		*next_key: "19", // next
		*prev_key: "20", // prev
		*play_key: "16", // play/pause
	}

	go h.run()
	go BindKeys(keymap)
	http.Handle("/", websocket.Handler(wsHandler))
	if err := http.ListenAndServe(*addr, nil); err != nil {
		log.Fatal("ListenAndServe:", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"html/template"
	"log"
	"math/rand"
	"net"
	"net/http"
	"os"
	"os/signal"
	"runtime"
	"sync"
	"syscall"
	"time"
)

const (
	indexHTML = `<!DOCTYPE html>

		<title>Welcome to my service</title>

Welcome!

A simple app using for examining telemetry options.

Overview

This is a toy application which makes calls to upstream services.

The upstream services might fail, or take a while to respond. This gives us "interesting" data to capture and then report on.

Responses

					{{range .}}

							{{.Key}} : {{.Value}}

					{{end}}

Support links

Source

Built by James Abley

`
)

type backend struct {
	server  *http.Server
	address string
	name    string
}

// KeyValue makes the ENV vars into a first-class data structure
type KeyValue struct {
	Key   string
	Value string
}

// KeyValues is a shorter way of referencing an array
type KeyValues []*KeyValue

var (
	tmpl                = template.Must(template.New("index.html").Parse(indexHTML))
	backendServiceNames = []string{
		"Navigation",
		"Content",
		"Search",
		"Product",
		"Price",
		"Shipping",
		"Identity",
		"Customer",
		"Basket",
		"Order",
	}
)

func main() {
	flag.Parse()

	port := getDefaultConfig("PORT", "8080")

	errorChan := make(chan error, 1)

	backends := newBackends(errorChan)

	srv := newMainServer(backends)

	signalChan := make(chan os.Signal, 1)
	signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)

	go listenAndServe(port, srv, errorChan)

	for {
		select {
		case err := <-errorChan:
			if err != nil {
				log.Fatal(err)
			}
		case s := <-signalChan:
			log.Println(fmt.Sprintf("Captured %v. Exiting ...", s))
			d := time.Now().Add(1 * time.Second)
			ctx, cancel := context.WithDeadline(context.Background(), d)
			defer cancel()
			srv.Shutdown(ctx)
			for _, b := range backends {
				b.server.Shutdown(ctx)
			}
			os.Exit(0)
		}
	}
}

func listenAndServe(port string, server *http.Server, errorChan chan<- error) {
	listener, err := newListener(port)
	if err != nil {
		errorChan <- err
		return
	}
	errorChan <- server.Serve(listener)
}

func newMainServer(backends []backend) *http.Server {
	serveMux := http.NewServeMux()

	serveMux.HandleFunc("/", mainHandler(backends))
	serveMux.HandleFunc("/_status", statusHandler)

	return newServer(serveMux)
}

func newBackends(errorChan chan<- error) []backend {
	backends := make([]backend, 10)

	for i := range backends {
		serveMux := http.NewServeMux()
		serveMux.HandleFunc("/", unreliableHandler(rand.Intn(5)+1))
		server := newServer(serveMux)
		listener, err := newListener("0")

		if err != nil {
			panic(err)
		}

		go func() {
			errorChan <- server.Serve(listener)
		}()

		backends[i] = backend{
			server:  server,
			address: listener.Addr().String(),
			name:    backendName(i),
		}
	}

	return backends
}

func backendName(i int) string {
	nameLen := len(backendServiceNames)
	return fmt.Sprintf("%s_%d", backendServiceNames[i%nameLen], i%nameLen)
}

func newListener(port string) (net.Listener, error) {
	return net.Listen("tcp", "0.0.0.0:"+port)
}

func newServer(serveMux http.Handler) *http.Server {
	return &http.Server{
		ReadTimeout:  5 * time.Second,
		WriteTimeout: 10 * time.Second,
		IdleTimeout:  120 * time.Second,
		Handler:      serveMux,
	}
}

func getDefaultConfig(name, fallback string) string {
	if val := os.Getenv(name); val != "" {
		return val
	}
	return fallback
}

func mainHandler(backends []backend) http.HandlerFunc {
	client := &http.Client{}

	return func(w http.ResponseWriter, r *http.Request) {
		fmt.Printf("Frontend received request\n")

		results := make(chan KeyValue, len(backends))

		var wg sync.WaitGroup

		for _, b := range backends {
			wg.Add(1)

			go func(b backend, results chan<- KeyValue) {
				defer wg.Done()
				// TODO(jabley): capture the response time
				// start := time.Now()
				// defer doSomething(b, time.Since(start))
				fetch(client, b, results)
			}(b, results)
		}

		wg.Wait()

		values := make([]KeyValue, len(backends))
		for i := range values {
			values[i] = <-results
		}

		w.Header().Set("Content-Type", "text/html; charset=UTF-8")
		w.Header().Set("Cache-Control", "private, no-cache, no-store, must-revalidate")

		if err := tmpl.Execute(w, values); err != nil {
			panic(err)
		}
	})
}

func fetch(client *http.Client, b backend, results chan<- KeyValue) {
	fmt.Printf("Sending request to backend %s\n", b.name)

	res, err := client.Get("http://" + b.address)

	fmt.Printf("Received response from backend %s\n", b.name)

	if err != nil {
		results <- KeyValue{b.name, err.Error()}
	} else {
		defer res.Body.Close()
		results <- KeyValue{b.name, res.Status}
	}
}

func statusHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json")
	w.Header().Set("Cache-Control", "private, no-cache, no-store, must-revalidate")
	w.WriteHeader(http.StatusOK)
	var mem runtime.MemStats
	runtime.ReadMemStats(&mem)
	json.NewEncoder(w).Encode(mem)
}

func unreliableHandler(percentageFailures int) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		fmt.Printf("Backend received request\n")

		if rand.Intn(100) < percentageFailures {
			w.WriteHeader(http.StatusBadRequest)
			w.Write([]byte(`{
  "errors": [
    {
      "status": "400",
      "source": { "pointer": "/data/attributes/first-name" },
      "title":  "Invalid Attribute",
      "detail": "First name must contain at least three characters."
    }
  ]
}`))
		} else {
			w.WriteHeader(http.StatusOK)
			w.Write([]byte(`{
  "data": [{
    "type": "articles",
    "id": "1",
    "attributes": {
      "title": "JSON API paints my bikeshed!",
      "body": "The shortest article. Ever.",
      "created": "2015-05-22T14:56:29.000Z",
      "updated": "2015-05-22T14:56:28.000Z"
    },
    "relationships": {
      "author": {
        "data": {"id": "42", "type": "people"}
      }
    }
  }],
  "included": [
    {
      "type": "people",
      "id": "42",
      "attributes": {
        "name": "John",
        "age": 80,
        "gender": "male"
      }
    }
  ]
}`))
		}
	}
}
<commit_msg>Add request ID in context<commit_after>package main

import (
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"html/template"
	"log"
	"math/rand"
	"net"
	"net/http"
	"os"
	"os/signal"
	"runtime"
	"sync"
	"syscall"
	"time"
)

type key int

const requestIDKey key = 0

const (
	indexHTML = `<!DOCTYPE html>

		<title>Welcome to my service</title>

Welcome!

A simple app using for examining telemetry options.

Overview

This is a toy application which makes calls to upstream services.

The upstream services might fail, or take a while to respond. This gives us "interesting" data to capture and then report on.

Responses

					{{range .}}

							{{.Key}} : {{.Value}}

					{{end}}

Support links

Source

Built by James Abley

`
)

type backend struct {
	server  *http.Server
	address string
	name    string
}

// KeyValue makes the ENV vars into a first-class data structure
type KeyValue struct {
	Key   string
	Value string
}

// KeyValues is a shorter way of referencing an array
type KeyValues []*KeyValue

var (
	tmpl                = template.Must(template.New("index.html").Parse(indexHTML))
	backendServiceNames = []string{
		"Navigation",
		"Content",
		"Search",
		"Product",
		"Price",
		"Shipping",
		"Identity",
		"Customer",
		"Basket",
		"Order",
	}
)

func main() {
	flag.Parse()

	port := getDefaultConfig("PORT", "8080")

	errorChan := make(chan error, 1)

	backends := newBackends(errorChan)

	srv := newMainServer(backends)

	signalChan := make(chan os.Signal, 1)
	signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)

	go listenAndServe(port, srv, errorChan)

	for {
		select {
		case err := <-errorChan:
			if err != nil {
				log.Fatal(err)
			}
		case s := <-signalChan:
			log.Println(fmt.Sprintf("Captured %v. Exiting ...", s))
			d := time.Now().Add(1 * time.Second)
			ctx, cancel := context.WithDeadline(context.Background(), d)
			defer cancel()
			srv.Shutdown(ctx)
			for _, b := range backends {
				b.server.Shutdown(ctx)
			}
			os.Exit(0)
		}
	}
}

func listenAndServe(port string, server *http.Server, errorChan chan<- error) {
	listener, err := newListener(port)
	if err != nil {
		errorChan <- err
		return
	}
	errorChan <- server.Serve(listener)
}

func newMainServer(backends []backend) *http.Server {
	serveMux := http.NewServeMux()

	serveMux.Handle("/", requestID(mainHandler(backends)))
	serveMux.Handle("/_status", statusHandler())

	return newServer(serveMux)
}

func newBackends(errorChan chan<- error) []backend {
	backends := make([]backend, 10)

	for i := range backends {
		serveMux := http.NewServeMux()
		serveMux.Handle("/", unreliableHandler(rand.Intn(5)+1))
		server := newServer(serveMux)
		listener, err := newListener("0")

		if err != nil {
			panic(err)
		}

		go func() {
			errorChan <- server.Serve(listener)
		}()

		backends[i] = backend{
			server:  server,
			address: listener.Addr().String(),
			name:    backendName(i),
		}
	}

	return backends
}

func backendName(i int) string {
	nameLen := len(backendServiceNames)
	return fmt.Sprintf("%s_%d", backendServiceNames[i%nameLen], i%nameLen)
}

func newListener(port string) (net.Listener, error) {
	return net.Listen("tcp", "0.0.0.0:"+port)
}

func newServer(serveMux http.Handler) *http.Server {
	return &http.Server{
		ReadTimeout:  5 * time.Second,
		WriteTimeout: 10 * time.Second,
		IdleTimeout:  120 * time.Second,
		Handler:      serveMux,
	}
}

func getDefaultConfig(name, fallback string) string {
	if val := os.Getenv(name); val != "" {
		return val
	}
	return fallback
}

func requestID(next http.Handler) http.Handler {
	return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
		ctx := newContextWithRequestID(r.Context(), r)
		next.ServeHTTP(rw, r.WithContext(ctx))
	})
}

func newContextWithRequestID(ctx context.Context, r *http.Request) context.Context {
	reqID := r.Header.Get("X-Request-ID")
	if reqID == "" {
		reqID = generateRandomID()
	}

	return context.WithValue(ctx, requestIDKey, reqID)
}

func generateRandomID() string {
	b := make([]byte, 16)
	_, err := rand.Read(b)
	if err != nil {
		panic(err)
	}
	// Horrible magic numbers due to UUID spec in RFC4122
	b[6] = (b[6] & 0xF) | (byte(4) << 4)
	b[8] = (b[8] | 0x40) & 0x7F
	return fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:])
}

func mainHandler(backends []backend) http.Handler {
	tr := &http.Transport{
		ResponseHeaderTimeout: 2 * time.Second,
	}
	client := &http.Client{Transport: tr}

	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Printf("Frontend received request\n")

		results := make(chan KeyValue, len(backends))

		var wg sync.WaitGroup

		for _, b := range backends {
			wg.Add(1)

			go func(b backend, results chan<- KeyValue) {
				defer wg.Done()
				// TODO(jabley): capture the response time
				// start := time.Now()
				// defer doSomething(b, time.Since(start))
				fetch(client, b, results)
			}(b, results)
		}

		wg.Wait()

		values := make([]KeyValue, len(backends))
		for i := range values {
			values[i] = <-results
		}

		w.Header().Set("Content-Type", "text/html; charset=UTF-8")
		w.Header().Set("Cache-Control", "private, no-cache, no-store, must-revalidate")

		if err := tmpl.Execute(w, values); err != nil {
			panic(err)
		}
	})
}

func fetch(client *http.Client, b backend, results chan<- KeyValue) {
	fmt.Printf("Sending request to backend %s\n", b.name)

	res, err := client.Get("http://" + b.address)

	fmt.Printf("Received response from backend %s\n", b.name)

	if err != nil {
		results <- KeyValue{b.name, err.Error()}
	} else {
		defer res.Body.Close()
		results <- KeyValue{b.name, res.Status}
	}
}

func statusHandler() http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		w.Header().Set("Cache-Control", "private, no-cache, no-store, must-revalidate")
		w.WriteHeader(http.StatusOK)
		var mem runtime.MemStats
		runtime.ReadMemStats(&mem)
		json.NewEncoder(w).Encode(mem)
	})
}

func unreliableHandler(percentageFailures int) http.Handler {

	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Printf("Backend received request\n")

		if rand.Intn(100) < percentageFailures {
			w.WriteHeader(http.StatusBadRequest)
			w.Write([]byte(`{
  "errors": [
    {
      "status": "400",
      "source": { "pointer": "/data/attributes/first-name" },
      "title":  "Invalid Attribute",
      "detail": "First name must contain at least three characters."
    }
  ]
}`))
		} else {
			w.WriteHeader(http.StatusOK)
			w.Write([]byte(`{
  "data": [{
    "type": "articles",
    "id": "1",
    "attributes": {
      "title": "JSON API paints my bikeshed!",
      "body": "The shortest article. Ever.",
      "created": "2015-05-22T14:56:29.000Z",
      "updated": "2015-05-22T14:56:28.000Z"
    },
    "relationships": {
      "author": {
        "data": {"id": "42", "type": "people"}
      }
    }
  }],
  "included": [
    {
      "type": "people",
      "id": "42",
      "attributes": {
        "name": "John",
        "age": 80,
        "gender": "male"
      }
    }
  ]
}`))
		}
	})
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/gorilla/mux"
	"github.com/minotar/minecraft"
	"github.com/op/go-logging"
	"image"
	"net/http"
	_ "net/http/pprof"
	"os"
	"runtime/debug"
	"strconv"
	"time"
)

const (
	DefaultSize = uint(180)
	MaxSize     = uint(300)
	MinSize     = uint(8)

	SkinCache

	ListenOn = ":9999"

	Minutes            uint = 60
	Hours                   = 60 * Minutes
	Days                    = 24 * Hours
	TimeoutActualSkin       = 2 * Days
	TimeoutFailedFetch      = 15 * Minutes

	MinotarVersion = "2.1"
)

type NotFoundHandler struct{}

func (h NotFoundHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(404)
	fmt.Fprintf(w, "404 not found")
}

func notFoundPage(w http.ResponseWriter, r *http.Request) {
	nfh := NotFoundHandler{}
	nfh.ServeHTTP(w, r)
}
func serverErrorPage(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(500)
	fmt.Fprintf(w, "500 internal server error")
}

func rationalizeSize(inp string) uint {
	out64, err := strconv.ParseUint(inp, 10, 0)
	out := uint(out64)
	if err != nil {
		return DefaultSize
	} else if out > MaxSize {
		return MaxSize
	} else if out < MinSize {
		return MinSize
	}
	return out
}

func addCacheTimeoutHeader(w http.ResponseWriter, timeout uint) {
	w.Header().Add("Cache-Control", fmt.Sprintf("max-age=%d", timeout))
}

func timeBetween(timeA time.Time, timeB time.Time) int64 {
	// millis between two timestamps

	if timeB.Before(timeA) {
		timeA, timeB = timeB, timeA
	}
	return timeB.Sub(timeA).Nanoseconds() / 1000000
}

func fetchImageProcessThen(callback func(minecraft.Skin) (image.Image, error)) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		timeReqStart := time.Now()

		vars := mux.Vars(r)

		username := vars["username"]
		size := rationalizeSize(vars["size"])
		ok := true

		skin := fetchSkin(username)
		var err error

		timeFetch := time.Now()

		img, err := callback(skin)
		if err != nil {
			serverErrorPage(w, r)
			return
		}
		timeProcess := time.Now()

		imgResized := Resize(size, size, img)
		timeResize := time.Now()

		w.Header().Add("Content-Type", "image/png")
		w.Header().Add("X-Requested", "processed")
		w.Header().Add("X-Skin-Hash", skin.Hash)
		var timeout uint
		if ok {
			w.Header().Add("X-Result", "ok")
			timeout = TimeoutActualSkin
		} else {
			w.Header().Add("X-Result", "failed")
			timeout = TimeoutFailedFetch
		}

		timing := fmt.Sprintf("%d+%d+%d=%dms", timeBetween(timeReqStart, timeFetch), timeBetween(timeFetch, timeProcess), timeBetween(timeProcess, timeResize), timeBetween(timeReqStart, timeResize))

		w.Header().Add("X-Timing", timing)
		addCacheTimeoutHeader(w, timeout)
		WritePNG(w, imgResized)

		log.Info("Serving skin for " + username + " (" + timing + ") md5: " + skin.Hash)
	}
}
func skinPage(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)

	username := vars["username"]

	skin := fetchSkin(username)

	w.Header().Add("Content-Type", "image/png")
	w.Header().Add("X-Requested", "skin")
	w.Header().Add("X-Result", "ok")

	WritePNG(w, skin.Image)
}
func downloadPage(w http.ResponseWriter, r *http.Request) {
	headers := w.Header()
	headers.Add("Content-Disposition", "attachment; filename=\"skin.png\"")
	skinPage(w, r)
}

func fetchSkin(username string) minecraft.Skin {
	skin, err := minecraft.FetchSkinFromUrl(username)
	if err != nil {
		log.Error("Failed to get skin for " + username + " from Mojang (" + err.Error() + ")")
		skin, _ = minecraft.FetchSkinForChar()
	}

	return skin

	/* We're not using this for now due to rate limiting restrictions
	skin, err := minecraft.GetSkin(minecraft.User{Name: username})
	if err != nil {
		// Problem with the returned image, probably means we have an incorrect username
		// Hit the accounts api
		user, err := minecraft.GetUser(username)

		if err != nil {
			// There's no account for this person, serve char
			skin, _ = minecraft.FetchSkinForChar()
		} else {
			// Get valid skin
			skin, err = minecraft.GetSkin(user)
			if err != nil {
				// Their skin somehow errored, fallback
				skin, _ = minecraft.FetchSkinForChar()
			}
		}
	}

	return skin
	*/
}

var log = logging.MustGetLogger("imgd")
var format = "[%{time:15:04:05.000000}] %{level:.4s} %{message}"

func main() {
	logBackend := logging.NewLogBackend(os.Stdout, "", 0)
	logging.SetBackend(logBackend)
	logging.SetFormatter(logging.MustStringFormatter(format))

	debug.SetGCPercent(10)

	avatarPage := fetchImageProcessThen(func(skin minecraft.Skin) (image.Image, error) {
		return GetHead(skin)
	})
	helmPage := fetchImageProcessThen(func(skin minecraft.Skin) (image.Image, error) {
		return GetHelm(skin)
	})

	r := mux.NewRouter()
	r.NotFoundHandler = NotFoundHandler{}

	r.HandleFunc("/avatar/{username:"+minecraft.ValidUsernameRegex+"}{extension:(.png)?}", avatarPage)
	r.HandleFunc("/avatar/{username:"+minecraft.ValidUsernameRegex+"}/{size:[0-9]+}{extension:(.png)?}", avatarPage)

	r.HandleFunc("/helm/{username:"+minecraft.ValidUsernameRegex+"}{extension:(.png)?}", helmPage)
	r.HandleFunc("/helm/{username:"+minecraft.ValidUsernameRegex+"}/{size:[0-9]+}{extension:(.png)?}", helmPage)

	r.HandleFunc("/download/{username:"+minecraft.ValidUsernameRegex+"}{extension:(.png)?}", downloadPage)

	r.HandleFunc("/skin/{username:"+minecraft.ValidUsernameRegex+"}{extension:(.png)?}", skinPage)

	r.HandleFunc("/version", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "%s", MinotarVersion)
	})

	r.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Location", "https://minotar.net/")
	})

	http.Handle("/", r)
	err := http.ListenAndServe(ListenOn, nil)
	log.Critical(err.Error())
}
<commit_msg>https://minotar.net/ redirect requires a 302<commit_after>package main

import (
	"fmt"
	"github.com/gorilla/mux"
	"github.com/minotar/minecraft"
	"github.com/op/go-logging"
	"image"
	"net/http"
	_ "net/http/pprof"
	"os"
	"runtime/debug"
	"strconv"
	"time"
)

const (
	DefaultSize = uint(180)
	MaxSize     = uint(300)
	MinSize     = uint(8)

	SkinCache

	ListenOn = ":9999"

	Minutes            uint = 60
	Hours                   = 60 * Minutes
	Days                    = 24 * Hours
	TimeoutActualSkin       = 2 * Days
	TimeoutFailedFetch      = 15 * Minutes

	MinotarVersion = "2.1"
)

type NotFoundHandler struct{}

func (h NotFoundHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(404)
	fmt.Fprintf(w, "404 not found")
}

func notFoundPage(w http.ResponseWriter, r *http.Request) {
	nfh := NotFoundHandler{}
	nfh.ServeHTTP(w, r)
}
func serverErrorPage(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(500)
	fmt.Fprintf(w, "500 internal server error")
}

func rationalizeSize(inp string) uint {
	out64, err := strconv.ParseUint(inp, 10, 0)
	out := uint(out64)
	if err != nil {
		return DefaultSize
	} else if out > MaxSize {
		return MaxSize
	} else if out < MinSize {
		return MinSize
	}
	return out
}

func addCacheTimeoutHeader(w http.ResponseWriter, timeout uint) {
	w.Header().Add("Cache-Control", fmt.Sprintf("max-age=%d", timeout))
}

func timeBetween(timeA time.Time, timeB time.Time) int64 {
	// millis between two timestamps

	if timeB.Before(timeA) {
		timeA, timeB = timeB, timeA
	}
	return timeB.Sub(timeA).Nanoseconds() / 1000000
}

func fetchImageProcessThen(callback func(minecraft.Skin) (image.Image, error)) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		timeReqStart := time.Now()

		vars := mux.Vars(r)

		username := vars["username"]
		size := rationalizeSize(vars["size"])
		ok := true

		skin := fetchSkin(username)
		var err error

		timeFetch := time.Now()

		img, err := callback(skin)
		if err != nil {
			serverErrorPage(w, r)
			return
		}
		timeProcess := time.Now()

		imgResized := Resize(size, size, img)
		timeResize := time.Now()

		w.Header().Add("Content-Type", "image/png")
		w.Header().Add("X-Requested", "processed")
		w.Header().Add("X-Skin-Hash", skin.Hash)
		var timeout uint
		if ok {
			w.Header().Add("X-Result", "ok")
			timeout = TimeoutActualSkin
		} else {
			w.Header().Add("X-Result", "failed")
			timeout = TimeoutFailedFetch
		}

		timing := fmt.Sprintf("%d+%d+%d=%dms", timeBetween(timeReqStart, timeFetch), timeBetween(timeFetch, timeProcess), timeBetween(timeProcess, timeResize), timeBetween(timeReqStart, timeResize))

		w.Header().Add("X-Timing", timing)
		addCacheTimeoutHeader(w, timeout)
		WritePNG(w, imgResized)

		log.Info("Serving skin for " + username + " (" + timing + ") md5: " + skin.Hash)
	}
}
func skinPage(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)

	username := vars["username"]

	skin := fetchSkin(username)

	w.Header().Add("Content-Type", "image/png")
	w.Header().Add("X-Requested", "skin")
	w.Header().Add("X-Result", "ok")

	WritePNG(w, skin.Image)
}
func downloadPage(w http.ResponseWriter, r *http.Request) {
	headers := w.Header()
	headers.Add("Content-Disposition", "attachment; filename=\"skin.png\"")
	skinPage(w, r)
}

func fetchSkin(username string) minecraft.Skin {
	skin, err := minecraft.FetchSkinFromUrl(username)
	if err != nil {
		log.Error("Failed to get skin for " + username + " from Mojang (" + err.Error() + ")")
		skin, _ = minecraft.FetchSkinForChar()
	}

	return skin

	/* We're not using this for now due to rate limiting restrictions
	skin, err := minecraft.GetSkin(minecraft.User{Name: username})
	if err != nil {
		// Problem with the returned image, probably means we have an incorrect username
		// Hit the accounts api
		user, err := minecraft.GetUser(username)

		if err != nil {
			// There's no account for this person, serve char
			skin, _ = minecraft.FetchSkinForChar()
		} else {
			// Get valid skin
			skin, err = minecraft.GetSkin(user)
			if err != nil {
				// Their skin somehow errored, fallback
				skin, _ = minecraft.FetchSkinForChar()
			}
		}
	}

	return skin
	*/
}

var log = logging.MustGetLogger("imgd")
var format = "[%{time:15:04:05.000000}] %{level:.4s} %{message}"

func main() {
	logBackend := logging.NewLogBackend(os.Stdout, "", 0)
	logging.SetBackend(logBackend)
	logging.SetFormatter(logging.MustStringFormatter(format))

	debug.SetGCPercent(10)

	avatarPage := fetchImageProcessThen(func(skin minecraft.Skin) (image.Image, error) {
		return GetHead(skin)
	})
	helmPage := fetchImageProcessThen(func(skin minecraft.Skin) (image.Image, error) {
		return GetHelm(skin)
	})

	r := mux.NewRouter()
	r.NotFoundHandler = NotFoundHandler{}

	r.HandleFunc("/avatar/{username:"+minecraft.ValidUsernameRegex+"}{extension:(.png)?}", avatarPage)
	r.HandleFunc("/avatar/{username:"+minecraft.ValidUsernameRegex+"}/{size:[0-9]+}{extension:(.png)?}", avatarPage)

	r.HandleFunc("/helm/{username:"+minecraft.ValidUsernameRegex+"}{extension:(.png)?}", helmPage)
	r.HandleFunc("/helm/{username:"+minecraft.ValidUsernameRegex+"}/{size:[0-9]+}{extension:(.png)?}", helmPage)

	r.HandleFunc("/download/{username:"+minecraft.ValidUsernameRegex+"}{extension:(.png)?}", downloadPage)

	r.HandleFunc("/skin/{username:"+minecraft.ValidUsernameRegex+"}{extension:(.png)?}", skinPage)

	r.HandleFunc("/version", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "%s", MinotarVersion)
	})

	r.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Location", "https://minotar.net/")
		w.WriteHeader(302)
	})

	http.Handle("/", r)
	err := http.ListenAndServe(ListenOn, nil)
	log.Critical(err.Error())
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"
)

func showOnsenList(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Onsen List")
}

func showKika(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "pc")
}
func showChiiia12(w http.ResponseWriter,r *http.Request){
	fmt.Fprintf(w,"chiiia12")
}

func main() {
	http.HandleFunc("/onsen/list", showOnsenList)
	http.HandleFunc("/onsen/list2", showOnsenList2)
	http.HandleFunc("/chiiia12",showChiiia12)
	http.HandleFunc("/kika", showKika)
	http.ListenAndServe(":8080", nil)
}
<commit_msg>modified error<commit_after>package main

import (
	"fmt"
	"net/http"
)

func showOnsenList(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Onsen List")
}

func showOnsenList2(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Onsen List")
}

func showKika(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "pc")
}
func showChiiia12(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "chiiia12")
}

func main() {
	http.HandleFunc("/onsen/list", showOnsenList)
	http.HandleFunc("/onsen/list2", showOnsenList2)
	http.HandleFunc("/chiiia12", showChiiia12)
	http.HandleFunc("/kika", showKika)
	http.ListenAndServe(":8080", nil)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"

	"code.google.com/p/gogoprotobuf/proto"

	pb "github.com/dgryski/carbonzipper/carbonzipperpb"
)

type zipper string

var Zipper zipper

var timeFormats = []string{"15:04 20060102", "20060102", "01/02/06"}

// dateParamToEpoch turns a passed string parameter into an epoch we can send to the Zipper
func dateParamToEpoch(s string, d int64) string {

	if s == "" {
		// return the default if nothing was passed
		return strconv.Itoa(int(d))
	}

	// relative timestamp
	if s[0] == '-' {

		j := 1
		for j < len(s) && s[j] >= '0' && s[j] <= '9' {
			j++
		}
		offsetStr, unitStr := s[:j], s[j:]

		var units time.Duration
		switch unitStr {
		case "s", "sec", "secs", "second", "seconds":
			units = time.Second
		case "min", "minute", "minutes":
			units = time.Minute
		case "h", "hour", "hours":
			units = time.Hour
		case "d", "day", "days":
			units = 24 * time.Hour
		case "mon", "month", "months":
			units = 30 * 24 * time.Hour
		case "y", "year", "years":
			units = 365 * 24 * time.Hour
		}

		offset, err := strconv.Atoi(offsetStr)
		if err != nil {
			return strconv.Itoa(int(d))
		}

		return strconv.Itoa(int(timeNow().Add(time.Duration(offset) * units).Unix()))

	}

	_, err := strconv.Atoi(s)
	if err == nil && len(s) > 8 {
		return s // We got a timestamp so returning it
	}

	if strings.Contains(s, "_") {
		s = strings.Replace(s, "_", " ", 1) // Go can't parse _ in date strings
	}

	for _, format := range timeFormats {
		t, err := time.Parse(format, s)
		if err == nil {
			return strconv.Itoa(int(t.Unix()))
		}
	}
	return strconv.Itoa(int(d))
}

// FIXME(dgryski): extract the http.Get + unproto code into its own function

func (z zipper) Find(metric string) (pb.GlobResponse, error) {

	u, _ := url.Parse(string(z) + "/metrics/find/")

	u.RawQuery = url.Values{
		"query":  []string{metric},
		"format": []string{"protobuf"},
	}.Encode()

	resp, err := http.Get(u.String())
	if err != nil {
		log.Printf("Find: http.Get: %+v\n", err)
		return pb.GlobResponse{}, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Printf("Find: ioutil.ReadAll: %+v\n", err)
		return pb.GlobResponse{}, err
	}

	var pbresp pb.GlobResponse

	err = proto.Unmarshal(body, &pbresp)
	if err != nil {
		log.Printf("Find: proto.Unmarshal: %+v\n", err)
		return pb.GlobResponse{}, err
	}

	return pbresp, nil
}

func (z zipper) Render(metric, from, until string) (pb.FetchResponse, error) {

	u, _ := url.Parse(string(z) + "/render/")

	u.RawQuery = url.Values{
		"target": []string{metric},
		"format": []string{"protobuf"},
		"from":   []string{from},
		"until":  []string{until},
	}.Encode()

	resp, err := http.Get(u.String())
	if err != nil {
		log.Printf("Render: http.Get: %s: %+v\n", metric, err)
		return pb.FetchResponse{}, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Printf("Render: ioutil.ReadAll: %s: %+v\n", metric, err)
		return pb.FetchResponse{}, err
	}

	var pbresp pb.FetchResponse

	err = proto.Unmarshal(body, &pbresp)
	if err != nil {
		log.Printf("Render: proto.Unmarshal: %s: %+v\n", metric, err)
		return pb.FetchResponse{}, err
	}

	return pbresp, nil
}

type limiter chan struct{}

func (l limiter) enter() { l <- struct{}{} }
func (l limiter) leave() { <-l }

var Limiter limiter

// for testing
var timeNow = time.Now

type graphitePoint struct {
	value float64
	t     int32
}

func (g graphitePoint) MarshalJSON() ([]byte, error) {
	// TODO(dgryski): fmt.Sprintf() is slow, use strconv.Append{Float,Int}
	if math.IsNaN(g.value) {
		return []byte(fmt.Sprintf("[null,%d]", g.t)), nil
	}
	return []byte(fmt.Sprintf("[%g,%d]", g.value, g.t)), nil
}

type jsonResponse struct {
	Target     string          `json:"target"`
	Datapoints []graphitePoint `json:"datapoints"`
}

func renderHandler(w http.ResponseWriter, r *http.Request) {

	r.ParseForm()
	targets := r.Form["target"]
	from := r.FormValue("from")
	until := r.FormValue("until")

	// normalize from and until values
	from = dateParamToEpoch(from, timeNow().Add(-24*time.Hour).Unix())
	until = dateParamToEpoch(until, timeNow().Unix())

	var results []*pb.FetchResponse
	// query zipper for find

	for _, target := range targets {

		exp, e, err := parseExpr(target)
		if err != nil || e != "" {
			http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
			return
		}

		metricMap := make(map[string][]*pb.FetchResponse)

		for _, metric := range exp.metrics() {

			// TODO(dgryski): add a caching layer here

			// query zipper for find
			glob, err := Zipper.Find(metric)
			if err != nil {
				continue
			}

			// for each server in find response query render
			rch := make(chan *pb.FetchResponse, len(glob.GetMatches()))
			for _, m := range glob.GetMatches() {
				go func(m *pb.GlobMatch) {
					Limiter.enter()
					var rptr *pb.FetchResponse
					if m.GetIsLeaf() {
						r, err := Zipper.Render(m.GetPath(), from, until)
						if err == nil {
							rptr = &r
						}
					}
					rch <- rptr
					Limiter.leave()
				}(m)
			}

			for i := 0; i < len(glob.GetMatches()); i++ {
				r := <-rch
				if r != nil {
					metricMap[metric] = append(metricMap[metric], r)
				}
			}
		}

		exprs := evalExpr(exp, metricMap)
		results = append(results, exprs...)
	}

	w.Header().Set("Content-Type", "application/json")

	var jresults []jsonResponse

	for _, r := range results {
		if r == nil {
			log.Println("skipping nil result")
		}
		datapoints := make([]graphitePoint, 0, len(r.Values))
		t := *r.StartTime
		for i, v := range r.Values {
			if r.IsAbsent[i] {
				v = math.NaN()
			}
			datapoints = append(datapoints, graphitePoint{value: v, t: t})
			t += *r.StepTime
		}
		jresults = append(jresults, jsonResponse{Target: r.GetName(), Datapoints: datapoints})
	}

	jEnc := json.NewEncoder(w)
	jEnc.Encode(jresults)
}

func main() {

	z := flag.String("z", "", "zipper")
	port := flag.Int("p", 8080, "port")
	l := flag.Int("l", 20, "concurrency limit")

	flag.Parse()

	if *z == "" {
		log.Fatal("no zipper (-z) provided")
	}

	if p := os.Getenv("PORT"); p != "" {
		*port, _ = strconv.Atoi(p)
	}

	Limiter = make(chan struct{}, *l)

	if *z == "" {
		log.Fatal("no zipper provided")
	}

	if _, err := url.Parse(*z); err != nil {
		log.Fatal("unable to parze zipper:", err)
	}

	log.Println("using zipper", *z)

	Zipper = zipper(*z)

	http.HandleFunc("/render/", renderHandler)

	log.Println("listening on port", *port)
	log.Fatalln(http.ListenAndServe(":"+strconv.Itoa(*port), nil))

}
<commit_msg>add weeks to time interval<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"

	"code.google.com/p/gogoprotobuf/proto"

	pb "github.com/dgryski/carbonzipper/carbonzipperpb"
)

type zipper string

var Zipper zipper

var timeFormats = []string{"15:04 20060102", "20060102", "01/02/06"}

// dateParamToEpoch turns a passed string parameter into an epoch we can send to the Zipper
func dateParamToEpoch(s string, d int64) string {

	if s == "" {
		// return the default if nothing was passed
		return strconv.Itoa(int(d))
	}

	// relative timestamp
	if s[0] == '-' {

		j := 1
		for j < len(s) && s[j] >= '0' && s[j] <= '9' {
			j++
		}
		offsetStr, unitStr := s[:j], s[j:]

		var units time.Duration
		switch unitStr {
		case "s", "sec", "secs", "second", "seconds":
			units = time.Second
		case "min", "minute", "minutes":
			units = time.Minute
		case "h", "hour", "hours":
			units = time.Hour
		case "d", "day", "days":
			units = 24 * time.Hour
		case "w", "week", "weeks":
			units = 7 * 24 * time.Hour
		case "mon", "month", "months":
			units = 30 * 24 * time.Hour
		case "y", "year", "years":
			units = 365 * 24 * time.Hour
		}

		offset, err := strconv.Atoi(offsetStr)
		if err != nil {
			return strconv.Itoa(int(d))
		}

		return strconv.Itoa(int(timeNow().Add(time.Duration(offset) * units).Unix()))

	}

	_, err := strconv.Atoi(s)
	if err == nil && len(s) > 8 {
		return s // We got a timestamp so returning it
	}

	if strings.Contains(s, "_") {
		s = strings.Replace(s, "_", " ", 1) // Go can't parse _ in date strings
	}

	for _, format := range timeFormats {
		t, err := time.Parse(format, s)
		if err == nil {
			return strconv.Itoa(int(t.Unix()))
		}
	}
	return strconv.Itoa(int(d))
}

// FIXME(dgryski): extract the http.Get + unproto code into its own function

func (z zipper) Find(metric string) (pb.GlobResponse, error) {

	u, _ := url.Parse(string(z) + "/metrics/find/")

	u.RawQuery = url.Values{
		"query":  []string{metric},
		"format": []string{"protobuf"},
	}.Encode()

	resp, err := http.Get(u.String())
	if err != nil {
		log.Printf("Find: http.Get: %+v\n", err)
		return pb.GlobResponse{}, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Printf("Find: ioutil.ReadAll: %+v\n", err)
		return pb.GlobResponse{}, err
	}

	var pbresp pb.GlobResponse

	err = proto.Unmarshal(body, &pbresp)
	if err != nil {
		log.Printf("Find: proto.Unmarshal: %+v\n", err)
		return pb.GlobResponse{}, err
	}

	return pbresp, nil
}

func (z zipper) Render(metric, from, until string) (pb.FetchResponse, error) {

	u, _ := url.Parse(string(z) + "/render/")

	u.RawQuery = url.Values{
		"target": []string{metric},
		"format": []string{"protobuf"},
		"from":   []string{from},
		"until":  []string{until},
	}.Encode()

	resp, err := http.Get(u.String())
	if err != nil {
		log.Printf("Render: http.Get: %s: %+v\n", metric, err)
		return pb.FetchResponse{}, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Printf("Render: ioutil.ReadAll: %s: %+v\n", metric, err)
		return pb.FetchResponse{}, err
	}

	var pbresp pb.FetchResponse

	err = proto.Unmarshal(body, &pbresp)
	if err != nil {
		log.Printf("Render: proto.Unmarshal: %s: %+v\n", metric, err)
		return pb.FetchResponse{}, err
	}

	return pbresp, nil
}

type limiter chan struct{}

func (l limiter) enter() { l <- struct{}{} }
func (l limiter) leave() { <-l }

var Limiter limiter

// for testing
var timeNow = time.Now

type graphitePoint struct {
	value float64
	t     int32
}

func (g graphitePoint) MarshalJSON() ([]byte, error) {
	// TODO(dgryski): fmt.Sprintf() is slow, use strconv.Append{Float,Int}
	if math.IsNaN(g.value) {
		return []byte(fmt.Sprintf("[null,%d]", g.t)), nil
	}
	return []byte(fmt.Sprintf("[%g,%d]", g.value, g.t)), nil
}

type jsonResponse struct {
	Target     string          `json:"target"`
	Datapoints []graphitePoint `json:"datapoints"`
}

func renderHandler(w http.ResponseWriter, r *http.Request) {

	r.ParseForm()
	targets := r.Form["target"]
	from := r.FormValue("from")
	until := r.FormValue("until")

	// normalize from and until values
	from = dateParamToEpoch(from, timeNow().Add(-24*time.Hour).Unix())
	until = dateParamToEpoch(until, timeNow().Unix())

	var results []*pb.FetchResponse
	// query zipper for find

	for _, target := range targets {

		exp, e, err := parseExpr(target)
		if err != nil || e != "" {
			http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
			return
		}

		metricMap := make(map[string][]*pb.FetchResponse)

		for _, metric := range exp.metrics() {

			// TODO(dgryski): add a caching layer here

			// query zipper for find
			glob, err := Zipper.Find(metric)
			if err != nil {
				continue
			}

			// for each server in find response query render
			rch := make(chan *pb.FetchResponse, len(glob.GetMatches()))
			for _, m := range glob.GetMatches() {
				go func(m *pb.GlobMatch) {
					Limiter.enter()
					var rptr *pb.FetchResponse
					if m.GetIsLeaf() {
						r, err := Zipper.Render(m.GetPath(), from, until)
						if err == nil {
							rptr = &r
						}
					}
					rch <- rptr
					Limiter.leave()
				}(m)
			}

			for i := 0; i < len(glob.GetMatches()); i++ {
				r := <-rch
				if r != nil {
					metricMap[metric] = append(metricMap[metric], r)
				}
			}
		}

		exprs := evalExpr(exp, metricMap)
		results = append(results, exprs...)
	}

	w.Header().Set("Content-Type", "application/json")

	var jresults []jsonResponse

	for _, r := range results {
		if r == nil {
			log.Println("skipping nil result")
		}
		datapoints := make([]graphitePoint, 0, len(r.Values))
		t := *r.StartTime
		for i, v := range r.Values {
			if r.IsAbsent[i] {
				v = math.NaN()
			}
			datapoints = append(datapoints, graphitePoint{value: v, t: t})
			t += *r.StepTime
		}
		jresults = append(jresults, jsonResponse{Target: r.GetName(), Datapoints: datapoints})
	}

	jEnc := json.NewEncoder(w)
	jEnc.Encode(jresults)
}

func main() {

	z := flag.String("z", "", "zipper")
	port := flag.Int("p", 8080, "port")
	l := flag.Int("l", 20, "concurrency limit")

	flag.Parse()

	if *z == "" {
		log.Fatal("no zipper (-z) provided")
	}

	if p := os.Getenv("PORT"); p != "" {
		*port, _ = strconv.Atoi(p)
	}

	Limiter = make(chan struct{}, *l)

	if *z == "" {
		log.Fatal("no zipper provided")
	}

	if _, err := url.Parse(*z); err != nil {
		log.Fatal("unable to parze zipper:", err)
	}

	log.Println("using zipper", *z)

	Zipper = zipper(*z)

	http.HandleFunc("/render/", renderHandler)

	log.Println("listening on port", *port)
	log.Fatalln(http.ListenAndServe(":"+strconv.Itoa(*port), nil))

}
<|endoftext|>