<commit_before>package main

import (
	"flag"
	"math/rand"
	"net/http"
	"time"

	"github.com/gorilla/mux"
	"github.com/sirupsen/logrus"
	"github.com/urfave/negroni"

	"github.com/ivan1993spb/snake-server/connections"
	"github.com/ivan1993spb/snake-server/handlers"
	"github.com/ivan1993spb/snake-server/middlewares"
)

const (
	defaultAddress     = ":8080"
	defaultGroupsLimit = 10
	defaultConnsLimit  = 100
)

var (
	address     string
	groupsLimit int
	connsLimit  int
	seed        int64
	flagJSONLog bool
	logLevel    string
)

func init() {
	flag.StringVar(&address, "address", defaultAddress, "address to serve")
	flag.IntVar(&groupsLimit, "groups-limit", defaultGroupsLimit, "groups limit")
	flag.IntVar(&connsLimit, "conns-limit", defaultConnsLimit, "web-socket connections limit")
	flag.Int64Var(&seed, "seed", time.Now().UnixNano(), "random seed")
	flag.BoolVar(&flagJSONLog, "log-json", false, "use json format for logger")
	flag.StringVar(&logLevel, "log-level", "info", "set log level: panic, fatal, error, warning (warn), info or debug")
	flag.Parse()
}

func logger() logrus.FieldLogger {
	logger := logrus.New()
	if flagJSONLog {
		logger.Formatter = &logrus.JSONFormatter{}
	}
	if level, err := logrus.ParseLevel(logLevel); err != nil {
		logger.SetLevel(logrus.InfoLevel)
	} else {
		logger.SetLevel(level)
	}
	return logger
}

func main() {
	logger := logger()

	logger.WithFields(logrus.Fields{
		"conns_limit":  connsLimit,
		"groups_limit": groupsLimit,
		"seed":         seed,
		"log_level":    logLevel,
	}).Info("preparing to start server")

	rand.Seed(seed)

	groupManager, err := connections.NewConnectionGroupManager(logger, groupsLimit, connsLimit)
	if err != nil {
		logger.Fatalln("cannot create connections group manager:", err)
	}

	r := mux.NewRouter()
	r.Path(handlers.URLRouteCreateGame).Methods(handlers.MethodCreateGame).Handler(handlers.NewCreateGameHandler(logger, groupManager))
	r.Path(handlers.URLRouteGetGameByID).Methods(handlers.MethodGetGame).Handler(handlers.NewGetGameHandler(logger, groupManager))
	r.Path(handlers.URLRouteDeleteGameByID).Methods(handlers.MethodDeleteGame).Handler(handlers.NewDeleteGameHandler(logger, groupManager))
	r.Path(handlers.URLRouteGetGames).Methods(handlers.MethodGetGames).Handler(handlers.NewGetGamesHandler(logger, groupManager))
	r.Path(handlers.URLRouteGameWebSocketByID).Methods(handlers.MethodGame).Handler(handlers.NewGameWebSocketHandler(logger, groupManager))

	n := negroni.New(middlewares.NewRecovery(logger), middlewares.NewLogger(logger))
	n.UseHandler(r)

	logger.WithField("address", address).Info("starting server")

	if err := http.ListenAndServe(address, n); err != nil {
		logger.Fatalf("server error: %s", err)
	}
}
<commit_msg>Create limits: groups and connections<commit_after>package main

import (
	"flag"
	"math/rand"
	"net/http"
	"time"

	"github.com/gorilla/mux"
	"github.com/sirupsen/logrus"
	"github.com/urfave/negroni"

	"github.com/ivan1993spb/snake-server/connections"
	"github.com/ivan1993spb/snake-server/handlers"
	"github.com/ivan1993spb/snake-server/middlewares"
)

const (
	defaultAddress     = ":8080"
	defaultGroupsLimit = 100
	defaultConnsLimit  = 1000
)

var (
	address     string
	groupsLimit int
	connsLimit  int
	seed        int64
	flagJSONLog bool
	logLevel    string
)

func init() {
	flag.StringVar(&address, "address", defaultAddress, "address to serve")
	flag.IntVar(&groupsLimit, "groups-limit", defaultGroupsLimit, "groups limit")
	flag.IntVar(&connsLimit, "conns-limit", defaultConnsLimit, "web-socket connections limit")
	flag.Int64Var(&seed, "seed", time.Now().UnixNano(), "random seed")
	flag.BoolVar(&flagJSONLog, "log-json", false, "use json format for logger")
	flag.StringVar(&logLevel, "log-level", "info", "set log level: panic, fatal, error, warning (warn), info or debug")
	flag.Parse()
}

func logger() logrus.FieldLogger {
	logger := logrus.New()
	if flagJSONLog {
		logger.Formatter = &logrus.JSONFormatter{}
	}
	if level, err := logrus.ParseLevel(logLevel); err != nil {
		logger.SetLevel(logrus.InfoLevel)
	} else {
		logger.SetLevel(level)
	}
	return logger
}

func main() {
	logger := logger()

	logger.WithFields(logrus.Fields{
		"conns_limit":  connsLimit,
		"groups_limit": groupsLimit,
		"seed":         seed,
		"log_level":    logLevel,
	}).Info("preparing to start server")

	rand.Seed(seed)

	groupManager, err := connections.NewConnectionGroupManager(logger, groupsLimit, connsLimit)
	if err != nil {
		logger.Fatalln("cannot create connections group manager:", err)
	}

	r := mux.NewRouter()
	r.Path(handlers.URLRouteCreateGame).Methods(handlers.MethodCreateGame).Handler(handlers.NewCreateGameHandler(logger, groupManager))
	r.Path(handlers.URLRouteGetGameByID).Methods(handlers.MethodGetGame).Handler(handlers.NewGetGameHandler(logger, groupManager))
	r.Path(handlers.URLRouteDeleteGameByID).Methods(handlers.MethodDeleteGame).Handler(handlers.NewDeleteGameHandler(logger, groupManager))
	r.Path(handlers.URLRouteGetGames).Methods(handlers.MethodGetGames).Handler(handlers.NewGetGamesHandler(logger, groupManager))
	r.Path(handlers.URLRouteGameWebSocketByID).Methods(handlers.MethodGame).Handler(handlers.NewGameWebSocketHandler(logger, groupManager))

	n := negroni.New(middlewares.NewRecovery(logger), middlewares.NewLogger(logger))
	n.UseHandler(r)

	logger.WithField("address", address).Info("starting server")

	if err := http.ListenAndServe(address, n); err != nil {
		logger.Fatalf("server error: %s", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os/exec"
	"time"

	"github.com/c9s/goprocinfo/linux"
)

type Usage interface {
	Usage() (float32, error)
}

type UsageFunc func() (float32, error)

func (f UsageFunc) Usage() (float32, error) {
	return f()
}

var MemoryUsage Usage = UsageFunc(func() (percent float32, err error) {
	var mi *linux.MemInfo
	if mi, err = linux.ReadMemInfo("/proc/meminfo"); err != nil {
		return
	}

	free := mi.MemFree + mi.Cached + mi.Buffers
	percent = (float32(free) / float32(mi.MemTotal)) * 100.0
	return
})

func getStats() (*linux.CPUStat, error) {
	if stat, err := linux.ReadStat("/proc/stat"); err != nil {
		return nil, err
	} else {
		return &stat.CPUStatAll, nil
	}
}

type CPUUsage struct {
	sampleTime time.Duration
}

func NewCPUUsage(sampleTime time.Duration) *CPUUsage {
	return &CPUUsage{sampleTime}
}

func (u *CPUUsage) Usage() (percent float32, err error) {
	var pstat, cstat *linux.CPUStat

	if pstat, err = getStats(); err != nil {
		return
	}

	time.Sleep(u.sampleTime)

	if cstat, err = getStats(); err != nil {
		return
	}

	// Calculate the percentage total CPU usage (adapted from
	// <http://stackoverflow.com/questions/23367857/accurate-calculation-of-cpu-usage-given-in-percentage-in-linux>).
	prevIdle := pstat.Idle + pstat.IOWait
	idle := cstat.Idle + cstat.IOWait

	prevNonIdle := pstat.User + pstat.Nice + pstat.System + pstat.IRQ + pstat.SoftIRQ + pstat.Steal
	nonIdle := cstat.User + cstat.Nice + cstat.System + cstat.IRQ + cstat.SoftIRQ + cstat.Steal

	prevTotal := prevIdle + prevNonIdle
	total := idle + nonIdle

	// Differentiate: actual values minus the previous one
	totald := total - prevTotal
	idled := idle - prevIdle

	percent = (float32(totald-idled) / float32(totald)) * 100
	return
}

type Threshold struct {
	LastValue float32 // Last usage value
	Threshold float32 // Percentage threshold
	Metric    Usage   // Source to be checked
	Err       error   // Errors encountered
}

func NewThreshold(metric Usage, threshold float32) *Threshold {
	return &Threshold{
		Threshold: threshold,
		Metric:    metric,
	}
}

func (t *Threshold) Exceeded() bool {
	u, err := t.Metric.Usage()
	if err != nil {
		t.Err = err
		return true
	}

	t.LastValue = u
	return t.LastValue > t.Threshold
}

/*type Check struct {
	Thresholds []
}*/

func main() {
	thresholds := []*Threshold{
		NewThreshold(NewCPUUsage(1*time.Second), 75),
		NewThreshold(MemoryUsage, 60),
	}

	for i := 0; i < 20; i++ {

	Loop:
		for {
			for j, t := range thresholds {
				if t.Exceeded() {
					log.Printf("%d usage = %f%%: waiting", j, t.LastValue)
					continue Loop
				}
			}

			break Loop // No thresholds are exceeded.
		}

		log.Printf("Command %d executing", i)
		go func(index int) {
			cmd := exec.Command("sh", "-c", "timeout 10s yes > /dev/null")
			if err := cmd.Run(); err != nil {
				log.Printf("Command %d failed: %s", index, err)
			} else {
				log.Printf("Command %d succeeded", index)
			}
		}(i)
	}

	// Wait for goroutines to finish.
	select {}
}
<commit_msg>Renamed Usage to Metric to better represent its function.<commit_after>package main

import (
	"log"
	"os/exec"
	"time"

	"github.com/c9s/goprocinfo/linux"
)

type Metricer interface {
	Metric() (float32, error)
}

type MetricFunc func() (float32, error)

func (f MetricFunc) Metric() (float32, error) {
	return f()
}

var MemoryMetric Metricer = MetricFunc(func() (percent float32, err error) {
	var mi *linux.MemInfo
	if mi, err = linux.ReadMemInfo("/proc/meminfo"); err != nil {
		return
	}

	free := mi.MemFree + mi.Cached + mi.Buffers
	percent = (float32(free) / float32(mi.MemTotal)) * 100.0
	return
})

func getStats() (*linux.CPUStat, error) {
	if stat, err := linux.ReadStat("/proc/stat"); err != nil {
		return nil, err
	} else {
		return &stat.CPUStatAll, nil
	}
}

type CPUMetric struct {
	sampleTime time.Duration
}

func NewCPUMetric(sampleTime time.Duration) *CPUMetric {
	return &CPUMetric{sampleTime}
}

func (u *CPUMetric) Metric() (percent float32, err error) {
	var pstat, cstat *linux.CPUStat

	if pstat, err = getStats(); err != nil {
		return
	}

	time.Sleep(u.sampleTime)

	if cstat, err = getStats(); err != nil {
		return
	}

	// Calculate the percentage total CPU usage (adapted from
	// <http://stackoverflow.com/questions/23367857/accurate-calculation-of-cpu-usage-given-in-percentage-in-linux>).
	prevIdle := pstat.Idle + pstat.IOWait
	idle := cstat.Idle + cstat.IOWait

	prevNonIdle := pstat.User + pstat.Nice + pstat.System + pstat.IRQ + pstat.SoftIRQ + pstat.Steal
	nonIdle := cstat.User + cstat.Nice + cstat.System + cstat.IRQ + cstat.SoftIRQ + cstat.Steal

	prevTotal := prevIdle + prevNonIdle
	total := idle + nonIdle

	// Differentiate: actual values minus the previous one
	totald := total - prevTotal
	idled := idle - prevIdle

	percent = (float32(totald-idled) / float32(totald)) * 100
	return
}

type Threshold struct {
	LastValue float32  // Last usage value
	Threshold float32  // Percentage threshold
	Metric    Metricer // Source to be checked
	Err       error    // Errors encountered
}

func NewThreshold(metric Metricer, threshold float32) *Threshold {
	return &Threshold{
		Threshold: threshold,
		Metric:    metric,
	}
}

func (t *Threshold) Exceeded() bool {
	u, err := t.Metric.Metric()
	if err != nil {
		t.Err = err
		return true
	}

	t.LastValue = u
	return t.LastValue > t.Threshold
}

/*type Check struct {
	Thresholds []
}*/

func main() {
	thresholds := []*Threshold{
		NewThreshold(NewCPUMetric(1*time.Second), 75),
		NewThreshold(MemoryMetric, 60),
	}

	for i := 0; i < 20; i++ {

	Loop:
		for {
			for j, t := range thresholds {
				if t.Exceeded() {
					log.Printf("%d usage = %f%%: waiting", j, t.LastValue)
					continue Loop
				}
			}

			break Loop // No thresholds are exceeded.
		}

		log.Printf("Command %d executing", i)
		go func(index int) {
			cmd := exec.Command("sh", "-c", "timeout 10s yes > /dev/null")
			if err := cmd.Run(); err != nil {
				log.Printf("Command %d failed: %s", index, err)
			} else {
				log.Printf("Command %d succeeded", index)
			}
		}(i)
	}

	// Wait for goroutines to finish.
	select {}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"net/http"
	_ "net/http/pprof"
	"os"
	"time"

	"github.com/apex/log"
	"github.com/apex/log/handlers/text"
	"github.com/harlow/nsqd-discovery/dnscfg"
	"github.com/harlow/nsqd-discovery/httpcfg"
)

var (
	ldPort      = flag.Int("lookupd-tcp-port", 4160, "The nsqlookupd TCP port")
	dnsAddr     = flag.String("lookupd-dns-address", "", "The nsqlookupd DNS entry")
	cfgAddr     = flag.String("config-http-address", "", "The config address")
	httpAddrCfg = flag.Bool("config-addresses-as-http", false, "Config nsqlookupd http addresses")
)

func main() {
	flag.Parse()

	if *cfgAddr == "" {
		fmt.Println("required flag not provided: --config-http-address")
		os.Exit(1)
	}

	if *dnsAddr == "" {
		fmt.Println("required flag not provided: --lookupd-dns-address")
		os.Exit(1)
	}

	log.SetHandler(text.New(os.Stderr))
	ctx := log.WithFields(log.Fields{
		"cfgAddr": *cfgAddr,
		"dnsAddr": *dnsAddr,
		"ldPort":  *ldPort,
	})

	ips, err := dnscfg.Get(dnsAddr, ldPort)
	if err != nil {
		ctx.WithError(err).Error("dns lookup")
	}

	if len(ips) == 0 {
		ctx.Error("no ip addresses found")
	}

	cfgURL := "http://" + *cfgAddr + "/config/nsqlookupd_tcp_addresses"
	if *httpAddrCfg {
		cfgURL = "http://" + *cfgAddr + "/config/nsqlookupd_http_addresses"
	}

	err = httpcfg.Set(cfgURL, ips)
	if err != nil {
		ctx.WithError(err).Error("setting config")
	} else {
		ctx.WithField("ips", ips).Info("setting config")
	}

	go configLoop(ctx, cfgURL)

	http.ListenAndServe(":6060", nil)
}

// continue looking at dns entry for changes in config
func configLoop(ctx log.Interface, cfgURL string) {
	ticker := time.Tick(15 * time.Second)

	for {
		select {
		case <-ticker:
			newIPs, err := dnscfg.Get(dnsAddr, ldPort)
			if err != nil {
				ctx.WithError(err).Error("dns lookup")
				continue
			}

			if len(newIPs) == 0 {
				ctx.Error("no ip addresses found")
				continue
			}

			oldIPs, err := httpcfg.Get(cfgURL)
			if err != nil {
				ctx.WithError(err).Error("getting config")
				continue
			}

			if eq(newIPs, oldIPs) {
				ctx.Info("config up to date")
				continue
			}

			err = httpcfg.Set(cfgURL, newIPs)
			if err != nil {
				ctx.WithError(err).Error("setting config")
				continue
			}

			ctx.WithField("ips", newIPs).Info("setting config")
		}
	}
}

// test equality of two slices
func eq(a []string, b []string) bool {
	if len(a) != len(b) {
		return false
	}
	for _, v := range a {
		if !contains(b, v) {
			return false
		}
	}
	return true
}

// test if slice contains string
func contains(s []string, e string) bool {
	for _, a := range s {
		if a == e {
			return true
		}
	}
	return false
}
<commit_msg>add --check-interval (#9)<commit_after>package main

import (
	"flag"
	"fmt"
	"net/http"
	_ "net/http/pprof"
	"os"
	"time"

	"github.com/apex/log"
	"github.com/apex/log/handlers/text"
	"github.com/harlow/nsqd-discovery/dnscfg"
	"github.com/harlow/nsqd-discovery/httpcfg"
)

var (
	ldPort        = flag.Int("lookupd-tcp-port", 4160, "The nsqlookupd TCP port")
	dnsAddr       = flag.String("lookupd-dns-address", "", "The nsqlookupd DNS entry")
	cfgAddr       = flag.String("config-http-address", "", "The config address")
	httpAddrCfg   = flag.Bool("config-addresses-as-http", false, "Config nsqlookupd http addresses")
	checkInterval = flag.Int("check-interval", 15, "Time interval for checks/update")
)

func main() {
	flag.Parse()

	if *cfgAddr == "" {
		fmt.Println("required flag not provided: --config-http-address")
		os.Exit(1)
	}

	if *dnsAddr == "" {
		fmt.Println("required flag not provided: --lookupd-dns-address")
		os.Exit(1)
	}

	log.SetHandler(text.New(os.Stderr))
	ctx := log.WithFields(log.Fields{
		"cfgAddr": *cfgAddr,
		"dnsAddr": *dnsAddr,
		"ldPort":  *ldPort,
	})

	ips, err := dnscfg.Get(dnsAddr, ldPort)
	if err != nil {
		ctx.WithError(err).Error("dns lookup")
	}

	if len(ips) == 0 {
		ctx.Error("no ip addresses found")
	}

	cfgURL := "http://" + *cfgAddr + "/config/nsqlookupd_tcp_addresses"
	if *httpAddrCfg {
		cfgURL = "http://" + *cfgAddr + "/config/nsqlookupd_http_addresses"
	}

	err = httpcfg.Set(cfgURL, ips)
	if err != nil {
		ctx.WithError(err).Error("setting config")
	} else {
		ctx.WithField("ips", ips).Info("setting config")
	}

	go configLoop(ctx, cfgURL)

	http.ListenAndServe(":6060", nil)
}

// continue looking at dns entry for changes in config
func configLoop(ctx log.Interface, cfgURL string) {
	ticker := time.Tick(time.Duration(*checkInterval) * time.Second)

	for {
		select {
		case <-ticker:
			newIPs, err := dnscfg.Get(dnsAddr, ldPort)
			if err != nil {
				ctx.WithError(err).Error("dns lookup")
				continue
			}

			if len(newIPs) == 0 {
				ctx.Error("no ip addresses found")
				continue
			}

			oldIPs, err := httpcfg.Get(cfgURL)
			if err != nil {
				ctx.WithError(err).Error("getting config")
				continue
			}

			if eq(newIPs, oldIPs) {
				ctx.Info("config up to date")
				continue
			}

			err = httpcfg.Set(cfgURL, newIPs)
			if err != nil {
				ctx.WithError(err).Error("setting config")
				continue
			}

			ctx.WithField("ips", newIPs).Info("setting config")
		}
	}
}

// test equality of two slices
func eq(a []string, b []string) bool {
	if len(a) != len(b) {
		return false
	}
	for _, v := range a {
		if !contains(b, v) {
			return false
		}
	}
	return true
}

// test if slice contains string
func contains(s []string, e string) bool {
	for _, a := range s {
		if a == e {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package main

import(
	"fmt"
	"net/http"
	"net/url"
	"os"
   "github.com/Jeffail/gabs" // more easier json parsing
	"io/ioutil"
	"strings"
	)

func helpHandler(w http.ResponseWriter, r *http.Request) {
// Help screen for those accessing url '/'
   fmt.Fprintf(w, "Welcome to BijBling\nValid targets:\n\n")
   fmt.Fprintf(w, "/alert :: reflects websocket alerts from Librato\n")
}

func alertHandler(w http.ResponseWriter, r *http.Request) {
// Handler for url path '/alert', this transforms an alert from librato
// into an annotation stream back inside librato
	luser := os.Getenv("BIJ_USER")
   lpass := os.Getenv("BIJ_TOKEN")

	//panic if we don't have librato credentials
	if luser == `` || lpass == `` {
		fmt.Printf("ERROR::Export BIJ_USER and BIJ_TOKEN with your librato credentials")
	}

	//parse out the server name and alert name from the alert payload
	payload,_ := ioutil.ReadAll(r.Body)
	raw,_ := url.QueryUnescape(string(payload))
	fmt.Printf("Raw input\n\n %s",raw)
	parsed_payload,_ := gabs.ParseJSON(payload)
	account,_ := parsed_payload.Path("account").Data().(string)
   alert, _ := parsed_payload.Path("alert.name").Data().(string)
	url_path := fmt.Sprintf("https://metrics-api.librato.com/v1/annotations/alerts.%s", alert)

	//Only librato.com accounts may use
	if strings.HasSuffix(account,"librato.com"){
		fmt.Printf("Dropping request from %s",account)
		return
	}

   violators, _ := parsed_payload.S("violations").ChildrenMap()
	for host, _ := range violators {
		//for each host that violated the threshold, fire off an annotation
		//into the annotation stream named for this alert
		form := url.Values{}
		form.Set("title", fmt.Sprintf("Alert Fired :: %s ",alert))
		form.Set("source", host)
		form.Set("description", fmt.Sprintf("Alert: %s fired on host %s",alert, host))
		req,_ := http.NewRequest("POST", url_path, strings.NewReader(form.Encode()))
		defer req.Body.Close()
		req.SetBasicAuth(luser,lpass)
		req.Header.Add("Content-Type", "application/x-www-form-urlencoded")

		fmt.Printf("Alert: %s fired on host %s\n", alert, host)
		//post the annotation
		client := new(http.Client)
		resp, err := client.Do(req)

		//log the response from librato-api
		if err != nil {
			fmt.Printf("ERROR :: %s\n",err)
		}
		resp_body,_ := ioutil.ReadAll(resp.Body)
		fmt.Printf("Annotation response: %s\n", resp.Status)
		fmt.Printf("Annotation response body: %s\n", resp_body)
	}
}

//lets kick this pig
func main() {
	port := fmt.Sprintf(":%s",os.Getenv("PORT"))
	fmt.Printf("Starting on port %s",port)
   http.HandleFunc("/", helpHandler)
   http.HandleFunc("/alert", alertHandler)
   http.ListenAndServe(port, nil)
}
<commit_msg>changing to support postform<commit_after>package main

import(
	"fmt"
	"net/http"
	"net/url"
	"os"
   "github.com/Jeffail/gabs" // more easier json parsing
	"io/ioutil"
	"strings"
	)

func helpHandler(w http.ResponseWriter, r *http.Request) {
// Help screen for those accessing url '/'
   fmt.Fprintf(w, "Welcome to BijBling\nValid targets:\n\n")
   fmt.Fprintf(w, "/alert :: reflects websocket alerts from Librato\n")
}

func alertHandler(w http.ResponseWriter, r *http.Request) {
// Handler for url path '/alert', this transforms an alert from librato
// into an annotation stream back inside librato
	luser := os.Getenv("BIJ_USER")
   lpass := os.Getenv("BIJ_TOKEN")

	//panic if we don't have librato credentials
	if luser == `` || lpass == `` {
		fmt.Printf("ERROR::Export BIJ_USER and BIJ_TOKEN with your librato credentials")
	}

	//parse out the server name and alert name from the alert payload
	r_form := r.ParseForm()
	payload := r_form.Get("payload")
	parsed_payload,_ := gabs.ParseJSON(payload)
	account,_ := parsed_payload.Path("account").Data().(string)
   alert, _ := parsed_payload.Path("alert.name").Data().(string)
	url_path := fmt.Sprintf("https://metrics-api.librato.com/v1/annotations/alerts.%s", alert)

	//Only librato.com accounts may use
	if strings.HasSuffix(account,"librato.com"){
		fmt.Printf("Dropping request from %s",account)
		return
	}

   violators, _ := parsed_payload.S("violations").ChildrenMap()
	for host, _ := range violators {
		//for each host that violated the threshold, fire off an annotation
		//into the annotation stream named for this alert
		form := url.Values{}
		form.Set("title", fmt.Sprintf("Alert Fired :: %s ",alert))
		form.Set("source", host)
		form.Set("description", fmt.Sprintf("Alert: %s fired on host %s",alert, host))
		req,_ := http.NewRequest("POST", url_path, strings.NewReader(form.Encode()))
		defer req.Body.Close()
		req.SetBasicAuth(luser,lpass)
		req.Header.Add("Content-Type", "application/x-www-form-urlencoded")

		fmt.Printf("Alert: %s fired on host %s\n", alert, host)
		//post the annotation
		client := new(http.Client)
		resp, err := client.Do(req)

		//log the response from librato-api
		if err != nil {
			fmt.Printf("ERROR :: %s\n",err)
		}
		resp_body,_ := ioutil.ReadAll(resp.Body)
		fmt.Printf("Annotation response: %s\n", resp.Status)
		fmt.Printf("Annotation response body: %s\n", resp_body)
	}
}

//lets kick this pig
func main() {
	port := fmt.Sprintf(":%s",os.Getenv("PORT"))
	fmt.Printf("Starting on port %s",port)
   http.HandleFunc("/", helpHandler)
   http.HandleFunc("/alert", alertHandler)
   http.ListenAndServe(port, nil)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"os/signal"
)

const (
	VERSION                   = "1.0.0"
	defaultPlaceholder        = "{{}}"
)

var placeholder string

var usage = `fzz allows you to run a command interactively.

Usage:

	fzz command

The command MUST include the placeholder '{{}}'.

Arguments:

	-v		Print version and exit
`

func printUsage() {
	fmt.Printf(usage)
}

func main() {
	flVersion := flag.Bool("v", false, "Print fzz version and quit")
	flag.Usage = printUsage
	flag.Parse()

	if *flVersion {
		fmt.Printf("fzz %s\n", VERSION)
		os.Exit(0)
	}

	if len(flag.Args()) < 2 {
		fmt.Fprintf(os.Stderr, usage)
		os.Exit(1)
	}

	if placeholder = os.Getenv("FZZ_PLACEHOLDER"); placeholder == "" {
		placeholder = defaultPlaceholder
	}

	if !validPlaceholder(placeholder) {
		fmt.Fprintln(os.Stderr, "Placeholder is not valid, needs even number of characters")
		os.Exit(1)
	}

	input, args := extractInput(flag.Args(), placeholder)

	if !containsPlaceholder(args, placeholder) {
		fmt.Fprintln(os.Stderr, "No placeholder in arguments")
		os.Exit(1)
	}

	tty, err := NewTTY()
	if err != nil {
		log.Fatal(err)
	}
	defer tty.resetState()

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	go func() {
		<-c
		tty.resetState()
		os.Exit(1)
	}()
	tty.setSttyState("cbreak", "-echo")

	stdinbuf := bytes.Buffer{}
	if isPipe(os.Stdin) {
		io.Copy(&stdinbuf, os.Stdin)
	}

	printer := NewPrinter(tty, tty.cols, tty.rows-1) // prompt is one row

	fzz := &Fzz{
		printer:     printer,
		tty:         tty,
		stdinbuf:    &stdinbuf,
		input:       []byte(input),
		placeholder: placeholder,
		args:        args,
	}
	fzz.Loop()
}
<commit_msg>Update the version to 1.0.1<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"os/signal"
)

const (
	VERSION                   = "1.0.1"
	defaultPlaceholder        = "{{}}"
)

var placeholder string

var usage = `fzz allows you to run a command interactively.

Usage:

	fzz command

The command MUST include the placeholder '{{}}'.

Arguments:

	-v		Print version and exit
`

func printUsage() {
	fmt.Printf(usage)
}

func main() {
	flVersion := flag.Bool("v", false, "Print fzz version and quit")
	flag.Usage = printUsage
	flag.Parse()

	if *flVersion {
		fmt.Printf("fzz %s\n", VERSION)
		os.Exit(0)
	}

	if len(flag.Args()) < 2 {
		fmt.Fprintf(os.Stderr, usage)
		os.Exit(1)
	}

	if placeholder = os.Getenv("FZZ_PLACEHOLDER"); placeholder == "" {
		placeholder = defaultPlaceholder
	}

	if !validPlaceholder(placeholder) {
		fmt.Fprintln(os.Stderr, "Placeholder is not valid, needs even number of characters")
		os.Exit(1)
	}

	input, args := extractInput(flag.Args(), placeholder)

	if !containsPlaceholder(args, placeholder) {
		fmt.Fprintln(os.Stderr, "No placeholder in arguments")
		os.Exit(1)
	}

	tty, err := NewTTY()
	if err != nil {
		log.Fatal(err)
	}
	defer tty.resetState()

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	go func() {
		<-c
		tty.resetState()
		os.Exit(1)
	}()
	tty.setSttyState("cbreak", "-echo")

	stdinbuf := bytes.Buffer{}
	if isPipe(os.Stdin) {
		io.Copy(&stdinbuf, os.Stdin)
	}

	printer := NewPrinter(tty, tty.cols, tty.rows-1) // prompt is one row

	fzz := &Fzz{
		printer:     printer,
		tty:         tty,
		stdinbuf:    &stdinbuf,
		input:       []byte(input),
		placeholder: placeholder,
		args:        args,
	}
	fzz.Loop()
}
<|endoftext|>


<commit_before>// Copyright 2016 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"runtime"

	"github.com/caixw/typing/app"
	"github.com/issue9/logs"
)

const usage = `typing 一个简单博客程序。
源代码以MIT开源许可，并发布于github: https://github.com/caixw/typing

命令行语法：
 typing [options]

 options:
  -h      显示帮助信息；
  -v      显示程序版本信息；
  -appdir 指定程序的数据存放路径，未指定，则为当前运行目录。`

func main() {
	help := flag.Bool("h", false, "显示当前信息")
	version := flag.Bool("v", false, "显示程序的版本信息")
	appdir := flag.String("appdir", "./testdata", "指定运行的数据目录")
	flag.Parse()

	if *help {
		fmt.Println(usage)
		return
	}

	if *version {
		fmt.Println(app.Version, "build with", runtime.Version())
		return
	}

	p := vars.NewPath(*appdir)

	// 初始化日志
	err := logs.InitFromXMLFile(p.ConfLogs)
	if err != nil {
		panic(err)
	}

	logs.Fatal(app.Run(p))
}
<commit_msg>修正引用错误<commit_after>// Copyright 2016 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"runtime"

	"github.com/caixw/typing/app"
	"github.com/caixw/typing/vars"
	"github.com/issue9/logs"
)

const usage = `typing 一个简单博客程序。
源代码以MIT开源许可，并发布于github: https://github.com/caixw/typing

命令行语法：
 typing [options]

 options:
  -h      显示帮助信息；
  -v      显示程序版本信息；
  -appdir 指定程序的数据存放路径，未指定，则为当前运行目录。`

func main() {
	help := flag.Bool("h", false, "显示当前信息")
	version := flag.Bool("v", false, "显示程序的版本信息")
	appdir := flag.String("appdir", "./testdata", "指定运行的数据目录")
	flag.Parse()

	if *help {
		fmt.Println(usage)
		return
	}

	if *version {
		fmt.Println(vars.Version, "build with", runtime.Version())
		return
	}

	p := vars.NewPath(*appdir)

	// 初始化日志
	err := logs.InitFromXMLFile(p.ConfLogs)
	if err != nil {
		panic(err)
	}

	logs.Fatal(app.Run(p))
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"crypto/sha1"
	"encoding/hex"
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
	"sync"

	"code.google.com/p/go.net/idna"
	"github.com/domainr/go-whois/whois"
)

var (
	v, quick      bool
	concurrency   int
	zones         []string
	prefixes      []string
	_, FILE, _, _ = runtime.Caller(0)
	DIR           = filepath.Dir(FILE)
)

func init() {
	flag.BoolVar(&v, "v", false, "verbose output (to stderr)")
	flag.BoolVar(&quick, "quick", false, "Only work on a subset of zones")
	flag.IntVar(&concurrency, "concurrency", 8, "Set maximum number of concurrent requests")
}

func main() {
	flag.Parse()

	if err := main1(); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

func main1() error {
	var err error
	zones, err = readLines("zones.txt")
	if err != nil {
		return err
	}
	prefixes, err = readLines("prefixes.txt")
	if err != nil {
		return err
	}

	// Quick for debugging?
	if quick {
		fmt.Fprintf(os.Stderr, "Quick mode enabled\n")
		zones = []string{"com", "net", "org", "co", "io", "nr"}
	}

	re := regexp.MustCompile(`^[^\.]+\.`)

	domains := make(map[string]bool, len(zones)*len(prefixes))
	for _, zone := range zones {
		for _, prefix := range prefixes {
			domain := prefix + "." + zone
			domains[domain] = true
			req, err := whois.Resolve(domain)
			if err == nil {
				hostParent := re.ReplaceAllLiteralString(req.Host, "")
				if _, ok := domains[hostParent]; !ok {
					fmt.Fprintf(os.Stderr, "  + %s\n", hostParent)
					domains[hostParent] = true
				}
			}
		}
	}

	fmt.Fprintf(os.Stderr, "Querying whois for %d domains (%d prefixes × %d zones + extras)\n", len(domains), len(prefixes), len(zones))

	responses := make(chan *whois.Response)
	limiter := make(chan struct{}, concurrency) // semaphore to limit concurrency
	for domain, _ := range domains {
		go func(domain string) {
			var res *whois.Response

			limiter <- struct{}{} // acquire semaphore
			defer func() {        // release semaphore
				responses <- res
				<-limiter
			}()

			req, err := whois.Resolve(domain)
			if err != nil {
				return
			}

			if v {
				fmt.Fprintf(os.Stderr, "Fetching %s from %s\n", req.Query, req.Host)
			}
			res, err = req.Fetch()
			if err != nil {
				fmt.Fprintf(os.Stderr, "Error fetching whois for %s: %s\n", req.Query, err)
				return
			}
		}(domain)
	}

	// Collect from goroutines
	var wg sync.WaitGroup
	wg.Add(len(domains))
	for i := 0; i < len(domains); i++ {
		go func() {
			res := <-responses
			defer wg.Done()

			if res == nil {
				return
			}

			dir := filepath.Join(DIR, "data", "responses", res.Host)
			err := os.MkdirAll(dir, os.ModePerm)
			if err != nil {
				fmt.Fprintf(os.Stderr, "Error creating response directory for %s: %s\n", res.Host, err)
				return
			}

			fn := filepath.Join(dir, (sha1hex(res.Body) + ".mime"))
			f, err := os.Create(fn)
			if err != nil {
				fmt.Fprintf(os.Stderr, "Error creating response file for %s: %s\n", res.Query, err)
				return
			}
			defer f.Close()
			res.WriteMIME(f)
		}()
	}
	wg.Wait()

	return nil
}

var whitespaceAndComments = regexp.MustCompile(`\s+|#.+$`)

func readLines(fn string) ([]string, error) {
	fmt.Fprintf(os.Stderr, "Reading %s\n", fn)
	f, err := os.Open(filepath.Join(DIR, "data", fn))
	if err != nil {
		return nil, err
	}
	defer f.Close()

	var out []string
	s := bufio.NewScanner(f)
	for s.Scan() {
		line := whitespaceAndComments.ReplaceAllLiteralString(s.Text(), "")
		if line == "" {
			continue
		}
		if line, ierr := idna.ToASCII(line); ierr == nil {
			out = append(out, line)
		}
	}
	return out, s.Err()
}

func sha1hex(buf []byte) string {
	h := sha1.New()
	h.Write(buf)
	return strings.ToLower(hex.EncodeToString(h.Sum(nil)))
}
<commit_msg>Avoid writing an empty body.<commit_after>package main

import (
	"bufio"
	"crypto/sha1"
	"encoding/hex"
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
	"sync"

	"code.google.com/p/go.net/idna"
	"github.com/domainr/go-whois/whois"
)

var (
	v, quick      bool
	concurrency   int
	zones         []string
	prefixes      []string
	_, FILE, _, _ = runtime.Caller(0)
	DIR           = filepath.Dir(FILE)
)

func init() {
	flag.BoolVar(&v, "v", false, "verbose output (to stderr)")
	flag.BoolVar(&quick, "quick", false, "Only work on a subset of zones")
	flag.IntVar(&concurrency, "concurrency", 8, "Set maximum number of concurrent requests")
}

func main() {
	flag.Parse()

	if err := main1(); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

func main1() error {
	var err error
	zones, err = readLines("zones.txt")
	if err != nil {
		return err
	}
	prefixes, err = readLines("prefixes.txt")
	if err != nil {
		return err
	}

	// Quick for debugging?
	if quick {
		fmt.Fprintf(os.Stderr, "Quick mode enabled\n")
		zones = []string{"com", "net", "org", "co", "io", "nr"}
	}

	re := regexp.MustCompile(`^[^\.]+\.`)

	domains := make(map[string]bool, len(zones)*len(prefixes))
	for _, zone := range zones {
		for _, prefix := range prefixes {
			domain := prefix + "." + zone
			domains[domain] = true
			req, err := whois.Resolve(domain)
			if err == nil {
				hostParent := re.ReplaceAllLiteralString(req.Host, "")
				if _, ok := domains[hostParent]; !ok {
					fmt.Fprintf(os.Stderr, "  + %s\n", hostParent)
					domains[hostParent] = true
				}
			}
		}
	}

	fmt.Fprintf(os.Stderr, "Querying whois for %d domains (%d prefixes × %d zones + extras)\n", len(domains), len(prefixes), len(zones))

	responses := make(chan *whois.Response)
	limiter := make(chan struct{}, concurrency) // semaphore to limit concurrency
	for domain, _ := range domains {
		go func(domain string) {
			var res *whois.Response

			limiter <- struct{}{} // acquire semaphore
			defer func() {        // release semaphore
				responses <- res
				<-limiter
			}()

			req, err := whois.Resolve(domain)
			if err != nil {
				return
			}

			if v {
				fmt.Fprintf(os.Stderr, "Fetching %s from %s\n", req.Query, req.Host)
			}
			res, err = req.Fetch()
			if err != nil {
				fmt.Fprintf(os.Stderr, "Error fetching whois for %s: %s\n", req.Query, err)
				return
			}
		}(domain)
	}

	// Collect from goroutines
	var wg sync.WaitGroup
	wg.Add(len(domains))
	for i := 0; i < len(domains); i++ {
		go func() {
			res := <-responses
			defer wg.Done()

			if res == nil {
				return
			}

			if res.Host == "" {
				fmt.Fprintf(os.Stderr, "Response for %q had no host\n", res.Query)
				return
			}

			if len(res.Body) == 0 {
				fmt.Fprintf(os.Stderr, "Response for %q had empty body\n", res.Query)
				return
			}

			dir := filepath.Join(DIR, "data", "responses", res.Host)
			err := os.MkdirAll(dir, os.ModePerm)
			if err != nil {
				fmt.Fprintf(os.Stderr, "Error creating response directory for %s: %s\n", res.Host, err)
				return
			}

			fn := filepath.Join(dir, (sha1hex(res.Body) + ".mime"))
			f, err := os.Create(fn)
			if err != nil {
				fmt.Fprintf(os.Stderr, "Error creating response file for %s: %s\n", res.Query, err)
				return
			}
			defer f.Close()
			res.WriteMIME(f)
		}()
	}
	wg.Wait()

	return nil
}

var whitespaceAndComments = regexp.MustCompile(`\s+|#.+$`)

func readLines(fn string) ([]string, error) {
	fmt.Fprintf(os.Stderr, "Reading %s\n", fn)
	f, err := os.Open(filepath.Join(DIR, "data", fn))
	if err != nil {
		return nil, err
	}
	defer f.Close()

	var out []string
	s := bufio.NewScanner(f)
	for s.Scan() {
		line := whitespaceAndComments.ReplaceAllLiteralString(s.Text(), "")
		if line == "" {
			continue
		}
		if line, ierr := idna.ToASCII(line); ierr == nil {
			out = append(out, line)
		}
	}
	return out, s.Err()
}

func sha1hex(buf []byte) string {
	h := sha1.New()
	h.Write(buf)
	return strings.ToLower(hex.EncodeToString(h.Sum(nil)))
}
<|endoftext|>


<commit_before>/*
Force12.io is a package that monitors demand for resource in a system and then scales and repurposes
containers, based on agreed "quality of service" contracts, to best handle that demand within the constraints of your existing VM or physical infrastructure (for v1).

Force12 is defined to optimize the use of existing physical and VM resources instantly. VMs cannot be scaled in real time (it takes several minutes) and new physical machines take even longer. However, containers can be started or stopped at sub second speeds, allowing your infrastructure to adapt itself in real time to meet system demands.

Force12 is aimed at effectively using the resources you have right now - your existing VMs or physical servers - by using them as optimally as possible.

The Force12 approach is analogous to the way that a router dynamically optimises the use of a physical network. A router is limited by the capacity of the lines physically connected to it. Adding additional capacity is a physical process and takes time. Routers therefore make decisions in real time about which packets will be prioritized on a particular line based on the packet's priority (defined by a "quality of service" contract).

For example, at times of high bandwidth usage a router might prioritize VOIP traffic over web browsing in real time.

Containers allow Force12 to make similar "instant" judgements on service prioritisation within your existing infrastructure. Routers make very simplistic
judgments because they have limited time and cpu and they act at a per packet level. Force12 has the capability of making far more
sophisticated judgements, although even fairly simple ones will still provide a significant new service.

This prototype is a bare bones implementation of Force12.io that recognises only 1 demand types
randomised demand for a priority 1 service (the current value stored in Consul's key value store) aka "client". WHen this minimum demand has been met then a P2 service will
be run (aka "server).

These demand type examples have been chosen purely for simplicity of demonstration. In the future more demand types
will be offered

V1 - Force12.io reacts to increased demand by starting/stopping containers on the slaves already in play.

Note - this version of Force12 starts and stops containers on a Mesos cluser using Marathon as the scheduler
*/

package main

import (
	"log"
	"os"
	"time"

	"bitbucket.org/force12io/force12-scheduler/marathon"
	"bitbucket.org/force12io/force12-scheduler/scheduler"
)

//CONSTANTS
const const_sleep = 100     //milliseconds
const const_stopsleep = 250 //milliseconds pause between stopping and restarting containers
const const_clientdemandstart int = 5
const const_serverdemandstart int = 4
const const_maxcontainers int = 9

//EXPORTED STRUCTS
type Demand struct {
	sched scheduler.Scheduler

	//  mu  sync.Mutex   // plan ahead for concurrency on this potentially shared resource, this is go after all
	clientdemand     int // indicates number of clients demanded (from dynamo table in prototype)
	serverdemand     int // indicates server demand (deduced in prototype)
	clientsrequested int // indicates how many clients we've tried to kick off.
	serversrequested int // indicates how many servers we've tried to kick off.
}

////////////////////////////////////////////////////////////////////////////////////////
// set
// Setter, returns what was there (client, server)
// if provided value is -1 don't update, demand will always be between 0 and const_maxcontainers
func (d *Demand) set(client, server int) (int, int) {
	//d.mu.Lock()
	serverold := d.serverdemand
	clientold := d.clientdemand
	if server != -1 {
		d.serverdemand = server
	}
	if client != -1 {
		d.clientdemand = client
	}
	//d.mu.Unlock()
	return clientold, serverold
}

////////////////////////////////////////////////////////////////////////////////////////
// get
// Getter, returns client, server AEC - Combine this with the set to reduce code
func (d *Demand) get() (int, int) {
	return d.clientdemand, d.serverdemand
}

////////////////////////////////////////////////////////////////////////////////////////
// handle
//
// Called in response to a change in demand to process it.
//
// output err - true if success (AEC currently assumes success).
//
// Note that handle will make any judgment on what to do with a demand
// change, including potentially nothing.
//
func (d *Demand) handle() bool {
	log.Println("Handle demand change.")

	// AEC NOTE THIS FUNCTION NEEDS TO BE HEAVILY REWRITTEN TO HANDLE ECS
	// WHEN WE PORT THAT OVER TO THE SAME STRUCTURE.
	// THe reason is that all the code we wrote to handle stopping before
	// starting etc.. is handled directly by Marathon so that code
	// from the old scheduler needs to go behind the scheduler interface
	err := d.sched.StopStartNTasks(os.Getenv("CLIENT_TASK"), os.Getenv("CLIENT_FAMILY"), d.clientdemand, d.clientsrequested)
	if err != nil {
		log.Printf("Failed to start client tasks. %v", err)
	}
	d.sched.StopStartNTasks(os.Getenv("SERVER_TASK"), os.Getenv("SERVER_FAMILY"), d.serverdemand, d.serversrequested)
	if err != nil {
		log.Printf("Failed to start server tasks. %v", err)
	}

	return false
}

////////////////////////////////////////////////////////////////////////////////////////
// update
//
// Called periodically to check for changes in demand and update accordingly.
//
// output changed? - true if demand changed
//
// Note that this routine will return any change. It makes no judgement on whether that change is
// significant. handle() will determine that.
//
func (d *Demand) update() bool {
	//log.Println("demand update check.")
	var demandchange bool = false

	container_count, err := d.sched.GetContainerCount(os.Getenv("CLIENT_ID"))
	if err != nil {
		log.Printf("Failed to get container count. %v", err)
		return false
	}
	//log.Printf("container count %v\n", container_count)

	//Update our saved client demand
	oldcli, _ := d.set(container_count, const_maxcontainers-container_count)

	//Has the demand changed?
	demandchange = (container_count != oldcli)

	if demandchange {
		log.Println("demandchange from, to ", oldcli, container_count)
	}

	return demandchange
}

////////////////////////////////////////////////////////////////////////////////////////
// MAIN
//
func main() {
	////////////////////////////////////////////////////////////////////////////////////////
	// For the simple prototype, Force12.io sits in a loop checking for demand changes every X milliseconds
	// In phase 2 we'll add a reactive mode where appropriate.
	//
	// Note - we don't route messages from demandcheckers to demandhandlers using channels because we want new values
	// to override old values. Queued history is of no importance here.
	//
	// Also for simplicity this first release is concurrency free (single threaded)
	currentdemand := Demand{
		sched: marathon.NewMarathonScheduler(),
	}
	currentdemand.set(const_clientdemandstart, const_serverdemandstart)
	//var errflag bool = false
	var demandchangeflag bool
	//uncomment code below to output logs to file, but there's nothing clever in here to limit file size
	//f, err := os.OpenFile("testlogfile", os.O_RDWR | os.O_CREATE | os.O_APPEND, 0666)
	//if err != nil {
	//   panic(err)
	//}
	//defer f.Close()

	//log.SetOutput(f)
	log.Println("This is a test log entry")

	// Initialise container types
	err := currentdemand.sched.InitScheduler(os.Getenv("CLIENT_TASK"))
	if err != nil {
		log.Printf("Failed to start client task. %v", err)
		return
	}
	err = currentdemand.sched.InitScheduler(os.Getenv("SERVER_TASK"))
	if err != nil {
		log.Printf("Failed to start server task. %v", err)
		return
	}

	// Find out how many containers we currently have running and get their details
	// Note have decided to do this periodically as a reset as we are getting mysteriously out of whack on ECS
	currentdemand.clientsrequested, currentdemand.serversrequested, err = currentdemand.sched.CountAllTasks()
	if err != nil {
		log.Printf("Failed to count tasks. %v", err)
	}

	//Now we can talk to the DB to check our client demand
	demandchangeflag = currentdemand.update()
	demandchangeflag = true

	for {
		switch {
		case demandchangeflag:
			demandchangeflag = false
			//make any changes dictated by this new demand level
			currentdemand.clientsrequested, currentdemand.serversrequested, err = currentdemand.sched.CountAllTasks()
			if err != nil {
				log.Printf("Failed to count tasks. %v", err)
			}
			//To trace out turn _ = errFlag
			_ = currentdemand.handle()
			//log.Println("demand change. result:", errflag)

		default:
			//log.Println("    .")
			var sleep time.Duration
			sleep = const_sleep * time.Millisecond
			time.Sleep(sleep)
			//Update currentdemand with latest client and server demand, if changed, set flag
			demandchangeflag = currentdemand.update()
		}
	}
}
<commit_msg>Moving the hard-coding of the high-priority container name<commit_after>/*
Force12.io is a package that monitors demand for resource in a system and then scales and repurposes
containers, based on agreed "quality of service" contracts, to best handle that demand within the constraints of your existing VM or physical infrastructure (for v1).

Force12 is defined to optimize the use of existing physical and VM resources instantly. VMs cannot be scaled in real time (it takes several minutes) and new physical machines take even longer. However, containers can be started or stopped at sub second speeds, allowing your infrastructure to adapt itself in real time to meet system demands.

Force12 is aimed at effectively using the resources you have right now - your existing VMs or physical servers - by using them as optimally as possible.

The Force12 approach is analogous to the way that a router dynamically optimises the use of a physical network. A router is limited by the capacity of the lines physically connected to it. Adding additional capacity is a physical process and takes time. Routers therefore make decisions in real time about which packets will be prioritized on a particular line based on the packet's priority (defined by a "quality of service" contract).

For example, at times of high bandwidth usage a router might prioritize VOIP traffic over web browsing in real time.

Containers allow Force12 to make similar "instant" judgements on service prioritisation within your existing infrastructure. Routers make very simplistic
judgments because they have limited time and cpu and they act at a per packet level. Force12 has the capability of making far more
sophisticated judgements, although even fairly simple ones will still provide a significant new service.

This prototype is a bare bones implementation of Force12.io that recognises only 1 demand types
randomised demand for a priority 1 service (the current value stored in Consul's key value store) aka "client". WHen this minimum demand has been met then a P2 service will
be run (aka "server).

These demand type examples have been chosen purely for simplicity of demonstration. In the future more demand types
will be offered

V1 - Force12.io reacts to increased demand by starting/stopping containers on the slaves already in play.

Note - this version of Force12 starts and stops containers on a Mesos cluser using Marathon as the scheduler
*/

package main

import (
	"log"
	"os"
	"time"

	"bitbucket.org/force12io/force12-scheduler/marathon"
	"bitbucket.org/force12io/force12-scheduler/scheduler"
)

//CONSTANTS
const const_sleep = 100     //milliseconds
const const_stopsleep = 250 //milliseconds pause between stopping and restarting containers
const const_clientdemandstart int = 5
const const_serverdemandstart int = 4
const const_maxcontainers int = 9

//EXPORTED STRUCTS
type Demand struct {
	sched scheduler.Scheduler

	//  mu  sync.Mutex   // plan ahead for concurrency on this potentially shared resource, this is go after all
	clientdemand     int // indicates number of clients demanded (from dynamo table in prototype)
	serverdemand     int // indicates server demand (deduced in prototype)
	clientsrequested int // indicates how many clients we've tried to kick off.
	serversrequested int // indicates how many servers we've tried to kick off.
}

////////////////////////////////////////////////////////////////////////////////////////
// set
// Setter, returns what was there (client, server)
// if provided value is -1 don't update, demand will always be between 0 and const_maxcontainers
func (d *Demand) set(client, server int) (int, int) {
	//d.mu.Lock()
	serverold := d.serverdemand
	clientold := d.clientdemand
	if server != -1 {
		d.serverdemand = server
	}
	if client != -1 {
		d.clientdemand = client
	}
	//d.mu.Unlock()
	return clientold, serverold
}

////////////////////////////////////////////////////////////////////////////////////////
// get
// Getter, returns client, server AEC - Combine this with the set to reduce code
func (d *Demand) get() (int, int) {
	return d.clientdemand, d.serverdemand
}

////////////////////////////////////////////////////////////////////////////////////////
// handle
//
// Called in response to a change in demand to process it.
//
// output err - true if success (AEC currently assumes success).
//
// Note that handle will make any judgment on what to do with a demand
// change, including potentially nothing.
//
func (d *Demand) handle() bool {
	log.Println("Handle demand change.")

	// AEC NOTE THIS FUNCTION NEEDS TO BE HEAVILY REWRITTEN TO HANDLE ECS
	// WHEN WE PORT THAT OVER TO THE SAME STRUCTURE.
	// THe reason is that all the code we wrote to handle stopping before
	// starting etc.. is handled directly by Marathon so that code
	// from the old scheduler needs to go behind the scheduler interface
	err := d.sched.StopStartNTasks(os.Getenv("CLIENT_TASK"), os.Getenv("CLIENT_FAMILY"), d.clientdemand, d.clientsrequested)
	if err != nil {
		log.Printf("Failed to start client tasks. %v", err)
	}
	d.sched.StopStartNTasks(os.Getenv("SERVER_TASK"), os.Getenv("SERVER_FAMILY"), d.serverdemand, d.serversrequested)
	if err != nil {
		log.Printf("Failed to start server tasks. %v", err)
	}

	return false
}

////////////////////////////////////////////////////////////////////////////////////////
// update
//
// Called periodically to check for changes in demand and update accordingly.
//
// output changed? - true if demand changed
//
// Note that this routine will return any change. It makes no judgement on whether that change is
// significant. handle() will determine that.
//
func (d *Demand) update() bool {
	//log.Println("demand update check.")
	var demandchange bool = false

	container_count, err := d.sched.GetContainerCount("priority1-demand")
	if err != nil {
		log.Printf("Failed to get container count. %v", err)
		return false
	}
	//log.Printf("container count %v\n", container_count)

	//Update our saved client demand
	oldcli, _ := d.set(container_count, const_maxcontainers-container_count)

	//Has the demand changed?
	demandchange = (container_count != oldcli)

	if demandchange {
		log.Println("demandchange from, to ", oldcli, container_count)
	}

	return demandchange
}

////////////////////////////////////////////////////////////////////////////////////////
// MAIN
//
func main() {
	////////////////////////////////////////////////////////////////////////////////////////
	// For the simple prototype, Force12.io sits in a loop checking for demand changes every X milliseconds
	// In phase 2 we'll add a reactive mode where appropriate.
	//
	// Note - we don't route messages from demandcheckers to demandhandlers using channels because we want new values
	// to override old values. Queued history is of no importance here.
	//
	// Also for simplicity this first release is concurrency free (single threaded)
	currentdemand := Demand{
		sched: marathon.NewMarathonScheduler(),
	}
	currentdemand.set(const_clientdemandstart, const_serverdemandstart)
	//var errflag bool = false
	var demandchangeflag bool
	//uncomment code below to output logs to file, but there's nothing clever in here to limit file size
	//f, err := os.OpenFile("testlogfile", os.O_RDWR | os.O_CREATE | os.O_APPEND, 0666)
	//if err != nil {
	//   panic(err)
	//}
	//defer f.Close()

	//log.SetOutput(f)
	log.Println("This is a test log entry")

	// Initialise container types
	err := currentdemand.sched.InitScheduler(os.Getenv("CLIENT_TASK"))
	if err != nil {
		log.Printf("Failed to start client task. %v", err)
		return
	}
	err = currentdemand.sched.InitScheduler(os.Getenv("SERVER_TASK"))
	if err != nil {
		log.Printf("Failed to start server task. %v", err)
		return
	}

	// Find out how many containers we currently have running and get their details
	// Note have decided to do this periodically as a reset as we are getting mysteriously out of whack on ECS
	currentdemand.clientsrequested, currentdemand.serversrequested, err = currentdemand.sched.CountAllTasks()
	if err != nil {
		log.Printf("Failed to count tasks. %v", err)
	}

	//Now we can talk to the DB to check our client demand
	demandchangeflag = currentdemand.update()
	demandchangeflag = true

	for {
		switch {
		case demandchangeflag:
			demandchangeflag = false
			//make any changes dictated by this new demand level
			currentdemand.clientsrequested, currentdemand.serversrequested, err = currentdemand.sched.CountAllTasks()
			if err != nil {
				log.Printf("Failed to count tasks. %v", err)
			}
			//To trace out turn _ = errFlag
			_ = currentdemand.handle()
			//log.Println("demand change. result:", errflag)

		default:
			//log.Println("    .")
			var sleep time.Duration
			sleep = const_sleep * time.Millisecond
			time.Sleep(sleep)
			//Update currentdemand with latest client and server demand, if changed, set flag
			demandchangeflag = currentdemand.update()
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"encoding/json"
	"flag"
	"html/template"
	"log"
	"net/http"
	"os"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/samherrmann/pingpong/config"
	"github.com/samherrmann/pingpong/ping"
)

var (
	port             *int
	interval         *int
	insecure         *bool
	configFileName   *string
	nodeStatesBuffer = NewNodeStatesBuffer()

	// timeout is the network node-polling
	// timeout in seconds.
	timeout int

	// version is the version of this software,
	// and should be initialized through the
	// following build flags:
	// go build -ldflags "-X main.version=version"
	version = "latest"
)

func main() {
	log.Println("Running pingpong version " + version)

	parseFlags()
	postFlagsParsingInit()

	err := parseConfigFile()
	if err != nil {
		log.Printf("Error while parsing config file: %v", err)
		return
	}

	err = registerUI()
	if err != nil {
		log.Printf("Error while registering UI: %v", err)
		return
	}

	registerAPI()
	monitorNodes()

	err = listenAndServe()
	if err != nil {
		log.Printf("Error while listening for requests: %v", err)
		return
	}
}

// parseConfigFile decodes the configuration JSON
// file into a map.
func parseConfigFile() error {
	// Attempt to parse config file. If successful,
	// exit immediately.
	parseErr := config.ParseFile(*configFileName)
	if parseErr == nil {
		return nil
	}

	// Does file actually exist? If yes, return parse-error
	if _, err := os.Stat(*configFileName); err == nil {
		return parseErr
	}

	// Are we attempting to load the default config file?
	// If no, return parse-error
	if *configFileName != config.DefaultFileName {
		return parseErr
	}

	// Default config file is requested but does
	// not exist, so let's help the user out and
	// create one for them.
	err := config.WriteDefaultFile()
	if err != nil {
		return err
	}

	// Now try parsing the generated config file.
	err = config.ParseFile(*configFileName)
	if err != nil {
		return err
	}
	log.Println("Config file \"" + *configFileName + "\" was not found. A default file was created and loaded instead.")
	return nil
}

func parseFlags() {
	port = flag.Int("port", 8080, "The port on which to access the results.")
	interval = flag.Int("interval", 60, "The interval between each UI refresh in seconds.")
	insecure = flag.Bool("insecure", false, "If set, will not verify the servers' certificate chain and host name.")
	configFileName = flag.String("config", config.DefaultFileName, "The name of the configuration file.")
	flag.Parse()
}

// postFlagsParsingInit performs initialization work
// that depends on values that are provided through
// command-line flags.
func postFlagsParsingInit() {
	timeout = *interval / 2
}

//go:generate go run build/template/main.go

// registerUI registers an HTTP handler function that presents the results
// in a web user interface
func registerUI() error {
	tpl, err := template.New("index.html").Parse(indexHTML)
	if err != nil {
		return err
	}

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		tpl.Execute(w, &ViewModel{*interval, nodeStatesBuffer.Get()})
	})
	return nil
}

// registerAPI registers an HTTP handler function that provides  the results
// in JSON format
func registerAPI() {
	http.HandleFunc("/json", func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		json.NewEncoder(w).Encode(nodeStatesBuffer.Get())
	})
}

// monitorNodes calls the getHTTPStatus function for
// all addresses in the config file that have a "http"
// prefix, and calls the ping function for all other
// provided addresses. This function is executed in a
// goroutine and is therefore non-blocking.
func monitorNodes() {
	go func() {
		for {
			states := &NodeStates{}

			for name, url := range config.Nodes {
				state := &NodeState{}
				state.Name = name
				state.URL = url

				if strings.HasPrefix(state.URL, "http") {
					state.Method = "HTTP/S"

					code, err := getHTTPStatus(state.URL)
					if err != nil {
						state.Note = err.Error()
					}

					if code >= 200 && code < 300 {
						state.IsOK = true
					}

				} else {
					state.Method = "Ping"

					err := ping.Run(state.URL, timeout)
					if err != nil {
						state.Note = err.Error()
					} else {
						state.IsOK = true
					}
				}

				*states = append(*states, *state)
			}
			sort.Sort(states)
			nodeStatesBuffer.Set(states)
			time.Sleep(time.Duration(*interval) * time.Second)
		}
	}()
}

// getHTTPStatus issues an HTTP GET call to the specified URL
// and returns the HTTP status code. 0 is returned along with
// an error if the HTTP call could not be completed successfully.
func getHTTPStatus(url string) (code int, err error) {
	res, err := httpClient().Get(url)
	if err != nil {
		return 0, err
	}
	defer res.Body.Close()
	return res.StatusCode, nil
}

// httpClient returns a HTTP client that does not
// verify a server's certificate chain and host name
// if the "insecure" command-line flag in set to true.
func httpClient() *http.Client {

	timeout := time.Duration(timeout) * time.Second

	if *insecure {
		transport := &http.Transport{
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: true,
			},
		}
		return &http.Client{
			Transport: transport,
			Timeout:   timeout,
		}
	}
	return &http.Client{Timeout: timeout}
}

func listenAndServe() error {
	log.Println("Listening on port " + strconv.Itoa(*port))
	return http.ListenAndServe(":"+strconv.Itoa(*port), nil)
}
<commit_msg>Ignore HTTP status codes<commit_after>package main

import (
	"crypto/tls"
	"encoding/json"
	"flag"
	"html/template"
	"log"
	"net/http"
	"os"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/samherrmann/pingpong/config"
	"github.com/samherrmann/pingpong/ping"
)

var (
	port             *int
	interval         *int
	insecure         *bool
	configFileName   *string
	nodeStatesBuffer = NewNodeStatesBuffer()

	// timeout is the network node-polling
	// timeout in seconds.
	timeout int

	// version is the version of this software,
	// and should be initialized through the
	// following build flags:
	// go build -ldflags "-X main.version=version"
	version = "latest"
)

func main() {
	log.Println("Running pingpong version " + version)

	parseFlags()
	postFlagsParsingInit()

	err := parseConfigFile()
	if err != nil {
		log.Printf("Error while parsing config file: %v", err)
		return
	}

	err = registerUI()
	if err != nil {
		log.Printf("Error while registering UI: %v", err)
		return
	}

	registerAPI()
	monitorNodes()

	err = listenAndServe()
	if err != nil {
		log.Printf("Error while listening for requests: %v", err)
		return
	}
}

// parseConfigFile decodes the configuration JSON
// file into a map.
func parseConfigFile() error {
	// Attempt to parse config file. If successful,
	// exit immediately.
	parseErr := config.ParseFile(*configFileName)
	if parseErr == nil {
		return nil
	}

	// Does file actually exist? If yes, return parse-error
	if _, err := os.Stat(*configFileName); err == nil {
		return parseErr
	}

	// Are we attempting to load the default config file?
	// If no, return parse-error
	if *configFileName != config.DefaultFileName {
		return parseErr
	}

	// Default config file is requested but does
	// not exist, so let's help the user out and
	// create one for them.
	err := config.WriteDefaultFile()
	if err != nil {
		return err
	}

	// Now try parsing the generated config file.
	err = config.ParseFile(*configFileName)
	if err != nil {
		return err
	}
	log.Println("Config file \"" + *configFileName + "\" was not found. A default file was created and loaded instead.")
	return nil
}

func parseFlags() {
	port = flag.Int("port", 8080, "The port on which to access the results.")
	interval = flag.Int("interval", 60, "The interval between each UI refresh in seconds.")
	insecure = flag.Bool("insecure", false, "If set, will not verify the servers' certificate chain and host name.")
	configFileName = flag.String("config", config.DefaultFileName, "The name of the configuration file.")
	flag.Parse()
}

// postFlagsParsingInit performs initialization work
// that depends on values that are provided through
// command-line flags.
func postFlagsParsingInit() {
	timeout = *interval / 2
}

//go:generate go run build/template/main.go

// registerUI registers an HTTP handler function that presents the results
// in a web user interface
func registerUI() error {
	tpl, err := template.New("index.html").Parse(indexHTML)
	if err != nil {
		return err
	}

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		tpl.Execute(w, &ViewModel{*interval, nodeStatesBuffer.Get()})
	})
	return nil
}

// registerAPI registers an HTTP handler function that provides  the results
// in JSON format
func registerAPI() {
	http.HandleFunc("/json", func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		json.NewEncoder(w).Encode(nodeStatesBuffer.Get())
	})
}

// monitorNodes calls the getHTTPStatus function for
// all addresses in the config file that have a "http"
// prefix, and calls the ping function for all other
// provided addresses. This function is executed in a
// goroutine and is therefore non-blocking.
func monitorNodes() {
	go func() {
		for {
			states := &NodeStates{}

			for name, url := range config.Nodes {
				state := &NodeState{}
				state.Name = name
				state.URL = url

				if strings.HasPrefix(state.URL, "http") {
					state.Method = "HTTP/S"

					_, err := getHTTPStatus(state.URL)
					if err != nil {
						state.Note = err.Error()
					} else {
						state.IsOK = true
					}

				} else {
					state.Method = "Ping"

					err := ping.Run(state.URL, timeout)
					if err != nil {
						state.Note = err.Error()
					} else {
						state.IsOK = true
					}
				}

				*states = append(*states, *state)
			}
			sort.Sort(states)
			nodeStatesBuffer.Set(states)
			time.Sleep(time.Duration(*interval) * time.Second)
		}
	}()
}

// getHTTPStatus issues an HTTP GET call to the specified URL
// and returns the HTTP status code. 0 is returned along with
// an error if the HTTP call could not be completed successfully.
func getHTTPStatus(url string) (code int, err error) {
	res, err := httpClient().Get(url)
	if err != nil {
		return 0, err
	}
	defer res.Body.Close()
	return res.StatusCode, nil
}

// httpClient returns a HTTP client that does not
// verify a server's certificate chain and host name
// if the "insecure" command-line flag in set to true.
func httpClient() *http.Client {

	timeout := time.Duration(timeout) * time.Second

	if *insecure {
		transport := &http.Transport{
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: true,
			},
		}
		return &http.Client{
			Transport: transport,
			Timeout:   timeout,
		}
	}
	return &http.Client{Timeout: timeout}
}

func listenAndServe() error {
	log.Println("Listening on port " + strconv.Itoa(*port))
	return http.ListenAndServe(":"+strconv.Itoa(*port), nil)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"os/signal"
	"time"
)

func main() {
	killSignal := make(chan os.Signal, 1)
	signal.Notify(killSignal, os.Interrupt, os.Kill)
mainLoop:
	for {
		client, agentConn, err := setupSshClient()
		if err != nil {
			log.Println("Ssh connection error, trying again in five seconds", err)
			agentConn.Close()
			time.Sleep(5 * time.Second)
			continue
		}

		// Setup ssh tunnels to the remote host
		quit := make(chan struct{})

		err = setupTunnel(quit, client, Config.LocalNfsdPort, Config.RemoteNfsdPort)
		if err != nil {
			log.Println("Unable to create tunnel, trying again in five seconds", err)
			client.Close()
			agentConn.Close()
			close(quit)
			time.Sleep(5 * time.Second)
			continue
		}

		err = setupTunnel(quit, client, Config.LocalMountdPort, Config.RemoteMountdPort)
		if err != nil {
			log.Println("Unable to create tunnel, trying again in five seconds", err)
			client.Close()
			agentConn.Close()
			close(quit)
			time.Sleep(5 * time.Second)
			continue
		}

		if !nfsIsMounted() {
			mountNfs()
		}

		// Create a channel we can select on for when the ssh connection closes
		clientWaitChan := make(chan struct{})
		go func() {
			client.Wait()
			close(clientWaitChan)
		}()

		go func() {
			for {
				waiter := time.After(1 * time.Second)
				select {
				case <-waiter:
				case <-quit:
					return
				}
				fmt.Printf("\r[METER] 10s avg. throughput: %s/second. 1min avg. throughput: %s/second.", Meter.GetHumanReadablePer10Seconds(), Meter.GetHumanReadablePerMinute())
			}
		}()

		select {
		case <-killSignal:
			unmountNfs()
			break mainLoop
		case <-clientWaitChan:
			log.Println("Ssh connection closed, restarting in five seconds")
			agentConn.Close()
			close(quit)
			time.Sleep(5 * time.Second)
		}
	}
	log.Fatalln("Interrupted! Unmounted the NFS and quiting..")
}
<commit_msg>Added conditional unmount based on current status<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"os/signal"
	"time"
)

func main() {
	killSignal := make(chan os.Signal, 1)
	signal.Notify(killSignal, os.Interrupt, os.Kill)
mainLoop:
	for {
		client, agentConn, err := setupSshClient()
		if err != nil {
			log.Println("Ssh connection error, trying again in five seconds", err)
			agentConn.Close()
			time.Sleep(5 * time.Second)
			continue
		}

		// Setup ssh tunnels to the remote host
		quit := make(chan struct{})

		err = setupTunnel(quit, client, Config.LocalNfsdPort, Config.RemoteNfsdPort)
		if err != nil {
			log.Println("Unable to create tunnel, trying again in five seconds", err)
			client.Close()
			agentConn.Close()
			close(quit)
			time.Sleep(5 * time.Second)
			continue
		}

		err = setupTunnel(quit, client, Config.LocalMountdPort, Config.RemoteMountdPort)
		if err != nil {
			log.Println("Unable to create tunnel, trying again in five seconds", err)
			client.Close()
			agentConn.Close()
			close(quit)
			time.Sleep(5 * time.Second)
			continue
		}

		if !nfsIsMounted() {
			mountNfs()
		}

		// Create a channel we can select on for when the ssh connection closes
		clientWaitChan := make(chan struct{})
		go func() {
			client.Wait()
			close(clientWaitChan)
		}()

		go func() {
			for {
				waiter := time.After(1 * time.Second)
				select {
				case <-waiter:
				case <-quit:
					return
				}
				fmt.Printf("\r[METER] 10s avg. throughput: %s/second. 1min avg. throughput: %s/second.", Meter.GetHumanReadablePer10Seconds(), Meter.GetHumanReadablePerMinute())
			}
		}()

		select {
		case <-killSignal:
			if nfsIsMounted() {
				unmountNfs()
			}
			break mainLoop
		case <-clientWaitChan:
			log.Println("Ssh connection closed, restarting in five seconds")
			agentConn.Close()
			close(quit)
			time.Sleep(5 * time.Second)
		}
	}
	log.Fatalln("Interrupted! Unmounted the NFS and quiting..")
}
<|endoftext|>


<commit_before>package main

import (
    "fmt"
    "log"
    "net/http"
)

const listenAddr = "localhost:8080"

func main() {
    fmt.Println("Starting YAPC server!")

    http.HandleFunc("/", handler)

    err := http.ListenAndServe(listenAddr, nil)
    if err != nil {
        log.Fatal(err)
    }

    fmt.Println("Finishing YAPC server!")
}

func handler(w http.ResponseWriter, r *http.Request) {
    fmt.Fprint(w, "Hello, web")
}
<commit_msg>YAPC reads the request, sends a new request to upstream and does the data pipeline to the downstream.<commit_after>package main

import (
    "fmt"
    "log"
    "net/http"
    "io"
)

const listenAddr = ":8080"

func main() {
    fmt.Println("Starting YAPC server!")

    http.HandleFunc("/", handler)

    err := http.ListenAndServe(listenAddr, nil)
    if err != nil {
        log.Fatal(err)
    }

    fmt.Println("Finishing YAPC server!")
}

func handler(w http.ResponseWriter, r *http.Request) {
    url := r.URL.String()
    response, err := http.Get(url)
    if err != nil {
        log.Fatal(err)
    }

    defer response.Body.Close()

    // Using io.Copy to dump the upstream response to the downstream.
    _, err = io.Copy(w, response.Body)
    if err != nil {
        log.Fatal(err)
    }
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	//	"github.com/davecgh/go-spew/spew"
	"github.com/k0kubun/twitter"
	"github.com/mingderwang/userstream"
	"github.com/parnurzeal/gorequest"
	"os"
	"strconv"
	"strings"
)

func main() {
	var CONSUMER_KEY = os.Getenv("CONSUMER_KEY")
	var CONSUMER_SECRET = os.Getenv("CONSUMER_SECRET")
	var ACCESS_TOKEN = os.Getenv("ACCESS_TOKEN")
	var ACCESS_TOKEN_SECRET = os.Getenv("ACCESS_TOKEN_SECRET")
	client := &userstream.Client{
		ConsumerKey:       CONSUMER_KEY,
		ConsumerSecret:    CONSUMER_SECRET,
		AccessToken:       ACCESS_TOKEN,
		AccessTokenSecret: ACCESS_TOKEN_SECRET,
	}

	client.UserStream(func(event interface{}) {
		switch event.(type) {
		case *twitter.Tweet:
			tweet := event.(*twitter.Tweet)
			fmt.Printf("%s: %s\n", tweet.User.ScreenName, tweet.Text)
		case *userstream.Delete:
			tweetDelete := event.(*userstream.Delete)
			fmt.Printf("[delete] %d\n", tweetDelete.Id)
		case *userstream.Favorite:
			favorite := event.(*userstream.Favorite)
			fmt.Printf("[favorite] %s => %s : %s\n",
				favorite.Source.ScreenName, favorite.Target.ScreenName, favorite.TargetObject.Text)
		case *userstream.Unfavorite:
			unfavorite := event.(*userstream.Unfavorite)
			fmt.Printf("[unfavorite] %s => %s : %s\n",
				unfavorite.Source.ScreenName, unfavorite.Target.ScreenName, unfavorite.TargetObject.Text)
		case *userstream.Follow:
			follow := event.(*userstream.Follow)
			fmt.Printf("[follow] %s => %s\n", follow.Source.ScreenName, follow.Target.ScreenName)
		case *userstream.Unfollow:
			unfollow := event.(*userstream.Unfollow)
			fmt.Printf("[unfollow] %s => %s\n", unfollow.Source.ScreenName, unfollow.Target.ScreenName)
		case *userstream.ListMemberAdded:
			listMemberAdded := event.(*userstream.ListMemberAdded)
			fmt.Printf("[list_member_added] %s (%s)\n",
				listMemberAdded.TargetObject.FullName, listMemberAdded.TargetObject.Description)
		case *userstream.ListMemberRemoved:
			listMemberRemoved := event.(*userstream.ListMemberRemoved)
			fmt.Printf("[list_member_removed] %s (%s)\n",
				listMemberRemoved.TargetObject.FullName, listMemberRemoved.TargetObject.Description)
		case *userstream.Record:
			directMessage := event.(*userstream.Record)
			sendRequest(directMessage.DirectMessage.Sender.ScreenName, directMessage.DirectMessage.Text)
		}
	})
}

func stringify(data string) (tag string, schema string) {
	jsonString := strings.SplitN(data, ":", 2)
	//	spew.Dump(jsonString)
	if len(jsonString) == 2 {
		str := strconv.Quote(jsonString[1])
		return jsonString[0], str
	} else {
		return "", ""
	}
}

func sendRequest(userName string, jsonSchemaWithTag string) {
	request := gorequest.New()
	if tag, schema := stringify(jsonSchemaWithTag); tag == "" && schema == "" {
		fmt.Println("error")
	} else {
		str := `{"domainName":"` + userName + `","typeName":` + tag + `,"jsonSchema":` + schema + `}`
		fmt.Printf("%s", str)
		resp, _, _ := request.Post("http://log4security.com:8080/onion").
			Set("Content-Type", "application/json").
			Send(str).
			End()
		//		spew.Dump(resp)
	}
}
<commit_msg>tmp, but get id already<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/davecgh/go-spew/spew"
	"github.com/k0kubun/twitter"
	"github.com/mingderwang/userstream"
	"github.com/parnurzeal/gorequest"
	"net/http"
	"os"
	"strconv"
	"strings"
)

type Onion struct {
	Ginger_Created int32 `json:"ginger_created"`
	Ginger_Id      int32 `json:"ginger_id" gorm:"primary_key"`

	DomainName string `json:"domainName"`
	TypeName   string `json:"typeName"`
	JsonSchema string `json:"jsonSchema"`
}

func main() {
	var CONSUMER_KEY = os.Getenv("CONSUMER_KEY")
	var CONSUMER_SECRET = os.Getenv("CONSUMER_SECRET")
	var ACCESS_TOKEN = os.Getenv("ACCESS_TOKEN")
	var ACCESS_TOKEN_SECRET = os.Getenv("ACCESS_TOKEN_SECRET")
	client := &userstream.Client{
		ConsumerKey:       CONSUMER_KEY,
		ConsumerSecret:    CONSUMER_SECRET,
		AccessToken:       ACCESS_TOKEN,
		AccessTokenSecret: ACCESS_TOKEN_SECRET,
	}

	client.UserStream(func(event interface{}) {
		switch event.(type) {
		case *twitter.Tweet:
			tweet := event.(*twitter.Tweet)
			fmt.Printf("%s: %s\n", tweet.User.ScreenName, tweet.Text)
		case *userstream.Delete:
			tweetDelete := event.(*userstream.Delete)
			fmt.Printf("[delete] %d\n", tweetDelete.Id)
		case *userstream.Favorite:
			favorite := event.(*userstream.Favorite)
			fmt.Printf("[favorite] %s => %s : %s\n",
				favorite.Source.ScreenName, favorite.Target.ScreenName, favorite.TargetObject.Text)
		case *userstream.Unfavorite:
			unfavorite := event.(*userstream.Unfavorite)
			fmt.Printf("[unfavorite] %s => %s : %s\n",
				unfavorite.Source.ScreenName, unfavorite.Target.ScreenName, unfavorite.TargetObject.Text)
		case *userstream.Follow:
			follow := event.(*userstream.Follow)
			fmt.Printf("[follow] %s => %s\n", follow.Source.ScreenName, follow.Target.ScreenName)
		case *userstream.Unfollow:
			unfollow := event.(*userstream.Unfollow)
			fmt.Printf("[unfollow] %s => %s\n", unfollow.Source.ScreenName, unfollow.Target.ScreenName)
		case *userstream.ListMemberAdded:
			listMemberAdded := event.(*userstream.ListMemberAdded)
			fmt.Printf("[list_member_added] %s (%s)\n",
				listMemberAdded.TargetObject.FullName, listMemberAdded.TargetObject.Description)
		case *userstream.ListMemberRemoved:
			listMemberRemoved := event.(*userstream.ListMemberRemoved)
			fmt.Printf("[list_member_removed] %s (%s)\n",
				listMemberRemoved.TargetObject.FullName, listMemberRemoved.TargetObject.Description)
		case *userstream.Record:
			directMessage := event.(*userstream.Record)
			sendRequest(directMessage.DirectMessage.Sender.ScreenName, directMessage.DirectMessage.Text)
		}
	})
}

func stringify(data string) (tag string, schema string) {
	jsonString := strings.SplitN(data, ":", 2)
	//	spew.Dump(jsonString)
	if len(jsonString) == 2 {
		str := strconv.Quote(jsonString[1])
		return jsonString[0], str
	} else {
		return "", ""
	}
}

func sendRequest(userName string, jsonSchemaWithTag string) {
	request := gorequest.New()
	if tag, schema := stringify(jsonSchemaWithTag); tag == "" && schema == "" {
		fmt.Println("error")
	} else {
		str := `{"domainName":"` + userName + `","typeName":` + tag + `,"jsonSchema":` + schema + `}`
		fmt.Printf("%s", str)
		resp, body, err := request.Post("http://log4security.com:8080/onion").
			Set("Content-Type", "application/json").
			Send(str).End()
		if err != nil {
			panic(err)
		}
		spew.Dump(body)
		spew.Dump(resp)
		target := Onion{}
		processResponser(resp, &target)
		spew.Dump(target.Ginger_Id)
		sendRequestByIdForBuild(string(target.Ginger_Id))
	}
}

func sendRequestByIdForBuild(idString string) {

}

func processResponser(response *http.Response, target *Onion) {
	json.NewDecoder(response.Body).Decode(&target)
}

func getJson(url string, target interface{}) error {
	r, err := http.Get(url)
	if err != nil {
		return err
	}
	defer r.Body.Close()

	return json.NewDecoder(r.Body).Decode(target)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"log"
	"math/rand"

	"github.com/AdRoll/goamz/aws"
	"github.com/AdRoll/goamz/sqs"
)

//QuestionQName : name of the questions SQS Queue
const QuestionQName = "demo-questions"

//AnswerQName : name of the answers SQS Queue
const AnswerQName = "demo-answers"

//SQS instance of the sqs var from goamz
var SQS *sqs.SQS

//Question : struct that holds the data for the math problem.
type Question struct {
	Num1 int
	Num2 int
}

var bufferCount = 50 //Playing with this get's different speep through puts.
//The mbAir needs it around 50 or it starts erroring out.
//The mbPro can handle higher values, but diminishing returns.

//But, even set at 50, the mbAir can put 100,000 items in the queu in  48 seconds.

var sem = make(chan bool, bufferCount)

func main() {
}

func doRun(public string, secret string, maxworkers int) error {

	auth := aws.Auth{AccessKey: public, SecretKey: secret}
	region := aws.Region{}
	region.Name = "us-west-2"
	region.SQSEndpoint = "http://sqs.us-west-2.amazonaws.com"
	SQS = sqs.New(auth, region)
	if SQS == nil {
		return fmt.Errorf("Can't get sqs reference for %v %v", auth, region)
	}
	questionq, getErr := SQS.GetQueue(QuestionQName)
	if getErr != nil {
		fmt.Println(QuestionQName)
		return getErr
	}
	msgSlice := make([]sqs.Message, 0, 10) //A slice that holds up to 10 messages
	msgAll := [][]sqs.Message{}

	for i := 0; i < 100000; i++ {
		num1 := rand.Intn(9) + 1
		num2 := rand.Intn(9 + 1)
		q := Question{num1, num2}
		jsonQ, _ := json.Marshal(&q)
		msg := sqs.Message{Body: base64.StdEncoding.EncodeToString(jsonQ)}
		msgSlice = append(msgSlice, msg)
		if len(msgSlice) == 10 {
			msgAll = append(msgAll, msgSlice)
			msgSlice = []sqs.Message{}
		}
	}
	for _, s := range msgAll {
		s := s //It's idomatic go I swear! http://golang.org/doc/effective_go.html#channels
		sem <- true
		go func(sl10 []sqs.Message) {
			addToQuestionQ(sl10, *questionq)
			defer func() { <-sem }()
		}(s)
	}
	for i := 0; i < cap(sem); i++ {
		sem <- true
	}
	return nil
}

func addToQuestionQ(msgList []sqs.Message, sqsQ sqs.Queue) {
	_, respErr := sqsQ.SendMessageBatch(msgList)
	if respErr != nil {
		log.Println("ERROR:", respErr)
	}
}
<commit_msg>not sure about multi clients. added some comments.<commit_after>package main

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"log"
	"math/rand"

	"github.com/AdRoll/goamz/aws"
	"github.com/AdRoll/goamz/sqs"
)

//QuestionQName : name of the questions SQS Queue
const QuestionQName = "demo-questions"

//AnswerQName : name of the answers SQS Queue
const AnswerQName = "demo-answers"

//SQS instance of the sqs var from goamz
var SQS *sqs.SQS

//Question : struct that holds the data for the math problem.
type Question struct {
	Num1 int
	Num2 int
}

var bufferCount = 50 //Playing with this get's different speep through puts.
//The mbAir needs it around 50 or it starts erroring out.
//The mbPro can handle higher values, but diminishing returns.

//But, even set at 50, the mbAir can put 100,000 items in the queu in  48 seconds.

var sem = make(chan bool, bufferCount)

func main() {
}

func getQueue(name, public, secret string) (*sqs.Queue, error) {
	auth := aws.Auth{AccessKey: public, SecretKey: secret}
	region := aws.Region{}
	region.Name = "us-west-2"
	region.SQSEndpoint = "http://sqs.us-west-2.amazonaws.com"
	SQS = sqs.New(auth, region)
	if SQS == nil {
		return nil, fmt.Errorf("Can't get sqs reference for %v %v", auth, region)
	}
	return SQS.GetQueue(name)
}

func doRun(public string, secret string, maxworkers int) error {
	questionq, getErr := getQueue(QuestionQName, public, secret)

	if getErr != nil {
		fmt.Println(QuestionQName)
		return getErr
	}
	msgSlice := make([]sqs.Message, 0, 10) //A slice that holds up to 10 messages
	msgAll := [][]sqs.Message{}

	for i := 0; i < 100000; i++ {
		num1 := rand.Intn(9) + 1
		num2 := rand.Intn(9 + 1)
		q := Question{num1, num2}
		jsonQ, _ := json.Marshal(&q)
		msg := sqs.Message{Body: base64.StdEncoding.EncodeToString(jsonQ)}
		msgSlice = append(msgSlice, msg)
		if len(msgSlice) == 10 {
			msgAll = append(msgAll, msgSlice)
			msgSlice = []sqs.Message{}
		}
	}
	for _, s := range msgAll {
		s := s //It's idomatic go I swear! http://golang.org/doc/effective_go.html#channels

		//Using the Semaphore
		sem <- true
		go func(sl10 []sqs.Message) {
			addToQuestionQ(sl10, *questionq)
			defer func() { <-sem }()
		}(s)
		//******************

		//Running sequentially.
		//addToQuestionQ(s, *questionq)
		//******************
	}
	for i := 0; i < cap(sem); i++ {
		sem <- true
	}
	return nil
}

func addToQuestionQ(msgList []sqs.Message, sqsQ sqs.Queue) {
	_, respErr := sqsQ.SendMessageBatch(msgList)
	if respErr != nil {
		log.Println("ERROR:", respErr)
	}
}
<|endoftext|>