<commit_before>package main

import (
	"flag"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/facebookgo/httpcontrol"
	"github.com/mjibson/moggio/server"

	// codecs
	_ "github.com/mjibson/moggio/codec/flac"
	_ "github.com/mjibson/moggio/codec/gme"
	_ "github.com/mjibson/moggio/codec/mpa"
	_ "github.com/mjibson/moggio/codec/nsf"
	_ "github.com/mjibson/moggio/codec/rar"
	_ "github.com/mjibson/moggio/codec/vorbis"
	_ "github.com/mjibson/moggio/codec/wav"

	// protocols
	// _ "github.com/mjibson/moggio/protocol/bandcamp"

	_ "github.com/mjibson/moggio/protocol/file"
	_ "github.com/mjibson/moggio/protocol/gmusic"
	"github.com/mjibson/moggio/protocol/soundcloud"
	_ "github.com/mjibson/moggio/protocol/stream"
)

var (
	flagAddr       = flag.String("addr", ":6601", "listen address")
	flagDrive      = flag.String("drive", "792434736327-0pup5skbua0gbfld4min3nfv2reairte.apps.googleusercontent.com:OsN_bydWG45resaU0PPiDmtK", "Google Drive API credentials of the form ClientID:ClientSecret")
	flagDropbox    = flag.String("dropbox", "rnhpqsbed2q2ezn:ldref688unj74ld", "Dropbox API credentials of the form ClientID:ClientSecret")
	flagSoundcloud = flag.String("soundcloud", "ec28c2226a0838d01edc6ed0014e462e:a115e94029d698f541960c8dc8560978", "SoundCloud API credentials of the form ClientID:ClientSecret")
	flagDev        = flag.Bool("dev", false, "enable dev mode")
	stateFile      = flag.String("state", "", "specify non-default statefile location")
)

func main() {
	flag.Parse()
	http.DefaultClient = &http.Client{
		Transport: &httpcontrol.Transport{
			ResponseHeaderTimeout: time.Second * 3,
			MaxTries:              3,
			RetryAfterTimeout:     true,
		},
	}
	redir := *flagAddr
	if strings.HasPrefix(redir, ":") {
		redir = "localhost" + redir
	}
	redir = "http://" + redir + "/api/oauth/"
	if *flagDrive != "" {
		sp := strings.Split(*flagDrive, ":")
		if len(sp) != 2 {
			log.Fatal("bad drive string %s", *flagDrive)
		}
		//drive.Init(sp[0], sp[1], redir)
	}
	if *flagDropbox != "" {
		sp := strings.Split(*flagDropbox, ":")
		if len(sp) != 2 {
			log.Fatal("bad drive string %s", *flagDropbox)
		}
		//dropbox.Init(sp[0], sp[1], redir)
	}
	if *flagSoundcloud != "" {
		sp := strings.Split(*flagSoundcloud, ":")
		if len(sp) != 2 {
			log.Fatal("bad drive string %s", *flagSoundcloud)
		}
		soundcloud.Init(sp[0], sp[1], redir)
	}
	if *stateFile == "" {
		switch {
		case *flagDev:
			*stateFile = "moggio.state"
		case runtime.GOOS == "windows":
			dir := filepath.Join(os.Getenv("APPDATA"), "moggio")
			if err := os.MkdirAll(dir, 0600); err != nil {
				log.Fatal(err)
			}
			*stateFile = filepath.Join(dir, "moggio.state")
		default:
			*stateFile = filepath.Join(os.Getenv("HOME"), ".moggio.state")
		}
	}
	log.Fatal(server.ListenAndServe(*stateFile, *flagAddr, *flagDev))
}

//go:generate browserify -t [ reactify --es6 ] server/static/src/nav.js -o server/static/js/moggio.js
<commit_msg>remove unused protocols<commit_after>package main

import (
	"flag"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/facebookgo/httpcontrol"
	"github.com/mjibson/moggio/server"

	// codecs
	_ "github.com/mjibson/moggio/codec/flac"
	_ "github.com/mjibson/moggio/codec/gme"
	_ "github.com/mjibson/moggio/codec/mpa"
	_ "github.com/mjibson/moggio/codec/nsf"
	_ "github.com/mjibson/moggio/codec/rar"
	_ "github.com/mjibson/moggio/codec/vorbis"
	_ "github.com/mjibson/moggio/codec/wav"

	// protocols
	// _ "github.com/mjibson/moggio/protocol/bandcamp"

	_ "github.com/mjibson/moggio/protocol/file"
	_ "github.com/mjibson/moggio/protocol/stream"
)

var (
	flagAddr  = flag.String("addr", ":6601", "listen address")
	flagDev   = flag.Bool("dev", false, "enable dev mode")
	stateFile = flag.String("state", "", "specify non-default statefile location")
)

func main() {
	flag.Parse()
	http.DefaultClient = &http.Client{
		Transport: &httpcontrol.Transport{
			ResponseHeaderTimeout: time.Second * 3,
			MaxTries:              3,
			RetryAfterTimeout:     true,
		},
	}
	redir := *flagAddr
	if strings.HasPrefix(redir, ":") {
		redir = "localhost" + redir
	}
	redir = "http://" + redir + "/api/oauth/"
	if *stateFile == "" {
		switch {
		case *flagDev:
			*stateFile = "moggio.state"
		case runtime.GOOS == "windows":
			dir := filepath.Join(os.Getenv("APPDATA"), "moggio")
			if err := os.MkdirAll(dir, 0600); err != nil {
				log.Fatal(err)
			}
			*stateFile = filepath.Join(dir, "moggio.state")
		default:
			*stateFile = filepath.Join(os.Getenv("HOME"), ".moggio.state")
		}
	}
	log.Fatal(server.ListenAndServe(*stateFile, *flagAddr, *flagDev))
}

//go:generate browserify -t [ reactify --es6 ] server/static/src/nav.js -o server/static/js/moggio.js
<|endoftext|>


<commit_before>//Created By Bipul Wagle
//Bipul Wagle. All rights reserved.

package main

import(
  "github.com/bipulw/ISSync/ISSync"
  "fmt"
  "encoding/json"
  "net/http"
)

type User struct{
        Name string      `json:"name,omitempty"`
        Age int          `json:"age,omitempty"`
        Updated_at string`json:"updated_at,omitempty"`
}

func (self User) CreateResponseObj() ISSync.ISSyncModelInterface{
        obj := self
        obj.Name = ""
        return obj
}

func (self User) ReadResponseObj() ISSync.ISSyncModelInterface{
        obj := self
        obj.Age = 0
        return obj
}

func (self User) UpdateResponseObj() ISSync.ISSyncModelInterface{
        obj := self
        obj.Updated_at = ""
        return obj
}

func (self User) DeleteResponseObj() ISSync.ISSyncModelInterface{
        obj := self
        return obj
}

func (self User) Create() bool{
        fmt.Println("create")
        return false
}

func (self User) Update() bool{
        fmt.Println("update")
        return false
}

func (self User) Delete() bool{
        fmt.Println("delete")
        return false
}

func (self User) Read() bool{
        fmt.Println("read")
        return false

}

func (self User) print(){
        fmt.Println("Name       :",self.Name);
        fmt.Println("Age        :",self.Age);
        fmt.Println("Updated at :",self.Updated_at);
}
var _ ISSync.ISSyncModelInterface = (*User)(nil)

// func processRequest(w http.ResponseWriter, r *http.Request){
//   ISSync.processRequest(w,r)
// }
func generateObject(req ISSync.ISSyncRequest) (ISSync.ISSyncModelInterface){
        //example usage
        if req.ModelID == "user"{
                var usrHoldr User
                jsonInBytes := []byte(req.Data)
                if err := json.Unmarshal(jsonInBytes, &usrHoldr); err != nil {
                        return nil
                }
                return usrHoldr
        }
        return nil
}

func main() {
        //http.HandleFunc("/curd", processRequest)
        ISSync.SetObjectGenerator(generateObject)
        http.HandleFunc("/crud", ISSync.ProcessRequest)
}

// func processResquestWorking(jsonString string){
//      var parsed ISSyncRequest
//      var response ISSyncBaseResponse
//``
//      jsonByte := []byte(jsonString)
//      err := json.Unmarshal(jsonByte, &parsed)
//              if err != nil{
//                      response.Error = true
//                      response.Error_msg = "Invalid Requst - Not Json"
//              }else{
//                      obj := generateObject(parsed)
//                      if obj == nil{
//                              response.Error = true
//                              response.Error_msg = "Error No Object Relation Found"
//                      }else{
//                              switch parsed.Crud {
//                              case "create":
//                                      response = create(obj)
//                              case "update":
//                                      response = update(obj)
//                              case "read":
//                                      response = read(obj)
//                              case "delete":
//                                      response = delete(obj)
//                              }
//                      }
//              }
//      fmt.Println(getJson(response))
//
// }

// func olDmain() {
//      fmt.Println("Hello, playground")
//      usr := new(User)
//      usr.Name = "bipul wagle"
//      usr.Age = 21
//      usr.Updated_at = "today"
//      jsonResult, err:= json.Marshal(&usr)
//      fmt.Println("string" , string(jsonResult), "erro ", err)
//      create(usr)
//      update(usr)
//      delete(usr)
//      read(usr)
//      jsonUsr := []byte("{\"age\":21,\"updated_at\":\"today\"}")
//      usrHoldr := new(User);
//      if err := json.Unmarshal(jsonUsr, usrHoldr); err != nil {
//              panic(err)
//      }
//      usrHoldr.print()
//      http.HandleFunc("/curd", processRequest)
//      req := new(ISSyncRequest)
//      req.Crud = "update"
//      req.ModelID = "user"
//      req.Data = "{\"age\":21,\"updated_at\":\"today\"}"
//
//      fmt.Println(getJson(req))
//      processResquestWorking(getJson(req))
//      processResquestWorking("testing")
//      //processResquestWorking()
//      //processRequest();
// }
<commit_msg>updated main.go<commit_after>//Created By Bipul Wagle
//Bipul Wagle. All rights reserved.

package main

import(
  "./ISSync"
  "fmt"
  "encoding/json"
  "net/http"
)

type User struct{
        Name string      `json:"name,omitempty"`
        Age int          `json:"age,omitempty"`
        Updated_at string`json:"updated_at,omitempty"`
}

func (self User) CreateResponseObj() ISSync.ISSyncModelInterface{
        obj := self
        obj.Name = ""
        return obj
}

func (self User) ReadResponseObj() ISSync.ISSyncModelInterface{
        obj := self
        obj.Age = 0
        return obj
}

func (self User) UpdateResponseObj() ISSync.ISSyncModelInterface{
        obj := self
        obj.Updated_at = ""
        return obj
}

func (self User) DeleteResponseObj() ISSync.ISSyncModelInterface{
        obj := self
        return obj
}

func (self User) Create() bool{
        fmt.Println("create")
        return false
}

func (self User) Update() bool{
        fmt.Println("update")
        return false
}

func (self User) Delete() bool{
        fmt.Println("delete")
        return false
}

func (self User) Read() bool{
        fmt.Println("read")
        return false

}

func (self User) print(){
        fmt.Println("Name       :",self.Name);
        fmt.Println("Age        :",self.Age);
        fmt.Println("Updated at :",self.Updated_at);
}
var _ ISSync.ISSyncModelInterface = (*User)(nil)

// func processRequest(w http.ResponseWriter, r *http.Request){
//   ISSync.processRequest(w,r)
// }
func generateObject(req ISSync.ISSyncRequest) (ISSync.ISSyncModelInterface){
        //example usage
        if req.ModelID == "user"{
                var usrHoldr User
                jsonInBytes := []byte(req.Data)
                if err := json.Unmarshal(jsonInBytes, &usrHoldr); err != nil {
                        return nil
                }
                return usrHoldr
        }
        return nil
}

func main() {
        //http.HandleFunc("/curd", processRequest)
        ISSync.SetObjectGenerator(generateObject)
        http.HandleFunc("/crud", ISSync.ProcessRequest)
}

// func processResquestWorking(jsonString string){
//      var parsed ISSyncRequest
//      var response ISSyncBaseResponse
//``
//      jsonByte := []byte(jsonString)
//      err := json.Unmarshal(jsonByte, &parsed)
//              if err != nil{
//                      response.Error = true
//                      response.Error_msg = "Invalid Requst - Not Json"
//              }else{
//                      obj := generateObject(parsed)
//                      if obj == nil{
//                              response.Error = true
//                              response.Error_msg = "Error No Object Relation Found"
//                      }else{
//                              switch parsed.Crud {
//                              case "create":
//                                      response = create(obj)
//                              case "update":
//                                      response = update(obj)
//                              case "read":
//                                      response = read(obj)
//                              case "delete":
//                                      response = delete(obj)
//                              }
//                      }
//              }
//      fmt.Println(getJson(response))
//
// }

// func olDmain() {
//      fmt.Println("Hello, playground")
//      usr := new(User)
//      usr.Name = "bipul wagle"
//      usr.Age = 21
//      usr.Updated_at = "today"
//      jsonResult, err:= json.Marshal(&usr)
//      fmt.Println("string" , string(jsonResult), "erro ", err)
//      create(usr)
//      update(usr)
//      delete(usr)
//      read(usr)
//      jsonUsr := []byte("{\"age\":21,\"updated_at\":\"today\"}")
//      usrHoldr := new(User);
//      if err := json.Unmarshal(jsonUsr, usrHoldr); err != nil {
//              panic(err)
//      }
//      usrHoldr.print()
//      http.HandleFunc("/curd", processRequest)
//      req := new(ISSyncRequest)
//      req.Crud = "update"
//      req.ModelID = "user"
//      req.Data = "{\"age\":21,\"updated_at\":\"today\"}"
//
//      fmt.Println(getJson(req))
//      processResquestWorking(getJson(req))
//      processResquestWorking("testing")
//      //processResquestWorking()
//      //processRequest();
// }
<|endoftext|>


<commit_before>package main

import "github.com/howeyc/fsnotify"
import "log"
import "flag"
import "fmt"
import "path/filepath"
import "os"
import "os/exec"
import "regexp"

func Subfolders(path string) (paths []string) {
	filepath.Walk(path, func(newPath string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if info.IsDir() {
			name := info.Name()
			// skip folders that begin with a dot
			hidden := filepath.HasPrefix(name, ".") && name != "." && name != ".."
			if hidden {
				return filepath.SkipDir
			} else {
				paths = append(paths, newPath)
			}
		}
		return nil
	})
	return paths
}

func main() {
	flag.Parse()
	var buildFlag = flag.Bool("b",false,"build")
	var testFlag = flag.Bool("t",false,"test")
	args := flag.Args()

	var dirs = []string{}
	var cmds = []string{}

	if *testFlag {
		cmds = []string{ "go", "test" }
	} else if *buildFlag {
		cmds = []string{ "go", "build" }
	} else {
		var takeDir = true
		for _, a := range args {
			if a == "--" {
				takeDir = false
				continue
			}
			if takeDir {
				dirs = append(dirs, a)
			} else {
				cmds = append(cmds, a)
			}
		}
	}

	if len(cmds) == 0 {
		fmt.Println("No command specified")
		os.Exit(2)
	}

	fmt.Println("Watching", dirs, "for", cmds)

	watcher, err := fsnotify.NewWatcher()

	if err != nil {
		log.Fatal(err)
	}

	for _, dir := range dirs {
		subfolders := Subfolders(dir)
		for _, f := range subfolders {
			err = watcher.WatchFlags(f, fsnotify.FSN_CREATE|fsnotify.FSN_MODIFY)
			if err != nil {
				log.Fatal(err)
			}
		}
	}

	var cmd *exec.Cmd

	for {
		select {
		case e := <-watcher.Event:
			matched, err := regexp.MatchString("\\.go$", e.Name)
			if err != nil {
				log.Println(err)
			}

			if !matched {
				continue
			}

			log.Println("Event:", e)

			if cmd != nil {
				err := cmd.Process.Kill()
				if err != nil {
					log.Println(err)
				}
			}
			cmd = exec.Command(cmds[0], cmds[1:]...)
			cmd.Stdout = os.Stdout
			cmd.Stderr = os.Stderr
			cmd.Start()
		case err := <-watcher.Error:
			log.Println("Error:", err)
		}
	}

	watcher.Close()
}
<commit_msg>print options<commit_after>package main

import "github.com/howeyc/fsnotify"
import "log"
import "flag"
import "fmt"
import "path/filepath"
import "os"
import "os/exec"
import "regexp"

func Subfolders(path string) (paths []string) {
	filepath.Walk(path, func(newPath string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if info.IsDir() {
			name := info.Name()
			// skip folders that begin with a dot
			hidden := filepath.HasPrefix(name, ".") && name != "." && name != ".."
			if hidden {
				return filepath.SkipDir
			} else {
				paths = append(paths, newPath)
			}
		}
		return nil
	})
	return paths
}

func main() {
	var helpFlag = flag.Bool("h",false,"help")
	var buildFlag = flag.Bool("b",false,"build")
	var testFlag = flag.Bool("t",false,"test")
	flag.Parse()
	args := flag.Args()

	var dirs = []string{}
	var cmds = []string{}

	if *helpFlag {
		fmt.Println("Usage: gomon [options] [dir] [-- command]")
		fmt.Println("   -b build")
		fmt.Println("   -t test")
		fmt.Println("   -h help")
		os.Exit(0)
	} else if *testFlag {
		cmds = []string{ "go", "test" }
	} else if *buildFlag {
		cmds = []string{ "go", "build" }
	} else {
		var takeDir = true
		for _, a := range args {
			if a == "--" {
				takeDir = false
				continue
			}
			if takeDir {
				dirs = append(dirs, a)
			} else {
				cmds = append(cmds, a)
			}
		}
	}

	if len(cmds) == 0 {
		fmt.Println("No command specified")
		os.Exit(2)
	}

	fmt.Println("Watching", dirs, "for", cmds)

	watcher, err := fsnotify.NewWatcher()

	if err != nil {
		log.Fatal(err)
	}

	for _, dir := range dirs {
		subfolders := Subfolders(dir)
		for _, f := range subfolders {
			err = watcher.WatchFlags(f, fsnotify.FSN_CREATE|fsnotify.FSN_MODIFY)
			if err != nil {
				log.Fatal(err)
			}
		}
	}

	var cmd *exec.Cmd

	for {
		select {
		case e := <-watcher.Event:
			matched, err := regexp.MatchString("\\.go$", e.Name)
			if err != nil {
				log.Println(err)
			}

			if !matched {
				continue
			}

			log.Println("Event:", e)

			if cmd != nil {
				err := cmd.Process.Kill()
				if err != nil {
					log.Println(err)
				}
			}
			cmd = exec.Command(cmds[0], cmds[1:]...)
			cmd.Stdout = os.Stdout
			cmd.Stderr = os.Stderr
			cmd.Start()
		case err := <-watcher.Error:
			log.Println("Error:", err)
		}
	}

	watcher.Close()
}
<|endoftext|>


<commit_before>package main

import (
	"net/http"
	"sync"
	"log"
	"os"
)

var mut = sync.Mutex{}

func main() {

	fileserver := http.FileServer(http.Dir("./"))
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Header().Add("cache-control", "no-cache")
		fileserver.ServeHTTP(w, r)
	})
  log.Fatal(http.ListenAndServe(":"+os.Args[1], nil))
}
<commit_msg>made 8000 default, like python<commit_after>package main

import (
	"net/http"
	"sync"
	"log"
	"os"
)

var mut = sync.Mutex{}

func main() {

	fileserver := http.FileServer(http.Dir("./"))
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Header().Add("cache-control", "no-cache")
		fileserver.ServeHTTP(w, r)
	})

  port := ":8000"
  if len(os.Args) > 1 {
    port = ":"+os.Args[1]
  }
  log.Fatal(http.ListenAndServe(port, nil))
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"os"

	"github.com/joushou/sshmux"

	"golang.org/x/crypto/ssh"
)

func usage() {
	fmt.Printf("Usage: \n")
	fmt.Printf("   %s conf\n", os.Args[0])
}

type Host struct {
	Address string   `json:"address"`
	Users   []string `json:"users"`
	NoAuth  bool     `json:"noAuth"`
}

type Conf struct {
	Address  string `json:"address"`
	HostKey  string `json:"hostkey"`
	AuthKeys string `json:"authkeys"`
	Hosts    []Host `json:"hosts"`
}

func parseConf(filename string) (*Conf, error) {
	f, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}

	c := &Conf{}
	err = json.Unmarshal(f, c)
	if err != nil {
		return nil, err
	}

	return c, nil
}

func parseAuthFile(filename string) ([]*sshmux.User, error) {
	var users []*sshmux.User

	authFile, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}

	// Parse authfile as authorized_key

	for len(authFile) > 0 {
		var (
			pk      ssh.PublicKey
			comment string
		)

		pk, comment, _, authFile, err = ssh.ParseAuthorizedKey(authFile)
		if err != nil {
			return nil, err
		}

		u := &sshmux.User{
			PublicKey: pk,
			Name:      comment,
		}

		users = append(users, u)
	}

	return users, nil
}

func main() {
	// Config
	if len(os.Args) != 2 {
		usage()
		return
	}

	conf := os.Args[1]

	c, err := parseConf(conf)
	if err != nil {
		panic(err)
	}

	hostPrivateKey, err := ioutil.ReadFile(c.HostKey)
	if err != nil {
		panic(err)
	}

	hostSigner, err := ssh.ParsePrivateKey(hostPrivateKey)
	if err != nil {
		panic(err)
	}

	users, err := parseAuthFile(c.AuthKeys)
	if err != nil {
		panic(err)
	}

	hasDefaults := false
	for _, h := range c.Hosts {
		if h.NoAuth {
			hasDefaults = true
			break
		}
	}

	// sshmux setup
	auth := func(c ssh.ConnMetadata, key ssh.PublicKey) (*sshmux.User, error) {
		t := key.Type()
		k := key.Marshal()
		for i := range users {
			candidate := users[i].PublicKey
			if t == candidate.Type() && bytes.Compare(k, candidate.Marshal()) == 0 {
				return users[i], nil
			}
		}

		if hasDefaults {
			return nil, nil
		}

		log.Printf("%s: access denied (username: %s)", c.RemoteAddr(), c.User())
		return nil, errors.New("access denied")
	}

	setup := func(session *sshmux.Session) error {
		if session.User != nil {
			log.Printf("%s: %s authorized (username: %s)", session.Conn.RemoteAddr(), session.User.Name, session.Conn.User())
		} else {
			log.Printf("%s: unknown user authorized (username: %s)", session.Conn.RemoteAddr(), session.Conn.User())
		}

	outer:
		for _, h := range c.Hosts {
			if h.NoAuth {
				session.Remotes = append(session.Remotes, h.Address)
				continue outer
			}

			for _, u := range h.Users {
				if u == session.User.Name {
					session.Remotes = append(session.Remotes, h.Address)
					continue outer
				}
			}
		}
		return nil
	}

	server := sshmux.New(hostSigner, auth, setup)
	server.Selected = func(session *sshmux.Session, remote string) error {
		log.Printf("%s: %s connecting to %s", session.Conn.RemoteAddr(), session.User.Name, remote)
		return nil
	}

	// Set up listener
	l, err := net.Listen("tcp", c.Address)
	if err != nil {
		panic(err)
	}

	server.Serve(l)
}
<commit_msg>Fix logging when user is unknown<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"os"

	"github.com/joushou/sshmux"

	"golang.org/x/crypto/ssh"
)

func usage() {
	fmt.Printf("Usage: \n")
	fmt.Printf("   %s conf\n", os.Args[0])
}

type Host struct {
	Address string   `json:"address"`
	Users   []string `json:"users"`
	NoAuth  bool     `json:"noAuth"`
}

type Conf struct {
	Address  string `json:"address"`
	HostKey  string `json:"hostkey"`
	AuthKeys string `json:"authkeys"`
	Hosts    []Host `json:"hosts"`
}

func parseConf(filename string) (*Conf, error) {
	f, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}

	c := &Conf{}
	err = json.Unmarshal(f, c)
	if err != nil {
		return nil, err
	}

	return c, nil
}

func parseAuthFile(filename string) ([]*sshmux.User, error) {
	var users []*sshmux.User

	authFile, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}

	// Parse authfile as authorized_key

	for len(authFile) > 0 {
		var (
			pk      ssh.PublicKey
			comment string
		)

		pk, comment, _, authFile, err = ssh.ParseAuthorizedKey(authFile)
		if err != nil {
			return nil, err
		}

		u := &sshmux.User{
			PublicKey: pk,
			Name:      comment,
		}

		users = append(users, u)
	}

	return users, nil
}

func main() {
	// Config
	if len(os.Args) != 2 {
		usage()
		return
	}

	conf := os.Args[1]

	c, err := parseConf(conf)
	if err != nil {
		panic(err)
	}

	hostPrivateKey, err := ioutil.ReadFile(c.HostKey)
	if err != nil {
		panic(err)
	}

	hostSigner, err := ssh.ParsePrivateKey(hostPrivateKey)
	if err != nil {
		panic(err)
	}

	users, err := parseAuthFile(c.AuthKeys)
	if err != nil {
		panic(err)
	}

	hasDefaults := false
	for _, h := range c.Hosts {
		if h.NoAuth {
			hasDefaults = true
			break
		}
	}

	// sshmux setup
	auth := func(c ssh.ConnMetadata, key ssh.PublicKey) (*sshmux.User, error) {
		t := key.Type()
		k := key.Marshal()
		for i := range users {
			candidate := users[i].PublicKey
			if t == candidate.Type() && bytes.Compare(k, candidate.Marshal()) == 0 {
				return users[i], nil
			}
		}

		if hasDefaults {
			return nil, nil
		}

		log.Printf("%s: access denied (username: %s)", c.RemoteAddr(), c.User())
		return nil, errors.New("access denied")
	}

	setup := func(session *sshmux.Session) error {
		var username string
		if session.User != nil {
			username = session.User.Name
		} else {
			username = "unknown user"
		}
		log.Printf("%s: %s authorized (username: %s)", session.Conn.RemoteAddr(), username, session.Conn.User())

	outer:
		for _, h := range c.Hosts {
			if h.NoAuth {
				session.Remotes = append(session.Remotes, h.Address)
				continue outer
			}

			for _, u := range h.Users {
				if u == session.User.Name {
					session.Remotes = append(session.Remotes, h.Address)
					continue outer
				}
			}
		}
		return nil
	}

	server := sshmux.New(hostSigner, auth, setup)
	server.Selected = func(session *sshmux.Session, remote string) error {
		var username string
		if session.User != nil {
			username = session.User.Name
		} else {
			username = "unknown user"
		}
		log.Printf("%s: %s connecting to %s", session.Conn.RemoteAddr(), username, remote)
		return nil
	}

	// Set up listener
	l, err := net.Listen("tcp", c.Address)
	if err != nil {
		panic(err)
	}

	server.Serve(l)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"encoding/binary"
	"flag"
	"io"
	"log"
	"math"
	"os"
	"path/filepath"
	"runtime"
	"sync"
	"syscall"
)

type block struct {
	filePath    string
	numBytes    uint16
	buffer      []byte
	startOfFile bool
	endOfFile   bool
	uid         int
	gid         int
	mode        os.FileMode
}

var blockSize uint16
var verbose bool
var logger *log.Logger

const dataBlockFlag byte = 1 << 0
const startOfFileFlag byte = 1 << 1
const endOfFileFlag byte = 1 << 2

func main() {
	extract := flag.Bool("x", false, "extract archive")
	create := flag.Bool("c", false, "create archive")
	inputFileName := flag.String("i", "", "input file for extraction; defaults to stdin (-x only)")
	outputFileName := flag.String("o", "", "output file for creation; defaults to stdout (-c only)")
	requestedBlockSize := flag.Uint("block-size", 4096, "internal block-size (-c only)")
	dirReaderCount := flag.Int("dir-readers", 16, "number of simultaneous directory readers (-c only)")
	fileReaderCount := flag.Int("file-readers", 16, "number of simultaneous file readers (-c only)")
	directoryScanQueueSize := flag.Int("queue-dir", 128, "queue size for scanning directories (-c only)")
	fileReadQueueSize := flag.Int("queue-read", 128, "queue size for reading files (-c only)")
	fileWriteQueueSize := flag.Int("queue-write", 128, "queue size for archive write (-c only); increasing can cause increased memory usage")
	multiCpu := flag.Int("multicpu", 1, "maximum number of CPUs that can be executing simultaneously")
	exclude := flag.String("exclude", "", "file patterns to exclude (eg. core.*); can be path list separated (eg. : in Linux) for multiple excludes (-c only)")
	flag.BoolVar(&verbose, "v", false, "verbose output on stderr")
	flag.Parse()

	runtime.GOMAXPROCS(*multiCpu)

	logger = log.New(os.Stderr, "", 0)

	if *requestedBlockSize > math.MaxUint16 {
		logger.Fatalln("block-size must be less than or equal to", math.MaxUint16)
	}
	blockSize = uint16(*requestedBlockSize)

	if *extract {
		var inputFile *os.File
		if *inputFileName != "" {
			file, err := os.Open(*inputFileName)
			if err != nil {
				logger.Fatalln("Error opening input file:", err.Error())
			}
			inputFile = file
		} else {
			inputFile = os.Stdin
		}

		bufferedInputFile := bufio.NewReader(inputFile)
		archiveReader(bufferedInputFile)
		inputFile.Close()

	} else if *create {
		if flag.NArg() == 0 {
			logger.Fatalln("Directories to archive must be specified")
		}

		var directoryScanQueue = make(chan string, *directoryScanQueueSize)
		var fileReadQueue = make(chan string, *fileReadQueueSize)
		var fileWriteQueue = make(chan block, *fileWriteQueueSize)
		var workInProgress sync.WaitGroup
		var excludes = filepath.SplitList(*exclude)

		var outputFile *os.File
		if *outputFileName != "" {
			file, err := os.Create(*outputFileName)
			if err != nil {
				logger.Fatalln("Error creating output file:", err.Error())
			}
			outputFile = file
		} else {
			outputFile = os.Stdout
		}

		bufferedOutputFile := bufio.NewWriter(outputFile)
		go archiveWriter(bufferedOutputFile, fileWriteQueue, &workInProgress)
		for i := 0; i < *dirReaderCount; i++ {
			go directoryScanner(directoryScanQueue, fileReadQueue, excludes, &workInProgress)
		}
		for i := 0; i < *fileReaderCount; i++ {
			go fileReader(fileReadQueue, fileWriteQueue, &workInProgress)
		}

		for i := 0; i < flag.NArg(); i++ {
			workInProgress.Add(1)
			directoryScanQueue <- flag.Arg(i)
		}

		workInProgress.Wait()
		close(directoryScanQueue)
		close(fileReadQueue)
		close(fileWriteQueue)
		bufferedOutputFile.Flush()
		outputFile.Close()
	} else {
		logger.Fatalln("extract (-x) or create (-c) flag must be provided")
	}
}

func directoryScanner(directoryScanQueue chan string, fileReadQueue chan string, excludePatterns []string, workInProgress *sync.WaitGroup) {
	for directoryPath := range directoryScanQueue {
		if verbose {
			logger.Println(directoryPath)
		}

		directory, err := os.Open(directoryPath)
		if err == nil {
			for fileName := range readdirnames(int(directory.Fd())) {
				workInProgress.Add(1)
				filePath := filepath.Join(directoryPath, fileName)

				excludeFile := false
				for _, excludePattern := range excludePatterns {
					match, err := filepath.Match(excludePattern, filePath)
					if err == nil && match {
						excludeFile = true
						break
					}
				}
				if excludeFile {
					logger.Println("skipping excluded file", filePath)
					workInProgress.Done()
					continue
				}

				fileInfo, err := os.Lstat(filePath)
				if err != nil {
					logger.Println("unable to lstat file", err.Error())
					workInProgress.Done()
					continue
				} else if (fileInfo.Mode() & os.ModeSymlink) != 0 {
					logger.Println("skipping symbolic link", filePath)
					workInProgress.Done()
					continue
				}

				if fileInfo.IsDir() {
					directoryScanQueue <- filePath
				} else {
					fileReadQueue <- filePath
				}
			}

			directory.Close()
		} else {
			logger.Println("directory read error:", err.Error())
		}

		workInProgress.Done()
	}
}

func fileReader(fileReadQueue <-chan string, fileWriterQueue chan block, workInProgress *sync.WaitGroup) {
	for filePath := range fileReadQueue {
		if verbose {
			logger.Println(filePath)
		}

		file, err := os.Open(filePath)
		if err == nil {

			var uid int = 0
			var gid int = 0
			var mode os.FileMode = 0
			fi, err := file.Stat()
			if err != nil {
				logger.Println("file stat error; uid/gid/mode will be incorrect:", err.Error())
			} else {
				mode = fi.Mode()
				stat_t := fi.Sys().(*syscall.Stat_t)
				if stat_t != nil {
					uid = int(stat_t.Uid)
					gid = int(stat_t.Gid)
				} else {
					logger.Println("unable to find file uid/gid")
				}
			}

			workInProgress.Add(1)
			fileWriterQueue <- block{filePath, 0, nil, true, false, uid, gid, mode}

			bufferedFile := bufio.NewReader(file)

			for {
				buffer := make([]byte, blockSize)
				bytesRead, err := bufferedFile.Read(buffer)
				if err == io.EOF {
					break
				} else if err != nil {
					logger.Println("file read error; file contents will be incomplete:", err.Error())
					break
				}

				workInProgress.Add(1)
				fileWriterQueue <- block{filePath, uint16(bytesRead), buffer, false, false, 0, 0, 0}
			}

			workInProgress.Add(1)
			fileWriterQueue <- block{filePath, 0, nil, false, true, 0, 0, 0}

			file.Close()
		} else {
			logger.Println("file open error:", err.Error())
		}

		workInProgress.Done()
	}
}

func archiveWriter(output io.Writer, fileWriterQueue <-chan block, workInProgress *sync.WaitGroup) {
	flags := make([]byte, 1)

	for block := range fileWriterQueue {
		filePath := []byte(block.filePath)
		err := binary.Write(output, binary.BigEndian, uint16(len(filePath)))
		if err != nil {
			logger.Fatalln("Archive write error:", err.Error())
		}
		_, err = output.Write(filePath)
		if err != nil {
			logger.Fatalln("Archive write error:", err.Error())
		}

		if block.startOfFile {
			flags[0] = startOfFileFlag
			_, err = output.Write(flags)
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}
			err = binary.Write(output, binary.BigEndian, uint32(block.uid))
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}
			err = binary.Write(output, binary.BigEndian, uint32(block.gid))
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}
			err = binary.Write(output, binary.BigEndian, block.mode)
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}
		} else if block.endOfFile {
			flags[0] = endOfFileFlag
			_, err = output.Write(flags)
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}
		} else {
			flags[0] = dataBlockFlag
			_, err = output.Write(flags)
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}

			err = binary.Write(output, binary.BigEndian, uint16(block.numBytes))
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}

			_, err = output.Write(block.buffer[:block.numBytes])
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}
		}

		workInProgress.Done()
	}
}

func archiveReader(file io.Reader) {
	var workInProgress sync.WaitGroup
	fileOutputChan := make(map[string]chan block)

	for {
		var pathSize uint16
		err := binary.Read(file, binary.BigEndian, &pathSize)
		if err == io.EOF {
			break
		} else if err != nil {
			logger.Fatalln("Archive read error:", err.Error())
		}

		buf := make([]byte, pathSize)
		_, err = io.ReadFull(file, buf)
		if err != nil {
			logger.Fatalln("Archive read error:", err.Error())
		}
		filePath := string(buf)

		flag := make([]byte, 1)
		_, err = io.ReadFull(file, flag)
		if err != nil {
			logger.Fatalln("Archive read error:", err.Error())
		}

		if flag[0] == startOfFileFlag {
			var uid uint32
			var gid uint32
			var mode os.FileMode

			err = binary.Read(file, binary.BigEndian, &uid)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}

			err = binary.Read(file, binary.BigEndian, &gid)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}

			err = binary.Read(file, binary.BigEndian, &mode)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}

			c := make(chan block, 1)
			fileOutputChan[filePath] = c
			workInProgress.Add(1)
			go writeFile(c, &workInProgress)
			c <- block{filePath, 0, nil, true, false, int(uid), int(gid), mode}
		} else if flag[0] == endOfFileFlag {
			c := fileOutputChan[filePath]
			c <- block{filePath, 0, nil, false, true, 0, 0, 0}
			close(c)
			delete(fileOutputChan, filePath)
		} else if flag[0] == dataBlockFlag {
			var blockSize uint16
			err = binary.Read(file, binary.BigEndian, &blockSize)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}

			blockData := make([]byte, blockSize)
			_, err = io.ReadFull(file, blockData)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}

			c := fileOutputChan[filePath]
			c <- block{filePath, blockSize, blockData, false, false, 0, 0, 0}
		} else {
			logger.Fatalln("Archive error: unrecognized block flag", flag[0])
		}
	}

	workInProgress.Wait()
}

func writeFile(blockSource chan block, workInProgress *sync.WaitGroup) {
	var file *os.File = nil
	var bufferedFile *bufio.Writer
	for block := range blockSource {
		if block.startOfFile {
			if verbose {
				logger.Println(block.filePath)
			}

			dir, _ := filepath.Split(block.filePath)
			err := os.MkdirAll(dir, os.ModeDir|0755)
			if err != nil {
				logger.Fatalln("Directory create error:", err.Error())
			}

			tmp, err := os.Create(block.filePath)
			if err != nil {
				logger.Fatalln("File create error:", err.Error())
			}
			file = tmp
			bufferedFile = bufio.NewWriter(file)

			err = file.Chown(block.uid, block.gid)
			if err != nil {
				logger.Println("Unable to chown file to", block.uid, "/", block.gid, ":", err.Error())
			}
			err = file.Chmod(block.mode)
			if err != nil {
				logger.Println("Unable to chmod file to", block.mode, ":", err.Error())
			}
		} else if block.endOfFile {
			bufferedFile.Flush()
			file.Close()
			file = nil
		} else {
			_, err := bufferedFile.Write(block.buffer[:block.numBytes])
			if err != nil {
				logger.Fatalln("File write error:", err.Error())
			}
		}
	}
	workInProgress.Done()
}

// Copy of os.Readdirnames for UNIX systems, but modified to return results
// as found through a channel rather than in one large array.
func readdirnames(fd int) chan string {
	retval := make(chan string)
	go func(fd int) {
		var buf []byte = make([]byte, blockSize)
		var nbuf int
		var bufp int

		for {
			// Refill the buffer if necessary
			if bufp >= nbuf {
				bufp = 0
				var errno error
				nbuf, errno = syscall.ReadDirent(fd, buf)
				if errno != nil {
					err := os.NewSyscallError("readdirent", errno)
					logger.Println("error reading directory:", err.Error())
					break
				}
				if nbuf <= 0 {
					break // EOF
				}
			}

			// Drain the buffer
			var nb, nc int
			names := make([]string, 0, 100)
			nb, nc, names = syscall.ParseDirent(buf[bufp:nbuf], -1, names)
			bufp += nb

			for i := 0; i < nc; i++ {
				retval <- names[i]
			}
		}

		close(retval)
	}(fd)
	return retval
}
<commit_msg>switch block bools to a blockType enum<commit_after>package main

import (
	"bufio"
	"encoding/binary"
	"flag"
	"io"
	"log"
	"math"
	"os"
	"path/filepath"
	"runtime"
	"sync"
	"syscall"
)

type blockType int
const (
	blockTypeData = iota
	blockTypeStartOfFile
	blockTypeEndOfFile
	blockTypeDirectory
)

type block struct {
	filePath    string
	numBytes    uint16
	buffer      []byte
	blockType   blockType
	uid         int
	gid         int
	mode        os.FileMode
}

var blockSize uint16
var verbose bool
var logger *log.Logger

const dataBlockFlag byte = 1 << 0
const startOfFileFlag byte = 1 << 1
const endOfFileFlag byte = 1 << 2

func main() {
	extract := flag.Bool("x", false, "extract archive")
	create := flag.Bool("c", false, "create archive")
	inputFileName := flag.String("i", "", "input file for extraction; defaults to stdin (-x only)")
	outputFileName := flag.String("o", "", "output file for creation; defaults to stdout (-c only)")
	requestedBlockSize := flag.Uint("block-size", 4096, "internal block-size (-c only)")
	dirReaderCount := flag.Int("dir-readers", 16, "number of simultaneous directory readers (-c only)")
	fileReaderCount := flag.Int("file-readers", 16, "number of simultaneous file readers (-c only)")
	directoryScanQueueSize := flag.Int("queue-dir", 128, "queue size for scanning directories (-c only)")
	fileReadQueueSize := flag.Int("queue-read", 128, "queue size for reading files (-c only)")
	fileWriteQueueSize := flag.Int("queue-write", 128, "queue size for archive write (-c only); increasing can cause increased memory usage")
	multiCpu := flag.Int("multicpu", 1, "maximum number of CPUs that can be executing simultaneously")
	exclude := flag.String("exclude", "", "file patterns to exclude (eg. core.*); can be path list separated (eg. : in Linux) for multiple excludes (-c only)")
	flag.BoolVar(&verbose, "v", false, "verbose output on stderr")
	flag.Parse()

	runtime.GOMAXPROCS(*multiCpu)

	logger = log.New(os.Stderr, "", 0)

	if *requestedBlockSize > math.MaxUint16 {
		logger.Fatalln("block-size must be less than or equal to", math.MaxUint16)
	}
	blockSize = uint16(*requestedBlockSize)

	if *extract {
		var inputFile *os.File
		if *inputFileName != "" {
			file, err := os.Open(*inputFileName)
			if err != nil {
				logger.Fatalln("Error opening input file:", err.Error())
			}
			inputFile = file
		} else {
			inputFile = os.Stdin
		}

		bufferedInputFile := bufio.NewReader(inputFile)
		archiveReader(bufferedInputFile)
		inputFile.Close()

	} else if *create {
		if flag.NArg() == 0 {
			logger.Fatalln("Directories to archive must be specified")
		}

		var directoryScanQueue = make(chan string, *directoryScanQueueSize)
		var fileReadQueue = make(chan string, *fileReadQueueSize)
		var fileWriteQueue = make(chan block, *fileWriteQueueSize)
		var workInProgress sync.WaitGroup
		var excludes = filepath.SplitList(*exclude)

		var outputFile *os.File
		if *outputFileName != "" {
			file, err := os.Create(*outputFileName)
			if err != nil {
				logger.Fatalln("Error creating output file:", err.Error())
			}
			outputFile = file
		} else {
			outputFile = os.Stdout
		}

		bufferedOutputFile := bufio.NewWriter(outputFile)
		go archiveWriter(bufferedOutputFile, fileWriteQueue, &workInProgress)
		for i := 0; i < *dirReaderCount; i++ {
			go directoryScanner(directoryScanQueue, fileReadQueue, excludes, &workInProgress)
		}
		for i := 0; i < *fileReaderCount; i++ {
			go fileReader(fileReadQueue, fileWriteQueue, &workInProgress)
		}

		for i := 0; i < flag.NArg(); i++ {
			workInProgress.Add(1)
			directoryScanQueue <- flag.Arg(i)
		}

		workInProgress.Wait()
		close(directoryScanQueue)
		close(fileReadQueue)
		close(fileWriteQueue)
		bufferedOutputFile.Flush()
		outputFile.Close()
	} else {
		logger.Fatalln("extract (-x) or create (-c) flag must be provided")
	}
}

func directoryScanner(directoryScanQueue chan string, fileReadQueue chan string, excludePatterns []string, workInProgress *sync.WaitGroup) {
	for directoryPath := range directoryScanQueue {
		if verbose {
			logger.Println(directoryPath)
		}

		directory, err := os.Open(directoryPath)
		if err == nil {
			for fileName := range readdirnames(int(directory.Fd())) {
				workInProgress.Add(1)
				filePath := filepath.Join(directoryPath, fileName)

				excludeFile := false
				for _, excludePattern := range excludePatterns {
					match, err := filepath.Match(excludePattern, filePath)
					if err == nil && match {
						excludeFile = true
						break
					}
				}
				if excludeFile {
					logger.Println("skipping excluded file", filePath)
					workInProgress.Done()
					continue
				}

				fileInfo, err := os.Lstat(filePath)
				if err != nil {
					logger.Println("unable to lstat file", err.Error())
					workInProgress.Done()
					continue
				} else if (fileInfo.Mode() & os.ModeSymlink) != 0 {
					logger.Println("skipping symbolic link", filePath)
					workInProgress.Done()
					continue
				}

				if fileInfo.IsDir() {
					directoryScanQueue <- filePath
				} else {
					fileReadQueue <- filePath
				}
			}

			directory.Close()
		} else {
			logger.Println("directory read error:", err.Error())
		}

		workInProgress.Done()
	}
}

func fileReader(fileReadQueue <-chan string, fileWriterQueue chan block, workInProgress *sync.WaitGroup) {
	for filePath := range fileReadQueue {
		if verbose {
			logger.Println(filePath)
		}

		file, err := os.Open(filePath)
		if err == nil {

			var uid int = 0
			var gid int = 0
			var mode os.FileMode = 0
			fi, err := file.Stat()
			if err != nil {
				logger.Println("file stat error; uid/gid/mode will be incorrect:", err.Error())
			} else {
				mode = fi.Mode()
				stat_t := fi.Sys().(*syscall.Stat_t)
				if stat_t != nil {
					uid = int(stat_t.Uid)
					gid = int(stat_t.Gid)
				} else {
					logger.Println("unable to find file uid/gid")
				}
			}

			workInProgress.Add(1)
			fileWriterQueue <- block{filePath, 0, nil, blockTypeStartOfFile, uid, gid, mode}

			bufferedFile := bufio.NewReader(file)

			for {
				buffer := make([]byte, blockSize)
				bytesRead, err := bufferedFile.Read(buffer)
				if err == io.EOF {
					break
				} else if err != nil {
					logger.Println("file read error; file contents will be incomplete:", err.Error())
					break
				}

				workInProgress.Add(1)
				fileWriterQueue <- block{filePath, uint16(bytesRead), buffer, blockTypeData, 0, 0, 0}
			}

			workInProgress.Add(1)
			fileWriterQueue <- block{filePath, 0, nil, blockTypeEndOfFile, 0, 0, 0}

			file.Close()
		} else {
			logger.Println("file open error:", err.Error())
		}

		workInProgress.Done()
	}
}

func archiveWriter(output io.Writer, fileWriterQueue <-chan block, workInProgress *sync.WaitGroup) {
	flags := make([]byte, 1)

	for block := range fileWriterQueue {
		filePath := []byte(block.filePath)
		err := binary.Write(output, binary.BigEndian, uint16(len(filePath)))
		if err != nil {
			logger.Fatalln("Archive write error:", err.Error())
		}
		_, err = output.Write(filePath)
		if err != nil {
			logger.Fatalln("Archive write error:", err.Error())
		}

		if block.blockType == blockTypeStartOfFile {
			flags[0] = startOfFileFlag
			_, err = output.Write(flags)
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}
			err = binary.Write(output, binary.BigEndian, uint32(block.uid))
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}
			err = binary.Write(output, binary.BigEndian, uint32(block.gid))
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}
			err = binary.Write(output, binary.BigEndian, block.mode)
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}
		} else if block.blockType == blockTypeEndOfFile {
			flags[0] = endOfFileFlag
			_, err = output.Write(flags)
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}
		} else if block.blockType == blockTypeData {
			flags[0] = dataBlockFlag
			_, err = output.Write(flags)
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}

			err = binary.Write(output, binary.BigEndian, uint16(block.numBytes))
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}

			_, err = output.Write(block.buffer[:block.numBytes])
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}
		} else {
			logger.Fatalln("Unexpected block type")
		}

		workInProgress.Done()
	}
}

func archiveReader(file io.Reader) {
	var workInProgress sync.WaitGroup
	fileOutputChan := make(map[string]chan block)

	for {
		var pathSize uint16
		err := binary.Read(file, binary.BigEndian, &pathSize)
		if err == io.EOF {
			break
		} else if err != nil {
			logger.Fatalln("Archive read error:", err.Error())
		}

		buf := make([]byte, pathSize)
		_, err = io.ReadFull(file, buf)
		if err != nil {
			logger.Fatalln("Archive read error:", err.Error())
		}
		filePath := string(buf)

		flag := make([]byte, 1)
		_, err = io.ReadFull(file, flag)
		if err != nil {
			logger.Fatalln("Archive read error:", err.Error())
		}

		if flag[0] == startOfFileFlag {
			var uid uint32
			var gid uint32
			var mode os.FileMode

			err = binary.Read(file, binary.BigEndian, &uid)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}

			err = binary.Read(file, binary.BigEndian, &gid)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}

			err = binary.Read(file, binary.BigEndian, &mode)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}

			c := make(chan block, 1)
			fileOutputChan[filePath] = c
			workInProgress.Add(1)
			go writeFile(c, &workInProgress)
			c <- block{filePath, 0, nil, blockTypeStartOfFile, int(uid), int(gid), mode}
		} else if flag[0] == endOfFileFlag {
			c := fileOutputChan[filePath]
			c <- block{filePath, 0, nil, blockTypeEndOfFile, 0, 0, 0}
			close(c)
			delete(fileOutputChan, filePath)
		} else if flag[0] == dataBlockFlag {
			var blockSize uint16
			err = binary.Read(file, binary.BigEndian, &blockSize)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}

			blockData := make([]byte, blockSize)
			_, err = io.ReadFull(file, blockData)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}

			c := fileOutputChan[filePath]
			c <- block{filePath, blockSize, blockData, blockTypeData, 0, 0, 0}
		} else {
			logger.Fatalln("Archive error: unrecognized block flag", flag[0])
		}
	}

	workInProgress.Wait()
}

func writeFile(blockSource chan block, workInProgress *sync.WaitGroup) {
	var file *os.File = nil
	var bufferedFile *bufio.Writer
	for block := range blockSource {
		if block.blockType == blockTypeStartOfFile {
			if verbose {
				logger.Println(block.filePath)
			}

			dir, _ := filepath.Split(block.filePath)
			err := os.MkdirAll(dir, os.ModeDir|0755)
			if err != nil {
				logger.Fatalln("Directory create error:", err.Error())
			}

			tmp, err := os.Create(block.filePath)
			if err != nil {
				logger.Fatalln("File create error:", err.Error())
			}
			file = tmp
			bufferedFile = bufio.NewWriter(file)

			err = file.Chown(block.uid, block.gid)
			if err != nil {
				logger.Println("Unable to chown file to", block.uid, "/", block.gid, ":", err.Error())
			}
			err = file.Chmod(block.mode)
			if err != nil {
				logger.Println("Unable to chmod file to", block.mode, ":", err.Error())
			}
		} else if block.blockType == blockTypeEndOfFile {
			bufferedFile.Flush()
			file.Close()
			file = nil
		} else {
			_, err := bufferedFile.Write(block.buffer[:block.numBytes])
			if err != nil {
				logger.Fatalln("File write error:", err.Error())
			}
		}
	}
	workInProgress.Done()
}

// Copy of os.Readdirnames for UNIX systems, but modified to return results
// as found through a channel rather than in one large array.
func readdirnames(fd int) chan string {
	retval := make(chan string)
	go func(fd int) {
		var buf []byte = make([]byte, blockSize)
		var nbuf int
		var bufp int

		for {
			// Refill the buffer if necessary
			if bufp >= nbuf {
				bufp = 0
				var errno error
				nbuf, errno = syscall.ReadDirent(fd, buf)
				if errno != nil {
					err := os.NewSyscallError("readdirent", errno)
					logger.Println("error reading directory:", err.Error())
					break
				}
				if nbuf <= 0 {
					break // EOF
				}
			}

			// Drain the buffer
			var nb, nc int
			names := make([]string, 0, 100)
			nb, nc, names = syscall.ParseDirent(buf[bufp:nbuf], -1, names)
			bufp += nb

			for i := 0; i < nc; i++ {
				retval <- names[i]
			}
		}

		close(retval)
	}(fd)
	return retval
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"math/rand"
	"sync"
	"time"
)

var wg sync.WaitGroup
var wga sync.WaitGroup

type alarm struct {
	bool
	*sync.Mutex
}

func (a *alarm) Set(value bool) {
	a.Lock()
	a.bool = value
	a.Unlock()
}

type person struct {
	Name    string
	AllDone bool
	*sync.Mutex
}

func (p *person) setAllDone(value bool) {
	defer p.Unlock()
	p.Lock()
	p.AllDone = value
}

type people []person

var alrm = alarm{false, &sync.Mutex{}}
var hsld = []person{}

func main() {
	fmt.Println("Let's go for walk!")
	Bob := person{"Bob", false, &sync.Mutex{}}
	Alice := person{"Alice", false, &sync.Mutex{}}
	hsld = []person{Bob, Alice}

	for k := range hsld {
		wg.Add(1)
		go hsld[k].dotask("getting ready", 1, 3, false)
	}
	wg.Wait()

	wga.Add(1)
	go setAlarm(5)

	for k := range hsld {
		wg.Add(1)
		go hsld[k].dotask("putting on shoes", 1, 3, true)
	}
	wg.Wait()
	alrm.Lock()
	if alrm.bool {
		fmt.Println("Crap!  The alarm is already set.")
	} else {
		fmt.Println("Exiting and locking door.")
	}
	alrm.Unlock()
	wga.Wait()
}

func setAlarm(delay int) {
	fmt.Println("Arming alarm.")
	fmt.Println("Alarm is counting down.")
	time.Sleep(time.Duration(delay) * time.Second)
	alrm.Set(true)
	fmt.Println("Alarm armed.")
	for _, v := range hsld {
		v.Lock()
		if v.AllDone == false {
			fmt.Printf("Alarm set before %v was ready.\n", v.Name)
		}
		v.Unlock()
	}
	defer func() {
		wga.Done()
	}()
}

func (p *person) dotask(task string, min int, max int, setdone bool) {
	defer wg.Done()
	s := random(min, max)
	fmt.Println(p.Name, "started", task)
	time.Sleep(time.Duration(s) * time.Second)
	fmt.Println(p.Name, "spent", s, "seconds", task)
	if setdone {
		p.setAllDone(true)
	}
}

func random(min, max int) int {
	rand.Seed(time.Now().Unix())
	time.Sleep(10 * time.Microsecond)
	return rand.Intn(max-min) + min
}
<commit_msg>gotta leave as key ref<commit_after>package main

import (
	"fmt"
	"math/rand"
	"sync"
	"time"
)

var wg sync.WaitGroup
var wga sync.WaitGroup

type alarm struct {
	bool
	*sync.Mutex
}

func (a *alarm) Set(value bool) {
	a.Lock()
	a.bool = value
	a.Unlock()
}

type person struct {
	Name    string
	AllDone bool
	*sync.Mutex
}

func (p *person) setAllDone(value bool) {
	defer p.Unlock()
	p.Lock()
	p.AllDone = value
}

type people []person

var alrm = alarm{false, &sync.Mutex{}}
var hsld = []person{}

func main() {
	fmt.Println("Let's go for walk!")
	Bob := person{"Bob", false, &sync.Mutex{}}
	Alice := person{"Alice", false, &sync.Mutex{}}
	hsld = []person{Bob, Alice}

	for k := range hsld {
		wg.Add(1)
		go hsld[k].dotask("getting ready", 1, 3, false)
	}

	wg.Wait()

	wga.Add(1)
	go setAlarm(5)

	for k := range hsld {
		wg.Add(1)
		go hsld[k].dotask("putting on shoes", 1, 3, true)
	}
	wg.Wait()
	alrm.Lock()
	if alrm.bool {
		fmt.Println("Crap!  The alarm is already set.")
	} else {
		fmt.Println("Exiting and locking door.")
	}
	alrm.Unlock()
	wga.Wait()
}

func setAlarm(delay int) {
	fmt.Println("Arming alarm.")
	fmt.Println("Alarm is counting down.")
	time.Sleep(time.Duration(delay) * time.Second)
	alrm.Set(true)
	fmt.Println("Alarm armed.")
	for _, v := range hsld {
		v.Lock()
		if v.AllDone == false {
			fmt.Printf("Alarm set before %v was ready.\n", v.Name)
		}
		v.Unlock()
	}
	defer func() {
		wga.Done()
	}()
}

func (p *person) dotask(task string, min int, max int, setdone bool) {
	defer wg.Done()
	s := random(min, max)
	fmt.Println(p.Name, "started", task)
	time.Sleep(time.Duration(s) * time.Second)
	fmt.Println(p.Name, "spent", s, "seconds", task)
	if setdone {
		p.setAllDone(true)
	}
}

func random(min, max int) int {
	rand.Seed(time.Now().Unix())
	time.Sleep(10 * time.Microsecond)
	return rand.Intn(max-min) + min
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"

	"github.com/veandco/go-sdl2/sdl"
)

type EventRegion struct {
	Rect *sdl.Rect
}

func NewEventRegion(x, y, width, height int32) *EventRegion {
	er := EventRegion{Rect: &sdl.Rect{X: x, Y: y, W: width, H: height}}
	return &er
}

func (er *EventRegion) SDLRect() *sdl.Rect {
	return er.Rect
}

func (er *EventRegion) HitWhat(x int32, y int32) *EventRegion {
	if x < er.Rect.X || x > er.Rect.X+er.Rect.W {
		return nil
	}
	if y < er.Rect.Y || y > er.Rect.Y+er.Rect.H {
		return nil
	}
	return er
}

type RegionList []*EventRegion

func (rl RegionList) HitWhat(x int32, y int32) *EventRegion {
	for _, region := range rl {
		if what := region.HitWhat(x, y); what != nil {
			return what
		}
	}
	return nil
}

const CARD_WIDTH = 100
const CARD_HEIGHT = 180

const (
	INTERACTION_STATE_DEFAULT  = 0
	INTERACTION_STATE_DRAGGING = 1
)

func main() {
	interactionState := INTERACTION_STATE_DEFAULT
	fmt.Println(interactionState)
	var draggingWhat *EventRegion
	res := sdl.Init(sdl.INIT_VIDEO)
	log.Println(res)

	//eventRegions := make([]*EventRegion, 0)
	eventRegions := make(RegionList, 0)
	var x int32
	var y int32 = 20
	for i := 0; i < 8; i++ {
		x = int32(80 + (i*CARD_WIDTH + i*10))
		eventRegions = append(eventRegions, NewEventRegion(x, y, CARD_WIDTH, CARD_HEIGHT))
	}

	y = 520
	for i := 0; i < 8; i++ {
		x = int32(80 + (i*CARD_WIDTH + i*10))
		eventRegions = append(eventRegions, NewEventRegion(x, y, CARD_WIDTH, CARD_HEIGHT))
	}

	y = 280
	eventRegions = append(eventRegions, NewEventRegion(30, y, CARD_WIDTH, CARD_HEIGHT))
	eventRegions = append(eventRegions, NewEventRegion(312, y, CARD_WIDTH, CARD_HEIGHT))
	eventRegions = append(eventRegions, NewEventRegion(612, y, CARD_WIDTH, CARD_HEIGHT))
	eventRegions = append(eventRegions, NewEventRegion(894, y, CARD_WIDTH, CARD_HEIGHT))

	window := sdl.CreateWindow("Speed", sdl.WINDOWPOS_UNDEFINED, sdl.WINDOWPOS_UNDEFINED, 1024, 768, sdl.WINDOW_SHOWN)

	renderer := sdl.CreateRenderer(window, -1, sdl.RENDERER_ACCELERATED)
	if renderer == nil {
		log.Println("Failed to create renderer:", sdl.GetError())
		os.Exit(1)
	}

	imgCard42 := sdl.LoadBMP("42.bmp")
	log.Println(imgCard42)

	texture := renderer.CreateTextureFromSurface(imgCard42)
	if texture == nil {
		log.Println("Failed to create texture (42):", sdl.GetError())
	}

	src := sdl.Rect{0, 0, 100, 180}
	dst := sdl.Rect{100, 50, 100, 180}

	renderer.Clear()
	renderer.Copy(texture, &src, &dst)

	renderer.SetDrawColor(255, 255, 255, 255)
	for _, region := range eventRegions {
		renderer.DrawRect(region.SDLRect())
	}
	renderer.SetDrawColor(0, 0, 0, 255)

	renderer.Present()

	var event sdl.Event
	running := true
	for running {
		for event = sdl.PollEvent(); event != nil; event = sdl.PollEvent() {
			switch t := event.(type) {
			case *sdl.QuitEvent:
				running = false
			/*case *sdl.MouseMotionEvent:
			fmt.Printf("[%d ms] MouseMotion\ttype:%d\tid:%d\tx:%d\ty:%d\txrel:%d\tyrel:%d\n",
				t.Timestamp, t.Type, t.Which, t.X, t.Y, t.XRel, t.YRel)*/
			case *sdl.MouseButtonEvent:
				fmt.Printf("[%d ms] MouseButton\ttype:%d\tid:%d\tx:%d\ty:%d\tbutton:%d\tstate:%d\n",
					t.Timestamp, t.Type, t.Which, t.X, t.Y, t.Button, t.State)
				if t.Button == 1 {
					what := eventRegions.HitWhat(t.X, t.Y)
					if t.Type == 1025 {
						log.Println("MOUSE DOWN. Grab something.")
						if what != nil {
							interactionState = INTERACTION_STATE_DRAGGING
							draggingWhat = what
						}
					} else if t.Type == 1026 {
						log.Println("MOUSE UP. Drop it.")
						if draggingWhat != nil {
							log.Println("DROPPING", draggingWhat, "ON", what)
							interactionState = INTERACTION_STATE_DEFAULT
							draggingWhat = nil
						}

					}
				}
			case *sdl.MouseWheelEvent:
				fmt.Printf("[%d ms] MouseWheel\ttype:%d\tid:%d\tx:%d\ty:%d\n",
					t.Timestamp, t.Type, t.Which, t.X, t.Y)
			case *sdl.KeyUpEvent:
				fmt.Printf("[%d ms] Keyboard\ttype:%d\tsym:%c\tmodifiers:%d\tstate:%d\trepeat:%d\n",
					t.Timestamp, t.Type, t.Keysym.Sym, t.Keysym.Mod, t.State, t.Repeat)
			}
		}
	}

	window.Destroy()
}
<commit_msg>Experimenting with interfaces<commit_after>package main

import (
	"fmt"
	"log"
	"os"

	"github.com/veandco/go-sdl2/sdl"
)

type Region interface {
	X() int32
	Y() int32
	Width() int32
	Height() int32
}

type Intersectable interface {
	Intersect(Region) bool
}

type Point struct {
	x, y int32
}

func (p Point) X() int32 {
	return p.x
}

func (p Point) Y() int32 {
	return p.y
}

func (p Point) Width() int32 {
	return 1
}

func (p Point) Height() int32 {
	return 1
}

func (p *Point) Intersect(target Region) bool {
	if p.X() >= target.X() && p.X() <= target.X()+target.Width() &&
		p.Y() >= target.Y() && p.Y() <= target.Y()+target.Height() {
		return true
	}
	return false
}

type EventRegion struct {
	Rect *sdl.Rect
}

func NewEventRegion(x, y, width, height int32) *EventRegion {
	er := EventRegion{Rect: &sdl.Rect{X: x, Y: y, W: width, H: height}}
	return &er
}

func (er *EventRegion) X() int32 {
	return er.Rect.X
}

func (er *EventRegion) Y() int32 {
	return er.Rect.Y
}

func (er *EventRegion) Width() int32 {
	return er.Rect.W
}

func (er *EventRegion) Height() int32 {
	return er.Rect.H
}

func (er *EventRegion) SDLRect() *sdl.Rect {
	return er.Rect
}

func (er *EventRegion) Intersect(target Region) bool {
	if target.X() > er.X()+er.Width() || target.X()+target.Width() < er.X() {
		return false
	}
	if target.Y() > er.Y()+er.Height() || target.Y()+target.Height() < er.Y() {
		return false
	}
	return true
}

type RegionList []*EventRegion

func (rl RegionList) HitWhat(r Region) Region {
	for _, region := range rl {
		if intersect := region.Intersect(r); intersect {
			return region
		}
	}
	return nil
}

const CARD_WIDTH = 100
const CARD_HEIGHT = 180

const (
	INTERACTION_STATE_DEFAULT  = 0
	INTERACTION_STATE_DRAGGING = 1
)

func main() {
	interactionState := INTERACTION_STATE_DEFAULT
	fmt.Println(interactionState)
	var draggingWhat Region
	res := sdl.Init(sdl.INIT_VIDEO)
	log.Println(res)

	//eventRegions := make([]*EventRegion, 0)
	eventRegions := make(RegionList, 0)
	var x int32
	var y int32 = 20
	for i := 0; i < 8; i++ {
		x = int32(80 + (i*CARD_WIDTH + i*10))
		eventRegions = append(eventRegions, NewEventRegion(x, y, CARD_WIDTH, CARD_HEIGHT))
	}

	y = 520
	for i := 0; i < 8; i++ {
		x = int32(80 + (i*CARD_WIDTH + i*10))
		eventRegions = append(eventRegions, NewEventRegion(x, y, CARD_WIDTH, CARD_HEIGHT))
	}

	y = 280
	eventRegions = append(eventRegions, NewEventRegion(30, y, CARD_WIDTH, CARD_HEIGHT))
	eventRegions = append(eventRegions, NewEventRegion(312, y, CARD_WIDTH, CARD_HEIGHT))
	eventRegions = append(eventRegions, NewEventRegion(612, y, CARD_WIDTH, CARD_HEIGHT))
	eventRegions = append(eventRegions, NewEventRegion(894, y, CARD_WIDTH, CARD_HEIGHT))

	window := sdl.CreateWindow("Speed", sdl.WINDOWPOS_UNDEFINED, sdl.WINDOWPOS_UNDEFINED, 1024, 768, sdl.WINDOW_SHOWN)

	renderer := sdl.CreateRenderer(window, -1, sdl.RENDERER_ACCELERATED)
	if renderer == nil {
		log.Println("Failed to create renderer:", sdl.GetError())
		os.Exit(1)
	}

	imgCard42 := sdl.LoadBMP("42.bmp")
	log.Println(imgCard42)

	texture := renderer.CreateTextureFromSurface(imgCard42)
	if texture == nil {
		log.Println("Failed to create texture (42):", sdl.GetError())
	}

	src := sdl.Rect{0, 0, 100, 180}
	dst := sdl.Rect{100, 50, 100, 180}

	renderer.Clear()
	renderer.Copy(texture, &src, &dst)

	renderer.SetDrawColor(255, 255, 255, 255)
	for _, region := range eventRegions {
		renderer.DrawRect(region.SDLRect())
	}
	renderer.SetDrawColor(0, 0, 0, 255)

	renderer.Present()

	var p1 = Point{50, 50}
	var p2 = Point{50, 50}
	var p3 = Point{53, 53}
	var r1 *EventRegion = NewEventRegion(50, 50, 2, 2)
	var r2 *EventRegion = NewEventRegion(52, 10, 100, 100)
	fmt.Println(p1.Intersect(p2))
	fmt.Println(p2.Intersect(p1))
	fmt.Println(p3.Intersect(p1))
	fmt.Println(p3.Intersect(p2))
	fmt.Println(p3.Intersect(p3))
	fmt.Println(r1.Intersect(p1))
	fmt.Println(r1.Intersect(p3))
	fmt.Println(r1.Intersect(r2))

	var event sdl.Event
	running := true
	for running {
		for event = sdl.PollEvent(); event != nil; event = sdl.PollEvent() {
			switch t := event.(type) {
			case *sdl.QuitEvent:
				running = false
			/*case *sdl.MouseMotionEvent:
			fmt.Printf("[%d ms] MouseMotion\ttype:%d\tid:%d\tx:%d\ty:%d\txrel:%d\tyrel:%d\n",
				t.Timestamp, t.Type, t.Which, t.X, t.Y, t.XRel, t.YRel)*/
			case *sdl.MouseButtonEvent:
				fmt.Printf("[%d ms] MouseButton\ttype:%d\tid:%d\tx:%d\ty:%d\tbutton:%d\tstate:%d\n",
					t.Timestamp, t.Type, t.Which, t.X, t.Y, t.Button, t.State)
				if t.Button == 1 {
					point := &Point{t.X, t.Y}
					what := eventRegions.HitWhat(point)
					if t.Type == 1025 {
						log.Println("MOUSE DOWN. Grab something.")
						if what != nil {
							log.Println("GRABBING", what)
							interactionState = INTERACTION_STATE_DRAGGING
							draggingWhat = what
						}
					} else if t.Type == 1026 {
						log.Println("MOUSE UP. Drop it.")
						if draggingWhat != nil {
							log.Println("DROPPING", draggingWhat, "ON", what)
							interactionState = INTERACTION_STATE_DEFAULT
							draggingWhat = nil
						}

					}
				}
			case *sdl.MouseWheelEvent:
				fmt.Printf("[%d ms] MouseWheel\ttype:%d\tid:%d\tx:%d\ty:%d\n",
					t.Timestamp, t.Type, t.Which, t.X, t.Y)
			case *sdl.KeyUpEvent:
				fmt.Printf("[%d ms] Keyboard\ttype:%d\tsym:%c\tmodifiers:%d\tstate:%d\trepeat:%d\n",
					t.Timestamp, t.Type, t.Keysym.Sym, t.Keysym.Mod, t.State, t.Repeat)
			}
		}
	}

	window.Destroy()
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"net/http"
	"net/url"
	"os"
	"time"

	"github.com/fraenkel/candiedyaml"
	"github.com/garyburd/redigo/redis"

	proleroutes "github.com/winston-ci/prole/routes"

	"github.com/winston-ci/winston/api"
	apiroutes "github.com/winston-ci/winston/api/routes"
	"github.com/winston-ci/winston/builder"
	"github.com/winston-ci/winston/config"
	"github.com/winston-ci/winston/db"
	"github.com/winston-ci/winston/endpoint"
	"github.com/winston-ci/winston/server"
)

var configPath = flag.String(
	"config",
	"",
	"path to winston server config .yml",
)

var proleAddr = flag.String(
	"proleAddr",
	"127.0.0.1:4637",
	"address denoting the prole service",
)

var listenAddr = flag.String(
	"listenAddr",
	":8080",
	"port for the web server to listen on",
)

var apiListenAddr = flag.String(
	"apiListenAddr",
	":8081",
	"port for the api to listen on",
)

var peerAddr = flag.String(
	"peerAddr",
	"127.0.0.1:8081",
	"external address of the api server",
)

func main() {
	flag.Parse()

	if *configPath == "" {
		fatal(errors.New("must specify -config"))
	}

	configFile, err := os.Open(*configPath)
	if err != nil {
		fatal(err)
	}

	var config config.Config
	err = candiedyaml.NewDecoder(configFile).Decode(&config)
	if err != nil {
		fatal(err)
	}

	configFile.Close()

	redisDB := db.NewRedis(redis.NewPool(func() (redis.Conn, error) {
		return redis.DialTimeout("tcp", "127.0.0.1:6379", 5*time.Second, 0, 0)
	}, 20))

	winstonApiUrl := &url.URL{
		Scheme: "http",
		Host:   *peerAddr,
	}

	winstonEndpoint := endpoint.EndpointRoutes{
		URL:    winstonApiUrl,
		Routes: apiroutes.Routes,
	}

	proleURL := &url.URL{
		Scheme: "http",
		Host:   *proleAddr,
	}

	proleEndpoint := endpoint.EndpointRoutes{
		URL:    proleURL,
		Routes: proleroutes.Routes,
	}

	builder := builder.NewBuilder(redisDB, proleEndpoint, winstonEndpoint)

	serverHandler, err := server.New(config, redisDB, builder)
	if err != nil {
		fatal(err)
	}

	apiHandler, err := api.New(redisDB)
	if err != nil {
		fatal(err)
	}

	errs := make(chan error, 2)

	go func() {
		errs <- http.ListenAndServe(*listenAddr, serverHandler)
	}()

	go func() {
		errs <- http.ListenAndServe(*apiListenAddr, apiHandler)
	}()

	fatal(<-errs)
}

func fatal(err error) {
	println(err.Error())
	os.Exit(1)
}
<commit_msg>use URL instead of address for peer<commit_after>package main

import (
	"errors"
	"flag"
	"net/http"
	"net/url"
	"os"
	"time"

	"github.com/fraenkel/candiedyaml"
	"github.com/garyburd/redigo/redis"

	proleroutes "github.com/winston-ci/prole/routes"

	"github.com/winston-ci/winston/api"
	apiroutes "github.com/winston-ci/winston/api/routes"
	"github.com/winston-ci/winston/builder"
	"github.com/winston-ci/winston/config"
	"github.com/winston-ci/winston/db"
	"github.com/winston-ci/winston/endpoint"
	"github.com/winston-ci/winston/server"
)

var configPath = flag.String(
	"config",
	"",
	"path to winston server config .yml",
)

var proleURL = flag.String(
	"proleURL",
	"http://127.0.0.1:4637",
	"address denoting the prole service",
)

var peerURL = flag.String(
	"peerURL",
	"http://127.0.0.1:8081",
	"external URL of the api server",
)

var listenAddr = flag.String(
	"listenAddr",
	":8080",
	"port for the web server to listen on",
)

var apiListenAddr = flag.String(
	"apiListenAddr",
	":8081",
	"port for the api to listen on",
)

func main() {
	flag.Parse()

	if *configPath == "" {
		fatal(errors.New("must specify -config"))
	}

	configFile, err := os.Open(*configPath)
	if err != nil {
		fatal(err)
	}

	var config config.Config
	err = candiedyaml.NewDecoder(configFile).Decode(&config)
	if err != nil {
		fatal(err)
	}

	configFile.Close()

	redisDB := db.NewRedis(redis.NewPool(func() (redis.Conn, error) {
		return redis.DialTimeout("tcp", "127.0.0.1:6379", 5*time.Second, 0, 0)
	}, 20))

	winstonApiURL, err := url.Parse(*peerURL)
	if err != nil {
		fatal(err)
	}

	winstonEndpoint := endpoint.EndpointRoutes{
		URL:    winstonApiURL,
		Routes: apiroutes.Routes,
	}

	proleEarl, err := url.Parse(*proleURL)
	if err != nil {
		fatal(err)
	}

	proleEndpoint := endpoint.EndpointRoutes{
		URL:    proleEarl,
		Routes: proleroutes.Routes,
	}

	builder := builder.NewBuilder(redisDB, proleEndpoint, winstonEndpoint)

	serverHandler, err := server.New(config, redisDB, builder)
	if err != nil {
		fatal(err)
	}

	apiHandler, err := api.New(redisDB)
	if err != nil {
		fatal(err)
	}

	errs := make(chan error, 2)

	go func() {
		errs <- http.ListenAndServe(*listenAddr, serverHandler)
	}()

	go func() {
		errs <- http.ListenAndServe(*apiListenAddr, apiHandler)
	}()

	fatal(<-errs)
}

func fatal(err error) {
	println(err.Error())
	os.Exit(1)
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"net"
	"net/http"

	"go.uber.org/zap"

	"bufio"
	"io"

	"strings"

	"github.com/openfresh/plasma/config"
	"github.com/openfresh/plasma/log"
	"github.com/openfresh/plasma/protobuf"
	"github.com/openfresh/plasma/pubsub"
	"github.com/openfresh/plasma/server"
	"github.com/openfresh/plasma/subscriber"
	"github.com/soheilhy/cmux"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
)

type Service struct {
	Serve    func(l net.Listener) error
	Matchers []cmux.Matcher
}

const ELBHealthCheckUserAgent = "ELB-HealthChecker/"

func ELBHealthCheckMatcher() cmux.Matcher {
	return func(r io.Reader) bool {
		req, err := http.ReadRequest(bufio.NewReader(r))
		if err != nil {
			return false
		}
		userAgent := req.Header.Get("User-Agent")
		if strings.Contains(userAgent, ELBHealthCheckUserAgent) {
			return true
		}

		return false
	}
}

func plasmaListener(logger *zap.Logger, config config.Config) net.Listener {
	if config.TLS.CertFile != "" && config.TLS.KeyFile != "" {
		cer, err := tls.LoadX509KeyPair(config.TLS.CertFile, config.TLS.KeyFile)
		if err != nil {
			logger.Fatal("failed to load TLS credentials for TCP",
				zap.Error(err),
				zap.String("certFile", config.TLS.CertFile),
				zap.String("keyFile", config.TLS.KeyFile),
			)
		}

		tlsConfig := &tls.Config{
			Certificates: []tls.Certificate{cer},
		}

		l, err := tls.Listen("tcp", ":"+config.Port, tlsConfig)
		if err != nil {
			logger.Fatal("failed to listen TLS",
				zap.Error(err),
				zap.String("certFile", config.TLS.CertFile),
				zap.String("keyFile", config.TLS.KeyFile),
			)
		}
		logger.Info("enable TLS mode",
			zap.String("certFile", config.TLS.CertFile),
			zap.String("keyFile", config.TLS.KeyFile),
		)
		return l
	} else {
		l, err := net.Listen("tcp", ":"+config.Port)
		if err != nil {
			logger.Fatal("failed to listen",
				zap.Error(err),
				zap.String("port", config.Port),
			)
		}
		logger.Info("non TLS mode")
		return l
	}
}

func plasmaGRPCServerOption(logger *zap.Logger, certFile string, keyFile string) []grpc.ServerOption {
	var opts []grpc.ServerOption
	if certFile != "" && keyFile != "" {
		creds, err := credentials.NewServerTLSFromFile(certFile, keyFile)
		if err != nil {
			logger.Fatal("failed to load TLS credentials for gRPC",
				zap.Error(err),
				zap.String("certFile", certFile),
				zap.String("keyFile", certFile),
			)
		}
		opts = []grpc.ServerOption{grpc.Creds(creds)}
	}
	return opts
}

func main() {
	config := config.New()

	accessLogger, err := log.NewLogger(config.AccessLog)
	if err != nil {
		panic(err)
	}
	errorLogger, err := log.NewLogger(config.ErrorLog)
	if err != nil {
		panic(err)
	}

	l := plasmaListener(errorLogger, config)
	defer l.Close()

	pubsuber := pubsub.NewPubSub()

	sub, err := subscriber.New(pubsuber, errorLogger, config)
	if err != nil {
		errorLogger.Fatal("failed to create subscriber",
			zap.Error(err),
			zap.String("type", config.Subscriber.Type),
			zap.Duration("mockDuration", config.Subscriber.Mock.Interval),
			zap.Object("redis", config.Subscriber.Redis),
		)
	}
	go func() {
		sub := sub
		if err := sub.Subscribe(); err != nil {
			errorLogger.Fatal("failed to subscribe",
				zap.String("type", config.Subscriber.Type),
				zap.Object("redis", config.Subscriber.Redis),
				zap.Error(err),
			)
		}
	}()

	// For Native Client
	opts := plasmaGRPCServerOption(errorLogger, config.TLS.CertFile, config.TLS.KeyFile)
	grpcServer := grpc.NewServer(opts...)
	proto.RegisterStreamServiceServer(grpcServer, server.NewStreamServer(pubsuber, accessLogger, errorLogger, config))

	// For Web Front End
	sseServer := server.NewSSEServer(pubsuber, accessLogger, errorLogger, config)

	// For AWS ELB
	healthCheckServer := server.NewHealthCheckServer(accessLogger, errorLogger, config)

	services := []Service{
		{
			Serve: grpcServer.Serve,
			Matchers: []cmux.Matcher{
				cmux.HTTP2HeaderField("content-type", "application/grpc"),
			},
		},
		{
			Serve: healthCheckServer.Serve,
			Matchers: []cmux.Matcher{
				ELBHealthCheckMatcher(),
			},
		},
		{
			Serve: sseServer.Serve,
			Matchers: []cmux.Matcher{
				cmux.HTTP1(),
			},
		},
	}

	m := cmux.New(l)
	for _, service := range services {
		go service.Serve(m.Match(service.Matchers...))
	}

	m.Serve()
}
<commit_msg>remove duplicated tls encryption for gRPC<commit_after>package main

import (
	"crypto/tls"
	"net"
	"net/http"

	"go.uber.org/zap"

	"bufio"
	"io"

	"strings"

	"github.com/openfresh/plasma/config"
	"github.com/openfresh/plasma/log"
	"github.com/openfresh/plasma/protobuf"
	"github.com/openfresh/plasma/pubsub"
	"github.com/openfresh/plasma/server"
	"github.com/openfresh/plasma/subscriber"
	"github.com/soheilhy/cmux"
	"google.golang.org/grpc"
)

type Service struct {
	Serve    func(l net.Listener) error
	Matchers []cmux.Matcher
}

const ELBHealthCheckUserAgent = "ELB-HealthChecker/"

func ELBHealthCheckMatcher() cmux.Matcher {
	return func(r io.Reader) bool {
		req, err := http.ReadRequest(bufio.NewReader(r))
		if err != nil {
			return false
		}
		userAgent := req.Header.Get("User-Agent")
		if strings.Contains(userAgent, ELBHealthCheckUserAgent) {
			return true
		}

		return false
	}
}

func plasmaListener(logger *zap.Logger, config config.Config) net.Listener {
	l, err := net.Listen("tcp", ":"+config.Port)
	if err != nil {
		logger.Fatal("failed to listen",
			zap.Error(err),
			zap.String("port", config.Port),
		)
	}

	if config.TLS.CertFile != "" && config.TLS.KeyFile != "" {
		cer, err := tls.LoadX509KeyPair(config.TLS.CertFile, config.TLS.KeyFile)
		if err != nil {
			logger.Fatal("failed to load TLS credentials for TCP",
				zap.Error(err),
				zap.String("certFile", config.TLS.CertFile),
				zap.String("keyFile", config.TLS.KeyFile),
			)
		}

		logger.Info("enable TLS mode",
			zap.String("certFile", config.TLS.CertFile),
			zap.String("keyFile", config.TLS.KeyFile),
		)
		return tls.NewListener(l, &tls.Config{
			Certificates: []tls.Certificate{cer},
		})
	}

	logger.Info("non TLS mode")
	return l
}

func main() {
	config := config.New()

	accessLogger, err := log.NewLogger(config.AccessLog)
	if err != nil {
		panic(err)
	}
	errorLogger, err := log.NewLogger(config.ErrorLog)
	if err != nil {
		panic(err)
	}

	l := plasmaListener(errorLogger, config)
	defer l.Close()

	pubsuber := pubsub.NewPubSub()

	sub, err := subscriber.New(pubsuber, errorLogger, config)
	if err != nil {
		errorLogger.Fatal("failed to create subscriber",
			zap.Error(err),
			zap.String("type", config.Subscriber.Type),
			zap.Duration("mockDuration", config.Subscriber.Mock.Interval),
			zap.Object("redis", config.Subscriber.Redis),
		)
	}
	go func() {
		sub := sub
		if err := sub.Subscribe(); err != nil {
			errorLogger.Fatal("failed to subscribe",
				zap.String("type", config.Subscriber.Type),
				zap.Object("redis", config.Subscriber.Redis),
				zap.Error(err),
			)
		}
	}()

	// For Native Client
	grpcServer := grpc.NewServer()
	proto.RegisterStreamServiceServer(grpcServer, server.NewStreamServer(pubsuber, accessLogger, errorLogger, config))

	// For Web Front End
	sseServer := server.NewSSEServer(pubsuber, accessLogger, errorLogger, config)

	// For AWS ELB
	healthCheckServer := server.NewHealthCheckServer(accessLogger, errorLogger, config)

	services := []Service{
		{
			Serve: grpcServer.Serve,
			Matchers: []cmux.Matcher{
				cmux.HTTP2HeaderField("content-type", "application/grpc"),
			},
		},
		{
			Serve: healthCheckServer.Serve,
			Matchers: []cmux.Matcher{
				ELBHealthCheckMatcher(),
			},
		},
		{
			Serve: sseServer.Serve,
			Matchers: []cmux.Matcher{
				cmux.HTTP1(),
			},
		},
	}

	m := cmux.New(l)
	for _, service := range services {
		go service.Serve(m.Match(service.Matchers...))
	}

	m.Serve()
}
<|endoftext|>


<commit_before>// Copyright 2014 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"net"
	"net/http"
	"net/http/pprof"
	"net/url"
	"os"
	"os/signal"
	"path"
	"path/filepath"
	"strings"
	"syscall"
	"time"

	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/prometheus/common/promlog"
	"github.com/prometheus/common/route"
	"github.com/prometheus/common/version"
	"gopkg.in/alecthomas/kingpin.v2"

	dto "github.com/prometheus/client_model/go"
	promlogflag "github.com/prometheus/common/promlog/flag"

	api_v1 "github.com/prometheus/pushgateway/api/v1"
	"github.com/prometheus/pushgateway/asset"
	"github.com/prometheus/pushgateway/handler"
	"github.com/prometheus/pushgateway/storage"
)

func init() {
	prometheus.MustRegister(version.NewCollector("pushgateway"))
}

// logFunc in an adaptor to plug gokit logging into promhttp.HandlerOpts.
type logFunc func(...interface{}) error

func (lf logFunc) Println(v ...interface{}) {
	lf("msg", fmt.Sprintln(v...))
}

func main() {
	var (
		app = kingpin.New(filepath.Base(os.Args[0]), "The Pushgateway")

		listenAddress       = app.Flag("web.listen-address", "Address to listen on for the web interface, API, and telemetry.").Default(":9091").String()
		metricsPath         = app.Flag("web.telemetry-path", "Path under which to expose metrics.").Default("/metrics").String()
		externalURL         = app.Flag("web.external-url", "The URL under which the Pushgateway is externally reachable.").Default("").URL()
		routePrefix         = app.Flag("web.route-prefix", "Prefix for the internal routes of web endpoints. Defaults to the path of --web.external-url.").Default("").String()
		enableLifeCycle     = app.Flag("web.enable-lifecycle", "Enable shutdown via HTTP request.").Default("false").Bool()
		enableAdminAPI      = app.Flag("web.enable-admin-api", "Enable API endpoints for admin control actions.").Default("false").Bool()
		persistenceFile     = app.Flag("persistence.file", "File to persist metrics. If empty, metrics are only kept in memory.").Default("").String()
		persistenceInterval = app.Flag("persistence.interval", "The minimum interval at which to write out the persistence file.").Default("5m").Duration()
		pushUnchecked       = app.Flag("push.disable-consistency-check", "Do not check consistency of pushed metrics. DANGEROUS.").Default("false").Bool()
		promlogConfig       = promlog.Config{}
	)
	promlogflag.AddFlags(app, &promlogConfig)
	app.Version(version.Print("pushgateway"))
	app.HelpFlag.Short('h')
	kingpin.MustParse(app.Parse(os.Args[1:]))
	logger := promlog.New(&promlogConfig)

	*routePrefix = computeRoutePrefix(*routePrefix, *externalURL)
	externalPathPrefix := computeRoutePrefix("", *externalURL)

	level.Info(logger).Log("msg", "starting pushgateway", "version", version.Info())
	level.Info(logger).Log("build_context", version.BuildContext())
	level.Debug(logger).Log("msg", "external URL", "url", *externalURL)
	level.Debug(logger).Log("msg", "path prefix used externally", "path", externalPathPrefix)
	level.Debug(logger).Log("msg", "path prefix for internal routing", "path", *routePrefix)

	// flags is used to show command line flags on the status page.
	// Kingpin default flags are excluded as they would be confusing.
	flags := map[string]string{}
	boilerplateFlags := kingpin.New("", "").Version("")
	for _, f := range app.Model().Flags {
		if boilerplateFlags.GetFlag(f.Name) == nil {
			flags[f.Name] = f.Value.String()
		}
	}

	ms := storage.NewDiskMetricStore(*persistenceFile, *persistenceInterval, prometheus.DefaultGatherer, logger)

	// Create a Gatherer combining the DefaultGatherer and the metrics from the metric store.
	g := prometheus.Gatherers{
		prometheus.DefaultGatherer,
		prometheus.GathererFunc(func() ([]*dto.MetricFamily, error) { return ms.GetMetricFamilies(), nil }),
	}

	r := route.New()
	r.Get(*routePrefix+"/-/healthy", handler.Healthy(ms).ServeHTTP)
	r.Get(*routePrefix+"/-/ready", handler.Ready(ms).ServeHTTP)
	r.Get(
		path.Join(*routePrefix, *metricsPath),
		promhttp.HandlerFor(g, promhttp.HandlerOpts{
			ErrorLog: logFunc(level.Error(logger).Log),
		}).ServeHTTP,
	)

	if *enableAdminAPI {
		// To be consistent with Prometheus codebase and provide endpoint versioning, we use the same path
		// as Prometheus for its admin endpoints, even if this may feel excesive for just one simple endpoint
		// this will likely change over time.
		r.Put(*routePrefix+"/api/v1/admin/wipe", handler.WipeMetricStore(ms, logger).ServeHTTP)
	}

	// Handlers for pushing and deleting metrics.
	pushAPIPath := *routePrefix + "/metrics"
	for _, suffix := range []string{"", handler.Base64Suffix} {
		jobBase64Encoded := suffix == handler.Base64Suffix
		r.Put(pushAPIPath+"/job"+suffix+"/:job/*labels", handler.Push(ms, true, !*pushUnchecked, jobBase64Encoded, logger))
		r.Post(pushAPIPath+"/job"+suffix+"/:job/*labels", handler.Push(ms, false, !*pushUnchecked, jobBase64Encoded, logger))
		r.Del(pushAPIPath+"/job"+suffix+"/:job/*labels", handler.Delete(ms, jobBase64Encoded, logger))
		r.Put(pushAPIPath+"/job"+suffix+"/:job", handler.Push(ms, true, !*pushUnchecked, jobBase64Encoded, logger))
		r.Post(pushAPIPath+"/job"+suffix+"/:job", handler.Push(ms, false, !*pushUnchecked, jobBase64Encoded, logger))
		r.Del(pushAPIPath+"/job"+suffix+"/:job", handler.Delete(ms, jobBase64Encoded, logger))
	}
	r.Get(*routePrefix+"/static/*filepath", handler.Static(asset.Assets, *routePrefix).ServeHTTP)

	statusHandler := handler.Status(ms, asset.Assets, flags, externalPathPrefix, logger)
	r.Get(*routePrefix+"/status", statusHandler.ServeHTTP)
	r.Get(*routePrefix+"/", statusHandler.ServeHTTP)

	// Re-enable pprof.
	r.Get(*routePrefix+"/debug/pprof/*pprof", handlePprof)

	level.Info(logger).Log("listen_address", *listenAddress)
	l, err := net.Listen("tcp", *listenAddress)
	if err != nil {
		level.Error(logger).Log("err", err)
		os.Exit(1)
	}

	quitCh := make(chan struct{})
	quitHandler := func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "Requesting termination... Goodbye!")
		close(quitCh)
	}

	forbiddenAPINotEnabled := func(w http.ResponseWriter, _ *http.Request) {
		w.WriteHeader(http.StatusForbidden)
		w.Write([]byte("Lifecycle API is not enabled."))
	}

	if *enableLifeCycle {
		r.Put(*routePrefix+"/-/quit", quitHandler)
		r.Post(*routePrefix+"/-/quit", quitHandler)
	} else {
		r.Put(*routePrefix+"/-/quit", forbiddenAPINotEnabled)
		r.Post(*routePrefix+"/-/quit", forbiddenAPINotEnabled)
	}

	r.Get("/-/quit", func(w http.ResponseWriter, _ *http.Request) {
		w.WriteHeader(http.StatusMethodNotAllowed)
		w.Write([]byte("Only POST or PUT requests allowed."))
	})

	mux := http.NewServeMux()
	mux.Handle("/", r)

	buildInfo := map[string]string{
		"version":   version.Version,
		"revision":  version.Revision,
		"branch":    version.Branch,
		"buildUser": version.BuildUser,
		"buildDate": version.BuildDate,
		"goVersion": version.GoVersion,
	}

	apiv1 := api_v1.New(logger, ms, flags, buildInfo)

	apiPath := "/api"
	if *routePrefix != "/" {
		apiPath = *routePrefix + apiPath
	}

	av1 := route.New()
	apiv1.Register(av1)

	mux.Handle(apiPath+"/v1/", http.StripPrefix(apiPath+"/v1", av1))

	go closeListenerOnQuit(l, quitCh, logger)
	err = (&http.Server{Addr: *listenAddress, Handler: mux}).Serve(l)
	level.Error(logger).Log("msg", "HTTP server stopped", "err", err)
	// To give running connections a chance to submit their payload, we wait
	// for 1sec, but we don't want to wait long (e.g. until all connections
	// are done) to not delay the shutdown.
	time.Sleep(time.Second)
	if err := ms.Shutdown(); err != nil {
		level.Error(logger).Log("msg", "problem shutting down metric storage", "err", err)
	}
}

func handlePprof(w http.ResponseWriter, r *http.Request) {
	switch route.Param(r.Context(), "pprof") {
	case "/cmdline":
		pprof.Cmdline(w, r)
	case "/profile":
		pprof.Profile(w, r)
	case "/symbol":
		pprof.Symbol(w, r)
	default:
		pprof.Index(w, r)
	}
}

// computeRoutePrefix returns the effective route prefix based on the
// provided flag values for --web.route-prefix and
// --web.external-url. With prefix empty, the path of externalURL is
// used instead. A prefix "/" results in an empty returned prefix. Any
// non-empty prefix is normalized to start, but not to end, with "/".
func computeRoutePrefix(prefix string, externalURL *url.URL) string {
	if prefix == "" {
		prefix = externalURL.Path
	}

	if prefix == "/" {
		prefix = ""
	}

	if prefix != "" {
		prefix = "/" + strings.Trim(prefix, "/")
	}

	return prefix
}

// closeListenerOnQuite closes the provided listener upon closing the provided
// quitCh or upon receiving a SIGINT or SIGTERM.
func closeListenerOnQuit(l net.Listener, quitCh <-chan struct{}, logger log.Logger) {
	notifier := make(chan os.Signal, 1)
	signal.Notify(notifier, os.Interrupt, syscall.SIGTERM)

	select {
	case <-notifier:
		level.Info(logger).Log("msg", "received SIGINT/SIGTERM; exiting gracefully...")
		break
	case <-quitCh:
		level.Warn(logger).Log("msg", "received termination request via web service, exiting gracefully...")
		break
	}
	l.Close()
}
<commit_msg>Make /api/v1/wipe work again<commit_after>// Copyright 2014 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"net"
	"net/http"
	"net/http/pprof"
	"net/url"
	"os"
	"os/signal"
	"path"
	"path/filepath"
	"strings"
	"syscall"
	"time"

	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/prometheus/common/promlog"
	"github.com/prometheus/common/route"
	"github.com/prometheus/common/version"
	"gopkg.in/alecthomas/kingpin.v2"

	dto "github.com/prometheus/client_model/go"
	promlogflag "github.com/prometheus/common/promlog/flag"

	api_v1 "github.com/prometheus/pushgateway/api/v1"
	"github.com/prometheus/pushgateway/asset"
	"github.com/prometheus/pushgateway/handler"
	"github.com/prometheus/pushgateway/storage"
)

func init() {
	prometheus.MustRegister(version.NewCollector("pushgateway"))
}

// logFunc in an adaptor to plug gokit logging into promhttp.HandlerOpts.
type logFunc func(...interface{}) error

func (lf logFunc) Println(v ...interface{}) {
	lf("msg", fmt.Sprintln(v...))
}

func main() {
	var (
		app = kingpin.New(filepath.Base(os.Args[0]), "The Pushgateway")

		listenAddress       = app.Flag("web.listen-address", "Address to listen on for the web interface, API, and telemetry.").Default(":9091").String()
		metricsPath         = app.Flag("web.telemetry-path", "Path under which to expose metrics.").Default("/metrics").String()
		externalURL         = app.Flag("web.external-url", "The URL under which the Pushgateway is externally reachable.").Default("").URL()
		routePrefix         = app.Flag("web.route-prefix", "Prefix for the internal routes of web endpoints. Defaults to the path of --web.external-url.").Default("").String()
		enableLifeCycle     = app.Flag("web.enable-lifecycle", "Enable shutdown via HTTP request.").Default("false").Bool()
		enableAdminAPI      = app.Flag("web.enable-admin-api", "Enable API endpoints for admin control actions.").Default("false").Bool()
		persistenceFile     = app.Flag("persistence.file", "File to persist metrics. If empty, metrics are only kept in memory.").Default("").String()
		persistenceInterval = app.Flag("persistence.interval", "The minimum interval at which to write out the persistence file.").Default("5m").Duration()
		pushUnchecked       = app.Flag("push.disable-consistency-check", "Do not check consistency of pushed metrics. DANGEROUS.").Default("false").Bool()
		promlogConfig       = promlog.Config{}
	)
	promlogflag.AddFlags(app, &promlogConfig)
	app.Version(version.Print("pushgateway"))
	app.HelpFlag.Short('h')
	kingpin.MustParse(app.Parse(os.Args[1:]))
	logger := promlog.New(&promlogConfig)

	*routePrefix = computeRoutePrefix(*routePrefix, *externalURL)
	externalPathPrefix := computeRoutePrefix("", *externalURL)

	level.Info(logger).Log("msg", "starting pushgateway", "version", version.Info())
	level.Info(logger).Log("build_context", version.BuildContext())
	level.Debug(logger).Log("msg", "external URL", "url", *externalURL)
	level.Debug(logger).Log("msg", "path prefix used externally", "path", externalPathPrefix)
	level.Debug(logger).Log("msg", "path prefix for internal routing", "path", *routePrefix)

	// flags is used to show command line flags on the status page.
	// Kingpin default flags are excluded as they would be confusing.
	flags := map[string]string{}
	boilerplateFlags := kingpin.New("", "").Version("")
	for _, f := range app.Model().Flags {
		if boilerplateFlags.GetFlag(f.Name) == nil {
			flags[f.Name] = f.Value.String()
		}
	}

	ms := storage.NewDiskMetricStore(*persistenceFile, *persistenceInterval, prometheus.DefaultGatherer, logger)

	// Create a Gatherer combining the DefaultGatherer and the metrics from the metric store.
	g := prometheus.Gatherers{
		prometheus.DefaultGatherer,
		prometheus.GathererFunc(func() ([]*dto.MetricFamily, error) { return ms.GetMetricFamilies(), nil }),
	}

	r := route.New()
	r.Get(*routePrefix+"/-/healthy", handler.Healthy(ms).ServeHTTP)
	r.Get(*routePrefix+"/-/ready", handler.Ready(ms).ServeHTTP)
	r.Get(
		path.Join(*routePrefix, *metricsPath),
		promhttp.HandlerFor(g, promhttp.HandlerOpts{
			ErrorLog: logFunc(level.Error(logger).Log),
		}).ServeHTTP,
	)

	// Handlers for pushing and deleting metrics.
	pushAPIPath := *routePrefix + "/metrics"
	for _, suffix := range []string{"", handler.Base64Suffix} {
		jobBase64Encoded := suffix == handler.Base64Suffix
		r.Put(pushAPIPath+"/job"+suffix+"/:job/*labels", handler.Push(ms, true, !*pushUnchecked, jobBase64Encoded, logger))
		r.Post(pushAPIPath+"/job"+suffix+"/:job/*labels", handler.Push(ms, false, !*pushUnchecked, jobBase64Encoded, logger))
		r.Del(pushAPIPath+"/job"+suffix+"/:job/*labels", handler.Delete(ms, jobBase64Encoded, logger))
		r.Put(pushAPIPath+"/job"+suffix+"/:job", handler.Push(ms, true, !*pushUnchecked, jobBase64Encoded, logger))
		r.Post(pushAPIPath+"/job"+suffix+"/:job", handler.Push(ms, false, !*pushUnchecked, jobBase64Encoded, logger))
		r.Del(pushAPIPath+"/job"+suffix+"/:job", handler.Delete(ms, jobBase64Encoded, logger))
	}
	r.Get(*routePrefix+"/static/*filepath", handler.Static(asset.Assets, *routePrefix).ServeHTTP)

	statusHandler := handler.Status(ms, asset.Assets, flags, externalPathPrefix, logger)
	r.Get(*routePrefix+"/status", statusHandler.ServeHTTP)
	r.Get(*routePrefix+"/", statusHandler.ServeHTTP)

	// Re-enable pprof.
	r.Get(*routePrefix+"/debug/pprof/*pprof", handlePprof)

	level.Info(logger).Log("listen_address", *listenAddress)
	l, err := net.Listen("tcp", *listenAddress)
	if err != nil {
		level.Error(logger).Log("err", err)
		os.Exit(1)
	}

	quitCh := make(chan struct{})
	quitHandler := func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "Requesting termination... Goodbye!")
		close(quitCh)
	}

	forbiddenAPINotEnabled := func(w http.ResponseWriter, _ *http.Request) {
		w.WriteHeader(http.StatusForbidden)
		w.Write([]byte("Lifecycle API is not enabled."))
	}

	if *enableLifeCycle {
		r.Put(*routePrefix+"/-/quit", quitHandler)
		r.Post(*routePrefix+"/-/quit", quitHandler)
	} else {
		r.Put(*routePrefix+"/-/quit", forbiddenAPINotEnabled)
		r.Post(*routePrefix+"/-/quit", forbiddenAPINotEnabled)
	}

	r.Get("/-/quit", func(w http.ResponseWriter, _ *http.Request) {
		w.WriteHeader(http.StatusMethodNotAllowed)
		w.Write([]byte("Only POST or PUT requests allowed."))
	})

	mux := http.NewServeMux()
	mux.Handle("/", r)

	buildInfo := map[string]string{
		"version":   version.Version,
		"revision":  version.Revision,
		"branch":    version.Branch,
		"buildUser": version.BuildUser,
		"buildDate": version.BuildDate,
		"goVersion": version.GoVersion,
	}

	apiv1 := api_v1.New(logger, ms, flags, buildInfo)

	apiPath := "/api"
	if *routePrefix != "/" {
		apiPath = *routePrefix + apiPath
	}

	av1 := route.New()
	apiv1.Register(av1)
	if *enableAdminAPI {
		av1.Put("/admin/wipe", handler.WipeMetricStore(ms, logger).ServeHTTP)
	}

	mux.Handle(apiPath+"/v1/", http.StripPrefix(apiPath+"/v1", av1))

	go closeListenerOnQuit(l, quitCh, logger)
	err = (&http.Server{Addr: *listenAddress, Handler: mux}).Serve(l)
	level.Error(logger).Log("msg", "HTTP server stopped", "err", err)
	// To give running connections a chance to submit their payload, we wait
	// for 1sec, but we don't want to wait long (e.g. until all connections
	// are done) to not delay the shutdown.
	time.Sleep(time.Second)
	if err := ms.Shutdown(); err != nil {
		level.Error(logger).Log("msg", "problem shutting down metric storage", "err", err)
	}
}

func handlePprof(w http.ResponseWriter, r *http.Request) {
	switch route.Param(r.Context(), "pprof") {
	case "/cmdline":
		pprof.Cmdline(w, r)
	case "/profile":
		pprof.Profile(w, r)
	case "/symbol":
		pprof.Symbol(w, r)
	default:
		pprof.Index(w, r)
	}
}

// computeRoutePrefix returns the effective route prefix based on the
// provided flag values for --web.route-prefix and
// --web.external-url. With prefix empty, the path of externalURL is
// used instead. A prefix "/" results in an empty returned prefix. Any
// non-empty prefix is normalized to start, but not to end, with "/".
func computeRoutePrefix(prefix string, externalURL *url.URL) string {
	if prefix == "" {
		prefix = externalURL.Path
	}

	if prefix == "/" {
		prefix = ""
	}

	if prefix != "" {
		prefix = "/" + strings.Trim(prefix, "/")
	}

	return prefix
}

// closeListenerOnQuite closes the provided listener upon closing the provided
// quitCh or upon receiving a SIGINT or SIGTERM.
func closeListenerOnQuit(l net.Listener, quitCh <-chan struct{}, logger log.Logger) {
	notifier := make(chan os.Signal, 1)
	signal.Notify(notifier, os.Interrupt, syscall.SIGTERM)

	select {
	case <-notifier:
		level.Info(logger).Log("msg", "received SIGINT/SIGTERM; exiting gracefully...")
		break
	case <-quitCh:
		level.Warn(logger).Log("msg", "received termination request via web service, exiting gracefully...")
		break
	}
	l.Close()
}
<|endoftext|>